@elizaos/plugin-computeruse 2.0.3-beta.2 → 2.0.3-beta.4

package/dist/scene/a11y-provider.d.ts

@@ -0,0 +1,83 @@
+/**
+ * AccessibilityProvider — thin abstraction over platform-specific a11y trees.
+ *
+ * The scene-builder needs structured a11y nodes (role, label, bbox, actions)
+ * tagged with a stable id so the planner can say "click element a47" across
+ * turns. The existing `platform/a11y.ts::extractA11yTree()` returns a single
+ * flat string — useful for prompts but not structured. This module wraps it
+ * with a typed-node interface and adds:
+ *
+ *   - Native impls per-OS that prefer structured JSON output where the
+ *     platform supports it (AT-SPI emits structured data we just need to
+ *     marshal; UIA/AX similarly).
+ *   - A Wayland-compositor fallback (`hyprctl clients -j`, `swaymsg -t
+ *     get_tree`) so Linux Wayland-only environments still surface windowable
+ *     nodes even when AT-SPI is locked down.
+ *   - A `NullAccessibilityProvider` for platforms / contexts where a11y is
+ *     intentionally disabled (CI, headless runners).
+ *
+ * Stable id strategy:
+ *   - Each provider emits `a<displayId>-<seq>` IDs. The same logical element
+ *     keeps the same id across consecutive frames AS LONG AS the provider's
+ *     in-memory map is preserved — we re-key when role + label + bbox
+ *     intersect significantly with a previous frame's node. This is the
+ *     contract WS7's "click element a47" depends on.
+ *
+ * The Android `AccessibilityService` impl is owned by WS8 and registers via
+ * `setAccessibilityProvider()` at runtime — this module exposes the seam
+ * but does not ship a JS-side implementation.
+ */
+import type { SceneAxNode } from "./scene-types.js";
+export interface AccessibilityProvider {
+    readonly name: string;
+    /**
+     * Whether this provider can produce structured nodes on the current host.
+     * Used by `resolveAccessibilityProvider` to pick the best chain entry.
+     */
+    available(): boolean;
+    /**
+     * Capture the live a11y tree and return per-display node lists. Returns
+     * an empty array when no nodes are reachable (vs throwing) so the
+     * scene-builder always produces a Scene.
+     */
+    snapshot(): Promise<SceneAxNode[]>;
+}
+declare class NullAccessibilityProvider implements AccessibilityProvider {
+    readonly name = "null";
+    available(): boolean;
+    snapshot(): Promise<SceneAxNode[]>;
+}
+/**
+ * Replace the active provider (used by Android/WS8 to inject the native
+ * `AccessibilityService` adapter, and by tests).
+ */
+export declare function setAccessibilityProvider(provider: AccessibilityProvider): void;
+export declare function resolveAccessibilityProvider(): AccessibilityProvider;
+interface IdAssignerState {
+    seq: Map<number, number>;
+}
+export declare function makeIdAssigner(): IdAssignerState;
+export declare function assignAxId(state: IdAssignerState, displayId: number): string;
+export declare class LinuxAccessibilityProvider implements AccessibilityProvider {
+    readonly name = "linux";
+    available(): boolean;
+    snapshot(): Promise<SceneAxNode[]>;
+    private tryAtspi;
+    private tryWaylandCompositor;
+    private tryHyprland;
+    private trySway;
+}
+export declare function parseHyprlandClients(text: string): SceneAxNode[];
+export declare function parseSwayTree(text: string): SceneAxNode[];
+export declare class DarwinAccessibilityProvider implements AccessibilityProvider {
+    readonly name = "darwin";
+    available(): boolean;
+    snapshot(): Promise<SceneAxNode[]>;
+}
+export declare class WindowsAccessibilityProvider implements AccessibilityProvider {
+    readonly name = "win32";
+    available(): boolean;
+    snapshot(): Promise<SceneAxNode[]>;
+}
+export { NullAccessibilityProvider };
+//# sourceMappingURL=a11y-provider.d.ts.map

package/dist/scene/a11y-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"a11y-provider.d.ts","sourceRoot":"","sources":["../../src/scene/a11y-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEpD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,SAAS,IAAI,OAAO,CAAC;IACrB;;;;OAIG;IACH,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;CACpC;AAED,cAAM,yBAA0B,YAAW,qBAAqB;IAC9D,QAAQ,CAAC,IAAI,UAAU;IACvB,SAAS,IAAI,OAAO;IAGd,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;CAGzC;AAID;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,qBAAqB,GAC9B,IAAI,CAEN;AAED,wBAAgB,4BAA4B,IAAI,qBAAqB,CAOpE;AAED,UAAU,eAAe;IACvB,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1B;AAED,wBAAgB,cAAc,IAAI,eAAe,CAEhD;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAK5E;AAID,qBAAa,0BAA2B,YAAW,qBAAqB;IACtE,QAAQ,CAAC,IAAI,WAAW;IAExB,SAAS,IAAI,OAAO;IAUd,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IASxC,OAAO,CAAC,QAAQ;IA0DhB,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,WAAW;IAanB,OAAO,CAAC,OAAO;CAYhB;AAaD,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CAiChE;AAcD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,EAAE,CA4CzD;AAuBD,qBAAa,2BAA4B,YAAW,qBAAqB;IACvE,QAAQ,CAAC,IAAI,YAAY;IAEzB,SAAS,IAAI,OAAO;IAId,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;CAuDzC;AAID,qBAAa,4BAA6B,YAAW,qBAAqB;IACxE,QAAQ,CAAC,IAAI,WAAW;IAExB,SAAS,IAAI,OAAO;IAId,QAAQ,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;CAqDzC;AAED,OAAO,EAAE,yBAAyB,EAAE,CAAC"}

package/dist/scene/apps.d.ts

@@ -0,0 +1,39 @@
+/**
+ * App enumeration — join `listProcesses()` with `listWindows()` to produce
+ * a per-pid `SceneApp[]` shape with embedded windows.
+ *
+ * This is intentionally a join over what we already have. The existing
+ * `windows-list.ts` doesn't surface pid on every OS:
+ *   - Windows: `Get-Process` already keys by pid; `MainWindowTitle` lives on
+ *              the same row. The join is implicit.
+ *   - Linux  : `wmctrl -l -p` gives pid per window. We re-invoke wmctrl with
+ *              `-p` here when available; otherwise the per-pid map carries an
+ *              empty `windows` list.
+ *   - macOS  : AppleScript gives `name of proc` but not the pid. We do a
+ *              name-based join on `comm`. Close enough for "click in Safari"
+ *              — the planner has app-name + window-title to disambiguate.
+ *
+ * Edge cases:
+ *   - A process with no visible windows still appears in `apps[]` with an
+ *     empty `windows` list. The planner uses this to know "Slack is running
+ *     but minimized" without firing an extra query.
+ *   - A window with no resolvable pid (Linux X11 without `_NET_WM_PID`) maps
+ *     to a synthetic `{ pid: 0, name: <appField> }` bucket.
+ */
+import { listProcesses } from "../platform/process-list.js";
+import { listWindows } from "../platform/windows-list.js";
+import type { WindowInfo } from "../types.js";
+import type { SceneApp } from "./scene-types.js";
+export interface AppEnumerationDeps {
+    /** Override for tests. */
+    processes?: typeof listProcesses;
+    windows?: typeof listWindows;
+}
+export declare function enumerateApps(deps?: AppEnumerationDeps): SceneApp[];
+interface RawProc {
+    pid: number;
+    name: string;
+}
+export declare function joinAppsAndWindows(procs: RawProc[], windows: WindowInfo[], platform: "linux" | "darwin" | "win32" | string): SceneApp[];
+export {};
+//# sourceMappingURL=apps.d.ts.map

package/dist/scene/apps.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"apps.d.ts","sourceRoot":"","sources":["../../src/scene/apps.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAkB,MAAM,kBAAkB,CAAC;AAEjE,MAAM,WAAW,kBAAkB;IACjC,0BAA0B;IAC1B,SAAS,CAAC,EAAE,OAAO,aAAa,CAAC;IACjC,OAAO,CAAC,EAAE,OAAO,WAAW,CAAC;CAC9B;AAED,wBAAgB,aAAa,CAAC,IAAI,GAAE,kBAAuB,GAAG,QAAQ,EAAE,CAMvE;AAED,UAAU,OAAO;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd;AASD,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,OAAO,EAAE,EAChB,OAAO,EAAE,UAAU,EAAE,EACrB,QAAQ,EAAE,OAAO,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAC9C,QAAQ,EAAE,CA4FZ"}

package/dist/scene/dhash.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Difference-hash (dHash) implementation for cheap frame-level and
+ * block-level change detection in the WS6 scene-builder.
+ *
+ * Why dHash:
+ *   - 64-bit integer, hammable in two cycles, vastly cheaper than a structural
+ *     similarity index.
+ *   - Robust to small re-encodings (cursor jitter, anti-alias seams) which
+ *     dominate "no change" frames in a real session.
+ *   - Tunable: we run an 8×8 whole-frame hash for the cheap "did anything
+ *     happen?" gate, and a 16×16 block grid (each block ~128×128 of source)
+ *     for dirty-block re-OCR.
+ *
+ * Implementation notes:
+ *   - The PNG decoder here is intentionally minimal — it handles the formats
+ *     produced by every screenshot path we ship (color type 2 = RGB, 6 =
+ *     RGBA, 8-bit depth, non-interlaced). Anything else returns `null` and
+ *     the caller falls back to a coarser whole-frame byte hash.
+ *   - Pure functions, no I/O. Safe to test deterministically.
+ *
+ * Block-grid contract:
+ *   - The image is gridded into N×N blocks (default 16) covering the whole
+ *     frame; remainder pixels go to the right/bottom edges.
+ *   - Each block gets its own 8-bit "mini-hash" derived from row-wise grayscale
+ *     differences sampled at four points per block. Two blocks compare equal
+ *     iff their hashes match. This is intentionally a much coarser test than
+ *     a full per-block dHash — the goal is just "did this region change at
+ *     all?" not "how similar."
+ */
+export interface RawImage {
+    width: number;
+    height: number;
+    /** RGBA scanline data, row-major, 4 bytes/pixel, length = width*height*4. */
+    rgba: Buffer;
+}
+/**
+ * Decode a PNG buffer to RGBA. Returns null for unsupported variants
+ * (interlaced, palette-indexed, 16-bit depth) — caller handles that by
+ * falling back to a non-block-grid hash strategy.
+ */
+export declare function decodePng(png: Buffer): RawImage | null;
+/**
+ * 64-bit dHash of the whole frame. Returns `null` if the PNG can't be
+ * decoded; callers fall back to a byte-length comparison.
+ */
+export declare function frameDhash(png: Buffer): bigint | null;
+/**
+ * Hamming distance between two 64-bit dHashes. Two visually identical frames
+ * report 0. We treat anything < 5 as "no change" for the idle gate.
+ */
+export declare function hamming(a: bigint, b: bigint): number;
+export interface BlockGrid {
+    /** Cols × Rows block-hash matrix. Index by `row * cols + col`. */
+    hashes: Uint32Array;
+    cols: number;
+    rows: number;
+}
+/**
+ * Split a frame into a `cols x rows` grid and return a tiny per-block hash.
+ *
+ * We don't use a full dHash per block — instead we sample 4 luminance points
+ * per block and pack a 16-bit fingerprint. That's enough to spot "this block
+ * changed" with very few false-negatives in practice, and stays cheap when
+ * called every active-poll frame (4 Hz).
+ */
+export declare function blockGrid(png: Buffer, cols?: number, rows?: number): BlockGrid | null;
+export declare function blockGridFromImage(image: RawImage, cols?: number, rows?: number): BlockGrid;
+export interface DirtyBlock {
+    col: number;
+    row: number;
+    /** Pixel-space bbox `[x, y, w, h]` of this block in the source frame. */
+    bbox: [number, number, number, number];
+}
+/**
+ * Return the list of blocks whose hash changed between two grids.
+ *
+ * If `prev` is null (first frame) every block is dirty.
+ * If `imageWidth/Height` are passed, the result includes a pixel-space bbox
+ * for each dirty block so the caller can crop the source PNG to re-OCR only
+ * the changed regions.
+ */
+export declare function diffBlocks(prev: BlockGrid | null, current: BlockGrid, imageWidth?: number, imageHeight?: number): DirtyBlock[];
+/**
+ * Coalesce a list of dirty blocks into axis-aligned rectangles using a row-
+ * sweep merge. Adjacent dirty blocks in the same row collapse into a single
+ * horizontal strip, then strips in successive rows that fully overlap collapse
+ * vertically. This keeps the OS region-capture count tiny in the common case
+ * where a single text field or a banner area changes (1 region, not 12).
+ *
+ * Returns the rectangles in display-local pixel space when `imageWidth` /
+ * `imageHeight` are provided; otherwise in `col, row, colspan, rowspan`
+ * units (1,1,1,1 = a single block at grid position 0,0).
+ */
+export declare function coalesceDirtyBlocks(dirty: DirtyBlock[], grid: BlockGrid, imageWidth?: number, imageHeight?: number): Array<{
+    bbox: [number, number, number, number];
+}>;
+/**
+ * Read PNG dimensions without inflating IDAT. Cheap — only the IHDR chunk is
+ * touched. Returns null if the buffer isn't a recognizable PNG.
+ */
+export declare function pngDimensions(png: Buffer): {
+    width: number;
+    height: number;
+} | null;
+//# sourceMappingURL=dhash.d.ts.map

package/dist/scene/dhash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dhash.d.ts","sourceRoot":"","sources":["../../src/scene/dhash.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAQH,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAmEtD;AAwFD;;;GAGG;AACH,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAIrD;AAED;;;GAGG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQpD;AAED,MAAM,WAAW,SAAS;IACxB,kEAAkE;IAClE,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,SAAK,EAAE,IAAI,SAAK,GAAG,SAAS,GAAG,IAAI,CAI7E;AAED,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,QAAQ,EACf,IAAI,SAAK,EACT,IAAI,SAAK,GACR,SAAS,CA2BX;AAOD,MAAM,WAAW,UAAU;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CACxB,IAAI,EAAE,SAAS,GAAG,IAAI,EACtB,OAAO,EAAE,SAAS,EAClB,UAAU,CAAC,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,MAAM,GACnB,UAAU,EAAE,CAgCd;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,UAAU,EAAE,EACnB,IAAI,EAAE,SAAS,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,WAAW,CAAC,EAAE,MAAM,GACnB,KAAK,CAAC;IAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC,CAwEnD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,MAAM,GACV;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAoB1C"}

package/dist/scene/ocr-adapter.d.ts

@@ -0,0 +1,64 @@
+/**
+ * OCR adapter for the WS6 scene-builder.
+ *
+ * The scene-builder needs text-from-image extraction on full frames and
+ * (much more often) on cropped dirty blocks. There are two registry seams it
+ * can draw from, in priority order:
+ *
+ *   1. The coord-aware `CoordOcrProvider` (`mobile/ocr-provider.ts`,
+ *      `registerCoordOcrProvider` / `getCoordOcrProvider`). This is the
+ *      canonical seam: `@elizaos/plugin-vision` registers a bridge to its
+ *      hierarchical OCR (docTR / Apple Vision) at boot, returning blocks with
+ *      bbox + words + semantic position in display-absolute coords. Preferred
+ *      whenever a provider is registered.
+ *   2. The line-only `OcrProvider` registry (`listOcrProviders()`), used by
+ *      the on-device iOS Apple-Vision provider and unit-test fakes. Fallback
+ *      when no coord provider is registered.
+ *
+ * **Integration choice (justified):**
+ *
+ * plugin-computeruse cannot take a hard `@elizaos/plugin-vision` dependency —
+ * that creates a cycle (vision -> capture -> computeruse) and forces every
+ * computeruse consumer to install the vision OCR stack even when they only
+ * want desktop control. Instead, computeruse *publishes* both registries and
+ * a consumer (plugin-vision, or an integrator) registers a provider at
+ * startup. The chain degrades to "no OCR" when nothing is registered; the
+ * scene-builder logs that condition once.
+ *
+ * This module exposes:
+ *   - `runOcrOnPng(png, displayId, options)` — the scene-builder calls this.
+ *   - `runOcrOnRegions(...)` — same, but for cropped dirty blocks. Falls back
+ *     to whole-frame OCR if the provider can't crop in place.
+ *   - `setOcrLoggingHook(fn)` — the scene-builder injects a logger so this
+ *     module doesn't have to take a `@elizaos/core` dep itself.
+ */
+import type { CoordOcrProvider } from "../mobile/ocr-provider.js";
+import type { SceneOcrBox } from "./scene-types.js";
+export declare function setOcrLoggingHook(fn: (message: string) => void): void;
+export interface OcrAdapterIdState {
+    /** Per-display sequence counter so ids stay stable per Scene. */
+    perDisplay: Map<number, number>;
+}
+export declare function makeOcrIdState(): OcrAdapterIdState;
+export declare function nextOcrId(state: OcrAdapterIdState, displayId: number): string;
+/**
+ * Run OCR on a whole PNG buffer. Prefers the coord-aware provider; falls back
+ * to the line-only registry. Returns boxes tagged with `displayId` and stable
+ * `t<displayId>-<seq>` ids drawn from `idState`. Empty array if no provider is
+ * registered.
+ */
+export declare function runOcrOnPng(png: Buffer, displayId: number, idState: OcrAdapterIdState): Promise<SceneOcrBox[]>;
+/**
+ * Run OCR on a list of cropped region buffers, each tied to a bbox in the
+ * source frame. Used for dirty-block re-OCR.
+ *
+ * `crops[i].png` is a standalone PNG of the dirty region. `crops[i].bbox` is
+ * the region's location in the source frame (display-local). The returned
+ * boxes are translated back into display-local source coordinates.
+ */
+export declare function runOcrOnRegions(crops: Array<{
+    png: Buffer;
+    bbox: [number, number, number, number];
+}>, displayId: number, idState: OcrAdapterIdState): Promise<SceneOcrBox[]>;
+export type { CoordOcrProvider };
+//# sourceMappingURL=ocr-adapter.d.ts.map

package/dist/scene/ocr-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr-adapter.d.ts","sourceRoot":"","sources":["../../src/scene/ocr-adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,KAAK,EAEV,gBAAgB,EAGjB,MAAM,2BAA2B,CAAC;AAKnC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAGpD,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI,CAErE;AAED,MAAM,WAAW,iBAAiB;IAChC,iEAAiE;IACjE,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,wBAAgB,cAAc,IAAI,iBAAiB,CAElD;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAK7E;AAoBD;;;;;GAKG;AACH,wBAAsB,WAAW,CAC/B,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CAsCxB;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CACnC,KAAK,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAAC,EACrE,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,WAAW,EAAE,CAAC,CAsFxB;AAyCD,YAAY,EAAE,gBAAgB,EAAE,CAAC"}

package/dist/scene/scene-builder.d.ts

@@ -0,0 +1,107 @@
+/**
+ * WS6 — Scene Builder
+ *
+ * Produces a single compact `Scene` JSON object per scene update. The runtime
+ * lifecycle is:
+ *
+ *   1. Idle poll @ 1 Hz       — capture each display, compute frame dHash.
+ *                                If no change observed for >2s, skip OCR + AX.
+ *   2. Active poll @ 4 Hz     — capture, dHash, block-grid diff, run OCR on
+ *                                the dirty blocks only, fold in AX snapshot.
+ *   3. Agent turn (onAgentTurn) — full pipeline including the WS7 VLM hook.
+ *                                 We surface a clean entry point but do NOT
+ *                                 call the VLM here.
+ *
+ * Caches:
+ *   - Whole-display scene cache keyed by `(displayId, frameDhash)`, TTL 30s.
+ *   - Per-display previous BlockGrid kept in memory; dirtied on every active
+ *     frame.
+ *   - AX subtree cache: we re-snapshot every agent turn and every active
+ *     poll when no AX focus-change notifications are available (which is
+ *     the case on every desktop OS today). The AT-SPI signal route would
+ *     remove that polling — call out as a follow-up.
+ *
+ * The builder is dependency-injectable for tests: pass an in-memory
+ * `captureDisplays`, `enumerateApps`, `accessibilityProvider`, and OCR adapter
+ * to assert pipeline behavior without a real screen.
+ */
+import { EventEmitter } from "node:events";
+import type { DisplayCapture } from "../platform/capture.js";
+import type { DisplayDescriptor } from "../types.js";
+import { type AccessibilityProvider } from "./a11y-provider.js";
+import { type OcrAdapterIdState } from "./ocr-adapter.js";
+import type { Scene, SceneApp, SceneOcrBox, SceneVlmElement } from "./scene-types.js";
+export interface SceneBuilderDeps {
+    captureAll?: () => Promise<DisplayCapture[]>;
+    captureOne?: (displayId: number) => Promise<DisplayCapture>;
+    /**
+     * Capture a region within a display in display-local coordinates. Used by
+     * the dirty-block re-OCR fast path to avoid full-frame OCR when only a
+     * small slice of the screen changed. Default: `captureDisplayRegion`.
+     */
+    captureRegion?: (displayId: number, region: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    }) => Promise<DisplayCapture>;
+    enumerateApps?: () => SceneApp[];
+    listDisplays?: () => DisplayDescriptor[];
+    accessibilityProvider?: AccessibilityProvider;
+    runOcrOnFrame?: (png: Buffer, displayId: number, idState: OcrAdapterIdState) => Promise<SceneOcrBox[]>;
+    /**
+     * Run OCR on a list of cropped PNGs, each carrying its display-local bbox.
+     * Returned boxes have their bboxes translated back into source-frame
+     * display-local coordinates. Default: `runOcrOnRegions`.
+     */
+    runOcrOnCrops?: (crops: Array<{
+        png: Buffer;
+        bbox: [number, number, number, number];
+    }>, displayId: number, idState: OcrAdapterIdState) => Promise<SceneOcrBox[]>;
+    log?: (msg: string) => void;
+}
+export interface SceneUpdateEvent {
+    scene: Scene;
+    reason: "idle" | "active" | "agent-turn";
+}
+export declare class SceneBuilder extends EventEmitter {
+    private readonly deps;
+    private readonly perDisplay;
+    private readonly ocrIdState;
+    private latestScene;
+    private inFlight;
+    constructor(deps?: SceneBuilderDeps);
+    /**
+     * Pulse the pipeline. Mode chooses how much work to do:
+     *   - "idle"     — capture + dHash only; reuses cached OCR/AX if unchanged.
+     *   - "active"   — capture + dHash + dirty-block OCR + AX.
+     *   - "agent-turn" — full pipeline; WS7's `onAgentTurn` should call this.
+     *
+     * Returns the produced Scene. Subscribers are notified after.
+     */
+    tick(mode?: "idle" | "active" | "agent-turn"): Promise<Scene>;
+    /** Called by WS7 when a real agent turn starts. Always full pipeline. */
+    onAgentTurn(): Promise<Scene>;
+    /** Returns the most recently emitted Scene, or null before first tick. */
+    getCurrentScene(): Scene | null;
+    /**
+     * Populate the VLM annotations on the current scene (#9105 M3). The Brain /
+     * DirtyTileDescriber produce `vlm_scene` (a one-paragraph description) and
+     * `vlm_elements` (described tiles); these were previously always `null`
+     * because nothing ever wrote them. Persisting them here lets the next
+     * provider read carry the cheap understanding instead of re-describing, and
+     * they survive subsequent ticks via the `latestScene?.vlm_*` pass-through.
+     */
+    setVlmAnnotations(vlmScene: string | null, vlmElements: SceneVlmElement[] | null): void;
+    /** Subscribe to scene updates. Returns an unsubscribe function. */
+    subscribe(handler: (event: SceneUpdateEvent) => void): () => void;
+    private run;
+    private captureWithFallback;
+    private safeEnumerateApps;
+    private safeSnapshotAx;
+    private ensureState;
+}
+export declare function getDefaultSceneBuilder(): SceneBuilder;
+/** Test-only reset. */
+export declare function _resetDefaultSceneBuilderForTests(): void;
+//# sourceMappingURL=scene-builder.d.ts.map

package/dist/scene/scene-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scene-builder.d.ts","sourceRoot":"","sources":["../../src/scene/scene-builder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAO7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,KAAK,qBAAqB,EAE3B,MAAM,oBAAoB,CAAC;AAW5B,OAAO,EAEL,KAAK,iBAAiB,EAIvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,KAAK,EACV,KAAK,EACL,QAAQ,EAGR,WAAW,EACX,eAAe,EAChB,MAAM,kBAAkB,CAAC;AAM1B,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7C,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAC5D;;;;OAIG;IACH,aAAa,CAAC,EAAE,CACd,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,KAC5D,OAAO,CAAC,cAAc,CAAC,CAAC;IAC7B,aAAa,CAAC,EAAE,MAAM,QAAQ,EAAE,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,iBAAiB,EAAE,CAAC;IACzC,qBAAqB,CAAC,EAAE,qBAAqB,CAAC;IAC9C,aAAa,CAAC,EAAE,CACd,GAAG,EAAE,MAAM,EACX,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,iBAAiB,KACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC5B;;;;OAIG;IACH,aAAa,CAAC,EAAE,CACd,KAAK,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC,EACrE,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,iBAAiB,KACvB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;IAC5B,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AASD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,MAAM,GAAG,QAAQ,GAAG,YAAY,CAAC;CAC1C;AAED,qBAAa,YAAa,SAAQ,YAAY;IAC5C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAEnB;IACF,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsC;IACjE,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAuC;IAClE,OAAO,CAAC,WAAW,CAAsB;IACzC,OAAO,CAAC,QAAQ,CAA+B;gBAEnC,IAAI,GAAE,gBAAqB;IAkCvC;;;;;;;OAOG;IACG,IAAI,CACR,IAAI,GAAE,MAAM,GAAG,QAAQ,GAAG,YAAuB,GAChD,OAAO,CAAC,KAAK,CAAC;IAWjB,yEAAyE;IACnE,WAAW,IAAI,OAAO,CAAC,KAAK,CAAC;IAInC,0EAA0E;IAC1E,eAAe,IAAI,KAAK,GAAG,IAAI;IAI/B;;;;;;;OAOG;IACH,iBAAiB,CACf,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,WAAW,EAAE,eAAe,EAAE,GAAG,IAAI,GACpC,IAAI;IASP,mEAAmE;IACnE,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,MAAM,IAAI;YAQnD,GAAG;YA2LH,mBAAmB;IAuBjC,OAAO,CAAC,iBAAiB;YAWX,cAAc;IAW5B,OAAO,CAAC,WAAW;CAapB;AA2CD,wBAAgB,sBAAsB,IAAI,YAAY,CAGrD;AAED,uBAAuB;AACvB,wBAAgB,iCAAiC,IAAI,IAAI,CAExD"}

package/dist/scene/scene-types.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Scene representation produced by the WS6 scene-builder pipeline.
+ *
+ * One `Scene` describes the agent's full visual + structural context at a
+ * single moment. It is the contract WS7 (Brain) consumes to ground every
+ * coordinate-bearing action.
+ *
+ * All bbox coordinates are LOCAL to their `displayId`. WS5's
+ * `localToGlobal` / `globalToLocal` translate to the input-driver space
+ * before any click fires.
+ */
+import type { DisplayDescriptor, WindowInfo } from "../types.js";
+export interface SceneAppWindow {
+    id: string;
+    title: string;
+    bounds: [number, number, number, number];
+    displayId: number;
+}
+export interface SceneApp {
+    name: string;
+    pid: number;
+    windows: SceneAppWindow[];
+}
+export interface SceneFocusedWindow {
+    app: string;
+    pid: number | null;
+    bounds: [number, number, number, number];
+    title: string;
+    displayId: number;
+}
+export interface SceneOcrBox {
+    /** Stable id `t<displayId>-<seq>`. */
+    id: string;
+    text: string;
+    bbox: [number, number, number, number];
+    conf: number;
+    displayId: number;
+}
+export interface SceneAxNode {
+    id: string;
+    role: string;
+    label?: string;
+    bbox: [number, number, number, number];
+    actions: string[];
+    displayId: number;
+}
+export interface SceneVlmElement {
+    id: string;
+    kind: string;
+    desc: string;
+    bbox: [number, number, number, number];
+    displayId: number;
+}
+export interface Scene {
+    timestamp: number;
+    displays: DisplayDescriptor[];
+    focused_window: SceneFocusedWindow | null;
+    apps: SceneApp[];
+    ocr: SceneOcrBox[];
+    ax: SceneAxNode[];
+    /** Set by WS7's Brain when a VLM turn runs; `null` outside agent turns. */
+    vlm_scene: string | null;
+    vlm_elements: SceneVlmElement[] | null;
+}
+/**
+ * Window-info alias retained for completeness; the scene-builder also accepts
+ * the existing `WindowInfo` shape and folds it into `SceneAppWindow`.
+ */
+export type SceneWindowInfo = WindowInfo;
+//# sourceMappingURL=scene-types.d.ts.map

package/dist/scene/scene-types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scene-types.d.ts","sourceRoot":"","sources":["../../src/scene/scene-types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,cAAc,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,sCAAsC;IACtC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,KAAK;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,cAAc,EAAE,kBAAkB,GAAG,IAAI,CAAC;IAC1C,IAAI,EAAE,QAAQ,EAAE,CAAC;IACjB,GAAG,EAAE,WAAW,EAAE,CAAC;IACnB,EAAE,EAAE,WAAW,EAAE,CAAC;IAClB,2EAA2E;IAC3E,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,CAAC"}

package/dist/scene/screen-state.d.ts

@@ -0,0 +1,105 @@
+/**
+ * Unified `ScreenState` — one capture per turn (#9105 M3).
+ *
+ * Before this seam the same display was captured/encoded multiple times per
+ * turn: the SceneBuilder's dHash gate, the SceneBuilder's OCR pass, and the
+ * Brain's `encodeForBrain` each pulled their own frame. `ScreenStateStore`
+ * makes the capture a single shared artifact: it grabs one PNG per display,
+ * computes the frame dHash and the 16×16 block grid once, and hands every
+ * consumer (OCR adapter, DirtyTileDescriber, scene provider, Brain) the same
+ * `ScreenState`. Re-asking for a display within `freshnessMs` reuses the prior
+ * capture instead of hitting the OS again.
+ *
+ * Change detection rides the existing `scene/dhash.ts`: a refresh only emits a
+ * change event when the frame dHash moved by at least `hammingThreshold` bits,
+ * so subscribers (e.g. the continuous describer loop) don't re-run on a
+ * pixel-identical screen. `getStats()` exposes how many captures were served
+ * from cache vs taken fresh so a test can prove the per-turn saving.
+ *
+ * Pure + injectable: pass a `capture` fn to drive the store from synthetic PNGs
+ * with no real screen.
+ */
+import type { DisplayCapture } from "../platform/capture.js";
+import { type BlockGrid, type DirtyBlock } from "./dhash.js";
+/** Frame-dHash hamming distance below which two frames are "unchanged". */
+export declare const SCREEN_STATE_HAMMING_THRESHOLD = 5;
+/** Default reuse window: a capture younger than this is served from cache. */
+export declare const SCREEN_STATE_DEFAULT_FRESHNESS_MS = 400;
+/** A single shared per-display capture for one turn. */
+export interface ScreenState {
+    displayId: number;
+    /** ms epoch when the underlying frame was captured. */
+    capturedAt: number;
+    width: number;
+    height: number;
+    /** PNG bytes at backing-store resolution. */
+    png: Buffer;
+    /** 64-bit frame dHash, or null when the PNG could not be decoded. */
+    dhash: bigint | null;
+    /** 16×16 block grid of the frame, or null when undecodable. */
+    blockGrid: BlockGrid | null;
+    /**
+     * Blocks that changed vs the previously stored frame for this display.
+     * `null` on the very first capture (no prior to diff against). Bboxes are in
+     * display-local pixel space when dimensions were available.
+     */
+    dirtyBlocks: DirtyBlock[] | null;
+}
+export interface ScreenStateChange {
+    state: ScreenState;
+    /** dHash hamming distance from the previous frame (Infinity on first frame). */
+    distance: number;
+}
+/** Capture-accounting snapshot for a store. */
+export interface ScreenStateStats {
+    /** Fresh OS captures actually taken. */
+    captures: number;
+    /** Capture requests served from the freshness cache (no OS hit). */
+    cacheHits: number;
+    /** Refreshes that changed the screen enough to fire a change event. */
+    changes: number;
+}
+export interface ScreenStateStoreOptions {
+    /**
+     * Capture one display to a PNG. Defaults to the platform `captureDisplay`.
+     * Injected in tests to drive synthetic frames.
+     */
+    capture: (displayId: number) => Promise<DisplayCapture>;
+    /** Reuse window in ms. Default `SCREEN_STATE_DEFAULT_FRESHNESS_MS`. */
+    freshnessMs?: number;
+    /** Hamming threshold for "changed". Default `SCREEN_STATE_HAMMING_THRESHOLD`. */
+    hammingThreshold?: number;
+    /** Clock injection for deterministic freshness tests. Default `Date.now`. */
+    now?: () => number;
+}
+/**
+ * Owns the single shared capture per display. `ComputerUseService` holds one
+ * store; SceneBuilder, the Brain, and the DirtyTileDescriber loop all read it.
+ */
+export declare class ScreenStateStore {
+    private readonly capture;
+    private readonly freshnessMs;
+    private readonly hammingThreshold;
+    private readonly now;
+    private readonly states;
+    private readonly listeners;
+    private stats;
+    constructor(options: ScreenStateStoreOptions);
+    getStats(): ScreenStateStats;
+    /** Latest stored state for a display, or null if never captured. */
+    peek(displayId: number): ScreenState | null;
+    /**
+     * Return a `ScreenState` for `displayId`, reusing the last capture when it is
+     * younger than the freshness window. Pass `force` to always re-capture.
+     */
+    get(displayId: number, force?: boolean): Promise<ScreenState>;
+    /**
+     * Force a fresh capture, recompute dHash + block grid + dirty diff against the
+     * prior frame, store it, and emit a change event when the frame moved by at
+     * least the hamming threshold.
+     */
+    refresh(displayId: number): Promise<ScreenState>;
+    /** Subscribe to change events. Returns an unsubscribe function. */
+    onChange(listener: (change: ScreenStateChange) => void): () => void;
+}
+//# sourceMappingURL=screen-state.d.ts.map

package/dist/scene/screen-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-state.d.ts","sourceRoot":"","sources":["../../src/scene/screen-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EACL,KAAK,SAAS,EAEd,KAAK,UAAU,EAKhB,MAAM,YAAY,CAAC;AAEpB,2EAA2E;AAC3E,eAAO,MAAM,8BAA8B,IAAI,CAAC;AAChD,8EAA8E;AAC9E,eAAO,MAAM,iCAAiC,MAAM,CAAC;AAErD,wDAAwD;AACxD,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,6CAA6C;IAC7C,GAAG,EAAE,MAAM,CAAC;IACZ,qEAAqE;IACrE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,+DAA+D;IAC/D,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B;;;;OAIG;IACH,WAAW,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,WAAW,CAAC;IACnB,gFAAgF;IAChF,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,gBAAgB;IAC/B,wCAAwC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,uBAAuB;IACtC;;;OAGG;IACH,OAAO,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IACxD,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiD;IACzE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAkD;IAC5E,OAAO,CAAC,KAAK,CAA+D;gBAEhE,OAAO,EAAE,uBAAuB;IAQ5C,QAAQ,IAAI,gBAAgB;IAI5B,oEAAoE;IACpE,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI;IAI3C;;;OAGG;IACG,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,UAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;IASjE;;;;OAIG;IACG,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;IAoCtD,mEAAmE;IACnE,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,iBAAiB,KAAK,IAAI,GAAG,MAAM,IAAI;CAIpE"}

package/dist/scene/serialize.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Token-efficient Scene serializer.
+ *
+ * Used by the `scene` provider to render the current Scene into the agent
+ * prompt. We deliberately cap lists by display so a 4k-monitor session
+ * doesn't push the prompt to thousands of tokens:
+ *
+ *   - displays              : full list (small)
+ *   - focused_window        : full (small)
+ *   - apps                  : pid, name, window count, top-2 window titles
+ *   - ocr                   : top-N most-confident lines per display
+ *                             (default N = 24, configurable)
+ *   - ax                    : limited to the focused window's display subtree
+ *                             (default cap = 24)
+ *   - vlm_scene / elements  : passed through verbatim
+ *
+ * The output is fenced JSON for predictable downstream tokenization.
+ */
+import type { Scene } from "./scene-types.js";
+export interface SerializeOptions {
+    ocrTopN?: number;
+    axMax?: number;
+    appTopWindows?: number;
+    /** Cap total apps emitted. Prefers apps with at least one visible window. */
+    appMax?: number;
+}
+export declare function serializeSceneForPrompt(scene: Scene, options?: SerializeOptions): string;
+//# sourceMappingURL=serialize.d.ts.map

package/dist/scene/serialize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serialize.d.ts","sourceRoot":"","sources":["../../src/scene/serialize.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAE9C,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,KAAK,EACZ,OAAO,GAAE,gBAAqB,GAC7B,MAAM,CAqFR"}

package/dist/security/browser-script-policy.d.ts

@@ -0,0 +1,9 @@
+/** GHSA-rcvr-766c-4phv: arbitrary page.evaluate + AsyncFunction must not run agent-supplied code. */
+export declare const BROWSER_EXECUTE_DISABLED_MESSAGE: string;
+export declare class BrowserExecuteDisabledError extends Error {
+    readonly code: "browser_execute_disabled";
+    constructor(message?: string);
+}
+export declare function assertBrowserExecuteAllowed(): never;
+export declare function isBrowserExecuteAllowed(): boolean;
+//# sourceMappingURL=browser-script-policy.d.ts.map