@elizaos/plugin-vision 2.0.0-beta.1 → 2.0.3-beta.5

package/dist/describe-backpressure.d.ts

@@ -0,0 +1,90 @@
+export type MemoryPressureLevel = "nominal" | "low" | "critical";
+export type DescribePauseReason = "arbiter-pressure" | "memory-cap" | null;
+export interface DescribeBackpressureStats {
+    /** True while the describe step is currently being skipped. */
+    paused: boolean;
+    /** Last arbiter pressure level applied via `setPressure`. */
+    pressureLevel: MemoryPressureLevel;
+    /** Describe ticks skipped because of backpressure since construction. */
+    describesSkipped: number;
+    /** Count of paused<->active edges (telemetry / test signal). */
+    pauseTransitions: number;
+    /** RSS captured on the first describe tick, used as the local cap baseline. */
+    memoryBaselineBytes: number | null;
+    /** Latest sampled RSS growth over the captured baseline. */
+    memoryGrowthBytes: number | null;
+}
+export interface DescribeBackpressureDecision {
+    /** Run the expensive describe this tick? */
+    describe: boolean;
+    /** `"paused"`/`"active"` when this call flipped the state, else `null`. */
+    transitionedTo: "paused" | "active" | null;
+    /** Why we are paused (only meaningful when `describe === false`). */
+    reason: DescribePauseReason;
+    /** How long the current continuous pause has lasted, in ms. */
+    pausedForMs: number;
+    /** True when the caller should emit a throttled long-pause warning. */
+    warnPaused: boolean;
+}
+export interface DescribeBackpressureConfig {
+    /**
+     * RSS growth cap in bytes. The first describe tick captures the process RSS
+     * baseline; while sampled RSS exceeds `baseline + memoryCapBytes`, the
+     * describe step pauses. `0` or negative disables the local check — only the
+     * arbiter signal can pause describing.
+     */
+    memoryCapBytes?: number;
+    /**
+     * RSS sampler; defaults to `process.memoryUsage().rss`. Injected by tests so
+     * the cap can be exercised deterministically without allocating memory.
+     */
+    sampleRssBytes?: () => number;
+    /**
+     * How long a single arbiter pressure signal keeps the loop paused, in ms.
+     * Because the WS1 bridge delivers pressure but not recovery, the pause
+     * auto-clears after this window of silence. Default 15_000.
+     */
+    arbiterPauseCooldownMs?: number;
+    /** Continuous pause duration before a warning is requested. Default 60s. */
+    pauseWarningThresholdMs?: number;
+    /** Minimum interval between repeated long-pause warnings. Default 60s. */
+    pauseWarningIntervalMs?: number;
+    /** Clock, injectable for tests. Defaults to `Date.now`. */
+    now?: () => number;
+}
+export declare class DescribeBackpressureController {
+    private readonly memoryCapBytes;
+    private readonly sampleRssBytes;
+    private readonly arbiterPauseCooldownMs;
+    private readonly pauseWarningThresholdMs;
+    private readonly pauseWarningIntervalMs;
+    private readonly now;
+    private pressureLevel;
+    private pauseUntilMs;
+    private paused;
+    private describesSkipped;
+    private pauseTransitions;
+    private memoryBaselineBytes;
+    private latestMemoryGrowthBytes;
+    private pauseStartedAtMs;
+    private lastPauseWarningAtMs;
+    constructor(config?: DescribeBackpressureConfig);
+    /**
+     * Apply an arbiter memory-pressure level. A non-nominal level opens (or
+     * extends) the cooldown pause window; `nominal` clears it immediately (only
+     * arbiters that actually report recovery do this — the WS1 bridge relies on
+     * the cooldown instead).
+     */
+    setPressure(level: MemoryPressureLevel): void;
+    /**
+     * Decide whether the expensive describe step may run this tick. Call ONLY
+     * when a describe would otherwise happen (the change/time gate already
+     * passed), so the skip counter reflects real avoided work. Has side effects:
+     * updates the skip counter and the pause/resume transition state. The
+     * arbiter signal takes precedence over the local cap when both are active so
+     * the reported `reason` is the more authoritative one.
+     */
+    evaluate(): DescribeBackpressureDecision;
+    stats(): DescribeBackpressureStats;
+}
+//# sourceMappingURL=describe-backpressure.d.ts.map

package/dist/describe-backpressure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"describe-backpressure.d.ts","sourceRoot":"","sources":["../src/describe-backpressure.ts"],"names":[],"mappings":"AAmCA,MAAM,MAAM,mBAAmB,GAAG,SAAS,GAAG,KAAK,GAAG,UAAU,CAAC;AAEjE,MAAM,MAAM,mBAAmB,GAAG,kBAAkB,GAAG,YAAY,GAAG,IAAI,CAAC;AAE3E,MAAM,WAAW,yBAAyB;IACxC,+DAA+D;IAC/D,MAAM,EAAE,OAAO,CAAC;IAChB,6DAA6D;IAC7D,aAAa,EAAE,mBAAmB,CAAC;IACnC,yEAAyE;IACzE,gBAAgB,EAAE,MAAM,CAAC;IACzB,gEAAgE;IAChE,gBAAgB,EAAE,MAAM,CAAC;IACzB,+EAA+E;IAC/E,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,4DAA4D;IAC5D,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,4BAA4B;IAC3C,4CAA4C;IAC5C,QAAQ,EAAE,OAAO,CAAC;IAClB,2EAA2E;IAC3E,cAAc,EAAE,QAAQ,GAAG,QAAQ,GAAG,IAAI,CAAC;IAC3C,qEAAqE;IACrE,MAAM,EAAE,mBAAmB,CAAC;IAC5B,+DAA+D;IAC/D,WAAW,EAAE,MAAM,CAAC;IACpB,uEAAuE;IACvE,UAAU,EAAE,OAAO,CAAC;CACrB;AAED,MAAM,WAAW,0BAA0B;IACzC;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,MAAM,CAAC;IAC9B;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,4EAA4E;IAC5E,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,0EAA0E;IAC1E,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAMD,qBAAa,8BAA8B;IACzC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAe;IAC9C,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,uBAAuB,CAAS;IACjD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAS;IAChD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,mBAAmB,CAAuB;IAClD,OAAO,CAAC,uBAAuB,CAAuB;IACtD,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,oBAAoB,CAAK;gBAErB,MAAM,GAAE,0BAA+B;IAyBnD;;;;;OAKG;IACH,WAAW,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAS7C;;;;;;;OAOG;IACH,QAAQ,IAAI,4BAA4B;IA2DxC,KAAK,IAAI,yBAAyB;CAUnC"}

package/dist/dirty-tile-describer.d.ts

@@ -0,0 +1,102 @@
+/**
+ * DirtyTileDescriber — change-gated, per-tile screen description (#9105 M3).
+ *
+ * The dominant token cost in a CUA loop is re-describing a whole screen to a
+ * VLM every step even when almost nothing moved. The Brain already skips the
+ * describe entirely when the *whole frame* dHash is unchanged
+ * (`plugin-computeruse` `Brain` frame-dHash cache). This describer is the
+ * finer-grained tier: it splits a frame into tiles (via `screen-tiler.ts`),
+ * computes a per-tile perceptual hash, and only (re)describes tiles whose hash
+ * changed since the last frame — every unchanged tile reuses its cached
+ * description. So a single text field flipping characters re-describes one tile,
+ * not the entire screen.
+ *
+ * The describer is pure + injectable: the tile hash (`hashTile`) and the
+ * per-tile describe call (`describeTile`) are supplied by the caller. The real
+ * boot wiring injects `plugin-computeruse`'s `frameDhash` (the existing
+ * `scene/dhash.ts`) and a `runtime.useModel(IMAGE_DESCRIPTION)`-backed describe;
+ * tests inject deterministic fakes with no model and no native dHash. The
+ * counters (`describeCallsSaved`, `approxTokensSaved`) make the saving
+ * measurable so a test can assert it.
+ */
+import type { ScreenTile } from "./screen-tiler.js";
+/** Approx image tokens charged for one tile describe — used only for the saved-tokens estimate. */
+export declare const APPROX_TOKENS_PER_TILE = 256;
+/** A described tile: its source rectangle plus the VLM/OCR text for it. */
+export interface DescribedTile {
+    /** Tiler id, e.g. `tile-1-0`. */
+    id: string;
+    displayId: string;
+    /** Top-left of the tile in source display pixel space. */
+    sourceX: number;
+    sourceY: number;
+    sourceW: number;
+    sourceH: number;
+    /** The description text for this tile. */
+    description: string;
+    /** True when this tile's description was reused from cache (no describe call). */
+    cached: boolean;
+}
+export interface DirtyTileDescription {
+    /** One entry per tile, in tiler order. */
+    tiles: DescribedTile[];
+    /** Composed full-frame description (non-empty tile texts, source-order). */
+    vlmScene: string;
+    /** Per-tile elements suitable for `Scene.vlm_elements`. */
+    elements: DirtyTileElement[];
+}
+/** A described tile projected into the `Scene.vlm_elements` shape. */
+export interface DirtyTileElement {
+    id: string;
+    kind: string;
+    desc: string;
+    /** Display-local `[x, y, w, h]` of the tile. */
+    bbox: [number, number, number, number];
+    displayId: number;
+}
+/** Token-accounting snapshot for a describer. */
+export interface DirtyTileStats {
+    /** Tiles actually sent to the describe call. */
+    tilesDescribed: number;
+    /** Tiles served from the per-tile cache (no describe call). */
+    tilesSkipped: number;
+    /** Describe calls avoided by the cache (== tilesSkipped). */
+    describeCallsSaved: number;
+    /** Approx image tokens avoided (tilesSkipped × APPROX_TOKENS_PER_TILE). */
+    approxTokensSaved: number;
+}
+export interface DirtyTileDescriberDeps {
+    /**
+     * Perceptual hash of a tile PNG. Identical pixels MUST hash equal. The boot
+     * wiring passes `plugin-computeruse`'s `frameDhash`; `null` means "could not
+     * hash" and forces a (re)describe for that tile.
+     */
+    hashTile: (png: Buffer) => bigint | null;
+    /** Describe one tile's pixels. Only called for changed/new tiles. */
+    describeTile: (tile: ScreenTile) => Promise<string>;
+    /** Tiling options forwarded to `tileScreenshot`. */
+    maxEdge?: number;
+    overlapFraction?: number;
+    /** Tokens charged per describe, for the saved-tokens estimate. */
+    approxTokensPerTile?: number;
+}
+export declare class DirtyTileDescriber {
+    private readonly deps;
+    /** tileId → { hash, description } from the previous frame. */
+    private readonly cache;
+    private stats;
+    constructor(deps: DirtyTileDescriberDeps);
+    getStats(): DirtyTileStats;
+    /**
+     * Describe a frame, re-describing only tiles whose hash changed since the
+     * previous call. Unchanged tiles reuse their cached description.
+     */
+    describe(input: {
+        displayId: number;
+        width: number;
+        height: number;
+        pngBytes: Buffer;
+    }): Promise<DirtyTileDescription>;
+    private toDescribed;
+}
+//# sourceMappingURL=dirty-tile-describer.d.ts.map

package/dist/dirty-tile-describer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dirty-tile-describer.d.ts","sourceRoot":"","sources":["../src/dirty-tile-describer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAGpD,mGAAmG;AACnG,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,kFAAkF;IAClF,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,0CAA0C;IAC1C,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,QAAQ,EAAE,gBAAgB,EAAE,CAAC;CAC9B;AAED,sEAAsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,gDAAgD;IAChD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,iDAAiD;AACjD,MAAM,WAAW,cAAc;IAC7B,gDAAgD;IAChD,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,kBAAkB,EAAE,MAAM,CAAC;IAC3B,2EAA2E;IAC3E,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,sBAAsB;IACrC;;;;OAIG;IACH,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACzC,qEAAqE;IACrE,YAAY,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACpD,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,kBAAkB;IAajB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAZjC,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,KAAK,CAGlB;IACJ,OAAO,CAAC,KAAK,CAKX;gBAE2B,IAAI,EAAE,sBAAsB;IAEzD,QAAQ,IAAI,cAAc;IAI1B;;;OAGG;IACG,QAAQ,CAAC,KAAK,EAAE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;KAClB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAqEjC,OAAO,CAAC,WAAW;CAgBpB"}

package/dist/dirty-tile-scene.d.ts

@@ -0,0 +1,56 @@
+/**
+ * Wiring seam for the change-gated per-tile scene describe (#9105 efficiency).
+ *
+ * `DirtyTileDescriber` (dirty-tile-describer.ts) is pure + injectable: it owns
+ * the per-tile hash cache and the "only re-describe changed tiles" loop, but it
+ * does not know how to hash a tile or how to ask the VLM to describe one. This
+ * module supplies those two collaborators from the live runtime so
+ * `VisionService` can build a describer that re-describes only the screen
+ * regions that actually changed since the previous frame instead of paying for
+ * a whole-frame VLM pass every scene tick.
+ *
+ * Two collaborators:
+ *   - `hashTile`: a perceptual hash. We reuse plugin-computeruse's `frameDhash`
+ *     (the same dHash the Brain frame-cache uses), resolved via a best-effort
+ *     dynamic import so plugin-vision never eagerly pulls computeruse's module
+ *     graph at boot — exactly the idiom the OCR bridge already uses. When
+ *     computeruse is absent the resolve returns `null` and the caller degrades
+ *     to the existing full-frame describe.
+ *   - `describeTile`: one `runtime.useModel(IMAGE_DESCRIPTION, …)` call per
+ *     changed tile, built from a caller-supplied prompt + result normalizer so
+ *     the per-tile path reuses the same prompt plumbing as the full-frame path.
+ */
+import type { ScreenTile } from "./screen-tiler.js";
+/** PNG perceptual hash. Identical pixels MUST hash equal; `null` = undecodable. */
+export type FrameHash = (png: Buffer) => bigint | null;
+/** Per-tile describe call. Returns the model's description text for one tile. */
+export type TileDescribeFn = (tile: ScreenTile) => Promise<string>;
+export interface TileDescribeDeps {
+    /**
+     * Build the per-tile image URL the VLM is asked to describe. The tile carries
+     * PNG bytes (`tile.pngBytes`), so this is a `data:image/png;base64,…` URL.
+     */
+    buildTileImageUrl: (tile: ScreenTile) => string;
+    /**
+     * Build the per-tile prompt. Receives the tile so callers can include bounds.
+     * Async because the scene context is pulled from a peer provider per call.
+     */
+    buildTilePrompt: (tile: ScreenTile) => Promise<string>;
+    /** Invoke the IMAGE_DESCRIPTION model and return its raw result. */
+    invokeModel: (imageUrl: string, prompt: string) => Promise<unknown>;
+    /**
+     * Normalize a model result into a description string, or `null` when the
+     * result is unusable (sentinel / empty). A `null` result yields an empty tile
+     * description, which the describer treats as "nothing to compose for this
+     * tile" while still caching the (empty) result against the tile hash.
+     */
+    extractDescription: (result: unknown) => string | null;
+}
+/**
+ * Build a `describeTile` function bound to the runtime's IMAGE_DESCRIPTION
+ * model. The describer calls this only for tiles whose hash changed.
+ */
+export declare function createTileDescribeFn(deps: TileDescribeDeps): TileDescribeFn;
+/** Encode a tile's PNG bytes into a base64 data URL for the VLM. */
+export declare function tilePngToImageUrl(tile: ScreenTile): string;
+//# sourceMappingURL=dirty-tile-scene.d.ts.map

package/dist/dirty-tile-scene.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dirty-tile-scene.d.ts","sourceRoot":"","sources":["../src/dirty-tile-scene.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAEpD,mFAAmF;AACnF,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;AAEvD,iFAAiF;AACjF,MAAM,MAAM,cAAc,GAAG,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAEnE,MAAM,WAAW,gBAAgB;IAC/B;;;OAGG;IACH,iBAAiB,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,MAAM,CAAC;IAChD;;;OAGG;IACH,eAAe,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,oEAAoE;IACpE,WAAW,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE;;;;;OAKG;IACH,kBAAkB,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,GAAG,IAAI,CAAC;CACxD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,gBAAgB,GAAG,cAAc,CAO3E;AAED,oEAAoE;AACpE,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,UAAU,GAAG,MAAM,CAE1D"}

package/dist/entity-tracker.d.ts

@@ -0,0 +1,33 @@
+import { type IAgentRuntime } from "@elizaos/core";
+import type { DetectedObject, PersonInfo, TrackedEntity, WorldState } from "./types";
+export declare class EntityTracker {
+    private worldState;
+    private readonly POSITION_THRESHOLD;
+    private readonly MISSING_THRESHOLD;
+    private readonly CLEANUP_THRESHOLD;
+    constructor(worldId: string);
+    updateEntities(detectedObjects: DetectedObject[], people: PersonInfo[], faceProfiles?: Map<string, string>, // Maps person ID to face profile ID
+    runtime?: IAgentRuntime): Promise<TrackedEntity[]>;
+    private trackPerson;
+    private trackObject;
+    private findMatchingEntity;
+    private calculateDistance;
+    private updateWorldState;
+    private syncWithRuntime;
+    getWorldState(): WorldState;
+    getActiveEntities(): TrackedEntity[];
+    getEntity(entityId: string): TrackedEntity | undefined;
+    getRecentlyLeft(): Array<{
+        entity: TrackedEntity;
+        leftAt: number;
+    }>;
+    assignNameToEntity(entityId: string, name: string): boolean;
+    getStatistics(): {
+        totalEntities: number;
+        activeEntities: number;
+        recentlyLeft: number;
+        people: number;
+        objects: number;
+    };
+}
+//# sourceMappingURL=entity-tracker.d.ts.map

package/dist/entity-tracker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-tracker.d.ts","sourceRoot":"","sources":["../src/entity-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoB,KAAK,aAAa,EAAU,MAAM,eAAe,CAAC;AAC7E,OAAO,KAAK,EAEV,cAAc,EACd,UAAU,EACV,aAAa,EACb,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,qBAAa,aAAa;IACxB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;gBAE/B,OAAO,EAAE,MAAM;IAUrB,cAAc,CAClB,eAAe,EAAE,cAAc,EAAE,EACjC,MAAM,EAAE,UAAU,EAAE,EACpB,YAAY,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,oCAAoC;IACxE,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,aAAa,EAAE,CAAC;YAoCb,WAAW;YA8DX,WAAW;IAsDzB,OAAO,CAAC,kBAAkB;IA2C1B,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,gBAAgB;YAuCV,eAAe;IAqD7B,aAAa,IAAI,UAAU;IAI3B,iBAAiB,IAAI,aAAa,EAAE;IAMpC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAItD,eAAe,IAAI,KAAK,CAAC;QAAE,MAAM,EAAE,aAAa,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAanE,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO;IAa3D,aAAa,IAAI;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,MAAM,EAAE,MAAM,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB;CAUF"}

package/dist/face-detector-ggml.d.ts

@@ -0,0 +1,60 @@
+import type { BoundingBox } from "./types";
+/**
+ * Same shape the removed ONNX detector exported. Kept identical so callers can
+ * select the native backend without reshaping results.
+ */
+export interface MediaPipeFaceConfig {
+    modelUrl?: string;
+    modelSha256?: string | null;
+    modelDir?: string;
+    scoreThreshold?: number;
+    trusted?: boolean;
+}
+export interface MediaPipeFaceDetection {
+    bbox: BoundingBox;
+    confidence: number;
+    /**
+     * BlazeFace's 6 keypoints in canonical order:
+     *   0: left eye   1: right eye   2: nose tip
+     *   3: mouth      4: left ear    5: right ear
+     * Coordinates are in source-image absolute pixels.
+     */
+    keypoints?: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+/**
+ * ggml-backed BlazeFace face detector. Mirrors the
+ * `MediaPipeFaceDetector` compatibility surface — same constructor config,
+ * same `MediaPipeFaceDetection` output shape.
+ *
+ * Currently disabled (`isAvailable()` returns `false`) until the
+ * face-cpp model entries gain runtime implementations and a BlazeFace GGUF
+ * artifact lands.
+ */
+export declare class BlazeFaceGgmlDetector {
+    private readonly cfg;
+    private bindings;
+    private handle;
+    private initialized;
+    private initPromise;
+    constructor(config?: MediaPipeFaceConfig);
+    /**
+     * `true` only when both the native library AND the GGUF weights are
+     * on disk. Loading them happens lazily in `initialize()`.
+     */
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    private _initialize;
+    /**
+     * Detect faces in the given image buffer. The buffer can be any
+     * sharp-supported format (PNG, JPEG, raw); we resize/letterbox to
+     * the BlazeFace 128x128 input via sharp, run the native detector,
+     * then return source-pixel bboxes + 6 keypoints.
+     */
+    detect(imageBuffer: Buffer): Promise<MediaPipeFaceDetection[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-detector-ggml.d.ts.map

package/dist/face-detector-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-detector-ggml.d.ts","sourceRoot":"","sources":["../src/face-detector-ggml.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAO3C;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7C;AA4MD;;;;;;;;GAQG;AACH,qBAAa,qBAAqB;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAGlB;IACF,OAAO,CAAC,QAAQ,CAAmC;IACnD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;gBAErC,MAAM,GAAE,mBAAwB;IAQ5C;;;OAGG;WACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAOnB,WAAW;IAsBzB;;;;;OAKG;IACG,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAsD9D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAS/B"}

package/dist/face-detector-mediapipe.d.ts

@@ -0,0 +1,25 @@
+import type { BoundingBox } from "./types";
+export interface MediaPipeFaceConfig {
+    modelUrl?: string;
+    modelSha256?: string | null;
+    modelDir?: string;
+    scoreThreshold?: number;
+    trusted?: boolean;
+}
+export interface MediaPipeFaceDetection {
+    bbox: BoundingBox;
+    confidence: number;
+    keypoints?: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+export declare class MediaPipeFaceDetector {
+    constructor(_config?: MediaPipeFaceConfig);
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    detect(_imageBuffer: Buffer): Promise<MediaPipeFaceDetection[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-detector-mediapipe.d.ts.map

package/dist/face-detector-mediapipe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-detector-mediapipe.d.ts","sourceRoot":"","sources":["../src/face-detector-mediapipe.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,WAAW,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC7C;AAED,qBAAa,qBAAqB;gBAEpB,OAAO,GAAE,mBAAwB;WAIhC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAI5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAM3B,MAAM,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,EAAE,CAAC;IAM/D,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAG/B"}

package/dist/face-recognition-ggml.d.ts

@@ -0,0 +1,94 @@
+import { type MediaPipeFaceDetection } from "./face-detector-ggml";
+import type { BoundingBox, FaceProfile } from "./types";
+/**
+ * Configuration for the ggml-backed face embedder.
+ */
+export interface FaceEmbedGgmlConfig {
+    modelPath?: string;
+    modelDir?: string;
+}
+/**
+ * Cosine distance between two 128-d unit-norm embeddings. Matches
+ * `face_embed_distance` in the C library: 0 for identical, 1 for
+ * orthogonal, 2 for antipodal.
+ */
+export declare function cosineDistance(a: Float32Array, b: Float32Array): number;
+/**
+ * L2 distance between two 128-d embeddings. For unit-norm inputs this
+ * is sqrt(2 - 2*dot(a, b)), in [0, 2].
+ */
+export declare function l2Distance(a: Float32Array, b: Float32Array): number;
+/**
+ * ggml-backed 128-d face embedder: a 128-d L2-normalized descriptor per
+ * detected face, consumed by the `FaceRecognition` class below.
+ */
+export declare class FaceEmbedGgmlRecognizer {
+    private readonly cfg;
+    private bindings;
+    private handle;
+    private initialized;
+    private initPromise;
+    constructor(config?: FaceEmbedGgmlConfig);
+    /**
+     * `true` only when both the native library AND the GGUF weights are
+     * on disk.
+     */
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    private _initialize;
+    /**
+     * Compute a 128-d L2-normalized face embedding from an RGB(A) image
+     * buffer plus a detection record (bbox + BlazeFace landmarks).
+     *
+     * The image is decoded via sharp; pass any sharp-supported format
+     * (PNG, JPEG, raw). `detection` should come from
+     * `BlazeFaceGgmlDetector` so the keypoints already match the
+     * BlazeFace order.
+     */
+    embed(imageBuffer: Buffer, detection: MediaPipeFaceDetection): Promise<Float32Array>;
+    dispose(): Promise<void>;
+}
+/**
+ * A detected face: a native BlazeFace detection plus its 128-d ggml
+ * embedding. Mirrors the fields `VisionService` reads off each result.
+ * The native backend produces no expression / age-gender estimates, so
+ * those attributes are left to higher layers.
+ */
+export interface DetectedFace {
+    detection: {
+        box: BoundingBox;
+    };
+    descriptor: Float32Array;
+}
+/**
+ * Native ggml face recognition: BlazeFace detection + 128-d embedding +
+ * in-memory matching and persistence. When the native `libface` library
+ * or its GGUF weights are not on disk, detection returns an empty list
+ * (recognition is disabled, never faked). Matching and storage are pure
+ * JS and always available.
+ */
+export declare class FaceRecognition {
+    private readonly detector;
+    private readonly embedder;
+    private detectorAvailable;
+    private readonly faceLibrary;
+    private readonly FACE_MATCH_THRESHOLD;
+    private readonly MIN_FACE_SIZE;
+    /**
+     * Detect faces in a raw RGBA frame and compute an embedding for each.
+     * Returns an empty list when the native face backend is unavailable.
+     */
+    detectFaces(imageData: Buffer, width: number, height: number): Promise<DetectedFace[]>;
+    recognizeFace(descriptor: Float32Array): Promise<{
+        profileId: string;
+        distance: number;
+    } | null>;
+    addOrUpdateFace(descriptor: Float32Array, attributes?: Partial<FaceProfile>): Promise<string>;
+    getFaceProfile(profileId: string): FaceProfile | undefined;
+    getAllProfiles(): FaceProfile[];
+    saveFaceLibrary(filePath: string): Promise<void>;
+    loadFaceLibrary(filePath: string): Promise<void>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=face-recognition-ggml.d.ts.map

package/dist/face-recognition-ggml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"face-recognition-ggml.d.ts","sourceRoot":"","sources":["../src/face-recognition-ggml.ts"],"names":[],"mappings":"AAyBA,OAAO,EAEL,KAAK,sBAAsB,EAC5B,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,WAAW,EAAe,WAAW,EAAE,MAAM,SAAS,CAAC;AAkMrE;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CASvE;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAUnE;AAED;;;GAGG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA6C;IACjE,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;gBAErC,MAAM,GAAE,mBAAwB;IAO5C;;;OAGG;WACU,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAgB5C,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAOnB,WAAW;IAoBzB;;;;;;;;OAQG;IACG,KAAK,CACT,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,sBAAsB,GAChC,OAAO,CAAC,YAAY,CAAC;IAsBlB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAS/B;AAID;;;;;GAKG;AACH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE;QAAE,GAAG,EAAE,WAAW,CAAA;KAAE,CAAC;IAChC,UAAU,EAAE,YAAY,CAAC;CAC1B;AAED;;;;;;GAMG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,iBAAiB,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAG1B;IAGF,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAO;IAE5C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAM;IAEpC;;;OAGG;IACG,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,YAAY,EAAE,CAAC;IA+CpB,aAAa,CACjB,UAAU,EAAE,YAAY,GACvB,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAiBpD,eAAe,CACnB,UAAU,EAAE,YAAY,EACxB,UAAU,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,GAChC,OAAO,CAAC,MAAM,CAAC;IAgDlB,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS;IAI1D,cAAc,IAAI,WAAW,EAAE;IAIzB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAShD,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAahD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAI/B"}

package/dist/get-screen-elements.d.ts

@@ -0,0 +1,90 @@
+/**
+ * Pure element-merge core for the GET_SCREEN action (#9105 Slice 2 / M2).
+ *
+ * GET_SCREEN returns a cheap, token-frugal list of grounded, clickable screen
+ * elements unified from three sources: OCR text boxes, accessibility (AX)
+ * clickables, and (optionally) VLM-detected elements. This module is the
+ * deterministic heart of that envelope — it collapses the three sources into a
+ * single deduplicated, stably-ordered element list, recording each element's
+ * `groundingSources` provenance.
+ *
+ * Like the M1 OCR bridge (`computeruse-ocr-bridge.ts`), this is intentionally
+ * dependency-free and pure: the source types live in `@elizaos/plugin-computeruse`
+ * (`SceneOcrBox` / `SceneAxNode` / `SceneVlmElement`), but we describe their
+ * shapes STRUCTURALLY here rather than importing them, to keep the no-hard-dep
+ * rule. That also makes the merge engine fully unit-testable with zero
+ * environment, decoupled from the runtime/native/model wiring (Slice 3).
+ */
+/** Display-local bounding box `[x, y, w, h]`. */
+export type Bbox = readonly [number, number, number, number];
+/** Structural shape of computeruse's `SceneOcrBox`. */
+export interface OcrBoxLike {
+    readonly id: string;
+    readonly text: string;
+    readonly bbox: Bbox;
+    readonly conf?: number;
+    readonly displayId: number;
+}
+/** Structural shape of computeruse's `SceneAxNode`. */
+export interface AxNodeLike {
+    readonly id: string;
+    readonly role: string;
+    readonly label?: string;
+    readonly bbox: Bbox;
+    readonly actions?: readonly string[];
+    readonly displayId: number;
+}
+/** Structural shape of computeruse's `SceneVlmElement`. */
+export interface VlmElementLike {
+    readonly id: string;
+    readonly kind: string;
+    readonly desc: string;
+    readonly bbox: Bbox;
+    readonly displayId: number;
+}
+export type GroundingSource = "ocr" | "ax" | "vlm";
+/** A single unified, grounded screen element in the GET_SCREEN envelope. */
+export interface GetScreenElement {
+    /** Stable id, preferring the AX id, then OCR, then VLM. */
+    id: string;
+    /** Display-local bbox `[x, y, w, h]` of the representative (highest-priority) source. */
+    bbox: [number, number, number, number];
+    /** User-facing text/label: AX label, else OCR text, else VLM description. */
+    text: string;
+    /** Element kind/role when known: AX role, else VLM kind. */
+    kind?: string;
+    displayId: number;
+    /** AX actions when the element is accessibility-grounded. */
+    actions?: string[];
+    /** Provenance — every source that contributed to this element, in fixed
+     * `ocr < ax < vlm` order for stability. Always non-empty. */
+    groundingSources: GroundingSource[];
+}
+export interface MergeScreenInput {
+    readonly ocr?: readonly OcrBoxLike[];
+    readonly ax?: readonly AxNodeLike[];
+    readonly vlm?: readonly VlmElementLike[];
+}
+export interface MergeScreenOptions {
+    /** Boxes whose IoU exceeds this collapse into one element (default 0.6). */
+    readonly iouThreshold?: number;
+}
+/** Intersection-over-union of two `[x, y, w, h]` boxes. 0 when either is empty
+ * or they don't overlap. */
+export declare function bboxIou(a: Bbox, b: Bbox): number;
+/**
+ * Merge OCR boxes + AX clickables + VLM elements into one deduplicated,
+ * deterministically-ordered element list.
+ *
+ * - Elements from different sources whose bboxes overlap above `iouThreshold`
+ *   (and share a `displayId`) collapse into one element that records all
+ *   contributing sources in `groundingSources`.
+ * - Field precedence is AX > OCR > VLM (AX wins id/label/role; OCR text fills
+ *   in when AX has no label; VLM desc is the last resort).
+ * - Output order is top-to-bottom, then left-to-right, so the envelope is
+ *   stable across turns regardless of input ordering.
+ * - Degrades gracefully: any source may be absent/empty (e.g. accessibility off)
+ *   and the function never throws.
+ */
+export declare function mergeScreenElements(input: MergeScreenInput, options?: MergeScreenOptions): GetScreenElement[];
+//# sourceMappingURL=get-screen-elements.d.ts.map

package/dist/get-screen-elements.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-screen-elements.d.ts","sourceRoot":"","sources":["../src/get-screen-elements.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,iDAAiD;AACjD,MAAM,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAE7D,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,uDAAuD;AACvD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,IAAI,GAAG,KAAK,CAAC;AAEnD,4EAA4E;AAC5E,MAAM,WAAW,gBAAgB;IAC/B,2DAA2D;IAC3D,EAAE,EAAE,MAAM,CAAC;IACX,yFAAyF;IACzF,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB;iEAC6D;IAC7D,gBAAgB,EAAE,eAAe,EAAE,CAAC;CACrC;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACrC,QAAQ,CAAC,EAAE,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACpC,QAAQ,CAAC,GAAG,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;CAC1C;AAED,MAAM,WAAW,kBAAkB;IACjC,4EAA4E;IAC5E,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAChC;AAID;4BAC4B;AAC5B,wBAAgB,OAAO,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,GAAG,MAAM,CAchD;AAoBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,gBAAgB,EACvB,OAAO,GAAE,kBAAuB,GAC/B,gBAAgB,EAAE,CAkEpB"}

package/dist/get-screen.d.ts

@@ -0,0 +1,60 @@
+/**
+ * GET_SCREEN core (issue #9105 / M2) — token-frugal structured screen readout.
+ *
+ * Returns OCR text + grounded elements (id/text/bbox/semantic position) from a
+ * captured frame using the registered coord-OCR service (native Windows OCR /
+ * docTR / Apple Vision — zero LLM tokens). The raw image is OMITTED by default
+ * (`includeImage:false`) so a CUA loop can read the screen each tick without
+ * spending image tokens; it is only base64-attached when explicitly requested.
+ *
+ * Pure + injectable (the OCR service can be passed in) so it is unit-testable
+ * without a real capture or a registered provider.
+ */
+import { type OcrWithCoordsService } from "./ocr-with-coords.js";
+export interface GetScreenElement {
+    /**
+     * Monotonic 1-based Set-of-Marks number (reading order). A model can pick
+     * `[3]` instead of regressing raw coordinates.
+     */
+    index: number;
+    /** Stable per-result id. */
+    id: string;
+    text: string;
+    /** Display-absolute [x, y, width, height]. */
+    bbox: [number, number, number, number];
+    /** Click target — the integer center of `bbox`, the point `index` resolves to. */
+    center: {
+        x: number;
+        y: number;
+    };
+    semantic_position: string;
+    displayId: number;
+}
+export interface GetScreenResult {
+    op: "get_screen";
+    displayId: number;
+    width: number;
+    height: number;
+    /** When the source frame was captured (ms epoch). */
+    lastChangeTime: number;
+    /** True when a coord-OCR provider was available and ran. */
+    ocrAvailable: boolean;
+    ocrText: string;
+    elements: GetScreenElement[];
+    elementCount: number;
+    /** Base64 PNG — only present when `includeImage` was requested. */
+    image?: string;
+}
+export interface BuildGetScreenOptions {
+    pngBytes: Uint8Array;
+    displayId?: number;
+    includeImage?: boolean;
+    includeOcr?: boolean;
+    capturedAt?: number;
+    /** Override for tests; defaults to the registered coord-OCR service. */
+    ocrService?: OcrWithCoordsService | null;
+}
+export declare function buildGetScreen(opts: BuildGetScreenOptions): Promise<GetScreenResult>;
+/** Human-readable one-line summary for the agent reply. */
+export declare function summarizeGetScreen(r: GetScreenResult): string;
+//# sourceMappingURL=get-screen.d.ts.map