@elizaos/plugin-vision 2.0.0-beta.1 → 2.0.3-beta.6

package/dist/ocr-service.d.ts

@@ -0,0 +1,91 @@
+import type { BoundingBox, OCRResult, ScreenTile } from "./types";
+export type OCRBackendName = "doctr" | "apple-vision";
+export interface OCRServiceConfig {
+    /**
+     * Force a specific backend. If unset, the chain is:
+     *   1. Apple Vision (darwin only, when a provider has been registered)
+     *   2. doCTR (ggml-backed CRNN+DBNet via native/doctr.cpp)
+     *
+     * There is no tesseract / onnx fallback — the migration removed both.
+     * If neither backend can initialize, `initialize()` throws.
+     */
+    backend?: OCRBackendName;
+}
+export interface StructuredOCRData {
+    tables: Array<{
+        rows: string[][];
+        bbox: BoundingBox;
+    }>;
+    forms: Array<{
+        label: string;
+        value: string;
+        bbox: BoundingBox;
+    }>;
+    lists: Array<{
+        items: string[];
+        bbox: BoundingBox;
+    }>;
+}
+export declare function extractStructuredDataFromOCR(ocr: OCRResult): StructuredOCRData;
+/**
+ * External provider seam for the Apple Vision OCR backend.
+ *
+ * `plugin-vision` does not take a runtime dep on `@elizaos/plugin-computeruse`
+ * — that would invert the layering (computeruse is the higher-level seam).
+ * Instead, the runtime registers a provider here on iOS/macOS startup using
+ * `createIosVisionOcrProvider(...)` from
+ * `@elizaos/plugin-computeruse/mobile/ocr-provider`. Until a provider is
+ * registered, `AppleVisionBackend.extractText` throws so the chooser falls
+ * through to the doCTR ggml backend.
+ *
+ * The provider shape is intentionally structural so plugin-vision stays
+ * Node-importable on hosts that don't ship Capacitor.
+ */
+export interface AppleVisionOcrProvider {
+    /** Stable id used in logs/telemetry. */
+    readonly name: string;
+    /** True when the underlying bridge is registered and ready. */
+    available(): boolean;
+    /**
+     * Recognize text in the JPEG/PNG bytes. The plugin-computeruse iOS provider
+     * returns `OcrResult`; we map to plugin-vision's `OCRResult` shape inline.
+     */
+    recognize(input: {
+        kind: "bytes";
+        data: Uint8Array;
+    }): Promise<{
+        readonly lines: ReadonlyArray<{
+            readonly text: string;
+            readonly confidence: number;
+            readonly boundingBox: {
+                readonly x: number;
+                readonly y: number;
+                readonly width: number;
+                readonly height: number;
+            };
+        }>;
+        readonly fullText: string;
+    }>;
+}
+export declare function registerAppleVisionOcrProvider(provider: AppleVisionOcrProvider | null): void;
+export declare function getAppleVisionOcrProvider(): AppleVisionOcrProvider | null;
+/**
+ * Walk the priority chain and pick the first backend that initializes.
+ * Backend instances are cached; per-call we just dispatch to the active one.
+ */
+export declare class OCRService {
+    private backends;
+    private chosen;
+    private initialized;
+    private readonly forced?;
+    constructor(config?: OCRServiceConfig);
+    initialize(): Promise<void>;
+    extractText(imageBuffer: Buffer): Promise<OCRResult>;
+    extractFromTile(tile: ScreenTile): Promise<OCRResult>;
+    extractFromImage(imageBuffer: Buffer): Promise<OCRResult>;
+    extractStructuredData(imageBuffer: Buffer): Promise<StructuredOCRData>;
+    getActiveBackend(): OCRBackendName | null;
+    isInitialized(): boolean;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=ocr-service.d.ts.map

package/dist/ocr-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr-service.d.ts","sourceRoot":"","sources":["../src/ocr-service.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAElE,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,cAAc,CAAC;AAEtD,MAAM,WAAW,gBAAgB;IAC/B;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,cAAc,CAAC;CAC1B;AASD,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC;QAAC,IAAI,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IACvD,KAAK,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;IAClE,KAAK,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAC,IAAI,EAAE,WAAW,CAAA;KAAE,CAAC,CAAC;CACtD;AAqBD,wBAAgB,4BAA4B,CAC1C,GAAG,EAAE,SAAS,GACb,iBAAiB,CAiEnB;AAgBD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,sBAAsB;IACrC,wCAAwC;IACxC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,SAAS,IAAI,OAAO,CAAC;IACrB;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,UAAU,CAAA;KAAE,GAAG,OAAO,CAAC;QAC7D,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC;YAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;YAC5B,QAAQ,CAAC,WAAW,EAAE;gBACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;gBACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;gBACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;aACzB,CAAC;SACH,CAAC,CAAC;QACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B,CAAC,CAAC;CACJ;AAID,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,sBAAsB,GAAG,IAAI,GACtC,IAAI,CAON;AAED,wBAAgB,yBAAyB,IAAI,sBAAsB,GAAG,IAAI,CAEzE;AAqED;;;GAGG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,MAAM,CAA2B;IACzC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAiB;gBAE7B,MAAM,GAAE,gBAAqB;IAInC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAkD3B,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAiCpD,eAAe,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC;IAOrD,gBAAgB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAIzD,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAK5E,gBAAgB,IAAI,cAAc,GAAG,IAAI;IAIzC,aAAa,IAAI,OAAO;IAIlB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAW/B"}

package/dist/ocr-with-coords.d.ts

@@ -0,0 +1,103 @@
+/**
+ * OCR-with-coords — hierarchical (block / line / word) OCR with absolute
+ * source-display coordinates and a coarse semantic position label per
+ * recognized text element.
+ *
+ * Why this lives in plugin-vision:
+ *   - plugin-computeruse needs OCR with coordinates so action targets can be
+ *     computed in display-absolute coordinates without re-running detection.
+ *   - It cannot take a runtime dep on plugin-vision (which would invert the
+ *     layering: computeruse is the higher-level seam and plugin-vision must
+ *     stay Node-importable on hosts that don't ship the action surface).
+ *   - Mirroring the pattern used by `AppleVisionOcrProvider` in
+ *     `./ocr-service.ts`, plugin-vision exports a structural interface plus a
+ *     registry seam (`registerCoordOcrProvider` lives in
+ *     plugin-computeruse/src/mobile/ocr-provider.ts) that the runtime wires up
+ *     at boot.
+ *
+ * This file defines the canonical `OcrWithCoordsService` interface and the
+ * in-tree `RapidOcrCoordAdapter` provider. The adapter is backed by the
+ * existing `RapidOCRService` and computes `semantic_position`
+ * deterministically from the bbox center against tile-relative thirds. Native
+ * OCR providers can register the same interface without changing consumers.
+ */
+import { OCRService } from "./ocr-service";
+import type { BoundingBox } from "./types";
+/** Coarse 3x3 location of a text element relative to the source tile. */
+export type SemanticPosition = "upper-left" | "upper-center" | "upper-right" | "middle-left" | "center" | "middle-right" | "lower-left" | "lower-center" | "lower-right";
+export interface OcrWithCoordsWord {
+    readonly text: string;
+    /** Absolute source-display coordinates. */
+    readonly bbox: BoundingBox;
+    readonly semantic_position: SemanticPosition;
+}
+export interface OcrWithCoordsBlock {
+    readonly text: string;
+    /** Absolute source-display coordinates. */
+    readonly bbox: BoundingBox;
+    readonly words: ReadonlyArray<OcrWithCoordsWord>;
+    readonly semantic_position: SemanticPosition;
+}
+export interface OcrWithCoordsResult {
+    readonly blocks: ReadonlyArray<OcrWithCoordsBlock>;
+}
+export interface OcrWithCoordsInput {
+    /** Stable identifier of the source display. Echoed in logs only. */
+    readonly displayId: string;
+    /** Absolute X offset of the tile within the source display. */
+    readonly sourceX: number;
+    /** Absolute Y offset of the tile within the source display. */
+    readonly sourceY: number;
+    /** Encoded PNG bytes of the tile. */
+    readonly pngBytes: Uint8Array;
+}
+export interface OcrWithCoordsService {
+    readonly name: string;
+    describe(input: OcrWithCoordsInput): Promise<OcrWithCoordsResult>;
+}
+/**
+ * Map a bbox center to one of nine semantic positions using strict thirds
+ * against the tile dimensions. Pure function — exported for tests so the
+ * thirds rule has a single source of truth.
+ *
+ * Rule:
+ *   col = floor(centerX / (tileWidth / 3)) clamped to [0, 2]
+ *   row = floor(centerY / (tileHeight / 3)) clamped to [0, 2]
+ *   "middle" + "center" collapses to the literal "center".
+ *
+ * Inputs use tile-relative coordinates so the same function works for words
+ * inside their parent block too (callers can pass the parent block bbox as
+ * the tile dims for word-relative labeling, but for the canonical
+ * implementation here we always label against the source tile).
+ */
+export declare function computeSemanticPosition(args: {
+    readonly bbox: BoundingBox;
+    readonly tileWidth: number;
+    readonly tileHeight: number;
+}): SemanticPosition;
+export declare function registerOcrWithCoordsService(service: OcrWithCoordsService | null): void;
+export declare function getOcrWithCoordsService(): OcrWithCoordsService | null;
+/**
+ * Wraps the existing `RapidOCRService` and maps its line-level output to the
+ * hierarchical `OcrWithCoordsResult` shape, computing `semantic_position`
+ * deterministically against the source tile thirds.
+ */
+export declare class RapidOcrCoordAdapter implements OcrWithCoordsService {
+    private readonly impl;
+    readonly name = "rapid-coord-adapter";
+    constructor(impl?: Pick<OCRService, "extractText">);
+    describe(input: OcrWithCoordsInput): Promise<OcrWithCoordsResult>;
+}
+/**
+ * Read width/height from the PNG IHDR chunk without pulling in sharp on the
+ * test path. PNG signature is 8 bytes; IHDR begins at offset 8 with a 4-byte
+ * length, 4-byte type ("IHDR"), then 4-byte width and 4-byte height (BE).
+ *
+ * Throws on malformed input so a corrupt tile surfaces immediately rather
+ * than silently producing zero-sized semantic-position math.
+ */
+export declare function readPngDimensions(pngBytes: Uint8Array): Promise<{
+    width: number;
+    height: number;
+}>;
+//# sourceMappingURL=ocr-with-coords.d.ts.map

package/dist/ocr-with-coords.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr-with-coords.d.ts","sourceRoot":"","sources":["../src/ocr-with-coords.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAE3C,yEAAyE;AACzE,MAAM,MAAM,gBAAgB,GACxB,YAAY,GACZ,cAAc,GACd,aAAa,GACb,aAAa,GACb,QAAQ,GACR,cAAc,GACd,YAAY,GACZ,cAAc,GACd,aAAa,CAAC;AAElB,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;CAC9C;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,2CAA2C;IAC3C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,iBAAiB,CAAC,CAAC;IACjD,QAAQ,CAAC,iBAAiB,EAAE,gBAAgB,CAAC;CAC9C;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,kBAAkB,CAAC,CAAC;CACpD;AAED,MAAM,WAAW,kBAAkB;IACjC,oEAAoE;IACpE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,qCAAqC;IACrC,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;CAC/B;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;CACnE;AAeD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B,GAAG,gBAAgB,CAenB;AAYD,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,oBAAoB,GAAG,IAAI,GACnC,IAAI,CAON;AAED,wBAAgB,uBAAuB,IAAI,oBAAoB,GAAG,IAAI,CAErE;AAID;;;;GAIG;AACH,qBAAa,oBAAqB,YAAW,oBAAoB;IAI7D,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHvB,QAAQ,CAAC,IAAI,yBAAyB;gBAGnB,IAAI,GAAE,IAAI,CAAC,UAAU,EAAE,aAAa,CAAoB;IAGrE,QAAQ,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAuDxE;AAMD;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,UAAU,GACnB,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CA8B5C"}

package/dist/person-detector.d.ts

@@ -0,0 +1,17 @@
+import type { PersonInfo } from "./types";
+import { type YOLOConfig } from "./yolo-detector";
+export interface PersonDetectorConfig extends Omit<YOLOConfig, "classFilter"> {
+    /** Score threshold specifically for person detections (defaults to 0.4). */
+    scoreThreshold?: number;
+}
+export declare class PersonDetector {
+    private yolo;
+    private initialized;
+    constructor(config?: PersonDetectorConfig);
+    static isAvailable(): Promise<boolean>;
+    isInitialized(): boolean;
+    initialize(): Promise<void>;
+    detect(imageBuffer: Buffer): Promise<PersonInfo[]>;
+    dispose(): Promise<void>;
+}
+//# sourceMappingURL=person-detector.d.ts.map

package/dist/person-detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"person-detector.d.ts","sourceRoot":"","sources":["../src/person-detector.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,EAAE,KAAK,UAAU,EAAgB,MAAM,iBAAiB,CAAC;AAEhE,MAAM,WAAW,oBAAqB,SAAQ,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC;IAC3E,4EAA4E;IAC5E,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAAS;gBAEhB,MAAM,GAAE,oBAAyB;IAQ7C,MAAM,CAAC,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAItC,aAAa,IAAI,OAAO;IAIlB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,MAAM,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAclD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAI/B"}

package/dist/provider.d.ts

@@ -0,0 +1,3 @@
+import { type Provider } from "@elizaos/core";
+export declare const visionProvider: Provider;
+//# sourceMappingURL=provider.d.ts.map

package/dist/provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../src/provider.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,QAAQ,EAEd,MAAM,eAAe,CAAC;AAYvB,eAAO,MAAM,cAAc,EAAE,QAiW5B,CAAC"}

package/dist/routes.d.ts

@@ -0,0 +1,7 @@
+import type { Route } from "@elizaos/core";
+/** GET — drain the queue of pending capture requests for the renderer poller. */
+export declare const captureRequestsRoute: Route;
+/** POST — accept a captured frame (or a skip) for a queued request. */
+export declare const screenFrameRoute: Route;
+export declare const visionRoutes: Route[];
+//# sourceMappingURL=routes.d.ts.map

package/dist/routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"routes.d.ts","sourceRoot":"","sources":["../src/routes.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AAqC3C,iFAAiF;AACjF,eAAO,MAAM,oBAAoB,EAAE,KAWlC,CAAC;AAEF,uEAAuE;AACvE,eAAO,MAAM,gBAAgB,EAAE,KA2C9B,CAAC;AAEF,eAAO,MAAM,YAAY,EAAE,KAAK,EAA6C,CAAC"}

package/dist/screen-capture-bridge.d.ts

@@ -0,0 +1,51 @@
+import { type IAgentRuntime, Service } from "@elizaos/core";
+/** Service type used to resolve the bridge off the runtime. */
+export declare const SCREEN_CAPTURE_BRIDGE_SERVICE_TYPE = "vision-screen-capture-bridge";
+/** A single enqueued capture request, drained by the GET poll. */
+export interface ScreenCaptureRequest {
+    requestId: string;
+    createdAt: number;
+    displayId?: number;
+}
+/** Result of a completed capture, returned to `requestFrame` callers. */
+export interface ScreenCaptureFrame {
+    pngBytes: Uint8Array;
+    displayId: number;
+    capturedAt: number;
+}
+/**
+ * Renderer-pulled screen-capture bridge service.
+ *
+ * The agent calls `requestFrame()`; the renderer drains the queue via
+ * `takeRequests()` and delivers frames via `submitFrame()`.
+ */
+export declare class ScreenCaptureBridgeService extends Service {
+    static serviceType: string;
+    capabilityDescription: string;
+    private readonly queue;
+    private readonly pending;
+    private readonly timeoutMs;
+    constructor(runtime?: IAgentRuntime, timeoutMs?: number);
+    static start(runtime: IAgentRuntime): Promise<ScreenCaptureBridgeService>;
+    /**
+     * Enqueue a capture request and wait for the renderer to deliver a frame.
+     * Resolves `null` if no frame arrives within the timeout (never hangs).
+     */
+    requestFrame(displayId?: number): Promise<ScreenCaptureFrame | null>;
+    /** Drain and return all queued requests (for the GET poll). */
+    takeRequests(): ScreenCaptureRequest[];
+    /**
+     * Deliver a captured frame for a queued request. Returns false if the
+     * requestId is unknown or already expired/resolved.
+     */
+    submitFrame(requestId: string, base64: string, _format: string, _width: number, _height: number): boolean;
+    /**
+     * Resolve a queued request as a skip/failure so the agent's pending promise
+     * settles promptly (as `null`) instead of waiting the full timeout. The
+     * renderer calls this when a capture throws or is unavailable. Returns false
+     * for unknown/expired requestIds.
+     */
+    failFrame(requestId: string, reason: string): boolean;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=screen-capture-bridge.d.ts.map

package/dist/screen-capture-bridge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-capture-bridge.d.ts","sourceRoot":"","sources":["../src/screen-capture-bridge.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,KAAK,aAAa,EAAU,OAAO,EAAE,MAAM,eAAe,CAAC;AAEpE,+DAA+D;AAC/D,eAAO,MAAM,kCAAkC,iCACf,CAAC;AASjC,kEAAkE;AAClE,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,yEAAyE;AACzE,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,UAAU,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAQD;;;;;GAKG;AACH,qBAAa,0BAA2B,SAAQ,OAAO;IACrD,OAAgB,WAAW,EAAE,MAAM,CAAsC;IAChE,qBAAqB,SACyC;IAEvE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA8B;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqC;IAC7D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAGjC,OAAO,CAAC,EAAE,aAAa,EACvB,SAAS,GAAE,MAAiC;WAMjC,KAAK,CAChB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,0BAA0B,CAAC;IAItC;;;OAGG;IACH,YAAY,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IA2BpE,+DAA+D;IAC/D,YAAY,IAAI,oBAAoB,EAAE;IAItC;;;OAGG;IACH,WAAW,CACT,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,OAAO;IAcV;;;;;OAKG;IACH,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO;IAY/C,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ5B"}

package/dist/screen-capture.d.ts

@@ -0,0 +1,17 @@
+import type { ScreenCapture, ScreenTile, VisionConfig } from "./types";
+export declare class ScreenCaptureService {
+    private config;
+    private activeTileIndex;
+    private lastCapture;
+    constructor(config: VisionConfig);
+    getScreenInfo(): Promise<{
+        width: number;
+        height: number;
+    } | null>;
+    captureScreen(): Promise<ScreenCapture>;
+    private captureScreenToFile;
+    getActiveTile(): ScreenTile | null;
+    getAllTiles(): ScreenTile[];
+    getProcessedTiles(): ScreenTile[];
+}
+//# sourceMappingURL=screen-capture.d.ts.map

package/dist/screen-capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-capture.d.ts","sourceRoot":"","sources":["../src/screen-capture.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAoEvE,qBAAa,oBAAoB;IAC/B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,WAAW,CAA8B;gBAErC,MAAM,EAAE,YAAY;IAI1B,aAAa,IAAI,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IA0DlE,aAAa,IAAI,OAAO,CAAC,aAAa,CAAC;YAkF/B,mBAAmB;IA+CjC,aAAa,IAAI,UAAU,GAAG,IAAI;IAOlC,WAAW,IAAI,UAAU,EAAE;IAI3B,iBAAiB,IAAI,UAAU,EAAE;CAGlC"}

package/dist/screen-tiler.d.ts

@@ -0,0 +1,75 @@
+/**
+ * One output tile from `tileScreenshot`.
+ *
+ * - `sourceX/sourceY` are absolute pixel coords in the *source display's*
+ *   native space (i.e. the same space `displayId` is reported in by
+ *   `plugin-computeruse/src/platform/displays.ts`).
+ * - `tileW/tileH` are the actual rendered dimensions of `pngBytes` and may
+ *   equal `sourceW/sourceH` (no resize) — the tiler does not downscale; it
+ *   only crops. Resizing is the model preprocessor's job.
+ */
+export interface ScreenTile {
+    /** Stable id of the form `tile-<row>-<col>`. */
+    id: string;
+    /** Display this tile was sourced from. Stringified to keep types narrow. */
+    displayId: string;
+    /** Top-left X of the tile in the source display's pixel space. */
+    sourceX: number;
+    /** Top-left Y of the tile in the source display's pixel space. */
+    sourceY: number;
+    /** Width of the cropped region in source pixels. */
+    sourceW: number;
+    /** Height of the cropped region in source pixels. */
+    sourceH: number;
+    /** Pixel width of `pngBytes`. Equal to `sourceW` (no resize). */
+    tileW: number;
+    /** Pixel height of `pngBytes`. Equal to `sourceH` (no resize). */
+    tileH: number;
+    /** PNG-encoded crop. */
+    pngBytes: Buffer;
+}
+export interface TileScreenshotInput {
+    displayId: string;
+    width: number;
+    height: number;
+    pngBytes: Buffer;
+}
+export interface TileScreenshotOptions {
+    /** Maximum tile edge in pixels. Tiles never exceed this in either dim. */
+    maxEdge: number;
+    /**
+     * Fraction of `tileSize` that adjacent tiles overlap. 0.12 (default) is
+     * large enough to keep multi-glyph tokens intact across seams, small enough
+     * to keep tile count near minimum.
+     */
+    overlapFraction: number;
+}
+/** Default local-VLM tile budget for Gemma vision. */
+export declare const DEFAULT_MAX_EDGE = 1280;
+/** Default seam overlap (12%). */
+export declare const DEFAULT_OVERLAP_FRACTION = 0.12;
+/**
+ * Tile a captured screenshot into local-VLM-sized PNG patches with
+ * pixel-overlap between neighbours.
+ *
+ * Single-tile fast path: when both dims fit within `maxEdge`, the input is
+ * returned as a single `ScreenTile` whose pngBytes is the unmodified input.
+ *
+ * Grid path: chooses the smallest grid (cols, rows) such that no individual
+ * tile exceeds `maxEdge`, then computes a per-axis stride that yields
+ * `overlapFraction * tileSize` of overlap between adjacent tiles. The last
+ * column/row is anchored to the source's right/bottom edge so we never
+ * extend past the screen.
+ */
+export declare function tileScreenshot(input: TileScreenshotInput, opts?: TileScreenshotOptions): Promise<ScreenTile[]>;
+/**
+ * Map a (localX, localY) inside a tile back to the source display's
+ * absolute pixel coordinates. Use this to translate "model said click at
+ * (x, y) inside tile-0-1" into a coordinate the input driver can act on.
+ */
+export declare function reconstructAbsoluteCoords(tile: ScreenTile, localX: number, localY: number): {
+    displayId: string;
+    absoluteX: number;
+    absoluteY: number;
+};
+//# sourceMappingURL=screen-tiler.d.ts.map

package/dist/screen-tiler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"screen-tiler.d.ts","sourceRoot":"","sources":["../src/screen-tiler.ts"],"names":[],"mappings":"AAoBA;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACzB,gDAAgD;IAChD,EAAE,EAAE,MAAM,CAAC;IACX,4EAA4E;IAC5E,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,kEAAkE;IAClE,OAAO,EAAE,MAAM,CAAC;IAChB,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,OAAO,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,qBAAqB;IACpC,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,sDAAsD;AACtD,eAAO,MAAM,gBAAgB,OAAO,CAAC;AACrC,kCAAkC;AAClC,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C;;;;;;;;;;;;GAYG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,mBAAmB,EAC1B,IAAI,GAAE,qBAGL,GACA,OAAO,CAAC,UAAU,EAAE,CAAC,CAuEvB;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,UAAU,EAChB,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,GACb;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAgB7D"}

package/dist/service.d.ts

@@ -0,0 +1,176 @@
+import { type IAgentRuntime, Service, type ServiceTypeName } from "@elizaos/core";
+import { EntityTracker } from "./entity-tracker";
+import type { FaceRecognition } from "./face-recognition-ggml";
+import { type BoundingBox, type CameraInfo, type DetectedObject, type EnhancedSceneDescription, type SceneDescription, type ScreenCapture, type VisionFrame, VisionMode } from "./types";
+export interface VisionContextSnapshot {
+    openApps: string[];
+    focusedWindow: {
+        app: string;
+        title: string;
+        bbox: [number, number, number, number] | null;
+    } | null;
+    recentActions: Array<{
+        action: string;
+        ts: number;
+    }>;
+    currentTaskGoal: string | null;
+}
+/**
+ * A face that the ggml face-recognition pipeline matched to a known profile,
+ * shaped for the VLM prompt. `label` is the profile's display name when set,
+ * otherwise the opaque profile id. `bbox` is the detected face region.
+ */
+export interface RecognizedFace {
+    label: string;
+    bbox: BoundingBox;
+}
+export declare function buildSceneDescriptionPrompt(context: VisionContextSnapshot | null, ocrText?: string | null, detectedObjects?: DetectedObject[] | null, recognizedFaces?: RecognizedFace[] | null): string;
+export declare class VisionService extends Service {
+    static serviceType: ServiceTypeName;
+    capabilityDescription: string;
+    private visionConfig;
+    private camera;
+    private lastFrame;
+    private lastSceneDescription;
+    private frameProcessingInterval;
+    private screenProcessingInterval;
+    private isProcessing;
+    private isProcessingScreen;
+    private objectDetector;
+    private hasObjectDetection;
+    private faceRecognition;
+    private entityTracker;
+    private audioCapture;
+    private streamingAudioCapture;
+    private screenCapture;
+    private ocrService;
+    private lastScreenCapture;
+    private lastEnhancedScene;
+    private workerManager;
+    private lastTfUpdateTime;
+    private lastVlmUpdateTime;
+    private lastTfDescription;
+    private readonly describeBackpressure;
+    private arbiterUnsubscribe;
+    private dirtyTileDescriber;
+    private dirtyTileDescriberInit;
+    private dirtyTileDescribeContext;
+    private readonly DEFAULT_CONFIG;
+    constructor(runtime?: IAgentRuntime);
+    private parseConfig;
+    static start(runtime: IAgentRuntime): Promise<VisionService>;
+    private checkCameraTools;
+    private initialize;
+    private initializeScreenVision;
+    private initializeCameraVision;
+    private initializeAudioCapture;
+    private storeAudioTranscription;
+    private startProcessing;
+    /**
+     * Subscribe the describe-backpressure controller to WS1 memory-pressure
+     * events. Resolves the arbiter dynamically (no hard dependency on
+     * `@elizaos/plugin-local-inference`); when none is registered the controller
+     * still pauses on the local RSS cap. Idempotent — a prior subscription is
+     * released first so a restart doesn't double-subscribe.
+     */
+    private attachMemoryArbiter;
+    /**
+     * Clear the arbiter-driven pause. Called by the WS1 bridge consumer when
+     * pressure returns to nominal; also exposed so an embedder can resume the
+     * describe loop explicitly.
+     */
+    resumeDescribeLoop(): void;
+    /** Current describe-backpressure stats (telemetry / tests). */
+    getBackpressureStats(): import("./describe-backpressure").DescribeBackpressureStats;
+    private startFrameProcessing;
+    private captureAndProcessFrame;
+    private processFrameData;
+    private calculatePixelChange;
+    private updateSceneDescription;
+    /**
+     * Normalize the various shapes that `useModel(IMAGE_DESCRIPTION, …)` may
+     * return into a non-empty string. Returns `null` when the result is the
+     * "I'm unable to analyze images" sentinel or empty.
+     */
+    private extractDescriptionFromUseModel;
+    /**
+     * Pull the latest desktop scene context from plugin-computeruse's
+     * VisionContextProvider when registered. Returns `null` when no provider is
+     * available (or when the lookup fails) so the VLM still receives a valid
+     * prompt — the context block is purely additive.
+     */
+    private collectVisionContext;
+    private collectCurrentOcrTextForPrompt;
+    /**
+     * Resolve the change-gated per-tile describer, building it once. Returns
+     * `null` (and degrades to full-frame describe) when no perceptual hash is
+     * available — i.e. plugin-computeruse's `frameDhash` cannot be imported.
+     *
+     * The dHash is resolved via a best-effort dynamic import so plugin-vision
+     * never eagerly pulls computeruse's module graph at boot (same idiom as the
+     * coord-OCR bridge wiring in `index.ts`).
+     */
+    private ensureDirtyTileDescriber;
+    private resolveFrameHash;
+    /**
+     * Build the per-tile describe call bound to this service's runtime. Reads the
+     * current frame's prompt context (`dirtyTileDescribeContext`) so each tile is
+     * described with the same context the full-frame path would use.
+     */
+    private buildTileDescribeFn;
+    /**
+     * Per-tile incremental scene describe. Re-describes only the tiles whose
+     * perceptual hash changed since the previous frame; unchanged tiles reuse
+     * their cached description. Returns `null` when the per-tile path is
+     * unavailable or yields no usable text, so the caller falls back to the
+     * full-frame describe.
+     */
+    private describeSceneWithDirtyTiles;
+    private describeSceneWithVLM;
+    private describeSceneWithVLMInTrajectory;
+    private detectMotionObjects;
+    private mergeAdjacentObjects;
+    private classifyObjectBySize;
+    private detectPeopleFromMotion;
+    private startScreenProcessing;
+    private captureAndProcessScreen;
+    private analyzeTile;
+    private updateEnhancedSceneDescription;
+    getCurrentFrame(): Promise<VisionFrame | null>;
+    getSceneDescription(): Promise<SceneDescription | null>;
+    getEnhancedSceneDescription(): Promise<EnhancedSceneDescription | null>;
+    getScreenCapture(): Promise<ScreenCapture | null>;
+    getVisionMode(): VisionMode;
+    /**
+     * Enable the camera input. If screen is already active, switches to BOTH;
+     * otherwise to CAMERA.
+     */
+    enableCamera(): Promise<void>;
+    /**
+     * Disable the camera input. Keeps screen capture if active; otherwise OFF.
+     */
+    disableCamera(): Promise<void>;
+    /**
+     * Enable screen capture. If displayIds are passed, the first id wins as
+     * the `displayIndex` (multi-display capture is still single-display
+     * upstream).
+     */
+    enableScreen(displayIds?: number[]): Promise<void>;
+    /**
+     * Disable screen capture. Keeps camera if active; otherwise OFF.
+     */
+    disableScreen(): Promise<void>;
+    setVisionMode(mode: VisionMode): Promise<void>;
+    private stopProcessing;
+    getCameraInfo(): CameraInfo | null;
+    isActive(): boolean;
+    private calculateBoxOverlap;
+    getEntityTracker(): EntityTracker;
+    getFaceRecognition(): Promise<FaceRecognition>;
+    stop(): Promise<void>;
+    private findCamera;
+    private listCameras;
+    private createCameraDevice;
+    captureImage(): Promise<Buffer | null>;
+}
+//# sourceMappingURL=service.d.ts.map

package/dist/service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../src/service.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,KAAK,aAAa,EAGlB,OAAO,EACP,KAAK,eAAe,EAErB,MAAM,eAAe,CAAC;AAiBvB,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAU/D,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,wBAAwB,EAE7B,KAAK,gBAAgB,EACrB,KAAK,aAAa,EAIlB,KAAK,WAAW,EAChB,UAAU,EAEX,MAAM,SAAS,CAAC;AAmBjB,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,EAAE;QACb,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,IAAI,CAAC;KAC/C,GAAG,IAAI,CAAC;IACT,aAAa,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAWD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,WAAW,CAAC;CACnB;AAgCD,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,EACvB,eAAe,CAAC,EAAE,cAAc,EAAE,GAAG,IAAI,EACzC,eAAe,CAAC,EAAE,cAAc,EAAE,GAAG,IAAI,GACxC,MAAM,CAaR;AA6DD,qBAAa,aAAc,SAAQ,OAAO;IACxC,OAAgB,WAAW,EAAE,eAAe,CAA4B;IAC/D,qBAAqB,SACgD;IAE9E,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAA6B;IAC3C,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,oBAAoB,CAAiC;IAC7D,OAAO,CAAC,uBAAuB,CAA+B;IAC9D,OAAO,CAAC,wBAAwB,CAA+B;IAC/D,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,kBAAkB,CAAS;IAInC,OAAO,CAAC,eAAe,CAAgC;IACvD,OAAO,CAAC,aAAa,CAAgB;IACrC,OAAO,CAAC,YAAY,CAAoC;IACxD,OAAO,CAAC,qBAAqB,CAA6C;IAG1E,OAAO,CAAC,aAAa,CAAuB;IAC5C,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,iBAAiB,CAA8B;IACvD,OAAO,CAAC,iBAAiB,CAAyC;IAGlE,OAAO,CAAC,aAAa,CAAoC;IAGzD,OAAO,CAAC,gBAAgB,CAAK;IAC7B,OAAO,CAAC,iBAAiB,CAAK;IAC9B,OAAO,CAAC,iBAAiB,CAAM;IAM/B,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAiC;IACtE,OAAO,CAAC,kBAAkB,CAA6B;IAMvD,OAAO,CAAC,kBAAkB,CAAmC;IAC7D,OAAO,CAAC,sBAAsB,CAAS;IAIvC,OAAO,CAAC,wBAAwB,CAGiB;IAGjD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAc7B;gBAEU,OAAO,CAAC,EAAE,aAAa;IAyCnC,OAAO,CAAC,WAAW;WA0EN,KAAK,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;YAMpD,gBAAgB;YA2BhB,UAAU;YA0DV,sBAAsB;YAyCtB,sBAAsB;YAgCtB,sBAAsB;YAmHtB,uBAAuB;IAmBrC,OAAO,CAAC,eAAe;IAuBvB;;;;;;OAMG;IACH,OAAO,CAAC,mBAAmB;IAoB3B;;;;OAIG;IACI,kBAAkB,IAAI,IAAI;IAIjC,+DAA+D;IACxD,oBAAoB;IAI3B,OAAO,CAAC,oBAAoB;YAoBd,sBAAsB;YAuCtB,gBAAgB;YA+BhB,oBAAoB;YA+BpB,sBAAsB;IAyWpC;;;;OAIG;IACH,OAAO,CAAC,8BAA8B;IAsBtC;;;;;OAKG;YACW,oBAAoB;IAclC,OAAO,CAAC,8BAA8B;IAatC;;;;;;;;OAQG;YACW,wBAAwB;YAuBxB,gBAAgB;IAoB9B;;;;OAIG;IACH,OAAO,CAAC,mBAAmB;IAsB3B;;;;;;OAMG;YACW,2BAA2B;YAwB3B,oBAAoB;YAoBpB,gCAAgC;YAoFhC,mBAAmB;IA2EjC,OAAO,CAAC,oBAAoB;IA6E5B,OAAO,CAAC,oBAAoB;YAgBd,sBAAsB;IA6CpC,OAAO,CAAC,qBAAqB;YAoBf,uBAAuB;YAoBvB,WAAW;YAkBX,8BAA8B;IAkD/B,eAAe,IAAI,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;IAI9C,mBAAmB,IAAI,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAIvD,2BAA2B,IAAI,OAAO,CAAC,wBAAwB,GAAG,IAAI,CAAC;IAUvE,gBAAgB,IAAI,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC;IAIvD,aAAa,IAAI,UAAU;IAIlC;;;OAGG;IACU,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAS1C;;OAEG;IACU,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAS3C;;;;OAIG;IACU,YAAY,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAa/D;;OAEG;IACU,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC;IAS9B,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAoC3D,OAAO,CAAC,cAAc;IAiBf,aAAa,IAAI,UAAU,GAAG,IAAI;IAYlC,QAAQ,IAAI,OAAO;IAK1B,OAAO,CAAC,mBAAmB;IAmBpB,gBAAgB,IAAI,aAAa;IAI3B,kBAAkB,IAAI,OAAO,CAAC,eAAe,CAAC;IAQrD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YA0Cb,UAAU;YAkCV,WAAW;IAyEzB,OAAO,CAAC,kBAAkB;IAkFb,YAAY,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;CAapD"}

package/dist/set-of-marks-provider.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Bridge plugin-vision's Set-of-Marks fusion into plugin-computeruse's
+ * `SetOfMarksProvider` registry seam (#9170 M9).
+ *
+ * Mirrors `computeruse-ocr-bridge.ts`: plugin-vision owns the GGUF YOLO icon
+ * detector and the OCR engines; plugin-computeruse exposes a registration slot
+ * and consumes whatever is registered from `detect_elements`, with NO hard
+ * dependency on plugin-vision. The provider is built here and registered at
+ * boot via a best-effort dynamic import (see `index.ts`).
+ *
+ * Pure + injectable: the YOLO detector and OCR resolver are passed in, so the
+ * fusion wiring is unit-testable with fakes and degrades gracefully when the
+ * GGUF detector or OCR engine is unavailable (icons or text simply absent).
+ */
+import { getOcrWithCoordsService } from "./ocr-with-coords.js";
+import { type DetectedObjectLike, type SetOfMarksOptions, type SomMark } from "./som.js";
+/** Structural shape of computeruse's `SetOfMarksInput`. */
+export interface SetOfMarksInputLike {
+    readonly displayId: string;
+    readonly sourceX: number;
+    readonly sourceY: number;
+    readonly pngBytes: Uint8Array;
+    readonly renderOverlay?: boolean;
+}
+/** Structural shape of computeruse's `SetOfMarksResult`. */
+export interface SetOfMarksResultLike {
+    readonly marks: ReadonlyArray<SomMark>;
+    readonly overlayPngBase64?: string;
+}
+/** Structural shape of computeruse's `SetOfMarksProvider`. */
+export interface SetOfMarksProviderLike {
+    readonly name: string;
+    describe(input: SetOfMarksInputLike): Promise<SetOfMarksResultLike>;
+}
+export type RegisterSetOfMarksProvider = (provider: SetOfMarksProviderLike | null) => void;
+export declare const VISION_SET_OF_MARKS_BRIDGE_NAME = "vision-set-of-marks-bridge";
+export interface SetOfMarksProviderDeps {
+    /**
+     * Detect icon-ish boxes from PNG bytes (GGUF YOLO). Returns `[]` when the
+     * detector is unavailable — Set-of-Marks then falls back to text-only marks.
+     */
+    readonly detectIcons?: (pngBytes: Uint8Array) => Promise<DetectedObjectLike[]>;
+    /** Resolve the OCR-with-coords service (defaults to the registered one). */
+    readonly resolveOcr?: typeof getOcrWithCoordsService;
+    /** Fusion tuning forwarded to `buildSetOfMarks`. */
+    readonly options?: SetOfMarksOptions;
+}
+/**
+ * Lazily-instantiated default GGUF YOLO icon detector. Best-effort: if the
+ * native bindings or GGUF weights are missing, every call resolves to `[]` so
+ * Set-of-Marks degrades to OCR-only text marks instead of throwing.
+ */
+export declare function createDefaultIconDetector(): (pngBytes: Uint8Array) => Promise<DetectedObjectLike[]>;
+/**
+ * Build a `SetOfMarksProvider`-shaped bridge that fuses GGUF YOLO icon
+ * detections + OCR text blocks into a numbered mark set (and optional overlay).
+ */
+export declare function buildVisionSetOfMarksProvider(deps?: SetOfMarksProviderDeps): SetOfMarksProviderLike;
+/**
+ * Register the vision Set-of-Marks bridge into computeruse's seam. Idempotent
+ * (last-call-wins). Returns true once registered.
+ */
+export declare function wireComputerUseSetOfMarksBridge(register: RegisterSetOfMarksProvider, deps?: SetOfMarksProviderDeps): boolean;
+//# sourceMappingURL=set-of-marks-provider.d.ts.map