@elizaos/plugin-computeruse 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/mobile/mobile-computer-interface.d.ts

@@ -0,0 +1,142 @@
+/**
+ * WS7 ↔ WS8 — `MobileComputerInterface` adapts the WS7 `ComputerInterface`
+ * port to the Android `AndroidComputerUseBridge` so the cascade + dispatcher
+ * run unchanged on mobile.
+ *
+ * Mapping (display-local pixel coords → Android screen-pixel gestures):
+ *
+ *   leftClick({x, y})          → dispatchGesture({type:"tap", x, y})
+ *   doubleClick({x, y})        → dispatchGesture(tap) twice (no native double-tap)
+ *   rightClick({x, y})         → dispatchGesture(tap) [no right-click on Android;
+ *                                cascade should prefer longClick semantics via
+ *                                the AX node's `longClick` action]
+ *   dragTo({x, y}) / drag(path)→ dispatchGesture({type:"swipe", x1,y1,x2,y2})
+ *   scroll({x, y, dx, dy})     → dispatchGesture(swipe) anchored at (x, y),
+ *                                direction inverted (scrolling DOWN visually
+ *                                means swiping UP physically)
+ *   pressKey({key:"back"})     → performGlobalAction("back")
+ *   pressKey({key:"home"})     → performGlobalAction("home")
+ *   hotkey                     → not supported on Android; throws
+ *   typeText                   → setText({text}) against the focused editable
+ *                                AccessibilityNodeInfo.
+ *
+ * `getScreenSize`, `getCursorPosition`, and the coord-conversion helpers
+ * keep their desktop behavior — they're metadata calls, not input.
+ *
+ * Errors:
+ *   Every method that calls the bridge propagates `ok:false` as a thrown
+ *   `Error` whose message carries the `code` + `message`. The WS7 dispatcher
+ *   maps that to `ActionResult.error.driver_error`, exactly like desktop.
+ */
+import type { ComputerInterface, CursorPosition, DisplayPoint, DragPath, MouseButton, ScreenshotResult, ScrollDelta } from "../actor/computer-interface.js";
+import type { Scene, SceneAxNode } from "../scene/scene-types.js";
+import type { DisplayDescriptor } from "../types.js";
+import type { AndroidComputerUseBridge } from "./android-bridge.js";
+export interface MobileComputerInterfaceDeps {
+    /** Capacitor plugin handle (null when off-platform). */
+    getBridge: () => AndroidComputerUseBridge | null;
+    /** Latest scene accessor — used for `getAccessibilityTree`. */
+    getScene?: () => Scene | null;
+    /** Display descriptor; mobile devices have exactly one. */
+    getDisplay?: () => DisplayDescriptor;
+    /** Internal cursor-position state, mostly for tests. */
+    cursorState?: {
+        current: CursorPosition;
+    };
+    /**
+     * Override fetched screenshot bytes. Defaults to a one-shot
+     * `bridge.captureFrame()` so the cascade's pull contract just works.
+     */
+    decodeJpeg?: (b64: string) => Buffer;
+    /**
+     * Map a WS7 key name (`"back"`, `"home"`, `"recents"`, `"notifications"`)
+     * to an Android `performGlobalAction` invocation. Other keys throw —
+     * Android has no equivalent of arbitrary keystrokes from a non-system app.
+     */
+    globalActionMap?: ReadonlyMap<string, "back" | "home" | "recents" | "notifications">;
+}
+export declare class MobileComputerInterface implements ComputerInterface {
+    private readonly deps;
+    private readonly cursorState;
+    constructor(deps: MobileComputerInterfaceDeps);
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(path: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(_args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(_args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(_args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+    private dispatchTap;
+    private dispatchSwipe;
+    private moveTracker;
+    private requireBridge;
+    private requireDisplayId;
+    private requireFiniteCoords;
+    private resolveGlobalAction;
+    private getDisplay;
+}
+/** Convenience factory. */
+export declare function makeMobileComputerInterface(deps: MobileComputerInterfaceDeps): ComputerInterface;
+//# sourceMappingURL=mobile-computer-interface.d.ts.map

package/dist/mobile/mobile-computer-interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mobile-computer-interface.d.ts","sourceRoot":"","sources":["../../src/mobile/mobile-computer-interface.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAGH,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,WAAW,EACZ,MAAM,gCAAgC,CAAC;AACxC,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAEV,wBAAwB,EAEzB,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,2BAA2B;IAC1C,wDAAwD;IACxD,SAAS,EAAE,MAAM,wBAAwB,GAAG,IAAI,CAAC;IACjD,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,IAAI,CAAC;IAC9B,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,iBAAiB,CAAC;IACrC,wDAAwD;IACxD,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,cAAc,CAAA;KAAE,CAAC;IAC1C;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,CAAC;IACrC;;;;OAIG;IACH,eAAe,CAAC,EAAE,WAAW,CAC3B,MAAM,EACN,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,eAAe,CAC9C,CAAC;CACH;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAA8B;IACnD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAA8B;gBAE9C,IAAI,EAAE,2BAA2B;IASvC,UAAU,CACd,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAChC,OAAO,CAAC,gBAAgB,CAAC;IAiBtB,SAAS,CACb,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC7C,OAAO,CAAC,IAAI,CAAC;IAOV,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAItE,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9C,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,MAAM,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAc1C,IAAI,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BnC,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM7C,KAAK,CAAC,KAAK,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAI5C,QAAQ,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAU/C,QAAQ,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAc9C,MAAM,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAQhD,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA+BzC,QAAQ,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAapE,UAAU,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAe5E,aAAa,CAAC,KAAK,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAKrE,iBAAiB,IAAI,cAAc;IAInC,mBAAmB,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAgB5B,uBAAuB,CAAC,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAgBlC,oBAAoB,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,WAAW,EAAE;YASnD,WAAW;YASX,aAAa;IAoB3B,OAAO,CAAC,WAAW;IAMnB,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;IAsB3B,OAAO,CAAC,UAAU;CAUnB;AAED,2BAA2B;AAC3B,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,2BAA2B,GAChC,iBAAiB,CAEnB"}

package/dist/mobile/mobile-screen-capture.d.ts

@@ -0,0 +1,64 @@
+/**
+ * WS8 — `MobileScreenCaptureSource` adapts the Android `captureFrame()`
+ * bridge call into a WS5-shape `DisplayCapture` so the WS7 cascade can run
+ * unmodified on Android. iOS uses ReplayKit and is treated separately; this
+ * module focuses on Android because that's where the consumer-build cascade
+ * actually runs.
+ *
+ * Contract:
+ *   - `captureDisplay(displayId?)` returns the latest screen frame as
+ *     `{ display, frame }` — the frame is the decoded JPEG byte buffer
+ *     exactly as the Kotlin ImageReader pipeline emits it. The Brain encodes
+ *     it as a `data:image/jpeg;base64,...` URL; downstream model adapters
+ *     do not require PNG specifically, only "image".
+ *   - `captureAllDisplays()` returns a single-element array since mobile
+ *     devices report one logical display. Multi-display Android (DeX,
+ *     foldables in two-pane mode) is out of scope for v1.
+ *   - On bridge `ok:false`, the call rejects with an `Error` whose message
+ *     carries the bridge's `code` and `message`. The cascade surfaces this
+ *     through `safeCapture` so the agent loop reports it as
+ *     `reason: "error"`.
+ *
+ * The capture path is intentionally pull-based — Kotlin keeps the latest
+ * frame in a ring-buffer, and this module just drains it. The fps knob is
+ * set when MediaProjection was started; this module doesn't manage it.
+ */
+import type { DisplayCapture } from "../platform/capture.js";
+import type { DisplayDescriptor } from "../types.js";
+import type { AndroidComputerUseBridge, CapturedScreenFrame } from "./android-bridge.js";
+export declare const ANDROID_LOGICAL_DISPLAY_ID: 0;
+export interface MobileScreenCaptureSourceDeps {
+    /** Returns the Capacitor `ComputerUse` plugin handle, or null when off-platform. */
+    getBridge: () => AndroidComputerUseBridge | null;
+    /**
+     * Override the display descriptor — primarily for tests. In production,
+     * the descriptor is derived from the captured frame's width/height (the
+     * device only knows its own size).
+     */
+    getDisplay?: (frame: CapturedScreenFrame | null) => DisplayDescriptor;
+    /**
+     * Override how raw JPEG bytes are produced from the base64 payload. The
+     * default uses Node's `Buffer.from(..., "base64")`; tests can inject a
+     * decoder when running in a browser-only context.
+     */
+    decodeJpeg?: (jpegBase64: string) => Buffer;
+}
+/**
+ * Pull-based capture source for Android MediaProjection. Mirrors the public
+ * surface of `captureAllDisplays` / `captureDisplay` from
+ * `platform/capture.ts` so the WS7 cascade can substitute it via DI.
+ */
+export declare class MobileScreenCaptureSource {
+    private readonly deps;
+    constructor(deps: MobileScreenCaptureSourceDeps);
+    /**
+     * Drain the latest frame for the (single) logical Android display.
+     * `displayId` is honored only when it equals `ANDROID_LOGICAL_DISPLAY_ID`;
+     * any other id throws — mirrors `captureDisplay`'s unknown-display
+     * behavior on desktop.
+     */
+    captureDisplay(displayId?: number): Promise<DisplayCapture>;
+    /** Single-element array — mobile devices have one logical display. */
+    captureAllDisplays(): Promise<DisplayCapture[]>;
+}
+//# sourceMappingURL=mobile-screen-capture.d.ts.map

package/dist/mobile/mobile-screen-capture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mobile-screen-capture.d.ts","sourceRoot":"","sources":["../../src/mobile/mobile-screen-capture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EACV,wBAAwB,EACxB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAE7B,eAAO,MAAM,0BAA0B,EAAG,CAAU,CAAC;AAErD,MAAM,WAAW,6BAA6B;IAC5C,oFAAoF;IACpF,SAAS,EAAE,MAAM,wBAAwB,GAAG,IAAI,CAAC;IACjD;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,KAAK,iBAAiB,CAAC;IACtE;;;;OAIG;IACH,UAAU,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,KAAK,MAAM,CAAC;CAC7C;AAED;;;;GAIG;AACH,qBAAa,yBAAyB;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAAJ,IAAI,EAAE,6BAA6B;IAEhE;;;;;OAKG;IACG,cAAc,CAClB,SAAS,GAAE,MAAmC,GAC7C,OAAO,CAAC,cAAc,CAAC;IA2B1B,sEAAsE;IAChE,kBAAkB,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;CAGtD"}

package/dist/mobile/ocr-provider.d.ts

@@ -0,0 +1,187 @@
+/**
+ * OCR provider chain — used by WS6 (scene-builder) and any other consumer
+ * that needs text-from-image extraction.
+ *
+ * Defined here in plugin-computeruse so the mobile bridge can publish the
+ * iOS Apple Vision implementation alongside the rest of the iOS surface.
+ * WS6 will register additional providers (cloud, Tesseract fallback) and
+ * pick a provider per-call via `selectOcrProvider`.
+ *
+ * Contract:
+ *   - `name`         : stable string id for routing/telemetry.
+ *   - `priority`     : higher wins when multiple providers report `available`.
+ *   - `available()`  : cheap synchronous availability probe; the registry
+ *                      caches the result for the process lifetime.
+ *   - `recognize()`  : async OCR call. Throws on hard failures so callers can
+ *                      fall back to the next provider; never returns empty
+ *                      lines silently.
+ */
+import type { IosComputerUseBridge } from "./ios-bridge.js";
+export interface OcrLine {
+    readonly text: string;
+    readonly confidence: number;
+    readonly boundingBox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+}
+export interface OcrResult {
+    readonly lines: readonly OcrLine[];
+    readonly fullText: string;
+    readonly elapsedMs: number;
+    readonly providerName: string;
+    readonly languagesUsed: readonly string[];
+}
+export interface OcrRecognizeOptions {
+    readonly languages?: readonly string[];
+    readonly recognitionLevel?: "fast" | "accurate";
+    readonly minimumTextHeight?: number;
+}
+export interface OcrProvider {
+    readonly name: string;
+    readonly priority: number;
+    available(): boolean;
+    recognize(input: OcrInput, options?: OcrRecognizeOptions): Promise<OcrResult>;
+}
+/**
+ * Image input. Either a base64-encoded PNG/JPEG or raw bytes. The provider
+ * is responsible for normalizing into whatever its native side expects.
+ */
+export type OcrInput = {
+    readonly kind: "base64";
+    readonly data: string;
+} | {
+    readonly kind: "bytes";
+    readonly data: Uint8Array;
+};
+export declare function registerOcrProvider(provider: OcrProvider): void;
+export declare function unregisterOcrProvider(name: string): void;
+export declare function listOcrProviders(): readonly OcrProvider[];
+/**
+ * Returns the highest-priority provider that reports `available()`. Throws if
+ * none are available — callers must handle that explicitly rather than
+ * silently degrading. WS6's scene-builder catches and reports.
+ */
+export declare function selectOcrProvider(): OcrProvider;
+/**
+ * Builds an OcrProvider that delegates to the Capacitor `ComputerUse` plugin's
+ * `visionOcr` method. Pass in a getter that lazily resolves the bridge so this
+ * module stays free of Capacitor imports (which would break Node test runs).
+ */
+export declare function createIosVisionOcrProvider(getBridge: () => IosComputerUseBridge | null, options?: {
+    readonly priority?: number;
+}): OcrProvider;
+/**
+ * Test helper. Drops every provider from the registry; callers re-register
+ * in `beforeEach`.
+ */
+export declare function _resetOcrProvidersForTests(): void;
+/**
+ * Hierarchical (block / line / word) OCR with absolute source-display
+ * coordinates and a coarse 3x3 semantic position label per element. This is
+ * a *separate* slot from `OcrProvider` above — that one is line-only and
+ * shaped around Apple Vision's API, while this one carries the structure
+ * that plugin-computeruse needs to compute action targets without re-running
+ * detection.
+ *
+ * The provider implementation lives in `@elizaos/plugin-vision`'s
+ * `ocr-with-coords.ts`. plugin-computeruse intentionally does not take a
+ * runtime dep on plugin-vision (computeruse is the higher-level seam), so
+ * the runtime registers a provider here at boot.
+ *
+ * The current in-tree provider is the RapidOCR-backed adapter registered by
+ * `@elizaos/plugin-vision`; native providers can register the same interface
+ * when they are available.
+ */
+export interface CoordOcrSemantic {
+    readonly position: "upper-left" | "upper-center" | "upper-right" | "middle-left" | "center" | "middle-right" | "lower-left" | "lower-center" | "lower-right";
+}
+export interface CoordOcrWord {
+    readonly text: string;
+    readonly bbox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+    readonly semantic_position: CoordOcrSemantic["position"];
+}
+export interface CoordOcrBlock {
+    readonly text: string;
+    readonly bbox: {
+        readonly x: number;
+        readonly y: number;
+        readonly width: number;
+        readonly height: number;
+    };
+    readonly words: ReadonlyArray<CoordOcrWord>;
+    readonly semantic_position: CoordOcrSemantic["position"];
+}
+export interface CoordOcrResult {
+    readonly blocks: ReadonlyArray<CoordOcrBlock>;
+}
+export interface CoordOcrInput {
+    readonly displayId: string;
+    readonly sourceX: number;
+    readonly sourceY: number;
+    readonly pngBytes: Uint8Array;
+}
+export interface CoordOcrProvider {
+    readonly name: string;
+    describe(input: CoordOcrInput): Promise<CoordOcrResult>;
+}
+/**
+ * Register the hierarchical / coord-aware OCR provider. Idempotent — last
+ * call wins so a hot-reload of the bridge swaps cleanly. Pass `null` to
+ * unregister.
+ */
+export declare function registerCoordOcrProvider(provider: CoordOcrProvider | null): void;
+export declare function getCoordOcrProvider(): CoordOcrProvider | null;
+/**
+ * Set-of-Marks grounding (trycua/cua's OmniParser technique): fuse GGUF YOLO
+ * icon detections + OCR text boxes into ONE deduplicated, 1-indexed set of
+ * numbered targets, optionally with a numbered-overlay PNG. The VLM picks a
+ * *number* instead of regressing raw pixels.
+ *
+ * The fusion + overlay implementation lives in `@elizaos/plugin-vision`
+ * (`som.ts`) because that package owns the YOLO detector and OCR engines.
+ * computeruse exposes this registration slot (same no-hard-dep pattern as the
+ * CoordOcrProvider above); `detect_elements` consumes whatever is registered.
+ */
+export interface SetOfMarksMark {
+    /** 1-indexed mark number shown in the overlay. */
+    readonly index: number;
+    /** Display-local box `[x, y, w, h]`. */
+    readonly bbox: readonly [number, number, number, number];
+    /** Box center `[x, y]` — the click target the number resolves to. */
+    readonly center: readonly [number, number];
+    readonly source: "icon" | "text";
+    readonly label?: string;
+    readonly score: number;
+}
+export interface SetOfMarksResult {
+    readonly marks: ReadonlyArray<SetOfMarksMark>;
+    /** Base64 PNG of the numbered overlay (present only when requested). */
+    readonly overlayPngBase64?: string;
+}
+export interface SetOfMarksInput {
+    readonly displayId: string;
+    readonly sourceX: number;
+    readonly sourceY: number;
+    readonly pngBytes: Uint8Array;
+    /** Render and return the numbered-overlay PNG. Default false (marks only). */
+    readonly renderOverlay?: boolean;
+}
+export interface SetOfMarksProvider {
+    readonly name: string;
+    describe(input: SetOfMarksInput): Promise<SetOfMarksResult>;
+}
+/**
+ * Register the Set-of-Marks provider. Idempotent — last call wins. Pass `null`
+ * to unregister.
+ */
+export declare function registerSetOfMarksProvider(provider: SetOfMarksProvider | null): void;
+export declare function getSetOfMarksProvider(): SetOfMarksProvider | null;
+//# sourceMappingURL=ocr-provider.d.ts.map

package/dist/mobile/ocr-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ocr-provider.d.ts","sourceRoot":"","sources":["../../src/mobile/ocr-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EACV,oBAAoB,EAIrB,MAAM,iBAAiB,CAAC;AAEzB,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE;QACpB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,SAAS,OAAO,EAAE,CAAC;IACnC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;IAChD,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,SAAS,IAAI,OAAO,CAAC;IACrB,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;CAC/E;AAED;;;GAGG;AACH,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAA;CAAE,CAAC;AAM1D,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,WAAW,GAAG,IAAI,CAE/D;AAED,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAExD;AAED,wBAAgB,gBAAgB,IAAI,SAAS,WAAW,EAAE,CAEzD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,IAAI,WAAW,CAO/C;AAID;;;;GAIG;AACH,wBAAgB,0BAA0B,CACxC,SAAS,EAAE,MAAM,oBAAoB,GAAG,IAAI,EAC5C,OAAO,GAAE;IAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3C,WAAW,CA2Cb;AAsCD;;;GAGG;AACH,wBAAgB,0BAA0B,IAAI,IAAI,CAEjD;AAID;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,QAAQ,EACb,YAAY,GACZ,cAAc,GACd,aAAa,GACb,aAAa,GACb,QAAQ,GACR,cAAc,GACd,YAAY,GACZ,cAAc,GACd,aAAa,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,QAAQ,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE;QACb,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;IACF,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,YAAY,CAAC,CAAC;IAC5C,QAAQ,CAAC,iBAAiB,EAAE,gBAAgB,CAAC,UAAU,CAAC,CAAC;CAC1D;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACzD;AAID;;;;GAIG;AACH,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,gBAAgB,GAAG,IAAI,GAChC,IAAI,CAEN;AAED,wBAAgB,mBAAmB,IAAI,gBAAgB,GAAG,IAAI,CAE7D;AAID;;;;;;;;;;GAUG;AACH,MAAM,WAAW,cAAc;IAC7B,kDAAkD;IAClD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACzD,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3C,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;IACjC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;IAC9C,wEAAwE;IACxE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,8EAA8E;IAC9E,QAAQ,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CAC7D;AAID;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,kBAAkB,GAAG,IAAI,GAClC,IAAI,CAEN;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE"}

package/dist/mobile/ocr-provider.js

@@ -0,0 +1,111 @@
+// src/mobile/ocr-provider.ts
+var REGISTRY = new Map;
+function registerOcrProvider(provider) {
+  REGISTRY.set(provider.name, provider);
+}
+function unregisterOcrProvider(name) {
+  REGISTRY.delete(name);
+}
+function listOcrProviders() {
+  return [...REGISTRY.values()].sort((a, b) => b.priority - a.priority);
+}
+function selectOcrProvider() {
+  for (const provider of listOcrProviders()) {
+    if (provider.available())
+      return provider;
+  }
+  throw new Error("No OCR provider available. Register at least one provider before calling selectOcrProvider().");
+}
+function createIosVisionOcrProvider(getBridge, options = {}) {
+  return {
+    name: "ios-apple-vision",
+    priority: options.priority ?? 100,
+    available() {
+      return getBridge() !== null;
+    },
+    async recognize(input, recognizeOptions) {
+      const bridge = getBridge();
+      if (!bridge) {
+        throw new Error("ios-apple-vision provider invoked but Capacitor ComputerUse plugin is not registered.");
+      }
+      const imageBase64 = toBase64(input);
+      const visionOptions = {
+        ...recognizeOptions?.languages ? { languages: recognizeOptions.languages } : {},
+        ...recognizeOptions?.recognitionLevel ? { recognitionLevel: recognizeOptions.recognitionLevel } : {},
+        ...recognizeOptions?.minimumTextHeight !== undefined ? { minimumTextHeight: recognizeOptions.minimumTextHeight } : {}
+      };
+      const result = await bridge.visionOcr({
+        imageBase64,
+        options: visionOptions
+      });
+      if (!result.ok) {
+        const failure = result;
+        throw new Error(`ios-apple-vision OCR failed: ${failure.code} — ${failure.message}`);
+      }
+      return mapVisionResult(result.data);
+    }
+  };
+}
+function toBase64(input) {
+  if (input.kind === "base64")
+    return input.data;
+  return uint8ArrayToBase64(input.data);
+}
+function uint8ArrayToBase64(bytes) {
+  if (typeof Buffer !== "undefined") {
+    return Buffer.from(bytes).toString("base64");
+  }
+  let binary = "";
+  for (let i = 0;i < bytes.length; i += 1) {
+    binary += String.fromCharCode(bytes[i]);
+  }
+  return btoa(binary);
+}
+function mapVisionResult(result) {
+  return {
+    lines: result.lines.map(mapLine),
+    fullText: result.fullText,
+    elapsedMs: result.elapsedMs,
+    providerName: "ios-apple-vision",
+    languagesUsed: result.languagesUsed
+  };
+}
+function mapLine(line) {
+  return {
+    text: line.text,
+    confidence: line.confidence,
+    boundingBox: line.boundingBox
+  };
+}
+function _resetOcrProvidersForTests() {
+  REGISTRY.clear();
+}
+var registeredCoordOcrProvider = null;
+function registerCoordOcrProvider(provider) {
+  registeredCoordOcrProvider = provider;
+}
+function getCoordOcrProvider() {
+  return registeredCoordOcrProvider;
+}
+var registeredSetOfMarksProvider = null;
+function registerSetOfMarksProvider(provider) {
+  registeredSetOfMarksProvider = provider;
+}
+function getSetOfMarksProvider() {
+  return registeredSetOfMarksProvider;
+}
+export {
+  unregisterOcrProvider,
+  selectOcrProvider,
+  registerSetOfMarksProvider,
+  registerOcrProvider,
+  registerCoordOcrProvider,
+  listOcrProviders,
+  getSetOfMarksProvider,
+  getCoordOcrProvider,
+  createIosVisionOcrProvider,
+  _resetOcrProvidersForTests
+};
+
+//# debugId=ECB47B4DF356B75E64756E2164756E21
+//# sourceMappingURL=ocr-provider.js.map

package/dist/mobile/ocr-provider.js.map

@@ -0,0 +1,10 @@
+{
+  "version": 3,
+  "sources": ["../../src/mobile/ocr-provider.ts"],
+  "sourcesContent": [
+    "/**\n * OCR provider chain — used by WS6 (scene-builder) and any other consumer\n * that needs text-from-image extraction.\n *\n * Defined here in plugin-computeruse so the mobile bridge can publish the\n * iOS Apple Vision implementation alongside the rest of the iOS surface.\n * WS6 will register additional providers (cloud, Tesseract fallback) and\n * pick a provider per-call via `selectOcrProvider`.\n *\n * Contract:\n *   - `name`         : stable string id for routing/telemetry.\n *   - `priority`     : higher wins when multiple providers report `available`.\n *   - `available()`  : cheap synchronous availability probe; the registry\n *                      caches the result for the process lifetime.\n *   - `recognize()`  : async OCR call. Throws on hard failures so callers can\n *                      fall back to the next provider; never returns empty\n *                      lines silently.\n */\n\nimport type {\n  IosComputerUseBridge,\n  VisionOcrLine,\n  VisionOcrOptions,\n  VisionOcrResult,\n} from \"./ios-bridge.js\";\n\nexport interface OcrLine {\n  readonly text: string;\n  readonly confidence: number;\n  readonly boundingBox: {\n    readonly x: number;\n    readonly y: number;\n    readonly width: number;\n    readonly height: number;\n  };\n}\n\nexport interface OcrResult {\n  readonly lines: readonly OcrLine[];\n  readonly fullText: string;\n  readonly elapsedMs: number;\n  readonly providerName: string;\n  readonly languagesUsed: readonly string[];\n}\n\nexport interface OcrRecognizeOptions {\n  readonly languages?: readonly string[];\n  readonly recognitionLevel?: \"fast\" | \"accurate\";\n  readonly minimumTextHeight?: number;\n}\n\nexport interface OcrProvider {\n  readonly name: string;\n  readonly priority: number;\n  available(): boolean;\n  recognize(input: OcrInput, options?: OcrRecognizeOptions): Promise<OcrResult>;\n}\n\n/**\n * Image input. Either a base64-encoded PNG/JPEG or raw bytes. The provider\n * is responsible for normalizing into whatever its native side expects.\n */\nexport type OcrInput =\n  | { readonly kind: \"base64\"; readonly data: string }\n  | { readonly kind: \"bytes\"; readonly data: Uint8Array };\n\n// ── Registry ─────────────────────────────────────────────────────────────────\n\nconst REGISTRY = new Map<string, OcrProvider>();\n\nexport function registerOcrProvider(provider: OcrProvider): void {\n  REGISTRY.set(provider.name, provider);\n}\n\nexport function unregisterOcrProvider(name: string): void {\n  REGISTRY.delete(name);\n}\n\nexport function listOcrProviders(): readonly OcrProvider[] {\n  return [...REGISTRY.values()].sort((a, b) => b.priority - a.priority);\n}\n\n/**\n * Returns the highest-priority provider that reports `available()`. Throws if\n * none are available — callers must handle that explicitly rather than\n * silently degrading. WS6's scene-builder catches and reports.\n */\nexport function selectOcrProvider(): OcrProvider {\n  for (const provider of listOcrProviders()) {\n    if (provider.available()) return provider;\n  }\n  throw new Error(\n    \"No OCR provider available. Register at least one provider before calling selectOcrProvider().\",\n  );\n}\n\n// ── iOS Apple Vision provider ────────────────────────────────────────────────\n\n/**\n * Builds an OcrProvider that delegates to the Capacitor `ComputerUse` plugin's\n * `visionOcr` method. Pass in a getter that lazily resolves the bridge so this\n * module stays free of Capacitor imports (which would break Node test runs).\n */\nexport function createIosVisionOcrProvider(\n  getBridge: () => IosComputerUseBridge | null,\n  options: { readonly priority?: number } = {},\n): OcrProvider {\n  return {\n    name: \"ios-apple-vision\",\n    priority: options.priority ?? 100,\n    available(): boolean {\n      return getBridge() !== null;\n    },\n    async recognize(input, recognizeOptions): Promise<OcrResult> {\n      const bridge = getBridge();\n      if (!bridge) {\n        throw new Error(\n          \"ios-apple-vision provider invoked but Capacitor ComputerUse plugin is not registered.\",\n        );\n      }\n      const imageBase64 = toBase64(input);\n      const visionOptions: VisionOcrOptions = {\n        ...(recognizeOptions?.languages\n          ? { languages: recognizeOptions.languages }\n          : {}),\n        ...(recognizeOptions?.recognitionLevel\n          ? { recognitionLevel: recognizeOptions.recognitionLevel }\n          : {}),\n        ...(recognizeOptions?.minimumTextHeight !== undefined\n          ? { minimumTextHeight: recognizeOptions.minimumTextHeight }\n          : {}),\n      };\n      const result = await bridge.visionOcr({\n        imageBase64,\n        options: visionOptions,\n      });\n      if (!result.ok) {\n        // Narrow to the failure arm explicitly — some consumer tsconfigs\n        // run with `strict: false`, which disables discriminated-union\n        // narrowing on `!result.ok` and surfaces TS2339 on `result.code`\n        // / `result.message`. The runtime invariant is unchanged.\n        const failure = result as Extract<typeof result, { ok: false }>;\n        throw new Error(\n          `ios-apple-vision OCR failed: ${failure.code} — ${failure.message}`,\n        );\n      }\n      return mapVisionResult(result.data);\n    },\n  };\n}\n\nfunction toBase64(input: OcrInput): string {\n  if (input.kind === \"base64\") return input.data;\n  return uint8ArrayToBase64(input.data);\n}\n\nfunction uint8ArrayToBase64(bytes: Uint8Array): string {\n  if (typeof Buffer !== \"undefined\") {\n    return Buffer.from(bytes).toString(\"base64\");\n  }\n  let binary = \"\";\n  for (let i = 0; i < bytes.length; i += 1) {\n    binary += String.fromCharCode(bytes[i]);\n  }\n  // btoa is the only universal browser/Capacitor path.\n  // eslint-disable-next-line no-undef\n  return btoa(binary);\n}\n\nfunction mapVisionResult(result: VisionOcrResult): OcrResult {\n  return {\n    lines: result.lines.map(mapLine),\n    fullText: result.fullText,\n    elapsedMs: result.elapsedMs,\n    providerName: \"ios-apple-vision\",\n    languagesUsed: result.languagesUsed,\n  };\n}\n\nfunction mapLine(line: VisionOcrLine): OcrLine {\n  return {\n    text: line.text,\n    confidence: line.confidence,\n    boundingBox: line.boundingBox,\n  };\n}\n\n/**\n * Test helper. Drops every provider from the registry; callers re-register\n * in `beforeEach`.\n */\nexport function _resetOcrProvidersForTests(): void {\n  REGISTRY.clear();\n}\n\n// ── Coord-aware OCR provider slot ────────────────────────────────────────────\n\n/**\n * Hierarchical (block / line / word) OCR with absolute source-display\n * coordinates and a coarse 3x3 semantic position label per element. This is\n * a *separate* slot from `OcrProvider` above — that one is line-only and\n * shaped around Apple Vision's API, while this one carries the structure\n * that plugin-computeruse needs to compute action targets without re-running\n * detection.\n *\n * The provider implementation lives in `@elizaos/plugin-vision`'s\n * `ocr-with-coords.ts`. plugin-computeruse intentionally does not take a\n * runtime dep on plugin-vision (computeruse is the higher-level seam), so\n * the runtime registers a provider here at boot.\n *\n * The current in-tree provider is the RapidOCR-backed adapter registered by\n * `@elizaos/plugin-vision`; native providers can register the same interface\n * when they are available.\n */\nexport interface CoordOcrSemantic {\n  readonly position:\n    | \"upper-left\"\n    | \"upper-center\"\n    | \"upper-right\"\n    | \"middle-left\"\n    | \"center\"\n    | \"middle-right\"\n    | \"lower-left\"\n    | \"lower-center\"\n    | \"lower-right\";\n}\n\nexport interface CoordOcrWord {\n  readonly text: string;\n  readonly bbox: {\n    readonly x: number;\n    readonly y: number;\n    readonly width: number;\n    readonly height: number;\n  };\n  readonly semantic_position: CoordOcrSemantic[\"position\"];\n}\n\nexport interface CoordOcrBlock {\n  readonly text: string;\n  readonly bbox: {\n    readonly x: number;\n    readonly y: number;\n    readonly width: number;\n    readonly height: number;\n  };\n  readonly words: ReadonlyArray<CoordOcrWord>;\n  readonly semantic_position: CoordOcrSemantic[\"position\"];\n}\n\nexport interface CoordOcrResult {\n  readonly blocks: ReadonlyArray<CoordOcrBlock>;\n}\n\nexport interface CoordOcrInput {\n  readonly displayId: string;\n  readonly sourceX: number;\n  readonly sourceY: number;\n  readonly pngBytes: Uint8Array;\n}\n\nexport interface CoordOcrProvider {\n  readonly name: string;\n  describe(input: CoordOcrInput): Promise<CoordOcrResult>;\n}\n\nlet registeredCoordOcrProvider: CoordOcrProvider | null = null;\n\n/**\n * Register the hierarchical / coord-aware OCR provider. Idempotent — last\n * call wins so a hot-reload of the bridge swaps cleanly. Pass `null` to\n * unregister.\n */\nexport function registerCoordOcrProvider(\n  provider: CoordOcrProvider | null,\n): void {\n  registeredCoordOcrProvider = provider;\n}\n\nexport function getCoordOcrProvider(): CoordOcrProvider | null {\n  return registeredCoordOcrProvider;\n}\n\n// ── Set-of-Marks provider slot (#9170 M9) ────────────────────────────────────\n\n/**\n * Set-of-Marks grounding (trycua/cua's OmniParser technique): fuse GGUF YOLO\n * icon detections + OCR text boxes into ONE deduplicated, 1-indexed set of\n * numbered targets, optionally with a numbered-overlay PNG. The VLM picks a\n * *number* instead of regressing raw pixels.\n *\n * The fusion + overlay implementation lives in `@elizaos/plugin-vision`\n * (`som.ts`) because that package owns the YOLO detector and OCR engines.\n * computeruse exposes this registration slot (same no-hard-dep pattern as the\n * CoordOcrProvider above); `detect_elements` consumes whatever is registered.\n */\nexport interface SetOfMarksMark {\n  /** 1-indexed mark number shown in the overlay. */\n  readonly index: number;\n  /** Display-local box `[x, y, w, h]`. */\n  readonly bbox: readonly [number, number, number, number];\n  /** Box center `[x, y]` — the click target the number resolves to. */\n  readonly center: readonly [number, number];\n  readonly source: \"icon\" | \"text\";\n  readonly label?: string;\n  readonly score: number;\n}\n\nexport interface SetOfMarksResult {\n  readonly marks: ReadonlyArray<SetOfMarksMark>;\n  /** Base64 PNG of the numbered overlay (present only when requested). */\n  readonly overlayPngBase64?: string;\n}\n\nexport interface SetOfMarksInput {\n  readonly displayId: string;\n  readonly sourceX: number;\n  readonly sourceY: number;\n  readonly pngBytes: Uint8Array;\n  /** Render and return the numbered-overlay PNG. Default false (marks only). */\n  readonly renderOverlay?: boolean;\n}\n\nexport interface SetOfMarksProvider {\n  readonly name: string;\n  describe(input: SetOfMarksInput): Promise<SetOfMarksResult>;\n}\n\nlet registeredSetOfMarksProvider: SetOfMarksProvider | null = null;\n\n/**\n * Register the Set-of-Marks provider. Idempotent — last call wins. Pass `null`\n * to unregister.\n */\nexport function registerSetOfMarksProvider(\n  provider: SetOfMarksProvider | null,\n): void {\n  registeredSetOfMarksProvider = provider;\n}\n\nexport function getSetOfMarksProvider(): SetOfMarksProvider | null {\n  return registeredSetOfMarksProvider;\n}\n"
+  ],
+  "mappings": ";AAoEA,IAAM,WAAW,IAAI;AAEd,SAAS,mBAAmB,CAAC,UAA6B;AAAA,EAC/D,SAAS,IAAI,SAAS,MAAM,QAAQ;AAAA;AAG/B,SAAS,qBAAqB,CAAC,MAAoB;AAAA,EACxD,SAAS,OAAO,IAAI;AAAA;AAGf,SAAS,gBAAgB,GAA2B;AAAA,EACzD,OAAO,CAAC,GAAG,SAAS,OAAO,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,WAAW,EAAE,QAAQ;AAAA;AAQ/D,SAAS,iBAAiB,GAAgB;AAAA,EAC/C,WAAW,YAAY,iBAAiB,GAAG;AAAA,IACzC,IAAI,SAAS,UAAU;AAAA,MAAG,OAAO;AAAA,EACnC;AAAA,EACA,MAAM,IAAI,MACR,+FACF;AAAA;AAUK,SAAS,0BAA0B,CACxC,WACA,UAA0C,CAAC,GAC9B;AAAA,EACb,OAAO;AAAA,IACL,MAAM;AAAA,IACN,UAAU,QAAQ,YAAY;AAAA,IAC9B,SAAS,GAAY;AAAA,MACnB,OAAO,UAAU,MAAM;AAAA;AAAA,SAEnB,UAAS,CAAC,OAAO,kBAAsC;AAAA,MAC3D,MAAM,SAAS,UAAU;AAAA,MACzB,IAAI,CAAC,QAAQ;AAAA,QACX,MAAM,IAAI,MACR,uFACF;AAAA,MACF;AAAA,MACA,MAAM,cAAc,SAAS,KAAK;AAAA,MAClC,MAAM,gBAAkC;AAAA,WAClC,kBAAkB,YAClB,EAAE,WAAW,iBAAiB,UAAU,IACxC,CAAC;AAAA,WACD,kBAAkB,mBAClB,EAAE,kBAAkB,iBAAiB,iBAAiB,IACtD,CAAC;AAAA,WACD,kBAAkB,sBAAsB,YACxC,EAAE,mBAAmB,iBAAiB,kBAAkB,IACxD,CAAC;AAAA,MACP;AAAA,MACA,MAAM,SAAS,MAAM,OAAO,UAAU;AAAA,QACpC;AAAA,QACA,SAAS;AAAA,MACX,CAAC;AAAA,MACD,IAAI,CAAC,OAAO,IAAI;AAAA,QAKd,MAAM,UAAU;AAAA,QAChB,MAAM,IAAI,MACR,gCAAgC,QAAQ,UAAS,QAAQ,SAC3D;AAAA,MACF;AAAA,MACA,OAAO,gBAAgB,OAAO,IAAI;AAAA;AAAA,EAEtC;AAAA;AAGF,SAAS,QAAQ,CAAC,OAAyB;AAAA,EACzC,IAAI,MAAM,SAAS;AAAA,IAAU,OAAO,MAAM;AAAA,EAC1C,OAAO,mBAAmB,MAAM,IAAI;AAAA;AAGtC,SAAS,kBAAkB,CAAC,OAA2B;AAAA,EACrD,IAAI,OAAO,WAAW,aAAa;AAAA,IACjC,OAAO,OAAO,KAAK,KAAK,EAAE,SAAS,QAAQ;AAAA,EAC7C;AAAA,EACA,IAAI,SAAS;AAAA,EACb,SAAS,IAAI,EAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;AAAA,IACxC,UAAU,OAAO,aAAa,MAAM,EAAE;AAAA,EACxC;AAAA,EAGA,OAAO,KAAK,MAAM;AAAA;AAGpB,SAAS,eAAe,CAAC,QAAoC;AAAA,EAC3D,OAAO;AAAA,IACL,OAAO,OAAO,MAAM,IAAI,OAAO;AAAA,IAC/B,UAAU,OAAO;AAAA,IACjB,WAAW,OAAO;AAAA,IAClB,cAAc;AAAA,IACd,eAAe,OAAO;AAAA,EACxB;AAAA;AAGF,SAAS,OAAO,CAAC,MAA8B;AAAA,EAC7C,OAAO;AAAA,IACL,MAAM,KAAK;AAAA,IACX,YAAY,KAAK;AAAA,IACjB,aAAa,KAAK;AAAA,EACpB;AAAA;AAOK,SAAS,0BAA0B,GAAS;AAAA,EACjD,SAAS,MAAM;AAAA;AA0EjB,IAAI,6BAAsD;AAOnD,SAAS,wBAAwB,CACtC,UACM;AAAA,EACN,6BAA6B;AAAA;AAGxB,SAAS,mBAAmB,GAA4B;AAAA,EAC7D,OAAO;AAAA;AAgDT,IAAI,+BAA0D;AAMvD,SAAS,0BAA0B,CACxC,UACM;AAAA,EACN,+BAA+B;AAAA;AAG1B,SAAS,qBAAqB,GAA8B;AAAA,EACjE,OAAO;AAAA;",
+  "debugId": "ECB47B4DF356B75E64756E2164756E21",
+  "names": []
+}

package/dist/osworld/action-converter.d.ts

@@ -7,11 +7,14 @@
  */
 import type { DesktopActionParams } from "../types.js";
 import type { OSWorldAction } from "./types.js";
+export interface OSWorldPointerState {
+    mouseDownAt?: [number, number];
+}
 /**
  * Convert an OSWorld computer_13 action to a DesktopActionParams.
  * Returns null for WAIT/DONE/FAIL (control flow, not desktop actions).
  */
-export declare function fromOSWorldAction(action: OSWorldAction): DesktopActionParams | null;
+export declare function fromOSWorldAction(action: OSWorldAction, pointerState?: OSWorldPointerState): DesktopActionParams | null;
 /**
  * Convert a DesktopActionParams to an OSWorld computer_13 action.
  */

package/dist/osworld/action-converter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"action-converter.d.ts","sourceRoot":"","sources":["../../src/osworld/action-converter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,GACpB,mBAAmB,GAAG,IAAI,CAgG5B;AAID;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,aAAa,CAwE1E;AAID;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAkHtE"}
+{"version":3,"file":"action-converter.d.ts","sourceRoot":"","sources":["../../src/osworld/action-converter.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD,MAAM,WAAW,mBAAmB;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,EACrB,YAAY,CAAC,EAAE,mBAAmB,GACjC,mBAAmB,GAAG,IAAI,CA8G5B;AAID;;GAEG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,aAAa,CAwE1E;AAID;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CA+GtE"}

package/dist/osworld/adapter.d.ts

@@ -21,6 +21,7 @@ export declare class OSWorldAdapter {
     private service;
     private config;
     private stepCount;
+    private pointerState;
     private trajectory;
     constructor(service: ComputerUseService, config?: Partial<OSWorldAgentConfig>);
     /**

package/dist/osworld/adapter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/osworld/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAM9E,OAAO,KAAK,EAEV,aAAa,EACb,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,UAAU,CAIV;gBAGN,OAAO,EAAE,kBAAkB,EAC3B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IActC;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA2BtE;;;OAGG;IACG,aAAa,CACjB,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAyBjE;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IA2BjE;;;OAGG;IACG,IAAI,CACR,MAAM,EAAE,aAAa,GAAG,MAAM,EAC9B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,iBAAiB,CAAC;IAuC7B;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;OAEG;IACH,aAAa;gBA3KH,aAAa,GAAG,MAAM;qBACjB,kBAAkB;mBACpB,MAAM;;IA6KnB;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,eAAe,IAAI,OAAO;IAI1B;;OAEG;IACH,SAAS,IAAI,kBAAkB;IAM/B,OAAO,CAAC,KAAK;CAGd"}
+{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/osworld/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAM9E,OAAO,KAAK,EACV,aAAa,EACb,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EAClB,MAAM,YAAY,CAAC;AAEpB,qBAAa,cAAc;IACzB,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,MAAM,CAAqB;IACnC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,YAAY,CAA2B;IAC/C,OAAO,CAAC,UAAU,CAIV;gBAGN,OAAO,EAAE,kBAAkB,EAC3B,MAAM,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC;IActC;;OAEG;IACG,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,CAAC;IA2BtE;;;OAGG;IACG,aAAa,CACjB,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IAyBjE;;OAEG;IACG,gBAAgB,CACpB,IAAI,EAAE,MAAM,GACX,OAAO,CAAC;QAAE,QAAQ,EAAE,OAAO,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,CAAC;IA2BjE;;;OAGG;IACG,IAAI,CACR,MAAM,EAAE,aAAa,GAAG,MAAM,EAC9B,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,iBAAiB,CAAC;IAuC7B;;OAEG;IACH,KAAK,IAAI,IAAI;IAMb;;OAEG;IACH,aAAa;gBA5KH,aAAa,GAAG,MAAM;qBACjB,kBAAkB;mBACpB,MAAM;;IA8KnB;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,eAAe,IAAI,OAAO;IAI1B;;OAEG;IACH,SAAS,IAAI,kBAAkB;IAM/B,OAAO,CAAC,KAAK;CAGd"}

package/dist/parity/index.d.ts

@@ -0,0 +1,9 @@
+/**
+ * trycua/cua parity tooling (#9170 M14) — public surface.
+ *
+ * The machine-checkable capability matrix + its validator, and the ScreenSpot
+ * grounding harness. The OSWorld benchmark adapter lives under `src/osworld/`.
+ */
+export { type OsCoverage, type OsName, PARITY_MATRIX, type ParityCapability, type ParityCoverageByOs, type ParityStatus, type ParityValidationProblem, type ParityValidationResult, parityCoverageByOs, parityMatrixSummary, validateParityCoverage, validateParityMatrix, } from "./parity-matrix.js";
+export { pointInBbox, type ScreenSpotPrediction, type ScreenSpotSample, type ScreenSpotSampleResult, type ScreenSpotScore, scoreScreenSpot, } from "./screenspot.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/parity/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/parity/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,KAAK,UAAU,EACf,KAAK,MAAM,EACX,aAAa,EACb,KAAK,gBAAgB,EACrB,KAAK,kBAAkB,EACvB,KAAK,YAAY,EACjB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,kBAAkB,EAClB,mBAAmB,EACnB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,WAAW,EACX,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,EAC3B,KAAK,eAAe,EACpB,eAAe,GAChB,MAAM,iBAAiB,CAAC"}