@elizaos/plugin-computeruse 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/actor/computer-interface.d.ts

@@ -0,0 +1,276 @@
+/**
+ * WS7 — BaseComputerInterface (TypeScript port).
+ *
+ * Shape ported from trycua/cua's `BaseComputerInterface` (MIT). The original
+ * provides one standard API surface for screenshot + mouse + keyboard + screen
+ * geometry that the Brain/Actor cascade can call without knowing what OS or
+ * driver is underneath.
+ *
+ *   Origin: https://github.com/trycua/cua/blob/main/libs/python/computer/
+ *           computer/interface/base.py
+ *   License: MIT, Copyright (c) 2025 trycua
+ *
+ * This is NOT a verbatim re-implementation — Eliza's underlying drivers
+ * (`platform/driver.ts`, `platform/capture.ts`) already cover most of the same
+ * surface. We give the cascade one well-typed seam so tests can swap in a
+ * fake driver, and so the Brain doesn't need to know about display-id
+ * resolution, retina scaling, or coord-source switches.
+ *
+ * Coordinate contract (matches WS5):
+ *   - Every method that takes `(displayId, x, y)` uses LOCAL pixel coords for
+ *     that display. The interface routes those through `localToGlobalDefault`
+ *     before any input fires.
+ *   - VLMs producing coords in image-space (i.e. against the downscaled max-
+ *     pixels frame) MUST first call `toScreenCoordinates(...)` to get the
+ *     real OS-pixel coord. The inverse `toScreenshotCoordinates(...)` is for
+ *     logging / replay.
+ *   - `displayId` must be a known display from `listDisplays()`. Unknown ids
+ *     throw — this is the safety net the dispatch layer relies on.
+ */
+import { type DisplayCapture } from "../platform/capture.js";
+import type { Scene, SceneAxNode } from "../scene/scene-types.js";
+import type { DisplayDescriptor } from "../types.js";
+/** A POSIX-style mouse button. */
+export type MouseButton = "left" | "middle" | "right";
+export interface ScreenshotResult {
+    displayId: number;
+    /** PNG bytes at backing-store resolution. */
+    frame: Buffer;
+    /** Backing-store / logical-pixel ratio. */
+    scaleFactor: number;
+    /** [x, y, w, h] in OS-global pixel space, for the display this frame belongs to. */
+    bounds: [number, number, number, number];
+}
+export interface DisplayPoint {
+    displayId: number;
+    /** LOCAL pixel coords inside the display's logical bounds. */
+    x: number;
+    y: number;
+}
+export interface DragPath {
+    displayId: number;
+    path: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+export interface ScrollDelta {
+    displayId: number;
+    x: number;
+    y: number;
+    /** Negative = scroll up/left, positive = scroll down/right. */
+    dx: number;
+    dy: number;
+}
+export interface CursorPosition {
+    displayId: number;
+    x: number;
+    y: number;
+}
+/**
+ * The single seam the cascade calls. All methods are safe to await
+ * concurrently within one display (driver semantics permitting); callers
+ * serialize at the dispatch layer.
+ */
+export interface ComputerInterface {
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(path: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    /**
+     * Convert from VLM-image-space (the downscaled max-pixels frame the model
+     * was shown) into OS-LOCAL-pixel-space for the same display.
+     *
+     * `(imgX, imgY)` are pixel coords inside an `imgW × imgH` image. Returned
+     * `(x, y)` are the same physical point but in the display's logical bounds.
+     */
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    /** Inverse of `toScreenCoordinates`. */
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    /**
+     * Return the AX (accessibility) tree the scene-builder snapshot already
+     * collected. We don't re-snapshot here — that's WS6's job. Cascade calls
+     * this to enumerate clickable AX nodes by display.
+     */
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+}
+/**
+ * Reference implementation that delegates to `platform/driver.ts` and
+ * `platform/capture.ts`. The constructor takes a `Scene` accessor so the
+ * AX tree is read from the latest WS6 snapshot without re-walking the OS.
+ */
+export interface ComputerInterfaceDeps {
+    /** Latest scene accessor — used for `getAccessibilityTree`. */
+    getScene?: () => Scene | null;
+    /** Capture override (mostly for tests). */
+    capture?: (displayId: number) => Promise<DisplayCapture>;
+    /** Driver overrides for tests. */
+    driver?: Partial<{
+        click: (x: number, y: number) => Promise<void>;
+        doubleClick: (x: number, y: number) => Promise<void>;
+        rightClick: (x: number, y: number) => Promise<void>;
+        mouseMove: (x: number, y: number) => Promise<void>;
+        mouseDown: (x: number, y: number, button: MouseButton) => Promise<void>;
+        mouseUp: (x: number, y: number, button: MouseButton) => Promise<void>;
+        drag: (x1: number, y1: number, x2: number, y2: number) => Promise<void>;
+        dragPath: (path: Array<{
+            x: number;
+            y: number;
+        }>) => Promise<void>;
+        scroll: (x: number, y: number, direction: "up" | "down" | "left" | "right", amount: number) => Promise<void>;
+        type: (text: string) => Promise<void>;
+        keyPress: (key: string) => Promise<void>;
+        keyCombo: (combo: string) => Promise<void>;
+        keyDown: (key: string) => Promise<void>;
+        keyUp: (key: string) => Promise<void>;
+    }>;
+    listDisplays?: () => DisplayDescriptor[];
+    /**
+     * Where the implementation should remember the last cursor target.
+     * Set whenever a movement-bearing call resolves successfully.
+     */
+    cursorState?: {
+        current: CursorPosition;
+    };
+}
+export declare class DefaultComputerInterface implements ComputerInterface {
+    private readonly deps;
+    constructor(deps?: ComputerInterfaceDeps);
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(args: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+    private primaryId;
+    private requireDisplay;
+    private toGlobalChecked;
+}
+/** Convenience factory used by the cascade. */
+export declare function makeComputerInterface(deps?: ComputerInterfaceDeps): ComputerInterface;
+//# sourceMappingURL=computer-interface.d.ts.map

package/dist/actor/computer-interface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"computer-interface.d.ts","sourceRoot":"","sources":["../../src/actor/computer-interface.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAkB,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAkB7E,OAAO,KAAK,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAErD,kCAAkC;AAClC,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;AAEtD,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,6CAA6C;IAC7C,KAAK,EAAE,MAAM,CAAC;IACd,2CAA2C;IAC3C,WAAW,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CAC1C;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAED,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,KAAK,CAAC;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,+DAA+D;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACX;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,IAAI,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACrE,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvE,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,KAAK,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,QAAQ,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/C,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,UAAU,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvE,aAAa,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACrE,iBAAiB,IAAI,cAAc,CAAC;IAEpC;;;;;;OAMG;IACH,mBAAmB,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAE7B,wCAAwC;IACxC,uBAAuB,CAAC,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAEnC;;;;OAIG;IACH,oBAAoB,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,WAAW,EAAE,CAAC;CACnE;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IACpC,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,MAAM,KAAK,GAAG,IAAI,CAAC;IAC9B,2CAA2C;IAC3C,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IACzD,kCAAkC;IAClC,MAAM,CAAC,EAAE,OAAO,CAAC;QACf,KAAK,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QAC/C,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACrD,UAAU,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,SAAS,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACnD,SAAS,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACxE,OAAO,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACtE,IAAI,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACxE,QAAQ,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACnE,MAAM,EAAE,CACN,CAAC,EAAE,MAAM,EACT,CAAC,EAAE,MAAM,EACT,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,EAC3C,MAAM,EAAE,MAAM,KACX,OAAO,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACtC,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3C,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;KACvC,CAAC,CAAC;IACH,YAAY,CAAC,EAAE,MAAM,iBAAiB,EAAE,CAAC;IACzC;;;OAGG;IACH,WAAW,CAAC,EAAE;QAAE,OAAO,EAAE,cAAc,CAAA;KAAE,CAAC;CAC3C;AAED,qBAAa,wBAAyB,YAAW,iBAAiB;IAChE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAKnB;gBAEU,IAAI,GAAE,qBAA0B;IA4BtC,UAAU,CACd,IAAI,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAChC,OAAO,CAAC,gBAAgB,CAAC;IAWtB,SAAS,CACb,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC7C,OAAO,CAAC,IAAI,CAAC;IAcV,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAYtE,SAAS,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9C,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO/C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAO9C,MAAM,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAc1C,IAAI,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BnC,OAAO,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7C,KAAK,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK3C,QAAQ,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK/C,QAAQ,CAAC,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAK9C,MAAM,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/C,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBzC,QAAQ,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAapE,UAAU,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5E,aAAa,CAAC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAKpE,iBAAiB,IAAI,cAAc;IAInC,mBAAmB,CAAC,IAAI,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE;IAe5B,uBAAuB,CAAC,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,CAAC;QAClB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAelC,oBAAoB,CAAC,IAAI,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,WAAW,EAAE;IAOjE,OAAO,CAAC,SAAS;IAKjB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,eAAe;CAexB;AAED,+CAA+C;AAC/C,wBAAgB,qBAAqB,CACnC,IAAI,GAAE,qBAA0B,GAC/B,iBAAiB,CAEnB"}

package/dist/actor/dispatch.d.ts

@@ -0,0 +1,24 @@
+/**
+ * WS7 — Dispatch layer.
+ *
+ * Single entry point that validates a resolved `ProposedAction` and routes
+ * it to a `ComputerInterface`. Errors are returned as structured
+ * `ActionResult.error` values — we don't let exceptions escape the cascade
+ * boundary, so the agent loop can recover by re-prompting the Brain.
+ *
+ * Validation:
+ *   - displayId must exist
+ *   - point coords (when present) must be inside the display's local bounds
+ *   - text/key/keys/dx/dy must match the action kind
+ *
+ * No business logic lives here. The dispatcher is dumb on purpose.
+ */
+import type { DisplayDescriptor } from "../types.js";
+import type { ComputerInterface } from "./computer-interface.js";
+import type { ActionResult, ProposedAction } from "./types.js";
+export interface DispatchDeps {
+    interface: ComputerInterface;
+    listDisplays: () => DisplayDescriptor[];
+}
+export declare function dispatch(action: ProposedAction, deps: DispatchDeps): Promise<ActionResult>;
+//# sourceMappingURL=dispatch.d.ts.map

package/dist/actor/dispatch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dispatch.d.ts","sourceRoot":"","sources":["../../src/actor/dispatch.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,KAAK,EAAE,iBAAiB,EAAgB,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE/D,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,iBAAiB,CAAC;IAC7B,YAAY,EAAE,MAAM,iBAAiB,EAAE,CAAC;CACzC;AAED,wBAAsB,QAAQ,CAC5B,MAAM,EAAE,cAAc,EACtB,IAAI,EAAE,YAAY,GACjB,OAAO,CAAC,YAAY,CAAC,CAmJvB"}

package/dist/actor/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * WS7 — Brain / Actor / Cascade public surface.
+ */
+export { type Actor, type ActorGroundArgs, OcrCoordinateGroundingActor, OsAtlasProActor, type OsAtlasProActorOptions, resolveReference, } from "./actor.js";
+export { type AgentDispatchContext, type AgentMiddleware, type AgentRunContext, type AgentRunSummary, type AgentStepContext, type AgentStepDecision, type BudgetCapOptions, createBudgetCapMiddleware, createImageRetentionMiddleware, createOperatorNormalizerMiddleware, createTrajectoryMiddleware, type ImageRetentionMiddleware, normalizeProposedAction, type TrajectoryEntry, type TrajectoryMiddleware, } from "./agent-callbacks.js";
+export { _resetAgentLoopsForTests, AGENT_LOOP_SETTING, type AgentLoop, type AgentLoopDeps, type AgentLoopRegistration, type AgentStepInput, createAgentLoop, DEFAULT_AGENT_LOOP_MODEL, LocalGrounderLoop, listAgentLoops, matchesModelFamily, type PredictClickInput, registerAgentLoop, selectAgentLoopRegistration, unregisterAgentLoop, } from "./agent-loop.js";
+export { BRAIN_MAX_PIXELS, BRAIN_MAX_ROIS, Brain, type BrainDeps, type BrainInput, BrainParseError, brainPromptFor, encodeForBrain, parseBrainOutput, } from "./brain.js";
+export { Cascade, type CascadeDeps, type CascadeInput, getRegisteredActor, setActor, } from "./cascade.js";
+export { type ComputerInterface, type ComputerInterfaceDeps, type CursorPosition, DefaultComputerInterface, type DisplayPoint, type DragPath, type MouseButton, makeComputerInterface, type ScreenshotResult, type ScrollDelta, } from "./computer-interface.js";
+export { type DispatchDeps, dispatch } from "./dispatch.js";
+export type { ActionResult as ActorActionResult, BrainActionKind, BrainOutput, BrainProposedAction, BrainRoi, CascadeResult, GroundingResult, ProposedAction, ReferenceTarget, } from "./types.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/actor/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actor/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EACL,KAAK,KAAK,EACV,KAAK,eAAe,EACpB,2BAA2B,EAC3B,eAAe,EACf,KAAK,sBAAsB,EAC3B,gBAAgB,GACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,yBAAyB,EACzB,8BAA8B,EAC9B,kCAAkC,EAClC,0BAA0B,EAC1B,KAAK,wBAAwB,EAC7B,uBAAuB,EACvB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,KAAK,SAAS,EACd,KAAK,aAAa,EAClB,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,eAAe,EACf,wBAAwB,EACxB,iBAAiB,EACjB,cAAc,EACd,kBAAkB,EAClB,KAAK,iBAAiB,EACtB,iBAAiB,EACjB,2BAA2B,EAC3B,mBAAmB,GACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,KAAK,EACL,KAAK,SAAS,EACd,KAAK,UAAU,EACf,eAAe,EACf,cAAc,EACd,cAAc,EACd,gBAAgB,GACjB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,OAAO,EACP,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,kBAAkB,EAClB,QAAQ,GACT,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,wBAAwB,EACxB,KAAK,YAAY,EACjB,KAAK,QAAQ,EACb,KAAK,WAAW,EAChB,qBAAqB,EACrB,KAAK,gBAAgB,EACrB,KAAK,WAAW,GACjB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,KAAK,YAAY,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC5D,YAAY,EACV,YAAY,IAAI,iBAAiB,EACjC,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,QAAQ,EACR,aAAa,EACb,eAAe,EACf,cAAc,EACd,eAAe,GAChB,MAAM,YAAY,CAAC"}

package/dist/actor/types.d.ts

@@ -0,0 +1,94 @@
+/**
+ * WS7 — Shared types for Brain / Actor / Cascade / Dispatch.
+ *
+ * These are the JSON contracts the planner and the cascade exchange. The
+ * Brain MUST emit `BrainOutput`; the cascade resolves it into a concrete
+ * `ProposedAction` with display-local OS-pixel coords; the dispatcher
+ * validates and forwards it through `ComputerInterface`.
+ */
+import type { SceneAxNode, SceneOcrBox } from "../scene/scene-types.js";
+export type BrainActionKind = "click" | "double_click" | "right_click" | "type" | "hotkey" | "key" | "scroll" | "drag" | "wait" | "finish";
+export interface BrainRoi {
+    displayId: number;
+    /** [x, y, w, h] in DISPLAY-LOCAL pixel space. */
+    bbox: [number, number, number, number];
+    reason: string;
+}
+export interface BrainProposedAction {
+    kind: BrainActionKind;
+    /** Stable id of an OCR box (`t<displayId>-<seq>`) or AX node (`a<displayId>-<seq>`). */
+    ref?: string;
+    /**
+     * Action-specific args. For `type`: { text }. For `hotkey`: { keys }. For
+     * `key`: { key }. For `scroll`: { dx, dy }. For `drag`: { from: {x,y}, to:
+     * {x,y} }. Coordinates here are display-local pixel space.
+     */
+    args?: Record<string, unknown>;
+    rationale: string;
+}
+export interface BrainOutput {
+    scene_summary: string;
+    target_display_id: number;
+    /** Up to N (cap enforced by cascade) ROIs the Brain wants the Actor to ground. */
+    roi: BrainRoi[];
+    proposed_action: BrainProposedAction;
+}
+/**
+ * Concrete action after the cascade has resolved coords. The dispatcher
+ * accepts only this shape.
+ */
+export interface ProposedAction {
+    kind: BrainActionKind;
+    displayId: number;
+    /** Display-local pixel coords, resolved by the cascade. */
+    x?: number;
+    y?: number;
+    /** For drag: start point. */
+    startX?: number;
+    startY?: number;
+    text?: string;
+    key?: string;
+    keys?: string[];
+    dx?: number;
+    dy?: number;
+    ref?: string;
+    rationale: string;
+}
+export interface ActionResult {
+    success: boolean;
+    /** Structured error code; absent on success. */
+    error?: {
+        code: "unknown_display" | "out_of_bounds" | "invalid_args" | "driver_error" | "brain_error" | "actor_error" | "timeout" | "internal";
+        message: string;
+    };
+    /** Action that was actually issued (mirrors input on success). */
+    issued?: ProposedAction;
+}
+export interface CascadeResult {
+    scene_summary: string;
+    proposed: ProposedAction;
+    /** ROIs the Brain produced, kept for trajectory logging. */
+    rois: BrainRoi[];
+}
+/**
+ * What the Actor returns when grounding a ROI / ref to a coord.
+ * `coords` are display-local pixel space.
+ */
+export interface GroundingResult {
+    displayId: number;
+    x: number;
+    y: number;
+    /** 0..1 confidence — informational only; the dispatcher does not gate on it. */
+    confidence: number;
+    /** Human-readable explanation, for trajectory + retries. */
+    reason: string;
+}
+/** Picked OCR/AX target — used by the deterministic OCR/AX grounding actor. */
+export interface ReferenceTarget {
+    displayId: number;
+    bbox: [number, number, number, number];
+    kind: "ocr" | "ax";
+    label: string;
+    source: SceneOcrBox | SceneAxNode;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/actor/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/actor/types.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAExE,MAAM,MAAM,eAAe,GACvB,OAAO,GACP,cAAc,GACd,aAAa,GACb,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,MAAM,GACN,MAAM,GACN,QAAQ,CAAC;AAEb,MAAM,WAAW,QAAQ;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,eAAe,CAAC;IACtB,wFAAwF;IACxF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kFAAkF;IAClF,GAAG,EAAE,QAAQ,EAAE,CAAC;IAChB,eAAe,EAAE,mBAAmB,CAAC;CACtC;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,eAAe,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,2DAA2D;IAC3D,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,KAAK,CAAC,EAAE;QACN,IAAI,EACA,iBAAiB,GACjB,eAAe,GACf,cAAc,GACd,cAAc,GACd,aAAa,GACb,aAAa,GACb,SAAS,GACT,UAAU,CAAC;QACf,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,kEAAkE;IAClE,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,cAAc,CAAC;IACzB,4DAA4D;IAC5D,IAAI,EAAE,QAAQ,EAAE,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,gFAAgF;IAChF,UAAU,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,+EAA+E;AAC/E,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,EAAE,KAAK,GAAG,IAAI,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,WAAW,GAAG,WAAW,CAAC;CACnC"}

package/dist/approval-manager.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"approval-manager.d.ts","sourceRoot":"","sources":["../src/approval-manager.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AA+BpB,KAAK,gBAAgB,GAAG;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAMF,KAAK,gBAAgB,GAAG,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;AAE7D,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,YAAY,CAEnE;AAED,qBAAa,0BAA0B;IACrC,OAAO,CAAC,IAAI,CAAgC;IAC5C,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAIzB;;IAMF,OAAO,IAAI,YAAY;IAIvB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY;IASnC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAY3C,SAAS,IAAI,OAAO;IAIpB,eAAe,CACb,OAAO,EAAE,MAAM,EACf,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GACvC,OAAO,CAAC,gBAAgB,CAAC;IAgB5B,WAAW,IAAI,gBAAgB;IAe/B,eAAe,CACb,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,OAAO,EACjB,MAAM,CAAC,EAAE,MAAM,GACd,kBAAkB,GAAG,IAAI;IAsB5B,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAQhC,SAAS,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,IAAI;IAQjD,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,IAAI;CAMb"}
+{"version":3,"file":"approval-manager.d.ts","sourceRoot":"","sources":["../src/approval-manager.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAEjB,MAAM,YAAY,CAAC;AAqCpB,KAAK,gBAAgB,GAAG;IACtB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAMF,KAAK,gBAAgB,GAAG,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;AAE7D,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,YAAY,CAEnE;AAED,qBAAa,0BAA0B;IACrC,OAAO,CAAC,IAAI,CAAiC;IAC7C,OAAO,CAAC,OAAO,CAA4C;IAC3D,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAIzB;;IAMF,OAAO,IAAI,YAAY;IAIvB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY;IASnC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAY3C,SAAS,IAAI,OAAO;IAIpB,eAAe,CACb,OAAO,EAAE,MAAM,EACf,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GACvC,OAAO,CAAC,gBAAgB,CAAC;IAgB5B,WAAW,IAAI,gBAAgB;IAe/B,eAAe,CACb,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE,OAAO,EACjB,MAAM,CAAC,EAAE,MAAM,GACd,kBAAkB,GAAG,IAAI;IAsB5B,SAAS,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAQhC,SAAS,CAAC,QAAQ,EAAE,gBAAgB,GAAG,MAAM,IAAI;IAQjD,OAAO,CAAC,UAAU;IAYlB,OAAO,CAAC,UAAU;IAalB,OAAO,CAAC,IAAI;CAMb"}

package/dist/index.d.ts

@@ -2,8 +2,10 @@
  * @elizaos/plugin-computeruse
  *
  * Desktop automation plugin for elizaOS agents — screenshots, mouse/keyboard
- * control, browser CDP automation, terminal access, file operations, and
- * window management.
+ * control, browser CDP automation, and window management.
+ *
+ * File operations belong on the FILE action; shell/terminal access belongs on
+ * the SHELL action. They are not exposed by this plugin.
  *
  * Deeply ported from coasty-ai/open-computer-use (Apache 2.0).
  *
@@ -23,11 +25,22 @@ import type { Plugin } from "@elizaos/core";
 export declare const computerUsePlugin: Plugin;
 export declare const computerusePlugin: Plugin;
 export default computerUsePlugin;
-export { ComputerUseService } from "./services/computer-use-service.js";
-export { captureDesktopScreenshot, commandExists, detectDesktopControlCapabilities, getDesktopPlatformName, isHeadfulGuiAvailable, listDesktopWindows, performDesktopClick, performDesktopDoubleClick, performDesktopKeypress, performDesktopMouseMove, performDesktopScroll, performDesktopTextInput, } from "./services/desktop-control.js";
-export type { DesktopControlCapabilities, DesktopControlCapability, DesktopInputButton, DesktopScreenshotRegion, DesktopWindowInfo, } from "./services/desktop-control.js";
+export { type ComputerUseAgentParams, type ComputerUseAgentReport, computerUseAgentAction, runComputerUseAgentLoop, } from "./actions/use-computer-agent.js";
+export * from "./actor/index.js";
+export * from "./mobile/index.js";
+export { listProcesses, type ProcessInfo, parsePsOutput, parseWindowsProcessJson, } from "./platform/process-list.js";
+export { sceneProvider } from "./providers/scene.js";
 export { handleComputerUseRoutes } from "./routes/computer-use-routes.js";
 export { handleSandboxRoute } from "./routes/sandbox-routes.js";
+export { type AccessibilityProvider, DarwinAccessibilityProvider, LinuxAccessibilityProvider, NullAccessibilityProvider, parseHyprlandClients, parseSwayTree, resolveAccessibilityProvider, setAccessibilityProvider, WindowsAccessibilityProvider, } from "./scene/a11y-provider.js";
+export { enumerateApps, joinAppsAndWindows } from "./scene/apps.js";
+export { type BlockGrid, blockGrid, type DirtyBlock, decodePng, diffBlocks, frameDhash, hamming, } from "./scene/dhash.js";
+export { _resetDefaultSceneBuilderForTests, getDefaultSceneBuilder, SceneBuilder, type SceneBuilderDeps, type SceneUpdateEvent, } from "./scene/scene-builder.js";
+export type { Scene, SceneApp, SceneAppWindow, SceneAxNode, SceneFocusedWindow, SceneOcrBox, SceneVlmElement, } from "./scene/scene-types.js";
+export { serializeSceneForPrompt } from "./scene/serialize.js";
+export { ComputerUseService } from "./services/computer-use-service.js";
+export type { DesktopControlCapabilities, DesktopControlCapability, DesktopInputButton, DesktopScreenshotRegion, DesktopWindowInfo, } from "./services/desktop-control.js";
+export { captureDesktopScreenshot, commandExists, detectDesktopControlCapabilities, getDesktopPlatformName, isHeadfulGuiAvailable, listDesktopWindows, performDesktopClick, performDesktopDoubleClick, performDesktopKeypress, performDesktopMouseMove, performDesktopScroll, performDesktopTextInput, } from "./services/desktop-control.js";
+export { VISION_CONTEXT_SERVICE_TYPE, VISION_CONTEXT_TASK_GOAL_CACHE_KEY, type VisionContext, type VisionContextBBox, type VisionContextFocusedWindow, VisionContextProvider, type VisionContextRecentAction, } from "./services/vision-context-provider.js";
 export type { ActionHistoryEntry, ApprovalMode, ApprovalResolution, ApprovalSnapshot, BrowserActionParams, BrowserActionResult, BrowserActionType, BrowserInfo, BrowserState, BrowserTab, ClickableElement, ComputerActionResult, ComputerUseConfig, ComputerUseResult, DesktopActionParams, DesktopActionType, FileActionParams, FileActionResult, FileActionType, FileEntry, PendingApproval, PermissionType, PlatformCapabilities, ScreenRegion, ScreenSize, TerminalActionParams, TerminalActionResult, TerminalActionType, WindowActionParams, WindowActionResult, WindowActionType, WindowInfo, } from "./types.js";
-export * from "./register-routes.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAS,MAAM,eAAe,CAAC;AAsCnD,eAAO,MAAM,iBAAiB,EAAE,MA2B/B,CAAC;AAEF,eAAO,MAAM,iBAAiB,QAAoB,CAAC;AAEnD,eAAe,iBAAiB,CAAC;AAEjC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,gCAAgC,EAChC,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,yBAAyB,EACzB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,+BAA+B,CAAC;AACvC,YAAY,EACV,0BAA0B,EAC1B,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAEhE,YAAY,EACV,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,GACX,MAAM,YAAY,CAAC;AACpB,cAAc,sBAAsB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAS,MAAM,eAAe,CAAC;AA0CnD,eAAO,MAAM,iBAAiB,EAAE,MAsC/B,CAAC;AAEF,eAAO,MAAM,iBAAiB,QAAoB,CAAC;AAEnD,eAAe,iBAAiB,CAAC;AAEjC,OAAO,EACL,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC3B,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,iCAAiC,CAAC;AAEzC,cAAc,kBAAkB,CAAC;AAEjC,cAAc,mBAAmB,CAAC;AAClC,OAAO,EACL,aAAa,EACb,KAAK,WAAW,EAChB,aAAa,EACb,uBAAuB,GACxB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAChE,OAAO,EACL,KAAK,qBAAqB,EAC1B,2BAA2B,EAC3B,0BAA0B,EAC1B,yBAAyB,EACzB,oBAAoB,EACpB,aAAa,EACb,4BAA4B,EAC5B,wBAAwB,EACxB,4BAA4B,GAC7B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACpE,OAAO,EACL,KAAK,SAAS,EACd,SAAS,EACT,KAAK,UAAU,EACf,SAAS,EACT,UAAU,EACV,UAAU,EACV,OAAO,GACR,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,iCAAiC,EACjC,sBAAsB,EACtB,YAAY,EACZ,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,GACtB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,KAAK,EACL,QAAQ,EACR,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,WAAW,EACX,eAAe,GAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AACxE,YAAY,EACV,0BAA0B,EAC1B,wBAAwB,EACxB,kBAAkB,EAClB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,wBAAwB,EACxB,aAAa,EACb,gCAAgC,EAChC,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,yBAAyB,EACzB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,2BAA2B,EAC3B,kCAAkC,EAClC,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,0BAA0B,EAC/B,qBAAqB,EACrB,KAAK,yBAAyB,GAC/B,MAAM,uCAAuC,CAAC;AAE/C,YAAY,EACV,kBAAkB,EAClB,YAAY,EACZ,kBAAkB,EAClB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,oBAAoB,EACpB,iBAAiB,EACjB,iBAAiB,EACjB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,eAAe,EACf,cAAc,EACd,oBAAoB,EACpB,YAAY,EACZ,UAAU,EACV,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,kBAAkB,EAClB,kBAAkB,EAClB,gBAAgB,EAChB,UAAU,GACX,MAAM,YAAY,CAAC"}