@elizaos/plugin-computeruse 2.0.3-beta.2 → 2.0.3-beta.4

package/dist/actor/aosp-input-actor.d.ts

@@ -0,0 +1,87 @@
+/**
+ * WS7 ↔ AOSP — Privileged-input actor.
+ *
+ * In the consumer build the cascade routes gestures through
+ * `MobileComputerInterface` → `AccessibilityGestureDescription` — which is
+ * coarse and blocks on touch-recognizer state in some apps (banking, DRM
+ * video, anything that sets `filterTouchesWhenObscured`).
+ *
+ * In an AOSP system-app build (see `docs/AOSP_SYSTEM_APP.md`), the
+ * privileged path uses `InputManager.injectInputEvent()` directly. That
+ * path lives behind `AospPrivilegedBridge.injectMotionEvent(...)` on the
+ * Kotlin side; the consumer-flavor `AospPrivilegedBridge` exports
+ * `createIfAvailable(): null` so this actor stays inert until a real AOSP
+ * bridge is linked in.
+ *
+ * `AospInputActor` maps a resolved WS7 `ProposedAction` (display-local
+ * pixel coords) into the privileged-bridge calls. It does NOT implement
+ * the `Actor` "grounding" contract — grounding stays with the OCR/AX or
+ * VLM actor; this is purely an *input-dispatch* shim. It's surfaced as
+ * an alternative to `ComputerInterface` for AOSP builds: the agent loop
+ * picks `AospInputActor.execute(action)` instead of `dispatch(action, {
+ * interface, ... })` when the privileged bridge is available.
+ */
+import type { ActionResult, ProposedAction } from "./types.js";
+/** Minimal Kotlin-side surface this actor needs from the AOSP build. */
+export interface AospPrivilegedInputBridge {
+    /**
+     * Inject a single motion event at the InputManager level. `action` follows
+     * `MotionEvent.ACTION_*` constants (DOWN=0, UP=1, MOVE=2). `downTimeMs`
+     * is the original-touch timestamp the gesture started at, in `uptimeMillis`
+     * units. Implementations enforce the INJECT_EVENTS permission.
+     */
+    injectMotionEvent(args: {
+        x: number;
+        y: number;
+        action: number;
+        downTimeMs: number;
+    }): Promise<{
+        ok: boolean;
+    }>;
+    /** Capture the primary display frame buffer synchronously. JPEG bytes. */
+    captureDisplayFrameBuffer?(): Promise<Uint8Array>;
+}
+export interface AospInputActorDeps {
+    /** Returns the AOSP bridge handle, or null in consumer builds. */
+    getBridge: () => AospPrivilegedInputBridge | null;
+    /** Override the clock for tests. */
+    now?: () => number;
+}
+/**
+ * Motion-event action constants matching `android.view.MotionEvent.ACTION_*`.
+ * Re-exported here so callers don't need to import Android Kotlin enums.
+ */
+export declare const MOTION_EVENT_ACTION_DOWN: 0;
+export declare const MOTION_EVENT_ACTION_UP: 1;
+export declare const MOTION_EVENT_ACTION_MOVE: 2;
+/**
+ * Translate a cascade-resolved `ProposedAction` into one or more
+ * `injectMotionEvent` calls. Returns the same `ActionResult` envelope the
+ * desktop dispatcher uses — invalid args or driver errors do not throw.
+ *
+ * Behavior parity with `dispatch.ts`:
+ *   - unknown action.kind → invalid_args
+ *   - missing coords      → invalid_args
+ *   - bridge throw        → driver_error
+ *   - bridge ok:false     → driver_error
+ *
+ * Coverage:
+ *   - click / double_click / right_click → tap(s)
+ *   - drag                              → DOWN at start, MOVE/UP at end
+ *   - scroll                            → swipe (DOWN, MOVE, UP)
+ *   - wait / finish                     → success: true (no input event)
+ *   - type / key / hotkey               → invalid_args (use AccessibilityNodeInfo
+ *                                         or a separate keymap actor; out of
+ *                                         scope for this privileged path).
+ */
+export declare class AospInputActor {
+    private readonly deps;
+    readonly name = "aosp-input";
+    constructor(deps: AospInputActorDeps);
+    execute(action: ProposedAction): Promise<ActionResult>;
+    private tap;
+    private swipe;
+    private must;
+    private now;
+}
+//# sourceMappingURL=aosp-input-actor.d.ts.map

package/dist/actor/aosp-input-actor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aosp-input-actor.d.ts","sourceRoot":"","sources":["../../src/actor/aosp-input-actor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAK/D,wEAAwE;AACxE,MAAM,WAAW,yBAAyB;IACxC;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE;QACtB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC7B,0EAA0E;IAC1E,yBAAyB,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,SAAS,EAAE,MAAM,yBAAyB,GAAG,IAAI,CAAC;IAClD,oCAAoC;IACpC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAG,CAAU,CAAC;AACnD,eAAO,MAAM,sBAAsB,EAAG,CAAU,CAAC;AACjD,eAAO,MAAM,wBAAwB,EAAG,CAAU,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,cAAc;IAGb,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,QAAQ,CAAC,IAAI,gBAAgB;gBAEA,IAAI,EAAE,kBAAkB;IAE/C,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;YAwH9C,GAAG;YA0BH,KAAK;YAsCL,IAAI;IAYlB,OAAO,CAAC,GAAG;CAGZ"}

package/dist/actor/brain.d.ts

@@ -0,0 +1,195 @@
+/**
+ * WS7 — Brain (full-screen reasoning).
+ *
+ * Sends one image per display (each downscaled to ~1.3 MP, the OS-Atlas /
+ * Qwen3-VL `max_pixels` convention) to `runtime.useModel(IMAGE_DESCRIPTION,
+ * ...)`. The model is prompted to emit a JSON `BrainOutput` describing:
+ *   - the scene in one paragraph,
+ *   - which display to act on,
+ *   - up to N ROIs the Actor should zoom into,
+ *   - a single proposed action with rationale.
+ *
+ * The Brain itself doesn't dispatch — it just produces `BrainOutput`. The
+ * cascade ("ScreenSeekeR") is the orchestrator that takes a `BrainOutput`,
+ * optionally calls the Actor on cropped ROIs, and produces a concrete
+ * `ProposedAction` for the dispatcher.
+ *
+ * Image transport contract: we pass `imageUrl` as a `data:image/png;base64,...`
+ * URL. The WS2 MemoryArbiter intercepts at `ModelType.IMAGE_DESCRIPTION` and
+ * routes through its content-hash cache, so identical frames don't burn
+ * inference budget twice.
+ *
+ * Image policy (#9105): the compact scene already carries OCR text + AX boxes,
+ * which can suffice to pick the next target, so the `"on-escalation"` policy
+ * plans from that text-only context with NO image — routed through a TEXT model,
+ * since every IMAGE_DESCRIPTION provider rejects an empty imageUrl — and
+ * attaches the ~1.3 MP frame only when the planned target cannot be grounded
+ * against the OCR/AX boxes. The DEFAULT is `"always"` (legacy: image on every
+ * call) until a real-model CUA trajectory validates imageless planning accuracy;
+ * operators opt into `"on-escalation"` via the `COMPUTERUSE_BRAIN_IMAGE_POLICY`
+ * setting. `"never"` never attaches pixels.
+ *
+ * Parse strictness:
+ *   - We try to parse the response as JSON (either the literal string or
+ *     `result.description`).
+ *   - On first parse failure, retry once with a stricter prompt.
+ *   - On second failure, throw `BrainParseError` — the cascade surfaces this
+ *     as a structured `ActionResult.error` and aborts the turn cleanly.
+ */
+import { type IAgentRuntime, type ImageDescriptionResult } from "@elizaos/core";
+import type { DisplayCapture } from "../platform/capture.js";
+import type { Scene } from "../scene/scene-types.js";
+import type { BrainOutput } from "./types.js";
+export declare const BRAIN_MAX_PIXELS = 1310720;
+export declare const BRAIN_MAX_ROIS = 2;
+/** Bound on the per-Brain dHash→BrainOutput cache (LRU-ish, oldest evicted). */
+export declare const BRAIN_DHASH_CACHE_MAX = 16;
+/**
+ * dHash Hamming threshold for cached-plan reuse (#9581 continuous-understanding
+ * tuning). Exact-equality (distance 0) re-burned the IMAGE_DESCRIPTION model on
+ * cosmetically-identical frames — cursor jitter, a blinking caret, anti-aliasing,
+ * and tiny scroll noise all flip a few dHash bits.
+ *
+ * This mirrors `SCREEN_STATE_HAMMING_THRESHOLD`: distances below the threshold
+ * are unchanged; distances at or above it are changed and must re-plan.
+ */
+export declare const BRAIN_DHASH_HAMMING_THRESHOLD = 5;
+/**
+ * Image-token estimate per source pixel for a vision model with the Qwen3-VL /
+ * OS-Atlas `max_pixels` convention: one visual token ≈ a 28×28 (≈750 px) patch.
+ * Used only to quantify the saving when a frame is *not* attached (#9105).
+ */
+export declare const BRAIN_PIXELS_PER_IMAGE_TOKEN = 750;
+/**
+ * When to attach the raw screenshot to the planning model (#9105):
+ *   - `"always"`     — attach the pixels on every call (legacy behaviour). Default.
+ *   - `"on-escalation"` — plan from the compact OCR/AX scene with no image
+ *                         first (routed through a TEXT model); attach pixels
+ *                         only when the planned target cannot be grounded
+ *                         against the OCR/AX boxes or a strict-retry fires.
+ *   - `"never"`      — never attach the screenshot; plan from the scene alone.
+ */
+export type BrainImagePolicy = "always" | "on-escalation" | "never";
+/**
+ * Default is `"always"` (proven legacy behaviour). `"on-escalation"` cuts the
+ * dominant per-frame image-token cost but plans the first pass blind to the
+ * pixels, so it stays opt-in (via the `COMPUTERUSE_BRAIN_IMAGE_POLICY` runtime
+ * setting / env var) until a real-model CUA trajectory validates its accuracy.
+ */
+export declare const DEFAULT_BRAIN_IMAGE_POLICY: BrainImagePolicy;
+/**
+ * Resolve the Brain image policy from the `COMPUTERUSE_BRAIN_IMAGE_POLICY`
+ * runtime setting / env var, falling back to {@link DEFAULT_BRAIN_IMAGE_POLICY}.
+ * The operator escape hatch to enable imageless planning without a code change
+ * once it is validated against a real model.
+ */
+export declare function resolveBrainImagePolicy(runtime: IAgentRuntime | null): BrainImagePolicy;
+/** Token-accounting snapshot for a Brain instance (#9105 M3). */
+export interface BrainStats {
+    /** IMAGE_DESCRIPTION model calls actually issued. */
+    invocations: number;
+    /** Describe calls served from the frame-dHash cache (no model call). */
+    cacheHits: number;
+    /**
+     * Model calls issued with NO screenshot attached (the scene's OCR/AX text
+     * sufficed). Each one saved roughly one full-frame image's worth of tokens.
+     */
+    imagelessCalls: number;
+    /**
+     * Estimated image tokens NOT sent because of imageless calls — the sum of
+     * `(width * height) / BRAIN_PIXELS_PER_IMAGE_TOKEN` over every imageless
+     * call, capped per frame at `BRAIN_MAX_PIXELS`.
+     */
+    estImageTokensSaved: number;
+}
+export declare class BrainParseError extends Error {
+    readonly raw: string;
+    constructor(message: string, raw: string);
+}
+export interface BrainDeps {
+    /** Optional override for tests — bypasses runtime.useModel. */
+    invokeModel?: (args: {
+        imageUrl: string;
+        prompt: string;
+        displayId: number;
+    }) => Promise<string | ImageDescriptionResult>;
+    /**
+     * When to attach the raw screenshot to the planning model (#9105). Defaults
+     * to `"always"` (legacy); see {@link resolveBrainImagePolicy} for the
+     * operator opt-in to `"on-escalation"`.
+     */
+    imagePolicy?: BrainImagePolicy;
+}
+export interface BrainInput {
+    scene: Scene;
+    goal: string;
+    /**
+     * Per-display capture buffers. If a display from `scene.displays` is
+     * missing here, the Brain skips it. The cascade is responsible for
+     * supplying these alongside the scene.
+     */
+    captures: Map<number, DisplayCapture>;
+}
+/**
+ * Pure description of a "Brain" call. Created by `Cascade.runCascade` and
+ * test fixtures.
+ */
+export declare class Brain {
+    private readonly runtime;
+    private readonly deps;
+    /**
+     * Frame-dHash → BrainOutput cache. The WS2 MemoryArbiter only dedups the
+     * IMAGE_DESCRIPTION call for LOCAL backends; the remote/cloud path bypasses
+     * it, so an identical screen re-burns tokens every step. This call-site cache
+     * skips the model entirely when the same frame is observed for the same goal,
+     * cutting the dominant CUA-loop token cost regardless of backend (#9105 M3).
+     */
+    private readonly dhashCache;
+    private readonly imagePolicy;
+    private invocations;
+    private cacheHits;
+    private imagelessCalls;
+    private estImageTokensSaved;
+    constructor(runtime: IAgentRuntime | null, deps?: BrainDeps);
+    /** Token-accounting snapshot (model calls, cache hits, imageless savings). */
+    getStats(): BrainStats;
+    private cacheKey;
+    /**
+     * Return a cached plan for the same goal whose frame is within
+     * `BRAIN_DHASH_HAMMING_THRESHOLD` bits of `dh` — a near-identical screen, not
+     * just a byte-identical one. On a hit the entry is moved to the end (LRU), so
+     * a steadily-evolving screen keeps its most-recent close match warm.
+     */
+    private findCached;
+    private rememberOutput;
+    observeAndPlan(input: BrainInput): Promise<BrainOutput>;
+    /**
+     * True when the imageless plan needs no screenshot to dispatch: a
+     * non-coordinate action (type/hotkey/key/wait/finish) carries everything in
+     * its args, and a coordinate action is fine when its `ref`/rationale resolves
+     * to a concrete OCR/AX box. When the target cannot be grounded we escalate to
+     * the pixels — correctness over token saving.
+     */
+    private resolvesWithoutImage;
+    private recordImageless;
+    private invoke;
+}
+export declare function brainPromptFor(compactSceneJson: string, goal: string, strict: boolean): string;
+export declare function parseBrainOutput(raw: string): BrainOutput;
+/**
+ * Encode a PNG buffer for transport to the IMAGE_DESCRIPTION model. We don't
+ * resize here — `runtime.useModel` adapters (and any vLLM backends behind
+ * them) handle the `max_pixels` downscale. The constant `BRAIN_MAX_PIXELS`
+ * is exported for the cascade so it can crop ROIs at the right native
+ * resolution before invoking the Actor.
+ */
+export declare function encodeForBrain(png: Buffer): Promise<string>;
+/**
+ * Estimate the visual tokens a frame would have cost the planning model, for
+ * the "tokens saved by going imageless" telemetry (#9105). The backends apply a
+ * `max_pixels` downscale, so the per-frame estimate is capped at
+ * `BRAIN_MAX_PIXELS`. Falls back to the cap when the PNG header is unreadable —
+ * an attached frame would have been downscaled to that ceiling regardless.
+ */
+export declare function estimateImageTokens(png: Buffer): number;
+//# sourceMappingURL=brain.d.ts.map

package/dist/actor/brain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain.d.ts","sourceRoot":"","sources":["../../src/actor/brain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAG5B,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,KAAK,EAAE,WAAW,EAAiC,MAAM,YAAY,CAAC;AAE7E,eAAO,MAAM,gBAAgB,UAAY,CAAC;AAC1C,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,gFAAgF;AAChF,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC;;;;;;;;GAQG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAC/C;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,eAAe,GAAG,OAAO,CAAC;AAEpE;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,EAAE,gBAA2B,CAAC;AAErE;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,aAAa,GAAG,IAAI,GAC5B,gBAAgB,CAOlB;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAmDD,qBAAa,eAAgB,SAAQ,KAAK;aAGtB,GAAG,EAAE,MAAM;gBAD3B,OAAO,EAAE,MAAM,EACC,GAAG,EAAE,MAAM;CAK9B;AAED,MAAM,WAAW,SAAS;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,KAAK,OAAO,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;IAC/C;;;;OAIG;IACH,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CACvC;AAED;;;GAGG;AACH,qBAAa,KAAK;IAqBd,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IArBvB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAKnB;IACR,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;IAC/C,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,mBAAmB,CAAK;gBAGb,OAAO,EAAE,aAAa,GAAG,IAAI,EAC7B,IAAI,GAAE,SAAc;IAKvC,8EAA8E;IAC9E,QAAQ,IAAI,UAAU;IAStB,OAAO,CAAC,QAAQ;IAShB;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAoBlB,OAAO,CAAC,cAAc;IAmBhB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IA4I7D;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,eAAe;YAKT,MAAM;CA4BrB;AAID,wBAAgB,cAAc,CAC5B,gBAAgB,EAAE,MAAM,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,GACd,MAAM,CAgCR;AAID,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CA6FzD;AAkBD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEjE;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMvD"}

package/dist/actor/cascade.d.ts

@@ -0,0 +1,92 @@
+/**
+ * WS7 — ScreenSeekeR cascade.
+ *
+ * Step 1: Brain looks at the full-screen scene for `target_display_id`.
+ * Step 2: For each ROI (up to BRAIN_MAX_ROIS), crop the *native resolution*
+ *         region from the captured PNG and hand it to the Actor for
+ *         fine grounding. The Actor returns display-local coords.
+ * Step 3: Combine the Brain's proposed action with the Actor's coordinates
+ *         (or fall back to the OCR/AX deterministic actor on `ref`) and
+ *         produce a single `ProposedAction` for the dispatcher.
+ *
+ * Cropping notes:
+ *   - We do NOT decode the PNG. The cropped buffer is what we hand to the
+ *     Actor. For the built-in OCR/AX actor, the crop is just a pass-through.
+ *   - When a real PNG cropper is wired in (sharp / native module), this
+ *     module is the place to add it: `cropPngToRoi(frame, bbox)`.
+ *   - The cascade tests use the actual frame bytes; assertions are on the
+ *     resolved coords and the order of Actor calls.
+ */
+import type { DisplayCapture } from "../platform/capture.js";
+import type { Scene } from "../scene/scene-types.js";
+import type { Actor } from "./actor.js";
+import { type Brain } from "./brain.js";
+import type { BrainRoi, CascadeResult } from "./types.js";
+export interface CascadeDeps {
+    brain: Brain;
+    actor?: Actor | null;
+    /** Cropper override (mostly tests). Returns a Buffer for the bbox region. */
+    crop?: (frame: Buffer, bbox: [number, number, number, number]) => Buffer;
+}
+export interface CascadeInput {
+    scene: Scene;
+    goal: string;
+    /** Per-display PNG captures, keyed by displayId. */
+    captures: Map<number, DisplayCapture>;
+}
+/** Grounding-cache accounting (#9105 M5). */
+export interface CascadeGroundStats {
+    /** Grounding resolutions served from the per-Scene cache. */
+    hits: number;
+    /** Grounding resolutions that ran the full resolve (OCR/AX + optional actor). */
+    misses: number;
+}
+export declare class Cascade {
+    private readonly deps;
+    /**
+     * Per-Scene grounding cache (predict/ground split, #9105 M5). Grounding the
+     * same target (ref or rationale) on the same Scene is deterministic, so the
+     * cheap GROUND step is memoized — re-grounding within a turn skips a repeat
+     * `resolveReference` (OCR/AX) scan and a repeat (possibly model-backed)
+     * `actor.ground` call. Keyed by Scene timestamp so a new screen invalidates.
+     */
+    private readonly groundCache;
+    private groundStats;
+    constructor(deps: CascadeDeps);
+    /** Grounding cache hit/miss snapshot for token/work accounting. */
+    getGroundStats(): CascadeGroundStats;
+    run(input: CascadeInput): Promise<CascadeResult>;
+    /**
+     * Grounding-only entry (the `predict_click` half of the predict/ground split,
+     * #9105 M5 / #9170 M10). Resolves a `ref` (OCR/AX id) or free-form
+     * `instruction` to a display-local coordinate WITHOUT running the Brain —
+     * agent loops that do their own step planning (Anthropic / OpenAI
+     * computer-use) call this to reuse our deterministic OCR/AX + actor grounding
+     * and its per-Scene cache. Returns `null` when nothing can be grounded.
+     */
+    groundTarget(args: {
+        scene: Scene;
+        captures: Map<number, DisplayCapture>;
+        targetDisplayId: number;
+        ref?: string;
+        instruction?: string;
+        /** Optional ROI to ground inside when no `ref` is available. */
+        roi?: BrainRoi;
+    }): Promise<{
+        displayId: number;
+        x: number;
+        y: number;
+    } | null>;
+    private resolveBrainOutput;
+    private resolveCoords;
+    /** Drop cache entries from any Scene other than `timestamp`, then store. */
+    private rememberGround;
+    private resolveCoordsUncached;
+    private coordsForRef;
+    private groundReference;
+    private groundRoi;
+    private cropFrame;
+}
+export declare function setActor(actor: Actor | null): void;
+export declare function getRegisteredActor(): Actor | null;
+//# sourceMappingURL=cascade.d.ts.map

package/dist/actor/cascade.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cascade.d.ts","sourceRoot":"","sources":["../../src/actor/cascade.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAmB,MAAM,YAAY,CAAC;AAEzD,OAAO,EAAkB,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,KAAK,EAEV,QAAQ,EACR,aAAa,EAEd,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACrB,6EAA6E;IAC7E,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;CAC1E;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CACvC;AAED,6CAA6C;AAC7C,MAAM,WAAW,kBAAkB;IACjC,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,OAAO;IAcN,OAAO,CAAC,QAAQ,CAAC,IAAI;IAbjC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAGxB;IACJ,OAAO,CAAC,WAAW,CAA8C;gBAEpC,IAAI,EAAE,WAAW;IAE9C,mEAAmE;IACnE,cAAc,IAAI,kBAAkB;IAI9B,GAAG,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC;IAStD;;;;;;;OAOG;IACG,YAAY,CAAC,IAAI,EAAE;QACvB,KAAK,EAAE,KAAK,CAAC;QACb,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACtC,eAAe,EAAE,MAAM,CAAC;QACxB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gEAAgE;QAChE,GAAG,CAAC,EAAE,QAAQ,CAAC;KAChB,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;YAsBjD,kBAAkB;YAwJlB,aAAa;IA2B3B,4EAA4E;IAC5E,OAAO,CAAC,cAAc;YAYR,qBAAqB;YAkCrB,YAAY;YAUZ,eAAe;YAiDf,SAAS;IAiCvB,OAAO,CAAC,SAAS;CAYlB;AAQD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,GAAG,IAAI,CAElD;AACD,wBAAgB,kBAAkB,IAAI,KAAK,GAAG,IAAI,CAEjD"}

package/dist/actor/computer-interface.d.ts

@@ -0,0 +1,276 @@
+/**
+ * WS7 — BaseComputerInterface (TypeScript port).
+ *
+ * Shape ported from trycua/cua's `BaseComputerInterface` (MIT). The original
+ * provides one standard API surface for screenshot + mouse + keyboard + screen
+ * geometry that the Brain/Actor cascade can call without knowing what OS or
+ * driver is underneath.
+ *
+ *   Origin: https://github.com/trycua/cua/blob/main/libs/python/computer/
+ *           computer/interface/base.py
+ *   License: MIT, Copyright (c) 2025 trycua
+ *
+ * This is NOT a verbatim re-implementation — Eliza's underlying drivers
+ * (`platform/driver.ts`, `platform/capture.ts`) already cover most of the same
+ * surface. We give the cascade one well-typed seam so tests can swap in a
+ * fake driver, and so the Brain doesn't need to know about display-id
+ * resolution, retina scaling, or coord-source switches.
+ *
+ * Coordinate contract (matches WS5):
+ *   - Every method that takes `(displayId, x, y)` uses LOCAL pixel coords for
+ *     that display. The interface routes those through `localToGlobalDefault`
+ *     before any input fires.
+ *   - VLMs producing coords in image-space (i.e. against the downscaled max-
+ *     pixels frame) MUST first call `toScreenCoordinates(...)` to get the
+ *     real OS-pixel coord. The inverse `toScreenshotCoordinates(...)` is for
+ *     logging / replay.
+ *   - `displayId` must be a known display from `listDisplays()`. Unknown ids
+ *     throw — this is the safety net the dispatch layer relies on.
+ */
+import { type DisplayCapture } from "../platform/capture.js";
+import type { Scene, SceneAxNode } from "../scene/scene-types.js";
+import type { DisplayDescriptor } from "../types.js";
+/** A POSIX-style mouse button. */
+export type MouseButton = "left" | "middle" | "right";
+export interface ScreenshotResult {
+    displayId: number;
+    /** PNG bytes at backing-store resolution. */
+    frame: Buffer;
+    /** Backing-store / logical-pixel ratio. */
+    scaleFactor: number;
+    /** [x, y, w, h] in OS-global pixel space, for the display this frame belongs to. */
+    bounds: [number, number, number, number];
+}
+export interface DisplayPoint {
+    displayId: number;
+    /** LOCAL pixel coords inside the display's logical bounds. */
+    x: number;
+    y: number;
+}
+export interface DragPath {
+    displayId: number;
+    path: Array<{
+        x: number;
+        y: number;
+    }>;
+}
+export interface ScrollDelta {
+    displayId: number;
+    x: number;
+    y: number;
+    /** Negative = scroll up/left, positive = scroll down/right. */
+    dx: number;
+    dy: number;
+}
+export interface CursorPosition {
+    displayId: number;
+    x: number;
+    y: number;
+}
+/**
+ * The single seam the cascade calls. All methods are safe to await
+ * concurrently within one display (driver semantics permitting); callers
+ * serialize at the dispatch layer.
+ */
+export interface ComputerInterface {
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(path: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    /**
+     * Convert from VLM-image-space (the downscaled max-pixels frame the model
+     * was shown) into OS-LOCAL-pixel-space for the same display.
+     *
+     * `(imgX, imgY)` are pixel coords inside an `imgW × imgH` image. Returned
+     * `(x, y)` are the same physical point but in the display's logical bounds.
+     */
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    /** Inverse of `toScreenCoordinates`. */
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    /**
+     * Return the AX (accessibility) tree the scene-builder snapshot already
+     * collected. We don't re-snapshot here — that's WS6's job. Cascade calls
+     * this to enumerate clickable AX nodes by display.
+     */
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+}
+/**
+ * Reference implementation that delegates to `platform/driver.ts` and
+ * `platform/capture.ts`. The constructor takes a `Scene` accessor so the
+ * AX tree is read from the latest WS6 snapshot without re-walking the OS.
+ */
+export interface ComputerInterfaceDeps {
+    /** Latest scene accessor — used for `getAccessibilityTree`. */
+    getScene?: () => Scene | null;
+    /** Capture override (mostly for tests). */
+    capture?: (displayId: number) => Promise<DisplayCapture>;
+    /** Driver overrides for tests. */
+    driver?: Partial<{
+        click: (x: number, y: number) => Promise<void>;
+        doubleClick: (x: number, y: number) => Promise<void>;
+        rightClick: (x: number, y: number) => Promise<void>;
+        mouseMove: (x: number, y: number) => Promise<void>;
+        mouseDown: (x: number, y: number, button: MouseButton) => Promise<void>;
+        mouseUp: (x: number, y: number, button: MouseButton) => Promise<void>;
+        drag: (x1: number, y1: number, x2: number, y2: number) => Promise<void>;
+        dragPath: (path: Array<{
+            x: number;
+            y: number;
+        }>) => Promise<void>;
+        scroll: (x: number, y: number, direction: "up" | "down" | "left" | "right", amount: number) => Promise<void>;
+        type: (text: string) => Promise<void>;
+        keyPress: (key: string) => Promise<void>;
+        keyCombo: (combo: string) => Promise<void>;
+        keyDown: (key: string) => Promise<void>;
+        keyUp: (key: string) => Promise<void>;
+    }>;
+    listDisplays?: () => DisplayDescriptor[];
+    /**
+     * Where the implementation should remember the last cursor target.
+     * Set whenever a movement-bearing call resolves successfully.
+     */
+    cursorState?: {
+        current: CursorPosition;
+    };
+}
+export declare class DefaultComputerInterface implements ComputerInterface {
+    private readonly deps;
+    constructor(deps?: ComputerInterfaceDeps);
+    screenshot(opts?: {
+        displayId?: number;
+    }): Promise<ScreenshotResult>;
+    mouseDown(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    mouseUp(point: DisplayPoint & {
+        button?: MouseButton;
+    }): Promise<void>;
+    leftClick(point: DisplayPoint): Promise<void>;
+    rightClick(point: DisplayPoint): Promise<void>;
+    doubleClick(point: DisplayPoint): Promise<void>;
+    moveCursor(point: DisplayPoint): Promise<void>;
+    dragTo(point: DisplayPoint): Promise<void>;
+    drag(args: DragPath): Promise<void>;
+    keyDown(args: {
+        key: string;
+    }): Promise<void>;
+    keyUp(args: {
+        key: string;
+    }): Promise<void>;
+    typeText(args: {
+        text: string;
+    }): Promise<void>;
+    pressKey(args: {
+        key: string;
+    }): Promise<void>;
+    hotkey(args: {
+        keys: string[];
+    }): Promise<void>;
+    scroll(delta: ScrollDelta): Promise<void>;
+    scrollUp(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    scrollDown(args: {
+        displayId: number;
+        clicks: number;
+    }): Promise<void>;
+    getScreenSize(args: {
+        displayId: number;
+    }): {
+        w: number;
+        h: number;
+    };
+    getCursorPosition(): CursorPosition;
+    toScreenCoordinates(args: {
+        displayId: number;
+        imgX: number;
+        imgY: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        x: number;
+        y: number;
+    };
+    toScreenshotCoordinates(args: {
+        displayId: number;
+        x: number;
+        y: number;
+        imgW: number;
+        imgH: number;
+    }): {
+        imgX: number;
+        imgY: number;
+    };
+    getAccessibilityTree(args: {
+        displayId?: number;
+    }): SceneAxNode[];
+    private primaryId;
+    private requireDisplay;
+    private toGlobalChecked;
+}
+/** Convenience factory used by the cascade. */
+export declare function makeComputerInterface(deps?: ComputerInterfaceDeps): ComputerInterface;
+//# sourceMappingURL=computer-interface.d.ts.map