@elizaos/plugin-computeruse 2.0.0-beta.1 → 2.0.3-beta.3

package/dist/actor/agent-loop.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Agent-loop registry (#9170 M10).
+ *
+ * trycua/cua selects an agent *loop* from a model string: an `anthropic/...`
+ * model routes to the Claude computer-use loop, `openai/computer-use-preview`
+ * routes to the OpenAI operator loop, an OmniParser/grounder string routes to a
+ * local set-of-marks loop, etc. Each loop implements the same two-call seam —
+ * `predict_step` (observe + plan the next action) and `predict_click` (ground a
+ * target to a coordinate) — so the runner is decoupled from *how* a step is
+ * produced.
+ *
+ * elizaOS shipped a single hardcoded Brain→Cascade (ScreenSeekeR). This module
+ * replaces that hardcoding with a registry:
+ *   - `AgentLoop` — the `predictStep` / `predictClick` seam.
+ *   - `registerAgentLoop` — register a loop keyed by a model-string matcher.
+ *   - `createAgentLoop(modelString, deps)` — pick the highest-priority matching
+ *     loop and instantiate it.
+ *
+ * The built-in `local-grounder` loop wraps the existing Brain→Cascade and
+ * exposes the M5 grounding cache through `predictClick`. Anthropic / OpenAI
+ * computer-use loops are *pluggable*: a provider plugin calls
+ * `registerAgentLoop` with `matchesModelFamily("anthropic")` (etc.) and its own
+ * `predictStep`. With none registered, every model string falls through to the
+ * local grounder (which always matches at the lowest priority).
+ */
+import type { IAgentRuntime } from "@elizaos/core";
+import type { DisplayCapture } from "../platform/capture.js";
+import type { Scene } from "../scene/scene-types.js";
+import { type Actor } from "./actor.js";
+import { Brain } from "./brain.js";
+import type { CascadeResult, GroundingResult } from "./types.js";
+/** Default loop model-string — the local OCR/AX + actor grounder. */
+export declare const DEFAULT_AGENT_LOOP_MODEL = "local-grounder";
+/** Setting / env key the runner reads to choose a loop. */
+export declare const AGENT_LOOP_SETTING = "COMPUTER_USE_AGENT_LOOP";
+export interface AgentStepInput {
+    scene: Scene;
+    goal: string;
+    captures: Map<number, DisplayCapture>;
+}
+export interface PredictClickInput {
+    scene: Scene;
+    captures: Map<number, DisplayCapture>;
+    targetDisplayId: number;
+    /** OCR/AX id to ground (`t<d>-<n>` / `a<d>-<n>`). */
+    ref?: string;
+    /** Free-form instruction when no ref is available. */
+    instruction?: string;
+}
+/**
+ * The two-call seam every loop implements. `predictStep` plans the next
+ * concrete action; `predictClick` grounds a target to a coordinate (used by
+ * loops that plan elsewhere but reuse our grounding, and by callers that want
+ * grounding without a full step).
+ */
+/**
+ * Per-run model-call accounting (#9105). `invocations` counts the token-bearing
+ * model calls a loop actually issued; `cacheHits` counts calls served without a
+ * model round-trip. Reported once per run as `evt:"computeruse.agent.tokens"`.
+ */
+export interface AgentLoopStats {
+    /** Token-bearing model calls actually issued during the run. */
+    invocations: number;
+    /** Calls served from cache (no model call, no tokens). */
+    cacheHits: number;
+    /** Model calls issued with no screenshot attached (#9105). */
+    imagelessCalls: number;
+    /** Estimated image tokens not sent because of imageless calls (#9105). */
+    estImageTokensSaved: number;
+}
+export interface AgentLoop {
+    readonly name: string;
+    predictStep(input: AgentStepInput): Promise<CascadeResult>;
+    predictClick(input: PredictClickInput): Promise<GroundingResult | null>;
+    /** Per-run model-call accounting, when the loop tracks it (#9105). */
+    getStats?(): AgentLoopStats;
+}
+export interface AgentLoopDeps {
+    runtime: IAgentRuntime | null;
+    /** Latest-scene accessor for the default actor. */
+    getScene: () => Scene | null;
+    /** Brain override (mostly tests). */
+    brain?: Brain;
+    /** Actor override (mostly tests). */
+    actor?: Actor | null;
+}
+export interface AgentLoopRegistration {
+    /** Stable id for telemetry + explicit selection. */
+    readonly name: string;
+    /** True when this loop handles `modelString`. */
+    matches: (modelString: string) => boolean;
+    /** Instantiate the loop for a run. */
+    create: (deps: AgentLoopDeps) => AgentLoop;
+    /** Higher wins when multiple registrations match. Default 0. */
+    priority?: number;
+}
+/**
+ * Wraps the existing ScreenSeekeR (Brain → Cascade). `predictStep` is the full
+ * observe→plan→ground cascade; `predictClick` calls the cascade's grounding-only
+ * path so the M5 per-Scene grounding cache is shared across both.
+ */
+export declare class LocalGrounderLoop implements AgentLoop {
+    readonly name = "local-grounder";
+    private readonly cascade;
+    private readonly brain;
+    constructor(deps: AgentLoopDeps);
+    predictStep(input: AgentStepInput): Promise<CascadeResult>;
+    predictClick(input: PredictClickInput): Promise<GroundingResult | null>;
+    /** Grounding cache hit/miss snapshot (delegates to the wrapped cascade). */
+    getGroundStats(): import("./cascade.js").CascadeGroundStats;
+    /** Model-call accounting from the wrapped Brain (#9105). */
+    getStats(): AgentLoopStats;
+}
+/**
+ * A matcher for a provider family — `anthropic`, `openai`, `google`, … A
+ * pluggable loop registers with `matches: matchesModelFamily("anthropic")` so a
+ * model string like `anthropic/claude-...` or `claude-3-7-sonnet` routes to it.
+ */
+export declare function matchesModelFamily(family: string): (modelString: string) => boolean;
+/** Register (or replace, by name) an agent-loop. */
+export declare function registerAgentLoop(registration: AgentLoopRegistration): void;
+export declare function unregisterAgentLoop(name: string): void;
+export declare function listAgentLoops(): readonly AgentLoopRegistration[];
+/**
+ * Pick the registration for a model string: the highest-priority one whose
+ * `matches` returns true. The local grounder's match-anything floor guarantees
+ * a result, so this never throws.
+ */
+export declare function selectAgentLoopRegistration(modelString: string): AgentLoopRegistration;
+/** Resolve + instantiate the loop for a model string. */
+export declare function createAgentLoop(modelString: string, deps: AgentLoopDeps): AgentLoop;
+/** Test helper — restore the registry to just the built-in local grounder. */
+export declare function _resetAgentLoopsForTests(): void;
+//# sourceMappingURL=agent-loop.d.ts.map

package/dist/actor/agent-loop.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-loop.d.ts","sourceRoot":"","sources":["../../src/actor/agent-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,KAAK,KAAK,EAA+B,MAAM,YAAY,CAAC;AACrE,OAAO,EAAE,KAAK,EAA2B,MAAM,YAAY,CAAC;AAE5D,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEjE,qEAAqE;AACrE,eAAO,MAAM,wBAAwB,mBAAmB,CAAC;AAEzD,2DAA2D;AAC3D,eAAO,MAAM,kBAAkB,4BAA4B,CAAC;AAE5D,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACtC,eAAe,EAAE,MAAM,CAAC;IACxB,qDAAqD;IACrD,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;GAKG;AACH;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,SAAS,EAAE,MAAM,CAAC;IAClB,8DAA8D;IAC9D,cAAc,EAAE,MAAM,CAAC;IACvB,0EAA0E;IAC1E,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC3D,YAAY,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IACxE,sEAAsE;IACtE,QAAQ,CAAC,IAAI,cAAc,CAAC;CAC7B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,aAAa,GAAG,IAAI,CAAC;IAC9B,mDAAmD;IACnD,QAAQ,EAAE,MAAM,KAAK,GAAG,IAAI,CAAC;IAC7B,qCAAqC;IACrC,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,qCAAqC;IACrC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iDAAiD;IACjD,OAAO,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAAC;IAC1C,sCAAsC;IACtC,MAAM,EAAE,CAAC,IAAI,EAAE,aAAa,KAAK,SAAS,CAAC;IAC3C,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAID;;;;GAIG;AACH,qBAAa,iBAAkB,YAAW,SAAS;IACjD,QAAQ,CAAC,IAAI,oBAA4B;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;gBAElB,IAAI,EAAE,aAAa;IAc/B,WAAW,CAAC,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,aAAa,CAAC;IAIpD,YAAY,CAChB,KAAK,EAAE,iBAAiB,GACvB,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAkBlC,4EAA4E;IAC5E,cAAc;IAId,4DAA4D;IAC5D,QAAQ,IAAI,cAAc;CAG3B;AAID;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,GACb,CAAC,WAAW,EAAE,MAAM,KAAK,OAAO,CAYlC;AAMD,oDAAoD;AACpD,wBAAgB,iBAAiB,CAAC,YAAY,EAAE,qBAAqB,GAAG,IAAI,CAE3E;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAEtD;AAED,wBAAgB,cAAc,IAAI,SAAS,qBAAqB,EAAE,CAIjE;AA0BD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,WAAW,EAAE,MAAM,GAClB,qBAAqB,CAevB;AAED,yDAAyD;AACzD,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,IAAI,EAAE,aAAa,GAClB,SAAS,CAEX;AAED,8EAA8E;AAC9E,wBAAgB,wBAAwB,IAAI,IAAI,CAG/C"}

package/dist/actor/aosp-input-actor.d.ts

@@ -0,0 +1,87 @@
+/**
+ * WS7 ↔ AOSP — Privileged-input actor.
+ *
+ * In the consumer build the cascade routes gestures through
+ * `MobileComputerInterface` → `AccessibilityGestureDescription` — which is
+ * coarse and blocks on touch-recognizer state in some apps (banking, DRM
+ * video, anything that sets `filterTouchesWhenObscured`).
+ *
+ * In an AOSP system-app build (see `docs/AOSP_SYSTEM_APP.md`), the
+ * privileged path uses `InputManager.injectInputEvent()` directly. That
+ * path lives behind `AospPrivilegedBridge.injectMotionEvent(...)` on the
+ * Kotlin side; the consumer-flavor `AospPrivilegedBridge` exports
+ * `createIfAvailable(): null` so this actor stays inert until a real AOSP
+ * bridge is linked in.
+ *
+ * `AospInputActor` maps a resolved WS7 `ProposedAction` (display-local
+ * pixel coords) into the privileged-bridge calls. It does NOT implement
+ * the `Actor` "grounding" contract — grounding stays with the OCR/AX or
+ * VLM actor; this is purely an *input-dispatch* shim. It's surfaced as
+ * an alternative to `ComputerInterface` for AOSP builds: the agent loop
+ * picks `AospInputActor.execute(action)` instead of `dispatch(action, {
+ * interface, ... })` when the privileged bridge is available.
+ */
+import type { ActionResult, ProposedAction } from "./types.js";
+/** Minimal Kotlin-side surface this actor needs from the AOSP build. */
+export interface AospPrivilegedInputBridge {
+    /**
+     * Inject a single motion event at the InputManager level. `action` follows
+     * `MotionEvent.ACTION_*` constants (DOWN=0, UP=1, MOVE=2). `downTimeMs`
+     * is the original-touch timestamp the gesture started at, in `uptimeMillis`
+     * units. Implementations enforce the INJECT_EVENTS permission.
+     */
+    injectMotionEvent(args: {
+        x: number;
+        y: number;
+        action: number;
+        downTimeMs: number;
+    }): Promise<{
+        ok: boolean;
+    }>;
+    /** Capture the primary display frame buffer synchronously. JPEG bytes. */
+    captureDisplayFrameBuffer?(): Promise<Uint8Array>;
+}
+export interface AospInputActorDeps {
+    /** Returns the AOSP bridge handle, or null in consumer builds. */
+    getBridge: () => AospPrivilegedInputBridge | null;
+    /** Override the clock for tests. */
+    now?: () => number;
+}
+/**
+ * Motion-event action constants matching `android.view.MotionEvent.ACTION_*`.
+ * Re-exported here so callers don't need to import Android Kotlin enums.
+ */
+export declare const MOTION_EVENT_ACTION_DOWN: 0;
+export declare const MOTION_EVENT_ACTION_UP: 1;
+export declare const MOTION_EVENT_ACTION_MOVE: 2;
+/**
+ * Translate a cascade-resolved `ProposedAction` into one or more
+ * `injectMotionEvent` calls. Returns the same `ActionResult` envelope the
+ * desktop dispatcher uses — invalid args or driver errors do not throw.
+ *
+ * Behavior parity with `dispatch.ts`:
+ *   - unknown action.kind → invalid_args
+ *   - missing coords      → invalid_args
+ *   - bridge throw        → driver_error
+ *   - bridge ok:false     → driver_error
+ *
+ * Coverage:
+ *   - click / double_click / right_click → tap(s)
+ *   - drag                              → DOWN at start, MOVE/UP at end
+ *   - scroll                            → swipe (DOWN, MOVE, UP)
+ *   - wait / finish                     → success: true (no input event)
+ *   - type / key / hotkey               → invalid_args (use AccessibilityNodeInfo
+ *                                         or a separate keymap actor; out of
+ *                                         scope for this privileged path).
+ */
+export declare class AospInputActor {
+    private readonly deps;
+    readonly name = "aosp-input";
+    constructor(deps: AospInputActorDeps);
+    execute(action: ProposedAction): Promise<ActionResult>;
+    private tap;
+    private swipe;
+    private must;
+    private now;
+}
+//# sourceMappingURL=aosp-input-actor.d.ts.map

package/dist/actor/aosp-input-actor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aosp-input-actor.d.ts","sourceRoot":"","sources":["../../src/actor/aosp-input-actor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAK/D,wEAAwE;AACxE,MAAM,WAAW,yBAAyB;IACxC;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE;QACtB,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC7B,0EAA0E;IAC1E,yBAAyB,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,kBAAkB;IACjC,kEAAkE;IAClE,SAAS,EAAE,MAAM,yBAAyB,GAAG,IAAI,CAAC;IAClD,oCAAoC;IACpC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAG,CAAU,CAAC;AACnD,eAAO,MAAM,sBAAsB,EAAG,CAAU,CAAC;AACjD,eAAO,MAAM,wBAAwB,EAAG,CAAU,CAAC;AAEnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,cAAc;IAGb,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,QAAQ,CAAC,IAAI,gBAAgB;gBAEA,IAAI,EAAE,kBAAkB;IAE/C,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC;YAwH9C,GAAG;YA0BH,KAAK;YAsCL,IAAI;IAYlB,OAAO,CAAC,GAAG;CAGZ"}

package/dist/actor/brain.d.ts

@@ -0,0 +1,195 @@
+/**
+ * WS7 — Brain (full-screen reasoning).
+ *
+ * Sends one image per display (each downscaled to ~1.3 MP, the OS-Atlas /
+ * Qwen3-VL `max_pixels` convention) to `runtime.useModel(IMAGE_DESCRIPTION,
+ * ...)`. The model is prompted to emit a JSON `BrainOutput` describing:
+ *   - the scene in one paragraph,
+ *   - which display to act on,
+ *   - up to N ROIs the Actor should zoom into,
+ *   - a single proposed action with rationale.
+ *
+ * The Brain itself doesn't dispatch — it just produces `BrainOutput`. The
+ * cascade ("ScreenSeekeR") is the orchestrator that takes a `BrainOutput`,
+ * optionally calls the Actor on cropped ROIs, and produces a concrete
+ * `ProposedAction` for the dispatcher.
+ *
+ * Image transport contract: we pass `imageUrl` as a `data:image/png;base64,...`
+ * URL. The WS2 MemoryArbiter intercepts at `ModelType.IMAGE_DESCRIPTION` and
+ * routes through its content-hash cache, so identical frames don't burn
+ * inference budget twice.
+ *
+ * Image policy (#9105): the compact scene already carries OCR text + AX boxes,
+ * which can suffice to pick the next target, so the `"on-escalation"` policy
+ * plans from that text-only context with NO image — routed through a TEXT model,
+ * since every IMAGE_DESCRIPTION provider rejects an empty imageUrl — and
+ * attaches the ~1.3 MP frame only when the planned target cannot be grounded
+ * against the OCR/AX boxes. The DEFAULT is `"always"` (legacy: image on every
+ * call) until a real-model CUA trajectory validates imageless planning accuracy;
+ * operators opt into `"on-escalation"` via the `COMPUTERUSE_BRAIN_IMAGE_POLICY`
+ * setting. `"never"` never attaches pixels.
+ *
+ * Parse strictness:
+ *   - We try to parse the response as JSON (either the literal string or
+ *     `result.description`).
+ *   - On first parse failure, retry once with a stricter prompt.
+ *   - On second failure, throw `BrainParseError` — the cascade surfaces this
+ *     as a structured `ActionResult.error` and aborts the turn cleanly.
+ */
+import { type IAgentRuntime, type ImageDescriptionResult } from "@elizaos/core";
+import type { DisplayCapture } from "../platform/capture.js";
+import type { Scene } from "../scene/scene-types.js";
+import type { BrainOutput } from "./types.js";
+export declare const BRAIN_MAX_PIXELS = 1310720;
+export declare const BRAIN_MAX_ROIS = 2;
+/** Bound on the per-Brain dHash→BrainOutput cache (LRU-ish, oldest evicted). */
+export declare const BRAIN_DHASH_CACHE_MAX = 16;
+/**
+ * dHash Hamming threshold for cached-plan reuse (#9581 continuous-understanding
+ * tuning). Exact-equality (distance 0) re-burned the IMAGE_DESCRIPTION model on
+ * cosmetically-identical frames — cursor jitter, a blinking caret, anti-aliasing,
+ * and tiny scroll noise all flip a few dHash bits.
+ *
+ * This mirrors `SCREEN_STATE_HAMMING_THRESHOLD`: distances below the threshold
+ * are unchanged; distances at or above it are changed and must re-plan.
+ */
+export declare const BRAIN_DHASH_HAMMING_THRESHOLD = 5;
+/**
+ * Image-token estimate per source pixel for a vision model with the Qwen3-VL /
+ * OS-Atlas `max_pixels` convention: one visual token ≈ a 28×28 (≈750 px) patch.
+ * Used only to quantify the saving when a frame is *not* attached (#9105).
+ */
+export declare const BRAIN_PIXELS_PER_IMAGE_TOKEN = 750;
+/**
+ * When to attach the raw screenshot to the planning model (#9105):
+ *   - `"always"`     — attach the pixels on every call (legacy behaviour). Default.
+ *   - `"on-escalation"` — plan from the compact OCR/AX scene with no image
+ *                         first (routed through a TEXT model); attach pixels
+ *                         only when the planned target cannot be grounded
+ *                         against the OCR/AX boxes or a strict-retry fires.
+ *   - `"never"`      — never attach the screenshot; plan from the scene alone.
+ */
+export type BrainImagePolicy = "always" | "on-escalation" | "never";
+/**
+ * Default is `"always"` (proven legacy behaviour). `"on-escalation"` cuts the
+ * dominant per-frame image-token cost but plans the first pass blind to the
+ * pixels, so it stays opt-in (via the `COMPUTERUSE_BRAIN_IMAGE_POLICY` runtime
+ * setting / env var) until a real-model CUA trajectory validates its accuracy.
+ */
+export declare const DEFAULT_BRAIN_IMAGE_POLICY: BrainImagePolicy;
+/**
+ * Resolve the Brain image policy from the `COMPUTERUSE_BRAIN_IMAGE_POLICY`
+ * runtime setting / env var, falling back to {@link DEFAULT_BRAIN_IMAGE_POLICY}.
+ * The operator escape hatch to enable imageless planning without a code change
+ * once it is validated against a real model.
+ */
+export declare function resolveBrainImagePolicy(runtime: IAgentRuntime | null): BrainImagePolicy;
+/** Token-accounting snapshot for a Brain instance (#9105 M3). */
+export interface BrainStats {
+    /** IMAGE_DESCRIPTION model calls actually issued. */
+    invocations: number;
+    /** Describe calls served from the frame-dHash cache (no model call). */
+    cacheHits: number;
+    /**
+     * Model calls issued with NO screenshot attached (the scene's OCR/AX text
+     * sufficed). Each one saved roughly one full-frame image's worth of tokens.
+     */
+    imagelessCalls: number;
+    /**
+     * Estimated image tokens NOT sent because of imageless calls — the sum of
+     * `(width * height) / BRAIN_PIXELS_PER_IMAGE_TOKEN` over every imageless
+     * call, capped per frame at `BRAIN_MAX_PIXELS`.
+     */
+    estImageTokensSaved: number;
+}
+export declare class BrainParseError extends Error {
+    readonly raw: string;
+    constructor(message: string, raw: string);
+}
+export interface BrainDeps {
+    /** Optional override for tests — bypasses runtime.useModel. */
+    invokeModel?: (args: {
+        imageUrl: string;
+        prompt: string;
+        displayId: number;
+    }) => Promise<string | ImageDescriptionResult>;
+    /**
+     * When to attach the raw screenshot to the planning model (#9105). Defaults
+     * to `"always"` (legacy); see {@link resolveBrainImagePolicy} for the
+     * operator opt-in to `"on-escalation"`.
+     */
+    imagePolicy?: BrainImagePolicy;
+}
+export interface BrainInput {
+    scene: Scene;
+    goal: string;
+    /**
+     * Per-display capture buffers. If a display from `scene.displays` is
+     * missing here, the Brain skips it. The cascade is responsible for
+     * supplying these alongside the scene.
+     */
+    captures: Map<number, DisplayCapture>;
+}
+/**
+ * Pure description of a "Brain" call. Created by `Cascade.runCascade` and
+ * test fixtures.
+ */
+export declare class Brain {
+    private readonly runtime;
+    private readonly deps;
+    /**
+     * Frame-dHash → BrainOutput cache. The WS2 MemoryArbiter only dedups the
+     * IMAGE_DESCRIPTION call for LOCAL backends; the remote/cloud path bypasses
+     * it, so an identical screen re-burns tokens every step. This call-site cache
+     * skips the model entirely when the same frame is observed for the same goal,
+     * cutting the dominant CUA-loop token cost regardless of backend (#9105 M3).
+     */
+    private readonly dhashCache;
+    private readonly imagePolicy;
+    private invocations;
+    private cacheHits;
+    private imagelessCalls;
+    private estImageTokensSaved;
+    constructor(runtime: IAgentRuntime | null, deps?: BrainDeps);
+    /** Token-accounting snapshot (model calls, cache hits, imageless savings). */
+    getStats(): BrainStats;
+    private cacheKey;
+    /**
+     * Return a cached plan for the same goal whose frame is within
+     * `BRAIN_DHASH_HAMMING_THRESHOLD` bits of `dh` — a near-identical screen, not
+     * just a byte-identical one. On a hit the entry is moved to the end (LRU), so
+     * a steadily-evolving screen keeps its most-recent close match warm.
+     */
+    private findCached;
+    private rememberOutput;
+    observeAndPlan(input: BrainInput): Promise<BrainOutput>;
+    /**
+     * True when the imageless plan needs no screenshot to dispatch: a
+     * non-coordinate action (type/hotkey/key/wait/finish) carries everything in
+     * its args, and a coordinate action is fine when its `ref`/rationale resolves
+     * to a concrete OCR/AX box. When the target cannot be grounded we escalate to
+     * the pixels — correctness over token saving.
+     */
+    private resolvesWithoutImage;
+    private recordImageless;
+    private invoke;
+}
+export declare function brainPromptFor(compactSceneJson: string, goal: string, strict: boolean): string;
+export declare function parseBrainOutput(raw: string): BrainOutput;
+/**
+ * Encode a PNG buffer for transport to the IMAGE_DESCRIPTION model. We don't
+ * resize here — `runtime.useModel` adapters (and any vLLM backends behind
+ * them) handle the `max_pixels` downscale. The constant `BRAIN_MAX_PIXELS`
+ * is exported for the cascade so it can crop ROIs at the right native
+ * resolution before invoking the Actor.
+ */
+export declare function encodeForBrain(png: Buffer): Promise<string>;
+/**
+ * Estimate the visual tokens a frame would have cost the planning model, for
+ * the "tokens saved by going imageless" telemetry (#9105). The backends apply a
+ * `max_pixels` downscale, so the per-frame estimate is capped at
+ * `BRAIN_MAX_PIXELS`. Falls back to the cap when the PNG header is unreadable —
+ * an attached frame would have been downscaled to that ceiling regardless.
+ */
+export declare function estimateImageTokens(png: Buffer): number;
+//# sourceMappingURL=brain.d.ts.map

package/dist/actor/brain.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brain.d.ts","sourceRoot":"","sources":["../../src/actor/brain.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,sBAAsB,EAG5B,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,KAAK,EAAE,WAAW,EAAiC,MAAM,YAAY,CAAC;AAE7E,eAAO,MAAM,gBAAgB,UAAY,CAAC;AAC1C,eAAO,MAAM,cAAc,IAAI,CAAC;AAChC,gFAAgF;AAChF,eAAO,MAAM,qBAAqB,KAAK,CAAC;AACxC;;;;;;;;GAQG;AACH,eAAO,MAAM,6BAA6B,IAAI,CAAC;AAC/C;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,MAAM,gBAAgB,GAAG,QAAQ,GAAG,eAAe,GAAG,OAAO,CAAC;AAEpE;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,EAAE,gBAA2B,CAAC;AAErE;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,aAAa,GAAG,IAAI,GAC5B,gBAAgB,CAOlB;AAED,iEAAiE;AACjE,MAAM,WAAW,UAAU;IACzB,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAmDD,qBAAa,eAAgB,SAAQ,KAAK;aAGtB,GAAG,EAAE,MAAM;gBAD3B,OAAO,EAAE,MAAM,EACC,GAAG,EAAE,MAAM;CAK9B;AAED,MAAM,WAAW,SAAS;IACxB,+DAA+D;IAC/D,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;KACnB,KAAK,OAAO,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC;IAC/C;;;;OAIG;IACH,WAAW,CAAC,EAAE,gBAAgB,CAAC;CAChC;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb;;;;OAIG;IACH,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CACvC;AAED;;;GAGG;AACH,qBAAa,KAAK;IAqBd,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;IArBvB;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,UAAU,CAKnB;IACR,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmB;IAC/C,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,mBAAmB,CAAK;gBAGb,OAAO,EAAE,aAAa,GAAG,IAAI,EAC7B,IAAI,GAAE,SAAc;IAKvC,8EAA8E;IAC9E,QAAQ,IAAI,UAAU;IAStB,OAAO,CAAC,QAAQ;IAShB;;;;;OAKG;IACH,OAAO,CAAC,UAAU;IAoBlB,OAAO,CAAC,cAAc;IAmBhB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IA4I7D;;;;;;OAMG;IACH,OAAO,CAAC,oBAAoB;IAa5B,OAAO,CAAC,eAAe;YAKT,MAAM;CA4BrB;AAID,wBAAgB,cAAc,CAC5B,gBAAgB,EAAE,MAAM,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,OAAO,GACd,MAAM,CAgCR;AAID,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,WAAW,CA6FzD;AAkBD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAEjE;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMvD"}

package/dist/actor/cascade.d.ts

@@ -0,0 +1,92 @@
+/**
+ * WS7 — ScreenSeekeR cascade.
+ *
+ * Step 1: Brain looks at the full-screen scene for `target_display_id`.
+ * Step 2: For each ROI (up to BRAIN_MAX_ROIS), crop the *native resolution*
+ *         region from the captured PNG and hand it to the Actor for
+ *         fine grounding. The Actor returns display-local coords.
+ * Step 3: Combine the Brain's proposed action with the Actor's coordinates
+ *         (or fall back to the OCR/AX deterministic actor on `ref`) and
+ *         produce a single `ProposedAction` for the dispatcher.
+ *
+ * Cropping notes:
+ *   - We do NOT decode the PNG. The cropped buffer is what we hand to the
+ *     Actor. For the built-in OCR/AX actor, the crop is just a pass-through.
+ *   - When a real PNG cropper is wired in (sharp / native module), this
+ *     module is the place to add it: `cropPngToRoi(frame, bbox)`.
+ *   - The cascade tests use the actual frame bytes; assertions are on the
+ *     resolved coords and the order of Actor calls.
+ */
+import type { DisplayCapture } from "../platform/capture.js";
+import type { Scene } from "../scene/scene-types.js";
+import type { Actor } from "./actor.js";
+import { type Brain } from "./brain.js";
+import type { BrainRoi, CascadeResult } from "./types.js";
+export interface CascadeDeps {
+    brain: Brain;
+    actor?: Actor | null;
+    /** Cropper override (mostly tests). Returns a Buffer for the bbox region. */
+    crop?: (frame: Buffer, bbox: [number, number, number, number]) => Buffer;
+}
+export interface CascadeInput {
+    scene: Scene;
+    goal: string;
+    /** Per-display PNG captures, keyed by displayId. */
+    captures: Map<number, DisplayCapture>;
+}
+/** Grounding-cache accounting (#9105 M5). */
+export interface CascadeGroundStats {
+    /** Grounding resolutions served from the per-Scene cache. */
+    hits: number;
+    /** Grounding resolutions that ran the full resolve (OCR/AX + optional actor). */
+    misses: number;
+}
+export declare class Cascade {
+    private readonly deps;
+    /**
+     * Per-Scene grounding cache (predict/ground split, #9105 M5). Grounding the
+     * same target (ref or rationale) on the same Scene is deterministic, so the
+     * cheap GROUND step is memoized — re-grounding within a turn skips a repeat
+     * `resolveReference` (OCR/AX) scan and a repeat (possibly model-backed)
+     * `actor.ground` call. Keyed by Scene timestamp so a new screen invalidates.
+     */
+    private readonly groundCache;
+    private groundStats;
+    constructor(deps: CascadeDeps);
+    /** Grounding cache hit/miss snapshot for token/work accounting. */
+    getGroundStats(): CascadeGroundStats;
+    run(input: CascadeInput): Promise<CascadeResult>;
+    /**
+     * Grounding-only entry (the `predict_click` half of the predict/ground split,
+     * #9105 M5 / #9170 M10). Resolves a `ref` (OCR/AX id) or free-form
+     * `instruction` to a display-local coordinate WITHOUT running the Brain —
+     * agent loops that do their own step planning (Anthropic / OpenAI
+     * computer-use) call this to reuse our deterministic OCR/AX + actor grounding
+     * and its per-Scene cache. Returns `null` when nothing can be grounded.
+     */
+    groundTarget(args: {
+        scene: Scene;
+        captures: Map<number, DisplayCapture>;
+        targetDisplayId: number;
+        ref?: string;
+        instruction?: string;
+        /** Optional ROI to ground inside when no `ref` is available. */
+        roi?: BrainRoi;
+    }): Promise<{
+        displayId: number;
+        x: number;
+        y: number;
+    } | null>;
+    private resolveBrainOutput;
+    private resolveCoords;
+    /** Drop cache entries from any Scene other than `timestamp`, then store. */
+    private rememberGround;
+    private resolveCoordsUncached;
+    private coordsForRef;
+    private groundReference;
+    private groundRoi;
+    private cropFrame;
+}
+export declare function setActor(actor: Actor | null): void;
+export declare function getRegisteredActor(): Actor | null;
+//# sourceMappingURL=cascade.d.ts.map

package/dist/actor/cascade.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cascade.d.ts","sourceRoot":"","sources":["../../src/actor/cascade.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAmB,MAAM,YAAY,CAAC;AAEzD,OAAO,EAAkB,KAAK,KAAK,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,KAAK,EAEV,QAAQ,EACR,aAAa,EAEd,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC;IACrB,6EAA6E;IAC7E,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,MAAM,CAAC;CAC1E;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,KAAK,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,oDAAoD;IACpD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;CACvC;AAED,6CAA6C;AAC7C,MAAM,WAAW,kBAAkB;IACjC,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;IACb,iFAAiF;IACjF,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,OAAO;IAcN,OAAO,CAAC,QAAQ,CAAC,IAAI;IAbjC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAGxB;IACJ,OAAO,CAAC,WAAW,CAA8C;gBAEpC,IAAI,EAAE,WAAW;IAE9C,mEAAmE;IACnE,cAAc,IAAI,kBAAkB;IAI9B,GAAG,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,aAAa,CAAC;IAStD;;;;;;;OAOG;IACG,YAAY,CAAC,IAAI,EAAE;QACvB,KAAK,EAAE,KAAK,CAAC;QACb,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;QACtC,eAAe,EAAE,MAAM,CAAC;QACxB,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,gEAAgE;QAChE,GAAG,CAAC,EAAE,QAAQ,CAAC;KAChB,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;YAsBjD,kBAAkB;YAwJlB,aAAa;IA2B3B,4EAA4E;IAC5E,OAAO,CAAC,cAAc;YAYR,qBAAqB;YAkCrB,YAAY;YAUZ,eAAe;YAiDf,SAAS;IAiCvB,OAAO,CAAC,SAAS;CAYlB;AAQD,wBAAgB,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,GAAG,IAAI,CAElD;AACD,wBAAgB,kBAAkB,IAAI,KAAK,GAAG,IAAI,CAEjD"}