@elizaos/plugin-computeruse 2.0.0-beta.1 → 2.0.3-beta.3

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shaw Walters and elizaOS Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,80 @@
+# `@elizaos/plugin-computeruse`
+
+Desktop automation plugin for elizaOS agents — screenshots, mouse /
+keyboard control, browser CDP automation, window management, clipboard,
+and the OCR provider registry that other plugins contribute to.
+
+Ported from
+[`coasty-ai/open-computer-use`](https://github.com/coasty-ai/open-computer-use)
+(Apache 2.0).
+
+## Boundary with `@elizaos/plugin-vision`
+
+This plugin owns the OS surfaces:
+
+- screen / display capture (`src/platform/capture.ts`,
+  `src/platform/displays.ts`,
+  `ComputerUseService.captureScreen()`),
+- input + windows + clipboard + accessibility,
+- the OCR provider registries — `OcrProvider` (line-level) and
+  `CoordOcrProvider` (hierarchical with absolute coords), defined in
+  `src/mobile/ocr-provider.ts`.
+
+`@elizaos/plugin-vision` owns the camera pipeline, scene description
+via `runtime.useModel(IMAGE_DESCRIPTION)`, the screen tiler, the
+detector pipeline (faces / people / objects), and the OCR
+implementations themselves. plugin-vision *consumes* capture from this
+plugin via `runtime.getService("computeruse")` and *contributes* the
+hierarchical OCR adapter into this plugin's `registerCoordOcrProvider`
+seam at boot.
+
+Both seams are runtime feature-detected — neither package depends on
+the other.
+
+## Enabling
+
+- Config: `features.computeruse: true`
+- Env: `COMPUTER_USE_ENABLED=1`
+
+## Platform requirements
+
+| OS | Capture | Input |
+|----|---------|-------|
+| macOS | `screencapture` (built-in) | `cliclick` (`brew install cliclick`), AppleScript |
+| Linux | `import` (ImageMagick) / `scrot` | `xdotool` (`sudo apt install xdotool`) |
+| Windows | PowerShell + `System.Drawing` | PowerShell |
+| Browser | — | `puppeteer-core` + Chrome / Edge / Brave |
+
+## Surface
+
+- **Actions** — `COMPUTER_USE` (canonical screenshot / click / key /
+  scroll / etc.), `WINDOW` (list / focus / arrange / move /...), and
+  `COMPUTER_USE_AGENT` (high-level goal-driven autonomous desktop loop:
+  Brain → Cascade → dispatch up to `maxSteps` iterations).
+  Subactions of `COMPUTER_USE` and `WINDOW` are promoted to virtual
+  top-level actions (e.g. `COMPUTER_USE_CLICK`, `WINDOW_FOCUS`) so the
+  planner picks a specific verb directly from the catalogue.
+- **Services** — `ComputerUseService` (`serviceType = "computeruse"`)
+  and `VisionContextProvider`.
+- **Providers** — `computerStateProvider`, `sceneProvider`.
+- **Routes** — approval inbox + SSE stream + approval-mode toggle under
+  `/api/computer-use/...`.
+
+## File operations + shell
+
+File operations live on the FILE action; shell / terminal access lives
+on the SHELL action. They are **not** exposed by this plugin.
+
+## Further reading
+
+- [`docs/MULTI_MONITOR.md`](./docs/MULTI_MONITOR.md) — multi-display
+  capture and coordinate translation.
+- [`docs/SCENE_BUILDER.md`](./docs/SCENE_BUILDER.md) — how windows,
+  a11y, screen, and OCR are composed into a single `Scene`.
+- [`docs/IOS_CONSTRAINTS.md`](./docs/IOS_CONSTRAINTS.md) /
+  [`docs/ANDROID_CONSTRAINTS.md`](./docs/ANDROID_CONSTRAINTS.md) —
+  honest scope on mobile.
+- [`docs/MOBILE_ASSISTANT_ROUTING.md`](./docs/MOBILE_ASSISTANT_ROUTING.md)
+  — mobile request routing.
+- [`docs/AOSP_SYSTEM_APP.md`](./docs/AOSP_SYSTEM_APP.md) — AOSP
+  system-app deployment notes.

package/dist/actions/clipboard.d.ts

@@ -0,0 +1,22 @@
+/**
+ * CLIPBOARD parent action — read or write the host system clipboard.
+ *
+ * Routes through `driverReadClipboard` / `driverWriteClipboard`, which
+ * select the per-OS tool (pbcopy/pbpaste, wl-copy/wl-paste, xclip,
+ * PowerShell Set-Clipboard / Get-Clipboard).
+ *
+ * Subactions: `read`, `write`. The plugin index promotes them to virtual
+ * top-level actions (`CLIPBOARD_READ`, `CLIPBOARD_WRITE`) so the planner
+ * can pick a specific verb directly from the action catalogue.
+ */
+import type { Action } from "@elizaos/core";
+declare const CLIPBOARD_ACTIONS: readonly ["read", "write"];
+export type ClipboardActionType = (typeof CLIPBOARD_ACTIONS)[number];
+export interface ClipboardActionParams {
+    action: ClipboardActionType;
+    /** Text payload for `write`. Ignored for `read`. */
+    text?: string;
+}
+export declare const clipboardAction: Action;
+export {};
+//# sourceMappingURL=clipboard.d.ts.map

package/dist/actions/clipboard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"clipboard.d.ts","sourceRoot":"","sources":["../../src/actions/clipboard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EACV,MAAM,EAOP,MAAM,eAAe,CAAC;AASvB,QAAA,MAAM,iBAAiB,4BAA6B,CAAC;AACrD,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,CAAC,CAAC;AAErE,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,mBAAmB,CAAC;IAC5B,oDAAoD;IACpD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AA+ED,eAAO,MAAM,eAAe,EAAE,MAgI7B,CAAC"}

package/dist/actions/progress.d.ts

@@ -0,0 +1,26 @@
+import { type Content, type HandlerCallback } from "@elizaos/core";
+import type { ApprovalSnapshot, PendingApproval } from "../types.js";
+export declare const ACTION_PROGRESS_SOURCE = "action_progress";
+export declare const COMPUTER_USE_APPROVAL_SOURCE = "computeruse_approval";
+export interface StepProgressInput {
+    actionName: string;
+    step: number;
+    kind: string;
+    rationale?: string;
+    success?: boolean;
+    error?: string;
+    source?: string;
+}
+export interface ApprovalRelayService {
+    getApprovalSnapshot(): ApprovalSnapshot;
+    subscribeApprovals(listener: (snapshot: ApprovalSnapshot) => void): () => void;
+}
+export interface ApprovalRelayOptions {
+    ownerId?: string;
+}
+export declare function isStreamProgressEnabled(value: unknown): value is true;
+export declare function formatStepProgressText(step: number, kind: string, rationale?: string): string;
+export declare function buildStepProgressContent(input: StepProgressInput): Content;
+export declare function buildApprovalPromptContent(approval: PendingApproval, options?: ApprovalRelayOptions): Content;
+export declare function withApprovalRelay<T>(service: ApprovalRelayService, callback: HandlerCallback | undefined, run: () => Promise<T>, options?: ApprovalRelayOptions): Promise<T>;
+//# sourceMappingURL=progress.d.ts.map

package/dist/actions/progress.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../src/actions/progress.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,OAAO,EACZ,KAAK,eAAe,EAGrB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAErE,eAAO,MAAM,sBAAsB,oBAAoB,CAAC;AACxD,eAAO,MAAM,4BAA4B,yBAAyB,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACnC,mBAAmB,IAAI,gBAAgB,CAAC;IACxC,kBAAkB,CAChB,QAAQ,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAC7C,MAAM,IAAI,CAAC;CACf;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAErE;AAED,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,SAAS,CAAC,EAAE,MAAM,GACjB,MAAM,CAGR;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,iBAAiB,GAAG,OAAO,CAoB1E;AAYD,wBAAgB,0BAA0B,CACxC,QAAQ,EAAE,eAAe,EACzB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAoBT;AAED,wBAAsB,iBAAiB,CAAC,CAAC,EACvC,OAAO,EAAE,oBAAoB,EAC7B,QAAQ,EAAE,eAAe,GAAG,SAAS,EACrC,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACrB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,CAAC,CAAC,CAwCZ"}

package/dist/actions/use-computer-agent.d.ts

@@ -0,0 +1,113 @@
+/**
+ * WS7 — COMPUTER_USE_AGENT action.
+ *
+ * High-level "give me a goal, I'll click my way there" entry point. The
+ * planner emits one of these instead of the lower-level COMPUTER_USE_CLICK
+ * etc. when the right action isn't obvious from the prompt.
+ *
+ * Loop:
+ *   1. refresh scene (`agent-turn`)
+ *   2. capture per-display PNGs
+ *   3. Brain → Cascade → ProposedAction
+ *   4. dispatch into ComputerInterface
+ *   5. observe (auto-screenshot via the existing service flow happens for
+ *      ProposedAction.kind=click/etc; explicit captureAllDisplays after
+ *      every step)
+ *   6. repeat until `finish` or `maxSteps`
+ *
+ * Trajectory events are emitted as structured `logger.info` lines with a
+ * `evt: "computeruse.agent.step"` payload, which the trajectory-logger app
+ * picks up via standard log capture. When `streamProgress` is enabled, the
+ * same step boundary also emits a `HandlerCallback` status to the origin chat.
+ * We don't take a hard dependency on the trajectory-logger plugin from here.
+ */
+import { type Action, type Content, type IAgentRuntime } from "@elizaos/core";
+import { type AgentMiddleware, type TrajectoryEntry } from "../actor/agent-callbacks.js";
+import { type AgentLoop, type AgentLoopStats } from "../actor/agent-loop.js";
+import type { Brain } from "../actor/brain.js";
+import { type ComputerInterface } from "../actor/computer-interface.js";
+import { type DisplayCapture } from "../platform/capture.js";
+import type { ComputerUseService } from "../services/computer-use-service.js";
+export interface ComputerUseAgentParams {
+    goal: string;
+    maxSteps?: number;
+    /**
+     * When true, emit a chat message after each dispatched step so a long-running
+     * goal does not leave the origin chat silent for minutes (#8912). The action
+     * handler wires this to the runtime HandlerCallback; the loop itself calls
+     * per-step progress hooks.
+     */
+    streamProgress?: boolean;
+    /** Wall-clock budget (ms) — the loop aborts before a step that exceeds it. */
+    maxDurationMs?: number;
+    /**
+     * Image-retention window (#9170 M11): keep only the N most-recent steps'
+     * screenshots in the bounded history. Off (unbounded) when unset.
+     */
+    imageRetentionLast?: number;
+}
+/** One per-step progress event, surfaced when `streamProgress` is set. */
+export interface ComputerUseAgentStepProgress {
+    goal: string;
+    step: number;
+    maxSteps: number;
+    sceneSummary: string;
+    actionKind: string;
+    rationale: string;
+    rois: number;
+    result: {
+        success: boolean;
+        error?: string;
+    };
+}
+interface AgentDeps {
+    brain?: Brain;
+    /** Pre-built loop override (tests). Supersedes model-string selection. */
+    loop?: AgentLoop;
+    /** Loop model-string override (tests / explicit selection). */
+    loopModel?: string;
+    /**
+     * Callback middleware override (#9170 M11). When set, replaces the default
+     * pipeline (operator-normalizer + trajectory, plus budget/image-retention
+     * when configured via params).
+     */
+    middleware?: AgentMiddleware[];
+    /** Clock override (tests) — defaults to `Date.now`. */
+    now?: () => number;
+    computerInterface?: ComputerInterface;
+    captureAll?: () => Promise<DisplayCapture[]>;
+    /** Called after each dispatched step when `params.streamProgress` is set. */
+    onStepProgress?: (progress: ComputerUseAgentStepProgress) => Promise<void> | void;
+    /** Called with compact Content after each dispatched step when enabled. */
+    onCompactStepProgress?: (content: Content) => Promise<void> | void;
+}
+export interface ComputerUseAgentReport {
+    goal: string;
+    steps: Array<{
+        step: number;
+        sceneSummary: string;
+        actionKind: string;
+        rationale: string;
+        rois: number;
+        result: {
+            success: boolean;
+            error?: string;
+        };
+    }>;
+    finished: boolean;
+    reason: "finish" | "max_steps" | "error" | "budget";
+    error?: string;
+    /** Per-step transcript recorded by the trajectory middleware (#9170 M11). */
+    trajectory?: TrajectoryEntry[];
+    /** Per-run model-call accounting, when the loop reports it (#9105). */
+    modelStats?: AgentLoopStats;
+}
+export declare function formatComputerUseAgentProgress(progress: ComputerUseAgentStepProgress): string;
+/**
+ * Run one Brain/Cascade/Dispatch loop. Exported so tests can drive it
+ * without exercising the full Action plumbing.
+ */
+export declare function runComputerUseAgentLoop(runtime: IAgentRuntime | null, params: ComputerUseAgentParams, service: ComputerUseService, deps?: AgentDeps): Promise<ComputerUseAgentReport>;
+export declare const computerUseAgentAction: Action;
+export {};
+//# sourceMappingURL=use-computer-agent.d.ts.map

package/dist/actions/use-computer-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"use-computer-agent.d.ts","sourceRoot":"","sources":["../../src/actions/use-computer-agent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EACL,KAAK,MAAM,EAEX,KAAK,OAAO,EAGZ,KAAK,aAAa,EAInB,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,KAAK,eAAe,EAWpB,KAAK,eAAe,EACrB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,cAAc,EAGpB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EACL,KAAK,iBAAiB,EAEvB,MAAM,gCAAgC,CAAC;AAExC,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAS9E,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,8EAA8E;IAC9E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,0EAA0E;AAC1E,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC9C;AAED,UAAU,SAAS;IACjB,KAAK,CAAC,EAAE,KAAK,CAAC;IACd,0EAA0E;IAC1E,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,uDAAuD;IACvD,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,iBAAiB,CAAC;IACtC,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAC7C,6EAA6E;IAC7E,cAAc,CAAC,EAAE,CACf,QAAQ,EAAE,4BAA4B,KACnC,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC1B,2EAA2E;IAC3E,qBAAqB,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;CACpE;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,KAAK,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE;YAAE,OAAO,EAAE,OAAO,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;KAC9C,CAAC,CAAC;IACH,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,QAAQ,GAAG,WAAW,GAAG,OAAO,GAAG,QAAQ,CAAC;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,6EAA6E;IAC7E,UAAU,CAAC,EAAE,eAAe,EAAE,CAAC;IAC/B,uEAAuE;IACvE,UAAU,CAAC,EAAE,cAAc,CAAC;CAC7B;AAED,wBAAgB,8BAA8B,CAC5C,QAAQ,EAAE,4BAA4B,GACrC,MAAM,CAMR;AAmBD;;;GAGG;AACH,wBAAsB,uBAAuB,CAC3C,OAAO,EAAE,aAAa,GAAG,IAAI,EAC7B,MAAM,EAAE,sBAAsB,EAC9B,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,SAAc,GACnB,OAAO,CAAC,sBAAsB,CAAC,CAuLjC;AAqFD,eAAO,MAAM,sBAAsB,EAAE,MA+HpC,CAAC"}

package/dist/actions/use-computer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"use-computer.d.ts","sourceRoot":"","sources":["../../src/actions/use-computer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAOP,MAAM,eAAe,CAAC;AAuJvB,eAAO,MAAM,iBAAiB,EAAE,MA6O/B,CAAC"}
+{"version":3,"file":"use-computer.d.ts","sourceRoot":"","sources":["../../src/actions/use-computer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,MAAM,EAOP,MAAM,eAAe,CAAC;AAgLvB,eAAO,MAAM,iBAAiB,EAAE,MA6W/B,CAAC"}

package/dist/actions/window-handlers.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Pure handler for the WINDOW parent action. Takes the live
+ * `ComputerUseService` and resolved params, executes the underlying
+ * window-management call, and returns an `ActionResult`.
+ */
+import type { ActionResult, HandlerCallback } from "@elizaos/core";
+import type { ComputerUseService } from "../services/computer-use-service.js";
+import type { WindowActionParams } from "../types.js";
+import { type ApprovalRelayOptions } from "./progress.js";
+export declare function handleWindowOp(service: ComputerUseService, params: WindowActionParams, callback?: HandlerCallback, approvalOptions?: ApprovalRelayOptions): Promise<ActionResult>;
+//# sourceMappingURL=window-handlers.d.ts.map

package/dist/actions/window-handlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"window-handlers.d.ts","sourceRoot":"","sources":["../../src/actions/window-handlers.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AACnE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qCAAqC,CAAC;AAC9E,OAAO,KAAK,EAAE,kBAAkB,EAAsB,MAAM,aAAa,CAAC;AAE1E,OAAO,EAAE,KAAK,oBAAoB,EAAqB,MAAM,eAAe,CAAC;AA0B7E,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,kBAAkB,EAC1B,QAAQ,CAAC,EAAE,eAAe,EAC1B,eAAe,GAAE,oBAAyB,GACzC,OAAO,CAAC,YAAY,CAAC,CAwBvB"}

package/dist/actions/window.d.ts

@@ -0,0 +1,11 @@
+/**
+ * WINDOW parent action — manages local desktop windows (list / focus /
+ * switch / arrange / move / minimize / maximize / restore / close).
+ *
+ * Pointer and keyboard primitives live on COMPUTER_USE. File and shell
+ * operations live on the FILE and SHELL actions in their own plugins —
+ * this plugin no longer exposes them.
+ */
+import type { Action } from "@elizaos/core";
+export declare const windowAction: Action;
+//# sourceMappingURL=window.d.ts.map

package/dist/actions/window.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"window.d.ts","sourceRoot":"","sources":["../../src/actions/window.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EACV,MAAM,EAOP,MAAM,eAAe,CAAC;AAgEvB,eAAO,MAAM,YAAY,EAAE,MAkK1B,CAAC"}

package/dist/actor/actor.d.ts

@@ -0,0 +1,84 @@
+/**
+ * WS7 — Actor (optional fine-grained grounding).
+ *
+ * The Actor is responsible for converting a Brain-issued reference
+ * ("click the Save button I see in this crop") into concrete display-local
+ * pixel coords.
+ *
+ * Primary path — deterministic (no model):
+ *   `OcrCoordinateGroundingActor` resolves a `ref: "t<displayId>-<seq>"`
+ *   (OCR id) or `ref: "a<displayId>-<seq>"` (AX id) directly from the
+ *   Scene. Click point is bbox-center. No VLM call, fully reproducible,
+ *   what 99% of the cascade should use.
+ *
+ * Optional secondary path — VLM:
+ *   `OsAtlasProActor` is a typed adapter for an operator-provided model-server
+ *   endpoint (e.g. an OS-Atlas-Pro vLLM service). Unless a deployment
+ *   registers that endpoint, the cascade uses the OCR/AX grounding above.
+ *
+ * Register the active Actor on the cascade via `setActor(actor)` (see
+ * `cascade.ts`). If none is registered, the cascade uses the OCR/AX actor
+ * automatically.
+ */
+import type { Scene } from "../scene/scene-types.js";
+import type { GroundingResult, ReferenceTarget } from "./types.js";
+export interface ActorGroundArgs {
+    /** Display the Brain wants to act on. */
+    displayId: number;
+    /**
+     * Cropped image of the ROI at native resolution (PNG bytes). May be a
+     * empty Buffer when the deterministic grounding doesn't need image bytes.
+     */
+    croppedImage: Buffer;
+    /** Hint from the Brain: "the Save button in the dialog footer". */
+    hint: string;
+    /** Optional reference from `BrainProposedAction.ref`. */
+    ref?: string;
+}
+export interface Actor {
+    readonly name: string;
+    ground(args: ActorGroundArgs): Promise<GroundingResult>;
+}
+export declare class OcrCoordinateGroundingActor implements Actor {
+    private readonly getScene;
+    readonly name = "ocr-ax-grounding";
+    constructor(getScene: () => Scene | null);
+    ground(args: ActorGroundArgs): Promise<GroundingResult>;
+}
+/**
+ * Look up a scene element by stable id, OR by case-insensitive label match
+ * when an id is absent. Used by both the deterministic actor and the cascade
+ * dispatcher to validate Brain output.
+ */
+export declare function resolveReference(scene: Scene, ref: string | undefined, hint: string, preferredDisplay: number): ReferenceTarget | null;
+export interface OsAtlasProActorOptions {
+    /** Endpoint of the model server, e.g. `http://localhost:8000/v1`. */
+    endpoint: string;
+    /** Optional auth header. */
+    apiKey?: string;
+    /** Model identifier on the server. */
+    model?: string;
+    /** Override the HTTP fetch (mostly for tests). */
+    fetcher?: (input: string, init: {
+        body: string;
+        headers: Record<string, string>;
+    }) => Promise<{
+        ok: boolean;
+        status: number;
+        text: () => Promise<string>;
+    }>;
+}
+/**
+ * Adapter for a server-side OS-Atlas-Pro (or compatible) grounding model.
+ * Not wired into the cascade by default. The contract: POST a JSON payload
+ * with `{ image: base64, hint }`, expect `{ x, y, confidence }` in image
+ * coordinates of the crop. The cascade is responsible for converting those
+ * crop-local coords back to display-local before dispatch.
+ */
+export declare class OsAtlasProActor implements Actor {
+    private readonly opts;
+    readonly name = "osatlas-pro";
+    constructor(opts: OsAtlasProActorOptions);
+    ground(args: ActorGroundArgs): Promise<GroundingResult>;
+}
+//# sourceMappingURL=actor.d.ts.map

package/dist/actor/actor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"actor.d.ts","sourceRoot":"","sources":["../../src/actor/actor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,KAAK,EAA4B,MAAM,yBAAyB,CAAC;AAC/E,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEnE,MAAM,WAAW,eAAe;IAC9B,yCAAyC;IACzC,SAAS,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAC;IACrB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,yDAAyD;IACzD,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,KAAK;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,MAAM,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CACzD;AAID,qBAAa,2BAA4B,YAAW,KAAK;IAG3C,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAFrC,QAAQ,CAAC,IAAI,sBAAsB;gBAEN,QAAQ,EAAE,MAAM,KAAK,GAAG,IAAI;IAEnD,MAAM,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;CAsB9D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,MAAM,GAAG,SAAS,EACvB,IAAI,EAAE,MAAM,EACZ,gBAAgB,EAAE,MAAM,GACvB,eAAe,GAAG,IAAI,CA+BxB;AA0CD,MAAM,WAAW,sBAAsB;IACrC,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;IACjB,4BAA4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,kDAAkD;IAClD,OAAO,CAAC,EAAE,CACR,KAAK,EAAE,MAAM,EACb,IAAI,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,KACpD,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAA;KAAE,CAAC,CAAC;CAC5E;AAED;;;;;;GAMG;AACH,qBAAa,eAAgB,YAAW,KAAK;IAG/B,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,QAAQ,CAAC,IAAI,iBAAiB;gBAED,IAAI,EAAE,sBAAsB;IAQnD,MAAM,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;CAkD9D"}

package/dist/actor/agent-callbacks.d.ts

@@ -0,0 +1,128 @@
+/**
+ * Agent-loop callback middleware (#9170 M11).
+ *
+ * trycua/cua threads a callback pipeline through its agent loop: budget caps,
+ * image-retention (keep only the N most-recent screenshots in context),
+ * operator-normalization (clean up the model's proposed action), and trajectory
+ * recording. Each callback is a thin middleware that observes (and sometimes
+ * transforms or aborts) the loop without the loop knowing which middlewares are
+ * present.
+ *
+ * This module defines the `AgentMiddleware` hook set and four built-ins. The
+ * runner (`use-computer-agent.ts`) fires the hooks at fixed points:
+ *   onRunStart → [ beforeStep → onCaptures → transformProposed → afterStep ]* →
+ *   onRunEnd
+ *
+ * Middlewares are pure-by-default and composable: `runBeforeStep`, etc.,
+ * fold the list in order and short-circuit on the first abort.
+ */
+import type { DisplayCapture } from "../platform/capture.js";
+import type { CascadeResult, ProposedAction } from "./types.js";
+export interface AgentRunContext {
+    goal: string;
+    maxSteps: number;
+}
+export interface AgentStepContext {
+    step: number;
+    maxSteps: number;
+    goal: string;
+    /** Wall-clock ms since the run started (set by the runner). */
+    elapsedMs: number;
+}
+export interface AgentDispatchContext {
+    step: number;
+    goal: string;
+    proposed: CascadeResult;
+    dispatchSuccess: boolean;
+    error?: string;
+}
+/** Returned by `beforeStep` — abort halts the loop with `reason`. */
+export interface AgentStepDecision {
+    abort?: boolean;
+    reason?: string;
+}
+export interface AgentRunSummary {
+    goal: string;
+    steps: number;
+    finished: boolean;
+    reason: string;
+}
+export interface AgentMiddleware {
+    readonly name: string;
+    onRunStart?(ctx: AgentRunContext): void | Promise<void>;
+    /** Inspect/abort before a step runs (budget caps live here). */
+    beforeStep?(ctx: AgentStepContext): AgentStepDecision | Promise<AgentStepDecision>;
+    /** Observe the captured frames (image-retention bookkeeping). */
+    onCaptures?(captures: Map<number, DisplayCapture>, ctx: AgentStepContext): void | Promise<void>;
+    /** Transform the planned step before dispatch (operator-normalizer). */
+    transformProposed?(proposed: CascadeResult, ctx: AgentStepContext): CascadeResult | Promise<CascadeResult>;
+    /** Observe a dispatched step (trajectory recording). */
+    afterStep?(ctx: AgentDispatchContext): void | Promise<void>;
+    onRunEnd?(summary: AgentRunSummary): void | Promise<void>;
+}
+export declare function runOnRunStart(middlewares: readonly AgentMiddleware[], ctx: AgentRunContext): Promise<void>;
+/** Fold `beforeStep`; the FIRST abort wins (and names the middleware). */
+export declare function runBeforeStep(middlewares: readonly AgentMiddleware[], ctx: AgentStepContext): Promise<AgentStepDecision>;
+export declare function runOnCaptures(middlewares: readonly AgentMiddleware[], captures: Map<number, DisplayCapture>, ctx: AgentStepContext): Promise<void>;
+/** Fold `transformProposed` left-to-right; each sees the prior's output. */
+export declare function runTransformProposed(middlewares: readonly AgentMiddleware[], proposed: CascadeResult, ctx: AgentStepContext): Promise<CascadeResult>;
+export declare function runAfterStep(middlewares: readonly AgentMiddleware[], ctx: AgentDispatchContext): Promise<void>;
+export declare function runOnRunEnd(middlewares: readonly AgentMiddleware[], summary: AgentRunSummary): Promise<void>;
+export interface BudgetCapOptions {
+    /** Abort once this many steps have STARTED (independent of the loop's own
+     * maxSteps; use to cap below it, e.g. cost control). */
+    maxSteps?: number;
+    /** Abort once wall-clock elapsed exceeds this many ms. */
+    maxDurationMs?: number;
+}
+/**
+ * Halts the loop when a step or time budget is exhausted. Caps below the loop's
+ * own `maxSteps`, and adds a wall-clock cap the loop has no notion of.
+ */
+export declare function createBudgetCapMiddleware(options: BudgetCapOptions): AgentMiddleware;
+export interface ImageRetentionMiddleware extends AgentMiddleware {
+    /** The display-keyed captures retained from the most recent steps. */
+    retained(): Array<{
+        step: number;
+        displayIds: number[];
+    }>;
+}
+/**
+ * Bounds the screenshot history to the `keepLast` most-recent steps, mirroring
+ * cua's image-retention (older frames fall out of context to cap token cost).
+ * The runner forwards the per-step captures; this middleware keeps the bounded
+ * window that a model-history consumer should send.
+ */
+export declare function createImageRetentionMiddleware(options: {
+    keepLast: number;
+}): ImageRetentionMiddleware;
+/**
+ * Normalize the model's proposed action into the canonical dispatch shape:
+ * integer coordinates, trimmed type text, deduped/lowercased hotkey keys. Pure
+ * and idempotent — re-normalizing already-clean input is a no-op.
+ */
+export declare function normalizeProposedAction(action: ProposedAction): ProposedAction;
+/** Operator-normalizer middleware — cleans `proposed.proposed` before dispatch. */
+export declare function createOperatorNormalizerMiddleware(): AgentMiddleware;
+export interface TrajectoryEntry {
+    step: number;
+    goal: string;
+    actionKind: string;
+    rationale: string;
+    success: boolean;
+    error?: string;
+}
+export interface TrajectoryMiddleware extends AgentMiddleware {
+    /** The recorded trajectory so far. */
+    entries(): TrajectoryEntry[];
+}
+/**
+ * Records one entry per dispatched step. Independent of the existing
+ * `logger.info` trajectory events — this gives an in-memory transcript the
+ * caller can attach to the run report or persist.
+ */
+export declare function createTrajectoryMiddleware(options?: {
+    /** Also emit a debug log line per step. Default false. */
+    log?: boolean;
+}): TrajectoryMiddleware;
+//# sourceMappingURL=agent-callbacks.d.ts.map

package/dist/actor/agent-callbacks.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent-callbacks.d.ts","sourceRoot":"","sources":["../../src/actor/agent-callbacks.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,aAAa,CAAC;IACxB,eAAe,EAAE,OAAO,CAAC;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qEAAqE;AACrE,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,gEAAgE;IAChE,UAAU,CAAC,CACT,GAAG,EAAE,gBAAgB,GACpB,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAClD,iEAAiE;IACjE,UAAU,CAAC,CACT,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACrC,GAAG,EAAE,gBAAgB,GACpB,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,wEAAwE;IACxE,iBAAiB,CAAC,CAChB,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,gBAAgB,GACpB,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;IAC1C,wDAAwD;IACxD,SAAS,CAAC,CAAC,GAAG,EAAE,oBAAoB,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,QAAQ,CAAC,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3D;AAID,wBAAsB,aAAa,CACjC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,GAAG,EAAE,eAAe,GACnB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,0EAA0E;AAC1E,wBAAsB,aAAa,CACjC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAW5B;AAED,wBAAsB,aAAa,CACjC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,EACrC,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,4EAA4E;AAC5E,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,QAAQ,EAAE,aAAa,EACvB,GAAG,EAAE,gBAAgB,GACpB,OAAO,CAAC,aAAa,CAAC,CAMxB;AAED,wBAAsB,YAAY,CAChC,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,GAAG,EAAE,oBAAoB,GACxB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,WAAW,CAC/B,WAAW,EAAE,SAAS,eAAe,EAAE,EACvC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,IAAI,CAAC,CAEf;AAID,MAAM,WAAW,gBAAgB;IAC/B;4DACwD;IACxD,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,gBAAgB,GACxB,eAAe,CAsBjB;AAID,MAAM,WAAW,wBAAyB,SAAQ,eAAe;IAC/D,sEAAsE;IACtE,QAAQ,IAAI,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;CAC3D;AAED;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,EAAE;IACtD,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,wBAAwB,CAgB3B;AASD;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,cAAc,GACrB,cAAc,CAiBhB;AAED,mFAAmF;AACnF,wBAAgB,kCAAkC,IAAI,eAAe,CAUpE;AAID,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAqB,SAAQ,eAAe;IAC3D,sCAAsC;IACtC,OAAO,IAAI,eAAe,EAAE,CAAC;CAC9B;AAED;;;;GAIG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,CAAC,EAAE;IACnD,0DAA0D;IAC1D,GAAG,CAAC,EAAE,OAAO,CAAC;CACf,GAAG,oBAAoB,CAwBvB"}