@zenalexa/unicli 0.211.2 → 0.213.0-beta.2

package/dist/transport/adapters/cua.js

@@ -0,0 +1,663 @@
+/**
+ * CuaTransport — screenshot + VLM "Computer Use" transport.
+ *
+ * Unlike the other transports, CUA has no single protocol — it wraps a
+ * pluggable "backend" that implements the primitive perception/action
+ * verbs against a real computer-use model or emulator. v0.212 ships four
+ * backends selected by `CUA_BACKEND`:
+ *
+ *   - `anthropic`   Anthropic `computer_20251124` beta tool, `ANTHROPIC_API_KEY`
+ *   - `trycua`      trycua/cua compute-server v0.1.9+, `TRYCUA_API_KEY`
+ *   - `opencua`     OpenCUA-72B via local vLLM, `OPENCUA_ENDPOINT`
+ *   - `scrapybara`  Scrapybara Computer API, `SCRAPYBARA_API_KEY`
+ *   - `mock`        in-memory stub used by tests and when no key is present
+ *
+ * The real network paths to each provider are stubbed in v0.212 — they are
+ * gated behind the corresponding env keys and throw a structured
+ * `config_error` envelope if selected without credentials. The MockBackend
+ * is always available so the offline test suite runs without secrets.
+ *
+ * Design contract:
+ *  - `action()` NEVER throws
+ *  - all backends implement the same primitive verb surface (below)
+ *  - selection is deterministic: `selectCuaBackend()` returns the same
+ *    backend for the same env — no hidden state
+ */
+import { err, exitCodeFor, ok } from "../../core/envelope.js";
+// ── Supported pipeline steps ────────────────────────────────────────────
+const CUA_STEPS = [
+    "cua_snapshot",
+    "cua_click",
+    "cua_type",
+    "cua_key",
+    "cua_scroll",
+    "cua_drag",
+    "cua_wait",
+    "cua_assert",
+    "cua_ask",
+    "cua_backend",
+    "cua_launch",
+];
+const CUA_CAPABILITY = {
+    steps: CUA_STEPS,
+    snapshotFormats: ["screenshot"],
+    mutatesHost: true,
+};
+// ── Mock backend (offline, deterministic) ───────────────────────────────
+/** Fixed 1x1 transparent PNG used by {@link MockBackend.snapshot}. */
+const MOCK_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkAAIAAAoAAv/lxKUAAAAASUVORK5CYII=";
+/**
+ * MockBackend — deterministic, offline CUA backend used when no provider
+ * credentials are present and by every test in this file. Records the
+ * sequence of calls on `history` so tests can assert against it without
+ * having to stub `fetch`.
+ */
+export class MockBackend {
+    name = "mock";
+    history = [];
+    record(verb, ...args) {
+        this.history.push({ verb, args });
+    }
+    async snapshot() {
+        this.record("snapshot");
+        return { base64: MOCK_PNG_BASE64, width: 1, height: 1, mime: "image/png" };
+    }
+    async click(x, y, button = "left") {
+        this.record("click", x, y, button);
+    }
+    async type(text) {
+        this.record("type", text);
+    }
+    async key(key) {
+        this.record("key", key);
+    }
+    async scroll(dx, dy) {
+        this.record("scroll", dx, dy);
+    }
+    async drag(fromX, fromY, toX, toY) {
+        this.record("drag", fromX, fromY, toX, toY);
+    }
+    async wait(ms) {
+        this.record("wait", ms);
+        // Intentionally not calling setTimeout — tests want determinism.
+    }
+    async ask(question) {
+        this.record("ask", question);
+        return "yes";
+    }
+    async launch(app) {
+        this.record("launch", app);
+    }
+}
+// ── Real-backend stubs ──────────────────────────────────────────────────
+/**
+ * Base class used by every real backend. All verbs throw `BackendNotReadyError`
+ * by default; subclasses override what they support.
+ *
+ * Honesty note (2026-04-15): v0.212 ships `Anthropic`, `Trycua`, `OpenCua`
+ * and `Scrapybara` as **interface stubs**. They declare the shape the real
+ * backends will take, fail fast with a structured envelope when selected
+ * without their credentials, and let downstream tests pin the Mock
+ * backend for offline determinism. Full network implementations are
+ * tracked in `docs/ROADMAP.md` as v0.213 "Gagarin" targets — see there
+ * for the design of the screen-source ↔ planner composition that real
+ * Anthropic computer-use requires.
+ *
+ * Users who want a real backend today can implement the `CuaBackend`
+ * interface in their own code and inject it via
+ * {@link CuaTransport.setBackend}, or provide a factory through the
+ * `opts.backend` of the transport constructor. That surface is stable.
+ */
+class BackendNotReadyError extends Error {
+    backend;
+    verb;
+    hint;
+    constructor(backend, verb, hint) {
+        super(`cua backend "${backend}" is a v0.212 stub — ${verb} not implemented (${hint})`);
+        this.backend = backend;
+        this.verb = verb;
+        this.hint = hint;
+        this.name = "BackendNotReadyError";
+    }
+}
+/**
+ * Anthropic `computer_use` tool stub. Selected when `CUA_BACKEND=anthropic`
+ * and `ANTHROPIC_API_KEY` is set.
+ *
+ * IMPORTANT — two-layer architecture: the Anthropic Messages API is a
+ * planner, not a screen capture service. A production Anthropic backend
+ * MUST be composed with a screenshot source (macOS `screencapture`,
+ * trycua sandbox, scrapybara VM) because the model receives screenshots
+ * from the client and returns `tool_use` actions. v0.212 ships this as
+ * a stub; the composition layer lands in v0.213 via a new
+ * `AnthropicPlanner(screenshotSource)` constructor — see
+ * `docs/ROADMAP.md` > v0.213 "Gagarin".
+ *
+ * `tool_version` defaults to `"computer_20260301"` (the successor to
+ * `computer_20251124` in the Sonnet 4.6 rollout); override via env
+ * `ANTHROPIC_CUA_TOOL_VERSION` so operators can follow Anthropic's
+ * release cadence without a code change.
+ */
+export class AnthropicBackend {
+    name = "anthropic";
+    apiKey;
+    model;
+    toolVersion;
+    constructor(apiKey, model = process.env.ANTHROPIC_CUA_MODEL ?? "claude-sonnet-4-6", toolVersion = process.env.ANTHROPIC_CUA_TOOL_VERSION ??
+        "computer_20260301") {
+        this.apiKey = apiKey;
+        this.model = model;
+        this.toolVersion = toolVersion;
+    }
+    async snapshot() {
+        throw new BackendNotReadyError(this.name, "snapshot", `the Anthropic Messages API does not capture screens; pair ${this.name} with a screenshot source (screencapture/trycua/scrapybara) — v0.213 Gagarin target`);
+    }
+    async click() {
+        throw new BackendNotReadyError(this.name, "click", `will POST /v1/messages with tool "${this.toolVersion}" action "left_click" in v0.213`);
+    }
+    async type() {
+        throw new BackendNotReadyError(this.name, "type", `will POST /v1/messages with tool "${this.toolVersion}" action "type" in v0.213`);
+    }
+    async key() {
+        throw new BackendNotReadyError(this.name, "key", `will POST /v1/messages with tool "${this.toolVersion}" action "key" in v0.213`);
+    }
+    async scroll() {
+        throw new BackendNotReadyError(this.name, "scroll", `will POST /v1/messages with tool "${this.toolVersion}" action "scroll" in v0.213`);
+    }
+    async drag() {
+        throw new BackendNotReadyError(this.name, "drag", `will POST /v1/messages with tool "${this.toolVersion}" action "left_click_drag" in v0.213`);
+    }
+    async wait(ms) {
+        await new Promise((r) => setTimeout(r, Math.max(0, ms)));
+    }
+    async ask() {
+        throw new BackendNotReadyError(this.name, "ask", `will POST /v1/messages without a tool for plain Q&A in v0.213; supply your own backend via CuaTransport.setBackend() today`);
+    }
+}
+/** trycua/cua compute-server v0.1.9+. Activated when `CUA_BACKEND=trycua`. */
+export class TrycuaBackend {
+    name = "trycua";
+    apiKey;
+    endpoint;
+    constructor(apiKey, endpoint = "https://api.cua.dev") {
+        this.apiKey = apiKey;
+        this.endpoint = endpoint;
+    }
+    async snapshot() {
+        throw new BackendNotReadyError(this.name, "snapshot", "implement GET /v1/screen against the trycua compute-server");
+    }
+    async click() {
+        throw new BackendNotReadyError(this.name, "click", "POST /v1/click");
+    }
+    async type() {
+        throw new BackendNotReadyError(this.name, "type", "POST /v1/type");
+    }
+    async key() {
+        throw new BackendNotReadyError(this.name, "key", "POST /v1/key");
+    }
+    async scroll() {
+        throw new BackendNotReadyError(this.name, "scroll", "POST /v1/scroll");
+    }
+    async drag() {
+        throw new BackendNotReadyError(this.name, "drag", "POST /v1/drag");
+    }
+    async wait(ms) {
+        await new Promise((r) => setTimeout(r, Math.max(0, ms)));
+    }
+}
+/** OpenCUA-72B via local vLLM. Activated when `CUA_BACKEND=opencua`. */
+export class OpenCuaBackend {
+    endpoint;
+    name = "opencua";
+    constructor(endpoint) {
+        this.endpoint = endpoint;
+    }
+    async snapshot() {
+        throw new BackendNotReadyError(this.name, "snapshot", "implement vLLM OpenCUA-72B screenshot call at " + this.endpoint);
+    }
+    async click() {
+        throw new BackendNotReadyError(this.name, "click", "implement vLLM action");
+    }
+    async type() {
+        throw new BackendNotReadyError(this.name, "type", "implement vLLM action");
+    }
+    async key() {
+        throw new BackendNotReadyError(this.name, "key", "implement vLLM action");
+    }
+    async scroll() {
+        throw new BackendNotReadyError(this.name, "scroll", "implement vLLM action");
+    }
+    async drag() {
+        throw new BackendNotReadyError(this.name, "drag", "implement vLLM action");
+    }
+    async wait(ms) {
+        await new Promise((r) => setTimeout(r, Math.max(0, ms)));
+    }
+}
+/** Scrapybara Computer API. Activated when `CUA_BACKEND=scrapybara`. */
+export class ScrapybaraBackend {
+    name = "scrapybara";
+    apiKey;
+    endpoint;
+    constructor(apiKey, endpoint = "https://api.scrapybara.com") {
+        this.apiKey = apiKey;
+        this.endpoint = endpoint;
+    }
+    async snapshot() {
+        throw new BackendNotReadyError(this.name, "snapshot", "implement Scrapybara screenshot API");
+    }
+    async click() {
+        throw new BackendNotReadyError(this.name, "click", "Scrapybara click");
+    }
+    async type() {
+        throw new BackendNotReadyError(this.name, "type", "Scrapybara type");
+    }
+    async key() {
+        throw new BackendNotReadyError(this.name, "key", "Scrapybara key");
+    }
+    async scroll() {
+        throw new BackendNotReadyError(this.name, "scroll", "Scrapybara scroll");
+    }
+    async drag() {
+        throw new BackendNotReadyError(this.name, "drag", "Scrapybara drag");
+    }
+    async wait(ms) {
+        await new Promise((r) => setTimeout(r, Math.max(0, ms)));
+    }
+}
+/**
+ * Pure function — given an env snapshot, return the backend to use.
+ *
+ * Selection rules (first match wins):
+ *   1. `CUA_BACKEND=anthropic`  and ANTHROPIC_API_KEY  → AnthropicBackend
+ *   2. `CUA_BACKEND=trycua`     and TRYCUA_API_KEY     → TrycuaBackend
+ *   3. `CUA_BACKEND=opencua`    and OPENCUA_ENDPOINT   → OpenCuaBackend
+ *   4. `CUA_BACKEND=scrapybara` and SCRAPYBARA_API_KEY → ScrapybaraBackend
+ *   5. otherwise                                       → MockBackend
+ *
+ * If the user names a backend but the required env is missing, we return a
+ * MockBackend tagged with a `missing` sentinel so the caller can emit a
+ * structured `config_error` envelope on first use.
+ */
+export function selectCuaBackend(env = process.env) {
+    const requested = (env.CUA_BACKEND ?? "").toLowerCase();
+    switch (requested) {
+        case "anthropic":
+            if (env.ANTHROPIC_API_KEY)
+                return new AnthropicBackend(env.ANTHROPIC_API_KEY);
+            return new MockBackend(); // ANTHROPIC_API_KEY missing → fallback
+        case "trycua":
+            if (env.TRYCUA_API_KEY)
+                return new TrycuaBackend(env.TRYCUA_API_KEY);
+            return new MockBackend();
+        case "opencua":
+            if (env.OPENCUA_ENDPOINT)
+                return new OpenCuaBackend(env.OPENCUA_ENDPOINT);
+            return new MockBackend();
+        case "scrapybara":
+            if (env.SCRAPYBARA_API_KEY)
+                return new ScrapybaraBackend(env.SCRAPYBARA_API_KEY);
+            return new MockBackend();
+        case "mock":
+        case "":
+            return new MockBackend();
+        default:
+            // Unknown backend name — fall back to mock so action() can envelope.
+            return new MockBackend();
+    }
+}
+/**
+ * CuaTransport routes `cua_*` pipeline steps to the active backend, wraps
+ * every outcome in an envelope, and never throws from `action()`.
+ */
+export class CuaTransport {
+    kind = "cua";
+    capability = CUA_CAPABILITY;
+    backend;
+    lastSnapshot = undefined;
+    constructor(opts = {}) {
+        this.backend = opts.backend ?? selectCuaBackend(opts.env);
+    }
+    /** Introspect the active backend (used by tests + `cua_backend` step). */
+    get activeBackendName() {
+        return this.backend.name;
+    }
+    /** Swap backends at runtime — the `cua_backend` step uses this. */
+    setBackend(backend) {
+        this.backend = backend;
+    }
+    async open(_ctx) {
+        // No-op — CUA backends don't need pipeline vars today.
+    }
+    async snapshot(opts) {
+        const format = opts?.format ?? "screenshot";
+        if (this.lastSnapshot && format === this.lastSnapshot.format)
+            return this.lastSnapshot;
+        try {
+            const raw = await this.backend.snapshot();
+            const snap = {
+                format: "screenshot",
+                data: Buffer.from(raw.base64, "base64"),
+                width: raw.width,
+                height: raw.height,
+            };
+            this.lastSnapshot = snap;
+            return snap;
+        }
+        catch {
+            // Never throw from snapshot — return an empty JSON envelope.
+            return { format: "json", data: JSON.stringify({ ok: false }) };
+        }
+    }
+    async action(req) {
+        const start = Date.now();
+        try {
+            const envelope = await this.dispatch(req);
+            envelope.elapsedMs = Date.now() - start;
+            return envelope;
+        }
+        catch (e) {
+            const msg = e instanceof Error ? e.message : String(e);
+            return err({
+                transport: "cua",
+                step: 0,
+                action: req.kind,
+                reason: `unexpected error in cua.${req.kind}: ${msg}`,
+                suggestion: "check backend configuration and env vars",
+                retryable: false,
+            });
+        }
+    }
+    async close() {
+        this.lastSnapshot = undefined;
+    }
+    // ── internals ────────────────────────────────────────────────────────
+    async dispatch(req) {
+        switch (req.kind) {
+            case "cua_snapshot":
+                return this.doSnapshot();
+            case "cua_click":
+                return this.doClick(req.params);
+            case "cua_type":
+                return this.doType(req.params);
+            case "cua_key":
+                return this.doKey(req.params);
+            case "cua_scroll":
+                return this.doScroll(req.params);
+            case "cua_drag":
+                return this.doDrag(req.params);
+            case "cua_wait":
+                return this.doWait(req.params);
+            case "cua_assert":
+                return this.doAssert(req.params);
+            case "cua_ask":
+                return this.doAsk(req.params);
+            case "cua_backend":
+                return this.doBackendInfo();
+            case "cua_launch":
+                return this.doLaunch(req.params);
+            default:
+                return err({
+                    transport: "cua",
+                    step: 0,
+                    action: req.kind,
+                    reason: `unsupported action "${req.kind}" for cua transport`,
+                    suggestion: `cua transport supports: ${CUA_STEPS.join(", ")}`,
+                    minimum_capability: `cua.${req.kind}`,
+                    exit_code: exitCodeFor("usage_error"),
+                });
+        }
+    }
+    envelopeFromBackendError(verb, e) {
+        const msg = e instanceof Error ? e.message : String(e);
+        const notReady = e instanceof BackendNotReadyError;
+        return err({
+            transport: "cua",
+            step: 0,
+            action: verb,
+            reason: msg,
+            suggestion: notReady
+                ? `install or configure the \`${this.backend.name}\` backend before calling cua.${verb}`
+                : "inspect backend error and retry",
+            minimum_capability: `cua.${verb}`,
+            retryable: !notReady,
+            exit_code: notReady
+                ? exitCodeFor("config_error")
+                : exitCodeFor("service_unavailable"),
+        });
+    }
+    async doSnapshot() {
+        try {
+            const raw = await this.backend.snapshot();
+            this.lastSnapshot = {
+                format: "screenshot",
+                data: Buffer.from(raw.base64, "base64"),
+                width: raw.width,
+                height: raw.height,
+            };
+            return ok({
+                backend: this.backend.name,
+                width: raw.width,
+                height: raw.height,
+                base64: raw.base64,
+            });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("snapshot", e);
+        }
+    }
+    async doClick(params) {
+        const x = typeof params.x === "number" ? params.x : undefined;
+        const y = typeof params.y === "number" ? params.y : undefined;
+        if (x === undefined || y === undefined) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_click",
+                reason: "cua_click requires numeric params.x and params.y",
+                suggestion: "pass { x, y } coordinates in pixel space",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        const button = params.button === "right" ? "right" : "left";
+        try {
+            await this.backend.click(x, y, button);
+            return ok({ x, y, button });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("click", e);
+        }
+    }
+    async doType(params) {
+        const text = typeof params.text === "string" ? params.text : undefined;
+        if (text === undefined) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_type",
+                reason: "cua_type requires params.text (string)",
+                suggestion: "pass the literal text to type",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        try {
+            await this.backend.type(text);
+            return ok({ text });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("type", e);
+        }
+    }
+    async doKey(params) {
+        const key = typeof params.key === "string" ? params.key : undefined;
+        if (!key) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_key",
+                reason: "cua_key requires params.key (string)",
+                suggestion: 'pass a named key, e.g. "Return", "cmd+a"',
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        try {
+            await this.backend.key(key);
+            return ok({ key });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("key", e);
+        }
+    }
+    async doScroll(params) {
+        const dx = typeof params.dx === "number" ? params.dx : 0;
+        const dy = typeof params.dy === "number" ? params.dy : 0;
+        try {
+            await this.backend.scroll(dx, dy);
+            return ok({ dx, dy });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("scroll", e);
+        }
+    }
+    async doDrag(params) {
+        const fromX = typeof params.fromX === "number" ? params.fromX : undefined;
+        const fromY = typeof params.fromY === "number" ? params.fromY : undefined;
+        const toX = typeof params.toX === "number" ? params.toX : undefined;
+        const toY = typeof params.toY === "number" ? params.toY : undefined;
+        if (fromX === undefined ||
+            fromY === undefined ||
+            toX === undefined ||
+            toY === undefined) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_drag",
+                reason: "cua_drag requires params.fromX/fromY/toX/toY numbers",
+                suggestion: "pass all four drag endpoints as numeric pixel coords",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        try {
+            await this.backend.drag(fromX, fromY, toX, toY);
+            return ok({ fromX, fromY, toX, toY });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("drag", e);
+        }
+    }
+    async doWait(params) {
+        const ms = typeof params.ms === "number"
+            ? params.ms
+            : typeof params.seconds === "number"
+                ? params.seconds * 1000
+                : 0;
+        try {
+            await this.backend.wait(Math.max(0, ms));
+            return ok({ ms });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("wait", e);
+        }
+    }
+    async doAssert(params) {
+        // The CUA assert is a ask→boolean — the VLM decides whether the
+        // screen satisfies the predicate. With MockBackend it always passes.
+        const predicate = typeof params.predicate === "string" ? params.predicate : undefined;
+        if (!predicate) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_assert",
+                reason: "cua_assert requires params.predicate (string)",
+                suggestion: "pass a natural-language predicate for the VLM",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        try {
+            const answer = this.backend.ask
+                ? await this.backend.ask(`Does the screen satisfy: ${predicate}?`)
+                : "yes";
+            const truthy = /^(y|yes|true|1)$/i.test(answer.trim());
+            if (!truthy) {
+                return err({
+                    transport: "cua",
+                    step: 0,
+                    action: "cua_assert",
+                    reason: `VLM did not confirm predicate "${predicate}" (got ${answer})`,
+                    suggestion: "inspect the latest snapshot or refine the predicate",
+                    retryable: true,
+                });
+            }
+            return ok({ predicate, answer });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("assert", e);
+        }
+    }
+    async doAsk(params) {
+        const question = typeof params.question === "string" ? params.question : undefined;
+        if (!question) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_ask",
+                reason: "cua_ask requires params.question (string)",
+                suggestion: "pass a natural-language question for the VLM",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        if (!this.backend.ask) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_ask",
+                reason: `backend ${this.backend.name} does not implement ask()`,
+                suggestion: "use the anthropic or mock backend, or add ask() to the backend",
+                minimum_capability: "cua.ask",
+                exit_code: exitCodeFor("service_unavailable"),
+            });
+        }
+        try {
+            const answer = await this.backend.ask(question);
+            return ok({ question, answer });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("ask", e);
+        }
+    }
+    async doBackendInfo() {
+        return ok({ backend: this.backend.name });
+    }
+    async doLaunch(params) {
+        const app = typeof params.app === "string" ? params.app : undefined;
+        if (!app) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_launch",
+                reason: "cua_launch requires params.app (string)",
+                suggestion: "pass the app name to launch",
+                exit_code: exitCodeFor("usage_error"),
+            });
+        }
+        if (!this.backend.launch) {
+            return err({
+                transport: "cua",
+                step: 0,
+                action: "cua_launch",
+                reason: `backend ${this.backend.name} does not support launch()`,
+                suggestion: "use desktop-ax.launch_app for native macOS launch, or Scrapybara sandbox",
+                minimum_capability: "cua.launch",
+                exit_code: exitCodeFor("service_unavailable"),
+            });
+        }
+        try {
+            await this.backend.launch(app);
+            return ok({ app });
+        }
+        catch (e) {
+            return this.envelopeFromBackendError("launch", e);
+        }
+    }
+}
+//# sourceMappingURL=cua.js.map