@beeos-ai/device-agent 0.4.0

package/dist/adapters/device-desktop.js

@@ -0,0 +1,101 @@
+/**
+ * Adapter — desktop mock device.
+ *
+ * Equivalent to the Python `mode=desktop` runtime: returns a fixed canvas,
+ * records every action into an in-memory log, and never touches the host.
+ *
+ * Used for:
+ *  - local UI development without ADB
+ *  - unit testing the application/inbound layers
+ *  - smoke-testing CI without a real device
+ */
+import { randomBytes } from "node:crypto";
+export class DesktopAdapter {
+    actions = [];
+    width;
+    height;
+    name;
+    osType;
+    connected = false;
+    constructor(opts = {}) {
+        this.width = opts.width ?? 1920;
+        this.height = opts.height ?? 1080;
+        this.name = opts.name ?? "Desktop (mock)";
+        this.osType = opts.os ?? defaultDesktopOs();
+    }
+    async connect() {
+        this.connected = true;
+    }
+    async disconnect() {
+        this.connected = false;
+    }
+    async getInfo() {
+        // ACP wire v2: per-action capability vocabulary. Desktop mock supports
+        // visual capture + tap/swipe/type but no Android-specific keys/install.
+        const capabilities = [
+            "screenshot",
+            "tap",
+            "swipe",
+            "type",
+            "shell",
+        ];
+        return {
+            type: this.osType,
+            name: this.name,
+            os: prettyOsName(this.osType),
+            width: this.width,
+            height: this.height,
+            capabilities,
+            osVersion: "mock",
+            density: 96,
+            metadata: { currentApp: "Mock", connected: String(this.connected) },
+        };
+    }
+    async capture() {
+        return {
+            width: this.width,
+            height: this.height,
+            imageBase64: TINY_PNG_BASE64,
+            imageFormat: "png",
+            uiXml: null,
+            currentApp: "Mock",
+            capturedAt: Date.now(),
+        };
+    }
+    async getUiTree() {
+        return [];
+    }
+    async apply(action) {
+        this.actions.push(action);
+    }
+    async install(_path) {
+        this.actions.push({ kind: "shell", command: `install:${_path}` });
+    }
+    async shell(command) {
+        this.actions.push({ kind: "shell", command });
+        return `mock-stdout:${randomBytes(4).toString("hex")}`;
+    }
+}
+function defaultDesktopOs() {
+    switch (process.platform) {
+        case "darwin":
+            return "desktop-macos";
+        case "win32":
+            return "desktop-windows";
+        default:
+            return "desktop-linux";
+    }
+}
+function prettyOsName(t) {
+    switch (t) {
+        case "desktop-macos":
+            return "macOS";
+        case "desktop-windows":
+            return "Windows";
+        case "desktop-linux":
+            return "Linux";
+    }
+}
+// A 1x1 transparent PNG. Constant so unit tests get deterministic output.
+const TINY_PNG_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
+//# sourceMappingURL=device-desktop.js.map

package/dist/adapters/device-desktop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"device-desktop.js","sourceRoot":"","sources":["../../src/adapters/device-desktop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAgB1C,MAAM,OAAO,cAAc;IAChB,OAAO,GAAa,EAAE,CAAC;IACf,KAAK,CAAS;IACd,MAAM,CAAS;IACf,IAAI,CAAS;IACb,MAAM,CAA2C;IAC1D,SAAS,GAAG,KAAK,CAAC;IAE1B,YAAY,OAA8B,EAAE;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC;QAClC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,gBAAgB,CAAC;QAC1C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,gBAAgB,EAAE,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,uEAAuE;QACvE,wEAAwE;QACxE,MAAM,YAAY,GAAuB;YACvC,YAAY;YACZ,KAAK;YACL,OAAO;YACP,MAAM;YACN,OAAO;SACR,CAAC;QACF,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,MAAM;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,EAAE,EAAE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY;YACZ,SAAS,EAAE,MAAM;YACjB,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE;SACpE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,OAAO;QACX,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,WAAW,EAAE,eAAe;YAC5B,WAAW,EAAE,KAAK;YAClB,KAAK,EAAE,IAAI;YACX,UAAU,EAAE,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;SACvB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS;QACb,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAAc;QACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,KAAa;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,KAAK,EAAE,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAAe;QACzB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;QAC9C,OAAO,eAAe,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IACzD,CAAC;CACF;AAED,SAAS,gBAAgB;IACvB,QAAQ,OAAO,CAAC,QAAQ,EAAE,CAAC;QACzB,KAAK,QAAQ;YACX,OAAO,eAAe,CAAC;QACzB,KAAK,OAAO;YACV,OAAO,iBAAiB,CAAC;QAC3B;YACE,OAAO,eAAe,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,CAA2C;IAC/D,QAAQ,CAAC,EAAE,CAAC;QACV,KAAK,eAAe;YAClB,OAAO,OAAO,CAAC;QACjB,KAAK,iBAAiB;YACpB,OAAO,SAAS,CAAC;QACnB,KAAK,eAAe;YAClB,OAAO,OAAO,CAAC;IACnB,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,MAAM,eAAe,GACnB,8FAA8F,CAAC"}

package/dist/adapters/identity-file.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Adapter — file-backed device identity (Ed25519).
+ *
+ * On-disk format follows the SDK convention used by `@beeos-ai/bridge-client`:
+ *   { "publicKey": "<base64>", "privateKey": "<base64>" }
+ *
+ * Default location: `~/.beeos/<id>.key`. The SDK's `loadKeyPair` is also
+ * permissive about a raw base64 32-byte private-key seed (the format used by
+ * K8s Secret mounts in production), so existing identity files keep working.
+ */
+import type { IDeviceIdentity, KeyPair } from "../ports/identity.js";
+export interface FileIdentityOptions {
+    /**
+     * Absolute path to the key file.
+     * If undefined, defaults to `~/.beeos/<id ?? "device-agent">.key`.
+     */
+    keyFile?: string;
+    /** Used to compute the default file name when `keyFile` is undefined. */
+    id?: string;
+}
+export declare class FileIdentity implements IDeviceIdentity {
+    private readonly keyFile;
+    private cached;
+    constructor(opts?: FileIdentityOptions);
+    /** Path of the underlying file (useful for diagnostics). */
+    get path(): string;
+    ensure(): Promise<KeyPair>;
+    publicKeyBase64(): Promise<string>;
+    fingerprint(): Promise<string>;
+}

package/dist/adapters/identity-file.js

@@ -0,0 +1,50 @@
+/**
+ * Adapter — file-backed device identity (Ed25519).
+ *
+ * On-disk format follows the SDK convention used by `@beeos-ai/bridge-client`:
+ *   { "publicKey": "<base64>", "privateKey": "<base64>" }
+ *
+ * Default location: `~/.beeos/<id>.key`. The SDK's `loadKeyPair` is also
+ * permissive about a raw base64 32-byte private-key seed (the format used by
+ * K8s Secret mounts in production), so existing identity files keep working.
+ */
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { fingerprint, loadKeyPair, publicKeyFromPrivate, } from "@beeos-ai/bridge-client";
+export class FileIdentity {
+    keyFile;
+    cached = null;
+    constructor(opts = {}) {
+        this.keyFile =
+            opts.keyFile ??
+                path.join(os.homedir(), ".beeos", `${opts.id ?? "device-agent"}.key`);
+    }
+    /** Path of the underlying file (useful for diagnostics). */
+    get path() {
+        return this.keyFile;
+    }
+    async ensure() {
+        if (this.cached)
+            return this.cached;
+        const dir = path.dirname(this.keyFile);
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+        // loadKeyPair generates + persists a fresh keypair when the file is missing.
+        const kp = loadKeyPair(this.keyFile);
+        if (!kp.publicKey || kp.publicKey.length === 0) {
+            kp.publicKey = publicKeyFromPrivate(kp.privateKey);
+        }
+        this.cached = kp;
+        return kp;
+    }
+    async publicKeyBase64() {
+        const kp = await this.ensure();
+        return Buffer.from(kp.publicKey).toString("base64");
+    }
+    async fingerprint() {
+        const kp = await this.ensure();
+        return fingerprint(kp.publicKey);
+    }
+}
+//# sourceMappingURL=identity-file.js.map

package/dist/adapters/identity-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"identity-file.js","sourceRoot":"","sources":["../../src/adapters/identity-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EACL,WAAW,EACX,WAAW,EACX,oBAAoB,GACrB,MAAM,yBAAyB,CAAC;AAajC,MAAM,OAAO,YAAY;IACN,OAAO,CAAS;IACzB,MAAM,GAAmB,IAAI,CAAC;IAEtC,YAAY,OAA4B,EAAE;QACxC,IAAI,CAAC,OAAO;YACV,IAAI,CAAC,OAAO;gBACZ,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,cAAc,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,4DAA4D;IAC5D,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACpC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC7E,6EAA6E;QAC7E,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/C,EAAE,CAAC,SAAS,GAAG,oBAAoB,CAAC,EAAE,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACjB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,KAAK,CAAC,eAAe;QACnB,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,WAAW;QACf,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO,WAAW,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;CACF"}

package/dist/adapters/transport-bridge-ws.d.ts

@@ -0,0 +1,39 @@
+/**
+ * Adapter — Bridge WebSocket transport.
+ *
+ * Wraps `@beeos-ai/bridge-client` (Ed25519 `agent_auth` handshake +
+ * auto-reconnect + heartbeat) and exposes the slim `IBridgeTransport` port
+ * the inbound ACP layer needs (raw string send + 4 events).
+ *
+ * Why a thin wrapper rather than re-using BridgeClient directly?
+ *  - keeps the application/inbound layers free from `events.EventEmitter`
+ *    typing leaks and lets us swap in an in-memory transport for tests
+ *  - exposes a stable surface even if the SDK adds new events
+ */
+import { type BridgeClientOptions } from "@beeos-ai/bridge-client";
+import type { IBridgeTransport } from "../ports/transport.js";
+export interface BridgeWsTransportOptions {
+    /** Bridge base URL, e.g. `wss://bridge.beeos.ai:18443`. */
+    bridgeUrl: string;
+    /** ACP service name registered on Bridge (default `acp`). */
+    service?: string;
+    /** Path to identity key file or pre-loaded keys. */
+    keyFile?: string;
+    privateKey?: Uint8Array;
+    publicKey?: Uint8Array;
+    /** Override reconnect / heartbeat policy. */
+    reconnect?: BridgeClientOptions["reconnect"];
+    heartbeat?: BridgeClientOptions["heartbeat"];
+    proxy?: string;
+}
+export declare class BridgeWsTransport implements IBridgeTransport {
+    private readonly client;
+    constructor(opts: BridgeWsTransportOptions);
+    connect(): Promise<void>;
+    send(data: string): Promise<void>;
+    close(): Promise<void>;
+    on(event: "connected", handler: () => void): void;
+    on(event: "disconnected", handler: (reason: string) => void): void;
+    on(event: "message", handler: (data: string) => void | Promise<void>): void;
+    on(event: "error", handler: (err: Error) => void): void;
+}

package/dist/adapters/transport-bridge-ws.js

@@ -0,0 +1,49 @@
+/**
+ * Adapter — Bridge WebSocket transport.
+ *
+ * Wraps `@beeos-ai/bridge-client` (Ed25519 `agent_auth` handshake +
+ * auto-reconnect + heartbeat) and exposes the slim `IBridgeTransport` port
+ * the inbound ACP layer needs (raw string send + 4 events).
+ *
+ * Why a thin wrapper rather than re-using BridgeClient directly?
+ *  - keeps the application/inbound layers free from `events.EventEmitter`
+ *    typing leaks and lets us swap in an in-memory transport for tests
+ *  - exposes a stable surface even if the SDK adds new events
+ */
+import { BridgeClient } from "@beeos-ai/bridge-client";
+export class BridgeWsTransport {
+    client;
+    constructor(opts) {
+        this.client = new BridgeClient({
+            bridgeUrl: opts.bridgeUrl,
+            service: opts.service ?? "acp",
+            keyFile: opts.keyFile,
+            privateKey: opts.privateKey,
+            publicKey: opts.publicKey,
+            reconnect: opts.reconnect,
+            heartbeat: opts.heartbeat,
+            proxy: opts.proxy,
+        });
+    }
+    async connect() {
+        await this.client.connect();
+    }
+    async send(data) {
+        await this.client.send(data);
+    }
+    async close() {
+        await this.client.close(true);
+    }
+    on(event, handler) {
+        if (event === "message") {
+            this.client.on("message", (data) => {
+                const str = typeof data === "string" ? data : Buffer.from(data).toString();
+                return handler(str);
+            });
+            return;
+        }
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        this.client.on(event, handler);
+    }
+}
+//# sourceMappingURL=transport-bridge-ws.js.map

package/dist/adapters/transport-bridge-ws.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"transport-bridge-ws.js","sourceRoot":"","sources":["../../src/adapters/transport-bridge-ws.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAA4B,MAAM,yBAAyB,CAAC;AAkBjF,MAAM,OAAO,iBAAiB;IACX,MAAM,CAAe;IAEtC,YAAY,IAA8B;QACxC,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;YAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,KAAK;YAC9B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;SAClB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAY;QACrB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAMD,EAAE,CAAC,KAAa,EAAE,OAAmD;QACnE,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,EAAE;gBACjC,MAAM,GAAG,GAAG,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC3E,OAAQ,OAA+C,CAAC,GAAG,CAAC,CAAC;YAC/D,CAAC,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,8DAA8D;QAC9D,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,KAAY,EAAE,OAAc,CAAC,CAAC;IAC/C,CAAC;CACF"}

package/dist/adapters/vlm-inner-loop.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Adapter — IVlm implementation backed by the existing inner VLM loop.
+ *
+ * The repo already ships a battle-tested screenshot→reason→act loop in
+ * `src/agent.ts` (`Agent`) plus a configurable `OpenAIClient` that picks the
+ * right `ModelAdapter` based on the model name (`gpt-4o`, `qwen-vl`,
+ * `ui-tars`, `gui-owl`, `mai-ui`, …).
+ *
+ * This file exposes that loop through the `IVlm` port. We do NOT re-implement
+ * the per-model parsing — instead we delegate, so the rich existing parsers /
+ * grounders / coordinate-system handling stay intact.
+ *
+ * The application layer therefore only sees one stable interface (`IVlm`) and
+ * stays decoupled from the OpenAI SDK and from per-model peculiarities.
+ */
+import type { ModelClient } from "../model/base.js";
+import type { IDevice } from "../ports/device.js";
+import type { IVlm } from "../ports/vlm.js";
+import type { VlmStep } from "../ports/agent-core.js";
+import type { Device } from "../device/base.js";
+export interface InnerLoopVlmOptions {
+    /** Model name forwarded to OpenAIClient (e.g. `gpt-4o`, `ui-tars-7b`). */
+    modelName?: string;
+    baseUrl?: string;
+    apiKey?: string;
+    maxSteps?: number;
+    lang?: "cn" | "en";
+    verbose?: boolean;
+    /**
+     * Caller can pre-construct a ModelClient for fully custom config; otherwise
+     * we instantiate a default OpenAIClient with adapter auto-detection.
+     */
+    modelClient?: ModelClient;
+}
+/**
+ * Wraps Agent + OpenAIClient as an IVlm. The IDevice **must** be one of the
+ * adapters in this package because the legacy Agent expects the old `Device`
+ * interface — we expose `.raw` on production adapters for that.
+ */
+export declare class InnerLoopVlm implements IVlm {
+    private agent;
+    private client;
+    private readonly inner;
+    private readonly cfg;
+    private readonly baseUrl?;
+    private readonly apiKey?;
+    private provider;
+    constructor(device: IDevice, opts?: InnerLoopVlmOptions);
+    get modelName(): string;
+    get providerName(): string;
+    /**
+     * Rebuild the underlying OpenAIClient + Agent with a new provider/model.
+     *
+     * `provider` is forwarded as the explicit adapter selector, so callers can
+     * force e.g. the `qwen-vl` adapter even when the model id alone would not
+     * disambiguate. Any in-flight `runTask` call keeps using the old client; the
+     * new client takes over from the next task.
+     */
+    setProvider(provider: string, model: string): Promise<void>;
+    runTask(opts: {
+        task: string;
+        maxSteps: number;
+        onText?: (text: string) => void | Promise<void>;
+        onStep?: (step: VlmStep & {
+            stepIndex: number;
+        }) => void | Promise<void>;
+        abortSignal?: AbortSignal;
+    }): Promise<{
+        finalMessage: string;
+        steps: number;
+        cancelled: boolean;
+    }>;
+}
+/**
+ * Best-effort `IDevice` → legacy `Device` unwrap.
+ *
+ * Production adapters expose a `.raw` of the legacy `Device` shape:
+ *   - `AndroidAdbAdapter.raw` → real `AndroidDevice` (full UI XML, etc.)
+ *   - `DesktopAdapter`        → no `.raw`; we synthesise an inline shim
+ *
+ * Tests / harness scripts that pass arbitrary `IDevice` stubs may also
+ * supply a `.raw` matching the legacy shape — that path is honoured too.
+ *
+ * Exported so callers (bootstrap, harness scripts) that build an
+ * `InProcMcpClient` from an `IDevice` get exactly the same behaviour the
+ * legacy `Agent` had with the same input.
+ */
+export declare function unwrapInnerDevice(d: IDevice): Device;

package/dist/adapters/vlm-inner-loop.js

@@ -0,0 +1,199 @@
+/**
+ * Adapter — IVlm implementation backed by the existing inner VLM loop.
+ *
+ * The repo already ships a battle-tested screenshot→reason→act loop in
+ * `src/agent.ts` (`Agent`) plus a configurable `OpenAIClient` that picks the
+ * right `ModelAdapter` based on the model name (`gpt-4o`, `qwen-vl`,
+ * `ui-tars`, `gui-owl`, `mai-ui`, …).
+ *
+ * This file exposes that loop through the `IVlm` port. We do NOT re-implement
+ * the per-model parsing — instead we delegate, so the rich existing parsers /
+ * grounders / coordinate-system handling stay intact.
+ *
+ * The application layer therefore only sees one stable interface (`IVlm`) and
+ * stays decoupled from the OpenAI SDK and from per-model peculiarities.
+ */
+import { Agent } from "../agent.js";
+import { OpenAIClient } from "../model/client.js";
+import { VlmError } from "../core/errors.js";
+import { AndroidAdbAdapter } from "./device-android-adb.js";
+import { DesktopAdapter } from "./device-desktop.js";
+/**
+ * Wraps Agent + OpenAIClient as an IVlm. The IDevice **must** be one of the
+ * adapters in this package because the legacy Agent expects the old `Device`
+ * interface — we expose `.raw` on production adapters for that.
+ */
+export class InnerLoopVlm {
+    agent;
+    client;
+    inner;
+    cfg;
+    baseUrl;
+    apiKey;
+    provider = "";
+    constructor(device, opts = {}) {
+        this.inner = unwrapInnerDevice(device);
+        this.cfg = {
+            maxSteps: opts.maxSteps,
+            lang: opts.lang,
+            verbose: opts.verbose,
+        };
+        this.baseUrl = opts.baseUrl;
+        this.apiKey = opts.apiKey;
+        this.client = opts.modelClient ?? new OpenAIClient({
+            modelName: opts.modelName,
+            baseUrl: opts.baseUrl,
+            apiKey: opts.apiKey,
+        });
+        this.agent = new Agent(this.client, this.inner, this.cfg);
+    }
+    get modelName() {
+        return this.client.modelName;
+    }
+    get providerName() {
+        return this.provider;
+    }
+    /**
+     * Rebuild the underlying OpenAIClient + Agent with a new provider/model.
+     *
+     * `provider` is forwarded as the explicit adapter selector, so callers can
+     * force e.g. the `qwen-vl` adapter even when the model id alone would not
+     * disambiguate. Any in-flight `runTask` call keeps using the old client; the
+     * new client takes over from the next task.
+     */
+    async setProvider(provider, model) {
+        if (!model)
+            throw new VlmError("setProvider: model is required");
+        const adapter = (provider || "").trim() || undefined;
+        this.client = new OpenAIClient({
+            modelName: model,
+            baseUrl: this.baseUrl,
+            apiKey: this.apiKey,
+            adapter,
+        });
+        this.agent = new Agent(this.client, this.inner, this.cfg);
+        this.provider = adapter ?? "";
+    }
+    async runTask(opts) {
+        if (opts.abortSignal?.aborted) {
+            return { finalMessage: "Cancelled before start.", steps: 0, cancelled: true };
+        }
+        try {
+            const result = await this.agent.run(opts.task, {
+                signal: opts.abortSignal,
+                maxSteps: opts.maxSteps,
+                onText: opts.onText,
+                onStep: opts.onStep
+                    ? async (step) => {
+                        await opts.onStep({ ...mapStep(step), stepIndex: step.index });
+                    }
+                    : undefined,
+            });
+            return {
+                finalMessage: result.finalText,
+                steps: result.steps,
+                cancelled: result.cancelled,
+            };
+        }
+        catch (e) {
+            if (opts.abortSignal?.aborted) {
+                return { finalMessage: "Cancelled.", steps: 0, cancelled: true };
+            }
+            throw new VlmError(`VLM task failed: ${e.message}`, e);
+        }
+    }
+}
+/**
+ * Project an internal `AgentStepRecord` (action shape: `{ actionType,
+ * params }`) onto the public `VlmStep` shape (action shape: discriminated
+ * `Action` union). The IVlm port treats `action` as opaque to the host —
+ * so a best-effort mapping that preserves `actionType` as `kind` and
+ * spreads params is enough for downstream consumers.
+ */
+function mapStep(step) {
+    const action = step.action
+        ? { kind: step.action.actionType, ...step.action.params }
+        : null;
+    return {
+        thinking: step.thinking,
+        action,
+        finished: step.finished,
+        message: step.message,
+    };
+}
+/**
+ * Best-effort `IDevice` → legacy `Device` unwrap.
+ *
+ * Production adapters expose a `.raw` of the legacy `Device` shape:
+ *   - `AndroidAdbAdapter.raw` → real `AndroidDevice` (full UI XML, etc.)
+ *   - `DesktopAdapter`        → no `.raw`; we synthesise an inline shim
+ *
+ * Tests / harness scripts that pass arbitrary `IDevice` stubs may also
+ * supply a `.raw` matching the legacy shape — that path is honoured too.
+ *
+ * Exported so callers (bootstrap, harness scripts) that build an
+ * `InProcMcpClient` from an `IDevice` get exactly the same behaviour the
+ * legacy `Agent` had with the same input.
+ */
+export function unwrapInnerDevice(d) {
+    if (d instanceof AndroidAdbAdapter)
+        return d.raw;
+    if (d instanceof DesktopAdapter)
+        return innerDesktopShim(d);
+    // Fallback: any IDevice with a public `.raw` of the legacy Device shape.
+    const raw = d.raw;
+    if (raw && typeof raw.takeScreenshot === "function")
+        return raw;
+    throw new VlmError("unwrapInnerDevice requires an IDevice that exposes a legacy `Device` driver via .raw (AndroidAdbAdapter / DesktopAdapter).");
+}
+/**
+ * Build a minimal legacy `Device` shim for the desktop mock so the inner Agent
+ * loop can still run end-to-end against it (tests, demos).
+ */
+function innerDesktopShim(d) {
+    return {
+        async takeScreenshot() {
+            const o = await d.capture();
+            return { base64Data: o.imageBase64, width: o.width, height: o.height };
+        },
+        async getCurrentApp() {
+            const i = await d.getInfo();
+            return i.metadata.currentApp ?? "Mock";
+        },
+        async getUiXml() {
+            return null;
+        },
+        async tap(x, y) {
+            await d.apply({ kind: "tap", x, y });
+        },
+        async doubleTap(x, y) {
+            await d.apply({ kind: "double_tap", x, y });
+        },
+        async longPress(x, y, durationMs) {
+            await d.apply({ kind: "long_press", x, y, durationMs });
+        },
+        async swipe(x1, y1, x2, y2, durationMs) {
+            await d.apply({ kind: "swipe", x1, y1, x2, y2, durationMs });
+        },
+        async inputText(text) {
+            await d.apply({ kind: "type", text });
+        },
+        async pressKey(keycode) {
+            await d.apply({ kind: "press_key", keycode });
+        },
+        async launchApp(name) {
+            await d.apply({ kind: "open_app", packageOrName: name });
+            return true;
+        },
+        async pressBack() {
+            await d.apply({ kind: "press_back" });
+        },
+        async pressHome() {
+            await d.apply({ kind: "press_home" });
+        },
+        async clearText() {
+            // no-op in mock
+        },
+    };
+}
+//# sourceMappingURL=vlm-inner-loop.js.map

package/dist/adapters/vlm-inner-loop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vlm-inner-loop.js","sourceRoot":"","sources":["../../src/adapters/vlm-inner-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,KAAK,EAAwB,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAMlD,OAAO,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAkBrD;;;;GAIG;AACH,MAAM,OAAO,YAAY;IACf,KAAK,CAAQ;IACb,MAAM,CAAc;IACX,KAAK,CAAS;IACd,GAAG,CAA+D;IAClE,OAAO,CAAU;IACjB,MAAM,CAAU;IACzB,QAAQ,GAAW,EAAE,CAAC;IAE9B,YAAY,MAAe,EAAE,OAA4B,EAAE;QACzD,IAAI,CAAC,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,GAAG,GAAG;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,YAAY,CAAC;YACjD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC;IAC/B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,QAAgB,EAAE,KAAa;QAC/C,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,QAAQ,CAAC,gCAAgC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QACrD,IAAI,CAAC,MAAM,GAAG,IAAI,YAAY,CAAC;YAC7B,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;SACR,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAMb;QACC,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;YAC9B,OAAO,EAAE,YAAY,EAAE,yBAAyB,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAChF,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE;gBAC7C,MAAM,EAAE,IAAI,CAAC,WAAW;gBACxB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACjB,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;wBACb,MAAM,IAAI,CAAC,MAAO,CAAC,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;oBAClE,CAAC;oBACH,CAAC,CAAC,SAAS;aACd,CAAC,CAAC;YACH,OAAO;gBACL,YAAY,EAAE,MAAM,CAAC,SAAS;gBAC9B,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,CAAC;gBAC9B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YACnE,CAAC;YACD,MAAM,IAAI,QAAQ,CAAC,oBAAqB,CAAW,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;CACF;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,IAAqB;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;QACxB,CAAC,CAAE,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAwB;QAChF,CAAC,CAAC,IAAI,CAAC;IACT,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM;QACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,OAAO,EAAE,IAAI,CAAC,OAAO;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,iBAAiB,CAAC,CAAU;IAC1C,IAAI,CAAC,YAAY,iBAAiB;QAAE,OAAO,CAAC,CAAC,GAAG,CAAC;IACjD,IAAI,CAAC,YAAY,cAAc;QAAE,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC;IAC5D,yEAAyE;IACzE,MAAM,GAAG,GAAI,CAAiC,CAAC,GAAG,CAAC;IACnD,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,cAAc,KAAK,UAAU;QAAE,OAAO,GAAG,CAAC;IAChE,MAAM,IAAI,QAAQ,CAChB,4HAA4H,CAC7H,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,CAAiB;IACzC,OAAO;QACL,KAAK,CAAC,cAAc;YAClB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;YAC5B,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACzE,CAAC;QACD,KAAK,CAAC,aAAa;YACjB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,QAAQ;YACZ,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACZ,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACvC,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;YAClB,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9C,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU;YAC9B,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC;QAC1D,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU;YACpC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI;YAClB,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,CAAC,QAAQ,CAAC,OAAO;YACpB,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,IAAI;YAClB,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,KAAK,CAAC,SAAS;YACb,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,CAAC,SAAS;YACb,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,CAAC,SAAS;YACb,gBAAgB;QAClB,CAAC;KACF,CAAC;AACJ,CAAC"}