@beeos-ai/device-agent 0.4.0

package/dist/agent-runtime/adapters/base.d.ts

@@ -0,0 +1,145 @@
+/**
+ * agent-runtime/adapters/base — minimal contract every model adapter speaks.
+ *
+ * Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/adapters/base.py
+ *
+ * Each adapter is a stateless* function from `(Observation, StepHistory[])`
+ * → `Prediction`, where `Prediction.action` is the unified `Action` schema
+ * the MCP server understands. Adapters MUST NOT touch the device or the
+ * MCP client themselves — they only translate observations into actions.
+ *
+ * (* "stateless" only WRT the agent loop: an adapter MAY hold its own LLM
+ * client, prompt cache, etc. as long as it tolerates concurrent runs by
+ * different `AgentLoop` instances.)
+ */
+import type { Action, Observation } from "@beeos-ai/device-common";
+/**
+ * Token / cost accounting reported by adapters per prediction. Aggregated by
+ * `AgentLoop` over a single run and surfaced via `AgentRunResult.usage`.
+ */
+export interface ModelUsage {
+    inputTokens: number;
+    outputTokens: number;
+    /** Cached prompt tokens, when the provider exposes them (OpenAI / Anthropic). */
+    cachedInputTokens?: number;
+    /** Approximate USD cost when the adapter knows the price; otherwise undefined. */
+    costUsd?: number;
+    /** Wall-clock time of the underlying model request. */
+    durationMs?: number;
+}
+/**
+ * One entry in the rolling history `AgentLoop` keeps so adapters can ground
+ * their next prediction on previous turns. The history is intentionally
+ * narrow — adapters that need richer context (full message arrays, image
+ * histories, scratchpad memory) should keep that internally.
+ */
+/**
+ * Outcome of dispatching this step's action. The `AgentLoop` writes one of:
+ *  - `"ok"`           — action dispatched to MCP and returned without error
+ *  - `"act_failed"`   — MCP `/act` rejected (HTTP error, ADB error, …)
+ *  - `"predict_failed"` — model adapter threw (no action produced)
+ *  - `"skipped"`      — pre-action sanity check vetoed dispatch
+ *  - `"terminal"`     — `done` / `fail` / `call_for_user` short-circuited the loop
+ *
+ * Used by reflectors and the planner replan path to decide whether to
+ * fire follow-up reasoning. Optional for backwards compatibility — older
+ * test fixtures predate the field.
+ */
+export type StepOutcome = "ok" | "act_failed" | "predict_failed" | "skipped" | "terminal";
+export interface StepHistory {
+    /** 1-based step index. */
+    stepIdx: number;
+    /** Action emitted in this step (`null` only for the very first observation). */
+    action: Action | null;
+    /** Free-form rationale or thinking string from the adapter. */
+    rationale: string;
+    /**
+     * Short summary of what the screen looked like (e.g. "Home screen") so
+     * adapters that don't keep image history can still reason about progress.
+     */
+    observationSummary: string;
+    /**
+     * Outcome of this step's action dispatch. See {@link StepOutcome}.
+     * Optional so existing test fixtures keep validating; new code paths
+     * (reflectors, replan context) MUST populate it.
+     */
+    actionResult?: StepOutcome;
+    /**
+     * Adapter-reported confidence in `[0, 1]` for this step's prediction.
+     * Copied from `Prediction.confidence`. Optional — adapters that can't
+     * estimate confidence (no logprob support) leave it undefined and the
+     * reflector falls back to its whitelist-only trigger.
+     */
+    confidence?: number;
+    /**
+     * Populated only when `actionResult === "act_failed"` — the MCP error
+     * message / status. Reflectors splice this into their prompt so the
+     * grounder can see *why* the previous step failed on retry.
+     */
+    actErrorMessage?: string;
+}
+export interface Prediction {
+    /** Predicted next action. Use `done` / `fail` / `call_for_user` to stop. */
+    action: Action;
+    /** Free-form thinking / reasoning string surfaced via hooks. */
+    rationale: string;
+    /** Adapter's self-reported confidence in [0, 1]; 0.5 when unknown. */
+    confidence: number;
+    /** Token / cost accounting for this single prediction. */
+    usage: ModelUsage;
+    /** Raw provider response (model output / tool-call payload). For debug. */
+    rawResponse: string;
+}
+/**
+ * Optional hooks an adapter may call during a long inference (streaming
+ * tokens, tool-call detected). Wired through to the AgentLoop's `onText`
+ * / `onReasoning` hooks.
+ */
+export interface PredictHooks {
+    /** Streaming token delta from the underlying model. Best-effort. */
+    onText?: (delta: string) => void | Promise<void>;
+    /** Reasoning tokens (separate channel for OpenAI o1-style models). */
+    onReasoning?: (delta: string) => void | Promise<void>;
+}
+export interface PredictOptions {
+    signal?: AbortSignal;
+    hooks?: PredictHooks;
+}
+export interface IModelAdapter {
+    /** Stable, lower-case identifier used by the registry (e.g. `openai`). */
+    readonly name: string;
+    /** Human-readable model identifier the adapter is configured against. */
+    readonly modelName?: string;
+    /**
+     * Whether this adapter consumes the per-step `Observation.uiDump`. When
+     * `false` (default — matches Open-AutoGLM upstream pure-vision flow,
+     * `phone_agent/agent.py:142-169` + `model/client.py:277-290` only feed
+     * `screenshot + {"current_app": ...}` text), the `AgentLoop` skips the
+     * `mcp.uiDump()` round-trip in `observe()`, saving 1–3 s per step on
+     * Android (uiautomator dump is not free on large apps).
+     *
+     * Set to `true` for adapters that splice the UI hierarchy XML into the
+     * model prompt (e.g. GUI-Owl, Qwen-VL with XML fine-tune, generic
+     * GPT-4V chains driven by HTML-style hierarchies).
+     */
+    readonly consumesUiDump?: boolean;
+    /**
+     * Predict the next `Action` given the current observation and history.
+     *
+     * Implementations SHOULD:
+     *   - tolerate `signal.aborted` via `AbortSignal.throwIfAborted()` on slow paths
+     *   - return a `wait` / `done` action rather than throwing for soft failures
+     *   - report token usage when the provider surfaces it
+     */
+    predictAction(observation: Observation, history: StepHistory[], opts?: PredictOptions): Promise<Prediction>;
+    /** Optional capability check; `undefined` means "trust the registry". */
+    supportsEnv?(env: Observation["env"]): boolean;
+    /** Release any underlying resources (HTTP agents, sockets, model pools). */
+    close?(): Promise<void>;
+}
+/**
+ * Convenience constructor — adapters that produce a fully-typed Action can
+ * skip filling in the boilerplate `usage`/`rawResponse` defaults.
+ */
+export declare function makePrediction(action: Action, init?: Partial<Omit<Prediction, "action">>): Prediction;

package/dist/agent-runtime/adapters/base.js

@@ -0,0 +1,29 @@
+/**
+ * agent-runtime/adapters/base — minimal contract every model adapter speaks.
+ *
+ * Mirrors the Python reference at
+ *   /Users/heat/workspace/deviceagent_research/deviceagent/agent_runtime/adapters/base.py
+ *
+ * Each adapter is a stateless* function from `(Observation, StepHistory[])`
+ * → `Prediction`, where `Prediction.action` is the unified `Action` schema
+ * the MCP server understands. Adapters MUST NOT touch the device or the
+ * MCP client themselves — they only translate observations into actions.
+ *
+ * (* "stateless" only WRT the agent loop: an adapter MAY hold its own LLM
+ * client, prompt cache, etc. as long as it tolerates concurrent runs by
+ * different `AgentLoop` instances.)
+ */
+/**
+ * Convenience constructor — adapters that produce a fully-typed Action can
+ * skip filling in the boilerplate `usage`/`rawResponse` defaults.
+ */
+export function makePrediction(action, init = {}) {
+    return {
+        action,
+        rationale: init.rationale ?? "",
+        confidence: init.confidence ?? action.confidence ?? 0.5,
+        usage: init.usage ?? { inputTokens: 0, outputTokens: 0 },
+        rawResponse: init.rawResponse ?? "",
+    };
+}
+//# sourceMappingURL=base.js.map

package/dist/agent-runtime/adapters/base.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"base.js","sourceRoot":"","sources":["../../../src/agent-runtime/adapters/base.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA8IH;;;GAGG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAc,EACd,OAA4C,EAAE;IAE9C,OAAO;QACL,MAAM;QACN,SAAS,EAAE,IAAI,CAAC,SAAS,IAAI,EAAE;QAC/B,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,IAAI,GAAG;QACvD,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;QACxD,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE;KACpC,CAAC;AACJ,CAAC"}

package/dist/agent-runtime/adapters/index.d.ts

@@ -0,0 +1,17 @@
+/**
+ * agent-runtime/adapters — barrel + side-effect import to populate the
+ * registry with the built-in adapters.
+ *
+ * Import this file (or any of its consumers, like `agent-runtime/core.ts`)
+ * exactly once at process start to ensure `mock`, `openai` and `ui-tars`
+ * are visible to `resolveAdapter`.
+ */
+export type { IModelAdapter, ModelUsage, PredictHooks, PredictOptions, Prediction, StepHistory, } from "./base.js";
+export { makePrediction } from "./base.js";
+export { available as availableAdapters, clearAdapters, registerAdapter, resolveAdapter, resolveAdapterSync, type AdapterDescriptor, } from "./registry.js";
+export { MockAdapter, type MockAdapterOptions } from "./mock.js";
+export { OpenAIAdapter, parsedToAction, type OpenAIAdapterOptions } from "./openai.js";
+export { UITarsAdapter } from "./ui-tars.js";
+import "./mock.js";
+import "./openai.js";
+import "./ui-tars.js";

package/dist/agent-runtime/adapters/index.js

@@ -0,0 +1,17 @@
+/**
+ * agent-runtime/adapters — barrel + side-effect import to populate the
+ * registry with the built-in adapters.
+ *
+ * Import this file (or any of its consumers, like `agent-runtime/core.ts`)
+ * exactly once at process start to ensure `mock`, `openai` and `ui-tars`
+ * are visible to `resolveAdapter`.
+ */
+export { makePrediction } from "./base.js";
+export { available as availableAdapters, clearAdapters, registerAdapter, resolveAdapter, resolveAdapterSync, } from "./registry.js";
+export { MockAdapter } from "./mock.js";
+export { OpenAIAdapter, parsedToAction } from "./openai.js";
+export { UITarsAdapter } from "./ui-tars.js";
+import "./mock.js";
+import "./openai.js";
+import "./ui-tars.js";
+//# sourceMappingURL=index.js.map

package/dist/agent-runtime/adapters/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/agent-runtime/adapters/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAUH,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EACL,SAAS,IAAI,iBAAiB,EAC9B,aAAa,EACb,eAAe,EACf,cAAc,EACd,kBAAkB,GAEnB,MAAM,eAAe,CAAC;AAEvB,OAAO,EAAE,WAAW,EAA2B,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,aAAa,EAAE,cAAc,EAA6B,MAAM,aAAa,CAAC;AACvF,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAE7C,OAAO,WAAW,CAAC;AACnB,OAAO,aAAa,CAAC;AACrB,OAAO,cAAc,CAAC"}

package/dist/agent-runtime/adapters/mock.d.ts

@@ -0,0 +1,44 @@
+/**
+ * agent-runtime/adapters/mock — deterministic adapter for tests + dry-runs.
+ *
+ * Mirrors `agent_runtime/adapters/mock.py` from the Python reference.
+ *
+ * The adapter walks a queued script of `Action`s; once exhausted it emits
+ * `done` with a generated final-answer string. This makes AgentLoop tests
+ * fully deterministic without touching real LLMs.
+ */
+import { type Action, type Observation } from "@beeos-ai/device-common";
+import type { IModelAdapter, Prediction, StepHistory } from "./base.js";
+export interface MockAdapterOptions {
+    /** Pre-baked actions to emit, in order. */
+    script?: Action[];
+    /**
+     * Final action to emit once `script` is exhausted. Defaults to a `done`
+     * with `{ message: "mock-finished" }`.
+     */
+    fallback?: Action;
+    /** Optional rationale prefix (e.g. "[mock]") prepended to each step. */
+    rationalePrefix?: string;
+    name?: string;
+    /**
+     * When `true`, the `AgentLoop` will fetch `mcp.uiDump()` per step. Defaults
+     * to `false` to mirror the upstream Open-AutoGLM pure-vision flow.
+     */
+    consumesUiDump?: boolean;
+}
+export declare class MockAdapter implements IModelAdapter {
+    readonly name: string;
+    readonly modelName = "mock";
+    readonly consumesUiDump: boolean;
+    private readonly script;
+    private readonly fallback;
+    private readonly rationalePrefix;
+    private cursor;
+    constructor(opts?: MockAdapterOptions);
+    /** Append additional actions to the running script (test helper). */
+    enqueue(...actions: Action[]): void;
+    /** Reset the script cursor (test helper). */
+    reset(): void;
+    supportsEnv(): boolean;
+    predictAction(_observation: Observation, history: StepHistory[]): Promise<Prediction>;
+}

package/dist/agent-runtime/adapters/mock.js

@@ -0,0 +1,54 @@
+/**
+ * agent-runtime/adapters/mock — deterministic adapter for tests + dry-runs.
+ *
+ * Mirrors `agent_runtime/adapters/mock.py` from the Python reference.
+ *
+ * The adapter walks a queued script of `Action`s; once exhausted it emits
+ * `done` with a generated final-answer string. This makes AgentLoop tests
+ * fully deterministic without touching real LLMs.
+ */
+import { makeAction } from "@beeos-ai/device-common";
+import { makePrediction } from "./base.js";
+import { registerAdapter } from "./registry.js";
+export class MockAdapter {
+    name;
+    modelName = "mock";
+    consumesUiDump;
+    script;
+    fallback;
+    rationalePrefix;
+    cursor = 0;
+    constructor(opts = {}) {
+        this.name = opts.name ?? "mock";
+        this.script = [...(opts.script ?? [])];
+        this.fallback = opts.fallback ?? makeAction("done", { payload: { message: "mock-finished" } });
+        this.rationalePrefix = opts.rationalePrefix ?? "";
+        this.consumesUiDump = opts.consumesUiDump ?? false;
+    }
+    /** Append additional actions to the running script (test helper). */
+    enqueue(...actions) {
+        this.script.push(...actions);
+    }
+    /** Reset the script cursor (test helper). */
+    reset() {
+        this.cursor = 0;
+    }
+    supportsEnv() {
+        return true;
+    }
+    async predictAction(_observation, history) {
+        const action = this.cursor < this.script.length ? this.script[this.cursor++] : structuredClone(this.fallback);
+        const rationale = `${this.rationalePrefix}step=${history.length + 1} action=${action.type}`.trim();
+        return makePrediction(action, {
+            rationale,
+            confidence: action.confidence ?? 1,
+            usage: { inputTokens: 0, outputTokens: 0 },
+            rawResponse: JSON.stringify(action),
+        });
+    }
+}
+registerAdapter({
+    name: "mock",
+    factory: () => new MockAdapter(),
+});
+//# sourceMappingURL=mock.js.map

package/dist/agent-runtime/adapters/mock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mock.js","sourceRoot":"","sources":["../../../src/agent-runtime/adapters/mock.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAiC,MAAM,yBAAyB,CAAC;AAGpF,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAoBhD,MAAM,OAAO,WAAW;IACb,IAAI,CAAS;IACb,SAAS,GAAG,MAAM,CAAC;IACnB,cAAc,CAAU;IAEhB,MAAM,CAAW;IACjB,QAAQ,CAAS;IACjB,eAAe,CAAS;IACjC,MAAM,GAAG,CAAC,CAAC;IAEnB,YAAY,OAA2B,EAAE;QACvC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,KAAK,CAAC;IACrD,CAAC;IAED,qEAAqE;IACrE,OAAO,CAAC,GAAG,OAAiB;QAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IAC/B,CAAC;IAED,6CAA6C;IAC7C,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,YAAyB,EAAE,OAAsB;QACnE,MAAM,MAAM,GACV,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACjG,MAAM,SAAS,GACb,GAAG,IAAI,CAAC,eAAe,QAAQ,OAAO,CAAC,MAAM,GAAG,CAAC,WAAW,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACnF,OAAO,cAAc,CAAC,MAAM,EAAE;YAC5B,SAAS;YACT,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,CAAC;YAClC,KAAK,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE;YAC1C,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;SACpC,CAAC,CAAC;IACL,CAAC;CACF;AAED,eAAe,CAAC;IACd,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,WAAW,EAAE;CACjC,CAAC,CAAC"}

package/dist/agent-runtime/adapters/openai.d.ts

@@ -0,0 +1,63 @@
+/**
+ * agent-runtime/adapters/openai — wraps the legacy `OpenAIClient` (which
+ * dispatches to the `model/adapters/*` registry) and exposes it under the
+ * new `IModelAdapter` contract.
+ *
+ * The translation has two parts:
+ *  - request side: build legacy `Message[]` from the new `Observation` +
+ *    `StepHistory` rolling context, then call `client.request()`.
+ *  - response side: call `client.parseResponse()` to recover the legacy
+ *    `ParsedAction`, then map onto the unified `Action` schema via
+ *    `parsedToAction()`. Coordinate-system metadata is preserved on
+ *    `payload._coordSystem` so the `AgentLoop` can scale to actual screen
+ *    pixels before forwarding to the MCP `/act` route.
+ */
+import { type Action, type Observation } from "@beeos-ai/device-common";
+import { type ModelConfig } from "../../model/client.js";
+import type { ModelClient } from "../../model/base.js";
+import { type ImageTransport } from "../../model/image/transport.js";
+import type { ParsedAction } from "../../types.js";
+import type { IModelAdapter, PredictOptions, Prediction, StepHistory } from "./base.js";
+export interface OpenAIAdapterOptions extends ModelConfig {
+    /**
+     * Force a specific legacy model adapter (`autoglm`, `uitars`, `qwenvl`, ...)
+     * — overrides the auto-detection that runs against `modelName`.
+     */
+    legacyAdapter?: string;
+    /** Display language for system prompts (forwarded to legacy adapters). */
+    lang?: "cn" | "en";
+    /**
+     * Reuse an existing ModelClient instead of constructing a new OpenAIClient.
+     * Used by the legacy `Agent` shim to avoid double allocation when an
+     * embedder already configured a fully custom client.
+     */
+    client?: ModelClient;
+    /**
+     * Strategy for delivering compressed VLM images to the model. Defaults
+     * to `DataUrlTransport` (inline base64) — boot-time wire-up may swap
+     * in `HttpUrlTransport` / `FallbackTransport` per fleet config.
+     */
+    imageTransport?: ImageTransport;
+    /**
+     * Override the `consumesUiDump` capability flag derived from the legacy
+     * adapter name. Useful when a downstream embedder feeds a UI-XML-aware
+     * system prompt into a generic VLM (e.g. GPT-4V) and wants the
+     * `AgentLoop` to populate `Observation.uiDump` for it.
+     */
+    consumesUiDump?: boolean;
+}
+export declare class OpenAIAdapter implements IModelAdapter {
+    readonly name: string;
+    readonly modelName: string;
+    readonly consumesUiDump: boolean;
+    private readonly client;
+    private readonly lang;
+    private readonly imageTransport;
+    private context;
+    private lastStepIdx;
+    constructor(opts?: OpenAIAdapterOptions);
+    supportsEnv(env: Observation["env"]): boolean;
+    predictAction(observation: Observation, history: StepHistory[], opts?: PredictOptions): Promise<Prediction>;
+    close(): Promise<void>;
+}
+export declare function parsedToAction(parsed: ParsedAction): Action;