@beeos-ai/device-agent 0.4.0

package/dist/adapters/agentcore-inner-loop.js

@@ -0,0 +1,72 @@
+/**
+ * Adapter — InnerLoopAgent (single-layer fallback).
+ *
+ * Wraps the existing `InnerLoopVlm` (which already implements the legacy
+ * `IVlm` port and drives the screenshot→reason→act loop in `agent.ts`) so
+ * that it can be plugged into the `IAgentCore` slot.
+ *
+ * Selected by `--legacy-inner` / `BEEOS_LEGACY_INNER=true`. Useful when
+ * the pi-coding-agent two-layer adapter isn't available (offline, custom
+ * model with no orchestrator support, …) — the agent still works as an
+ * end-to-end VLM controller, just without an outer text LLM picking
+ * between tools.
+ *
+ * The adapter:
+ *   - re-emits each VlmStep both as `onStep` (legacy compat) and as a
+ *     synthetic `ToolCallEvent` (`name = action.kind`) so consumers that
+ *     only care about tool calls keep working.
+ *   - forwards `setModel(provider, model)` to `IVlm.setProvider`.
+ */
+export class InnerLoopAgent {
+    vlm;
+    constructor(vlm) {
+        this.vlm = vlm;
+    }
+    get modelName() {
+        return this.vlm.modelName;
+    }
+    get providerName() {
+        return this.vlm.providerName;
+    }
+    async run(opts) {
+        const out = await this.vlm.runTask({
+            task: opts.task,
+            maxSteps: opts.maxSteps ?? 30,
+            abortSignal: opts.abortSignal,
+            onText: opts.onText,
+            onStep: async (step) => {
+                if (opts.onStep)
+                    await opts.onStep(step);
+                // Surface the same step as a synthetic ToolCallEvent so plugins
+                // wired against `onTool` keep a single code path. We emit
+                // start+end back-to-back because the legacy IVlm does not stream
+                // tool progress separately from the step boundary.
+                if (opts.onTool && step.action) {
+                    const id = `inner-${step.stepIndex}`;
+                    const toolName = step.action.kind;
+                    const args = actionParams(step.action);
+                    const start = { id, name: toolName, args, status: "start" };
+                    await opts.onTool(start);
+                    const end = {
+                        id,
+                        name: toolName,
+                        args,
+                        status: "end",
+                        result: step.thinking ?? "",
+                        isError: false,
+                    };
+                    await opts.onTool(end);
+                }
+            },
+        });
+        return out;
+    }
+    async setModel(provider, model) {
+        await this.vlm.setProvider(provider, model);
+    }
+}
+function actionParams(a) {
+    const { kind: _kind, ...rest } = a;
+    return rest;
+}
+//# sourceMappingURL=agentcore-inner-loop.js.map

package/dist/adapters/agentcore-inner-loop.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"agentcore-inner-loop.js","sourceRoot":"","sources":["../../src/adapters/agentcore-inner-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,MAAM,OAAO,cAAc;IACI;IAA7B,YAA6B,GAAS;QAAT,QAAG,GAAH,GAAG,CAAM;IAAG,CAAC;IAE1C,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;IAC5B,CAAC;IAED,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAyB;QACjC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;YACjC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;YAC7B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;gBACrB,IAAI,IAAI,CAAC,MAAM;oBAAE,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAEzC,gEAAgE;gBAChE,0DAA0D;gBAC1D,iEAAiE;gBACjE,mDAAmD;gBACnD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAC/B,MAAM,EAAE,GAAG,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;oBACrC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;oBAClC,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACvC,MAAM,KAAK,GAAkB,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;oBAC3E,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACzB,MAAM,GAAG,GAAkB;wBACzB,EAAE;wBACF,IAAI,EAAE,QAAQ;wBACd,IAAI;wBACJ,MAAM,EAAE,KAAK;wBACb,MAAM,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;wBAC3B,OAAO,EAAE,KAAK;qBACf,CAAC;oBACF,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;SACF,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,QAAgB,EAAE,KAAa;QAC5C,MAAM,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC9C,CAAC;CACF;AAED,SAAS,YAAY,CAAC,CAAmB;IACvC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,EAAE,GAAG,CAA+C,CAAC;IACjF,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/adapters/agentcore-pi.d.ts

@@ -0,0 +1,215 @@
+/**
+ * Adapter — PiOrchestratorAgent (two-layer outer orchestrator).
+ *
+ * Wraps `@mariozechner/pi-coding-agent`'s `AgentSession` so it can be
+ * plugged into the `IAgentCore` slot. The outer layer is the LLM that
+ * pi drives; the inner layer is whatever extensions are loaded from
+ * `<projectRoot>/.pi/extensions/` (e.g. `mobile_use`, `ground`,
+ * `ui_xml`). The adapter is **transport-agnostic** — it only knows about
+ * pi events and `AgentHost` callbacks.
+ *
+ * Key responsibilities:
+ *  - Lazy session bootstrap (extension/skill scanning, model resolution)
+ *    on first `run()` call.
+ *  - Map pi's session events to `IAgentCore` callbacks:
+ *      `message_update / text_delta`  → `onText`
+ *      `tool_execution_start/update/end` → `onTool` (start/update/end)
+ *      `message_end (assistant role)` → captures the final assistant text
+ *  - Translate `setModel(provider, model)` into pi-ai's `getModel()` and
+ *    forward to `session.setModel`. Hot-swap is best-effort — when model
+ *    resolution fails we keep the previous model and surface a warning.
+ *  - Honor `abortSignal` by calling `session.abort()`.
+ *  - Map ARouter / OpenRouter / OpenAI / Anthropic / Google base URLs to
+ *    pi-ai's known provider env vars (e.g. `OPENROUTER_API_KEY`) so
+ *    callers can pass a single `outer.apiKey` without exporting
+ *    provider-specific env names.
+ *
+ * NOT responsibilities:
+ *  - Loading extensions (caller passes the resourceLoader options).
+ *  - Setting up the device — pi extensions read device handles from
+ *    process env / DEVAGENT_HTTP_URL like before.
+ */
+import type { AgentCoreRunOptions, AgentCoreRunResult, IAgentCore } from "../ports/agent-core.js";
+import type { Logger } from "../logger/index.js";
+import type { IGrounder } from "../grounding/grounder.js";
+import type { IMcpRuntime } from "../mcp-client/runtime.js";
+export interface PiOuterConfig {
+    /** API key for the outer LLM. Mapped to `OPENROUTER_API_KEY` etc. */
+    apiKey?: string;
+    /** Base URL — used to infer the pi-ai provider. Default ARouter. */
+    baseUrl?: string;
+    /** Upstream model id (e.g. `moonshotai/kimi-k2.6`). */
+    model?: string;
+    /**
+     * Optional explicit pi-ai provider tag (`openrouter`, `openai`, …).
+     * When empty the provider is inferred from `baseUrl`.
+     */
+    provider?: string;
+}
+/**
+ * Minimal AgentSession surface that the adapter actually consumes.
+ * Mirrors `@mariozechner/pi-coding-agent` so we can swap in a fake in
+ * unit tests without pulling pi-ai's heavy types into the test boundary.
+ */
+export interface PiAgentSessionLike {
+    prompt(text: string): Promise<void>;
+    abort(): Promise<void>;
+    setModel(model: unknown): Promise<void>;
+    subscribe(listener: (event: PiSessionEvent) => void | Promise<void>): () => void;
+    dispose(): void;
+}
+/**
+ * Subset of pi's `AgentSessionEvent` we care about. Defined locally so
+ * tests don't need to import pi types and so the adapter is robust to
+ * upstream additions.
+ */
+export type PiSessionEvent = {
+    type: "message_start";
+    message: PiMessageLike;
+} | {
+    type: "message_update";
+    message: PiMessageLike;
+    assistantMessageEvent: PiAssistantEvent;
+} | {
+    type: "message_end";
+    message: PiMessageLike;
+} | {
+    type: "tool_execution_start";
+    toolCallId: string;
+    toolName: string;
+    args: unknown;
+} | {
+    type: "tool_execution_update";
+    toolCallId: string;
+    toolName: string;
+    args: unknown;
+    partialResult: unknown;
+} | {
+    type: "tool_execution_end";
+    toolCallId: string;
+    toolName: string;
+    result: unknown;
+    isError: boolean;
+} | {
+    type: string;
+    [k: string]: unknown;
+};
+interface PiMessageLike {
+    role: string;
+    content: unknown;
+}
+type PiAssistantEvent = {
+    type: "text_delta";
+    delta: string;
+    [k: string]: unknown;
+} | {
+    type: "thinking_delta";
+    delta: string;
+    [k: string]: unknown;
+} | {
+    type: string;
+    [k: string]: unknown;
+};
+/**
+ * Test seam — provide a custom session factory. Default factory loads
+ * `@mariozechner/pi-coding-agent` lazily so building tests / standalone
+ * binaries that skip the orchestrator never imports pi-ai at all.
+ */
+export interface PiSessionFactory {
+    (config: PiSessionFactoryConfig): Promise<PiAgentSessionLike>;
+}
+export interface PiSessionFactoryConfig {
+    projectRoot: string;
+    systemPrompt?: string;
+    /** Resolved pi-ai Model object (already URL-rewritten for ARouter). */
+    model?: unknown;
+}
+export interface PiOrchestratorAgentOptions {
+    /** Project root passed to pi's `DefaultResourceLoader`. */
+    projectRoot: string;
+    /** Outer-layer LLM config. */
+    outer: PiOuterConfig;
+    /**
+     * Optional override for pi's system prompt. Defaults to
+     * `ORCHESTRATOR_SYSTEM_PROMPT` (see `orchestrator-system-prompt.ts`).
+     */
+    systemPrompt?: string;
+    /** Structured logger; falls back to console.warn. */
+    logger?: Pick<Logger, "warn" | "info" | "debug">;
+    /** Inject a custom session factory (used by tests). */
+    createSession?: PiSessionFactory;
+    /** Override pi-ai `getModel`. Used by tests to skip the heavy import. */
+    resolveModel?: ModelResolver;
+    /**
+     * Resolves the active `IMcpRuntime` on every tool call. Typically
+     * `() => mcpClient` (the sibling `device-mcp-server` HTTP client) or an
+     * `InProcMcpClient` for local CLI usage. Resolved lazily so an ACP
+     * device switch (or autospawn restart) is reflected immediately on the
+     * next `mobile_use` / `ui_xml` / `ground` invocation.
+     */
+    mcpProvider?: () => IMcpRuntime;
+    /**
+     * Lazily build the grounder used by the `ground` extension. Called the
+     * first time the extension reads `getGrounder()` within a session.
+     * Typical implementation: `() => new HTTPGrounder(config.grounder)`.
+     */
+    grounderFactory?: () => IGrounder;
+    /**
+     * Run a `mobile_use` task end-to-end and return the final assistant
+     * message. Wraps `InnerLoopVlm.runTask`. Letting the bootstrap layer
+     * supply this keeps the heavy `OpenAIClient` instance shared with the
+     * legacy single-layer path (and lets tests stub it out).
+     */
+    mobileUseRunner?: (task: string, maxSteps: number, abortSignal?: AbortSignal) => Promise<string>;
+    /**
+     * Optional UI-XML formatter. When omitted the adapter calls
+     * `mcp.uiDump({ format: "raw" })` and feeds the result through
+     * `formatUiXmlToolOutput` — same behaviour the legacy `queryUiXml`
+     * used to provide on top of the raw `Device`. Tests can override to
+     * skip the dump path entirely.
+     */
+    uiXmlRunner?: (query?: string) => Promise<string>;
+}
+export interface ModelResolver {
+    (provider: string, modelId: string, baseUrl: string): unknown | undefined;
+}
+export declare class PiOrchestratorAgent implements IAgentCore {
+    private session;
+    private starting;
+    private outer;
+    private readonly projectRoot;
+    private readonly systemPrompt;
+    private readonly logger;
+    private readonly createSessionFactory;
+    private readonly resolveModelFn;
+    private readonly mcpProvider?;
+    private readonly grounderFactory?;
+    private readonly mobileUseRunner?;
+    private readonly uiXmlRunner?;
+    private cachedGrounder;
+    constructor(opts: PiOrchestratorAgentOptions);
+    get modelName(): string;
+    get providerName(): string | undefined;
+    run(opts: AgentCoreRunOptions): Promise<AgentCoreRunResult>;
+    setModel(provider: string, model: string): Promise<void>;
+    /** Tear down the underlying pi session. Idempotent. */
+    dispose(): Promise<void>;
+    private buildExtensionContext;
+    private ensureSession;
+}
+/**
+ * Map our generic outer config to pi-ai's env-var contract. pi-ai reads
+ * provider-specific keys (e.g. `OPENROUTER_API_KEY`) from `process.env`,
+ * so we transparently set them when the caller hands us `outer.apiKey`
+ * + a recognised baseUrl. Never overwrites a key the caller already set.
+ *
+ * Recognised hosts:
+ *   - openrouter.ai / arouter.ai       → OPENROUTER_API_KEY
+ *   - api.openai.com                   → OPENAI_API_KEY
+ *   - api.anthropic.com                → ANTHROPIC_API_KEY
+ *   - generativelanguage.googleapis.com → GEMINI_API_KEY
+ */
+export declare function applyOuterEnvMapping(outer: PiOuterConfig): void;
+/** Infer the pi-ai provider tag from a base URL. */
+export declare function inferPiProvider(baseUrl: string): string;
+export {};