@beeos-ai/device-agent 0.9.10 → 0.9.12

package/dist/ports/agent-core.d.ts

@@ -8,15 +8,16 @@
  *
  *   - ACP / Bridge / HTTP transports
  *   - SessionManager / TaskRunner accounting
- *   - the pi-coding-agent vs single-layer details
+ *   - the two-layer orchestrator vs single-layer details
  *
  * Plugins (ACP, HTTP debug, agent-sync, ...) interact with the core
  * indirectly through `AgentHost`. The host owns the IAgentCore instance
  * and dispatches events; plugins never call `IAgentCore.run` directly.
  *
  * Implementations:
- *  - `adapters/agentcore-pi.ts` — pi-coding-agent two-layer orchestrator
- *    (default; outer = arouter/kimi-k2.6, inner = autoglm-9b via mobile_use)
+ *  - `adapters/agentcore-runner.ts` — AgentRunner Hybrid Plan-Act (default)
+ *  - `adapters/agentcore-orchestrator.ts` — two-layer OrchestratorAgent
+ *    (outer tool-calling LLM, inner autoglm VLM via mobile_use; `--orchestrator`)
  *  - `adapters/agentcore-inner-loop.ts` — single-layer fallback that wraps
  *    `vlm-inner-loop.ts` (selected by `--legacy-inner`)
  */
@@ -119,7 +120,7 @@ export interface AgentCoreRunCallbacks {
 export interface AgentCoreRunResult {
     /** Final user-facing reply. */
     finalMessage: string;
-    /** Number of inner steps observed (0 for the pi adapter — opaque tools). */
+    /** Number of inner steps observed (0 for the orchestrator adapter — opaque tools). */
     steps: number;
     /** True iff the run terminated due to abort / timeout. */
     cancelled: boolean;
@@ -128,7 +129,7 @@ export interface AgentCoreRunResult {
 export interface AgentCoreRunOptions extends AgentCoreRunCallbacks {
     /** Natural language task / prompt. */
     task: string;
-    /** Soft cap on inner steps (legacy adapter); pi adapter may ignore. */
+    /** Soft cap on inner steps (legacy adapter); orchestrator may ignore. */
     maxSteps?: number;
     abortSignal?: AbortSignal;
 }

package/dist/shortcuts/types.d.ts

@@ -31,8 +31,7 @@
  *         shortcut's `execute(...)` body but are never advertised as
  *         outer-LLM tools. The convention is enforced by inspection:
  *         if you need to add a new shortcut that calls these, do so
- *         from inside `services/shortcuts/*.ts` only — see the
- *         file-level comment in `pi-context.ts`.
+ *         from inside `services/shortcuts/*.ts` only.
  *
  * Error contract:
  *
@@ -94,8 +93,7 @@ export type ShortcutResult<T = unknown> = ShortcutSuccess<T> | ShortcutFailure;
  * JSON schema fragment for a shortcut's parameter object. Kept as
  * `unknown` here so callers can supply either a TypeBox / Zod schema
  * or a hand-rolled JSON Schema; the registry forwards the value
- * unchanged to the outer LLM tool descriptor (see
- * `.pi/extensions/android-tools.ts`). Validation happens inside each
+ * unchanged to the outer LLM tool descriptor. Validation happens inside each
  * shortcut's `execute` body — there is no central validator, both
  * because the runtimes (TypeBox vs Zod vs raw JSON Schema) differ in
  * how they expose `validate(input)` and because the plan intentionally
@@ -139,8 +137,7 @@ export type ShortcutPlatform = "android" | "desktop" | "any";
 /**
  * Concrete shortcut. The execute method is given a ready-built
  * `ShortcutContext` so individual shortcuts don't have to hand-roll
- * MCP / device wiring (and the boundary in `pi-context.ts` stays
- * honest — see D9).
+ * MCP / device wiring.
  */
 export interface Shortcut<TArgs = unknown, TOut = unknown> {
     readonly descriptor: ShortcutDescriptor;

package/dist/shortcuts/types.js

@@ -31,8 +31,7 @@
  *         shortcut's `execute(...)` body but are never advertised as
  *         outer-LLM tools. The convention is enforced by inspection:
  *         if you need to add a new shortcut that calls these, do so
- *         from inside `services/shortcuts/*.ts` only — see the
- *         file-level comment in `pi-context.ts`.
+ *         from inside `services/shortcuts/*.ts` only.
  *
  * Error contract:
  *

package/dist/shortcuts/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/shortcuts/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/shortcuts/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@beeos-ai/device-agent",
-  "version": "0.9.10",
+  "version": "0.9.12",
   "description": "BeeOS device-agent — Android/desktop AI control agent (TypeScript) speaking ACP over Bridge WebSocket. Connects to a sibling device-mcp-server process for device control.",
   "license": "Apache-2.0",
   "type": "module",
@@ -23,8 +23,7 @@
     "dist",
     "README.md",
     "LICENSE",
-    "CHANGELOG.md",
-    ".pi"
+    "CHANGELOG.md"
   ],
   "publishConfig": {
     "access": "public"
@@ -56,7 +55,6 @@
     "@beeos-ai/message-sdk": "^1.0.0",
     "@fastify/cors": "^11.2.0",
     "@fastify/multipart": "^10.0.0",
-    "@mariozechner/pi-coding-agent": "0.70.2",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "dotenv": "^16.5.0",
     "fastify": "^5.8.5",
@@ -82,7 +80,6 @@
     "test": "vitest run",
     "test:watch": "vitest",
     "test:cov": "vitest run --coverage",
-    "chat": "tsx src/chat.ts",
     "run:wechat-fixture": "tsx scripts/synthesize-wechat-fixture.ts",
     "record:wechat-fixture": "tsx scripts/record-wechat-fixture.ts",
     "legacy:test:mobile": "tsx scripts/legacy/test-mobile-use.ts",

package/.pi/extensions/android-tools.ts

@@ -1,291 +0,0 @@
-/**
- * Pi extension: registers tools for the **outer** orchestrator.
- *
- * Two layers of tools coexist:
- *
- *   - **Legacy GUI primitives** — `mobile_use`, `ground`, `ui_xml`.
- *     These wrap the inner VLM loop and visual grounder; they are
- *     always present and don't depend on the shortcut runtime.
- *
- *   - **Hybrid Action Space (Phase 2)** — one outer-LLM tool per
- *     shortcut registered with `ShortcutKB`. Each shortcut's
- *     descriptor (id, displayName, description, jsonSchema) is
- *     mapped to a pi `registerTool({...})` call here. The LLM never
- *     sees raw `runShell` / `pushFile` — those are explicitly hidden
- *     behind the `Shortcut.execute` boundary (see D9 in the plan and
- *     the comment block in `pi-context.ts`).
- *
- * Why register dynamically instead of hard-coding 4 tools?
- *
- *   The shortcut KB is the source of truth — the bootstrap layer
- *   decides which shortcuts to register, and the extension picks up
- *   whatever is there at session-start. New shortcuts (or future
- *   plugin packages) auto-light up without an extension code change.
- *   The trade-off is that the descriptor *cannot* depend on context
- *   that varies per call — but JSON Schema + display name + free-form
- *   description is exactly that, so we don't lose anything.
- *
- * Extension lifecycle (since the dual-layer + MCP refactor):
- *
- *   PiOrchestratorAgent.ensureSession()
- *     └─ setPiExtensionContext({ getMcpClient, getGrounder, runMobileUse,
- *                                getUiXml, getShortcutKB, runShell, pushFile })
- *     └─ pi.loader.reload() → loads this file → pi.registerTool(...)
- *     └─ pi.session.prompt(...) → tools execute → context.* invoked
- *
- *   PiOrchestratorAgent.dispose()
- *     └─ setPiExtensionContext(null)
- *
- * The extension is the ONLY pi-coupling point. Bootstrap supplies the
- * concrete dependencies — this file stays free of process.env reads,
- * ad-hoc `new AndroidDevice()` / `new HTTPGrounder()` calls, and direct
- * MCP HTTP knowledge.
- */
-
-import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
-import { Type } from "@sinclair/typebox";
-import { getPiExtensionContext } from "../../src/pi-context.js";
-import type { Shortcut, ShortcutResult } from "../../src/shortcuts/index.js";
-
-/**
- * Server-side floor for `mobile_use` step budget.
- *
- * The outer orchestrator LLM frequently misjudges ADB / Android tasks as
- * "simple actions" and emits `max_steps: 5..15`, which the inner
- * `Agent.run` loop blows through before the task can finish — surfacing as
- * "Reached max steps (N)". Clamping upward at the tool boundary protects
- * the user without preventing the LLM from explicitly asking for more
- * steps when it judges a flow as complex.
- */
-const MIN_MOBILE_USE_STEPS = 30;
-
-export default function (pi: ExtensionAPI) {
-  // ── mobile_use ─────────────────────────────────────────────────────────────
-  pi.registerTool({
-    name: "mobile_use",
-    label: "Mobile Use",
-    description:
-      "Drive the connected Android phone with a natural-language task. " +
-      "A VLM runs its own screenshot→reason→act loop until it finishes or hits max-steps. " +
-      "Pass a full, concrete instruction with app-native UI text. " +
-      "For long text (especially Chinese), embed it verbatim and note '原样输入、不要翻译'. " +
-      "One call runs 20–50 steps autonomously; do NOT call repeatedly for the same task.",
-    promptSnippet: "mobile_use(task): drive Android phone via VLM screenshot→reason→act loop",
-    parameters: Type.Object({
-      task: Type.String({ description: "Full natural-language instruction in Chinese or English" }),
-      max_steps: Type.Integer({
-        description: "Maximum VLM steps for this call. Required. Typical range 20-50 depending on task complexity.",
-        minimum: 1,
-        maximum: 100,
-      }),
-    }),
-    async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
-      onUpdate?.({ content: [{ type: "text", text: `[mobile_use] ${params.task}\n` }], details: null });
-      if (signal?.aborted) throw new Error("aborted");
-
-      const ctx = getPiExtensionContext();
-      const requested = params.max_steps;
-      const effective = Math.max(MIN_MOBILE_USE_STEPS, requested);
-      if (effective !== requested) {
-        const log = (await import("../../src/logger/index.js")).getLogger();
-        log.warn(
-          { component: "mobile_use", requested, effective, floor: MIN_MOBILE_USE_STEPS },
-          "mobile_use.max_steps clamped up to floor",
-        );
-      }
-      const result = await ctx.runMobileUse(params.task, effective, signal ?? undefined);
-      return { content: [{ type: "text", text: result }], details: null };
-    },
-  });
-
-  // ── ground ─────────────────────────────────────────────────────────────────
-  // Visual grounding tool: find where a described UI element is on screen.
-  // Takes a screenshot from the connected device (or uses a caller-supplied image),
-  // calls the configured grounder VLM, and returns pixel coordinates.
-  pi.registerTool({
-    name: "ground",
-    label: "Visual Ground",
-    description:
-      "Locate a UI element on the Android phone screen and return its pixel coordinates. " +
-      "Describe the element in natural language (e.g. '确认按钮', 'Settings icon', 'search box'). " +
-      "If image_b64 is omitted, a fresh screenshot is taken automatically. " +
-      "Returns JSON: { x, y, width, height, raw } where x/y are absolute pixels from the top-left. " +
-      "Use this when you need to know where to tap without running the full mobile_use loop.",
-    promptSnippet: "ground(query, image_b64?): locate UI element on screen → {x, y}",
-    parameters: Type.Object({
-      query: Type.String({ description: "Natural-language description of the element to locate" }),
-      image_b64: Type.Optional(
-        Type.String({ description: "Base64-encoded JPEG/PNG image. If omitted, a screenshot is taken." }),
-      ),
-    }),
-    async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
-      onUpdate?.({ content: [{ type: "text", text: `[ground] "${params.query}"\n` }], details: null });
-      if (signal?.aborted) throw new Error("aborted");
-
-      const ctx = getPiExtensionContext();
-      let imageB64 = params.image_b64 ?? null;
-
-      if (!imageB64) {
-        const ss = await ctx.getMcpClient().screenshot();
-        imageB64 = ss.b64;
-      }
-
-      const result = await ctx.getGrounder().ground(imageB64, params.query);
-
-      return {
-        content: [{ type: "text", text: result ?? "Element not found" }],
-        details: null,
-      };
-    },
-  });
-
-  // ── ui_xml ─────────────────────────────────────────────────────────────────
-  pi.registerTool({
-    name: "ui_xml",
-    label: "UI XML",
-    description:
-      "Dump the Android UI hierarchy from the connected device and return it as text. " +
-      "Without a query: returns the raw uiautomator XML (truncated at 8000 chars). " +
-      "With a query: searches nodes by text / resource-id / content-desc / class name " +
-      "(case-insensitive substring match) and returns matching nodes as JSON with: " +
-      "text, resourceId, contentDesc, className, bounds ([x1,y1][x2,y2]), center {x,y}, clickable. " +
-      "Use this to inspect the current screen structure, find resource IDs, or check element states.",
-    promptSnippet: "ui_xml(query?): dump or search UI hierarchy → raw XML or [{text, bounds, center, ...}]",
-    parameters: Type.Object({
-      query: Type.Optional(
-        Type.String({ description: "Substring to match against text/resource-id/content-desc/class. Omit for raw XML." }),
-      ),
-    }),
-    async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
-      onUpdate?.({ content: [{ type: "text", text: `[ui_xml] query=${params.query ?? "(raw)"}\n` }], details: null });
-      if (signal?.aborted) throw new Error("aborted");
-
-      const ctx = getPiExtensionContext();
-      const out = await ctx.getUiXml(params.query);
-      return { content: [{ type: "text", text: out }], details: null };
-    },
-  });
-
-  // ── Hybrid Action Space (Phase 2): one tool per shortcut ─────────────────
-  // The shortcut KB may not be wired (legacy bootstrap, tests). When it
-  // isn't, we silently skip — the legacy primitives above remain
-  // functional, the orchestrator just doesn't see any shortcut tools.
-  // Failing loud here would block every legacy path that doesn't need
-  // the hybrid layer.
-  let shortcuts: Shortcut<unknown, unknown>[] = [];
-  try {
-    const ctx = getPiExtensionContext();
-    shortcuts = ctx.getShortcutKB().registry.list();
-  } catch {
-    shortcuts = [];
-  }
-
-  for (const shortcut of shortcuts) {
-    registerShortcutAsTool(pi, shortcut);
-  }
-
-  // ── mobile_use_brain_actor ─ (commented out, not exposed to orchestrator)
-  // pi.registerTool({ name: "mobile_use_brain_actor", ... execute: mobileUseBrainActor })
-
-  // ── mobile_use_grounding ─ (commented out, not exposed to orchestrator)
-  // pi.registerTool({ name: "mobile_use_grounding", ... execute: mobileUseGrounding })
-}
-
-/**
- * Map a `Shortcut` descriptor onto a pi `registerTool({...})` call.
- *
- * Each tool's `execute(...)` body forwards into the shortcut runtime
- * (`ctx.getShortcutKB().invoke(...)`), translating the structured
- * `ShortcutResult` into the LLM-visible content array. Failures are
- * **not** thrown — instead we return `{ content: [{type:"text", text}],
- * details: { error_code } }` so the LLM sees a structured failure code
- * (for example `"app_not_found"`) and can fall back to `mobile_use`
- * without a stack trace.
- */
-function registerShortcutAsTool(
-  pi: ExtensionAPI,
-  shortcut: Shortcut<unknown, unknown>,
-): void {
-  const d = shortcut.descriptor;
-  pi.registerTool({
-    name: d.id,
-    label: d.displayName,
-    description: d.description,
-    promptSnippet: `${d.id}: ${d.displayName.toLowerCase()} (${d.kind})`,
-    // pi-coding-agent accepts JSON-Schema-shaped objects directly —
-    // TypeBox's `Type.Object` returns one anyway. Forwarding the raw
-    // schema lets shortcut authors describe parameters without
-    // requiring a TypeBox dependency.
-    parameters: d.jsonSchema as never,
-    async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
-      onUpdate?.({
-        content: [{ type: "text", text: `[${d.id}] invoke\n` }],
-        details: null,
-      });
-      const ctx = getPiExtensionContext();
-      const invoker = ctx.getShortcutKB();
-      const log = (await import("../../src/logger/index.js")).getLogger();
-      // Pull the shared resource facade so shortcuts that need
-      // `ctx.resources.resolveUrl` (e.g. `download_to_device`) see
-      // the same registry+cache as the AgentHost auto-trigger. The
-      // accessor is optional; tests / standalone bootstraps that
-      // don't wire a facade get `undefined` and the shortcut surfaces
-      // `permission_denied` to the LLM (intentional fail-soft).
-      const resources = ctx.getShortcutResources?.();
-      const callerCtx = {
-        abortSignal: signal ?? new AbortController().signal,
-        log,
-        ...(resources ? { resources } : {}),
-      };
-      const res: ShortcutResult<unknown> = await invoker.invoke(d.id, params, callerCtx);
-      return formatShortcutResult(d.id, res);
-    },
-  });
-}
-
-/**
- * Translate a `ShortcutResult` into a pi tool result. We return a
- * **single text block** containing JSON the LLM can parse, plus a
- * `details` blob for the orchestrator-side hooks. Errors NEVER throw —
- * the LLM should be able to inspect the failure and choose a fallback
- * path (typically `mobile_use`). Throwing would skip pi's tool-error
- * recovery and bubble up as an unhandled exception in the chat.
- */
-function formatShortcutResult(
-  id: string,
-  res: ShortcutResult<unknown>,
-): { content: Array<{ type: "text"; text: string }>; details: unknown } {
-  if (res.ok) {
-    return {
-      content: [
-        {
-          type: "text",
-          text: JSON.stringify({ ok: true, output: res.output }, null, 2),
-        },
-      ],
-      details: { shortcut_id: id, ok: true, durationMs: res.durationMs ?? null },
-    };
-  }
-  return {
-    content: [
-      {
-        type: "text",
-        text: JSON.stringify(
-          {
-            ok: false,
-            code: res.code,
-            message: res.message,
-          },
-          null,
-          2,
-        ),
-      },
-    ],
-    details: {
-      shortcut_id: id,
-      ok: false,
-      code: res.code,
-      durationMs: res.durationMs ?? null,
-    },
-  };
-}

package/.pi/skills/ground-mobile-fallback/SKILL.md

@@ -1,50 +0,0 @@
----
-name: ground-mobile-fallback
-description: 当 mobile_use 失败或不可用时，指导外层编排 Agent 用 ground + adb（及可选 ui_xml）分步操作 Android：先理解界面、再视觉定位、再执行点击/滑动，作为一体化 VLM 回退路径。
----
-
-# mobile_use 不可用时的手机操作回退（ground + adb）
-
-## 何时启用
-
-在以下情况**不要**再盲目重试 `mobile_use`，改用本流程：
-
-- `mobile_use` 报错（API、超时、鉴权、设备断开等）。
-- 任务极短、只需 1～3 次精确点击，一体化循环性价比低或易跑偏。
-- 你需要**可解释**的坐标（例如调试、或要向用户说明点在哪里）。
-
-若 `ground` 所依赖的 Grounder（`GROUNDER_*` / 与 `GUI_AGENT_*` 共用的 VLM）也不可用，本回退无效，应先恢复网络与密钥或改任务。
-
-## 工具分工（外层负责编排）
-
-| 工具 | 作用 |
-|------|------|
-| `ui_xml` | 可选。看当前界面结构、文案、`bounds` / `center`，缩小「要点哪里」的描述范围。 |
-| `ground` | 对**当前屏**截图（或你传入的 `image_b64`），根据自然语言描述返回目标大致像素位置（模型原始输出多为类似 `[x, y]` 的坐标；以工具返回文本为准并解析）。 |
-| Bash + `adb` | 在解析出坐标后执行 `adb shell input tap x y`、`input swipe ...` 等（需已连接设备，必要时设置 `GUI_AGENT_DEVICE_ID`）。 |
-
-## 推荐流程
-
-1. **确认设备**：`adb devices`；多设备时指定 serial（与项目环境变量一致）。
-2. **（可选）** `ui_xml` 带 `query` 搜索按钮/列表项文案，确认目标在屏上且描述与界面语言一致。
-3. **`ground(query)`**：`query` 写具体、可区分（例如「设置列表里的 WLAN 项」而非「网络」）。避免一次描述多个无关目标。
-4. **解析坐标**：从返回文本中提取整数像素坐标；若有多个候选，优先与 `ui_xml` 的 `center` 接近的点。
-5. **`adb shell input tap x y`**。若未生效：等待动画后再 `ground` 一次（界面已变）或先 `ui_xml` 看是否被遮挡/未 clickable。
-6. **输入文字**：`adb shell input text '...'`（注意 shell 转义；复杂文本可改用系统剪贴板或拆分步骤）。
-7. **滑动**：需要时 `input swipe x1 y1 x2 y2 duration_ms`；长列表可多次滑动后再 `ground`。
-
-## 与 mobile_use 的配合
-
-- 正常情况仍以 **`mobile_use` 为主**（一次调用内多步截图→推理→执行）。
-- 仅当 `mobile_use` **明确失败**或**不适合**时，再切换到「`ui_xml` + `ground` + adb」的显式编排。
-- 回退路径步数多、对描述敏感；每步后根据新截图或 `ui_xml` 校验是否到达预期界面。
-
-## 环境变量提示（便于排查）
-
-- 设备：`GUI_AGENT_DEVICE_ID`
-- 截图与 Grounder 常与内层共用：`GUI_AGENT_BASE_URL`、`GUI_AGENT_API_KEY`、`GUI_AGENT_MODEL`；若单独配置 Grounder：`GROUNDER_BASE_URL`、`GROUNDER_API_KEY`、`GROUNDER_MODEL`
-
-## 禁止事项
-
-- 不要在未确认坐标的情况下连续乱点同一区域。
-- 不要用模糊到无法在一屏内唯一定位的 `query` 调用 `ground`（先 `ui_xml` 或滚动到目标大致出现）。