@beeos-ai/device-agent 0.9.9 → 0.9.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/dist/adapters/agentcore-inner-loop.d.ts +3 -4
  2. package/dist/adapters/agentcore-inner-loop.js +3 -4
  3. package/dist/adapters/agentcore-inner-loop.js.map +1 -1
  4. package/dist/adapters/agentcore-orchestrator.d.ts +36 -0
  5. package/dist/adapters/agentcore-orchestrator.js +203 -0
  6. package/dist/adapters/agentcore-orchestrator.js.map +1 -0
  7. package/dist/adapters/agentcore-runner.d.ts +3 -3
  8. package/dist/adapters/agentcore-runner.js +3 -3
  9. package/dist/adapters/app-guide.d.ts +16 -0
  10. package/dist/adapters/app-guide.js +101 -0
  11. package/dist/adapters/app-guide.js.map +1 -0
  12. package/dist/adapters/orchestrator-agent-prompt.d.ts +7 -0
  13. package/dist/adapters/orchestrator-agent-prompt.js +59 -0
  14. package/dist/adapters/orchestrator-agent-prompt.js.map +1 -0
  15. package/dist/adapters/vlm-inner-loop.d.ts +6 -0
  16. package/dist/adapters/vlm-inner-loop.js +3 -0
  17. package/dist/adapters/vlm-inner-loop.js.map +1 -1
  18. package/dist/agent-runtime/core.js +1 -1
  19. package/dist/agent-runtime/core.js.map +1 -1
  20. package/dist/agent-runtime/shortcuts/types.d.ts +2 -3
  21. package/dist/agent-runtime/shortcuts/types.js +2 -3
  22. package/dist/agent-runtime/shortcuts/types.js.map +1 -1
  23. package/dist/api.d.ts +1 -1
  24. package/dist/api.js +1 -1
  25. package/dist/bootstrap.d.ts +4 -4
  26. package/dist/bootstrap.js +47 -61
  27. package/dist/bootstrap.js.map +1 -1
  28. package/dist/config/loader.d.ts +2 -2
  29. package/dist/config/loader.js +2 -2
  30. package/dist/config/loader.js.map +1 -1
  31. package/dist/config/schema.d.ts +1 -1
  32. package/dist/config/schema.js +4 -5
  33. package/dist/config/schema.js.map +1 -1
  34. package/dist/device/apps.js +1 -1
  35. package/dist/device/apps.js.map +1 -1
  36. package/dist/fleet/domain/events.d.ts +22 -1
  37. package/dist/fleet/domain/fsm.d.ts +23 -1
  38. package/dist/fleet/domain/fsm.js +29 -0
  39. package/dist/fleet/domain/fsm.js.map +1 -1
  40. package/dist/fleet/supervisor.d.ts +13 -0
  41. package/dist/fleet/supervisor.js +90 -0
  42. package/dist/fleet/supervisor.js.map +1 -1
  43. package/dist/grounding/grounder.d.ts +1 -1
  44. package/dist/inbound/cli/main.d.ts +1 -1
  45. package/dist/inbound/cli/main.js +4 -4
  46. package/dist/inbound/cli/main.js.map +1 -1
  47. package/dist/inbound/cli/run-command.d.ts +6 -0
  48. package/dist/inbound/cli/run-command.js +94 -0
  49. package/dist/inbound/cli/run-command.js.map +1 -1
  50. package/dist/index.d.ts +2 -4
  51. package/dist/index.js +1 -2
  52. package/dist/index.js.map +1 -1
  53. package/dist/model/adapters/general.d.ts +11 -0
  54. package/dist/model/adapters/general.js +249 -0
  55. package/dist/model/adapters/general.js.map +1 -0
  56. package/dist/model/adapters/index.d.ts +1 -0
  57. package/dist/model/adapters/index.js +1 -0
  58. package/dist/model/adapters/index.js.map +1 -1
  59. package/dist/plugins/message-bus/ms-channel-notifier.d.ts +24 -0
  60. package/dist/plugins/message-bus/ms-channel-notifier.js +179 -12
  61. package/dist/plugins/message-bus/ms-channel-notifier.js.map +1 -1
  62. package/dist/ports/agent-core.d.ts +6 -5
  63. package/dist/shortcuts/types.d.ts +3 -6
  64. package/dist/shortcuts/types.js +1 -2
  65. package/dist/shortcuts/types.js.map +1 -1
  66. package/package.json +3 -6
  67. package/.pi/extensions/android-tools.ts +0 -291
  68. package/.pi/skills/.gitkeep +0 -0
  69. package/.pi/skills/ground-mobile-fallback/SKILL.md +0 -50
  70. package/dist/adapters/agentcore-pi.d.ts +0 -237
  71. package/dist/adapters/agentcore-pi.js +0 -512
  72. package/dist/adapters/agentcore-pi.js.map +0 -1
  73. package/dist/chat.d.ts +0 -16
  74. package/dist/chat.js +0 -185
  75. package/dist/chat.js.map +0 -1
  76. package/dist/debug/dump-system-prompt.d.ts +0 -1
  77. package/dist/debug/dump-system-prompt.js +0 -34
  78. package/dist/debug/dump-system-prompt.js.map +0 -1
  79. package/dist/orchestrator-system-prompt.d.ts +0 -32
  80. package/dist/orchestrator-system-prompt.js +0 -197
  81. package/dist/orchestrator-system-prompt.js.map +0 -1
  82. package/dist/pi-context.d.ts +0 -132
  83. package/dist/pi-context.js +0 -62
  84. package/dist/pi-context.js.map +0 -1
  85. package/dist/pi-local-skills.d.ts +0 -17
  86. package/dist/pi-local-skills.js +0 -24
  87. package/dist/pi-local-skills.js.map +0 -1
@@ -1,291 +0,0 @@
1
- /**
2
- * Pi extension: registers tools for the **outer** orchestrator.
3
- *
4
- * Two layers of tools coexist:
5
- *
6
- * - **Legacy GUI primitives** — `mobile_use`, `ground`, `ui_xml`.
7
- * These wrap the inner VLM loop and visual grounder; they are
8
- * always present and don't depend on the shortcut runtime.
9
- *
10
- * - **Hybrid Action Space (Phase 2)** — one outer-LLM tool per
11
- * shortcut registered with `ShortcutKB`. Each shortcut's
12
- * descriptor (id, displayName, description, jsonSchema) is
13
- * mapped to a pi `registerTool({...})` call here. The LLM never
14
- * sees raw `runShell` / `pushFile` — those are explicitly hidden
15
- * behind the `Shortcut.execute` boundary (see D9 in the plan and
16
- * the comment block in `pi-context.ts`).
17
- *
18
- * Why register dynamically instead of hard-coding 4 tools?
19
- *
20
- * The shortcut KB is the source of truth — the bootstrap layer
21
- * decides which shortcuts to register, and the extension picks up
22
- * whatever is there at session-start. New shortcuts (or future
23
- * plugin packages) auto-light up without an extension code change.
24
- * The trade-off is that the descriptor *cannot* depend on context
25
- * that varies per call — but JSON Schema + display name + free-form
26
- * description is exactly that, so we don't lose anything.
27
- *
28
- * Extension lifecycle (since the dual-layer + MCP refactor):
29
- *
30
- * PiOrchestratorAgent.ensureSession()
31
- * └─ setPiExtensionContext({ getMcpClient, getGrounder, runMobileUse,
32
- * getUiXml, getShortcutKB, runShell, pushFile })
33
- * └─ pi.loader.reload() → loads this file → pi.registerTool(...)
34
- * └─ pi.session.prompt(...) → tools execute → context.* invoked
35
- *
36
- * PiOrchestratorAgent.dispose()
37
- * └─ setPiExtensionContext(null)
38
- *
39
- * The extension is the ONLY pi-coupling point. Bootstrap supplies the
40
- * concrete dependencies — this file stays free of process.env reads,
41
- * ad-hoc `new AndroidDevice()` / `new HTTPGrounder()` calls, and direct
42
- * MCP HTTP knowledge.
43
- */
44
-
45
- import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
46
- import { Type } from "@sinclair/typebox";
47
- import { getPiExtensionContext } from "../../src/pi-context.js";
48
- import type { Shortcut, ShortcutResult } from "../../src/shortcuts/index.js";
49
-
50
- /**
51
- * Server-side floor for `mobile_use` step budget.
52
- *
53
- * The outer orchestrator LLM frequently misjudges ADB / Android tasks as
54
- * "simple actions" and emits `max_steps: 5..15`, which the inner
55
- * `Agent.run` loop blows through before the task can finish — surfacing as
56
- * "Reached max steps (N)". Clamping upward at the tool boundary protects
57
- * the user without preventing the LLM from explicitly asking for more
58
- * steps when it judges a flow as complex.
59
- */
60
- const MIN_MOBILE_USE_STEPS = 30;
61
-
62
- export default function (pi: ExtensionAPI) {
63
- // ── mobile_use ─────────────────────────────────────────────────────────────
64
- pi.registerTool({
65
- name: "mobile_use",
66
- label: "Mobile Use",
67
- description:
68
- "Drive the connected Android phone with a natural-language task. " +
69
- "A VLM runs its own screenshot→reason→act loop until it finishes or hits max-steps. " +
70
- "Pass a full, concrete instruction with app-native UI text. " +
71
- "For long text (especially Chinese), embed it verbatim and note '原样输入、不要翻译'. " +
72
- "One call runs 20–50 steps autonomously; do NOT call repeatedly for the same task.",
73
- promptSnippet: "mobile_use(task): drive Android phone via VLM screenshot→reason→act loop",
74
- parameters: Type.Object({
75
- task: Type.String({ description: "Full natural-language instruction in Chinese or English" }),
76
- max_steps: Type.Integer({
77
- description: "Maximum VLM steps for this call. Required. Typical range 20-50 depending on task complexity.",
78
- minimum: 1,
79
- maximum: 100,
80
- }),
81
- }),
82
- async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
83
- onUpdate?.({ content: [{ type: "text", text: `[mobile_use] ${params.task}\n` }], details: null });
84
- if (signal?.aborted) throw new Error("aborted");
85
-
86
- const ctx = getPiExtensionContext();
87
- const requested = params.max_steps;
88
- const effective = Math.max(MIN_MOBILE_USE_STEPS, requested);
89
- if (effective !== requested) {
90
- const log = (await import("../../src/logger/index.js")).getLogger();
91
- log.warn(
92
- { component: "mobile_use", requested, effective, floor: MIN_MOBILE_USE_STEPS },
93
- "mobile_use.max_steps clamped up to floor",
94
- );
95
- }
96
- const result = await ctx.runMobileUse(params.task, effective, signal ?? undefined);
97
- return { content: [{ type: "text", text: result }], details: null };
98
- },
99
- });
100
-
101
- // ── ground ─────────────────────────────────────────────────────────────────
102
- // Visual grounding tool: find where a described UI element is on screen.
103
- // Takes a screenshot from the connected device (or uses a caller-supplied image),
104
- // calls the configured grounder VLM, and returns pixel coordinates.
105
- pi.registerTool({
106
- name: "ground",
107
- label: "Visual Ground",
108
- description:
109
- "Locate a UI element on the Android phone screen and return its pixel coordinates. " +
110
- "Describe the element in natural language (e.g. '确认按钮', 'Settings icon', 'search box'). " +
111
- "If image_b64 is omitted, a fresh screenshot is taken automatically. " +
112
- "Returns JSON: { x, y, width, height, raw } where x/y are absolute pixels from the top-left. " +
113
- "Use this when you need to know where to tap without running the full mobile_use loop.",
114
- promptSnippet: "ground(query, image_b64?): locate UI element on screen → {x, y}",
115
- parameters: Type.Object({
116
- query: Type.String({ description: "Natural-language description of the element to locate" }),
117
- image_b64: Type.Optional(
118
- Type.String({ description: "Base64-encoded JPEG/PNG image. If omitted, a screenshot is taken." }),
119
- ),
120
- }),
121
- async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
122
- onUpdate?.({ content: [{ type: "text", text: `[ground] "${params.query}"\n` }], details: null });
123
- if (signal?.aborted) throw new Error("aborted");
124
-
125
- const ctx = getPiExtensionContext();
126
- let imageB64 = params.image_b64 ?? null;
127
-
128
- if (!imageB64) {
129
- const ss = await ctx.getMcpClient().screenshot();
130
- imageB64 = ss.b64;
131
- }
132
-
133
- const result = await ctx.getGrounder().ground(imageB64, params.query);
134
-
135
- return {
136
- content: [{ type: "text", text: result ?? "Element not found" }],
137
- details: null,
138
- };
139
- },
140
- });
141
-
142
- // ── ui_xml ─────────────────────────────────────────────────────────────────
143
- pi.registerTool({
144
- name: "ui_xml",
145
- label: "UI XML",
146
- description:
147
- "Dump the Android UI hierarchy from the connected device and return it as text. " +
148
- "Without a query: returns the raw uiautomator XML (truncated at 8000 chars). " +
149
- "With a query: searches nodes by text / resource-id / content-desc / class name " +
150
- "(case-insensitive substring match) and returns matching nodes as JSON with: " +
151
- "text, resourceId, contentDesc, className, bounds ([x1,y1][x2,y2]), center {x,y}, clickable. " +
152
- "Use this to inspect the current screen structure, find resource IDs, or check element states.",
153
- promptSnippet: "ui_xml(query?): dump or search UI hierarchy → raw XML or [{text, bounds, center, ...}]",
154
- parameters: Type.Object({
155
- query: Type.Optional(
156
- Type.String({ description: "Substring to match against text/resource-id/content-desc/class. Omit for raw XML." }),
157
- ),
158
- }),
159
- async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
160
- onUpdate?.({ content: [{ type: "text", text: `[ui_xml] query=${params.query ?? "(raw)"}\n` }], details: null });
161
- if (signal?.aborted) throw new Error("aborted");
162
-
163
- const ctx = getPiExtensionContext();
164
- const out = await ctx.getUiXml(params.query);
165
- return { content: [{ type: "text", text: out }], details: null };
166
- },
167
- });
168
-
169
- // ── Hybrid Action Space (Phase 2): one tool per shortcut ─────────────────
170
- // The shortcut KB may not be wired (legacy bootstrap, tests). When it
171
- // isn't, we silently skip — the legacy primitives above remain
172
- // functional, the orchestrator just doesn't see any shortcut tools.
173
- // Failing loud here would block every legacy path that doesn't need
174
- // the hybrid layer.
175
- let shortcuts: Shortcut<unknown, unknown>[] = [];
176
- try {
177
- const ctx = getPiExtensionContext();
178
- shortcuts = ctx.getShortcutKB().registry.list();
179
- } catch {
180
- shortcuts = [];
181
- }
182
-
183
- for (const shortcut of shortcuts) {
184
- registerShortcutAsTool(pi, shortcut);
185
- }
186
-
187
- // ── mobile_use_brain_actor ─ (commented out, not exposed to orchestrator)
188
- // pi.registerTool({ name: "mobile_use_brain_actor", ... execute: mobileUseBrainActor })
189
-
190
- // ── mobile_use_grounding ─ (commented out, not exposed to orchestrator)
191
- // pi.registerTool({ name: "mobile_use_grounding", ... execute: mobileUseGrounding })
192
- }
193
-
194
- /**
195
- * Map a `Shortcut` descriptor onto a pi `registerTool({...})` call.
196
- *
197
- * Each tool's `execute(...)` body forwards into the shortcut runtime
198
- * (`ctx.getShortcutKB().invoke(...)`), translating the structured
199
- * `ShortcutResult` into the LLM-visible content array. Failures are
200
- * **not** thrown — instead we return `{ content: [{type:"text", text}],
201
- * details: { error_code } }` so the LLM sees a structured failure code
202
- * (for example `"app_not_found"`) and can fall back to `mobile_use`
203
- * without a stack trace.
204
- */
205
- function registerShortcutAsTool(
206
- pi: ExtensionAPI,
207
- shortcut: Shortcut<unknown, unknown>,
208
- ): void {
209
- const d = shortcut.descriptor;
210
- pi.registerTool({
211
- name: d.id,
212
- label: d.displayName,
213
- description: d.description,
214
- promptSnippet: `${d.id}: ${d.displayName.toLowerCase()} (${d.kind})`,
215
- // pi-coding-agent accepts JSON-Schema-shaped objects directly —
216
- // TypeBox's `Type.Object` returns one anyway. Forwarding the raw
217
- // schema lets shortcut authors describe parameters without
218
- // requiring a TypeBox dependency.
219
- parameters: d.jsonSchema as never,
220
- async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
221
- onUpdate?.({
222
- content: [{ type: "text", text: `[${d.id}] invoke\n` }],
223
- details: null,
224
- });
225
- const ctx = getPiExtensionContext();
226
- const invoker = ctx.getShortcutKB();
227
- const log = (await import("../../src/logger/index.js")).getLogger();
228
- // Pull the shared resource facade so shortcuts that need
229
- // `ctx.resources.resolveUrl` (e.g. `download_to_device`) see
230
- // the same registry+cache as the AgentHost auto-trigger. The
231
- // accessor is optional; tests / standalone bootstraps that
232
- // don't wire a facade get `undefined` and the shortcut surfaces
233
- // `permission_denied` to the LLM (intentional fail-soft).
234
- const resources = ctx.getShortcutResources?.();
235
- const callerCtx = {
236
- abortSignal: signal ?? new AbortController().signal,
237
- log,
238
- ...(resources ? { resources } : {}),
239
- };
240
- const res: ShortcutResult<unknown> = await invoker.invoke(d.id, params, callerCtx);
241
- return formatShortcutResult(d.id, res);
242
- },
243
- });
244
- }
245
-
246
- /**
247
- * Translate a `ShortcutResult` into a pi tool result. We return a
248
- * **single text block** containing JSON the LLM can parse, plus a
249
- * `details` blob for the orchestrator-side hooks. Errors NEVER throw —
250
- * the LLM should be able to inspect the failure and choose a fallback
251
- * path (typically `mobile_use`). Throwing would skip pi's tool-error
252
- * recovery and bubble up as an unhandled exception in the chat.
253
- */
254
- function formatShortcutResult(
255
- id: string,
256
- res: ShortcutResult<unknown>,
257
- ): { content: Array<{ type: "text"; text: string }>; details: unknown } {
258
- if (res.ok) {
259
- return {
260
- content: [
261
- {
262
- type: "text",
263
- text: JSON.stringify({ ok: true, output: res.output }, null, 2),
264
- },
265
- ],
266
- details: { shortcut_id: id, ok: true, durationMs: res.durationMs ?? null },
267
- };
268
- }
269
- return {
270
- content: [
271
- {
272
- type: "text",
273
- text: JSON.stringify(
274
- {
275
- ok: false,
276
- code: res.code,
277
- message: res.message,
278
- },
279
- null,
280
- 2,
281
- ),
282
- },
283
- ],
284
- details: {
285
- shortcut_id: id,
286
- ok: false,
287
- code: res.code,
288
- durationMs: res.durationMs ?? null,
289
- },
290
- };
291
- }
File without changes
@@ -1,50 +0,0 @@
1
- ---
2
- name: ground-mobile-fallback
3
- description: 当 mobile_use 失败或不可用时,指导外层编排 Agent 用 ground + adb(及可选 ui_xml)分步操作 Android:先理解界面、再视觉定位、再执行点击/滑动,作为一体化 VLM 回退路径。
4
- ---
5
-
6
- # mobile_use 不可用时的手机操作回退(ground + adb)
7
-
8
- ## 何时启用
9
-
10
- 在以下情况**不要**再盲目重试 `mobile_use`,改用本流程:
11
-
12
- - `mobile_use` 报错(API、超时、鉴权、设备断开等)。
13
- - 任务极短、只需 1~3 次精确点击,一体化循环性价比低或易跑偏。
14
- - 你需要**可解释**的坐标(例如调试、或要向用户说明点在哪里)。
15
-
16
- 若 `ground` 所依赖的 Grounder(`GROUNDER_*` / 与 `GUI_AGENT_*` 共用的 VLM)也不可用,本回退无效,应先恢复网络与密钥或改任务。
17
-
18
- ## 工具分工(外层负责编排)
19
-
20
- | 工具 | 作用 |
21
- |------|------|
22
- | `ui_xml` | 可选。看当前界面结构、文案、`bounds` / `center`,缩小「要点哪里」的描述范围。 |
23
- | `ground` | 对**当前屏**截图(或你传入的 `image_b64`),根据自然语言描述返回目标大致像素位置(模型原始输出多为类似 `[x, y]` 的坐标;以工具返回文本为准并解析)。 |
24
- | Bash + `adb` | 在解析出坐标后执行 `adb shell input tap x y`、`input swipe ...` 等(需已连接设备,必要时设置 `GUI_AGENT_DEVICE_ID`)。 |
25
-
26
- ## 推荐流程
27
-
28
- 1. **确认设备**:`adb devices`;多设备时指定 serial(与项目环境变量一致)。
29
- 2. **(可选)** `ui_xml` 带 `query` 搜索按钮/列表项文案,确认目标在屏上且描述与界面语言一致。
30
- 3. **`ground(query)`**:`query` 写具体、可区分(例如「设置列表里的 WLAN 项」而非「网络」)。避免一次描述多个无关目标。
31
- 4. **解析坐标**:从返回文本中提取整数像素坐标;若有多个候选,优先与 `ui_xml` 的 `center` 接近的点。
32
- 5. **`adb shell input tap x y`**。若未生效:等待动画后再 `ground` 一次(界面已变)或先 `ui_xml` 看是否被遮挡/未 clickable。
33
- 6. **输入文字**:`adb shell input text '...'`(注意 shell 转义;复杂文本可改用系统剪贴板或拆分步骤)。
34
- 7. **滑动**:需要时 `input swipe x1 y1 x2 y2 duration_ms`;长列表可多次滑动后再 `ground`。
35
-
36
- ## 与 mobile_use 的配合
37
-
38
- - 正常情况仍以 **`mobile_use` 为主**(一次调用内多步截图→推理→执行)。
39
- - 仅当 `mobile_use` **明确失败**或**不适合**时,再切换到「`ui_xml` + `ground` + adb」的显式编排。
40
- - 回退路径步数多、对描述敏感;每步后根据新截图或 `ui_xml` 校验是否到达预期界面。
41
-
42
- ## 环境变量提示(便于排查)
43
-
44
- - 设备:`GUI_AGENT_DEVICE_ID`
45
- - 截图与 Grounder 常与内层共用:`GUI_AGENT_BASE_URL`、`GUI_AGENT_API_KEY`、`GUI_AGENT_MODEL`;若单独配置 Grounder:`GROUNDER_BASE_URL`、`GROUNDER_API_KEY`、`GROUNDER_MODEL`
46
-
47
- ## 禁止事项
48
-
49
- - 不要在未确认坐标的情况下连续乱点同一区域。
50
- - 不要用模糊到无法在一屏内唯一定位的 `query` 调用 `ground`(先 `ui_xml` 或滚动到目标大致出现)。
@@ -1,237 +0,0 @@
1
- /**
2
- * Adapter — PiOrchestratorAgent (two-layer outer orchestrator).
3
- *
4
- * Wraps `@mariozechner/pi-coding-agent`'s `AgentSession` so it can be
5
- * plugged into the `IAgentCore` slot. The outer layer is the LLM that
6
- * pi drives; the inner layer is whatever extensions are loaded from
7
- * `<projectRoot>/.pi/extensions/` (e.g. `mobile_use`, `ground`,
8
- * `ui_xml`). The adapter is **transport-agnostic** — it only knows about
9
- * pi events and `AgentHost` callbacks.
10
- *
11
- * Key responsibilities:
12
- * - Lazy session bootstrap (extension/skill scanning, model resolution)
13
- * on first `run()` call.
14
- * - Map pi's session events to `IAgentCore` callbacks:
15
- * `message_update / text_delta` → `onText`
16
- * `tool_execution_start/update/end` → `onTool` (start/update/end)
17
- * `message_end (assistant role)` → captures the final assistant text
18
- * - Translate `setModel(provider, model)` into pi-ai's `getModel()` and
19
- * forward to `session.setModel`. Hot-swap is best-effort — when model
20
- * resolution fails we keep the previous model and surface a warning.
21
- * - Honor `abortSignal` by calling `session.abort()`.
22
- * - Map ARouter / OpenRouter / OpenAI / Anthropic / Google base URLs to
23
- * pi-ai's known provider env vars (e.g. `OPENROUTER_API_KEY`) so
24
- * callers can pass a single `outer.apiKey` without exporting
25
- * provider-specific env names.
26
- *
27
- * NOT responsibilities:
28
- * - Loading extensions (caller passes the resourceLoader options).
29
- * - Setting up the device — pi extensions read device handles from
30
- * process env / DEVAGENT_HTTP_URL like before.
31
- */
32
- import type { AgentCoreRunOptions, AgentCoreRunResult, IAgentCore } from "../ports/agent-core.js";
33
- import type { Logger } from "../logger/index.js";
34
- import type { IGrounder } from "../grounding/grounder.js";
35
- import type { IMcpRuntime } from "../mcp-client/runtime.js";
36
- import type { ShortcutInvoker, ShortcutResourceFacade } from "../shortcuts/index.js";
37
- export interface PiOuterConfig {
38
- /** API key for the outer LLM. Mapped to `OPENROUTER_API_KEY` etc. */
39
- apiKey?: string;
40
- /** Base URL — used to infer the pi-ai provider. Default ARouter. */
41
- baseUrl?: string;
42
- /** Upstream model id (e.g. `moonshotai/kimi-k2.6`). */
43
- model?: string;
44
- /**
45
- * Optional explicit pi-ai provider tag (`openrouter`, `openai`, …).
46
- * When empty the provider is inferred from `baseUrl`.
47
- */
48
- provider?: string;
49
- }
50
- /**
51
- * Minimal AgentSession surface that the adapter actually consumes.
52
- * Mirrors `@mariozechner/pi-coding-agent` so we can swap in a fake in
53
- * unit tests without pulling pi-ai's heavy types into the test boundary.
54
- */
55
- export interface PiAgentSessionLike {
56
- prompt(text: string): Promise<void>;
57
- abort(): Promise<void>;
58
- setModel(model: unknown): Promise<void>;
59
- subscribe(listener: (event: PiSessionEvent) => void | Promise<void>): () => void;
60
- dispose(): void;
61
- }
62
- /**
63
- * Subset of pi's `AgentSessionEvent` we care about. Defined locally so
64
- * tests don't need to import pi types and so the adapter is robust to
65
- * upstream additions.
66
- */
67
- export type PiSessionEvent = {
68
- type: "message_start";
69
- message: PiMessageLike;
70
- } | {
71
- type: "message_update";
72
- message: PiMessageLike;
73
- assistantMessageEvent: PiAssistantEvent;
74
- } | {
75
- type: "message_end";
76
- message: PiMessageLike;
77
- } | {
78
- type: "tool_execution_start";
79
- toolCallId: string;
80
- toolName: string;
81
- args: unknown;
82
- } | {
83
- type: "tool_execution_update";
84
- toolCallId: string;
85
- toolName: string;
86
- args: unknown;
87
- partialResult: unknown;
88
- } | {
89
- type: "tool_execution_end";
90
- toolCallId: string;
91
- toolName: string;
92
- result: unknown;
93
- isError: boolean;
94
- } | {
95
- type: string;
96
- [k: string]: unknown;
97
- };
98
- interface PiMessageLike {
99
- role: string;
100
- content: unknown;
101
- }
102
- type PiAssistantEvent = {
103
- type: "text_delta";
104
- delta: string;
105
- [k: string]: unknown;
106
- } | {
107
- type: "thinking_delta";
108
- delta: string;
109
- [k: string]: unknown;
110
- } | {
111
- type: string;
112
- [k: string]: unknown;
113
- };
114
- /**
115
- * Test seam — provide a custom session factory. Default factory loads
116
- * `@mariozechner/pi-coding-agent` lazily so building tests / standalone
117
- * binaries that skip the orchestrator never imports pi-ai at all.
118
- */
119
- export interface PiSessionFactory {
120
- (config: PiSessionFactoryConfig): Promise<PiAgentSessionLike>;
121
- }
122
- export interface PiSessionFactoryConfig {
123
- projectRoot: string;
124
- systemPrompt?: string;
125
- /** Resolved pi-ai Model object (already URL-rewritten for ARouter). */
126
- model?: unknown;
127
- }
128
- export interface PiOrchestratorAgentOptions {
129
- /** Project root passed to pi's `DefaultResourceLoader`. */
130
- projectRoot: string;
131
- /** Outer-layer LLM config. */
132
- outer: PiOuterConfig;
133
- /**
134
- * Optional override for pi's system prompt. Defaults to
135
- * `ORCHESTRATOR_SYSTEM_PROMPT` (see `orchestrator-system-prompt.ts`).
136
- */
137
- systemPrompt?: string;
138
- /** Structured logger; falls back to console.warn. */
139
- logger?: Pick<Logger, "warn" | "info" | "debug">;
140
- /** Inject a custom session factory (used by tests). */
141
- createSession?: PiSessionFactory;
142
- /** Override pi-ai `getModel`. Used by tests to skip the heavy import. */
143
- resolveModel?: ModelResolver;
144
- /**
145
- * Resolves the active `IMcpRuntime` on every tool call. Typically
146
- * `() => mcpClient` (the sibling `device-mcp-server` HTTP client) or an
147
- * `InProcMcpClient` for local CLI usage. Resolved lazily so an ACP
148
- * device switch (or autospawn restart) is reflected immediately on the
149
- * next `mobile_use` / `ui_xml` / `ground` invocation.
150
- */
151
- mcpProvider?: () => IMcpRuntime;
152
- /**
153
- * Lazily build the grounder used by the `ground` extension. Called the
154
- * first time the extension reads `getGrounder()` within a session.
155
- * Typical implementation: `() => new HTTPGrounder(config.grounder)`.
156
- */
157
- grounderFactory?: () => IGrounder;
158
- /**
159
- * Run a `mobile_use` task end-to-end and return the final assistant
160
- * message. Wraps `InnerLoopVlm.runTask`. Letting the bootstrap layer
161
- * supply this keeps the heavy `OpenAIClient` instance shared with the
162
- * legacy single-layer path (and lets tests stub it out).
163
- */
164
- mobileUseRunner?: (task: string, maxSteps: number, abortSignal?: AbortSignal) => Promise<string>;
165
- /**
166
- * Optional UI-XML formatter. When omitted the adapter calls
167
- * `mcp.uiDump({ format: "raw" })` and feeds the result through
168
- * `formatUiXmlToolOutput` — same behaviour the legacy `queryUiXml`
169
- * used to provide on top of the raw `Device`. Tests can override to
170
- * skip the dump path entirely.
171
- */
172
- uiXmlRunner?: (query?: string) => Promise<string>;
173
- /**
174
- * Active shortcut runtime (Phase 2 of the GUI-Shortcut Hybrid Plan).
175
- * When set, the orchestrator extension renders one outer-LLM tool
176
- * per registered shortcut. When omitted, only the legacy
177
- * `mobile_use` / `ground` / `ui_xml` triad is exposed and any
178
- * extension that calls `getShortcutKB()` will throw — that's
179
- * intentional, the bootstrap layer is responsible for wiring this.
180
- */
181
- shortcutInvoker?: ShortcutInvoker;
182
- /**
183
- * Optional resource facade used by `ShortcutInvoker` callers (the
184
- * `download_to_device` builtin in particular). The bootstrap layer
185
- * supplies a lazy factory because the resource layer is constructed
186
- * AFTER the agent core in `bootstrap.ts`. Returning `undefined`
187
- * means the `download_to_device` shortcut will surface
188
- * `permission_denied` to the LLM, which is preferable to silently
189
- * misbehaving — but bootstrap should normally wire a facade.
190
- */
191
- shortcutResourcesFactory?: () => ShortcutResourceFacade | undefined;
192
- }
193
- export interface ModelResolver {
194
- (provider: string, modelId: string, baseUrl: string): unknown | undefined;
195
- }
196
- export declare class PiOrchestratorAgent implements IAgentCore {
197
- private session;
198
- private starting;
199
- private outer;
200
- private readonly projectRoot;
201
- private readonly systemPrompt;
202
- private readonly logger;
203
- private readonly createSessionFactory;
204
- private readonly resolveModelFn;
205
- private readonly mcpProvider?;
206
- private readonly grounderFactory?;
207
- private readonly mobileUseRunner?;
208
- private readonly uiXmlRunner?;
209
- private readonly shortcutInvoker?;
210
- private readonly shortcutResourcesFactory?;
211
- private cachedGrounder;
212
- constructor(opts: PiOrchestratorAgentOptions);
213
- get modelName(): string;
214
- get providerName(): string | undefined;
215
- run(opts: AgentCoreRunOptions): Promise<AgentCoreRunResult>;
216
- setModel(provider: string, model: string): Promise<void>;
217
- /** Tear down the underlying pi session. Idempotent. */
218
- dispose(): Promise<void>;
219
- private buildExtensionContext;
220
- private ensureSession;
221
- }
222
- /**
223
- * Map our generic outer config to pi-ai's env-var contract. pi-ai reads
224
- * provider-specific keys (e.g. `OPENROUTER_API_KEY`) from `process.env`,
225
- * so we transparently set them when the caller hands us `outer.apiKey`
226
- * + a recognised baseUrl. Never overwrites a key the caller already set.
227
- *
228
- * Recognised hosts:
229
- * - openrouter.ai / arouter.ai → OPENROUTER_API_KEY
230
- * - api.openai.com → OPENAI_API_KEY
231
- * - api.anthropic.com → ANTHROPIC_API_KEY
232
- * - generativelanguage.googleapis.com → GEMINI_API_KEY
233
- */
234
- export declare function applyOuterEnvMapping(outer: PiOuterConfig): void;
235
- /** Infer the pi-ai provider tag from a base URL. */
236
- export declare function inferPiProvider(baseUrl: string): string;
237
- export {};