@beeos-ai/device-agent 0.9.9 → 0.9.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/agentcore-inner-loop.d.ts +3 -4
- package/dist/adapters/agentcore-inner-loop.js +3 -4
- package/dist/adapters/agentcore-inner-loop.js.map +1 -1
- package/dist/adapters/agentcore-orchestrator.d.ts +36 -0
- package/dist/adapters/agentcore-orchestrator.js +203 -0
- package/dist/adapters/agentcore-orchestrator.js.map +1 -0
- package/dist/adapters/agentcore-runner.d.ts +3 -3
- package/dist/adapters/agentcore-runner.js +3 -3
- package/dist/adapters/app-guide.d.ts +16 -0
- package/dist/adapters/app-guide.js +101 -0
- package/dist/adapters/app-guide.js.map +1 -0
- package/dist/adapters/orchestrator-agent-prompt.d.ts +7 -0
- package/dist/adapters/orchestrator-agent-prompt.js +59 -0
- package/dist/adapters/orchestrator-agent-prompt.js.map +1 -0
- package/dist/adapters/vlm-inner-loop.d.ts +6 -0
- package/dist/adapters/vlm-inner-loop.js +3 -0
- package/dist/adapters/vlm-inner-loop.js.map +1 -1
- package/dist/agent-runtime/core.js +1 -1
- package/dist/agent-runtime/core.js.map +1 -1
- package/dist/agent-runtime/shortcuts/types.d.ts +2 -3
- package/dist/agent-runtime/shortcuts/types.js +2 -3
- package/dist/agent-runtime/shortcuts/types.js.map +1 -1
- package/dist/api.d.ts +1 -1
- package/dist/api.js +1 -1
- package/dist/bootstrap.d.ts +4 -4
- package/dist/bootstrap.js +47 -61
- package/dist/bootstrap.js.map +1 -1
- package/dist/config/loader.d.ts +2 -2
- package/dist/config/loader.js +2 -2
- package/dist/config/loader.js.map +1 -1
- package/dist/config/schema.d.ts +1 -1
- package/dist/config/schema.js +4 -5
- package/dist/config/schema.js.map +1 -1
- package/dist/device/apps.js +1 -1
- package/dist/device/apps.js.map +1 -1
- package/dist/fleet/domain/events.d.ts +22 -1
- package/dist/fleet/domain/fsm.d.ts +23 -1
- package/dist/fleet/domain/fsm.js +29 -0
- package/dist/fleet/domain/fsm.js.map +1 -1
- package/dist/fleet/supervisor.d.ts +13 -0
- package/dist/fleet/supervisor.js +90 -0
- package/dist/fleet/supervisor.js.map +1 -1
- package/dist/grounding/grounder.d.ts +1 -1
- package/dist/inbound/cli/main.d.ts +1 -1
- package/dist/inbound/cli/main.js +4 -4
- package/dist/inbound/cli/main.js.map +1 -1
- package/dist/inbound/cli/run-command.d.ts +6 -0
- package/dist/inbound/cli/run-command.js +94 -0
- package/dist/inbound/cli/run-command.js.map +1 -1
- package/dist/index.d.ts +2 -4
- package/dist/index.js +1 -2
- package/dist/index.js.map +1 -1
- package/dist/model/adapters/general.d.ts +11 -0
- package/dist/model/adapters/general.js +249 -0
- package/dist/model/adapters/general.js.map +1 -0
- package/dist/model/adapters/index.d.ts +1 -0
- package/dist/model/adapters/index.js +1 -0
- package/dist/model/adapters/index.js.map +1 -1
- package/dist/plugins/message-bus/ms-channel-notifier.d.ts +24 -0
- package/dist/plugins/message-bus/ms-channel-notifier.js +179 -12
- package/dist/plugins/message-bus/ms-channel-notifier.js.map +1 -1
- package/dist/ports/agent-core.d.ts +6 -5
- package/dist/shortcuts/types.d.ts +3 -6
- package/dist/shortcuts/types.js +1 -2
- package/dist/shortcuts/types.js.map +1 -1
- package/package.json +3 -6
- package/.pi/extensions/android-tools.ts +0 -291
- package/.pi/skills/.gitkeep +0 -0
- package/.pi/skills/ground-mobile-fallback/SKILL.md +0 -50
- package/dist/adapters/agentcore-pi.d.ts +0 -237
- package/dist/adapters/agentcore-pi.js +0 -512
- package/dist/adapters/agentcore-pi.js.map +0 -1
- package/dist/chat.d.ts +0 -16
- package/dist/chat.js +0 -185
- package/dist/chat.js.map +0 -1
- package/dist/debug/dump-system-prompt.d.ts +0 -1
- package/dist/debug/dump-system-prompt.js +0 -34
- package/dist/debug/dump-system-prompt.js.map +0 -1
- package/dist/orchestrator-system-prompt.d.ts +0 -32
- package/dist/orchestrator-system-prompt.js +0 -197
- package/dist/orchestrator-system-prompt.js.map +0 -1
- package/dist/pi-context.d.ts +0 -132
- package/dist/pi-context.js +0 -62
- package/dist/pi-context.js.map +0 -1
- package/dist/pi-local-skills.d.ts +0 -17
- package/dist/pi-local-skills.js +0 -24
- package/dist/pi-local-skills.js.map +0 -1
|
@@ -1,291 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pi extension: registers tools for the **outer** orchestrator.
|
|
3
|
-
*
|
|
4
|
-
* Two layers of tools coexist:
|
|
5
|
-
*
|
|
6
|
-
* - **Legacy GUI primitives** — `mobile_use`, `ground`, `ui_xml`.
|
|
7
|
-
* These wrap the inner VLM loop and visual grounder; they are
|
|
8
|
-
* always present and don't depend on the shortcut runtime.
|
|
9
|
-
*
|
|
10
|
-
* - **Hybrid Action Space (Phase 2)** — one outer-LLM tool per
|
|
11
|
-
* shortcut registered with `ShortcutKB`. Each shortcut's
|
|
12
|
-
* descriptor (id, displayName, description, jsonSchema) is
|
|
13
|
-
* mapped to a pi `registerTool({...})` call here. The LLM never
|
|
14
|
-
* sees raw `runShell` / `pushFile` — those are explicitly hidden
|
|
15
|
-
* behind the `Shortcut.execute` boundary (see D9 in the plan and
|
|
16
|
-
* the comment block in `pi-context.ts`).
|
|
17
|
-
*
|
|
18
|
-
* Why register dynamically instead of hard-coding 4 tools?
|
|
19
|
-
*
|
|
20
|
-
* The shortcut KB is the source of truth — the bootstrap layer
|
|
21
|
-
* decides which shortcuts to register, and the extension picks up
|
|
22
|
-
* whatever is there at session-start. New shortcuts (or future
|
|
23
|
-
* plugin packages) auto-light up without an extension code change.
|
|
24
|
-
* The trade-off is that the descriptor *cannot* depend on context
|
|
25
|
-
* that varies per call — but JSON Schema + display name + free-form
|
|
26
|
-
* description is exactly that, so we don't lose anything.
|
|
27
|
-
*
|
|
28
|
-
* Extension lifecycle (since the dual-layer + MCP refactor):
|
|
29
|
-
*
|
|
30
|
-
* PiOrchestratorAgent.ensureSession()
|
|
31
|
-
* └─ setPiExtensionContext({ getMcpClient, getGrounder, runMobileUse,
|
|
32
|
-
* getUiXml, getShortcutKB, runShell, pushFile })
|
|
33
|
-
* └─ pi.loader.reload() → loads this file → pi.registerTool(...)
|
|
34
|
-
* └─ pi.session.prompt(...) → tools execute → context.* invoked
|
|
35
|
-
*
|
|
36
|
-
* PiOrchestratorAgent.dispose()
|
|
37
|
-
* └─ setPiExtensionContext(null)
|
|
38
|
-
*
|
|
39
|
-
* The extension is the ONLY pi-coupling point. Bootstrap supplies the
|
|
40
|
-
* concrete dependencies — this file stays free of process.env reads,
|
|
41
|
-
* ad-hoc `new AndroidDevice()` / `new HTTPGrounder()` calls, and direct
|
|
42
|
-
* MCP HTTP knowledge.
|
|
43
|
-
*/
|
|
44
|
-
|
|
45
|
-
import type { ExtensionAPI, ExtensionContext } from "@mariozechner/pi-coding-agent";
|
|
46
|
-
import { Type } from "@sinclair/typebox";
|
|
47
|
-
import { getPiExtensionContext } from "../../src/pi-context.js";
|
|
48
|
-
import type { Shortcut, ShortcutResult } from "../../src/shortcuts/index.js";
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Server-side floor for `mobile_use` step budget.
|
|
52
|
-
*
|
|
53
|
-
* The outer orchestrator LLM frequently misjudges ADB / Android tasks as
|
|
54
|
-
* "simple actions" and emits `max_steps: 5..15`, which the inner
|
|
55
|
-
* `Agent.run` loop blows through before the task can finish — surfacing as
|
|
56
|
-
* "Reached max steps (N)". Clamping upward at the tool boundary protects
|
|
57
|
-
* the user without preventing the LLM from explicitly asking for more
|
|
58
|
-
* steps when it judges a flow as complex.
|
|
59
|
-
*/
|
|
60
|
-
const MIN_MOBILE_USE_STEPS = 30;
|
|
61
|
-
|
|
62
|
-
export default function (pi: ExtensionAPI) {
|
|
63
|
-
// ── mobile_use ─────────────────────────────────────────────────────────────
|
|
64
|
-
pi.registerTool({
|
|
65
|
-
name: "mobile_use",
|
|
66
|
-
label: "Mobile Use",
|
|
67
|
-
description:
|
|
68
|
-
"Drive the connected Android phone with a natural-language task. " +
|
|
69
|
-
"A VLM runs its own screenshot→reason→act loop until it finishes or hits max-steps. " +
|
|
70
|
-
"Pass a full, concrete instruction with app-native UI text. " +
|
|
71
|
-
"For long text (especially Chinese), embed it verbatim and note '原样输入、不要翻译'. " +
|
|
72
|
-
"One call runs 20–50 steps autonomously; do NOT call repeatedly for the same task.",
|
|
73
|
-
promptSnippet: "mobile_use(task): drive Android phone via VLM screenshot→reason→act loop",
|
|
74
|
-
parameters: Type.Object({
|
|
75
|
-
task: Type.String({ description: "Full natural-language instruction in Chinese or English" }),
|
|
76
|
-
max_steps: Type.Integer({
|
|
77
|
-
description: "Maximum VLM steps for this call. Required. Typical range 20-50 depending on task complexity.",
|
|
78
|
-
minimum: 1,
|
|
79
|
-
maximum: 100,
|
|
80
|
-
}),
|
|
81
|
-
}),
|
|
82
|
-
async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
|
|
83
|
-
onUpdate?.({ content: [{ type: "text", text: `[mobile_use] ${params.task}\n` }], details: null });
|
|
84
|
-
if (signal?.aborted) throw new Error("aborted");
|
|
85
|
-
|
|
86
|
-
const ctx = getPiExtensionContext();
|
|
87
|
-
const requested = params.max_steps;
|
|
88
|
-
const effective = Math.max(MIN_MOBILE_USE_STEPS, requested);
|
|
89
|
-
if (effective !== requested) {
|
|
90
|
-
const log = (await import("../../src/logger/index.js")).getLogger();
|
|
91
|
-
log.warn(
|
|
92
|
-
{ component: "mobile_use", requested, effective, floor: MIN_MOBILE_USE_STEPS },
|
|
93
|
-
"mobile_use.max_steps clamped up to floor",
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
const result = await ctx.runMobileUse(params.task, effective, signal ?? undefined);
|
|
97
|
-
return { content: [{ type: "text", text: result }], details: null };
|
|
98
|
-
},
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
// ── ground ─────────────────────────────────────────────────────────────────
|
|
102
|
-
// Visual grounding tool: find where a described UI element is on screen.
|
|
103
|
-
// Takes a screenshot from the connected device (or uses a caller-supplied image),
|
|
104
|
-
// calls the configured grounder VLM, and returns pixel coordinates.
|
|
105
|
-
pi.registerTool({
|
|
106
|
-
name: "ground",
|
|
107
|
-
label: "Visual Ground",
|
|
108
|
-
description:
|
|
109
|
-
"Locate a UI element on the Android phone screen and return its pixel coordinates. " +
|
|
110
|
-
"Describe the element in natural language (e.g. '确认按钮', 'Settings icon', 'search box'). " +
|
|
111
|
-
"If image_b64 is omitted, a fresh screenshot is taken automatically. " +
|
|
112
|
-
"Returns JSON: { x, y, width, height, raw } where x/y are absolute pixels from the top-left. " +
|
|
113
|
-
"Use this when you need to know where to tap without running the full mobile_use loop.",
|
|
114
|
-
promptSnippet: "ground(query, image_b64?): locate UI element on screen → {x, y}",
|
|
115
|
-
parameters: Type.Object({
|
|
116
|
-
query: Type.String({ description: "Natural-language description of the element to locate" }),
|
|
117
|
-
image_b64: Type.Optional(
|
|
118
|
-
Type.String({ description: "Base64-encoded JPEG/PNG image. If omitted, a screenshot is taken." }),
|
|
119
|
-
),
|
|
120
|
-
}),
|
|
121
|
-
async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
|
|
122
|
-
onUpdate?.({ content: [{ type: "text", text: `[ground] "${params.query}"\n` }], details: null });
|
|
123
|
-
if (signal?.aborted) throw new Error("aborted");
|
|
124
|
-
|
|
125
|
-
const ctx = getPiExtensionContext();
|
|
126
|
-
let imageB64 = params.image_b64 ?? null;
|
|
127
|
-
|
|
128
|
-
if (!imageB64) {
|
|
129
|
-
const ss = await ctx.getMcpClient().screenshot();
|
|
130
|
-
imageB64 = ss.b64;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
const result = await ctx.getGrounder().ground(imageB64, params.query);
|
|
134
|
-
|
|
135
|
-
return {
|
|
136
|
-
content: [{ type: "text", text: result ?? "Element not found" }],
|
|
137
|
-
details: null,
|
|
138
|
-
};
|
|
139
|
-
},
|
|
140
|
-
});
|
|
141
|
-
|
|
142
|
-
// ── ui_xml ─────────────────────────────────────────────────────────────────
|
|
143
|
-
pi.registerTool({
|
|
144
|
-
name: "ui_xml",
|
|
145
|
-
label: "UI XML",
|
|
146
|
-
description:
|
|
147
|
-
"Dump the Android UI hierarchy from the connected device and return it as text. " +
|
|
148
|
-
"Without a query: returns the raw uiautomator XML (truncated at 8000 chars). " +
|
|
149
|
-
"With a query: searches nodes by text / resource-id / content-desc / class name " +
|
|
150
|
-
"(case-insensitive substring match) and returns matching nodes as JSON with: " +
|
|
151
|
-
"text, resourceId, contentDesc, className, bounds ([x1,y1][x2,y2]), center {x,y}, clickable. " +
|
|
152
|
-
"Use this to inspect the current screen structure, find resource IDs, or check element states.",
|
|
153
|
-
promptSnippet: "ui_xml(query?): dump or search UI hierarchy → raw XML or [{text, bounds, center, ...}]",
|
|
154
|
-
parameters: Type.Object({
|
|
155
|
-
query: Type.Optional(
|
|
156
|
-
Type.String({ description: "Substring to match against text/resource-id/content-desc/class. Omit for raw XML." }),
|
|
157
|
-
),
|
|
158
|
-
}),
|
|
159
|
-
async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
|
|
160
|
-
onUpdate?.({ content: [{ type: "text", text: `[ui_xml] query=${params.query ?? "(raw)"}\n` }], details: null });
|
|
161
|
-
if (signal?.aborted) throw new Error("aborted");
|
|
162
|
-
|
|
163
|
-
const ctx = getPiExtensionContext();
|
|
164
|
-
const out = await ctx.getUiXml(params.query);
|
|
165
|
-
return { content: [{ type: "text", text: out }], details: null };
|
|
166
|
-
},
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
// ── Hybrid Action Space (Phase 2): one tool per shortcut ─────────────────
|
|
170
|
-
// The shortcut KB may not be wired (legacy bootstrap, tests). When it
|
|
171
|
-
// isn't, we silently skip — the legacy primitives above remain
|
|
172
|
-
// functional, the orchestrator just doesn't see any shortcut tools.
|
|
173
|
-
// Failing loud here would block every legacy path that doesn't need
|
|
174
|
-
// the hybrid layer.
|
|
175
|
-
let shortcuts: Shortcut<unknown, unknown>[] = [];
|
|
176
|
-
try {
|
|
177
|
-
const ctx = getPiExtensionContext();
|
|
178
|
-
shortcuts = ctx.getShortcutKB().registry.list();
|
|
179
|
-
} catch {
|
|
180
|
-
shortcuts = [];
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
for (const shortcut of shortcuts) {
|
|
184
|
-
registerShortcutAsTool(pi, shortcut);
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// ── mobile_use_brain_actor ─ (commented out, not exposed to orchestrator)
|
|
188
|
-
// pi.registerTool({ name: "mobile_use_brain_actor", ... execute: mobileUseBrainActor })
|
|
189
|
-
|
|
190
|
-
// ── mobile_use_grounding ─ (commented out, not exposed to orchestrator)
|
|
191
|
-
// pi.registerTool({ name: "mobile_use_grounding", ... execute: mobileUseGrounding })
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
/**
|
|
195
|
-
* Map a `Shortcut` descriptor onto a pi `registerTool({...})` call.
|
|
196
|
-
*
|
|
197
|
-
* Each tool's `execute(...)` body forwards into the shortcut runtime
|
|
198
|
-
* (`ctx.getShortcutKB().invoke(...)`), translating the structured
|
|
199
|
-
* `ShortcutResult` into the LLM-visible content array. Failures are
|
|
200
|
-
* **not** thrown — instead we return `{ content: [{type:"text", text}],
|
|
201
|
-
* details: { error_code } }` so the LLM sees a structured failure code
|
|
202
|
-
* (for example `"app_not_found"`) and can fall back to `mobile_use`
|
|
203
|
-
* without a stack trace.
|
|
204
|
-
*/
|
|
205
|
-
function registerShortcutAsTool(
|
|
206
|
-
pi: ExtensionAPI,
|
|
207
|
-
shortcut: Shortcut<unknown, unknown>,
|
|
208
|
-
): void {
|
|
209
|
-
const d = shortcut.descriptor;
|
|
210
|
-
pi.registerTool({
|
|
211
|
-
name: d.id,
|
|
212
|
-
label: d.displayName,
|
|
213
|
-
description: d.description,
|
|
214
|
-
promptSnippet: `${d.id}: ${d.displayName.toLowerCase()} (${d.kind})`,
|
|
215
|
-
// pi-coding-agent accepts JSON-Schema-shaped objects directly —
|
|
216
|
-
// TypeBox's `Type.Object` returns one anyway. Forwarding the raw
|
|
217
|
-
// schema lets shortcut authors describe parameters without
|
|
218
|
-
// requiring a TypeBox dependency.
|
|
219
|
-
parameters: d.jsonSchema as never,
|
|
220
|
-
async execute(_id, params, signal, onUpdate, _ctx: ExtensionContext) {
|
|
221
|
-
onUpdate?.({
|
|
222
|
-
content: [{ type: "text", text: `[${d.id}] invoke\n` }],
|
|
223
|
-
details: null,
|
|
224
|
-
});
|
|
225
|
-
const ctx = getPiExtensionContext();
|
|
226
|
-
const invoker = ctx.getShortcutKB();
|
|
227
|
-
const log = (await import("../../src/logger/index.js")).getLogger();
|
|
228
|
-
// Pull the shared resource facade so shortcuts that need
|
|
229
|
-
// `ctx.resources.resolveUrl` (e.g. `download_to_device`) see
|
|
230
|
-
// the same registry+cache as the AgentHost auto-trigger. The
|
|
231
|
-
// accessor is optional; tests / standalone bootstraps that
|
|
232
|
-
// don't wire a facade get `undefined` and the shortcut surfaces
|
|
233
|
-
// `permission_denied` to the LLM (intentional fail-soft).
|
|
234
|
-
const resources = ctx.getShortcutResources?.();
|
|
235
|
-
const callerCtx = {
|
|
236
|
-
abortSignal: signal ?? new AbortController().signal,
|
|
237
|
-
log,
|
|
238
|
-
...(resources ? { resources } : {}),
|
|
239
|
-
};
|
|
240
|
-
const res: ShortcutResult<unknown> = await invoker.invoke(d.id, params, callerCtx);
|
|
241
|
-
return formatShortcutResult(d.id, res);
|
|
242
|
-
},
|
|
243
|
-
});
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
/**
|
|
247
|
-
* Translate a `ShortcutResult` into a pi tool result. We return a
|
|
248
|
-
* **single text block** containing JSON the LLM can parse, plus a
|
|
249
|
-
* `details` blob for the orchestrator-side hooks. Errors NEVER throw —
|
|
250
|
-
* the LLM should be able to inspect the failure and choose a fallback
|
|
251
|
-
* path (typically `mobile_use`). Throwing would skip pi's tool-error
|
|
252
|
-
* recovery and bubble up as an unhandled exception in the chat.
|
|
253
|
-
*/
|
|
254
|
-
function formatShortcutResult(
|
|
255
|
-
id: string,
|
|
256
|
-
res: ShortcutResult<unknown>,
|
|
257
|
-
): { content: Array<{ type: "text"; text: string }>; details: unknown } {
|
|
258
|
-
if (res.ok) {
|
|
259
|
-
return {
|
|
260
|
-
content: [
|
|
261
|
-
{
|
|
262
|
-
type: "text",
|
|
263
|
-
text: JSON.stringify({ ok: true, output: res.output }, null, 2),
|
|
264
|
-
},
|
|
265
|
-
],
|
|
266
|
-
details: { shortcut_id: id, ok: true, durationMs: res.durationMs ?? null },
|
|
267
|
-
};
|
|
268
|
-
}
|
|
269
|
-
return {
|
|
270
|
-
content: [
|
|
271
|
-
{
|
|
272
|
-
type: "text",
|
|
273
|
-
text: JSON.stringify(
|
|
274
|
-
{
|
|
275
|
-
ok: false,
|
|
276
|
-
code: res.code,
|
|
277
|
-
message: res.message,
|
|
278
|
-
},
|
|
279
|
-
null,
|
|
280
|
-
2,
|
|
281
|
-
),
|
|
282
|
-
},
|
|
283
|
-
],
|
|
284
|
-
details: {
|
|
285
|
-
shortcut_id: id,
|
|
286
|
-
ok: false,
|
|
287
|
-
code: res.code,
|
|
288
|
-
durationMs: res.durationMs ?? null,
|
|
289
|
-
},
|
|
290
|
-
};
|
|
291
|
-
}
|
package/.pi/skills/.gitkeep
DELETED
|
File without changes
|
|
@@ -1,50 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: ground-mobile-fallback
|
|
3
|
-
description: 当 mobile_use 失败或不可用时,指导外层编排 Agent 用 ground + adb(及可选 ui_xml)分步操作 Android:先理解界面、再视觉定位、再执行点击/滑动,作为一体化 VLM 回退路径。
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# mobile_use 不可用时的手机操作回退(ground + adb)
|
|
7
|
-
|
|
8
|
-
## 何时启用
|
|
9
|
-
|
|
10
|
-
在以下情况**不要**再盲目重试 `mobile_use`,改用本流程:
|
|
11
|
-
|
|
12
|
-
- `mobile_use` 报错(API、超时、鉴权、设备断开等)。
|
|
13
|
-
- 任务极短、只需 1~3 次精确点击,一体化循环性价比低或易跑偏。
|
|
14
|
-
- 你需要**可解释**的坐标(例如调试、或要向用户说明点在哪里)。
|
|
15
|
-
|
|
16
|
-
若 `ground` 所依赖的 Grounder(`GROUNDER_*` / 与 `GUI_AGENT_*` 共用的 VLM)也不可用,本回退无效,应先恢复网络与密钥或改任务。
|
|
17
|
-
|
|
18
|
-
## 工具分工(外层负责编排)
|
|
19
|
-
|
|
20
|
-
| 工具 | 作用 |
|
|
21
|
-
|------|------|
|
|
22
|
-
| `ui_xml` | 可选。看当前界面结构、文案、`bounds` / `center`,缩小「要点哪里」的描述范围。 |
|
|
23
|
-
| `ground` | 对**当前屏**截图(或你传入的 `image_b64`),根据自然语言描述返回目标大致像素位置(模型原始输出多为类似 `[x, y]` 的坐标;以工具返回文本为准并解析)。 |
|
|
24
|
-
| Bash + `adb` | 在解析出坐标后执行 `adb shell input tap x y`、`input swipe ...` 等(需已连接设备,必要时设置 `GUI_AGENT_DEVICE_ID`)。 |
|
|
25
|
-
|
|
26
|
-
## 推荐流程
|
|
27
|
-
|
|
28
|
-
1. **确认设备**:`adb devices`;多设备时指定 serial(与项目环境变量一致)。
|
|
29
|
-
2. **(可选)** `ui_xml` 带 `query` 搜索按钮/列表项文案,确认目标在屏上且描述与界面语言一致。
|
|
30
|
-
3. **`ground(query)`**:`query` 写具体、可区分(例如「设置列表里的 WLAN 项」而非「网络」)。避免一次描述多个无关目标。
|
|
31
|
-
4. **解析坐标**:从返回文本中提取整数像素坐标;若有多个候选,优先与 `ui_xml` 的 `center` 接近的点。
|
|
32
|
-
5. **`adb shell input tap x y`**。若未生效:等待动画后再 `ground` 一次(界面已变)或先 `ui_xml` 看是否被遮挡/未 clickable。
|
|
33
|
-
6. **输入文字**:`adb shell input text '...'`(注意 shell 转义;复杂文本可改用系统剪贴板或拆分步骤)。
|
|
34
|
-
7. **滑动**:需要时 `input swipe x1 y1 x2 y2 duration_ms`;长列表可多次滑动后再 `ground`。
|
|
35
|
-
|
|
36
|
-
## 与 mobile_use 的配合
|
|
37
|
-
|
|
38
|
-
- 正常情况仍以 **`mobile_use` 为主**(一次调用内多步截图→推理→执行)。
|
|
39
|
-
- 仅当 `mobile_use` **明确失败**或**不适合**时,再切换到「`ui_xml` + `ground` + adb」的显式编排。
|
|
40
|
-
- 回退路径步数多、对描述敏感;每步后根据新截图或 `ui_xml` 校验是否到达预期界面。
|
|
41
|
-
|
|
42
|
-
## 环境变量提示(便于排查)
|
|
43
|
-
|
|
44
|
-
- 设备:`GUI_AGENT_DEVICE_ID`
|
|
45
|
-
- 截图与 Grounder 常与内层共用:`GUI_AGENT_BASE_URL`、`GUI_AGENT_API_KEY`、`GUI_AGENT_MODEL`;若单独配置 Grounder:`GROUNDER_BASE_URL`、`GROUNDER_API_KEY`、`GROUNDER_MODEL`
|
|
46
|
-
|
|
47
|
-
## 禁止事项
|
|
48
|
-
|
|
49
|
-
- 不要在未确认坐标的情况下连续乱点同一区域。
|
|
50
|
-
- 不要用模糊到无法在一屏内唯一定位的 `query` 调用 `ground`(先 `ui_xml` 或滚动到目标大致出现)。
|
|
@@ -1,237 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Adapter — PiOrchestratorAgent (two-layer outer orchestrator).
|
|
3
|
-
*
|
|
4
|
-
* Wraps `@mariozechner/pi-coding-agent`'s `AgentSession` so it can be
|
|
5
|
-
* plugged into the `IAgentCore` slot. The outer layer is the LLM that
|
|
6
|
-
* pi drives; the inner layer is whatever extensions are loaded from
|
|
7
|
-
* `<projectRoot>/.pi/extensions/` (e.g. `mobile_use`, `ground`,
|
|
8
|
-
* `ui_xml`). The adapter is **transport-agnostic** — it only knows about
|
|
9
|
-
* pi events and `AgentHost` callbacks.
|
|
10
|
-
*
|
|
11
|
-
* Key responsibilities:
|
|
12
|
-
* - Lazy session bootstrap (extension/skill scanning, model resolution)
|
|
13
|
-
* on first `run()` call.
|
|
14
|
-
* - Map pi's session events to `IAgentCore` callbacks:
|
|
15
|
-
* `message_update / text_delta` → `onText`
|
|
16
|
-
* `tool_execution_start/update/end` → `onTool` (start/update/end)
|
|
17
|
-
* `message_end (assistant role)` → captures the final assistant text
|
|
18
|
-
* - Translate `setModel(provider, model)` into pi-ai's `getModel()` and
|
|
19
|
-
* forward to `session.setModel`. Hot-swap is best-effort — when model
|
|
20
|
-
* resolution fails we keep the previous model and surface a warning.
|
|
21
|
-
* - Honor `abortSignal` by calling `session.abort()`.
|
|
22
|
-
* - Map ARouter / OpenRouter / OpenAI / Anthropic / Google base URLs to
|
|
23
|
-
* pi-ai's known provider env vars (e.g. `OPENROUTER_API_KEY`) so
|
|
24
|
-
* callers can pass a single `outer.apiKey` without exporting
|
|
25
|
-
* provider-specific env names.
|
|
26
|
-
*
|
|
27
|
-
* NOT responsibilities:
|
|
28
|
-
* - Loading extensions (caller passes the resourceLoader options).
|
|
29
|
-
* - Setting up the device — pi extensions read device handles from
|
|
30
|
-
* process env / DEVAGENT_HTTP_URL like before.
|
|
31
|
-
*/
|
|
32
|
-
import type { AgentCoreRunOptions, AgentCoreRunResult, IAgentCore } from "../ports/agent-core.js";
|
|
33
|
-
import type { Logger } from "../logger/index.js";
|
|
34
|
-
import type { IGrounder } from "../grounding/grounder.js";
|
|
35
|
-
import type { IMcpRuntime } from "../mcp-client/runtime.js";
|
|
36
|
-
import type { ShortcutInvoker, ShortcutResourceFacade } from "../shortcuts/index.js";
|
|
37
|
-
export interface PiOuterConfig {
|
|
38
|
-
/** API key for the outer LLM. Mapped to `OPENROUTER_API_KEY` etc. */
|
|
39
|
-
apiKey?: string;
|
|
40
|
-
/** Base URL — used to infer the pi-ai provider. Default ARouter. */
|
|
41
|
-
baseUrl?: string;
|
|
42
|
-
/** Upstream model id (e.g. `moonshotai/kimi-k2.6`). */
|
|
43
|
-
model?: string;
|
|
44
|
-
/**
|
|
45
|
-
* Optional explicit pi-ai provider tag (`openrouter`, `openai`, …).
|
|
46
|
-
* When empty the provider is inferred from `baseUrl`.
|
|
47
|
-
*/
|
|
48
|
-
provider?: string;
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
* Minimal AgentSession surface that the adapter actually consumes.
|
|
52
|
-
* Mirrors `@mariozechner/pi-coding-agent` so we can swap in a fake in
|
|
53
|
-
* unit tests without pulling pi-ai's heavy types into the test boundary.
|
|
54
|
-
*/
|
|
55
|
-
export interface PiAgentSessionLike {
|
|
56
|
-
prompt(text: string): Promise<void>;
|
|
57
|
-
abort(): Promise<void>;
|
|
58
|
-
setModel(model: unknown): Promise<void>;
|
|
59
|
-
subscribe(listener: (event: PiSessionEvent) => void | Promise<void>): () => void;
|
|
60
|
-
dispose(): void;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Subset of pi's `AgentSessionEvent` we care about. Defined locally so
|
|
64
|
-
* tests don't need to import pi types and so the adapter is robust to
|
|
65
|
-
* upstream additions.
|
|
66
|
-
*/
|
|
67
|
-
export type PiSessionEvent = {
|
|
68
|
-
type: "message_start";
|
|
69
|
-
message: PiMessageLike;
|
|
70
|
-
} | {
|
|
71
|
-
type: "message_update";
|
|
72
|
-
message: PiMessageLike;
|
|
73
|
-
assistantMessageEvent: PiAssistantEvent;
|
|
74
|
-
} | {
|
|
75
|
-
type: "message_end";
|
|
76
|
-
message: PiMessageLike;
|
|
77
|
-
} | {
|
|
78
|
-
type: "tool_execution_start";
|
|
79
|
-
toolCallId: string;
|
|
80
|
-
toolName: string;
|
|
81
|
-
args: unknown;
|
|
82
|
-
} | {
|
|
83
|
-
type: "tool_execution_update";
|
|
84
|
-
toolCallId: string;
|
|
85
|
-
toolName: string;
|
|
86
|
-
args: unknown;
|
|
87
|
-
partialResult: unknown;
|
|
88
|
-
} | {
|
|
89
|
-
type: "tool_execution_end";
|
|
90
|
-
toolCallId: string;
|
|
91
|
-
toolName: string;
|
|
92
|
-
result: unknown;
|
|
93
|
-
isError: boolean;
|
|
94
|
-
} | {
|
|
95
|
-
type: string;
|
|
96
|
-
[k: string]: unknown;
|
|
97
|
-
};
|
|
98
|
-
interface PiMessageLike {
|
|
99
|
-
role: string;
|
|
100
|
-
content: unknown;
|
|
101
|
-
}
|
|
102
|
-
type PiAssistantEvent = {
|
|
103
|
-
type: "text_delta";
|
|
104
|
-
delta: string;
|
|
105
|
-
[k: string]: unknown;
|
|
106
|
-
} | {
|
|
107
|
-
type: "thinking_delta";
|
|
108
|
-
delta: string;
|
|
109
|
-
[k: string]: unknown;
|
|
110
|
-
} | {
|
|
111
|
-
type: string;
|
|
112
|
-
[k: string]: unknown;
|
|
113
|
-
};
|
|
114
|
-
/**
|
|
115
|
-
* Test seam — provide a custom session factory. Default factory loads
|
|
116
|
-
* `@mariozechner/pi-coding-agent` lazily so building tests / standalone
|
|
117
|
-
* binaries that skip the orchestrator never imports pi-ai at all.
|
|
118
|
-
*/
|
|
119
|
-
export interface PiSessionFactory {
|
|
120
|
-
(config: PiSessionFactoryConfig): Promise<PiAgentSessionLike>;
|
|
121
|
-
}
|
|
122
|
-
export interface PiSessionFactoryConfig {
|
|
123
|
-
projectRoot: string;
|
|
124
|
-
systemPrompt?: string;
|
|
125
|
-
/** Resolved pi-ai Model object (already URL-rewritten for ARouter). */
|
|
126
|
-
model?: unknown;
|
|
127
|
-
}
|
|
128
|
-
export interface PiOrchestratorAgentOptions {
|
|
129
|
-
/** Project root passed to pi's `DefaultResourceLoader`. */
|
|
130
|
-
projectRoot: string;
|
|
131
|
-
/** Outer-layer LLM config. */
|
|
132
|
-
outer: PiOuterConfig;
|
|
133
|
-
/**
|
|
134
|
-
* Optional override for pi's system prompt. Defaults to
|
|
135
|
-
* `ORCHESTRATOR_SYSTEM_PROMPT` (see `orchestrator-system-prompt.ts`).
|
|
136
|
-
*/
|
|
137
|
-
systemPrompt?: string;
|
|
138
|
-
/** Structured logger; falls back to console.warn. */
|
|
139
|
-
logger?: Pick<Logger, "warn" | "info" | "debug">;
|
|
140
|
-
/** Inject a custom session factory (used by tests). */
|
|
141
|
-
createSession?: PiSessionFactory;
|
|
142
|
-
/** Override pi-ai `getModel`. Used by tests to skip the heavy import. */
|
|
143
|
-
resolveModel?: ModelResolver;
|
|
144
|
-
/**
|
|
145
|
-
* Resolves the active `IMcpRuntime` on every tool call. Typically
|
|
146
|
-
* `() => mcpClient` (the sibling `device-mcp-server` HTTP client) or an
|
|
147
|
-
* `InProcMcpClient` for local CLI usage. Resolved lazily so an ACP
|
|
148
|
-
* device switch (or autospawn restart) is reflected immediately on the
|
|
149
|
-
* next `mobile_use` / `ui_xml` / `ground` invocation.
|
|
150
|
-
*/
|
|
151
|
-
mcpProvider?: () => IMcpRuntime;
|
|
152
|
-
/**
|
|
153
|
-
* Lazily build the grounder used by the `ground` extension. Called the
|
|
154
|
-
* first time the extension reads `getGrounder()` within a session.
|
|
155
|
-
* Typical implementation: `() => new HTTPGrounder(config.grounder)`.
|
|
156
|
-
*/
|
|
157
|
-
grounderFactory?: () => IGrounder;
|
|
158
|
-
/**
|
|
159
|
-
* Run a `mobile_use` task end-to-end and return the final assistant
|
|
160
|
-
* message. Wraps `InnerLoopVlm.runTask`. Letting the bootstrap layer
|
|
161
|
-
* supply this keeps the heavy `OpenAIClient` instance shared with the
|
|
162
|
-
* legacy single-layer path (and lets tests stub it out).
|
|
163
|
-
*/
|
|
164
|
-
mobileUseRunner?: (task: string, maxSteps: number, abortSignal?: AbortSignal) => Promise<string>;
|
|
165
|
-
/**
|
|
166
|
-
* Optional UI-XML formatter. When omitted the adapter calls
|
|
167
|
-
* `mcp.uiDump({ format: "raw" })` and feeds the result through
|
|
168
|
-
* `formatUiXmlToolOutput` — same behaviour the legacy `queryUiXml`
|
|
169
|
-
* used to provide on top of the raw `Device`. Tests can override to
|
|
170
|
-
* skip the dump path entirely.
|
|
171
|
-
*/
|
|
172
|
-
uiXmlRunner?: (query?: string) => Promise<string>;
|
|
173
|
-
/**
|
|
174
|
-
* Active shortcut runtime (Phase 2 of the GUI-Shortcut Hybrid Plan).
|
|
175
|
-
* When set, the orchestrator extension renders one outer-LLM tool
|
|
176
|
-
* per registered shortcut. When omitted, only the legacy
|
|
177
|
-
* `mobile_use` / `ground` / `ui_xml` triad is exposed and any
|
|
178
|
-
* extension that calls `getShortcutKB()` will throw — that's
|
|
179
|
-
* intentional, the bootstrap layer is responsible for wiring this.
|
|
180
|
-
*/
|
|
181
|
-
shortcutInvoker?: ShortcutInvoker;
|
|
182
|
-
/**
|
|
183
|
-
* Optional resource facade used by `ShortcutInvoker` callers (the
|
|
184
|
-
* `download_to_device` builtin in particular). The bootstrap layer
|
|
185
|
-
* supplies a lazy factory because the resource layer is constructed
|
|
186
|
-
* AFTER the agent core in `bootstrap.ts`. Returning `undefined`
|
|
187
|
-
* means the `download_to_device` shortcut will surface
|
|
188
|
-
* `permission_denied` to the LLM, which is preferable to silently
|
|
189
|
-
* misbehaving — but bootstrap should normally wire a facade.
|
|
190
|
-
*/
|
|
191
|
-
shortcutResourcesFactory?: () => ShortcutResourceFacade | undefined;
|
|
192
|
-
}
|
|
193
|
-
export interface ModelResolver {
|
|
194
|
-
(provider: string, modelId: string, baseUrl: string): unknown | undefined;
|
|
195
|
-
}
|
|
196
|
-
export declare class PiOrchestratorAgent implements IAgentCore {
|
|
197
|
-
private session;
|
|
198
|
-
private starting;
|
|
199
|
-
private outer;
|
|
200
|
-
private readonly projectRoot;
|
|
201
|
-
private readonly systemPrompt;
|
|
202
|
-
private readonly logger;
|
|
203
|
-
private readonly createSessionFactory;
|
|
204
|
-
private readonly resolveModelFn;
|
|
205
|
-
private readonly mcpProvider?;
|
|
206
|
-
private readonly grounderFactory?;
|
|
207
|
-
private readonly mobileUseRunner?;
|
|
208
|
-
private readonly uiXmlRunner?;
|
|
209
|
-
private readonly shortcutInvoker?;
|
|
210
|
-
private readonly shortcutResourcesFactory?;
|
|
211
|
-
private cachedGrounder;
|
|
212
|
-
constructor(opts: PiOrchestratorAgentOptions);
|
|
213
|
-
get modelName(): string;
|
|
214
|
-
get providerName(): string | undefined;
|
|
215
|
-
run(opts: AgentCoreRunOptions): Promise<AgentCoreRunResult>;
|
|
216
|
-
setModel(provider: string, model: string): Promise<void>;
|
|
217
|
-
/** Tear down the underlying pi session. Idempotent. */
|
|
218
|
-
dispose(): Promise<void>;
|
|
219
|
-
private buildExtensionContext;
|
|
220
|
-
private ensureSession;
|
|
221
|
-
}
|
|
222
|
-
/**
|
|
223
|
-
* Map our generic outer config to pi-ai's env-var contract. pi-ai reads
|
|
224
|
-
* provider-specific keys (e.g. `OPENROUTER_API_KEY`) from `process.env`,
|
|
225
|
-
* so we transparently set them when the caller hands us `outer.apiKey`
|
|
226
|
-
* + a recognised baseUrl. Never overwrites a key the caller already set.
|
|
227
|
-
*
|
|
228
|
-
* Recognised hosts:
|
|
229
|
-
* - openrouter.ai / arouter.ai → OPENROUTER_API_KEY
|
|
230
|
-
* - api.openai.com → OPENAI_API_KEY
|
|
231
|
-
* - api.anthropic.com → ANTHROPIC_API_KEY
|
|
232
|
-
* - generativelanguage.googleapis.com → GEMINI_API_KEY
|
|
233
|
-
*/
|
|
234
|
-
export declare function applyOuterEnvMapping(outer: PiOuterConfig): void;
|
|
235
|
-
/** Infer the pi-ai provider tag from a base URL. */
|
|
236
|
-
export declare function inferPiProvider(baseUrl: string): string;
|
|
237
|
-
export {};
|