@beeos-ai/device-agent 0.9.9 → 0.9.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/agentcore-inner-loop.d.ts +3 -4
- package/dist/adapters/agentcore-inner-loop.js +3 -4
- package/dist/adapters/agentcore-inner-loop.js.map +1 -1
- package/dist/adapters/agentcore-orchestrator.d.ts +36 -0
- package/dist/adapters/agentcore-orchestrator.js +203 -0
- package/dist/adapters/agentcore-orchestrator.js.map +1 -0
- package/dist/adapters/agentcore-runner.d.ts +3 -3
- package/dist/adapters/agentcore-runner.js +3 -3
- package/dist/adapters/app-guide.d.ts +16 -0
- package/dist/adapters/app-guide.js +101 -0
- package/dist/adapters/app-guide.js.map +1 -0
- package/dist/adapters/orchestrator-agent-prompt.d.ts +7 -0
- package/dist/adapters/orchestrator-agent-prompt.js +59 -0
- package/dist/adapters/orchestrator-agent-prompt.js.map +1 -0
- package/dist/adapters/vlm-inner-loop.d.ts +6 -0
- package/dist/adapters/vlm-inner-loop.js +3 -0
- package/dist/adapters/vlm-inner-loop.js.map +1 -1
- package/dist/agent-runtime/core.js +1 -1
- package/dist/agent-runtime/core.js.map +1 -1
- package/dist/agent-runtime/shortcuts/types.d.ts +2 -3
- package/dist/agent-runtime/shortcuts/types.js +2 -3
- package/dist/agent-runtime/shortcuts/types.js.map +1 -1
- package/dist/api.d.ts +1 -1
- package/dist/api.js +1 -1
- package/dist/bootstrap.d.ts +4 -4
- package/dist/bootstrap.js +47 -61
- package/dist/bootstrap.js.map +1 -1
- package/dist/config/loader.d.ts +2 -2
- package/dist/config/loader.js +2 -2
- package/dist/config/loader.js.map +1 -1
- package/dist/config/schema.d.ts +1 -1
- package/dist/config/schema.js +4 -5
- package/dist/config/schema.js.map +1 -1
- package/dist/device/apps.js +1 -1
- package/dist/device/apps.js.map +1 -1
- package/dist/fleet/domain/events.d.ts +22 -1
- package/dist/fleet/domain/fsm.d.ts +23 -1
- package/dist/fleet/domain/fsm.js +29 -0
- package/dist/fleet/domain/fsm.js.map +1 -1
- package/dist/fleet/supervisor.d.ts +13 -0
- package/dist/fleet/supervisor.js +90 -0
- package/dist/fleet/supervisor.js.map +1 -1
- package/dist/grounding/grounder.d.ts +1 -1
- package/dist/inbound/cli/main.d.ts +1 -1
- package/dist/inbound/cli/main.js +4 -4
- package/dist/inbound/cli/main.js.map +1 -1
- package/dist/inbound/cli/run-command.d.ts +6 -0
- package/dist/inbound/cli/run-command.js +94 -0
- package/dist/inbound/cli/run-command.js.map +1 -1
- package/dist/index.d.ts +2 -4
- package/dist/index.js +1 -2
- package/dist/index.js.map +1 -1
- package/dist/model/adapters/general.d.ts +11 -0
- package/dist/model/adapters/general.js +249 -0
- package/dist/model/adapters/general.js.map +1 -0
- package/dist/model/adapters/index.d.ts +1 -0
- package/dist/model/adapters/index.js +1 -0
- package/dist/model/adapters/index.js.map +1 -1
- package/dist/plugins/message-bus/ms-channel-notifier.d.ts +24 -0
- package/dist/plugins/message-bus/ms-channel-notifier.js +179 -12
- package/dist/plugins/message-bus/ms-channel-notifier.js.map +1 -1
- package/dist/ports/agent-core.d.ts +6 -5
- package/dist/shortcuts/types.d.ts +3 -6
- package/dist/shortcuts/types.js +1 -2
- package/dist/shortcuts/types.js.map +1 -1
- package/package.json +3 -6
- package/.pi/extensions/android-tools.ts +0 -291
- package/.pi/skills/.gitkeep +0 -0
- package/.pi/skills/ground-mobile-fallback/SKILL.md +0 -50
- package/dist/adapters/agentcore-pi.d.ts +0 -237
- package/dist/adapters/agentcore-pi.js +0 -512
- package/dist/adapters/agentcore-pi.js.map +0 -1
- package/dist/chat.d.ts +0 -16
- package/dist/chat.js +0 -185
- package/dist/chat.js.map +0 -1
- package/dist/debug/dump-system-prompt.d.ts +0 -1
- package/dist/debug/dump-system-prompt.js +0 -34
- package/dist/debug/dump-system-prompt.js.map +0 -1
- package/dist/orchestrator-system-prompt.d.ts +0 -32
- package/dist/orchestrator-system-prompt.js +0 -197
- package/dist/orchestrator-system-prompt.js.map +0 -1
- package/dist/pi-context.d.ts +0 -132
- package/dist/pi-context.js +0 -62
- package/dist/pi-context.js.map +0 -1
- package/dist/pi-local-skills.d.ts +0 -17
- package/dist/pi-local-skills.js +0 -24
- package/dist/pi-local-skills.js.map +0 -1
package/dist/chat.js
DELETED
|
@@ -1,185 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Standalone interactive chat CLI for the two-layer agent.
|
|
3
|
-
*
|
|
4
|
-
* Same UX as before — `pnpm run chat` opens a REPL that drives the outer
|
|
5
|
-
* orchestrator (pi-coding-agent) over `ORCH_*` env vars, with the inner
|
|
6
|
-
* `mobile_use` / `ground` / `ui_xml` extensions loaded from
|
|
7
|
-
* `<projectRoot>/.pi/extensions/`. Internally it now goes through the
|
|
8
|
-
* `PiOrchestratorAgent` adapter so there is exactly **one** code path
|
|
9
|
-
* for "construct a pi session with our system prompt and our extension
|
|
10
|
-
* context", shared with the production `bootstrap.ts` flow.
|
|
11
|
-
*
|
|
12
|
-
* `chat.ts` does NOT start the ACP bridge / HTTP debug / agent-sync
|
|
13
|
-
* plugins; it's a developer convenience to manually drive the agent
|
|
14
|
-
* against a local device. For production runs use `pnpm start`.
|
|
15
|
-
*/
|
|
16
|
-
import "dotenv/config";
|
|
17
|
-
import readline from "node:readline/promises";
|
|
18
|
-
import { stdin as input, stdout as output } from "node:process";
|
|
19
|
-
import { dirname, resolve } from "path";
|
|
20
|
-
import { fileURLToPath } from "url";
|
|
21
|
-
import { loadConfig } from "./config/loader.js";
|
|
22
|
-
import { AndroidAdbAdapter } from "./adapters/device-android-adb.js";
|
|
23
|
-
import { DesktopAdapter } from "./adapters/device-desktop.js";
|
|
24
|
-
import { DeviceManager } from "./app/device-manager.js";
|
|
25
|
-
import { InnerLoopVlm, unwrapInnerDevice } from "./adapters/vlm-inner-loop.js";
|
|
26
|
-
import { PiOrchestratorAgent } from "./adapters/agentcore-pi.js";
|
|
27
|
-
import { HTTPGrounder } from "./grounding/grounder.js";
|
|
28
|
-
import { formatUiXmlToolOutput } from "./device/ui-xml.js";
|
|
29
|
-
import { InProcMcpClient, McpClient } from "./mcp-client/index.js";
|
|
30
|
-
const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
|
31
|
-
const GREY = "\x1b[90m";
|
|
32
|
-
const CYAN = "\x1b[36m";
|
|
33
|
-
const YELLOW = "\x1b[33m";
|
|
34
|
-
const GREEN = "\x1b[32m";
|
|
35
|
-
const RESET = "\x1b[0m";
|
|
36
|
-
async function main() {
|
|
37
|
-
// ── Resolve config (ORCH_* / GUI_AGENT_* / GROUNDER_* + ~/.beeos/config.json)
|
|
38
|
-
const config = loadConfig({}, process.env, {
|
|
39
|
-
warn: (msg) => process.stderr.write(`${YELLOW}[config]${RESET} ${msg}\n`),
|
|
40
|
-
});
|
|
41
|
-
// ── Build device + manager (same logic as bootstrap, no Bridge/ACP) ─────
|
|
42
|
-
const device = config.mode === "desktop"
|
|
43
|
-
? new DesktopAdapter()
|
|
44
|
-
: new AndroidAdbAdapter({ serial: config.adbSerial });
|
|
45
|
-
const devices = new DeviceManager();
|
|
46
|
-
await devices.register(device);
|
|
47
|
-
const info = await device.getInfo().catch(() => null);
|
|
48
|
-
console.log(`${GREY}[device] ${info?.name ?? "?"} (${info?.type ?? config.mode}) registered${RESET}`);
|
|
49
|
-
// ── Inner mobile_use VLM (shared between extension and any future legacy
|
|
50
|
-
// path so token / abort accounting stays consistent).
|
|
51
|
-
const vlm = new InnerLoopVlm(devices.active(), {
|
|
52
|
-
apiKey: config.mobileUse.apiKey || config.vlmApiKey || undefined,
|
|
53
|
-
baseUrl: config.mobileUse.baseUrl || config.vlmBaseUrl || undefined,
|
|
54
|
-
modelName: config.mobileUse.model || config.vlmModel || undefined,
|
|
55
|
-
maxSteps: config.maxSteps,
|
|
56
|
-
});
|
|
57
|
-
// ── MCP runtime resolver — when MCP_URL is set we delegate to the
|
|
58
|
-
// sibling device-mcp-server; otherwise we wrap the in-process driver
|
|
59
|
-
// so the chat REPL keeps working without extra setup.
|
|
60
|
-
//
|
|
61
|
-
// 0.4.0: device-mcp-server is a singleton multi-device process — pass
|
|
62
|
-
// `defaultDevice` so every tool call carries the right `device` arg and
|
|
63
|
-
// the server's BackendRegistry routes to the matching backend.
|
|
64
|
-
const defaultDevice = config.mode === "desktop"
|
|
65
|
-
? "desktop"
|
|
66
|
-
: config.adbSerial || undefined;
|
|
67
|
-
const mcpProvider = config.mcpUrl
|
|
68
|
-
? (() => {
|
|
69
|
-
const client = new McpClient({
|
|
70
|
-
baseUrl: config.mcpUrl,
|
|
71
|
-
defaultDevice,
|
|
72
|
-
});
|
|
73
|
-
return () => client;
|
|
74
|
-
})()
|
|
75
|
-
: () => {
|
|
76
|
-
const idevice = devices.active();
|
|
77
|
-
return new InProcMcpClient(unwrapInnerDevice(idevice), { idevice });
|
|
78
|
-
};
|
|
79
|
-
// ── Outer agent — drives pi-coding-agent through PiOrchestratorAgent ───
|
|
80
|
-
const agent = new PiOrchestratorAgent({
|
|
81
|
-
projectRoot: PROJECT_ROOT,
|
|
82
|
-
outer: {
|
|
83
|
-
apiKey: config.outer.apiKey || undefined,
|
|
84
|
-
baseUrl: config.outer.baseUrl || undefined,
|
|
85
|
-
model: config.outer.model || undefined,
|
|
86
|
-
provider: config.outer.provider || undefined,
|
|
87
|
-
},
|
|
88
|
-
logger: {
|
|
89
|
-
warn: (...args) => console.warn(`${YELLOW}[warn]${RESET}`, ...args),
|
|
90
|
-
info: (...args) => console.log(`${GREY}[info]${RESET}`, ...args),
|
|
91
|
-
debug: () => undefined,
|
|
92
|
-
},
|
|
93
|
-
mcpProvider,
|
|
94
|
-
grounderFactory: () => new HTTPGrounder({
|
|
95
|
-
baseUrl: config.grounder.baseUrl || undefined,
|
|
96
|
-
apiKey: config.grounder.apiKey || undefined,
|
|
97
|
-
model: config.grounder.model || undefined,
|
|
98
|
-
}),
|
|
99
|
-
mobileUseRunner: async (task, max, signal) => {
|
|
100
|
-
const r = await vlm.runTask({ task, maxSteps: max, abortSignal: signal });
|
|
101
|
-
return r.finalMessage;
|
|
102
|
-
},
|
|
103
|
-
uiXmlRunner: async (q) => {
|
|
104
|
-
const dump = await mcpProvider().uiDump({ format: "raw" });
|
|
105
|
-
if (!dump.supported) {
|
|
106
|
-
return "UI XML is not supported on this backend (use `ground` to locate elements)";
|
|
107
|
-
}
|
|
108
|
-
return formatUiXmlToolOutput(dump.content ?? "", q);
|
|
109
|
-
},
|
|
110
|
-
});
|
|
111
|
-
console.log(`${CYAN}device-agent interactive chat${RESET}`);
|
|
112
|
-
console.log(`${GREY} outer=${config.outer.model || "(default)"} mobileUse=${config.mobileUse.model || "(default)"} grounder=${config.grounder.model || "(default)"}${RESET}`);
|
|
113
|
-
console.log(`${GREY} type your task and press Enter; /q or /exit to quit${RESET}`);
|
|
114
|
-
console.log(`${GREY} Ctrl+C / Ctrl+D 也可退出${RESET}`);
|
|
115
|
-
console.log("─".repeat(60));
|
|
116
|
-
const rl = readline.createInterface({ input, output, terminal: true });
|
|
117
|
-
let turn = 0;
|
|
118
|
-
while (true) {
|
|
119
|
-
let line;
|
|
120
|
-
try {
|
|
121
|
-
line = (await rl.question(`\n${CYAN}you [${++turn}]>${RESET} `)).trim();
|
|
122
|
-
}
|
|
123
|
-
catch {
|
|
124
|
-
break; // Ctrl+D
|
|
125
|
-
}
|
|
126
|
-
if (!line) {
|
|
127
|
-
turn--;
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
|
-
if (line === "/q" || line === "/exit" || line === "exit" || line === "quit")
|
|
131
|
-
break;
|
|
132
|
-
if (line === "/clear") {
|
|
133
|
-
console.clear();
|
|
134
|
-
continue;
|
|
135
|
-
}
|
|
136
|
-
if (line === "/help") {
|
|
137
|
-
console.log(" /q | /exit quit\n /clear clear screen\n /help show this");
|
|
138
|
-
continue;
|
|
139
|
-
}
|
|
140
|
-
process.stdout.write(`${YELLOW}agent>${RESET} `);
|
|
141
|
-
let toolDepth = 0;
|
|
142
|
-
try {
|
|
143
|
-
await agent.run({
|
|
144
|
-
task: line,
|
|
145
|
-
onText: (delta) => {
|
|
146
|
-
process.stdout.write(delta);
|
|
147
|
-
},
|
|
148
|
-
onTool: (event) => {
|
|
149
|
-
if (event.status === "start") {
|
|
150
|
-
toolDepth++;
|
|
151
|
-
const indent = " ".repeat(toolDepth - 1);
|
|
152
|
-
process.stdout.write(`\n${GREY}${indent}▶ ${event.name}${RESET} ${GREY}${truncate(JSON.stringify(event.args), 140)}${RESET}\n`);
|
|
153
|
-
}
|
|
154
|
-
else if (event.status === "end") {
|
|
155
|
-
const indent = " ".repeat(Math.max(0, toolDepth - 1));
|
|
156
|
-
const tag = event.isError ? `${YELLOW}✗ err${RESET}` : `${GREEN}✓${RESET}`;
|
|
157
|
-
const preview = typeof event.result === "string"
|
|
158
|
-
? truncate(event.result, 200)
|
|
159
|
-
: truncate(JSON.stringify(event.result ?? ""), 200);
|
|
160
|
-
process.stdout.write(`${GREY}${indent}${tag} ${preview}${RESET}\n`);
|
|
161
|
-
toolDepth = Math.max(0, toolDepth - 1);
|
|
162
|
-
}
|
|
163
|
-
},
|
|
164
|
-
});
|
|
165
|
-
process.stdout.write("\n");
|
|
166
|
-
}
|
|
167
|
-
catch (e) {
|
|
168
|
-
console.error(`\n${YELLOW}[error]${RESET}`, e instanceof Error ? e.message : e);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
rl.close();
|
|
172
|
-
await agent.dispose();
|
|
173
|
-
await devices.disconnectAll().catch(() => undefined);
|
|
174
|
-
console.log(`\n${GREY}bye.${RESET}`);
|
|
175
|
-
process.exit(0);
|
|
176
|
-
}
|
|
177
|
-
function truncate(s, n) {
|
|
178
|
-
s = s.replace(/\s+/g, " ");
|
|
179
|
-
return s.length <= n ? s : s.slice(0, n - 1) + "…";
|
|
180
|
-
}
|
|
181
|
-
main().catch((e) => {
|
|
182
|
-
console.error(e);
|
|
183
|
-
process.exit(1);
|
|
184
|
-
});
|
|
185
|
-
//# sourceMappingURL=chat.js.map
|
package/dist/chat.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chat.js","sourceRoot":"","sources":["../src/chat.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,eAAe,CAAC;AAEvB,OAAO,QAAQ,MAAM,wBAAwB,CAAC;AAC9C,OAAO,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,IAAI,MAAM,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,SAAS,EAAoB,MAAM,uBAAuB,CAAC;AAIrF,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AAE5E,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,KAAK,UAAU,IAAI;IACjB,+EAA+E;IAC/E,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,EAAE,OAAO,CAAC,GAAG,EAAE;QACzC,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;KAC1E,CAAC,CAAC;IAEH,2EAA2E;IAC3E,MAAM,MAAM,GACV,MAAM,CAAC,IAAI,KAAK,SAAS;QACvB,CAAC,CAAC,IAAI,cAAc,EAAE;QACtB,CAAC,CAAC,IAAI,iBAAiB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,IAAI,aAAa,EAAE,CAAC;IACpC,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACtD,OAAO,CAAC,GAAG,CACT,GAAG,IAAI,YAAY,IAAI,EAAE,IAAI,IAAI,GAAG,KAAK,IAAI,EAAE,IAAI,IAAI,MAAM,CAAC,IAAI,eAAe,KAAK,EAAE,CACzF,CAAC;IAEF,0EAA0E;IAC1E,sDAAsD;IACtD,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE;QAC7C,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,IAAI,MAAM,CAAC,SAAS,IAAI,SAAS;QAChE,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,OAAO,IAAI,MAAM,CAAC,UAAU,IAAI,SAAS;QACnE,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,MAAM,CAAC,QAAQ,IAAI,SAAS;QACjE,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAC;IAEH,mEAAmE;IACnE,qEAAqE;IACrE,sDAAsD;IACtD,EAAE;IACF,sEAAsE;IACtE,wEAAwE;IACxE,+DAA+D;IAC/D,MAAM,aAAa,GACjB,MAAM,CAAC,IAAI,KAAK,SAAS;QACvB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,SAAS,IAAI,SAAS,CAAC;IACpC,MAAM,WAAW,GAAsB,MAAM,CAAC,MAAM;QAClD,CAAC,CAAC,CAAC,GAAG,EAAE;YACJ,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;gBAC3B,OAAO,EAAE,MAAM,CAAC,MAAM;gBACtB,aAAa;aACd,CAAC,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC;QACtB,CAAC,CAAC,EAAE;QACN,CAAC,CAAC,GAAG,EAAE;YACH,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACjC,OAAO,IAAI,eAAe,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC,CAAC;IAEN,0EAA0E;IAC1E,MAAM,KAAK,GAAG,IAAI,mBAAmB,CAAC;QACpC,WAAW,EAAE,YAAY;QACzB,KAAK,EAAE;YACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,SAAS;YACxC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,SAAS;YAC1C,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,SAAS;YACtC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,SAAS;SAC7C;QACD,MAAM,EAAE;YACN,IAAI,EAAE,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC;YAC9E,IAAI,EAAE,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,SAAS,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC;YAC3E,KAAK,EAAE,GAAG,EAAE,CAAC,SAAS;SACvB;QACD,WAAW;QACX,eAAe,EAAE,GAAG,EAAE,CACpB,IAAI,YAAY,CAAC;YACf,OAAO,EAAE,MAAM,CAAC,QAAQ,CAAC,OAAO,IAAI,SAAS;YAC7C,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,SAAS;YAC3C,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,SAAS;SAC1C,CAAC;QACJ,eAAe,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE;YAC3C,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,OAAO,CAAC,CAAC,YAAY,CAAC;QACxB,CAAC;QACD,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;YACvB,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3D,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBACpB,OAAO,2EAA2E,CAAC;YACrF,CAAC;YACD,OAAO,qBAAqB,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;QACtD,CAAC;KACF,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,gCAAgC,KAAK,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CACT,GAAG,IAAI,WAAW,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,WAAW,cAAc,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,WAAW,aAAa,MAAM,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,GAAG,KAAK,EAAE,CAClK,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,wDAAwD,KAAK,EAAE,CAAC,CAAC;IACpF,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,yBAAyB,KAAK,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAEvE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,IAAY,CAAC;QACjB,IAAI,CAAC;YACH,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,EAAE,IAAI,KAAK,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1E,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,CAAC,SAAS;QAClB,CAAC;QAED,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,IAAI,EAAE,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM;YAAE,MAAM;QACnF,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,SAAS;QACX,CAAC;QACD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,4EAA4E,CAAC,CAAC;YAC1F,SAAS;QACX,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,SAAS,KAAK,GAAG,CAAC,CAAC;QACjD,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,GAAG,CAAC;gBACd,IAAI,EAAE,IAAI;gBACV,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;oBAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC9B,CAAC;gBACD,MAAM,EAAE,CAAC,KAAoB,EAAE,EAAE;oBAC/B,IAAI,KAAK,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;wBAC7B,SAAS,EAAE,CAAC;wBACZ,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;wBAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,IAAI,GAAG,MAAM,KAAK,KAAK,CAAC,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,GAAG,KAAK,IAAI,CAC1G,CAAC;oBACJ,CAAC;yBAAM,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;wBAClC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;wBACvD,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC;wBAC3E,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;4BAC9B,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC;4BAC7B,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;wBACxD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,GAAG,MAAM,GAAG,GAAG,IAAI,OAAO,GAAG,KAAK,IAAI,CAAC,CAAC;wBACpE,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;oBACzC,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,KAAK,MAAM,UAAU,KAAK,EAAE,EAAE,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAC;IACX,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACtB,MAAM,OAAO,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,KAAK,EAAE,CAAC,CAAC;IACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,QAAQ,CAAC,CAAS,EAAE,CAAS;IACpC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC3B,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;AACrD,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IACjB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import "dotenv/config";
|
|
2
|
-
import { createAgentSession, DefaultResourceLoader, getAgentDir, } from "@mariozechner/pi-coding-agent";
|
|
3
|
-
import { dirname, resolve } from "path";
|
|
4
|
-
import { fileURLToPath } from "url";
|
|
5
|
-
import { ORCHESTRATOR_SYSTEM_PROMPT } from "../orchestrator-system-prompt.js";
|
|
6
|
-
import { projectPiSkillsLoaderOptions } from "../pi-local-skills.js";
|
|
7
|
-
const PROJECT_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..");
|
|
8
|
-
async function main() {
|
|
9
|
-
const loader = new DefaultResourceLoader({
|
|
10
|
-
cwd: PROJECT_ROOT,
|
|
11
|
-
agentDir: getAgentDir(),
|
|
12
|
-
systemPromptOverride: () => ORCHESTRATOR_SYSTEM_PROMPT,
|
|
13
|
-
...projectPiSkillsLoaderOptions(PROJECT_ROOT),
|
|
14
|
-
});
|
|
15
|
-
await loader.reload();
|
|
16
|
-
const { session } = await createAgentSession({ resourceLoader: loader });
|
|
17
|
-
const prompt = session.systemPrompt;
|
|
18
|
-
console.log("==== FINAL SYSTEM PROMPT ====");
|
|
19
|
-
console.log(prompt);
|
|
20
|
-
console.log(`\n==== END SYSTEM PROMPT (length=${prompt.length} chars) ====\n`);
|
|
21
|
-
// What's actually sent as the `tools` HTTP parameter (distinct from system prompt)
|
|
22
|
-
const tools = session.getAllTools();
|
|
23
|
-
console.log(`==== TOOLS PARAMETER (count=${tools.length}) ====`);
|
|
24
|
-
for (const t of tools) {
|
|
25
|
-
console.log(`\n[${t.name}] source=${t.sourceInfo?.source ?? "?"}`);
|
|
26
|
-
console.log(` description: ${t.description}`);
|
|
27
|
-
console.log(` parameters: ${JSON.stringify(t.parameters).slice(0, 400)}`);
|
|
28
|
-
}
|
|
29
|
-
console.log("\n==== END TOOLS ====");
|
|
30
|
-
session.dispose();
|
|
31
|
-
process.exit(0);
|
|
32
|
-
}
|
|
33
|
-
main().catch((e) => { console.error(e); process.exit(1); });
|
|
34
|
-
//# sourceMappingURL=dump-system-prompt.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dump-system-prompt.js","sourceRoot":"","sources":["../../src/debug/dump-system-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,CAAC;AACvB,OAAO,EACL,kBAAkB,EAClB,qBAAqB,EACrB,WAAW,GACZ,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AACpC,OAAO,EAAE,0BAA0B,EAAE,MAAM,kCAAkC,CAAC;AAC9E,OAAO,EAAE,4BAA4B,EAAE,MAAM,uBAAuB,CAAC;AAErE,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAElF,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,IAAI,qBAAqB,CAAC;QACvC,GAAG,EAAE,YAAY;QACjB,QAAQ,EAAE,WAAW,EAAE;QACvB,oBAAoB,EAAE,GAAG,EAAE,CAAC,0BAA0B;QACtD,GAAG,4BAA4B,CAAC,YAAY,CAAC;KAC9C,CAAC,CAAC;IACH,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IAEtB,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,kBAAkB,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzE,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,oCAAoC,MAAM,CAAC,MAAM,gBAAgB,CAAC,CAAC;IAE/E,mFAAmF;IACnF,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,+BAA+B,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,UAAU,EAAE,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAErC,OAAO,CAAC,OAAO,EAAE,CAAC;IAClB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Outer orchestrator system prompts (pi-coding-agent).
|
|
3
|
-
* Per-tool details live in tool `description` / schema, not here.
|
|
4
|
-
* Select via ORCH_LANG env var: "cn" (default) | "en".
|
|
5
|
-
*
|
|
6
|
-
* Phase 2 (GUI-Shortcut Hybrid Plan, MAS-Bench 2025 SOTA):
|
|
7
|
-
* The prompt below now codifies three GENERIC heuristics for
|
|
8
|
-
* choosing between GUI primitives (`mobile_use`) and structured
|
|
9
|
-
* shortcuts (`download_to_device`, `set_clipboard`, `open_deeplink`,
|
|
10
|
-
* `find_app_package`):
|
|
11
|
-
*
|
|
12
|
-
* 1. **prefer-shortcut-when-applicable** — when a structured
|
|
13
|
-
* shortcut covers the user's need, call it before resorting
|
|
14
|
-
* to mobile_use. Shortcuts complete in O(1) ADB calls vs
|
|
15
|
-
* mobile_use's O(N) screenshot loop.
|
|
16
|
-
* 2. **avoid-misuse** — shortcuts are not free-form. Never
|
|
17
|
-
* invent shortcut arguments; respect the JSON schema. If a
|
|
18
|
-
* shortcut's pre-conditions don't hold (no URL, no installed
|
|
19
|
-
* app), fall back to mobile_use rather than fabricating an
|
|
20
|
-
* argument.
|
|
21
|
-
* 3. **shortcut-failure-fallback-to-mobile_use** — every
|
|
22
|
-
* shortcut returns a structured `{ok:false, code:"..."}`
|
|
23
|
-
* result on failure (`app_not_found`, `intent_rejected`,
|
|
24
|
-
* `timeout`, `unsupported_platform`, …). When you see one,
|
|
25
|
-
* switch to `mobile_use` for the same goal. Do NOT retry
|
|
26
|
-
* the same shortcut with the same args.
|
|
27
|
-
*
|
|
28
|
-
* The prompt is **app-agnostic** by construction — no app names,
|
|
29
|
-
* no package names, no GUI path hints. The shortcuts themselves
|
|
30
|
-
* are also app-agnostic (e.g. `open_deeplink` accepts any URI).
|
|
31
|
-
*/
|
|
32
|
-
export declare const ORCHESTRATOR_SYSTEM_PROMPT: string;
|
|
@@ -1,197 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Outer orchestrator system prompts (pi-coding-agent).
|
|
3
|
-
* Per-tool details live in tool `description` / schema, not here.
|
|
4
|
-
* Select via ORCH_LANG env var: "cn" (default) | "en".
|
|
5
|
-
*
|
|
6
|
-
* Phase 2 (GUI-Shortcut Hybrid Plan, MAS-Bench 2025 SOTA):
|
|
7
|
-
* The prompt below now codifies three GENERIC heuristics for
|
|
8
|
-
* choosing between GUI primitives (`mobile_use`) and structured
|
|
9
|
-
* shortcuts (`download_to_device`, `set_clipboard`, `open_deeplink`,
|
|
10
|
-
* `find_app_package`):
|
|
11
|
-
*
|
|
12
|
-
* 1. **prefer-shortcut-when-applicable** — when a structured
|
|
13
|
-
* shortcut covers the user's need, call it before resorting
|
|
14
|
-
* to mobile_use. Shortcuts complete in O(1) ADB calls vs
|
|
15
|
-
* mobile_use's O(N) screenshot loop.
|
|
16
|
-
* 2. **avoid-misuse** — shortcuts are not free-form. Never
|
|
17
|
-
* invent shortcut arguments; respect the JSON schema. If a
|
|
18
|
-
* shortcut's pre-conditions don't hold (no URL, no installed
|
|
19
|
-
* app), fall back to mobile_use rather than fabricating an
|
|
20
|
-
* argument.
|
|
21
|
-
* 3. **shortcut-failure-fallback-to-mobile_use** — every
|
|
22
|
-
* shortcut returns a structured `{ok:false, code:"..."}`
|
|
23
|
-
* result on failure (`app_not_found`, `intent_rejected`,
|
|
24
|
-
* `timeout`, `unsupported_platform`, …). When you see one,
|
|
25
|
-
* switch to `mobile_use` for the same goal. Do NOT retry
|
|
26
|
-
* the same shortcut with the same args.
|
|
27
|
-
*
|
|
28
|
-
* The prompt is **app-agnostic** by construction — no app names,
|
|
29
|
-
* no package names, no GUI path hints. The shortcuts themselves
|
|
30
|
-
* are also app-agnostic (e.g. `open_deeplink` accepts any URI).
|
|
31
|
-
*/
|
|
32
|
-
const HEURISTICS_CN = `\
|
|
33
|
-
## Hybrid 动作空间启发式(GUI + Shortcut)
|
|
34
|
-
|
|
35
|
-
工具分两类:
|
|
36
|
-
- **GUI 原语**:\`mobile_use\` / \`ground\` / \`ui_xml\`(视觉推理,O(N) 步骤)
|
|
37
|
-
- **Shortcut**:\`download_to_device\` / \`set_clipboard\` / \`open_deeplink\` / \`find_app_package\`(结构化输入,O(1) 调用)
|
|
38
|
-
|
|
39
|
-
三条通用启发式(**与具体 App 无关**):
|
|
40
|
-
|
|
41
|
-
1. **能用 shortcut 就先用 shortcut**:
|
|
42
|
-
- 任务里出现 URL → 先 \`download_to_device\` 把文件落到设备
|
|
43
|
-
- 任务里出现长文本 / 含 emoji → 先 \`set_clipboard\` 再让 GUI 粘贴
|
|
44
|
-
- 任务里有可识别的 deeplink(\`xx://\`、\`geo:\`、\`https://\`) → 先 \`open_deeplink\`
|
|
45
|
-
- 需要包名 → 先 \`find_app_package\` 模糊查询
|
|
46
|
-
- shortcut 不成立时(无 URL、无文本、无 deeplink)→ 直接走 \`mobile_use\`
|
|
47
|
-
|
|
48
|
-
2. **不要伪造 shortcut 参数**:
|
|
49
|
-
- 严格按工具 \`parameters\` schema 调用
|
|
50
|
-
- 不知道某个参数的真实值时(如不知道目标 deeplink)→ 不要瞎猜,直接 \`mobile_use\`
|
|
51
|
-
|
|
52
|
-
3. **shortcut 失败 → fallback \`mobile_use\`**:
|
|
53
|
-
- shortcut 返回 \`{ok:false, code:"..."}\` 是结构化失败信号,不是异常
|
|
54
|
-
- 常见失败码:\`app_not_found\`(未安装目标应用)、\`intent_rejected\`(系统拒绝)、\`timeout\`、\`unsupported_platform\`(设备不是 Android 或 desktop 不支持)
|
|
55
|
-
- 看到失败码后**用同样的目标任务调用 \`mobile_use\`**,不要用同样参数重试 shortcut
|
|
56
|
-
|
|
57
|
-
4. **「发布 / 分享 / 上传」类任务的执行路径(app-agnostic, first-principles):**
|
|
58
|
-
|
|
59
|
-
当任务里同时出现 **「发 / 发布 / 分享 / 上传 / post / share / upload」** + **资源**(URL / 文件路径 / 长文本 / 媒体)时:
|
|
60
|
-
- **Intent = 主动创建新内容并把资源附加到目标 App 的内容流**,**不是**在已有内容里查找该资源
|
|
61
|
-
- 标准 GUI 路径(在任何 App 都通用):\`进入目标 App → 找到「+ / 创建 / 拍摄 / 编辑 / 发布」入口 → 选择 / 插入资源 → 写文案 → 提交\`
|
|
62
|
-
- **禁止**在已有内容流(feed / timeline / 列表 / 收件箱 / 个人主页)里 swipe / 浏览 / 查找该资源——这条路径根本不会成功,资源是你刚下载的,不在任何人的 timeline 里
|
|
63
|
-
- 反复触发 \`pre-sanity blocked: blind_swipe\` 是**强信号**:你**进错了页面**,停止 swipe,立刻调用 \`ui_xml\` 看清当前 Activity,然后 \`mobile_use\` 跳到正确的「创建入口」(如长按相机图标 / 点击右上角 + 号 / 拉起底部抽屉等),而不是继续滑动
|
|
64
|
-
|
|
65
|
-
**关于资源在相册 / picker 里的位置(强约束)**:\`Resources prepared\` 块里 \`mtime=now\` 意味着该资源是**刚刚**下载到设备的(已 trigger MediaScanner 写入 MediaStore)。Android 系统的相册 / 图片 picker / 文件管理器**默认按 \`date_added\` 倒序**排列,因此该资源一定出现在**最前面(第一张 / 第一行 / 最近列表的顶部)**。
|
|
66
|
-
|
|
67
|
-
- **MUST**: 进入 picker 后**第一个动作必须是 tap 第一项**(左上角第一个缩略图 / 列表第一行)。仅当第一项明显不是目标 MIME 类型(例如期望 image/webp 但第一项是视频缩略图)才考虑后续策略。
|
|
68
|
-
- **MUST NOT — 反搜索**:**绝不**在 picker 里通过文件名 / 关键字 / 文字搜索来找该资源。资源的文件名是机器生成的 \`beeos_<时间戳>_<hash>.<扩展名>\`,picker 的搜索功能基于平台元数据索引(人脸 / 地点 / 物体 OCR),用文件名搜索必然 0 结果。
|
|
69
|
-
- **MUST NOT — 反识图**:**不要识别图片内容**(如"这看起来是蜂巢/风景/头像")然后用关键字去搜索。**任务里的 URL 已经是 unique reference**,文件已落到 \`Resources prepared\` 块里给出的具体路径,picker 第一项就是它。"识图→搜索"是反模式,比"直接 tap 第一项"慢 10 倍且永远找不到。
|
|
70
|
-
|
|
71
|
-
`;
|
|
72
|
-
const CN = `\
|
|
73
|
-
# Android 手机自动化编排 Agent
|
|
74
|
-
|
|
75
|
-
> 工具的参数、限制、返回格式在 \`tools\` 载荷里定义,本提示词只讲**策略**。
|
|
76
|
-
|
|
77
|
-
## 工具选择
|
|
78
|
-
|
|
79
|
-
| 工具 | 适用场景 |
|
|
80
|
-
|------|----------|
|
|
81
|
-
| \`mobile_use\` | **GUI 主路径**。多步视觉操作交给它,内部跑 VLM 截图→推理→执行闭环。 |
|
|
82
|
-
| \`ground\` | 需要**精确像素坐标**时:补救 \`mobile_use\` 误触,或配合 adb 手动执行点击。 |
|
|
83
|
-
| \`ui_xml\` | 需要**查看控件树**(resource-id / bounds / clickable)时:调试或确认当前页面状态。 |
|
|
84
|
-
| \`download_to_device\` | 任务里带 URL,需要把文件落到设备相册 / 下载目录时使用。 |
|
|
85
|
-
| \`set_clipboard\` | 长文本 / 含 emoji 的输入:写到剪贴板再让 GUI 粘贴比直接键盘输入更可靠。 |
|
|
86
|
-
| \`open_deeplink\` | 已知 deeplink URI 时:直接跳转目标页,跳过多步导航。 |
|
|
87
|
-
| \`find_app_package\` | 通过 App 名片段查询包名(用于生成 deeplink、\`monkey\` 启动等)。 |
|
|
88
|
-
|
|
89
|
-
${HEURISTICS_CN}
|
|
90
|
-
## 调用规则
|
|
91
|
-
|
|
92
|
-
**\`mobile_use\`**
|
|
93
|
-
- 指令写完整、写具体:界面原文、应用名、需要输入的文字(注明"原样输入、不改写")。
|
|
94
|
-
- 单次调用完成一个连贯子任务;跨 App 的长流程可拆成多次顺序调用。
|
|
95
|
-
- 用 \`max_steps\` 控制步数:**每次调用必须在工具参数里显式传入**(Schema 必填);简单操作 15–25,复杂流程 30–50。
|
|
96
|
-
- 失败后先分析原因(设备状态、参数、当前界面),不要用相同参数盲目重试。
|
|
97
|
-
|
|
98
|
-
**\`ground\` / \`ui_xml\`**
|
|
99
|
-
- 典型组合:\`ui_xml\` 找到 resource-id → \`ground\` 定位像素 → adb 执行。
|
|
100
|
-
- 不能替代 \`mobile_use\` 主路径,只在需要原子级精度时介入。
|
|
101
|
-
|
|
102
|
-
## 多轮对话
|
|
103
|
-
|
|
104
|
-
记住之前的结论和已执行操作,不必每轮从零确认,除非用户改需求或上下文已变。
|
|
105
|
-
|
|
106
|
-
## 任务结束
|
|
107
|
-
|
|
108
|
-
用一两句话总结结果,用中文回复。
|
|
109
|
-
|
|
110
|
-
## 交流
|
|
111
|
-
|
|
112
|
-
用户问你是否具备某工具(如 \`mobile_use\`)时,明确回答可否并一句话说明用途;禁止仅用「hi」等无信息回复。
|
|
113
|
-
`;
|
|
114
|
-
const HEURISTICS_EN = `\
|
|
115
|
-
## Hybrid action-space heuristics (GUI + Shortcut)
|
|
116
|
-
|
|
117
|
-
Tools split into two camps:
|
|
118
|
-
- **GUI primitives** — \`mobile_use\` / \`ground\` / \`ui_xml\` (visual reasoning, O(N) steps)
|
|
119
|
-
- **Shortcuts** — \`download_to_device\` / \`set_clipboard\` / \`open_deeplink\` / \`find_app_package\` (structured input, O(1) calls)
|
|
120
|
-
|
|
121
|
-
Three generic heuristics (**app-agnostic**):
|
|
122
|
-
|
|
123
|
-
1. **Prefer shortcuts when applicable**
|
|
124
|
-
- Task contains a URL → call \`download_to_device\` first to land the file on the device
|
|
125
|
-
- Task contains long / emoji-heavy text → use \`set_clipboard\` then have the GUI paste
|
|
126
|
-
- Task contains a recognisable deeplink URI (\`xx://\`, \`geo:\`, \`https://\`) → \`open_deeplink\` first
|
|
127
|
-
- Need a package name → \`find_app_package\` substring lookup
|
|
128
|
-
- When no shortcut applies (no URL, no text, no deeplink) → go straight to \`mobile_use\`
|
|
129
|
-
|
|
130
|
-
2. **Do not fabricate shortcut arguments**
|
|
131
|
-
- Strict adherence to each tool's \`parameters\` JSON schema
|
|
132
|
-
- When a parameter's true value is unknown (e.g. you don't know the target deeplink) → do NOT guess, call \`mobile_use\` instead
|
|
133
|
-
|
|
134
|
-
3. **Shortcut failure → fall back to \`mobile_use\`**
|
|
135
|
-
- A shortcut return of \`{ok:false, code:"..."}\` is a structured signal, not an exception
|
|
136
|
-
- Common codes: \`app_not_found\` (target app not installed), \`intent_rejected\` (OS refused), \`timeout\`, \`unsupported_platform\` (device is not Android, or desktop can't handle the asset)
|
|
137
|
-
- On failure, **call \`mobile_use\` for the same goal**. Never retry the same shortcut with the same arguments.
|
|
138
|
-
|
|
139
|
-
4. **Intent classification for "post / share / upload" tasks (app-agnostic, first-principles):**
|
|
140
|
-
|
|
141
|
-
When the task contains **"post / share / upload / 发 / 发布 / 分享 / 上传"** + **a resource** (URL / file path / long text / media):
|
|
142
|
-
- **Intent = create new content and attach the resource to the target app's content stream**, NOT find that resource inside existing content
|
|
143
|
-
- Universal GUI path (works in every app): \`enter target app → locate "+ / create / camera / compose / publish" entry → select/insert the resource → write copy → submit\`
|
|
144
|
-
- **Never** swipe through feeds / timelines / lists / inboxes / profiles searching for the resource — that path can never succeed; the resource was just downloaded and is not in anyone's timeline
|
|
145
|
-
- Repeated \`pre-sanity blocked: blind_swipe\` is a **strong signal that you are on the wrong screen**. Stop swiping, call \`ui_xml\` to read the current Activity, then \`mobile_use\` to jump to the correct "create entry" (long-press the camera icon, tap the top-right "+", pull a bottom sheet, etc.). Do NOT keep scrolling.
|
|
146
|
-
|
|
147
|
-
**Resource position inside gallery / picker (hard constraint)**: \`mtime=now\` in the \`Resources prepared\` block means the resource was **just** downloaded to the device (MediaScanner has registered it in MediaStore). Android's gallery / image picker / file manager **sorts by \`date_added\` DESC by default**, so the resource is **always at the very top — first thumbnail / first row / top of "Recent"**.
|
|
148
|
-
|
|
149
|
-
- **MUST**: Once a picker is open, the **first action must be tap the first item** (top-left thumbnail / first list row). Only consider alternatives if the first item is obviously the wrong MIME (e.g. expected image/webp but the first slot is a video).
|
|
150
|
-
- **MUST NOT — anti-search**: **NEVER** search for the resource by filename / keyword / text inside a picker. The filename is machine-generated (\`beeos_<timestamp>_<hash>.<ext>\`), and picker search uses platform metadata indices (faces / places / object OCR), so a filename query yields 0 results.
|
|
151
|
-
- **MUST NOT — anti-vision-recognition**: **Do not look at the picture and try to recognise its content** ("this looks like a beehive / landscape / avatar") and then search by keyword. The URL in the task is already a **unique reference**, and the file is at the exact path shown in \`Resources prepared\`. The picker's first item *is* it. "Recognise → search" is an anti-pattern: 10× slower than "tap first item" and it will never succeed.
|
|
152
|
-
|
|
153
|
-
`;
|
|
154
|
-
const EN = `\
|
|
155
|
-
# Android device orchestration agent
|
|
156
|
-
|
|
157
|
-
> Tool parameters, limits, and return formats are in the \`tools\` payload. This prompt covers **strategy only**.
|
|
158
|
-
|
|
159
|
-
## Tool selection
|
|
160
|
-
|
|
161
|
-
| Tool | When to use |
|
|
162
|
-
|------|-------------|
|
|
163
|
-
| \`mobile_use\` | **GUI main path.** Multi-step visual operations; runs a VLM screenshot→reason→act loop autonomously. |
|
|
164
|
-
| \`ground\` | When you need **exact pixel coords**: recover from a mis-tap, or feed coords to adb manually. |
|
|
165
|
-
| \`ui_xml\` | When you need the **view hierarchy** (resource-id / bounds / clickable): debug or verify screen state. |
|
|
166
|
-
| \`download_to_device\` | Task references a URL and the result must land on the device (image / video / pdf / apk). |
|
|
167
|
-
| \`set_clipboard\` | Long / emoji-heavy text input — clipboard + GUI paste beats keystroke-by-keystroke entry. |
|
|
168
|
-
| \`open_deeplink\` | Known deeplink URI — jump straight to the target screen and skip GUI navigation. |
|
|
169
|
-
| \`find_app_package\` | Substring-lookup of installed package names (input for deeplink generation, \`monkey\`-launch, etc.). |
|
|
170
|
-
|
|
171
|
-
${HEURISTICS_EN}
|
|
172
|
-
## Calling rules
|
|
173
|
-
|
|
174
|
-
**\`mobile_use\`**
|
|
175
|
-
- Write instructions in full: exact on-screen labels, app names, verbatim text to type (note "paste verbatim, no paraphrase").
|
|
176
|
-
- One call = one coherent subtask; split cross-app flows into sequential calls.
|
|
177
|
-
- \`max_steps\` is **required on every call** (tool schema); use 15–25 for simple actions, 30–50 for complex flows.
|
|
178
|
-
- On failure: diagnose first (device state, args, current screen). Do not retry with the same call.
|
|
179
|
-
|
|
180
|
-
**\`ground\` / \`ui_xml\`**
|
|
181
|
-
- Typical pattern: \`ui_xml\` → find resource-id → \`ground\` → pixel coords → adb tap.
|
|
182
|
-
- These are atomic helpers, not replacements for the \`mobile_use\` main path.
|
|
183
|
-
|
|
184
|
-
## Multi-turn
|
|
185
|
-
|
|
186
|
-
Retain prior conclusions and actions; do not re-confirm from scratch unless intent or context changed.
|
|
187
|
-
|
|
188
|
-
## Task completion
|
|
189
|
-
|
|
190
|
-
Summarize in one or two sentences; match the user's language.
|
|
191
|
-
|
|
192
|
-
## Communication
|
|
193
|
-
|
|
194
|
-
If asked whether you can use a tool (e.g. \`mobile_use\`), answer clearly (yes/no) and one sentence on what it does; do not reply with meaningless fillers like "hi" only.
|
|
195
|
-
`;
|
|
196
|
-
export const ORCHESTRATOR_SYSTEM_PROMPT = (process.env.ORCH_LANG ?? "cn") === "en" ? EN : CN;
|
|
197
|
-
//# sourceMappingURL=orchestrator-system-prompt.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator-system-prompt.js","sourceRoot":"","sources":["../src/orchestrator-system-prompt.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCrB,CAAC;AAEF,MAAM,EAAE,GAAG;;;;;;;;;;;;;;;;;EAiBT,aAAa;;;;;;;;;;;;;;;;;;;;;;;;CAwBd,CAAC;AAEF,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuCrB,CAAC;AAEF,MAAM,EAAE,GAAG;;;;;;;;;;;;;;;;;EAiBT,aAAa;;;;;;;;;;;;;;;;;;;;;;;;CAwBd,CAAC;AAEF,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC"}
|
package/dist/pi-context.d.ts
DELETED
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pi extension context — process-wide service locator that bridges
|
|
3
|
-
* `PiOrchestratorAgent` (which knows about the sibling `device-mcp-server`,
|
|
4
|
-
* `config.grounder`, and the `mobile_use` runner) and the
|
|
5
|
-
* `.pi/extensions/*.ts` files that pi-coding-agent loads on startup.
|
|
6
|
-
*
|
|
7
|
-
* Why a singleton?
|
|
8
|
-
*
|
|
9
|
-
* pi-coding-agent's `ExtensionContext` (the value passed to
|
|
10
|
-
* `pi.registerTool({ execute })`) is a fixed shape we don't control — it
|
|
11
|
-
* holds the session id, abort signal, and update sink, but no slot for
|
|
12
|
-
* custom dependency injection. Without this module the only way to reach
|
|
13
|
-
* a screenshot from inside an extension was to read `process.env`
|
|
14
|
-
* and `new AndroidDevice(...)`, which (a) bypasses the sibling
|
|
15
|
-
* `device-mcp-server` (so ACP-driven device switches and out-of-process
|
|
16
|
-
* sandboxes are invisible) and (b) bypasses `config.grounder` /
|
|
17
|
-
* `config.mobileUse` (so user-supplied URLs and API keys silently lose).
|
|
18
|
-
*
|
|
19
|
-
* The singleton is **scoped to the lifetime of the active
|
|
20
|
-
* `PiOrchestratorAgent` session**: the adapter calls
|
|
21
|
-
* `setPiExtensionContext(ctx)` immediately before constructing the pi
|
|
22
|
-
* session (so `loader.reload()` and the first `prompt()` see a hot
|
|
23
|
-
* context) and `setPiExtensionContext(null)` on `dispose()`.
|
|
24
|
-
*
|
|
25
|
-
* Tests and harness scripts that call extensions outside of a pi session
|
|
26
|
-
* must call `setPiExtensionContext(...)` themselves; otherwise
|
|
27
|
-
* `getPiExtensionContext()` throws so we don't silently fall back to env
|
|
28
|
-
* defaults.
|
|
29
|
-
*/
|
|
30
|
-
import type { IMcpRuntime } from "./mcp-client/runtime.js";
|
|
31
|
-
import type { IGrounder } from "./grounding/grounder.js";
|
|
32
|
-
import type { ShortcutInvoker, ShortcutResourceFacade } from "./shortcuts/index.js";
|
|
33
|
-
export type { IGrounder };
|
|
34
|
-
export interface PiExtensionContext {
|
|
35
|
-
/**
|
|
36
|
-
* Active MCP runtime — typically the sibling `device-mcp-server` HTTP
|
|
37
|
-
* client. Implementations should re-resolve from the bootstrap layer on
|
|
38
|
-
* every call so an ACP `device/select` (or autospawn restart) is
|
|
39
|
-
* reflected immediately on the next tool call.
|
|
40
|
-
*/
|
|
41
|
-
getMcpClient(): IMcpRuntime;
|
|
42
|
-
/**
|
|
43
|
-
* Visual grounder configured from the outer agent's `config.grounder`.
|
|
44
|
-
* Re-built lazily on first access so the underlying HTTP client is
|
|
45
|
-
* shared across `ground` invocations within the same session.
|
|
46
|
-
*/
|
|
47
|
-
getGrounder(): IGrounder;
|
|
48
|
-
/**
|
|
49
|
-
* Run the inner mobile_use VLM loop end-to-end. Returns the final
|
|
50
|
-
* agent message (the same string the legacy `mobileUse(task, maxSteps)`
|
|
51
|
-
* helper returned). Implementations route through `AgentLoop` over the
|
|
52
|
-
* sibling MCP server so step/text callbacks reach the outer host.
|
|
53
|
-
*/
|
|
54
|
-
runMobileUse(task: string, maxSteps: number, abortSignal?: AbortSignal): Promise<string>;
|
|
55
|
-
/**
|
|
56
|
-
* Dump the current UI XML, optionally filtered by substring (matches
|
|
57
|
-
* text / resource-id / content-desc / class name, case-insensitive).
|
|
58
|
-
* Implementations call `mcp.uiDump({ format: "raw" })` and pass the
|
|
59
|
-
* result through `formatUiXmlToolOutput`.
|
|
60
|
-
*/
|
|
61
|
-
getUiXml(query?: string): Promise<string>;
|
|
62
|
-
/**
|
|
63
|
-
* The active shortcut invoker. The outer-LLM extension uses
|
|
64
|
-
* `getShortcutKB().registry.list()` to enumerate shortcuts (one
|
|
65
|
-
* descriptor → one outer-LLM tool) and `getShortcutKB().invoke(id, …)`
|
|
66
|
-
* to dispatch a call. Throws when no shortcut runtime is wired (e.g.
|
|
67
|
-
* tests that don't need the hybrid action space).
|
|
68
|
-
*/
|
|
69
|
-
getShortcutKB(): ShortcutInvoker;
|
|
70
|
-
/**
|
|
71
|
-
* Optional resource facade for shortcut runtime callers (chiefly
|
|
72
|
-
* `download_to_device`). Implementations return `undefined` when
|
|
73
|
-
* the bootstrap layer has not wired a resolver — in that case the
|
|
74
|
-
* shortcut surfaces `permission_denied` to the LLM, which is the
|
|
75
|
-
* intended fail-soft behaviour.
|
|
76
|
-
*
|
|
77
|
-
* The .pi extension passes the result of this accessor into the
|
|
78
|
-
* `InvokeCallerContext.resources` field on every shortcut invoke,
|
|
79
|
-
* so all shortcut implementations see the *same* facade as the
|
|
80
|
-
* AgentHost auto-trigger and consequently share the same cache.
|
|
81
|
-
*/
|
|
82
|
-
getShortcutResources?(): ShortcutResourceFacade | undefined;
|
|
83
|
-
/**
|
|
84
|
-
* INTERNAL-ONLY. Runs a shell command on the active device through
|
|
85
|
-
* the same MCP runtime `getMcpClient()` returns. Provided as a
|
|
86
|
-
* convenience for shortcut implementations that need a one-liner
|
|
87
|
-
* exec; calling it from a `pi.registerTool({ execute })` body is a
|
|
88
|
-
* code-review failure (see comment block above).
|
|
89
|
-
*
|
|
90
|
-
* Implementations MUST honor `abortSignal` and surface a structured
|
|
91
|
-
* `{ stdout, stderr, exitCode }` so callers can map exit codes to
|
|
92
|
-
* `ShortcutErrorCode` deterministically.
|
|
93
|
-
*/
|
|
94
|
-
runShell(command: string, opts?: {
|
|
95
|
-
timeoutMs?: number;
|
|
96
|
-
abortSignal?: AbortSignal;
|
|
97
|
-
}): Promise<{
|
|
98
|
-
stdout: string;
|
|
99
|
-
stderr: string;
|
|
100
|
-
exitCode: number;
|
|
101
|
-
}>;
|
|
102
|
-
/**
|
|
103
|
-
* INTERNAL-ONLY. Push raw bytes to an absolute device-local path via
|
|
104
|
-
* the MCP runtime's `fileWrite`. Used by `download_to_device` (and
|
|
105
|
-
* any future shortcut that needs to materialise a file on the
|
|
106
|
-
* device) — never wrap this in an outer-LLM tool. Throws when the
|
|
107
|
-
* active runtime does not implement `fileWrite` (e.g. legacy
|
|
108
|
-
* in-process bridges).
|
|
109
|
-
*/
|
|
110
|
-
pushFile(path: string, bytes: Buffer): Promise<void>;
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* Install (or clear, when `ctx` is `null`) the context that
|
|
114
|
-
* `getPiExtensionContext()` resolves. `PiOrchestratorAgent` calls this
|
|
115
|
-
* before booting the pi session and clears it on dispose.
|
|
116
|
-
*
|
|
117
|
-
* Tests / harness scripts that exercise extensions in isolation must
|
|
118
|
-
* `setPiExtensionContext(stub)` first and clear it on teardown to keep
|
|
119
|
-
* other tests deterministic.
|
|
120
|
-
*/
|
|
121
|
-
export declare function setPiExtensionContext(ctx: PiExtensionContext | null): void;
|
|
122
|
-
/**
|
|
123
|
-
* Resolve the active context. Throws when no `PiOrchestratorAgent`
|
|
124
|
-
* session is in flight — extensions are designed to run inside a pi
|
|
125
|
-
* session and should never be invoked from cold storage.
|
|
126
|
-
*/
|
|
127
|
-
export declare function getPiExtensionContext(): PiExtensionContext;
|
|
128
|
-
/**
|
|
129
|
-
* Inspect-without-throw helper. Returns the active context or `null`.
|
|
130
|
-
* Exposed mainly for diagnostics / debug routes.
|
|
131
|
-
*/
|
|
132
|
-
export declare function peekPiExtensionContext(): PiExtensionContext | null;
|