@beeos-ai/device-agent 0.9.13 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/device-compat.d.ts +14 -0
- package/dist/adapters/device-compat.js +71 -0
- package/dist/adapters/device-compat.js.map +1 -0
- package/dist/agent-runtime/adapters/index.d.ts +1 -3
- package/dist/agent-runtime/adapters/index.js +1 -3
- package/dist/agent-runtime/adapters/index.js.map +1 -1
- package/dist/agent-runtime/adapters/openai.d.ts +5 -5
- package/dist/agent-runtime/adapters/openai.js +19 -6
- package/dist/agent-runtime/adapters/openai.js.map +1 -1
- package/dist/agent-runtime/adapters/registry.d.ts +1 -1
- package/dist/agent-runtime/adapters/registry.js +3 -3
- package/dist/agent-runtime/adapters/registry.js.map +1 -1
- package/dist/agent-runtime/agent-runner.d.ts +1 -1
- package/dist/agent-runtime/factory.d.ts +22 -0
- package/dist/agent-runtime/factory.js +25 -0
- package/dist/agent-runtime/factory.js.map +1 -0
- package/dist/agent-runtime/index.d.ts +9 -4
- package/dist/agent-runtime/index.js +9 -4
- package/dist/agent-runtime/index.js.map +1 -1
- package/dist/{model → agent-runtime/models}/base.d.ts +1 -1
- package/dist/{model → agent-runtime/models}/base.js.map +1 -1
- package/dist/{model → agent-runtime/models}/client.d.ts +3 -3
- package/dist/{model → agent-runtime/models}/client.js +2 -2
- package/dist/agent-runtime/models/client.js.map +1 -0
- package/dist/agent-runtime/models/image/compress.js.map +1 -0
- package/dist/{model → agent-runtime/models}/image/transport.d.ts +2 -2
- package/dist/{model → agent-runtime/models}/image/transport.js +1 -1
- package/dist/agent-runtime/models/image/transport.js.map +1 -0
- package/dist/agent-runtime/models/vlm/autoglm.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/base.d.ts +1 -1
- package/dist/agent-runtime/models/vlm/base.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/general.js +19 -25
- package/dist/agent-runtime/models/vlm/general.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/guiowl.js +1 -1
- package/dist/agent-runtime/models/vlm/guiowl.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/index.d.ts +0 -2
- package/dist/agent-runtime/models/vlm/index.js +7 -0
- package/dist/agent-runtime/models/vlm/index.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/maiui.js +1 -1
- package/dist/agent-runtime/models/vlm/maiui.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm}/qwenvl.js +1 -1
- package/dist/agent-runtime/models/vlm/qwenvl.js.map +1 -0
- package/dist/{model/adapters → agent-runtime/models/vlm/shared}/tool-call.d.ts +1 -1
- package/dist/agent-runtime/models/vlm/shared/tool-call.js.map +1 -0
- package/dist/agent-runtime/modes/hybrid/index.d.ts +58 -0
- package/dist/{adapters/agentcore-runner.js → agent-runtime/modes/hybrid/index.js} +21 -54
- package/dist/agent-runtime/modes/hybrid/index.js.map +1 -0
- package/dist/agent-runtime/{planner → modes/hybrid/planner}/base.d.ts +2 -2
- package/dist/agent-runtime/modes/hybrid/planner/base.js.map +1 -0
- package/dist/agent-runtime/modes/hybrid/planner/llm-planner.js.map +1 -0
- package/dist/agent-runtime/modes/hybrid/planner/noop-planner.js.map +1 -0
- package/dist/agent-runtime/{planner → modes/hybrid/planner}/prompts.d.ts +2 -2
- package/dist/agent-runtime/modes/hybrid/planner/prompts.js.map +1 -0
- package/dist/{adapters → agent-runtime/modes/orchestrator}/app-guide.d.ts +1 -1
- package/dist/{adapters → agent-runtime/modes/orchestrator}/app-guide.js +51 -6
- package/dist/agent-runtime/modes/orchestrator/app-guide.js.map +1 -0
- package/dist/{adapters/agentcore-orchestrator.d.ts → agent-runtime/modes/orchestrator/index.d.ts} +5 -6
- package/dist/{adapters/agentcore-orchestrator.js → agent-runtime/modes/orchestrator/index.js} +30 -21
- package/dist/agent-runtime/modes/orchestrator/index.js.map +1 -0
- package/dist/{adapters/orchestrator-agent-prompt.js → agent-runtime/modes/orchestrator/prompt.js} +5 -11
- package/dist/agent-runtime/modes/orchestrator/prompt.js.map +1 -0
- package/dist/agent-runtime/plan-executor.d.ts +1 -1
- package/dist/agent-runtime/reflection.js +46 -36
- package/dist/agent-runtime/reflection.js.map +1 -1
- package/dist/agent-runtime/vlm.d.ts +17 -0
- package/dist/agent-runtime/vlm.js +18 -0
- package/dist/agent-runtime/vlm.js.map +1 -0
- package/dist/api.d.ts +9 -20
- package/dist/api.js +26 -33
- package/dist/api.js.map +1 -1
- package/dist/app/agent-host.js +11 -0
- package/dist/app/agent-host.js.map +1 -1
- package/dist/app/multi-device-host.d.ts +144 -0
- package/dist/app/multi-device-host.js +242 -0
- package/dist/app/multi-device-host.js.map +1 -0
- package/dist/app/plugin-commands/device-status-command.d.ts +28 -0
- package/dist/app/plugin-commands/device-status-command.js +101 -0
- package/dist/app/plugin-commands/device-status-command.js.map +1 -0
- package/dist/app/session-manager.d.ts +1 -0
- package/dist/app/session-manager.js +12 -11
- package/dist/app/session-manager.js.map +1 -1
- package/dist/bootstrap.d.ts +0 -2
- package/dist/bootstrap.js +131 -78
- package/dist/bootstrap.js.map +1 -1
- package/dist/config/loader.d.ts +4 -19
- package/dist/config/loader.js +64 -89
- package/dist/config/loader.js.map +1 -1
- package/dist/config/schema.d.ts +26 -40
- package/dist/config/schema.js +30 -57
- package/dist/config/schema.js.map +1 -1
- package/dist/core/task-outcome.d.ts +32 -0
- package/dist/core/task-outcome.js +68 -0
- package/dist/core/task-outcome.js.map +1 -0
- package/dist/fleet/adapters/process/in-process-manager.d.ts +74 -0
- package/dist/fleet/adapters/process/in-process-manager.js +212 -0
- package/dist/fleet/adapters/process/in-process-manager.js.map +1 -0
- package/dist/fleet/adapters/process/node-process-manager.d.ts +28 -0
- package/dist/fleet/adapters/process/node-process-manager.js +13 -2
- package/dist/fleet/adapters/process/node-process-manager.js.map +1 -1
- package/dist/fleet/supervisor.d.ts +21 -0
- package/dist/fleet/supervisor.js +204 -9
- package/dist/fleet/supervisor.js.map +1 -1
- package/dist/inbound/acp/handlers/models.js +0 -1
- package/dist/inbound/acp/handlers/models.js.map +1 -1
- package/dist/inbound/cli/fleet-command.d.ts +32 -0
- package/dist/inbound/cli/fleet-command.js +51 -2
- package/dist/inbound/cli/fleet-command.js.map +1 -1
- package/dist/inbound/cli/main.d.ts +3 -8
- package/dist/inbound/cli/main.js +14 -54
- package/dist/inbound/cli/main.js.map +1 -1
- package/dist/inbound/cli/run-command.d.ts +2 -2
- package/dist/inbound/cli/run-command.js +10 -9
- package/dist/inbound/cli/run-command.js.map +1 -1
- package/dist/index.d.ts +8 -7
- package/dist/index.js +4 -5
- package/dist/index.js.map +1 -1
- package/dist/plugins/message-bus/device-command.d.ts +70 -0
- package/dist/plugins/message-bus/device-command.js +197 -0
- package/dist/plugins/message-bus/device-command.js.map +1 -0
- package/dist/plugins/message-bus/index.d.ts +16 -0
- package/dist/plugins/message-bus/index.js +17 -0
- package/dist/plugins/message-bus/index.js.map +1 -1
- package/dist/plugins/message-bus/ms-channel-notifier.js +30 -2
- package/dist/plugins/message-bus/ms-channel-notifier.js.map +1 -1
- package/dist/ports/agent-core.d.ts +12 -6
- package/dist/ports/agent-host.d.ts +2 -0
- package/package.json +1 -1
- package/dist/adapters/agentcore-inner-loop.d.ts +0 -28
- package/dist/adapters/agentcore-inner-loop.js +0 -71
- package/dist/adapters/agentcore-inner-loop.js.map +0 -1
- package/dist/adapters/agentcore-orchestrator.js.map +0 -1
- package/dist/adapters/agentcore-runner.d.ts +0 -70
- package/dist/adapters/agentcore-runner.js.map +0 -1
- package/dist/adapters/app-guide.js.map +0 -1
- package/dist/adapters/orchestrator-agent-prompt.js.map +0 -1
- package/dist/adapters/vlm-inner-loop.d.ts +0 -94
- package/dist/adapters/vlm-inner-loop.js +0 -202
- package/dist/adapters/vlm-inner-loop.js.map +0 -1
- package/dist/agent-runtime/adapters/ui-tars.d.ts +0 -13
- package/dist/agent-runtime/adapters/ui-tars.js +0 -22
- package/dist/agent-runtime/adapters/ui-tars.js.map +0 -1
- package/dist/agent-runtime/planner/base.js.map +0 -1
- package/dist/agent-runtime/planner/llm-planner.js.map +0 -1
- package/dist/agent-runtime/planner/noop-planner.js.map +0 -1
- package/dist/agent-runtime/planner/prompts.js.map +0 -1
- package/dist/agent.d.ts +0 -110
- package/dist/agent.js +0 -162
- package/dist/agent.js.map +0 -1
- package/dist/grounding/grounder.d.ts +0 -46
- package/dist/grounding/grounder.js +0 -82
- package/dist/grounding/grounder.js.map +0 -1
- package/dist/model/adapters/autoglm.js.map +0 -1
- package/dist/model/adapters/base.js.map +0 -1
- package/dist/model/adapters/general.js.map +0 -1
- package/dist/model/adapters/guiowl.js.map +0 -1
- package/dist/model/adapters/index.js +0 -9
- package/dist/model/adapters/index.js.map +0 -1
- package/dist/model/adapters/maiui.js.map +0 -1
- package/dist/model/adapters/qwenvl.js.map +0 -1
- package/dist/model/adapters/showui.d.ts +0 -1
- package/dist/model/adapters/showui.js +0 -132
- package/dist/model/adapters/showui.js.map +0 -1
- package/dist/model/adapters/tool-call.js.map +0 -1
- package/dist/model/adapters/uitars.d.ts +0 -1
- package/dist/model/adapters/uitars.js +0 -176
- package/dist/model/adapters/uitars.js.map +0 -1
- package/dist/model/client.js.map +0 -1
- package/dist/model/image/compress.js.map +0 -1
- package/dist/model/image/transport.js.map +0 -1
- /package/dist/{model → agent-runtime/models}/base.js +0 -0
- /package/dist/{model → agent-runtime/models}/image/compress.d.ts +0 -0
- /package/dist/{model → agent-runtime/models}/image/compress.js +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/autoglm.d.ts +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/autoglm.js +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/base.js +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/general.d.ts +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/guiowl.d.ts +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/maiui.d.ts +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm}/qwenvl.d.ts +0 -0
- /package/dist/{model/adapters → agent-runtime/models/vlm/shared}/tool-call.js +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/base.js +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/llm-planner.d.ts +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/llm-planner.js +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/noop-planner.d.ts +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/noop-planner.js +0 -0
- /package/dist/agent-runtime/{planner → modes/hybrid/planner}/prompts.js +0 -0
- /package/dist/{adapters/orchestrator-agent-prompt.d.ts → agent-runtime/modes/orchestrator/prompt.d.ts} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"app-guide.js","sourceRoot":"","sources":["../../../../src/agent-runtime/modes/orchestrator/app-guide.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAWH,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC5C,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,UAAU;IACV,QAAQ;IACR,WAAW;CACZ,CAAC;AAOF,MAAM,MAAM,GAAmC;IAC7C,EAAE,EAAE;QACF,OAAO,EACL,0CAA0C;QAC5C,IAAI,EAAE;;;;;;;;;;;;uCAY6B;KACpC;IAED,GAAG,EAAE;QACH,OAAO,EACL,8CAA8C;QAChD,IAAI,EAAE;;;;;;;;;;;mDAWyC;KAChD;IAED,EAAE,EAAE;QACF,OAAO,EACL,8CAA8C;QAChD,IAAI,EAAE;;;;;;;;;;mDAUyC;KAChD;IAED,EAAE,EAAE;QACF,OAAO,EACL,8CAA8C;QAChD,IAAI,EAAE;;;;;;;;;;mDAUyC;KAChD;IAED,QAAQ,EAAE;QACR,OAAO,EACL,6CAA6C;QAC/C,IAAI,EAAE;;;;;;;;;;;qDAW2C;KAClD;IAED,MAAM,EAAE;QACN,OAAO,EACL,mDAAmD;QACrD,IAAI,EAAE;;;;;;;;;;;;0FAYgF;KACvF;IAED,SAAS,EAAE;QACT,OAAO,EACL,8DAA8D;QAChE,IAAI,EAAE;;;;;;;;;;;;kFAYwE;KAC/E;CACF,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,GAAW,EAAE,MAAe;IACtD,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,EAAkB,CAAC;IACvC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CACL,YAAY,GAAG,UAAU,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;YACpD,wBAAwB,CACzB,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACpC,OAAO,IAAI,GAAG,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC,IAAI,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,GAAG,OAAO,MAAM,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC;AACvD,CAAC"}
|
package/dist/{adapters/agentcore-orchestrator.d.ts → agent-runtime/modes/orchestrator/index.d.ts}
RENAMED
|
@@ -1,19 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* OrchestratorAgent — OpenAI tool-calling outer loop + mobile_use inner loop.
|
|
3
3
|
*
|
|
4
|
-
* Two-layer architecture
|
|
4
|
+
* Two-layer architecture:
|
|
5
5
|
* outer: plain OpenAI chat completions with tools[]
|
|
6
6
|
* inner: the existing AgentRunner (ReAct / Plan-Act) called via mobileUseRunner
|
|
7
7
|
*
|
|
8
8
|
* Loop is single-turn by design: the outer LLM may call get_app_guide
|
|
9
|
-
* any number of times, then calls mobile_use once → loop exits immediately
|
|
10
|
-
* regardless of success or failure.
|
|
9
|
+
* any number of times, then calls mobile_use once → loop exits immediately.
|
|
11
10
|
*
|
|
12
11
|
* Tools exposed to the outer LLM:
|
|
13
12
|
* - get_app_guide(app, action?) → static UI path hints
|
|
14
13
|
* - mobile_use(task) → calls mobileUseRunner, returns text
|
|
15
14
|
*/
|
|
16
|
-
import type { AgentCoreRunCallbacks, AgentCoreRunOptions, AgentCoreRunResult, IAgentCore } from "
|
|
15
|
+
import type { AgentCoreRunCallbacks, AgentCoreRunOptions, AgentCoreRunResult, IAgentCore } from "../../../ports/agent-core.js";
|
|
17
16
|
export interface OrchestratorOuterConfig {
|
|
18
17
|
apiKey?: string;
|
|
19
18
|
baseUrl?: string;
|
|
@@ -22,12 +21,12 @@ export interface OrchestratorOuterConfig {
|
|
|
22
21
|
export interface OrchestratorAgentOptions {
|
|
23
22
|
outer: OrchestratorOuterConfig;
|
|
24
23
|
/** Runs the inner ReAct loop and returns the final text. */
|
|
25
|
-
|
|
24
|
+
vlmRunner: (task: string, maxSteps: number, signal?: AbortSignal, callbacks?: AgentCoreRunCallbacks) => Promise<string>;
|
|
26
25
|
}
|
|
27
26
|
export declare class OrchestratorAgent implements IAgentCore {
|
|
28
27
|
private readonly client;
|
|
29
28
|
private _model;
|
|
30
|
-
private readonly
|
|
29
|
+
private readonly vlmRunner;
|
|
31
30
|
constructor(opts: OrchestratorAgentOptions);
|
|
32
31
|
get modelName(): string;
|
|
33
32
|
get providerName(): string;
|
package/dist/{adapters/agentcore-orchestrator.js → agent-runtime/modes/orchestrator/index.js}
RENAMED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* OrchestratorAgent — OpenAI tool-calling outer loop + mobile_use inner loop.
|
|
3
3
|
*
|
|
4
|
-
* Two-layer architecture
|
|
4
|
+
* Two-layer architecture:
|
|
5
5
|
* outer: plain OpenAI chat completions with tools[]
|
|
6
6
|
* inner: the existing AgentRunner (ReAct / Plan-Act) called via mobileUseRunner
|
|
7
7
|
*
|
|
8
8
|
* Loop is single-turn by design: the outer LLM may call get_app_guide
|
|
9
|
-
* any number of times, then calls mobile_use once → loop exits immediately
|
|
10
|
-
* regardless of success or failure.
|
|
9
|
+
* any number of times, then calls mobile_use once → loop exits immediately.
|
|
11
10
|
*
|
|
12
11
|
* Tools exposed to the outer LLM:
|
|
13
12
|
* - get_app_guide(app, action?) → static UI path hints
|
|
@@ -15,9 +14,10 @@
|
|
|
15
14
|
*/
|
|
16
15
|
import OpenAI from "openai";
|
|
17
16
|
import { getAppGuide, SUPPORTED_APPS } from "./app-guide.js";
|
|
18
|
-
import { ORCHESTRATOR_AGENT_SYSTEM_PROMPT } from "./
|
|
17
|
+
import { ORCHESTRATOR_AGENT_SYSTEM_PROMPT } from "./prompt.js";
|
|
18
|
+
import { taskOutcomeFromTerminator } from "../../../core/task-outcome.js";
|
|
19
19
|
/** Fixed inner-VLM step budget. The outer LLM does not control this. */
|
|
20
|
-
const
|
|
20
|
+
const VLM_MAX_STEPS = 30;
|
|
21
21
|
// ── Tool definitions ─────────────────────────────────────────────────────────
|
|
22
22
|
const TOOLS = [
|
|
23
23
|
{
|
|
@@ -70,10 +70,10 @@ const TOOLS = [
|
|
|
70
70
|
export class OrchestratorAgent {
|
|
71
71
|
client;
|
|
72
72
|
_model;
|
|
73
|
-
|
|
73
|
+
vlmRunner;
|
|
74
74
|
constructor(opts) {
|
|
75
75
|
this._model = opts.outer.model ?? "gpt-4o";
|
|
76
|
-
this.
|
|
76
|
+
this.vlmRunner = opts.vlmRunner;
|
|
77
77
|
this.client = new OpenAI({
|
|
78
78
|
apiKey: opts.outer.apiKey ?? process.env.OUTER_API_KEY ?? process.env.GUI_AGENT_API_KEY ?? "EMPTY",
|
|
79
79
|
baseURL: opts.outer.baseUrl ?? process.env.OUTER_BASE_URL ?? "https://api.openai.com/v1",
|
|
@@ -96,12 +96,10 @@ export class OrchestratorAgent {
|
|
|
96
96
|
{ role: "system", content: ORCHESTRATOR_AGENT_SYSTEM_PROMPT },
|
|
97
97
|
{ role: "user", content: task },
|
|
98
98
|
];
|
|
99
|
-
let
|
|
99
|
+
let vlmCalled = false;
|
|
100
100
|
let finalMessage = "";
|
|
101
101
|
try {
|
|
102
|
-
|
|
103
|
-
// the model emits a plain text reply (no tool calls).
|
|
104
|
-
while (!mobileUseCalled) {
|
|
102
|
+
while (!vlmCalled) {
|
|
105
103
|
if (abortSignal?.aborted)
|
|
106
104
|
break;
|
|
107
105
|
const response = await this.client.chat.completions.create({
|
|
@@ -115,13 +113,11 @@ export class OrchestratorAgent {
|
|
|
115
113
|
break;
|
|
116
114
|
const msg = choice.message;
|
|
117
115
|
messages.push(msg);
|
|
118
|
-
// Plain text reply — no tools called, we're done.
|
|
119
116
|
if (!msg.tool_calls || msg.tool_calls.length === 0) {
|
|
120
117
|
finalMessage = msg.content ?? "";
|
|
121
118
|
onText?.(finalMessage);
|
|
122
119
|
break;
|
|
123
120
|
}
|
|
124
|
-
// Process each tool call in sequence.
|
|
125
121
|
const toolResults = [];
|
|
126
122
|
for (const call of msg.tool_calls) {
|
|
127
123
|
const fn = call.function.name;
|
|
@@ -144,11 +140,11 @@ export class OrchestratorAgent {
|
|
|
144
140
|
onTool?.({ id: call.id, name: fn, args, status: "end", result: resultText });
|
|
145
141
|
}
|
|
146
142
|
else if (fn === "mobile_use") {
|
|
147
|
-
|
|
143
|
+
vlmCalled = true;
|
|
148
144
|
const innerTask = String(args.task ?? task);
|
|
149
|
-
const innerMax =
|
|
145
|
+
const innerMax = VLM_MAX_STEPS;
|
|
150
146
|
try {
|
|
151
|
-
resultText = await this.
|
|
147
|
+
resultText = await this.vlmRunner(innerTask, innerMax, abortSignal, {
|
|
152
148
|
onTool: opts.onTool,
|
|
153
149
|
onStep: opts.onStep,
|
|
154
150
|
onText: opts.onText,
|
|
@@ -176,8 +172,7 @@ export class OrchestratorAgent {
|
|
|
176
172
|
});
|
|
177
173
|
}
|
|
178
174
|
messages.push(...toolResults);
|
|
179
|
-
|
|
180
|
-
if (mobileUseCalled) {
|
|
175
|
+
if (vlmCalled) {
|
|
181
176
|
if (abortSignal?.aborted)
|
|
182
177
|
break;
|
|
183
178
|
const summaryResp = await this.client.chat.completions.create({
|
|
@@ -194,11 +189,25 @@ export class OrchestratorAgent {
|
|
|
194
189
|
}
|
|
195
190
|
catch (e) {
|
|
196
191
|
if (e.name === "AbortError") {
|
|
197
|
-
return {
|
|
192
|
+
return {
|
|
193
|
+
finalMessage: "",
|
|
194
|
+
steps: 0,
|
|
195
|
+
cancelled: true,
|
|
196
|
+
...taskOutcomeFromTerminator({ terminator: "cancelled" }),
|
|
197
|
+
};
|
|
198
198
|
}
|
|
199
199
|
throw e;
|
|
200
200
|
}
|
|
201
|
-
return {
|
|
201
|
+
return {
|
|
202
|
+
finalMessage,
|
|
203
|
+
steps: vlmCalled ? 1 : 0,
|
|
204
|
+
cancelled: false,
|
|
205
|
+
...taskOutcomeFromTerminator({
|
|
206
|
+
terminator: "done",
|
|
207
|
+
finalMessage,
|
|
208
|
+
taskAction: vlmCalled ? "mobile_use" : undefined,
|
|
209
|
+
}),
|
|
210
|
+
};
|
|
202
211
|
}
|
|
203
212
|
}
|
|
204
|
-
//# sourceMappingURL=
|
|
213
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/agent-runtime/modes/orchestrator/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC7D,OAAO,EAAE,gCAAgC,EAAE,MAAM,aAAa,CAAC;AAC/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,+BAA+B,CAAC;AAgB1E,wEAAwE;AACxE,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,gFAAgF;AAEhF,MAAM,KAAK,GAAiD;IAC1D;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,IAAI,EAAE,eAAe;YACrB,WAAW,EACT,2DAA2D;gBAC3D,6EAA6E;gBAC7E,mBAAmB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;gBAChD,iDAAiD;YACnD,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,GAAG,EAAE;wBACH,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,qBAAqB,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;qBAC9D;oBACD,MAAM,EAAE;wBACN,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;wBAClB,WAAW,EAAE,gFAAgF;qBAC9F;iBACF;gBACD,QAAQ,EAAE,CAAC,KAAK,CAAC;aAClB;SACF;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,IAAI,EAAE,YAAY;YAClB,WAAW,EACT,kEAAkE;gBAClE,mEAAmE;gBACnE,+EAA+E;gBAC/E,8EAA8E;YAChF,UAAU,EAAE;gBACV,IAAI,EAAE,QAAQ;gBACd,UAAU,EAAE;oBACV,IAAI,EAAE;wBACJ,IAAI,EAAE,QAAQ;wBACd,WAAW,EAAE,0DAA0D;qBACxE;iBACF;gBACD,QAAQ,EAAE,CAAC,MAAM,CAAC;aACnB;SACF;KACF;CACF,CAAC;AAEF,gFAAgF;AAEhF,MAAM,OAAO,iBAAiB;IACX,MAAM,CAAS;IACxB,MAAM,CAAS;IACN,SAAS,CAAwC;IAElE,YAAY,IAA8B;QACxC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC;QAC3C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC;YACvB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,OAAO;YAClG,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,2BAA2B;YACxF,OAAO,EAAE,OAAO;YAChB,UAAU,EAAE,CAAC;SACd,CAAC,CAAC;IACL,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,YAAY;QACd,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,SAAiB,EAAE,KAAa;QAC7C,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACtB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAyB;QACjC,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;QAE5D,MAAM,QAAQ,GAAyD;YACrE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,gCAAgC,EAAE;YAC7D,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;SAChC,CAAC;QAEF,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,IAAI,YAAY,GAAG,EAAE,CAAC;QAEtB,IAAI,CAAC;YACH,OAAO,CAAC,SAAS,EAAE,CAAC;gBAClB,IAAI,WAAW,EAAE,OAAO;oBAAE,MAAM;gBAEhC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CACxD;oBACE,KAAK,EAAE,IAAI,CAAC,MAAM;oBAClB,QAAQ;oBACR,KAAK,EAAE,KAAK;oBACZ,WAAW,EAAE,MAAM;iBACpB,EACD,EAAE,MAAM,EAAE,WAAW,EAAE,CACxB,CAAC;gBAEF,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACnC,IAAI,CAAC,MAAM;oBAAE,MAAM;gBAEnB,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC;gBAC3B,QAAQ,CAAC,IAAI,CAAC,GAAyD,CAAC,CAAC;gBAEzE,IAAI,CAAC,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACnD,YAAY,GAAG,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;oBACjC,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC;oBACvB,MAAM;gBACR,CAAC;gBAED,MAAM,WAAW,GAA6D,EAAE,CAAC;gBAEjF,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBAClC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;oBAC9B,IAAI,IAAI,GAA4B,EAAE,CAAC;oBACvC,IAAI,CAAC;wBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC;oBACrD,CAAC;oBAAC,MAAM,CAAC;wBACP,uCAAuC;oBACzC,CAAC;oBAED,MAAM,EAAE,CAAC;wBACP,EAAE,EAAE,IAAI,CAAC,EAAE;wBACX,IAAI,EAAE,EAAE;wBACR,IAAI;wBACJ,MAAM,EAAE,OAAO;qBAChB,CAAC,CAAC;oBAEH,IAAI,UAAkB,CAAC;oBAEvB,IAAI,EAAE,KAAK,eAAe,EAAE,CAAC;wBAC3B,UAAU,GAAG,WAAW,CACtB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,EACtB,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAC9C,CAAC;wBACF,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;oBAC/E,CAAC;yBAAM,IAAI,EAAE,KAAK,YAAY,EAAE,CAAC;wBAC/B,SAAS,GAAG,IAAI,CAAC;wBACjB,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC;wBAC5C,MAAM,QAAQ,GAAG,aAAa,CAAC;wBAE/B,IAAI,CAAC;4BACH,UAAU,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE;gCAClE,MAAM,EAAE,IAAI,CAAC,MAAM;gCACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gCACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gCACnB,OAAO,EAAE,IAAI,CAAC,OAAO;6BACtB,CAAC,CAAC;wBACL,CAAC;wBAAC,OAAO,CAAC,EAAE,CAAC;4BACX,UAAU,GAAG,oBAAqB,CAAW,CAAC,OAAO,EAAE,CAAC;4BACxD,OAAO,EAAE,CAAC;gCACR,IAAI,EAAE,kBAAkB;gCACxB,OAAO,EAAG,CAAW,CAAC,OAAO;gCAC7B,KAAK,EAAE,KAAK;6BACb,CAAC,CAAC;wBACL,CAAC;wBACD,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;oBAC/E,CAAC;yBAAM,CAAC;wBACN,UAAU,GAAG,iBAAiB,EAAE,EAAE,CAAC;wBACnC,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;oBAC9F,CAAC;oBAED,WAAW,CAAC,IAAI,CAAC;wBACf,IAAI,EAAE,MAAM;wBACZ,YAAY,EAAE,IAAI,CAAC,EAAE;wBACrB,OAAO,EAAE,UAAU;qBACpB,CAAC,CAAC;gBACL,CAAC;gBAED,QAAQ,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;gBAE9B,IAAI,SAAS,EAAE,CAAC;oBACd,IAAI,WAAW,EAAE,OAAO;wBAAE,MAAM;oBAChC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAC3D;wBACE,KAAK,EAAE,IAAI,CAAC,MAAM;wBAClB,QAAQ;wBACR,KAAK,EAAE,KAAK;wBACZ,WAAW,EAAE,MAAM;qBACpB,EACD,EAAE,MAAM,EAAE,WAAW,EAAE,CACxB,CAAC;oBACF,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAC;oBAC9D,MAAM,EAAE,CAAC,YAAY,CAAC,CAAC;oBACvB,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAK,CAAW,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBACvC,OAAO;oBACL,YAAY,EAAE,EAAE;oBAChB,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,IAAI;oBACf,GAAG,yBAAyB,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,CAAC;iBAC1D,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,CAAC;QACV,CAAC;QAED,OAAO;YACL,YAAY;YACZ,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACxB,SAAS,EAAE,KAAK;YAChB,GAAG,yBAAyB,CAAC;gBAC3B,UAAU,EAAE,MAAM;gBAClB,YAAY;gBACZ,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;aACjD,CAAC;SACH,CAAC;IACJ,CAAC;CACF"}
|
package/dist/{adapters/orchestrator-agent-prompt.js → agent-runtime/modes/orchestrator/prompt.js}
RENAMED
|
@@ -14,6 +14,7 @@ export const ORCHESTRATOR_AGENT_SYSTEM_PROMPT = `\
|
|
|
14
14
|
2. mobile_use 只能调用一次。调用后立刻结束,不再调用任何工具。
|
|
15
15
|
3. 禁止使用 bash / shell / exec 等工具直接运行 adb 命令。所有设备操作只能通过 mobile_use 完成。
|
|
16
16
|
4. get_app_guide 可多次调用(例如先查 App 能力概览再查 post 步骤),但 mobile_use 只能调用一次。
|
|
17
|
+
5. 对于发布类任务,如果点击「发布」/「Post」/「Share」后 App 自动回到首页、个人页、内容流首页或发布成功结果页,视为任务已完成,立即终止。不要再次点击发布入口,不要重复发布同一内容。
|
|
17
18
|
|
|
18
19
|
## mobile_use 的 task 参数格式(强制)
|
|
19
20
|
|
|
@@ -21,10 +22,11 @@ export const ORCHESTRATOR_AGENT_SYSTEM_PROMPT = `\
|
|
|
21
22
|
|
|
22
23
|
**第一部分:任务总览**(一句话概括目标,例如:「在 B站 搜索张雪峰并给第一个视频点赞」)
|
|
23
24
|
|
|
24
|
-
|
|
25
|
+
**第二部分:具体步骤**(编号列表,步骤数量不超过 12 条)
|
|
25
26
|
- 步骤粒度:每步对应一个原子 UI 操作或一次判断(「打开 App」「点击搜索框」「输入文字」「等待加载」「点击第一个视频」「点击点赞按钮」等)
|
|
26
27
|
- 不能过细(不拆分手势细节),不能过粗(不把多个操作合并成一步)
|
|
27
28
|
- 必须覆盖:启动 / 进入目标页面 → 核心操作 → 确认完成 → 异常处理(弹窗 / 权限 / 加载失败)
|
|
29
|
+
- 对于发布类任务,步骤中必须明确写出:发布按钮点击成功后,如果页面回到首页/个人页/信息流首页,直接判定成功并结束,禁止再次进入发布流程
|
|
28
30
|
|
|
29
31
|
示例格式:
|
|
30
32
|
\`\`\`
|
|
@@ -42,18 +44,10 @@ export const ORCHESTRATOR_AGENT_SYSTEM_PROMPT = `\
|
|
|
42
44
|
9. 点击第一个视频,进入视频播放页。
|
|
43
45
|
10. 等待视频播放页加载完成。
|
|
44
46
|
11. 如出现弹窗(会员购买、青少年模式等),关闭弹窗。
|
|
45
|
-
12.
|
|
46
|
-
13. 点击点赞按钮。
|
|
47
|
-
14. 确认点赞按钮已高亮 / 变色,表示点赞成功。
|
|
48
|
-
15. 如点击后无反应,等待 1–2 秒后重试一次。
|
|
49
|
-
16. 记录当前状态(点赞是否成功)。
|
|
50
|
-
17. 如需登录才能点赞,停止并说明无法完成。
|
|
51
|
-
18. 任务完成,不做其他操作。
|
|
52
|
-
19. 如页面跳转到意外位置,按返回键回到视频页面。
|
|
53
|
-
20. 最终确认:点赞图标处于激活状态,任务结束。
|
|
47
|
+
12. 点击点赞按钮并确认高亮;若需登录则停止说明无法完成,否则任务结束。
|
|
54
48
|
\`\`\`
|
|
55
49
|
|
|
56
50
|
## 任务结束
|
|
57
51
|
|
|
58
52
|
mobile_use 执行完毕后,用一两句话向用户汇报结果。`;
|
|
59
|
-
//# sourceMappingURL=
|
|
53
|
+
//# sourceMappingURL=prompt.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prompt.js","sourceRoot":"","sources":["../../../../src/agent-runtime/modes/orchestrator/prompt.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAEhD,MAAM,CAAC,MAAM,gCAAgC,GAAG;;;;;iBAK/B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+BAuCV,CAAC"}
|
|
@@ -34,7 +34,7 @@ import type { Action, Observation } from "@beeos-ai/device-common";
|
|
|
34
34
|
import type { IMcpRuntime } from "../mcp-client/runtime.js";
|
|
35
35
|
import type { IModelAdapter, ModelUsage, StepHistory } from "./adapters/base.js";
|
|
36
36
|
import { type AgentLoopTerminator, type AgentRunHooks } from "./core.js";
|
|
37
|
-
import type { IPlanner } from "./planner/base.js";
|
|
37
|
+
import type { IPlanner } from "./modes/hybrid/planner/base.js";
|
|
38
38
|
import type { HierarchicalReflector } from "./reflection.js";
|
|
39
39
|
export interface PlanExecutorOptions {
|
|
40
40
|
planner: IPlanner;
|
|
@@ -48,14 +48,9 @@ export class HierarchicalReflector {
|
|
|
48
48
|
if (recent.length === 4 && recent.every((a) => actionsEqual(a, recent[0]))) {
|
|
49
49
|
return `stalled: action ${recent[0].type} repeated 4 consecutive times`;
|
|
50
50
|
}
|
|
51
|
-
//
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
.map((h) => h.action?.type)
|
|
55
|
-
.filter((t) => Boolean(t));
|
|
56
|
-
if (recentTypes.filter((t) => t === "screenshot").length >= 5) {
|
|
57
|
-
return "stalled: screenshot-heavy trajectory with no actuation";
|
|
58
|
-
}
|
|
51
|
+
// [已弃用] screenshot-heavy stall 检测:连续6步中有5个 screenshot 动作时终止。
|
|
52
|
+
// 当前所有适配器(autoglm、general 等)截图均由 AgentLoop 被动触发,
|
|
53
|
+
// 模型不会主动发出 screenshot 动作,此规则永远不会触发,故弃用。
|
|
59
54
|
return null;
|
|
60
55
|
}
|
|
61
56
|
/**
|
|
@@ -117,10 +112,15 @@ export class HierarchicalReflector {
|
|
|
117
112
|
// Rule 1 — double-type without re-focus. If the previous `type`
|
|
118
113
|
// visibly altered the screen (e.g. text actually appeared), don't
|
|
119
114
|
// block; the grounder may legitimately want to keep typing.
|
|
120
|
-
|
|
115
|
+
// Rule 1 — 仅当两次 type 内容完全相同时才触发(重复卡死)。
|
|
116
|
+
// 原逻辑依赖 !screenChanged,无 hash 时永远为 true,会误杀合法的追加输入。
|
|
117
|
+
if (nextAction.type === "type" &&
|
|
118
|
+
prev.action.type === "type" &&
|
|
119
|
+
nextAction.payload.text ===
|
|
120
|
+
prev.action.payload.text) {
|
|
121
121
|
return {
|
|
122
122
|
code: "double_type_without_focus",
|
|
123
|
-
reason: "two consecutive `type` actions
|
|
123
|
+
reason: "two consecutive `type` actions with identical text — likely a stuck loop, focus probably wasn't established",
|
|
124
124
|
};
|
|
125
125
|
}
|
|
126
126
|
// Rule 2 — repeated tap.
|
|
@@ -145,34 +145,44 @@ export class HierarchicalReflector {
|
|
|
145
145
|
// hash known + screen progressed (or coords too far apart) → allow
|
|
146
146
|
}
|
|
147
147
|
else {
|
|
148
|
-
// legacy fallback for pure-vision adapters
|
|
149
|
-
if (dx <= 8 && dy <= 8) {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
}
|
|
157
|
-
// Rule 3 — blind swipe (same direction, no screenshot between, and
|
|
158
|
-
// — when hash data is available — no observable scroll progress).
|
|
159
|
-
if (nextAction.type === "swipe" &&
|
|
160
|
-
prev.action.type === "swipe" &&
|
|
161
|
-
isBoxTarget(nextAction.target) &&
|
|
162
|
-
isBoxTarget(prev.action.target) &&
|
|
163
|
-
!screenChanged) {
|
|
164
|
-
const sameDir = sign(nextAction.target.x2 - nextAction.target.x1) ===
|
|
165
|
-
sign(prev.action.target.x2 - prev.action.target.x1) &&
|
|
166
|
-
sign(nextAction.target.y2 - nextAction.target.y1) ===
|
|
167
|
-
sign(prev.action.target.y2 - prev.action.target.y1);
|
|
168
|
-
const sawScreenshotBetween = anyActionBetween(history, prev.idx, history.length, (a) => a.type === "screenshot");
|
|
169
|
-
if (sameDir && !sawScreenshotBetween) {
|
|
170
|
-
return {
|
|
171
|
-
code: "blind_swipe",
|
|
172
|
-
reason: "swipe in the same direction as the previous step with no intervening screenshot or scroll progress — re-ground before scrolling further",
|
|
173
|
-
};
|
|
148
|
+
// legacy fallback for pure-vision adapters — 此阶段暂不开启(无 hash 数据时对 autoglm 等纯视觉适配器过于激进)
|
|
149
|
+
// if (dx <= 8 && dy <= 8) {
|
|
150
|
+
// return {
|
|
151
|
+
// code: "double_tap_same_coords",
|
|
152
|
+
// reason: `two consecutive tap on (${prev.action.target.x},${prev.action.target.y}) → looks like a misclassified double_tap`,
|
|
153
|
+
// };
|
|
154
|
+
// }
|
|
174
155
|
}
|
|
175
156
|
}
|
|
157
|
+
// [已弃用] Rule 3 — blind_swipe:连续同向 swipe 且中间无 screenshot 时触发。
|
|
158
|
+
// 依赖 !screenChanged,无 hash 时永远为 true,会误杀合法的连续滚动操作。
|
|
159
|
+
// AgentLoop 被动截图不计入 sawScreenshotBetween,导致条件永远满足,故弃用。
|
|
160
|
+
// if (
|
|
161
|
+
// nextAction.type === "swipe" &&
|
|
162
|
+
// prev.action.type === "swipe" &&
|
|
163
|
+
// isBoxTarget(nextAction.target) &&
|
|
164
|
+
// isBoxTarget(prev.action.target) &&
|
|
165
|
+
// !screenChanged
|
|
166
|
+
// ) {
|
|
167
|
+
// const sameDir =
|
|
168
|
+
// sign(nextAction.target.x2 - nextAction.target.x1) ===
|
|
169
|
+
// sign(prev.action.target.x2 - prev.action.target.x1) &&
|
|
170
|
+
// sign(nextAction.target.y2 - nextAction.target.y1) ===
|
|
171
|
+
// sign(prev.action.target.y2 - prev.action.target.y1);
|
|
172
|
+
// const sawScreenshotBetween = anyActionBetween(
|
|
173
|
+
// history,
|
|
174
|
+
// prev.idx,
|
|
175
|
+
// history.length,
|
|
176
|
+
// (a) => a.type === "screenshot",
|
|
177
|
+
// );
|
|
178
|
+
// if (sameDir && !sawScreenshotBetween) {
|
|
179
|
+
// return {
|
|
180
|
+
// code: "blind_swipe",
|
|
181
|
+
// reason:
|
|
182
|
+
// "swipe in the same direction as the previous step with no intervening screenshot or scroll progress — re-ground before scrolling further",
|
|
183
|
+
// };
|
|
184
|
+
// }
|
|
185
|
+
// }
|
|
176
186
|
return null;
|
|
177
187
|
}
|
|
178
188
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reflection.js","sourceRoot":"","sources":["../../src/agent-runtime/reflection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAiDH,MAAM,OAAO,qBAAqB;IACf,OAAO,CAAiB;IAChC,eAAe,CAAoB;IACnC,mBAAmB,CAAwB;IAEpD,YAAY,gBAA+D;QACzE,IAAI,gBAAgB,IAAI,MAAM,IAAK,gBAAkC,EAAE,CAAC;YACtE,IAAI,CAAC,OAAO,GAAG,gBAAiC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAI,gBAA6D,IAAI,EAAE,CAAC;YAClF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;YAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACtD,CAAC;QACD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,6CAA6C;IAClE,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,OAAsB,EAAE,IAAY,EAAE,SAAiB;QAC/D,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEnD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAChD,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,oDAAoD;QACpD,MAAM,MAAM,GAAG,OAAO;aACnB,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,mBAAmB,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,+BAA+B,CAAC;QAC1E,CAAC;QAED,
|
|
1
|
+
{"version":3,"file":"reflection.js","sourceRoot":"","sources":["../../src/agent-runtime/reflection.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAiDH,MAAM,OAAO,qBAAqB;IACf,OAAO,CAAiB;IAChC,eAAe,CAAoB;IACnC,mBAAmB,CAAwB;IAEpD,YAAY,gBAA+D;QACzE,IAAI,gBAAgB,IAAI,MAAM,IAAK,gBAAkC,EAAE,CAAC;YACtE,IAAI,CAAC,OAAO,GAAG,gBAAiC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAI,gBAA6D,IAAI,EAAE,CAAC;YAClF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;YAC5B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;YAC5C,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACtD,CAAC;QACD,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,6CAA6C;IAClE,CAAC;IAED;;;OAGG;IACH,SAAS,CAAC,OAAsB,EAAE,IAAY,EAAE,SAAiB;QAC/D,IAAI,IAAI,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEnD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QAChD,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,oDAAoD;QACpD,MAAM,MAAM,GAAG,OAAO;aACnB,KAAK,CAAC,CAAC,CAAC,CAAC;aACT,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,OAAO,mBAAmB,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,+BAA+B,CAAC;QAC1E,CAAC;QAED,6DAA6D;QAC7D,iDAAiD;QACjD,wCAAwC;QAExC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAgCG;IACH,gBAAgB,CAAC,OAAsB,EAAE,UAAkB;QACzD,IAAI,CAAC,UAAU;YAAE,OAAO,IAAI,CAAC;QAE7B,qEAAqE;QACrE,IAAI,UAAU,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC1C,OAAO;gBACL,IAAI,EAAE,mBAAmB;gBACzB,MAAM,EACJ,0GAA0G;aAC7G,CAAC;QACJ,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACtC,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,qEAAqE;QACrE,oEAAoE;QACpE,8DAA8D;QAC9D,iDAAiD;QACjD,MAAM,QAAQ,GACZ,IAAI,CAAC,KAAK,CAAC,qBAAqB,KAAK,SAAS;YAC9C,IAAI,CAAC,KAAK,CAAC,oBAAoB,KAAK,SAAS,CAAC;QAChD,MAAM,aAAa,GACjB,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,qBAAqB,KAAK,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC;QACnF,MAAM,eAAe,GACnB,QAAQ,IAAI,IAAI,CAAC,KAAK,CAAC,qBAAqB,KAAK,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC;QAEnF,gEAAgE;QAChE,kEAAkE;QAClE,4DAA4D;QAC5D,uCAAuC;QACvC,oDAAoD;QACpD,IACE,UAAU,CAAC,IAAI,KAAK,MAAM;YAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,MAAM;YAC1B,UAAU,CAAC,OAA8B,CAAC,IAAI;gBAC5C,IAAI,CAAC,MAAM,CAAC,OAA8B,CAAC,IAAI,EAClD,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,2BAA2B;gBACjC,MAAM,EACJ,6GAA6G;aAChH,CAAC;QACJ,CAAC;QAED,yBAAyB;QACzB,kEAAkE;QAClE,sEAAsE;QACtE,gEAAgE;QAChE,wCAAwC;QACxC,oEAAoE;QACpE,IACE,UAAU,CAAC,IAAI,KAAK,KAAK;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK;YAC1B,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAChC,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EACjC,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAChE,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,eAAe,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,CAAC;oBAC5C,OAAO;wBACL,IAAI,EAAE,oBAAoB;wBAC1B,MAAM,EAAE,sBAAsB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,4CAA4C;qBACvH,CAAC;gBACJ,CAAC;gBACD,mEAAmE;YACrE,CAAC;iBAAM,CAAC;gBACN,sFAAsF;gBACtF,4BAA4B;gBAC5B,aAAa;gBACb,sCAAsC;gBACtC,kIAAkI;gBAClI,OAAO;gBACP,IAAI;YACN,CAAC;QACH,CAAC;QAED,6DAA6D;QAC7D,mDAAmD;QACnD,uDAAuD;QACvD,OAAO;QACP,mCAAmC;QACnC,oCAAoC;QACpC,sCAAsC;QACtC,uCAAuC;QACvC,mBAAmB;QACnB,MAAM;QACN,oBAAoB;QACpB,4DAA4D;QAC5D,+DAA+D;QAC/D,4DAA4D;QAC5D,6DAA6D;QAC7D,mDAAmD;QACnD,eAAe;QACf,gBAAgB;QAChB,sBAAsB;QACtB,sCAAsC;QACtC,OAAO;QACP,4CAA4C;QAC5C,eAAe;QACf,6BAA6B;QAC7B,gBAAgB;QAChB,qJAAqJ;QACrJ,SAAS;QACT,MAAM;QACN,IAAI;QAEJ,OAAO,IAAI,CAAC;IACd,CAAC;CACF;AAED,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAEhF,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAC;IACpD,OAAO,UAAU,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;AAC1C,CAAC;AAED,SAAS,YAAY,CAAC,CAAmB,EAAE,CAAmB;IAC5D,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3B,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,UAAU,CAAC,CAAU,EAAE,CAAU;IACxC,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IAC7F,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,CAAmB;IACxC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;AACnC,CAAC;AAED,SAAS,WAAW,CAClB,CAAmB;IAEnB,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;AACjC,CAAC;AAED,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CACtB,OAAsB;IAEtB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;QACvB,IAAI,CAAC;YAAE,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,gBAAgB,CACvB,OAAsB,EACtB,cAAsB,EACtB,YAAoB,EACpB,IAA4B;IAE5B,KAAK,IAAI,CAAC,GAAG,cAAc,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACvD,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IAChC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vlm — unified VLM loop entry point.
|
|
3
|
+
*
|
|
4
|
+
* Thin functional wrapper over `AgentRunner` that lets callers drive the
|
|
5
|
+
* screenshot→reason→act loop without instantiating a class. All three
|
|
6
|
+
* execution strategies (react / plan / react+replan) are selected by the
|
|
7
|
+
* same `mode` + `planner` heuristic as `AgentRunner`; this is just a
|
|
8
|
+
* more ergonomic call site for one-shot invocations.
|
|
9
|
+
*
|
|
10
|
+
* The OrchestratorAgent's inner loop, run-command.ts, and any future
|
|
11
|
+
* callers that need a plain function signature (vs. an `IAgentCore` port)
|
|
12
|
+
* should prefer this over constructing an `AgentRunner` directly.
|
|
13
|
+
*/
|
|
14
|
+
import { type AgentRunnerOptions, type AgentRunnerResult } from "./agent-runner.js";
|
|
15
|
+
export type { AgentRunnerOptions as VlmOptions };
|
|
16
|
+
export type { AgentRunnerResult as VlmResult };
|
|
17
|
+
export declare function vlm(opts: AgentRunnerOptions): Promise<AgentRunnerResult>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* vlm — unified VLM loop entry point.
|
|
3
|
+
*
|
|
4
|
+
* Thin functional wrapper over `AgentRunner` that lets callers drive the
|
|
5
|
+
* screenshot→reason→act loop without instantiating a class. All three
|
|
6
|
+
* execution strategies (react / plan / react+replan) are selected by the
|
|
7
|
+
* same `mode` + `planner` heuristic as `AgentRunner`; this is just a
|
|
8
|
+
* more ergonomic call site for one-shot invocations.
|
|
9
|
+
*
|
|
10
|
+
* The OrchestratorAgent's inner loop, run-command.ts, and any future
|
|
11
|
+
* callers that need a plain function signature (vs. an `IAgentCore` port)
|
|
12
|
+
* should prefer this over constructing an `AgentRunner` directly.
|
|
13
|
+
*/
|
|
14
|
+
import { AgentRunner } from "./agent-runner.js";
|
|
15
|
+
export async function vlm(opts) {
|
|
16
|
+
return new AgentRunner(opts).run();
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=vlm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vlm.js","sourceRoot":"","sources":["../../src/agent-runtime/vlm.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,WAAW,EAAmD,MAAM,mBAAmB,CAAC;AAKjG,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,IAAwB;IAChD,OAAO,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;AACrC,CAAC"}
|
package/dist/api.d.ts
CHANGED
|
@@ -1,27 +1,16 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* **NOT exported from `index.ts`.** This entry point predates the
|
|
5
|
-
* `IAgentCore` / `AgentHost` plugin architecture and is kept on disk
|
|
6
|
-
* solely so the existing one-shot harness scripts (`test-mobile-use.ts`,
|
|
7
|
-
* `test-mem.ts`) keep compiling. Production paths reach the same inner
|
|
8
|
-
* VLM through `InnerLoopVlm` (single-layer / `legacyInner`) or the
|
|
9
|
-
* `mobile_use` tool of `OrchestratorAgent` (two-layer).
|
|
10
|
-
*
|
|
11
|
-
* Internally we go through `InnerLoopVlm` so this file no longer
|
|
12
|
-
* duplicates the OpenAIClient construction logic — the loader resolves
|
|
13
|
-
* `config.mobileUse` from the same env / CLI flags / config-file source
|
|
14
|
-
* the rest of the agent uses.
|
|
2
|
+
* Legacy harness entry point — NOT exported from `index.ts`.
|
|
3
|
+
* Kept for scripts in `scripts/legacy/`. Production callers use `bootstrap()`.
|
|
15
4
|
*/
|
|
16
|
-
|
|
17
|
-
export
|
|
5
|
+
/** @deprecated — kept for harness scripts only; use bootstrap() in production. */
|
|
6
|
+
export interface AgentConfig {
|
|
7
|
+
maxSteps?: number;
|
|
8
|
+
lang?: "cn" | "en";
|
|
9
|
+
verbose?: boolean;
|
|
10
|
+
}
|
|
18
11
|
/**
|
|
19
12
|
* Drive the connected Android phone to complete a natural-language task.
|
|
20
13
|
*
|
|
21
|
-
* @
|
|
22
|
-
* Callers (including outer agents) should explicitly pass this value;
|
|
23
|
-
* only test:mobile reads GUI_AGENT_MAX_STEPS from env as a convenience.
|
|
24
|
-
*
|
|
25
|
-
* @internal — use `bootstrap()` or `InnerLoopVlm` directly in production code.
|
|
14
|
+
* @internal — use `bootstrap()` in production code.
|
|
26
15
|
*/
|
|
27
16
|
export declare function mobileUse(task: string, max_steps?: number): Promise<string>;
|
package/dist/api.js
CHANGED
|
@@ -1,48 +1,41 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
* **NOT exported from `index.ts`.** This entry point predates the
|
|
5
|
-
* `IAgentCore` / `AgentHost` plugin architecture and is kept on disk
|
|
6
|
-
* solely so the existing one-shot harness scripts (`test-mobile-use.ts`,
|
|
7
|
-
* `test-mem.ts`) keep compiling. Production paths reach the same inner
|
|
8
|
-
* VLM through `InnerLoopVlm` (single-layer / `legacyInner`) or the
|
|
9
|
-
* `mobile_use` tool of `OrchestratorAgent` (two-layer).
|
|
10
|
-
*
|
|
11
|
-
* Internally we go through `InnerLoopVlm` so this file no longer
|
|
12
|
-
* duplicates the OpenAIClient construction logic — the loader resolves
|
|
13
|
-
* `config.mobileUse` from the same env / CLI flags / config-file source
|
|
14
|
-
* the rest of the agent uses.
|
|
2
|
+
* Legacy harness entry point — NOT exported from `index.ts`.
|
|
3
|
+
* Kept for scripts in `scripts/legacy/`. Production callers use `bootstrap()`.
|
|
15
4
|
*/
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
5
|
+
import { McpClient, resolveDefaultMcpUrl } from "./mcp-client/client.js";
|
|
6
|
+
import { OpenAIClient } from "./agent-runtime/models/client.js";
|
|
7
|
+
import { OpenAIAdapter } from "./agent-runtime/adapters/openai.js";
|
|
8
|
+
import { AgentRunner } from "./agent-runtime/agent-runner.js";
|
|
18
9
|
import { loadConfig } from "./config/loader.js";
|
|
19
10
|
const DEFAULT_MAX_STEPS = 30;
|
|
20
11
|
/**
|
|
21
12
|
* Drive the connected Android phone to complete a natural-language task.
|
|
22
13
|
*
|
|
23
|
-
* @
|
|
24
|
-
* Callers (including outer agents) should explicitly pass this value;
|
|
25
|
-
* only test:mobile reads GUI_AGENT_MAX_STEPS from env as a convenience.
|
|
26
|
-
*
|
|
27
|
-
* @internal — use `bootstrap()` or `InnerLoopVlm` directly in production code.
|
|
14
|
+
* @internal — use `bootstrap()` in production code.
|
|
28
15
|
*/
|
|
29
16
|
export async function mobileUse(task, max_steps) {
|
|
30
17
|
const cfg = loadConfig({});
|
|
31
18
|
const max = max_steps ?? cfg.maxSteps ?? DEFAULT_MAX_STEPS;
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
19
|
+
const mcpUrl = process.env.MCP_URL ?? resolveDefaultMcpUrl();
|
|
20
|
+
const mcp = new McpClient({ baseUrl: mcpUrl });
|
|
21
|
+
await mcp.waitUntilReady({ attempts: 10, attemptDelayMs: 500 });
|
|
22
|
+
const client = new OpenAIClient({
|
|
23
|
+
apiKey: cfg.vlm.apiKey || undefined,
|
|
24
|
+
baseUrl: cfg.vlm.baseUrl || undefined,
|
|
25
|
+
modelName: cfg.vlm.model || undefined,
|
|
35
26
|
});
|
|
36
|
-
const
|
|
37
|
-
|
|
38
|
-
baseUrl: cfg.mobileUse.baseUrl || cfg.vlmBaseUrl || undefined,
|
|
39
|
-
modelName: cfg.mobileUse.model || cfg.vlmModel || undefined,
|
|
40
|
-
maxSteps: max,
|
|
27
|
+
const adapter = new OpenAIAdapter({
|
|
28
|
+
client,
|
|
41
29
|
lang: (process.env.GUI_AGENT_LANG ?? "cn"),
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
30
|
+
});
|
|
31
|
+
const runner = new AgentRunner({
|
|
32
|
+
task,
|
|
33
|
+
env: "mobile",
|
|
34
|
+
mcp,
|
|
35
|
+
adapter,
|
|
36
|
+
maxSteps: max,
|
|
37
|
+
});
|
|
38
|
+
const result = await runner.run();
|
|
39
|
+
return result.finalText;
|
|
47
40
|
}
|
|
48
41
|
//# sourceMappingURL=api.js.map
|
package/dist/api.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"api.js","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,SAAS,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,kCAAkC,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,oCAAoC,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAC9D,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAShD,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,SAAkB;IAC9D,MAAM,GAAG,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IAC3B,MAAM,GAAG,GAAG,SAAS,IAAI,GAAG,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAE3D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,oBAAoB,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/C,MAAM,GAAG,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;IAEhE,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;QAC9B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS;QACnC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,OAAO,IAAI,SAAS;QACrC,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,SAAS;KACtC,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,IAAI,aAAa,CAAC;QAChC,MAAM;QACN,IAAI,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAgB;KAC1D,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC;QAC7B,IAAI;QACJ,GAAG,EAAE,QAAQ;QACb,GAAG;QACH,OAAO;QACP,QAAQ,EAAE,GAAG;KACd,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;IAClC,OAAO,MAAM,CAAC,SAAS,CAAC;AAC1B,CAAC"}
|
package/dist/app/agent-host.js
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { randomUUID } from "node:crypto";
|
|
21
21
|
import { getLogger, runWithCorrelation } from "../logger/index.js";
|
|
22
|
+
import { deriveTaskOutcome, taskOutcomeFromTerminator } from "../core/task-outcome.js";
|
|
22
23
|
import { ResourceContextAugmenter, ResourceResolutionCache, } from "../resources/index.js";
|
|
23
24
|
import { ResolverError } from "../resources/resolvers/url-resolver.js";
|
|
24
25
|
import { recordResolution } from "../resources/telemetry.js";
|
|
@@ -174,6 +175,11 @@ export class AgentHostImpl {
|
|
|
174
175
|
finalMessage: error.message ?? String(error),
|
|
175
176
|
steps: 0,
|
|
176
177
|
cancelled: signal.aborted,
|
|
178
|
+
...taskOutcomeFromTerminator({
|
|
179
|
+
terminator: signal.aborted ? "cancelled" : "error",
|
|
180
|
+
finalMessage: error.message ?? String(error),
|
|
181
|
+
taskError: error.message ?? String(error),
|
|
182
|
+
}),
|
|
177
183
|
};
|
|
178
184
|
}
|
|
179
185
|
}
|
|
@@ -222,6 +228,11 @@ export class AgentHostImpl {
|
|
|
222
228
|
taskId,
|
|
223
229
|
turnId,
|
|
224
230
|
result,
|
|
231
|
+
outcome: deriveTaskOutcome({
|
|
232
|
+
taskId,
|
|
233
|
+
result,
|
|
234
|
+
...(error ? { error: error.message ?? String(error) } : {}),
|
|
235
|
+
}),
|
|
225
236
|
completedAt: Date.now(),
|
|
226
237
|
...(error ? { error: error.message ?? String(error) } : {}),
|
|
227
238
|
};
|