browser-autopilot 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/agent/loop.d.ts +5 -0
- package/dist/agent/loop.js +66 -51
- package/dist/agent/loop.js.map +1 -1
- package/dist/agent/run.js +1 -1
- package/dist/agent/run.js.map +1 -1
- package/dist/agent/tools.d.ts +126 -36
- package/dist/agent/tools.js +36 -38
- package/dist/agent/tools.js.map +1 -1
- package/dist/orchestrator.js +84 -10
- package/dist/orchestrator.js.map +1 -1
- package/dist/x11/agent.d.ts +64 -2
- package/dist/x11/agent.js +176 -59
- package/dist/x11/agent.js.map +1 -1
- package/dist/x11/input.d.ts +4 -0
- package/dist/x11/input.js +58 -8
- package/dist/x11/input.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
package/dist/agent/loop.d.ts
CHANGED
|
@@ -18,8 +18,13 @@ export interface AgentOptions {
|
|
|
18
18
|
onStep?: (step: StepRecord) => void;
|
|
19
19
|
sensitiveData?: Record<string, string>;
|
|
20
20
|
}
|
|
21
|
+
export interface AgentModeSwitchRequest {
|
|
22
|
+
mode: "x11";
|
|
23
|
+
reason: string;
|
|
24
|
+
}
|
|
21
25
|
export declare function runAgent(opts: AgentOptions): Promise<{
|
|
22
26
|
result: string | null;
|
|
23
27
|
success: boolean;
|
|
24
28
|
history: AgentHistory;
|
|
29
|
+
requestedModeSwitch: AgentModeSwitchRequest | null;
|
|
25
30
|
}>;
|
package/dist/agent/loop.js
CHANGED
|
@@ -25,6 +25,7 @@ RULES:
|
|
|
25
25
|
- Cookie/overlay dialogs: use 'evaluate' to remove them: document.querySelector('[class*=cookie]')?.remove()
|
|
26
26
|
- Prefer 'navigate' with direct URLs over clicking through menus when the URL is known.
|
|
27
27
|
- You can call multiple tools in sequence within a single step — each result is fed back to you.
|
|
28
|
+
- If the site is blocking automation, the DOM is unusable, or the task clearly needs literal mouse/keyboard control, call 'switch_to_x11' with a brief reason. X11 means local OS-level control of the browser window via real mouse and keyboard events, not DOM or CDP interaction.
|
|
28
29
|
- When done, use the 'done' tool.
|
|
29
30
|
|
|
30
31
|
For each step, think about:
|
|
@@ -33,54 +34,37 @@ For each step, think about:
|
|
|
33
34
|
3. What should you do next? (next goal)
|
|
34
35
|
4. What specific actions to take?`;
|
|
35
36
|
}
|
|
36
|
-
|
|
37
|
+
/**
|
|
38
|
+
* Wrap tool execute functions to spill large results to disk.
|
|
39
|
+
* The AI SDK lifecycle callbacks handle logging; this only handles
|
|
40
|
+
* result truncation so the LLM doesn't drown in huge tool output.
|
|
41
|
+
*/
|
|
42
|
+
function wrapToolsForSpilling(rawTools, dataDir) {
|
|
37
43
|
const wrapped = {};
|
|
38
44
|
for (const [name, t] of Object.entries(rawTools)) {
|
|
39
45
|
if (!t)
|
|
40
46
|
continue;
|
|
41
47
|
const originalExecute = t.execute;
|
|
42
|
-
if (!originalExecute) {
|
|
48
|
+
if (!originalExecute || name === "done" || name === "screenshot") {
|
|
43
49
|
wrapped[name] = t;
|
|
44
50
|
continue;
|
|
45
51
|
}
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
action.result = typeof toolResult === "string" ? toolResult : JSON.stringify(toolResult);
|
|
55
|
-
if (isDone) {
|
|
56
|
-
tracker.result = toolResult.text ?? action.result;
|
|
57
|
-
tracker.success = toolResult.success ?? true;
|
|
58
|
-
console.log(`\n✓ Task complete: ${tracker.result?.slice(0, 100)}`);
|
|
59
|
-
}
|
|
60
|
-
if (isScreenshot)
|
|
61
|
-
action.result = "(screenshot taken)";
|
|
62
|
-
if (!isDone && !isScreenshot && action.result.length > 4000) {
|
|
63
|
-
const spillPath = `${dataDir}/tool_result_${name}.txt`;
|
|
64
|
-
try {
|
|
65
|
-
mkdirSync(dataDir, { recursive: true });
|
|
66
|
-
writeFileSync(spillPath, action.result);
|
|
67
|
-
}
|
|
68
|
-
catch { }
|
|
69
|
-
return `${action.result.slice(0, 1500)}\n\n... (${action.result.length} chars — full: ${spillPath})`;
|
|
52
|
+
const spillingExecute = async (args, opts) => {
|
|
53
|
+
const toolResult = await originalExecute(args, opts);
|
|
54
|
+
const resultStr = typeof toolResult === "string" ? toolResult : JSON.stringify(toolResult);
|
|
55
|
+
if (resultStr.length > 4000) {
|
|
56
|
+
const spillPath = `${dataDir}/tool_result_${name}.txt`;
|
|
57
|
+
try {
|
|
58
|
+
mkdirSync(dataDir, { recursive: true });
|
|
59
|
+
writeFileSync(spillPath, resultStr);
|
|
70
60
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
catch (e) {
|
|
74
|
-
action.error = e.message;
|
|
75
|
-
console.log(` ✗ ${name} failed: ${e.message.slice(0, 100)}`);
|
|
76
|
-
throw e;
|
|
61
|
+
catch { }
|
|
62
|
+
return `${resultStr.slice(0, 1500)}\n\n... (${resultStr.length} chars — full: ${spillPath})`;
|
|
77
63
|
}
|
|
64
|
+
return toolResult;
|
|
78
65
|
};
|
|
79
|
-
// Preserve the full original tool (including internal AI SDK Symbols)
|
|
80
|
-
// and only override execute. Object.create + getOwnPropertyDescriptors
|
|
81
|
-
// copies symbols that spread {...t} drops.
|
|
82
66
|
const descriptors = Object.getOwnPropertyDescriptors(t);
|
|
83
|
-
descriptors.execute = { value:
|
|
67
|
+
descriptors.execute = { value: spillingExecute, writable: true, configurable: true, enumerable: true };
|
|
84
68
|
wrapped[name] = Object.create(Object.getPrototypeOf(t), descriptors);
|
|
85
69
|
}
|
|
86
70
|
return wrapped;
|
|
@@ -90,9 +74,11 @@ export async function runAgent(opts) {
|
|
|
90
74
|
const dataDir = config.browser.dataDir;
|
|
91
75
|
const gatewayModel = gateway(model);
|
|
92
76
|
const rawTools = { ...createBrowserTools(browser, dataDir), ...extraTools };
|
|
77
|
+
const tools = wrapToolsForSpilling(rawTools, dataDir);
|
|
93
78
|
const history = new AgentHistory();
|
|
94
79
|
let finalResult = null;
|
|
95
80
|
let finalSuccess = false;
|
|
81
|
+
let requestedModeSwitch = null;
|
|
96
82
|
for (let step = 0; step < maxSteps; step++) {
|
|
97
83
|
console.log(`\n${"─".repeat(50)}`);
|
|
98
84
|
console.log(`Step ${step + 1}/${maxSteps}`);
|
|
@@ -120,12 +106,7 @@ export async function runAgent(opts) {
|
|
|
120
106
|
else
|
|
121
107
|
nudge = "\n⚠️ You are repeating similar actions. Try a different tool or strategy. For text input, use the 'input' tool, not 'click'.";
|
|
122
108
|
}
|
|
123
|
-
const
|
|
124
|
-
actions: [],
|
|
125
|
-
result: null,
|
|
126
|
-
success: false,
|
|
127
|
-
};
|
|
128
|
-
const wrappedTools = wrapToolsForTracking(rawTools, dataDir, tracker);
|
|
109
|
+
const actions = [];
|
|
129
110
|
const messages = [{
|
|
130
111
|
role: "user",
|
|
131
112
|
content: [
|
|
@@ -157,11 +138,49 @@ export async function runAgent(opts) {
|
|
|
157
138
|
maxOutputTokens: 16384,
|
|
158
139
|
system: systemPrompt ?? buildSystemPrompt(dataDir),
|
|
159
140
|
messages,
|
|
160
|
-
tools
|
|
141
|
+
tools,
|
|
161
142
|
stopWhen: stepCountIs(maxActionsPerStep),
|
|
162
143
|
providerOptions: {
|
|
163
144
|
anthropic: { cacheControl: { type: "ephemeral" } },
|
|
164
145
|
},
|
|
146
|
+
experimental_onToolCallStart({ toolCall }) {
|
|
147
|
+
console.log(` → ${toolCall.toolName}(${JSON.stringify(toolCall.input).slice(0, 80)})`);
|
|
148
|
+
actions.push({ name: toolCall.toolName, params: (toolCall.input ?? {}) });
|
|
149
|
+
},
|
|
150
|
+
experimental_onToolCallFinish({ toolCall, durationMs, ...rest }) {
|
|
151
|
+
const action = [...actions].reverse().find((a) => a.name === toolCall.toolName);
|
|
152
|
+
if (!action)
|
|
153
|
+
return;
|
|
154
|
+
if ("error" in rest && rest.error) {
|
|
155
|
+
action.error = String(rest.error);
|
|
156
|
+
console.log(` ✗ ${toolCall.toolName} failed (${durationMs}ms): ${String(rest.error).slice(0, 100)}`);
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
159
|
+
const output = "output" in rest ? rest.output : undefined;
|
|
160
|
+
const resultStr = typeof output === "string" ? output : JSON.stringify(output);
|
|
161
|
+
if (toolCall.toolName === "done" && output && typeof output === "object") {
|
|
162
|
+
const done = output;
|
|
163
|
+
finalResult = done.text ?? resultStr;
|
|
164
|
+
finalSuccess = done.success ?? true;
|
|
165
|
+
action.result = resultStr;
|
|
166
|
+
console.log(`\n✓ Task complete: ${finalResult?.slice(0, 100)}`);
|
|
167
|
+
}
|
|
168
|
+
else if (toolCall.toolName === "switch_to_x11" && output && typeof output === "object") {
|
|
169
|
+
const request = output;
|
|
170
|
+
requestedModeSwitch = {
|
|
171
|
+
mode: request.mode ?? "x11",
|
|
172
|
+
reason: request.reason ?? "No reason provided.",
|
|
173
|
+
};
|
|
174
|
+
action.result = resultStr;
|
|
175
|
+
console.log(`\n↪ Requesting mode switch to X11: ${requestedModeSwitch.reason.slice(0, 120)}`);
|
|
176
|
+
}
|
|
177
|
+
else if (toolCall.toolName === "screenshot") {
|
|
178
|
+
action.result = "(screenshot taken)";
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
action.result = resultStr;
|
|
182
|
+
}
|
|
183
|
+
},
|
|
165
184
|
});
|
|
166
185
|
for (const s of result.steps) {
|
|
167
186
|
if (s.text) {
|
|
@@ -184,20 +203,16 @@ export async function runAgent(opts) {
|
|
|
184
203
|
}
|
|
185
204
|
catch (e) {
|
|
186
205
|
console.log(` LLM error: ${e.message.slice(0, 200)}`);
|
|
187
|
-
|
|
206
|
+
actions.push({ name: "llm_error", params: {}, error: e.message });
|
|
188
207
|
evaluation = `LLM call failed: ${e.message.slice(0, 100)}`;
|
|
189
208
|
}
|
|
190
|
-
if (tracker.result !== null) {
|
|
191
|
-
finalResult = tracker.result;
|
|
192
|
-
finalSuccess = tracker.success;
|
|
193
|
-
}
|
|
194
209
|
const record = {
|
|
195
210
|
step: step + 1,
|
|
196
211
|
url: state.url,
|
|
197
212
|
evaluation,
|
|
198
213
|
memory,
|
|
199
214
|
nextGoal,
|
|
200
|
-
actions
|
|
215
|
+
actions,
|
|
201
216
|
timestamp: Date.now(),
|
|
202
217
|
};
|
|
203
218
|
history.add(record);
|
|
@@ -208,11 +223,11 @@ export async function runAgent(opts) {
|
|
|
208
223
|
console.log(` 🧠 ${memory.slice(0, 120)}`);
|
|
209
224
|
if (nextGoal)
|
|
210
225
|
console.log(` 🎯 ${nextGoal.slice(0, 120)}`);
|
|
211
|
-
if (finalResult !== null)
|
|
226
|
+
if (finalResult !== null || requestedModeSwitch !== null)
|
|
212
227
|
break;
|
|
213
228
|
await browser.waitMs(500);
|
|
214
229
|
}
|
|
215
|
-
return { result: finalResult, success: finalSuccess, history };
|
|
230
|
+
return { result: finalResult, success: finalSuccess, history, requestedModeSwitch };
|
|
216
231
|
}
|
|
217
232
|
function maskSensitiveData(text, sensitiveData) {
|
|
218
233
|
let masked = text;
|
package/dist/agent/loop.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loop.js","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAqB,MAAM,IAAI,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"loop.js","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAqB,MAAM,IAAI,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAAE,YAAY,EAAmB,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAqB,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAoBtC,SAAS,iBAAiB,CAAC,OAAe;IACzC,OAAO;;;;;;;yFAOiF,OAAO,6EAA6E,OAAO,+BAA+B,OAAO;;;;;;;;;;;;;kCAaxL,CAAC;AACnC,CAAC;AAED;;;;GAIG;AACH,SAAS,oBAAoB,CAC5B,QAA6B,EAC7B,OAAe;IAEf,MAAM,OAAO,GAAwB,EAAE,CAAC;IAExC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,eAAe,GAAG,CAAC,CAAC,OAAO,CAAC;QAClC,IAAI,CAAC,eAAe,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;YAClE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAClB,SAAS;QACV,CAAC;QAED,MAAM,eAAe,GAAG,KAAK,EAAE,IAAS,EAAE,IAAS,EAAE,EAAE;YACtD,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACrD,MAAM,SAAS,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAE3F,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;gBAC7B,MAAM,SAAS,GAAG,GAAG,OAAO,gBAAgB,IAAI,MAAM,CAAC;gBACvD,IAAI,CAAC;oBAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;oBAAC,aAAa,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAA,CAAC;gBAC9F,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,YAAY,SAAS,CAAC,MAAM,kBAAkB,SAAS,GAAG,CAAC;YAC9F,CAAC;YAED,OAAO,UAAU,CAAC;QACnB,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,MAAM,CAAC,yBAAyB,CAAC,CAAC,CAAC,CAAC;QACxD,WAAW,CAAC,OAAO,GAAG,EAAE,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;IACtE,CAAC;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAkB;IAMhD,MAAM,EACL,IAAI,EACJ,OAAO,EACP,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,mBAAmB,EACtD,QAAQ,GAAG,EAAE,EACb,WAAW,GAAG,CAAC,EACf,iBAAiB,GAAG,EAAE,EACtB,UAAU,GAAG,EAAE,EACf,YAAY,EACZ,MAAM,EACN,aAAa,GAAG,EAAE,GAClB,GAAG,IAAI,CAAC;IAET,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IACvC,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,QAAQ,GAAG,EAAE,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,GAAG,UAAU,EAAE,CAAC;IAC5E,MAAM,KAAK,GAAG,oBAAoB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtD,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IAEnC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,IAAI,mBAAmB,GAAkC,IAAI,CAAC;IAE9D,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC;QAE5C,IAAI,OAAO,CAAC,mBAAmB,IAAI,WAAW,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,iBAAiB,WAAW,oCAAoC,CAAC,CAAC;YAC9E,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC;YACrG,MAAM;QACP,CAAC;QAED,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACpD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3B,SAAS;QACV,CAAC;QAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;QAE3C,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;gBAAE,KAAK,GAAG,mOAAmO,CAAC;;gBAChQ,KAAK,GAAG,8HAA8H,CAAC;QAC7I,CAAC;QAED,MAAM,OAAO,GAA0B,EAAE,CAAC;QAE1C,MAAM,QAAQ,GAAmB,CAAC;gBACjC,IAAI,EAAE,MAAM;gBACZ,OAAO,EAAE;oBACR;wBACC,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE;4BACL,SAAS,iBAAiB,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE;4BACjD,EAAE;4BACF,WAAW;4BACX,EAAE;4BACF,wBAAwB;4BACxB,SAAS;4BACT,KAAK;yBACL,CAAC,IAAI,CAAC,IAAI,CAAC;qBACZ;oBACD;wBACC,IAAI,EAAE,OAAO;wBACb,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC;wBACpD,SAAS,EAAE,YAAY;qBACvB;iBACD;aACD,CAAC,CAAC;QAEH,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC;gBACjC,KAAK,EAAE,YAAY;gBACnB,eAAe,EAAE,KAAK;gBACtB,MAAM,EAAE,YAAY,IAAI,iBAAiB,CAAC,OAAO,CAAC;gBAClD,QAAQ;gBACR,KAAK;gBACL,QAAQ,EAAE,WAAW,CAAC,iBAAiB,CAAC;gBACxC,eAAe,EAAE;oBAChB,SAAS,EAAE,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE;iBAClD;gBAED,4BAA4B,CAAC,EAAE,QAAQ,EAAE;oBACxC,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,CAAC,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;oBACxF,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAwB,EAAE,CAAC,CAAC;gBAClG,CAAC;gBAED,6BAA6B,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,IAAI,EAAE;oBAC9D,MAAM,MAAM,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,CAAgC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,QAAQ,CAAC,CAAC;oBAC/G,IAAI,CAAC,MAAM;wBAAE,OAAO;oBAEpB,IAAI,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBAClC,OAAO,CAAC,GAAG,CAAC,OAAO,QAAQ,CAAC,QAAQ,YAAY,UAAU,QAAQ,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;wBACtG,OAAO;oBACR,CAAC;oBAED,MAAM,MAAM,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;oBAC1D,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;oBAE/E,IAAI,QAAQ,CAAC,QAAQ,KAAK,MAAM,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;wBAC1E,MAAM,IAAI,GAAG,MAA8C,CAAC;wBAC5D,WAAW,GAAG,IAAI,CAAC,IAAI,IAAI,SAAS,CAAC;wBACrC,YAAY,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;wBACpC,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,sBAAsB,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACjE,CAAC;yBAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,eAAe,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;wBAC1F,MAAM,OAAO,GAAG,MAA2C,CAAC;wBAC5D,mBAAmB,GAAG;4BACrB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK;4BAC3B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,qBAAqB;yBAC/C,CAAC;wBACF,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;wBAC1B,OAAO,CAAC,GAAG,CAAC,sCAAsC,mBAAmB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBAC/F,CAAC;yBAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;wBAC/C,MAAM,CAAC,MAAM,GAAG,oBAAoB,CAAC;oBACtC,CAAC;yBAAM,CAAC;wBACP,MAAM,CAAC,MAAM,GAAG,SAAS,CAAC;oBAC3B,CAAC;gBACF,CAAC;aACD,CAAC,CAAC;YAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAC9B,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;oBACZ,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;oBACpB,IAAI,CAAC,UAAU,EAAE,CAAC;wBACjB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACvE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;wBAC7E,IAAI,SAAS;4BAAE,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAChD,IAAI,QAAQ;4BAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,SAAS;4BAAE,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAC9C,IAAI,CAAC,UAAU;4BAAE,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBAClD,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClE,UAAU,GAAG,oBAAoB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAe;YAC1B,IAAI,EAAE,IAAI,GAAG,CAAC;YACd,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,UAAU;YACV,MAAM;YACN,QAAQ;YACR,OAAO;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAEjB,IAAI,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,QAAQ;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,WAAW,KAAK,IAAI,IAAI,mBAAmB,KAAK,IAAI;YAAE,MAAM;QAEhE,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,mBAAmB,EAAE,CAAC;AACrF,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,aAAqC;IAC7E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1D,IAAI,KAAK;YAAE,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,GAAG,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC"}
|
package/dist/agent/run.js
CHANGED
|
@@ -20,7 +20,7 @@ async function main() {
|
|
|
20
20
|
const extraTools = {
|
|
21
21
|
get_totp_code: tool({
|
|
22
22
|
description: "Get a fresh TOTP 2FA code. Call RIGHT BEFORE entering — expires in 30s.",
|
|
23
|
-
|
|
23
|
+
inputSchema: z.object({}),
|
|
24
24
|
execute: async () => {
|
|
25
25
|
const code = authenticator.generate(creds.totpKey);
|
|
26
26
|
const remaining = 30 - (Math.floor(Date.now() / 1000) % 30);
|
package/dist/agent/run.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/agent/run.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,KAAK,UAAU,IAAI;IAClB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IACjC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAE/B,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IAEjC,MAAM,UAAU,GAAG;QAClB,aAAa,EAAE,IAAI,CAAC;YACnB,WAAW,EAAE,yEAAyE;YACtF,
|
|
1
|
+
{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/agent/run.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,KAAK,UAAU,IAAI;IAClB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IACjC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAE/B,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IAEjC,MAAM,UAAU,GAAG;QAClB,aAAa,EAAE,IAAI,CAAC;YACnB,WAAW,EAAE,yEAAyE;YACtF,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACzB,OAAO,EAAE,KAAK,IAAI,EAAE;gBACnB,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC5D,OAAO,cAAc,IAAI,eAAe,SAAS,IAAI,CAAC;YACvD,CAAC;SACD,CAAC;KACF,CAAC;IAEF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI;;;;qCAIH,KAAK,CAAC,QAAQ,WAAW,KAAK,CAAC,KAAK,cAAc,KAAK,CAAC,QAAQ;;;;;;;;;CASpG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC;QACnD,IAAI;QACJ,OAAO;QACP,UAAU;QACV,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC;QACjD,aAAa,EAAE;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,OAAO;SACvB;KACD,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3D,IAAI,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;AAC5B,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IAClB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
|
package/dist/agent/tools.d.ts
CHANGED
|
@@ -1,43 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Complete browser automation tools.
|
|
3
|
+
* Each tool maps to a CDP operation. Index-based element targeting.
|
|
4
|
+
*/
|
|
1
5
|
import { CDPBrowser } from "../browser/cdp.js";
|
|
2
6
|
export declare function createBrowserTools(browser: CDPBrowser, dataDir?: string): {
|
|
3
|
-
navigate: import("ai").Tool<
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
7
|
+
navigate: import("ai").Tool<{
|
|
8
|
+
url: string;
|
|
9
|
+
}, string>;
|
|
10
|
+
go_back: import("ai").Tool<{}, string>;
|
|
11
|
+
reload: import("ai").Tool<{}, string>;
|
|
12
|
+
search: import("ai").Tool<{
|
|
13
|
+
query: string;
|
|
14
|
+
}, string>;
|
|
15
|
+
click: import("ai").Tool<{
|
|
16
|
+
index: number;
|
|
17
|
+
}, string>;
|
|
18
|
+
click_at: import("ai").Tool<{
|
|
19
|
+
x: number;
|
|
20
|
+
y: number;
|
|
21
|
+
}, string>;
|
|
22
|
+
input: import("ai").Tool<{
|
|
23
|
+
text: string;
|
|
24
|
+
index: number;
|
|
25
|
+
clear: boolean;
|
|
26
|
+
}, string>;
|
|
27
|
+
send_keys: import("ai").Tool<{
|
|
28
|
+
keys: string;
|
|
29
|
+
}, string>;
|
|
30
|
+
scroll: import("ai").Tool<{
|
|
31
|
+
direction: "up" | "down";
|
|
32
|
+
pixels: number;
|
|
33
|
+
}, string>;
|
|
34
|
+
find_text: import("ai").Tool<{
|
|
35
|
+
text: string;
|
|
36
|
+
}, string>;
|
|
37
|
+
scroll_to_element: import("ai").Tool<{
|
|
38
|
+
index: number;
|
|
39
|
+
}, string>;
|
|
40
|
+
screenshot: import("ai").Tool<{}, {
|
|
15
41
|
type: string;
|
|
16
42
|
image_base64: string;
|
|
17
43
|
}>;
|
|
18
|
-
extract: import("ai").Tool<
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
44
|
+
extract: import("ai").Tool<{
|
|
45
|
+
query: string;
|
|
46
|
+
}, string>;
|
|
47
|
+
search_page: import("ai").Tool<{
|
|
48
|
+
query: string;
|
|
49
|
+
}, string>;
|
|
50
|
+
switch_tab: import("ai").Tool<{
|
|
51
|
+
tab_id: string;
|
|
52
|
+
}, string>;
|
|
53
|
+
close_tab: import("ai").Tool<{
|
|
54
|
+
tab_id: string;
|
|
55
|
+
}, string>;
|
|
56
|
+
new_tab: import("ai").Tool<{
|
|
57
|
+
url: string;
|
|
58
|
+
}, string>;
|
|
59
|
+
dropdown_options: import("ai").Tool<{
|
|
60
|
+
selector: string;
|
|
61
|
+
}, string>;
|
|
62
|
+
select_dropdown: import("ai").Tool<{
|
|
63
|
+
value: string;
|
|
64
|
+
selector: string;
|
|
65
|
+
}, string>;
|
|
66
|
+
upload_file: import("ai").Tool<{
|
|
67
|
+
selector: string;
|
|
68
|
+
file_path: string;
|
|
69
|
+
}, string>;
|
|
70
|
+
click_and_upload: import("ai").Tool<{
|
|
71
|
+
index: number;
|
|
72
|
+
file_path: string;
|
|
73
|
+
}, string>;
|
|
74
|
+
paste_content: import("ai").Tool<{
|
|
75
|
+
file_path: string;
|
|
76
|
+
editor_selector?: string | undefined;
|
|
77
|
+
}, string>;
|
|
78
|
+
paste_image: import("ai").Tool<{
|
|
79
|
+
file_path: string;
|
|
80
|
+
editor_selector?: string | undefined;
|
|
81
|
+
}, string>;
|
|
82
|
+
write_file: import("ai").Tool<{
|
|
83
|
+
path: string;
|
|
84
|
+
content: string;
|
|
85
|
+
}, string>;
|
|
86
|
+
read_file: import("ai").Tool<{
|
|
87
|
+
path: string;
|
|
88
|
+
}, string>;
|
|
89
|
+
type_text: import("ai").Tool<{
|
|
90
|
+
text: string;
|
|
91
|
+
}, string>;
|
|
92
|
+
save_page_snapshot: import("ai").Tool<{}, string>;
|
|
93
|
+
save_element_html: import("ai").Tool<{
|
|
94
|
+
index?: number | undefined;
|
|
95
|
+
selector?: string | undefined;
|
|
96
|
+
filename?: string | undefined;
|
|
97
|
+
}, string>;
|
|
98
|
+
shell: import("ai").Tool<{
|
|
99
|
+
command: string;
|
|
100
|
+
}, any>;
|
|
101
|
+
evaluate: import("ai").Tool<{
|
|
102
|
+
code: string;
|
|
103
|
+
}, string>;
|
|
104
|
+
handle_dialog: import("ai").Tool<{
|
|
105
|
+
accept: boolean;
|
|
106
|
+
text?: string | undefined;
|
|
107
|
+
}, string>;
|
|
108
|
+
wait: import("ai").Tool<{
|
|
109
|
+
seconds: number;
|
|
110
|
+
}, string>;
|
|
111
|
+
solve_captcha: import("ai").Tool<{
|
|
112
|
+
site_url: string;
|
|
113
|
+
site_key: string;
|
|
114
|
+
captcha_type: "recaptcha_v2" | "recaptcha_v3" | "hcaptcha" | "turnstile";
|
|
115
|
+
}, string>;
|
|
116
|
+
inject_captcha_token: import("ai").Tool<{
|
|
117
|
+
token: string;
|
|
118
|
+
}, string>;
|
|
119
|
+
solve_datadome: import("ai").Tool<{
|
|
120
|
+
site_url: string;
|
|
121
|
+
captcha_url: string;
|
|
122
|
+
user_agent?: string | undefined;
|
|
123
|
+
}, string>;
|
|
124
|
+
done: import("ai").Tool<{
|
|
125
|
+
success: boolean;
|
|
126
|
+
text: string;
|
|
127
|
+
}, {
|
|
128
|
+
done: boolean;
|
|
129
|
+
text: string;
|
|
130
|
+
success: boolean;
|
|
131
|
+
}>;
|
|
42
132
|
};
|
|
43
133
|
export type BrowserTools = ReturnType<typeof createBrowserTools>;
|