browser-autopilot 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/README.md +251 -0
  2. package/dist/agent/history.d.ts +41 -0
  3. package/dist/agent/history.js +98 -0
  4. package/dist/agent/history.js.map +1 -0
  5. package/dist/agent/loop.d.ts +34 -0
  6. package/dist/agent/loop.js +278 -0
  7. package/dist/agent/loop.js.map +1 -0
  8. package/dist/agent/run.d.ts +4 -0
  9. package/dist/agent/run.js +67 -0
  10. package/dist/agent/run.js.map +1 -0
  11. package/dist/agent/state.d.ts +37 -0
  12. package/dist/agent/state.js +82 -0
  13. package/dist/agent/state.js.map +1 -0
  14. package/dist/agent/tools.d.ts +414 -0
  15. package/dist/agent/tools.js +399 -0
  16. package/dist/agent/tools.js.map +1 -0
  17. package/dist/browser/cdp.d.ts +91 -0
  18. package/dist/browser/cdp.js +470 -0
  19. package/dist/browser/cdp.js.map +1 -0
  20. package/dist/browser/dom.d.ts +30 -0
  21. package/dist/browser/dom.js +79 -0
  22. package/dist/browser/dom.js.map +1 -0
  23. package/dist/browser/snapshot.d.ts +19 -0
  24. package/dist/browser/snapshot.js +70 -0
  25. package/dist/browser/snapshot.js.map +1 -0
  26. package/dist/captcha/solver.d.ts +20 -0
  27. package/dist/captcha/solver.js +101 -0
  28. package/dist/captcha/solver.js.map +1 -0
  29. package/dist/config.d.ts +36 -0
  30. package/dist/config.js +44 -0
  31. package/dist/config.js.map +1 -0
  32. package/dist/index.d.ts +20 -0
  33. package/dist/index.js +43 -0
  34. package/dist/index.js.map +1 -0
  35. package/dist/orchestrator.d.ts +33 -0
  36. package/dist/orchestrator.js +197 -0
  37. package/dist/orchestrator.js.map +1 -0
  38. package/dist/viewer/server.d.ts +14 -0
  39. package/dist/viewer/server.js +93 -0
  40. package/dist/viewer/server.js.map +1 -0
  41. package/dist/x11/agent.d.ts +34 -0
  42. package/dist/x11/agent.js +103 -0
  43. package/dist/x11/agent.js.map +1 -0
  44. package/dist/x11/chrome.d.ts +9 -0
  45. package/dist/x11/chrome.js +107 -0
  46. package/dist/x11/chrome.js.map +1 -0
  47. package/dist/x11/input.d.ts +13 -0
  48. package/dist/x11/input.js +75 -0
  49. package/dist/x11/input.js.map +1 -0
  50. package/dist/x11/login.d.ts +6 -0
  51. package/dist/x11/login.js +76 -0
  52. package/dist/x11/login.js.map +1 -0
  53. package/package.json +79 -0
@@ -0,0 +1,278 @@
1
+ /**
2
+ * Step-based LLM agent loop with multi-turn tool use.
3
+ *
4
+ * Each step:
5
+ * 1. Capture browser state (screenshot + indexed DOM + URL + tabs)
6
+ * 2. Send to LLM with history + state + screenshot
7
+ * 3. LLM returns tool calls
8
+ * 4. Execute tools, feed results back to LLM
9
+ * 5. LLM can chain more tool calls (seeing previous results)
10
+ * 6. Repeat tool loop until LLM emits end_turn or hits maxActionsPerStep
11
+ * 7. Record step in history, advance to next step
12
+ *
13
+ * The inner tool loop means the LLM can call get_buyer_profile,
14
+ * see the result, then call input(text="Marcus") — all within one step.
15
+ */
16
+ import Anthropic from "@anthropic-ai/sdk";
17
+ import { writeFileSync, mkdirSync } from "fs";
18
+ import { captureBrowserState, formatStateForLLM } from "./state.js";
19
+ import { AgentHistory } from "./history.js";
20
+ import { createBrowserTools } from "./tools.js";
21
+ import { config } from "../config.js";
22
+ function buildSystemPrompt(dataDir) {
23
+ return `You are a browser automation agent. You observe the page via screenshots and an indexed DOM tree, then take actions using tools.
24
+
25
+ RULES:
26
+ - Elements are referenced by index numbers like [1], [2], etc.
27
+ - To CLICK a button/link: use 'click' with the index. It auto-scrolls, checks occlusion, and falls back to JS click.
28
+ - To TYPE into a form input/textbox: use 'input' with the index AND text. Do NOT click first — 'input' handles focus.
29
+ - To TYPE into rich text editors (contenteditable, ProseMirror, Tiptap, etc.): first click on the editor area with 'click' or 'click_at', then use 'type_text' to type at the cursor. This is the most reliable way to interact with rich text editors.
30
+ - To understand page HTML structure: use 'save_page_snapshot' to save the full HTML to ${dataDir}/page.html, then use 'shell' to search it (e.g. grep -n 'contenteditable' ${dataDir}/page.html, sed -n '50,70p' ${dataDir}/page.html). This lets you explore the full HTML without loading it all into context.
31
+ - Use 'shell' to run bash commands (grep, sed, head, tail, wc, cat, etc.) on any saved files. This is your primary tool for exploring page structure, finding selectors, and debugging.
32
+ - If a click doesn't work after 2 tries, use 'evaluate' to run JS directly.
33
+ - Cookie/overlay dialogs: use 'evaluate' to remove them: document.querySelector('[class*=cookie]')?.remove()
34
+ - Prefer 'navigate' with direct URLs over clicking through menus when the URL is known.
35
+ - You can call multiple tools in sequence within a single step — each result is fed back to you.
36
+ - When done, use the 'done' tool.
37
+
38
+ For each step, think about:
39
+ 1. What happened in the previous step? (evaluation)
40
+ 2. What do you remember? (important facts for later)
41
+ 3. What should you do next? (next goal)
42
+ 4. What specific actions to take?`;
43
+ }
44
+ export async function runAgent(opts) {
45
+ const { task, browser, model = process.env.AGENT_MODEL ?? "claude-sonnet-4-6", maxSteps = 80, maxFailures = 5, maxActionsPerStep = 10, extraTools = {}, systemPrompt, onStep, sensitiveData = {}, } = opts;
46
+ const dataDir = config.browser.dataDir;
47
+ const client = new Anthropic({ timeout: 300_000 });
48
+ const tools = createBrowserTools(browser, dataDir);
49
+ const allTools = { ...tools, ...extraTools };
50
+ const history = new AgentHistory();
51
+ const toolDefs = buildToolDefinitions(allTools);
52
+ let result = null;
53
+ let success = false;
54
+ for (let step = 0; step < maxSteps; step++) {
55
+ console.log(`\n${"─".repeat(50)}`);
56
+ console.log(`Step ${step + 1}/${maxSteps}`);
57
+ if (history.consecutiveFailures >= maxFailures) {
58
+ console.log(`Max failures (${maxFailures}) reached. Forcing final response.`);
59
+ result = history.lastStep?.actions.map((a) => a.result ?? a.error).join("\n") ?? "Task failed.";
60
+ break;
61
+ }
62
+ let state;
63
+ try {
64
+ state = await captureBrowserState(browser);
65
+ }
66
+ catch (e) {
67
+ console.log(` State capture failed: ${e.message}`);
68
+ await browser.waitMs(2000);
69
+ continue;
70
+ }
71
+ const stateText = formatStateForLLM(state);
72
+ const historyText = history.formatForLLM();
73
+ const loop = history.detectLoop(3);
74
+ let nudge = "";
75
+ if (loop.isLoop) {
76
+ if (loop.loopCount >= 2)
77
+ nudge = "\n⚠️ STOP. You are stuck in a loop repeating the same actions. You MUST try a COMPLETELY DIFFERENT approach right now. If you were clicking, try 'input' or 'evaluate' instead. If nothing works, call 'done' with what you have.";
78
+ else
79
+ nudge = "\n⚠️ You are repeating similar actions. Try a different tool or strategy. For text input, use the 'input' tool, not 'click'.";
80
+ }
81
+ const userContent = [
82
+ { type: "text", text: [
83
+ `Task: ${maskSensitiveData(task, sensitiveData)}`,
84
+ "",
85
+ historyText,
86
+ "",
87
+ "Current browser state:",
88
+ stateText,
89
+ nudge,
90
+ ].join("\n") },
91
+ { type: "image", source: { type: "base64", media_type: "image/jpeg", data: state.screenshotBase64 } },
92
+ ];
93
+ // Multi-turn tool loop within this step
94
+ const actions = [];
95
+ let evaluation = "";
96
+ let memory = "";
97
+ let nextGoal = "";
98
+ const messages = [
99
+ { role: "user", content: userContent },
100
+ ];
101
+ try {
102
+ let actionsThisStep = 0;
103
+ while (actionsThisStep < maxActionsPerStep) {
104
+ const response = await client.messages.create({
105
+ model,
106
+ max_tokens: 16384,
107
+ system: [{ type: "text", text: systemPrompt ?? buildSystemPrompt(dataDir), cache_control: { type: "ephemeral" } }],
108
+ tools: toolDefs.map((t, i) => i === toolDefs.length - 1 ? { ...t, cache_control: { type: "ephemeral" } } : t),
109
+ messages,
110
+ });
111
+ // Extract text reasoning from this response
112
+ for (const block of response.content) {
113
+ if (block.type === "text" && !evaluation) {
114
+ const text = block.text;
115
+ const evalMatch = text.match(/(?:evaluation|eval)[:\s]*(.+?)(?:\n|$)/i);
116
+ const memMatch = text.match(/(?:memory|remember)[:\s]*(.+?)(?:\n|$)/i);
117
+ const goalMatch = text.match(/(?:next.?goal|plan|next)[:\s]*(.+?)(?:\n|$)/i);
118
+ if (evalMatch)
119
+ evaluation = evalMatch[1].trim();
120
+ if (memMatch)
121
+ memory = memMatch[1].trim();
122
+ if (goalMatch)
123
+ nextGoal = goalMatch[1].trim();
124
+ if (!evaluation)
125
+ evaluation = text.slice(0, 200);
126
+ }
127
+ }
128
+ // Collect tool calls from this response
129
+ const toolCalls = response.content.filter((b) => b.type === "tool_use");
130
+ if (toolCalls.length === 0)
131
+ break;
132
+ // Add assistant response to messages
133
+ messages.push({ role: "assistant", content: response.content });
134
+ // Execute each tool call and build tool results
135
+ const toolResults = [];
136
+ for (const tc of toolCalls) {
137
+ actionsThisStep++;
138
+ const action = { name: tc.name, params: tc.input };
139
+ actions.push(action);
140
+ console.log(` → ${tc.name}(${JSON.stringify(tc.input).slice(0, 80)})`);
141
+ try {
142
+ const toolFn = allTools[tc.name];
143
+ if (!toolFn) {
144
+ action.error = `Unknown tool: ${tc.name}`;
145
+ toolResults.push({ type: "tool_result", tool_use_id: tc.id, content: `Error: Unknown tool "${tc.name}"` });
146
+ continue;
147
+ }
148
+ const toolResult = await toolFn.execute(tc.input);
149
+ action.result = typeof toolResult === "string" ? toolResult : JSON.stringify(toolResult);
150
+ // Cap tool result size to prevent context explosion.
151
+ // Screenshots return base64 — send as image block, not text.
152
+ if (tc.name === "screenshot" && toolResult?.image_base64) {
153
+ toolResults.push({
154
+ type: "tool_result",
155
+ tool_use_id: tc.id,
156
+ content: [{ type: "image", source: { type: "base64", media_type: "image/png", data: toolResult.image_base64 } }],
157
+ });
158
+ action.result = "(screenshot taken)";
159
+ }
160
+ else if (action.result.length > 4000) {
161
+ // Save full result to file, send summary + pointer to LLM
162
+ const spillPath = `${config.browser.dataDir}/tool_result_${tc.name}.txt`;
163
+ try {
164
+ mkdirSync(config.browser.dataDir, { recursive: true });
165
+ writeFileSync(spillPath, action.result);
166
+ }
167
+ catch { }
168
+ const preview = action.result.slice(0, 1500);
169
+ toolResults.push({
170
+ type: "tool_result",
171
+ tool_use_id: tc.id,
172
+ content: `${preview}\n\n... (${action.result.length} chars total — full output saved to ${spillPath}, use shell to search)`,
173
+ });
174
+ }
175
+ else {
176
+ toolResults.push({ type: "tool_result", tool_use_id: tc.id, content: action.result });
177
+ }
178
+ if (tc.name === "done") {
179
+ result = toolResult.text ?? action.result;
180
+ success = toolResult.success ?? true;
181
+ console.log(`\n✓ Task complete: ${result?.slice(0, 100)}`);
182
+ }
183
+ }
184
+ catch (e) {
185
+ action.error = e.message;
186
+ console.log(` ✗ ${tc.name} failed: ${e.message.slice(0, 100)}`);
187
+ toolResults.push({ type: "tool_result", tool_use_id: tc.id, content: `Error: ${e.message}`, is_error: true });
188
+ }
189
+ }
190
+ // If done was called, break out
191
+ if (result !== null)
192
+ break;
193
+ // Feed tool results back to LLM for potential follow-up calls
194
+ messages.push({ role: "user", content: toolResults });
195
+ // If the LLM said stop_reason=end_turn, it's done with tool calls
196
+ if (response.stop_reason === "end_turn")
197
+ break;
198
+ }
199
+ }
200
+ catch (e) {
201
+ console.log(` LLM error: ${e.message.slice(0, 200)}`);
202
+ actions.push({ name: "llm_error", params: {}, error: e.message });
203
+ evaluation = `LLM call failed: ${e.message.slice(0, 100)}`;
204
+ }
205
+ const record = {
206
+ step: step + 1,
207
+ url: state.url,
208
+ evaluation,
209
+ memory,
210
+ nextGoal,
211
+ actions,
212
+ timestamp: Date.now(),
213
+ };
214
+ history.add(record);
215
+ onStep?.(record);
216
+ if (evaluation)
217
+ console.log(` 📝 ${evaluation.slice(0, 120)}`);
218
+ if (memory)
219
+ console.log(` 🧠 ${memory.slice(0, 120)}`);
220
+ if (nextGoal)
221
+ console.log(` 🎯 ${nextGoal.slice(0, 120)}`);
222
+ if (result !== null)
223
+ break;
224
+ await browser.waitMs(500);
225
+ }
226
+ return { result, success, history };
227
+ }
228
+ function buildToolDefinitions(tools) {
229
+ const defs = [];
230
+ for (const [name, t] of Object.entries(tools)) {
231
+ if (!t || !t.parameters)
232
+ continue;
233
+ const schema = t.parameters;
234
+ defs.push({
235
+ name,
236
+ description: t.description ?? name,
237
+ input_schema: zodToJsonSchema(schema),
238
+ });
239
+ }
240
+ return defs;
241
+ }
242
+ function zodToJsonSchema(schema) {
243
+ if (schema._def) {
244
+ const def = schema._def;
245
+ if (def.typeName === "ZodObject") {
246
+ const props = {};
247
+ const required = [];
248
+ for (const [key, val] of Object.entries(def.shape())) {
249
+ props[key] = zodToJsonSchema(val);
250
+ if (!val.isOptional())
251
+ required.push(key);
252
+ }
253
+ return { type: "object", properties: props, required };
254
+ }
255
+ if (def.typeName === "ZodString")
256
+ return { type: "string", description: def.description };
257
+ if (def.typeName === "ZodNumber")
258
+ return { type: "number", description: def.description };
259
+ if (def.typeName === "ZodBoolean")
260
+ return { type: "boolean", description: def.description };
261
+ if (def.typeName === "ZodEnum")
262
+ return { type: "string", enum: def.values, description: def.description };
263
+ if (def.typeName === "ZodDefault")
264
+ return zodToJsonSchema(def.innerType);
265
+ if (def.typeName === "ZodOptional")
266
+ return zodToJsonSchema(def.innerType);
267
+ }
268
+ return { type: "string" };
269
+ }
270
+ function maskSensitiveData(text, sensitiveData) {
271
+ let masked = text;
272
+ for (const [key, value] of Object.entries(sensitiveData)) {
273
+ if (value)
274
+ masked = masked.replaceAll(value, `<secret:${key}>`);
275
+ }
276
+ return masked;
277
+ }
278
+ //# sourceMappingURL=loop.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loop.js","sourceRoot":"","sources":["../../src/agent/loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE9C,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAAE,YAAY,EAAmB,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAqB,MAAM,YAAY,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAetC,SAAS,iBAAiB,CAAC,OAAe;IACzC,OAAO;;;;;;;yFAOiF,OAAO,6EAA6E,OAAO,+BAA+B,OAAO;;;;;;;;;;;;kCAYxL,CAAC;AACnC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAkB;IAKhD,MAAM,EACL,IAAI,EACJ,OAAO,EACP,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,mBAAmB,EACtD,QAAQ,GAAG,EAAE,EACb,WAAW,GAAG,CAAC,EACf,iBAAiB,GAAG,EAAE,EACtB,UAAU,GAAG,EAAE,EACf,YAAY,EACZ,MAAM,EACN,aAAa,GAAG,EAAE,GAClB,GAAG,IAAI,CAAC;IAET,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,EAAE,GAAG,KAAK,EAAE,GAAG,UAAU,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAEhD,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,QAAQ,IAAI,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC,CAAC;QAE5C,IAAI,OAAO,CAAC,mBAAmB,IAAI,WAAW,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,iBAAiB,WAAW,oCAAoC,CAAC,CAAC;YAC9E,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC;YAChG,MAAM;QACP,CAAC;QAED,IAAI,KAAK,CAAC;QACV,IAAI,CAAC;YACJ,KAAK,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACpD,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3B,SAAS;QACV,CAAC;QAED,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;QAE3C,MAAM,IAAI,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACnC,IAAI,KAAK,GAAG,EAAE,CAAC;QACf,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC;gBAAE,KAAK,GAAG,mOAAmO,CAAC;;gBAChQ,KAAK,GAAG,8HAA8H,CAAC;QAC7I,CAAC;QAED,MAAM,WAAW,GAAkC;YAClD,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;oBACrB,SAAS,iBAAiB,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE;oBACjD,EAAE;oBACF,WAAW;oBACX,EAAE;oBACF,wBAAwB;oBACxB,SAAS;oBACT,KAAK;iBACL,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACd,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,CAAC,gBAAgB,EAAE,EAAE;SACrG,CAAC;QAEF,wCAAwC;QACxC,MAAM,OAAO,GAA0F,EAAE,CAAC;QAC1G,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,EAAE,CAAC;QAElB,MAAM,QAAQ,GAA6B;YAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACtC,CAAC;QAEF,IAAI,CAAC;YACJ,IAAI,eAAe,GAAG,CAAC,CAAC;YAExB,OAAO,eAAe,GAAG,iBAAiB,EAAE,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;oBAC7C,KAAK;oBACL,UAAU,EAAE,KAAK;oBACjB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,IAAI,iBAAiB,CAAC,OAAO,CAAC,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC;oBAClH,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC7G,QAAQ;iBACR,CAAC,CAAC;gBAEH,4CAA4C;gBAC5C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;oBACtC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;wBAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;wBACxB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACxE,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;wBACvE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;wBAC7E,IAAI,SAAS;4BAAE,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAChD,IAAI,QAAQ;4BAAE,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAC1C,IAAI,SAAS;4BAAE,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;wBAC9C,IAAI,CAAC,UAAU;4BAAE,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBAClD,CAAC;gBACF,CAAC;gBAED,wCAAwC;gBACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAA+B,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;gBAErG,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM;gBAElC,qCAAqC;gBACrC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;gBAEhE,gDAAgD;gBAChD,MAAM,WAAW,GAAqC,EAAE,CAAC;gBAEzD,KAAK,MAAM,EAAE,IAAI,SAAS,EAAE,CAAC;oBAC5B,eAAe,EAAE,CAAC;oBAClB,MAAM,MAAM,GAA2B,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,KAA4B,EAAE,CAAC;oBAClG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAErB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;oBAExE,IAAI,CAAC;wBACJ,MAAM,MAAM,GAAG,QAAQ,CAAC,EAAE,CAAC,IAA6B,CAAC,CAAC;wBAC1D,IAAI,CAAC,MAAM,EAAE,CAAC;4BACb,MAAM,CAAC,KAAK,GAAG,iBAAiB,EAAE,CAAC,IAAI,EAAE,CAAC;4BAC1C,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;4BAC3G,SAAS;wBACV,CAAC;wBACD,MAAM,UAAU,GAAG,MAAO,MAAc,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC3D,MAAM,CAAC,MAAM,GAAG,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;wBAEzF,qDAAqD;wBACrD,6DAA6D;wBAC7D,IAAI,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,UAAU,EAAE,YAAY,EAAE,CAAC;4BAC1D,WAAW,CAAC,IAAI,CAAC;gCAChB,IAAI,EAAE,aAAa;gCACnB,WAAW,EAAE,EAAE,CAAC,EAAE;gCAClB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,IAAI,EAAE,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC;6BAChH,CAAC,CAAC;4BACH,MAAM,CAAC,MAAM,GAAG,oBAAoB,CAAC;wBACtC,CAAC;6BAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;4BACxC,0DAA0D;4BAC1D,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,gBAAgB,EAAE,CAAC,IAAI,MAAM,CAAC;4BACzE,IAAI,CAAC;gCACJ,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gCACvD,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;4BACzC,CAAC;4BAAC,MAAM,CAAC,CAAA,CAAC;4BACV,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;4BAC7C,WAAW,CAAC,IAAI,CAAC;gCAChB,IAAI,EAAE,aAAa;gCACnB,WAAW,EAAE,EAAE,CAAC,EAAE;gCAClB,OAAO,EAAE,GAAG,OAAO,YAAY,MAAM,CAAC,MAAM,CAAC,MAAM,uCAAuC,SAAS,wBAAwB;6BAC3H,CAAC,CAAC;wBACJ,CAAC;6BAAM,CAAC;4BACP,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;wBACvF,CAAC;wBAED,IAAI,EAAE,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;4BACxB,MAAM,GAAI,UAAkB,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,CAAC;4BACnD,OAAO,GAAI,UAAkB,CAAC,OAAO,IAAI,IAAI,CAAC;4BAC9C,OAAO,CAAC,GAAG,CAAC,sBAAsB,MAAM,EAAE,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC5D,CAAC;oBACF,CAAC;oBAAC,OAAO,CAAM,EAAE,CAAC;wBACjB,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC;wBACzB,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;wBACjE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC/G,CAAC;gBACF,CAAC;gBAED,gCAAgC;gBAChC,IAAI,MAAM,KAAK,IAAI;oBAAE,MAAM;gBAE3B,8DAA8D;gBAC9D,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC;gBAEtD,kEAAkE;gBAClE,IAAI,QAAQ,CAAC,WAAW,KAAK,UAAU;oBAAE,MAAM;YAChD,CAAC;QACF,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YACvD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YAClE,UAAU,GAAG,oBAAoB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAC5D,CAAC;QAED,MAAM,MAAM,GAAe;YAC1B,IAAI,EAAE,IAAI,GAAG,CAAC;YACd,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,UAAU;YACV,MAAM;YACN,QAAQ;YACR,OAAO;YACP,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACrB,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACpB,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAEjB,IAAI,UAAU;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAChE,IAAI,MAAM;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QACxD,IAAI,QAAQ;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAE5D,IAAI,MAAM,KAAK,IAAI;YAAE,MAAM;QAE3B,MAAM,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,oBAAoB,CAAC,KAA0B;IACvD,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/C,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU;YAAE,SAAS;QAClC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC;YACT,IAAI;YACJ,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,IAAI;YAClC,YAAY,EAAE,eAAe,CAAC,MAAM,CAAC;SACrC,CAAC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACb,CAAC;AAED,SAAS,eAAe,CAAC,MAAW;IACnC,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QACjB,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;QACxB,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,KAAK,GAAwB,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC;gBACtD,KAAK,CAAC,GAAG,CAAC,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;gBAClC,IAAI,CAAE,GAAW,CAAC,UAAU,EAAE;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpD,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QACxD,CAAC;QACD,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;QAC1F,IAAI,GAAG,CAAC,QAAQ,KAAK,WAAW;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;QAC1F,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;QAC5F,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC;QAC1G,IAAI,GAAG,CAAC,QAAQ,KAAK,YAAY;YAAE,OAAO,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzE,IAAI,GAAG,CAAC,QAAQ,KAAK,aAAa;YAAE,OAAO,eAAe,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC3B,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAY,EAAE,aAAqC;IAC7E,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;QAC1D,IAAI,KAAK;YAAE,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,GAAG,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,MAAM,CAAC;AACf,CAAC"}
@@ -0,0 +1,4 @@
1
+ /**
2
+ * CLI entrypoint for the browser-use agent (Phase 2).
3
+ */
4
+ export {};
@@ -0,0 +1,67 @@
1
+ /**
2
+ * CLI entrypoint for the browser-use agent (Phase 2).
3
+ */
4
+ import { authenticator } from "otplib";
5
+ import { z } from "zod";
6
+ import { tool } from "ai";
7
+ import { CDPBrowser } from "../browser/cdp.js";
8
+ import { runAgent } from "./loop.js";
9
+ import { config } from "../config.js";
10
+ async function main() {
11
+ console.log("=".repeat(50));
12
+ console.log("Encumbrance Agent");
13
+ console.log(` cdp: ${config.cdpUrl}`);
14
+ console.log("=".repeat(50));
15
+ const browser = new CDPBrowser();
16
+ await browser.connect();
17
+ console.log("CDP connected\n");
18
+ const creds = config.credentials;
19
+ const extraTools = {
20
+ get_totp_code: tool({
21
+ description: "Get a fresh TOTP 2FA code. Call RIGHT BEFORE entering — expires in 30s.",
22
+ parameters: z.object({}),
23
+ execute: async () => {
24
+ const code = authenticator.generate(creds.totpKey);
25
+ const remaining = 30 - (Math.floor(Date.now() / 1000) % 30);
26
+ return `TOTP code: ${code} (valid for ${remaining}s)`;
27
+ },
28
+ }),
29
+ };
30
+ const task = process.env.AGENT_TASK ?? `
31
+ You are connected to a Chrome browser that should be logged into Twitter/X.
32
+
33
+ 1. Navigate to https://x.com/home — if home timeline loads, you're logged in.
34
+ 2. If not logged in, use: username=${creds.username}, email=${creds.email}, password=${creds.password}, 2FA=get_totp_code tool.
35
+
36
+ After confirming login, navigate to https://developer.x.com and:
37
+ - Find or create app "SmithMediaBot"
38
+ - Use case (250+ chars): "Building an automated social media analytics and content scheduling platform that helps small businesses track engagement metrics, optimize posting times, and manage multiple brand accounts through the Twitter API for improved social media ROI and audience growth."
39
+ - Go to "Keys and tokens" → generate API Key + Secret, Access Token + Secret, Bearer Token
40
+ - CRITICAL: Copy ALL values BEFORE dismissing dialogs
41
+ - Call write_file with path="twitter_api_keys.json" and the keys as JSON content
42
+ - Call done with all keys in the result
43
+ `;
44
+ const { result, success, history } = await runAgent({
45
+ task,
46
+ browser,
47
+ extraTools,
48
+ maxSteps: parseInt(process.env.MAX_STEPS ?? "80"),
49
+ sensitiveData: {
50
+ password: creds.password,
51
+ totp_key: creds.totpKey,
52
+ },
53
+ });
54
+ console.log("\n" + "=".repeat(50));
55
+ console.log(`Success: ${success}`);
56
+ console.log(`Steps: ${history.totalSteps}`);
57
+ console.log(`URLs visited: ${history.allUrls.join(", ")}`);
58
+ if (result)
59
+ console.log(`\nResult:\n${result}`);
60
+ console.log("=".repeat(50));
61
+ await browser.disconnect();
62
+ }
63
+ main().catch((e) => {
64
+ console.error(e);
65
+ process.exit(1);
66
+ });
67
+ //# sourceMappingURL=run.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/agent/run.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,QAAQ,CAAC;AACvC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC;AAC1B,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,KAAK,UAAU,IAAI;IAClB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,OAAO,GAAG,IAAI,UAAU,EAAE,CAAC;IACjC,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAE/B,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC;IAEjC,MAAM,UAAU,GAAG;QAClB,aAAa,EAAE,IAAI,CAAC;YACnB,WAAW,EAAE,yEAAyE;YACtF,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,KAAK,IAAI,EAAE;gBACnB,MAAM,IAAI,GAAG,aAAa,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACnD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC5D,OAAO,cAAc,IAAI,eAAe,SAAS,IAAI,CAAC;YACvD,CAAC;SACD,CAAC;KACF,CAAC;IAEF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI;;;;qCAIH,KAAK,CAAC,QAAQ,WAAW,KAAK,CAAC,KAAK,cAAc,KAAK,CAAC,QAAQ;;;;;;;;;CASpG,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC;QACnD,IAAI;QACJ,OAAO;QACP,UAAU;QACV,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC;QACjD,aAAa,EAAE;YACd,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,OAAO;SACvB;KACD,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,GAAG,CAAC,UAAU,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,iBAAiB,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC3D,IAAI,MAAM;QAAE,OAAO,CAAC,GAAG,CAAC,cAAc,MAAM,EAAE,CAAC,CAAC;IAChD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;AAC5B,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;IAClB,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC"}
@@ -0,0 +1,37 @@
1
+ /**
2
+ * Browser state capture — what the LLM sees each step.
3
+ *
4
+ * Full DOM tree + page text are saved to dataDir files each step.
5
+ * The LLM gets a compact summary + knows it can use 'shell' to
6
+ * grep the full files for details.
7
+ */
8
+ import { CDPBrowser } from "../browser/cdp.js";
9
+ import type { IndexedElement } from "../browser/dom.js";
10
+ export interface BrowserState {
11
+ url: string;
12
+ title: string;
13
+ domTree: string;
14
+ elements: Map<number, IndexedElement>;
15
+ tabs: Array<{
16
+ id: string;
17
+ url: string;
18
+ title: string;
19
+ }>;
20
+ screenshotBase64: string;
21
+ scrollPosition: {
22
+ x: number;
23
+ y: number;
24
+ };
25
+ pendingDialogs: Array<{
26
+ type: string;
27
+ message: string;
28
+ }>;
29
+ }
30
+ export declare function captureBrowserState(browser: CDPBrowser): Promise<BrowserState>;
31
+ /**
32
+ * Format state for LLM context.
33
+ *
34
+ * Sends a compact DOM (interactive elements only, capped at 15k chars).
35
+ * The full DOM is always available at <dataDir>/dom.txt for shell grep.
36
+ */
37
+ export declare function formatStateForLLM(state: BrowserState, maxDOMChars?: number): string;
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Browser state capture — what the LLM sees each step.
3
+ *
4
+ * Full DOM tree + page text are saved to dataDir files each step.
5
+ * The LLM gets a compact summary + knows it can use 'shell' to
6
+ * grep the full files for details.
7
+ */
8
+ import { execSync } from "child_process";
9
+ import { writeFileSync, readFileSync, mkdirSync } from "fs";
10
+ import { config } from "../config.js";
11
+ function resizeScreenshot(base64Png, width = 1024, height = 768) {
12
+ try {
13
+ const inPath = "/tmp/ss_full.png";
14
+ const outPath = "/tmp/ss_resized.jpeg";
15
+ writeFileSync(inPath, Buffer.from(base64Png, "base64"));
16
+ execSync(`convert ${inPath} -resize ${width}x${height} -quality 70 ${outPath}`, { stdio: "pipe" });
17
+ return readFileSync(outPath).toString("base64");
18
+ }
19
+ catch {
20
+ return base64Png;
21
+ }
22
+ }
23
+ export async function captureBrowserState(browser) {
24
+ const [url, title, dom, tabs, rawScreenshot, scrollPos] = await Promise.all([
25
+ browser.currentUrl(),
26
+ browser.pageTitle(),
27
+ browser.getIndexedDOM(),
28
+ browser.refreshTabs(),
29
+ browser.screenshot(),
30
+ browser.evaluate("({ x: window.scrollX, y: window.scrollY })"),
31
+ ]);
32
+ const screenshotBase64 = resizeScreenshot(rawScreenshot);
33
+ // Save full DOM tree to file — agent can grep it with 'shell'
34
+ const dataDir = config.browser.dataDir;
35
+ try {
36
+ mkdirSync(dataDir, { recursive: true });
37
+ writeFileSync(`${dataDir}/dom.txt`, dom.text);
38
+ }
39
+ catch { }
40
+ return {
41
+ url,
42
+ title,
43
+ domTree: dom.text,
44
+ elements: dom.elements,
45
+ tabs,
46
+ screenshotBase64,
47
+ scrollPosition: scrollPos,
48
+ pendingDialogs: [...browser.pendingDialogs],
49
+ };
50
+ }
51
+ /**
52
+ * Format state for LLM context.
53
+ *
54
+ * Sends a compact DOM (interactive elements only, capped at 15k chars).
55
+ * The full DOM is always available at <dataDir>/dom.txt for shell grep.
56
+ */
57
+ export function formatStateForLLM(state, maxDOMChars = 15_000) {
58
+ const dataDir = config.browser.dataDir;
59
+ const parts = [];
60
+ parts.push(`URL: ${state.url}`);
61
+ parts.push(`Title: ${state.title}`);
62
+ if (state.tabs.length > 1) {
63
+ parts.push(`\nOpen tabs (${state.tabs.length}):`);
64
+ for (const t of state.tabs) {
65
+ parts.push(` - [${t.id.slice(0, 8)}] ${t.title} (${t.url})`);
66
+ }
67
+ }
68
+ if (state.pendingDialogs.length > 0) {
69
+ parts.push(`\n⚠️ Pending dialog: ${state.pendingDialogs[0].type} — "${state.pendingDialogs[0].message}"`);
70
+ }
71
+ parts.push(`\nScroll position: x=${state.scrollPosition.x}, y=${state.scrollPosition.y}`);
72
+ parts.push(`\nInteractive elements (use [N] to reference):`);
73
+ if (state.domTree.length > maxDOMChars) {
74
+ parts.push(state.domTree.slice(0, maxDOMChars));
75
+ parts.push(`\n... (${state.domTree.length - maxDOMChars} chars omitted — full DOM saved to ${dataDir}/dom.txt, use shell to search)`);
76
+ }
77
+ else {
78
+ parts.push(state.domTree);
79
+ }
80
+ return parts.join("\n");
81
+ }
82
+ //# sourceMappingURL=state.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"state.js","sourceRoot":"","sources":["../../src/agent/state.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAE5D,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AActC,SAAS,gBAAgB,CAAC,SAAiB,EAAE,KAAK,GAAG,IAAI,EAAE,MAAM,GAAG,GAAG;IACtE,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,kBAAkB,CAAC;QAClC,MAAM,OAAO,GAAG,sBAAsB,CAAC;QACvC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;QACxD,QAAQ,CAAC,WAAW,MAAM,YAAY,KAAK,IAAI,MAAM,gBAAgB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QACnG,OAAO,YAAY,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,SAAS,CAAC;IAClB,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmB;IAC5D,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC3E,OAAO,CAAC,UAAU,EAAE;QACpB,OAAO,CAAC,SAAS,EAAE;QACnB,OAAO,CAAC,aAAa,EAAE;QACvB,OAAO,CAAC,WAAW,EAAE;QACrB,OAAO,CAAC,UAAU,EAAE;QACpB,OAAO,CAAC,QAAQ,CAA2B,4CAA4C,CAAC;KACxF,CAAC,CAAC;IAEH,MAAM,gBAAgB,GAAG,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAEzD,8DAA8D;IAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IACvC,IAAI,CAAC;QACJ,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,aAAa,CAAC,GAAG,OAAO,UAAU,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IAEV,OAAO;QACN,GAAG;QACH,KAAK;QACL,OAAO,EAAE,GAAG,CAAC,IAAI;QACjB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,IAAI;QACJ,gBAAgB;QAChB,cAAc,EAAE,SAAS;QACzB,cAAc,EAAE,CAAC,GAAG,OAAO,CAAC,cAAc,CAAC;KAC3C,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAmB,EAAE,WAAW,GAAG,MAAM;IAC1E,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;IACvC,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;IAChC,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAEpC,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,gBAAgB,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;QAC/D,CAAC;IACF,CAAC;IAED,IAAI,KAAK,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;IAC3G,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,wBAAwB,KAAK,CAAC,cAAc,CAAC,CAAC,OAAO,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1F,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAE7D,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,WAAW,sCAAsC,OAAO,gCAAgC,CAAC,CAAC;IACvI,CAAC;SAAM,CAAC;QACP,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC"}