@agentruntime-labs/browser-agent 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +6 -0
  3. package/README.md +64 -34
  4. package/bin/browser-agent.js +1 -0
  5. package/cli/control.mjs +477 -0
  6. package/cli/daemon.mjs +143 -0
  7. package/cli/hosts.mjs +1 -0
  8. package/cli/main.mjs +24 -0
  9. package/cli/mcp.mjs +1 -0
  10. package/docs/ARCHITECTURE.md +73 -0
  11. package/docs/DESKTOP_MCP.md +41 -0
  12. package/docs/EXTENSION_MANIFEST.md +163 -0
  13. package/docs/HYBRID.md +12 -0
  14. package/docs/MANAGED_BROWSER.md +111 -0
  15. package/docs/MCP_TOOLS.md +486 -0
  16. package/docs/MESSAGES.md +272 -0
  17. package/docs/MVP.md +76 -0
  18. package/docs/PRIVACY_POLICY.md +50 -0
  19. package/docs/README.md +27 -0
  20. package/docs/REFERENCE_PROJECTS.md +24 -0
  21. package/docs/RELEASE_CHECKLIST.md +49 -0
  22. package/docs/REPEATED_DOCS.md +27 -0
  23. package/docs/SAFETY.md +92 -0
  24. package/docs/SHELL_QUOTING.md +30 -0
  25. package/docs/SPLIT_REPOS.md +46 -0
  26. package/docs/THIRD_PARTY_LICENSES.md +182 -0
  27. package/docs/VISION.md +38 -0
  28. package/docs/WEB_STORE_SUBMISSION.md +117 -0
  29. package/extension/approval.js +1 -0
  30. package/extension/assets/agentruntime-logo.svg +77 -0
  31. package/extension/assets/agentruntime-mark-512.png +0 -0
  32. package/extension/background.js +776 -62
  33. package/extension/cdp.js +577 -0
  34. package/extension/constants.js +1 -0
  35. package/extension/content.js +69 -9
  36. package/extension/devtools-grant.html +31 -0
  37. package/extension/devtools-grant.js +29 -0
  38. package/extension/icons/icon128.png +0 -0
  39. package/extension/icons/icon16.png +0 -0
  40. package/extension/icons/icon32.png +0 -0
  41. package/extension/icons/icon48.png +0 -0
  42. package/extension/manifest.json +50 -23
  43. package/extension/policy.js +102 -0
  44. package/extension/popup.html +190 -56
  45. package/extension/popup.js +376 -166
  46. package/extension/soft.js +301 -0
  47. package/extension/welcome.html +91 -0
  48. package/extension/welcome.js +13 -0
  49. package/package.json +6 -2
  50. package/server/dist/arBrowserPaths.d.ts.map +1 -1
  51. package/server/dist/arBrowserPaths.js +1 -0
  52. package/server/dist/arBrowserPaths.js.map +1 -1
  53. package/server/dist/automationBackend.d.ts +46 -0
  54. package/server/dist/automationBackend.d.ts.map +1 -0
  55. package/server/dist/automationBackend.js +20 -0
  56. package/server/dist/automationBackend.js.map +1 -0
  57. package/server/dist/automationSession.d.ts.map +1 -1
  58. package/server/dist/automationSession.js +1 -0
  59. package/server/dist/automationSession.js.map +1 -1
  60. package/server/dist/bridge.d.ts +17 -4
  61. package/server/dist/bridge.d.ts.map +1 -1
  62. package/server/dist/bridge.js +132 -33
  63. package/server/dist/bridge.js.map +1 -1
  64. package/server/dist/browserToolsExtras.d.ts +2 -2
  65. package/server/dist/browserToolsExtras.d.ts.map +1 -1
  66. package/server/dist/browserToolsExtras.js +70 -1
  67. package/server/dist/browserToolsExtras.js.map +1 -1
  68. package/server/dist/cdpTools.d.ts +5 -0
  69. package/server/dist/cdpTools.d.ts.map +1 -0
  70. package/server/dist/cdpTools.js +411 -0
  71. package/server/dist/cdpTools.js.map +1 -0
  72. package/server/dist/compositeTools.d.ts +2 -2
  73. package/server/dist/compositeTools.d.ts.map +1 -1
  74. package/server/dist/compositeTools.js.map +1 -1
  75. package/server/dist/debugSnapshot.d.ts +2 -2
  76. package/server/dist/debugSnapshot.d.ts.map +1 -1
  77. package/server/dist/debugSnapshot.js.map +1 -1
  78. package/server/dist/guardrails.d.ts.map +1 -1
  79. package/server/dist/guardrails.js +1 -0
  80. package/server/dist/guardrails.js.map +1 -1
  81. package/server/dist/humanLike.d.ts.map +1 -1
  82. package/server/dist/humanLike.js +1 -0
  83. package/server/dist/humanLike.js.map +1 -1
  84. package/server/dist/main.js +56 -15
  85. package/server/dist/main.js.map +1 -1
  86. package/server/dist/managedBridge.d.ts +58 -0
  87. package/server/dist/managedBridge.d.ts.map +1 -0
  88. package/server/dist/managedBridge.js +619 -0
  89. package/server/dist/managedBridge.js.map +1 -0
  90. package/server/dist/managedDom.d.ts +58 -0
  91. package/server/dist/managedDom.d.ts.map +1 -0
  92. package/server/dist/managedDom.js +310 -0
  93. package/server/dist/managedDom.js.map +1 -0
  94. package/server/dist/policyTiers.d.ts.map +1 -1
  95. package/server/dist/policyTiers.js +3 -1
  96. package/server/dist/policyTiers.js.map +1 -1
  97. package/server/dist/protocol.d.ts +4 -0
  98. package/server/dist/protocol.d.ts.map +1 -1
  99. package/server/dist/protocol.js +1 -0
  100. package/server/dist/protocol.js.map +1 -1
  101. package/server/dist/recorder.d.ts +22 -0
  102. package/server/dist/recorder.d.ts.map +1 -0
  103. package/server/dist/recorder.js +82 -0
  104. package/server/dist/recorder.js.map +1 -0
  105. package/server/dist/richEditorFill.d.ts.map +1 -1
  106. package/server/dist/richEditorFill.js +1 -0
  107. package/server/dist/richEditorFill.js.map +1 -1
  108. package/server/dist/screenshotCrop.d.ts.map +1 -1
  109. package/server/dist/screenshotCrop.js +1 -0
  110. package/server/dist/screenshotCrop.js.map +1 -1
  111. package/server/dist/sessionLog.d.ts +1 -0
  112. package/server/dist/sessionLog.d.ts.map +1 -1
  113. package/server/dist/sessionLog.js +1 -0
  114. package/server/dist/sessionLog.js.map +1 -1
  115. package/server/dist/sessionPhase.d.ts.map +1 -1
  116. package/server/dist/sessionPhase.js.map +1 -1
  117. package/server/dist/sessionSummary.d.ts.map +1 -1
  118. package/server/dist/sessionSummary.js +1 -0
  119. package/server/dist/sessionSummary.js.map +1 -1
  120. package/server/dist/shotUtil.d.ts +44 -0
  121. package/server/dist/shotUtil.d.ts.map +1 -0
  122. package/server/dist/shotUtil.js +89 -0
  123. package/server/dist/shotUtil.js.map +1 -0
  124. package/server/dist/skillsInstructions.d.ts.map +1 -1
  125. package/server/dist/skillsInstructions.js +1 -0
  126. package/server/dist/skillsInstructions.js.map +1 -1
  127. package/server/dist/snapshotText.d.ts +47 -0
  128. package/server/dist/snapshotText.d.ts.map +1 -0
  129. package/server/dist/snapshotText.js +99 -0
  130. package/server/dist/snapshotText.js.map +1 -0
  131. package/server/dist/structureFilter.d.ts.map +1 -1
  132. package/server/dist/structureFilter.js.map +1 -1
  133. package/server/dist/surveyCatalog.d.ts.map +1 -1
  134. package/server/dist/surveyCatalog.js +1 -0
  135. package/server/dist/surveyCatalog.js.map +1 -1
  136. package/server/dist/surveyRunner.d.ts.map +1 -1
  137. package/server/dist/surveyRunner.js.map +1 -1
  138. package/server/dist/toolHelpers.d.ts +27 -2
  139. package/server/dist/toolHelpers.d.ts.map +1 -1
  140. package/server/dist/toolHelpers.js +67 -1
  141. package/server/dist/toolHelpers.js.map +1 -1
  142. package/server/dist/toolRuntime.d.ts +4 -0
  143. package/server/dist/toolRuntime.d.ts.map +1 -1
  144. package/server/dist/toolRuntime.js +12 -6
  145. package/server/dist/toolRuntime.js.map +1 -1
  146. package/server/dist/toolSchemas.d.ts +37 -1
  147. package/server/dist/toolSchemas.d.ts.map +1 -1
  148. package/server/dist/toolSchemas.js +33 -1
  149. package/server/dist/toolSchemas.js.map +1 -1
  150. package/server/dist/tools.d.ts +2 -2
  151. package/server/dist/tools.d.ts.map +1 -1
  152. package/server/dist/tools.js +65 -11
  153. package/server/dist/tools.js.map +1 -1
  154. package/server/dist/verdictExtract.d.ts.map +1 -1
  155. package/server/dist/verdictExtract.js +1 -0
  156. package/server/dist/verdictExtract.js.map +1 -1
  157. package/server/dist/workflowTools.d.ts +2 -2
  158. package/server/dist/workflowTools.d.ts.map +1 -1
  159. package/server/dist/workflowTools.js +16 -4
  160. package/server/dist/workflowTools.js.map +1 -1
  161. package/skills/browser-agent/SKILL.md +89 -0
@@ -0,0 +1,477 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ import { execFileSync, spawn } from 'node:child_process';
3
+ import readline from 'node:readline';
4
+ import fs from 'node:fs';
5
+ import http from 'node:http';
6
+ import path from 'node:path';
7
+ import { fileURLToPath } from 'node:url';
8
+ import { infoFile, runDaemon } from './daemon.mjs';
9
+
10
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
11
+ const MAIN = path.join(__dirname, 'main.mjs');
12
+
13
+ function readInfo() {
14
+ try {
15
+ return JSON.parse(fs.readFileSync(infoFile(), 'utf8'));
16
+ } catch {
17
+ return null;
18
+ }
19
+ }
20
+
21
+ function request(info, method, urlPath, body, auth = true) {
22
+ return new Promise((resolve, reject) => {
23
+ const data = body === undefined ? undefined : JSON.stringify(body);
24
+ const req = http.request(
25
+ {
26
+ host: '127.0.0.1',
27
+ port: info.httpPort,
28
+ path: urlPath,
29
+ method,
30
+ headers: {
31
+ ...(auth ? { authorization: `Bearer ${info.token}` } : {}),
32
+ ...(data ? { 'content-type': 'application/json', 'content-length': Buffer.byteLength(data) } : {}),
33
+ },
34
+ timeout: 200_000,
35
+ },
36
+ (res) => {
37
+ let buf = '';
38
+ res.on('data', (c) => (buf += c));
39
+ res.on('end', () => {
40
+ try {
41
+ resolve({ status: res.statusCode, json: JSON.parse(buf) });
42
+ } catch {
43
+ reject(new Error(`bad response (${res.statusCode}): ${buf.slice(0, 200)}`));
44
+ }
45
+ });
46
+ }
47
+ );
48
+ req.on('error', reject);
49
+ req.on('timeout', () => req.destroy(new Error('daemon request timed out')));
50
+ if (data) req.write(data);
51
+ req.end();
52
+ });
53
+ }
54
+
55
+ async function alive(info) {
56
+ if (!info) return false;
57
+ try {
58
+ const r = await request(info, 'GET', '/health', undefined, false);
59
+ return r.status === 200;
60
+ } catch {
61
+ return false;
62
+ }
63
+ }
64
+
65
+ /** Return daemon info, starting a detached daemon if none is running. */
66
+ async function ensureDaemon() {
67
+ let info = readInfo();
68
+ if (await alive(info)) return info;
69
+ const child = spawn(process.execPath, [MAIN, 'daemon', '--detached'], {
70
+ detached: true,
71
+ stdio: 'ignore',
72
+ windowsHide: true,
73
+ });
74
+ child.unref();
75
+ for (let i = 0; i < 80; i++) {
76
+ await new Promise((r) => setTimeout(r, 250));
77
+ info = readInfo();
78
+ if (await alive(info)) return info;
79
+ }
80
+ throw new Error('daemon did not start (run `browser-agent daemon` to see errors)');
81
+ }
82
+
83
+ export async function callTool(tool, args = {}, profile) {
84
+ const info = await ensureDaemon();
85
+ const r = await request(info, 'POST', '/call', { tool, args, ...(profile ? { profile: String(profile) } : {}) });
86
+ if (r.status !== 200) throw new Error(r.json?.error ?? `daemon error ${r.status}`);
87
+ return r.json;
88
+ }
89
+
90
+ // ---- arg parsing ------------------------------------------------------------------------------
91
+
92
+ function parseValue(v) {
93
+ try {
94
+ return JSON.parse(v);
95
+ } catch {
96
+ return v;
97
+ }
98
+ }
99
+
100
+ /** Split argv into positionals and --flags (--flag value, --flag=value, --bool). */
101
+ export function parseArgs(argv) {
102
+ const pos = [];
103
+ const flags = {};
104
+ for (let i = 0; i < argv.length; i++) {
105
+ const a = argv[i];
106
+ if (a.startsWith('--')) {
107
+ const [k, inline] = a.slice(2).split(/=(.*)/s, 2);
108
+ if (inline !== undefined) flags[k] = parseValue(inline);
109
+ else if (argv[i + 1] !== undefined && !argv[i + 1].startsWith('--')) flags[k] = parseValue(argv[++i]);
110
+ else flags[k] = true;
111
+ } else pos.push(a);
112
+ }
113
+ return { pos, flags };
114
+ }
115
+
116
+ /** `{"a":1}` or key=value pairs -> object. */
117
+ export function argsFrom(rest) {
118
+ if (rest.length === 1 && rest[0].trim().startsWith('{')) return JSON.parse(rest[0]);
119
+ const o = {};
120
+ for (const r of rest) {
121
+ const i = r.indexOf('=');
122
+ if (i < 1) throw new Error(`expected key=value or a JSON object, got "${r}"`);
123
+ o[r.slice(0, i)] = parseValue(r.slice(i + 1));
124
+ }
125
+ return o;
126
+ }
127
+
128
+ // ---- output -----------------------------------------------------------------------------------
129
+
130
+ function show(out, flags) {
131
+ if (flags.json) {
132
+ console.log(JSON.stringify(out, null, 2));
133
+ return out.isError ? 1 : 0;
134
+ }
135
+ for (const c of out.content) {
136
+ if (c.type === 'image') console.log(`image: ${c.path}`);
137
+ else if (c.type === 'text') {
138
+ try {
139
+ const j = JSON.parse(c.text);
140
+ const { automationSessionId, tier, needsEvidenceAfterNavigate, ...rest } = j; // server bookkeeping
141
+ console.log(JSON.stringify(rest, null, 2));
142
+ } catch {
143
+ console.log(c.text);
144
+ }
145
+ }
146
+ }
147
+ return out.isError ? 1 : 0;
148
+ }
149
+
150
+ /** --on <text>: make the tab whose URL or title contains <text> this agent's default tab before the command. */
151
+ async function selectOn(flags) {
152
+ if (flags.on === undefined || flags.on === true) return null;
153
+ const out = await callTool('browser_use_tab', { match: String(flags.on) }, flags.profile);
154
+ return out.isError ? out : null;
155
+ }
156
+
157
+ async function run(tool, args, flags) {
158
+ const failed = await selectOn(flags);
159
+ if (failed) return show(failed, flags);
160
+ if (flags.tab !== undefined && args.tabId === undefined) args.tabId = Number(flags.tab);
161
+ const out = await callTool(tool, args, flags.profile);
162
+ return show(out, flags);
163
+ }
164
+
165
+ /** Text currently on the system clipboard (for --clip). */
166
+ export function clipboardText() {
167
+ try {
168
+ if (process.platform === 'win32') {
169
+ return execFileSync('powershell', ['-NoProfile', '-Command', 'Get-Clipboard -Raw'], { encoding: 'utf8', windowsHide: true }).trim();
170
+ }
171
+ if (process.platform === 'darwin') return execFileSync('pbpaste', { encoding: 'utf8' }).trim();
172
+ return execFileSync('xclip', ['-selection', 'clipboard', '-o'], { encoding: 'utf8' }).trim();
173
+ } catch {
174
+ throw new Error('could not read the clipboard');
175
+ }
176
+ }
177
+
178
+ /** A URL given as "-" is read from stdin (pipe it in: no shell escaping at all). */
179
+ async function urlArg(flags, value) {
180
+ if (flags.clip) return clipboardText();
181
+ if (value === '-') {
182
+ let buf = '';
183
+ process.stdin.setEncoding('utf8');
184
+ for await (const chunk of process.stdin) buf += chunk;
185
+ return buf.trim();
186
+ }
187
+ return value;
188
+ }
189
+
190
+ /** Split a typed line into words; "double" and 'single' quotes group words and a backslash escapes a space or quote. Nothing else is special. */
191
+ export function splitLine(line) {
192
+ const out = [];
193
+ let cur = '';
194
+ let quote = null;
195
+ let has = false;
196
+ for (let i = 0; i < line.length; i++) {
197
+ const ch = line[i];
198
+ // A backslash escapes a following space or quote (so \" and "\ " work); before anything else it is literal,
199
+ // which keeps Windows paths like C:\temp\x and \\server\share intact.
200
+ if (ch === '\\' && i + 1 < line.length && /[\s"']/.test(line[i + 1])) {
201
+ cur += line[++i];
202
+ has = true;
203
+ continue;
204
+ }
205
+ if (quote) {
206
+ if (ch === quote) quote = null;
207
+ else cur += ch;
208
+ } else if (ch === '"' || ch === "'") {
209
+ quote = ch;
210
+ has = true;
211
+ } else if (/\s/.test(ch)) {
212
+ if (cur !== '' || has) out.push(cur);
213
+ cur = '';
214
+ has = false;
215
+ } else cur += ch;
216
+ }
217
+ if (cur !== '' || has) out.push(cur);
218
+ return out;
219
+ }
220
+
221
+ // ---- commands ---------------------------------------------------------------------------------
222
+
223
+ export const CONTROL_USAGE = `Control the browser from a shell (no MCP host needed). The first command starts a background daemon;
224
+ the Chrome extension (v0.2.2+) auto-connects to it.
225
+
226
+ browser-agent daemon [--ws-port n] [--http-port n] run the daemon in the foreground
227
+ browser-agent stop stop the daemon
228
+ browser-agent status daemon + whether the extension is connected
229
+ browser-agent wait [--timeout 60] wait until the extension is connected
230
+ browser-agent tools [--json] [name] list tools (or show one tool's schema)
231
+ browser-agent call <tool> [json | key=value ...] call any tool
232
+ browser-agent profiles Chrome profiles that have the extension connected
233
+ browser-agent use <label> send following commands to that profile ('use ""' = default)
234
+ (any command) --profile <label> run just this command on that profile
235
+ browser-agent shell interactive prompt: no shell quoting problems with & ? % in URLs
236
+ (URLs with & ? % break in most shells unless quoted: see docs/SHELL_QUOTING.md)
237
+ browser-agent open <url> | open --clip | open - open a new tab and work on it (--clip: URL from the clipboard; '-': URL from stdin; same for nav)
238
+ browser-agent use-tab <text | tab id> work on the tab whose URL or title contains <text>
239
+ (any command) --on <text> run that command on the tab matching <text>
240
+
241
+ Shortcuts (all accept --tab <id> and --json):
242
+ tabs | windows | snapshot | text
243
+ nav <url> | click <ref> | dblclick <ref> | rclick <ref> | fill <ref> <text> | press <key>
244
+ drag <ref> <toRef> | scroll <up|down> | wait-for <text>
245
+ click --x n --y n [--on-shot] [--right] [--hold ms] [--audit] [--engine cdp] | wheel --x n --y n --dy n | wait-change [--change] [--region x,y,w,h] [--timeout s]
246
+ shot [--region x,y,w,h] [--scale n] [--full] [--restore] [--out file] [--mode auto|cdp|visible] (--restore: briefly un-minimize a minimized window)
247
+ viewport <mobile|tablet|laptop|desktop|WxH> | scheme <light|dark|none> | reset
248
+ record start | record stop <name> [--gif]
249
+ window open [url] [--width n --height n | --phone | --tablet] (--phone: real 375 px page in a popup window) | window focus <id> | window resize <id> --width n --height n
250
+ clip read | clip write <text>
251
+ `;
252
+
253
+ /** "x,y,w,h" in page pixels. */
254
+ function parseRegion(v) {
255
+ const p = String(v).split(',').map(Number);
256
+ if (p.length !== 4 || p.some((n) => !Number.isFinite(n)) || p[2] <= 0 || p[3] <= 0) throw new Error('--region needs x,y,w,h in page pixels, e.g. --region 100,40,200,60');
257
+ return { x: p[0], y: p[1], w: p[2], h: p[3] };
258
+ }
259
+
260
+ function parseSize(s) {
261
+ const m = /^(\d+)x(\d+)$/.exec(s);
262
+ return m ? { width: Number(m[1]), height: Number(m[2]) } : null;
263
+ }
264
+
265
+ export async function control(cmd, rest) {
266
+ const { pos, flags } = parseArgs(rest);
267
+ switch (cmd) {
268
+ case 'daemon': {
269
+ const info = await runDaemon({
270
+ wsPort: flags['ws-port'] ? Number(flags['ws-port']) : undefined,
271
+ httpPort: flags['http-port'] ? Number(flags['http-port']) : undefined,
272
+ });
273
+ if (!flags.detached) {
274
+ console.log(`browser-agent daemon: pid ${info.pid}, extension port ${info.wsPort}, control port ${info.httpPort}`);
275
+ console.log('The Chrome extension connects on its own within ~30 s. Ctrl+C to stop.');
276
+ }
277
+ return 'keep-running';
278
+ }
279
+ case 'stop': {
280
+ const info = readInfo();
281
+ if (!(await alive(info))) {
282
+ console.log('daemon is not running');
283
+ return 0;
284
+ }
285
+ await request(info, 'POST', '/stop', {});
286
+ console.log('daemon stopped');
287
+ return 0;
288
+ }
289
+ case 'status': {
290
+ const info = readInfo();
291
+ if (!(await alive(info))) {
292
+ console.log('daemon: not running (it starts on the first command)');
293
+ return 1;
294
+ }
295
+ const out = await callTool('browser_get_tabs', {});
296
+ const text = out.content.find((c) => c.type === 'text')?.text ?? '';
297
+ const connected = !out.isError;
298
+ console.log(`daemon: running (pid ${info.pid}, extension port ${info.wsPort}, control port ${info.httpPort})`);
299
+ console.log(`extension: ${connected ? 'connected' : 'NOT connected'}${connected ? '' : ' (reload the extension or click Connect in its popup)'}`);
300
+ if (!connected && flags.json) console.log(text);
301
+ return connected ? 0 : 1;
302
+ }
303
+ case 'wait': {
304
+ const timeout = Number(flags.timeout ?? 60) * 1000;
305
+ const t0 = Date.now();
306
+ while (Date.now() - t0 < timeout) {
307
+ const out = await callTool('browser_get_tabs', {});
308
+ if (!out.isError) {
309
+ console.log('extension connected');
310
+ return 0;
311
+ }
312
+ await new Promise((r) => setTimeout(r, 2000));
313
+ }
314
+ console.error('timed out waiting for the extension');
315
+ return 1;
316
+ }
317
+ case 'tools': {
318
+ const info = await ensureDaemon();
319
+ const r = await request(info, 'GET', '/tools');
320
+ const list = r.json.tools;
321
+ if (pos[0]) {
322
+ const t = list.find((x) => x.name === pos[0] || x.name === `browser_${pos[0]}`);
323
+ if (!t) {
324
+ console.error(`no tool "${pos[0]}"`);
325
+ return 1;
326
+ }
327
+ console.log(JSON.stringify(t, null, 2));
328
+ return 0;
329
+ }
330
+ if (flags.json) console.log(JSON.stringify(list, null, 2));
331
+ else for (const t of list) console.log(`${t.name.padEnd(32)} ${(t.description ?? '').split('. ')[0].slice(0, 90)}`);
332
+ return 0;
333
+ }
334
+ case 'call': {
335
+ if (!pos[0]) throw new Error('usage: browser-agent call <tool> [json | key=value ...]');
336
+ const tool = pos[0].startsWith('browser_') ? pos[0] : `browser_${pos[0]}`;
337
+ return run(tool, argsFrom(pos.slice(1)), flags);
338
+ }
339
+ case 'shell': {
340
+ console.log('browser-agent shell. Type commands as usual (open <url>, shot, tabs, use-tab <text>, ...). Quotes group words. "exit" quits.');
341
+ const rl = readline.createInterface({ input: process.stdin, output: process.stdout, prompt: 'browser-agent> ' });
342
+ rl.prompt();
343
+ for await (const line of rl) {
344
+ const words = splitLine(line.trim());
345
+ if (words.length === 0) {
346
+ rl.prompt();
347
+ continue;
348
+ }
349
+ if (['exit', 'quit', 'q'].includes(words[0])) break;
350
+ if (words[0] === 'help') {
351
+ console.log(CONTROL_USAGE);
352
+ rl.prompt();
353
+ continue;
354
+ }
355
+ try {
356
+ const r = await control(words[0], words.slice(1));
357
+ if (r === undefined) console.log(`unknown command "${words[0]}" (type help)`);
358
+ } catch (e) {
359
+ console.error(e?.message ?? e);
360
+ }
361
+ rl.prompt();
362
+ }
363
+ return 0;
364
+ }
365
+ case 'use-tab': {
366
+ if (flags.clear) return run('browser_use_tab', { clear: true }, flags);
367
+ const id = pos[0] !== undefined && /^\d+$/.test(pos[0]) ? { tabId: Number(pos[0]) } : { match: pos.join(' ') };
368
+ if (!id.tabId && !id.match) throw new Error('usage: browser-agent use-tab <url or title text | tab id> | --clear');
369
+ return run('browser_use_tab', id, flags);
370
+ }
371
+ case 'open':
372
+ return run('browser_tab_open', { url: await urlArg(flags, pos[0]), use: flags.use !== false }, flags);
373
+ case 'profiles':
374
+ return run('browser_profiles', {}, flags);
375
+ case 'use':
376
+ return run('browser_use_profile', { profile: pos[0] ?? '' }, flags);
377
+ case 'tabs':
378
+ return run('browser_get_tabs', {}, flags);
379
+ case 'windows':
380
+ return run('browser_window_list', {}, flags);
381
+ case 'snapshot':
382
+ return run('browser_snapshot', { includeText: flags.text === true }, flags);
383
+ case 'text':
384
+ return run('browser_get_page_text', {}, flags);
385
+ case 'nav':
386
+ return run('browser_navigate', { url: await urlArg(flags, pos[0]) }, flags);
387
+ case 'click':
388
+ // click --x 400 --y 300 [--on-shot] [--right] [--hold ms] [--audit] [--engine cdp] (coordinates, for canvases and games)
389
+ if (flags.x !== undefined || flags.y !== undefined) {
390
+ if (!Number.isFinite(Number(flags.x)) || !Number.isFinite(Number(flags.y))) throw new Error('usage: browser-agent click --x <n> --y <n> [--on-shot] [--right] [--hold ms] [--audit] [--engine cdp]');
391
+ return run('browser_click_at', {
392
+ x: Number(flags.x), y: Number(flags.y),
393
+ ...(flags.right ? { button: 'right' } : {}),
394
+ ...(flags.hold ? { holdMs: Number(flags.hold) } : {}),
395
+ ...(flags['on-shot'] ? { onShot: true } : {}),
396
+ ...(flags.audit ? { audit: true } : {}),
397
+ ...(flags.engine ? { engine: String(flags.engine) } : {}),
398
+ }, flags);
399
+ }
400
+ return run('browser_click_element', { ref: pos[0], ...(flags.right ? { button: 'right' } : {}) }, flags);
401
+ case 'wheel':
402
+ if (!Number.isFinite(Number(flags.x)) || !Number.isFinite(Number(flags.y))) throw new Error('usage: browser-agent wheel --x <n> --y <n> --dy <n> [--dx n] [--on-shot] [--engine cdp]');
403
+ return run('browser_wheel', {
404
+ x: Number(flags.x), y: Number(flags.y), deltaY: Number(flags.dy ?? 0), deltaX: Number(flags.dx ?? 0),
405
+ ...(flags['on-shot'] ? { onShot: true } : {}),
406
+ ...(flags.engine ? { engine: String(flags.engine) } : {}),
407
+ }, flags);
408
+ case 'wait-change':
409
+ return run('browser_wait_change', {
410
+ mode: flags.change ? 'change' : 'settle',
411
+ ...(flags.timeout ? { timeoutMs: Number(flags.timeout) * 1000 } : {}),
412
+ ...(flags.region ? { region: parseRegion(flags.region) } : {}),
413
+ }, flags);
414
+ case 'dblclick':
415
+ return run('browser_double_click', { ref: pos[0] }, flags);
416
+ case 'rclick':
417
+ return run('browser_right_click', { ref: pos[0] }, flags);
418
+ case 'fill':
419
+ return run('browser_fill_input', { ref: pos[0], value: pos.slice(1).join(' ') }, flags);
420
+ case 'press':
421
+ return run('browser_press_key', { key: pos[0] }, flags);
422
+ case 'drag':
423
+ return run('browser_drag', { ref: pos[0], toRef: pos[1] }, flags);
424
+ case 'scroll':
425
+ return run('browser_scroll', { direction: pos[0] ?? 'down', ...(flags.amount ? { amount: Number(flags.amount) } : {}) }, flags);
426
+ case 'wait-for':
427
+ return run('browser_wait_for', { textContains: pos.join(' '), timeoutMs: Number(flags.timeout ?? 10) * 1000 }, flags);
428
+ case 'shot': {
429
+ const failed = await selectOn(flags);
430
+ if (failed) return show(failed, flags);
431
+ const args = { ...(flags.region ? { region: parseRegion(flags.region) } : {}), ...(flags.scale ? { scale: Number(flags.scale) } : {}), mode: flags.mode ?? 'auto', fullPage: flags.full === true, restoreWindow: flags.restore === true, format: flags.png ? 'png' : 'jpeg' };
432
+ if (flags.tab !== undefined) args.tabId = Number(flags.tab);
433
+ const out = await callTool('browser_screenshot_visible', args, flags.profile);
434
+ const img = out.content.find((c) => c.type === 'image');
435
+ if (!img) return show(out, flags);
436
+ if (flags.out) {
437
+ fs.copyFileSync(img.path, String(flags.out));
438
+ console.log(`image: ${path.resolve(String(flags.out))}`);
439
+ } else console.log(`image: ${img.path}`);
440
+ return 0;
441
+ }
442
+ case 'viewport': {
443
+ const size = parseSize(pos[0] ?? '');
444
+ return run('browser_set_viewport', size ?? { preset: pos[0] }, flags);
445
+ }
446
+ case 'scheme':
447
+ return run('browser_set_color_scheme', { scheme: pos[0] }, flags);
448
+ case 'reset':
449
+ return run('browser_reset_emulation', {}, flags);
450
+ case 'record':
451
+ if (pos[0] === 'start') return run('browser_record_start', {}, flags);
452
+ if (pos[0] === 'stop') return run('browser_record_stop', { name: pos[1], gif: flags.gif === true }, flags);
453
+ throw new Error('usage: browser-agent record start | record stop <name> [--gif]');
454
+ case 'window':
455
+ if (pos[0] === 'open' && (flags.phone || flags.tablet)) {
456
+ // A normal Chrome window cannot be narrower than ~516 px; a popup-type window can, so --phone gives a real
457
+ // 375 px wide page (no zoom, no DevTools). Chrome adds ~16 px of frame, so ask for 16 px more than the page width.
458
+ const phone = flags.phone === true;
459
+ return run('browser_window_open', { url: pos[1], popup: phone, width: phone ? 391 : 784, height: phone ? 860 : 1000, ...(flags.focused === false ? { focused: false } : {}) }, flags);
460
+ }
461
+ if (pos[0] === 'open') return run('browser_window_open', { url: pos[1], ...(flags.width ? { width: Number(flags.width) } : {}), ...(flags.height ? { height: Number(flags.height) } : {}) }, flags);
462
+ if (pos[0] === 'focus') return run('browser_window_focus', { windowId: Number(pos[1]) }, flags);
463
+ if (pos[0] === 'resize') return run('browser_window_resize', { windowId: Number(pos[1]), ...(flags.width ? { width: Number(flags.width) } : {}), ...(flags.height ? { height: Number(flags.height) } : {}), ...(flags.state ? { state: flags.state } : {}) }, flags);
464
+ throw new Error('usage: browser-agent window open|focus|resize ...');
465
+ case 'clip':
466
+ if (pos[0] === 'read') return run('browser_clipboard_read', {}, flags);
467
+ if (pos[0] === 'write') return run('browser_clipboard_write', { text: pos.slice(1).join(' ') }, flags);
468
+ throw new Error('usage: browser-agent clip read | clip write <text>');
469
+ default:
470
+ return undefined;
471
+ }
472
+ }
473
+
474
+ export const CONTROL_COMMANDS = new Set([
475
+ 'daemon', 'stop', 'status', 'wait', 'tools', 'call', 'profiles', 'use', 'use-tab', 'open', 'shell', 'tabs', 'windows', 'snapshot', 'text', 'nav', 'click', 'dblclick',
476
+ 'rclick', 'fill', 'press', 'drag', 'scroll', 'wait-for', 'wheel', 'wait-change', 'shot', 'viewport', 'scheme', 'reset', 'record', 'window', 'clip',
477
+ ]);
package/cli/daemon.mjs ADDED
@@ -0,0 +1,143 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ import { randomBytes } from 'node:crypto';
3
+ import fs from 'node:fs';
4
+ import http from 'node:http';
5
+ import net from 'node:net';
6
+ import os from 'node:os';
7
+ import path from 'node:path';
8
+ import { fileURLToPath } from 'node:url';
9
+ import { Client } from '@modelcontextprotocol/sdk/client/index.js';
10
+ import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
11
+
12
+ const __dirname = path.dirname(fileURLToPath(import.meta.url));
13
+ export const SERVER_MAIN = path.resolve(__dirname, '..', 'server', 'dist', 'main.js');
14
+
15
+ export function arHome() {
16
+ return process.env.BROWSER_AUTOMATION_HOME || path.join(os.homedir(), '.ar_browser');
17
+ }
18
+ export function infoFile() {
19
+ return path.join(arHome(), 'daemon.json');
20
+ }
21
+ export function outDir() {
22
+ return path.join(arHome(), 'cli-out');
23
+ }
24
+
25
+ /** First free TCP port at or above `start` on 127.0.0.1. */
26
+ export function freePort(start, span = 40) {
27
+ const tryPort = (p) =>
28
+ new Promise((resolve) => {
29
+ const s = net.createServer();
30
+ s.once('error', () => resolve(false));
31
+ s.once('listening', () => s.close(() => resolve(true)));
32
+ s.listen(p, '127.0.0.1');
33
+ });
34
+ return (async () => {
35
+ for (let p = start; p < start + span; p++) {
36
+ if (await tryPort(p)) return p;
37
+ }
38
+ throw new Error(`no free port in ${start}-${start + span - 1}`);
39
+ })();
40
+ }
41
+
42
+ /** Turn an MCP tool result into JSON-friendly output; images are written to outDir and replaced by a path. */
43
+ export function flatten(result, dir, counter) {
44
+ const out = { isError: result.isError === true, content: [] };
45
+ for (const c of result.content ?? []) {
46
+ if (c.type === 'image') {
47
+ const ext = c.mimeType === 'image/png' ? 'png' : 'jpg';
48
+ const file = path.join(dir, `shot-${Date.now()}-${counter.n++}.${ext}`);
49
+ fs.writeFileSync(file, Buffer.from(c.data, 'base64'));
50
+ out.content.push({ type: 'image', path: file });
51
+ } else if (c.type === 'text') {
52
+ out.content.push({ type: 'text', text: c.text });
53
+ }
54
+ }
55
+ if (result.structuredContent) {
56
+ out.structured = JSON.parse(
57
+ JSON.stringify(result.structuredContent, (k, v) =>
58
+ k === 'dataBase64' && typeof v === 'string' ? `<${v.length} base64 chars, see image path>` : v
59
+ )
60
+ );
61
+ }
62
+ return out;
63
+ }
64
+
65
+ export async function runDaemon(opts = {}) {
66
+ if (!fs.existsSync(SERVER_MAIN)) {
67
+ throw new Error(`Server not built: ${SERVER_MAIN} (run npm run build in server/)`);
68
+ }
69
+ const wsPort = opts.wsPort ?? (await freePort(18795));
70
+ const httpPort = opts.httpPort ?? (await freePort(18890));
71
+ const dir = opts.out ?? outDir();
72
+ fs.mkdirSync(dir, { recursive: true });
73
+ fs.mkdirSync(arHome(), { recursive: true });
74
+ const token = randomBytes(16).toString('hex');
75
+
76
+ const transport = new StdioClientTransport({
77
+ command: process.execPath,
78
+ args: [SERVER_MAIN, '--insecure-no-token'],
79
+ env: { ...process.env, BROWSER_AUTOMATION_WS_PORT: String(wsPort) },
80
+ stderr: 'ignore',
81
+ });
82
+ const client = new Client({ name: 'browser-agent-cli', version: '0.2.0' });
83
+ await client.connect(transport);
84
+ const { tools } = await client.listTools();
85
+
86
+ const counter = { n: 0 };
87
+ const server = http.createServer(async (req, res) => {
88
+ const send = (code, obj) => {
89
+ res.writeHead(code, { 'content-type': 'application/json' });
90
+ res.end(JSON.stringify(obj));
91
+ };
92
+ if (req.method === 'GET' && req.url === '/health') {
93
+ return send(200, { ok: true, pid: process.pid, wsPort, tools: tools.length });
94
+ }
95
+ if (req.headers.authorization !== `Bearer ${token}`) {
96
+ return send(401, { error: 'unauthorized' });
97
+ }
98
+ if (req.method === 'GET' && req.url === '/tools') {
99
+ return send(200, {
100
+ tools: tools.map((t) => ({ name: t.name, description: t.description, inputSchema: t.inputSchema })),
101
+ });
102
+ }
103
+ if (req.method === 'POST' && req.url === '/stop') {
104
+ send(200, { ok: true });
105
+ setTimeout(shutdown, 50);
106
+ return;
107
+ }
108
+ if (req.method === 'POST' && req.url === '/call') {
109
+ let body = '';
110
+ for await (const chunk of req) body += chunk;
111
+ try {
112
+ const { tool, args, profile } = JSON.parse(body);
113
+ if (typeof profile === 'string' && profile !== '') {
114
+ // Route this call (and the following ones) to a Chrome profile; one agent acts at a time.
115
+ const sel = await client.callTool({ name: 'browser_use_profile', arguments: { profile } }, undefined, { timeout: 30_000 });
116
+ if (sel.isError) return send(200, flatten(sel, dir, counter));
117
+ }
118
+ const r = await client.callTool({ name: tool, arguments: args ?? {} }, undefined, { timeout: 180_000 });
119
+ return send(200, flatten(r, dir, counter));
120
+ } catch (e) {
121
+ return send(500, { error: String(e?.message ?? e) });
122
+ }
123
+ }
124
+ send(404, { error: 'not found' });
125
+ });
126
+ await new Promise((resolve) => server.listen(httpPort, '127.0.0.1', resolve));
127
+
128
+ const info = { pid: process.pid, httpPort, wsPort, token, outDir: dir, startedAt: new Date().toISOString() };
129
+ fs.writeFileSync(infoFile(), JSON.stringify(info, null, 2), { mode: 0o600 });
130
+
131
+ async function shutdown() {
132
+ try {
133
+ fs.unlinkSync(infoFile());
134
+ } catch {
135
+ /* already gone */
136
+ }
137
+ await client.close().catch(() => undefined);
138
+ server.close();
139
+ process.exit(0);
140
+ }
141
+ for (const sig of ['SIGINT', 'SIGTERM']) process.on(sig, shutdown);
142
+ return info;
143
+ }
package/cli/hosts.mjs CHANGED
@@ -1,3 +1,4 @@
1
+ // SPDX-License-Identifier: Apache-2.0
1
2
  import fs from 'node:fs';
2
3
  import os from 'node:os';
3
4
  import path from 'node:path';
package/cli/main.mjs CHANGED
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env node
2
+ // SPDX-License-Identifier: Apache-2.0
2
3
  import path from 'node:path';
3
4
  import fs from 'node:fs';
4
5
  import { fileURLToPath } from 'node:url';
@@ -18,6 +19,7 @@ Usage:
18
19
  browser-agent uninstall [options]
19
20
  browser-agent doctor
20
21
  browser-agent paths
22
+ browser-agent status | tabs | shot | call <tool> ... control the browser from a shell (see: browser-agent help-control)
21
23
 
22
24
  Install options:
23
25
  --cursor Wire Cursor (~/.cursor/mcp.json) (default: on)
@@ -125,11 +127,32 @@ function cmdPaths() {
125
127
  }
126
128
 
127
129
  const [cmd, ...rest] = process.argv.slice(2);
130
+ if (cmd === 'help-control') {
131
+ const { CONTROL_USAGE } = await import('./control.mjs');
132
+ console.log(CONTROL_USAGE);
133
+ process.exit(0);
134
+ }
128
135
  if (!cmd || cmd === '--help' || cmd === '-h') {
129
136
  usage();
130
137
  process.exit(cmd ? 0 : 1);
131
138
  }
132
139
 
140
+ const CONTROL = new Set([
141
+ 'daemon', 'stop', 'status', 'wait', 'tools', 'call', 'profiles', 'use', 'use-tab', 'open', 'shell', 'tabs', 'windows', 'snapshot', 'text', 'nav', 'click', 'dblclick',
142
+ 'rclick', 'fill', 'press', 'drag', 'scroll', 'wait-for', 'wheel', 'wait-change', 'shot', 'viewport', 'scheme', 'reset', 'record', 'window', 'clip',
143
+ ]);
144
+ if (CONTROL.has(cmd)) {
145
+ const { control } = await import('./control.mjs');
146
+ try {
147
+ const code = await control(cmd, rest);
148
+ if (code === 'keep-running') {
149
+ // daemon: stay alive; the HTTP server and child process keep the event loop busy.
150
+ } else process.exit(code ?? 0);
151
+ } catch (e) {
152
+ console.error(`browser-agent ${cmd}: ${e?.message ?? e}`);
153
+ process.exit(1);
154
+ }
155
+ } else {
133
156
  const flags = parseFlags(rest);
134
157
  if (flags.help) {
135
158
  usage();
@@ -154,3 +177,4 @@ switch (cmd) {
154
177
  usage();
155
178
  process.exit(1);
156
179
  }
180
+ }
package/cli/mcp.mjs CHANGED
@@ -1,3 +1,4 @@
1
+ // SPDX-License-Identifier: Apache-2.0
1
2
  import fs from 'node:fs';
2
3
  import path from 'node:path';
3
4