@agentruntime-labs/browser-agent 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +6 -0
  3. package/README.md +64 -34
  4. package/bin/browser-agent.js +1 -0
  5. package/cli/control.mjs +477 -0
  6. package/cli/daemon.mjs +143 -0
  7. package/cli/hosts.mjs +1 -0
  8. package/cli/main.mjs +24 -0
  9. package/cli/mcp.mjs +1 -0
  10. package/docs/ARCHITECTURE.md +73 -0
  11. package/docs/DESKTOP_MCP.md +41 -0
  12. package/docs/EXTENSION_MANIFEST.md +163 -0
  13. package/docs/HYBRID.md +12 -0
  14. package/docs/MANAGED_BROWSER.md +111 -0
  15. package/docs/MCP_TOOLS.md +486 -0
  16. package/docs/MESSAGES.md +272 -0
  17. package/docs/MVP.md +76 -0
  18. package/docs/PRIVACY_POLICY.md +50 -0
  19. package/docs/README.md +27 -0
  20. package/docs/REFERENCE_PROJECTS.md +24 -0
  21. package/docs/RELEASE_CHECKLIST.md +49 -0
  22. package/docs/REPEATED_DOCS.md +27 -0
  23. package/docs/SAFETY.md +92 -0
  24. package/docs/SHELL_QUOTING.md +30 -0
  25. package/docs/SPLIT_REPOS.md +46 -0
  26. package/docs/THIRD_PARTY_LICENSES.md +182 -0
  27. package/docs/VISION.md +38 -0
  28. package/docs/WEB_STORE_SUBMISSION.md +117 -0
  29. package/extension/approval.js +1 -0
  30. package/extension/assets/agentruntime-logo.svg +77 -0
  31. package/extension/assets/agentruntime-mark-512.png +0 -0
  32. package/extension/background.js +776 -62
  33. package/extension/cdp.js +577 -0
  34. package/extension/constants.js +1 -0
  35. package/extension/content.js +69 -9
  36. package/extension/devtools-grant.html +31 -0
  37. package/extension/devtools-grant.js +29 -0
  38. package/extension/icons/icon128.png +0 -0
  39. package/extension/icons/icon16.png +0 -0
  40. package/extension/icons/icon32.png +0 -0
  41. package/extension/icons/icon48.png +0 -0
  42. package/extension/manifest.json +50 -23
  43. package/extension/policy.js +102 -0
  44. package/extension/popup.html +190 -56
  45. package/extension/popup.js +376 -166
  46. package/extension/soft.js +301 -0
  47. package/extension/welcome.html +91 -0
  48. package/extension/welcome.js +13 -0
  49. package/package.json +6 -2
  50. package/server/dist/arBrowserPaths.d.ts.map +1 -1
  51. package/server/dist/arBrowserPaths.js +1 -0
  52. package/server/dist/arBrowserPaths.js.map +1 -1
  53. package/server/dist/automationBackend.d.ts +46 -0
  54. package/server/dist/automationBackend.d.ts.map +1 -0
  55. package/server/dist/automationBackend.js +20 -0
  56. package/server/dist/automationBackend.js.map +1 -0
  57. package/server/dist/automationSession.d.ts.map +1 -1
  58. package/server/dist/automationSession.js +1 -0
  59. package/server/dist/automationSession.js.map +1 -1
  60. package/server/dist/bridge.d.ts +17 -4
  61. package/server/dist/bridge.d.ts.map +1 -1
  62. package/server/dist/bridge.js +132 -33
  63. package/server/dist/bridge.js.map +1 -1
  64. package/server/dist/browserToolsExtras.d.ts +2 -2
  65. package/server/dist/browserToolsExtras.d.ts.map +1 -1
  66. package/server/dist/browserToolsExtras.js +70 -1
  67. package/server/dist/browserToolsExtras.js.map +1 -1
  68. package/server/dist/cdpTools.d.ts +5 -0
  69. package/server/dist/cdpTools.d.ts.map +1 -0
  70. package/server/dist/cdpTools.js +411 -0
  71. package/server/dist/cdpTools.js.map +1 -0
  72. package/server/dist/compositeTools.d.ts +2 -2
  73. package/server/dist/compositeTools.d.ts.map +1 -1
  74. package/server/dist/compositeTools.js.map +1 -1
  75. package/server/dist/debugSnapshot.d.ts +2 -2
  76. package/server/dist/debugSnapshot.d.ts.map +1 -1
  77. package/server/dist/debugSnapshot.js.map +1 -1
  78. package/server/dist/guardrails.d.ts.map +1 -1
  79. package/server/dist/guardrails.js +1 -0
  80. package/server/dist/guardrails.js.map +1 -1
  81. package/server/dist/humanLike.d.ts.map +1 -1
  82. package/server/dist/humanLike.js +1 -0
  83. package/server/dist/humanLike.js.map +1 -1
  84. package/server/dist/main.js +56 -15
  85. package/server/dist/main.js.map +1 -1
  86. package/server/dist/managedBridge.d.ts +58 -0
  87. package/server/dist/managedBridge.d.ts.map +1 -0
  88. package/server/dist/managedBridge.js +619 -0
  89. package/server/dist/managedBridge.js.map +1 -0
  90. package/server/dist/managedDom.d.ts +58 -0
  91. package/server/dist/managedDom.d.ts.map +1 -0
  92. package/server/dist/managedDom.js +310 -0
  93. package/server/dist/managedDom.js.map +1 -0
  94. package/server/dist/policyTiers.d.ts.map +1 -1
  95. package/server/dist/policyTiers.js +3 -1
  96. package/server/dist/policyTiers.js.map +1 -1
  97. package/server/dist/protocol.d.ts +4 -0
  98. package/server/dist/protocol.d.ts.map +1 -1
  99. package/server/dist/protocol.js +1 -0
  100. package/server/dist/protocol.js.map +1 -1
  101. package/server/dist/recorder.d.ts +22 -0
  102. package/server/dist/recorder.d.ts.map +1 -0
  103. package/server/dist/recorder.js +82 -0
  104. package/server/dist/recorder.js.map +1 -0
  105. package/server/dist/richEditorFill.d.ts.map +1 -1
  106. package/server/dist/richEditorFill.js +1 -0
  107. package/server/dist/richEditorFill.js.map +1 -1
  108. package/server/dist/screenshotCrop.d.ts.map +1 -1
  109. package/server/dist/screenshotCrop.js +1 -0
  110. package/server/dist/screenshotCrop.js.map +1 -1
  111. package/server/dist/sessionLog.d.ts +1 -0
  112. package/server/dist/sessionLog.d.ts.map +1 -1
  113. package/server/dist/sessionLog.js +1 -0
  114. package/server/dist/sessionLog.js.map +1 -1
  115. package/server/dist/sessionPhase.d.ts.map +1 -1
  116. package/server/dist/sessionPhase.js.map +1 -1
  117. package/server/dist/sessionSummary.d.ts.map +1 -1
  118. package/server/dist/sessionSummary.js +1 -0
  119. package/server/dist/sessionSummary.js.map +1 -1
  120. package/server/dist/shotUtil.d.ts +44 -0
  121. package/server/dist/shotUtil.d.ts.map +1 -0
  122. package/server/dist/shotUtil.js +89 -0
  123. package/server/dist/shotUtil.js.map +1 -0
  124. package/server/dist/skillsInstructions.d.ts.map +1 -1
  125. package/server/dist/skillsInstructions.js +1 -0
  126. package/server/dist/skillsInstructions.js.map +1 -1
  127. package/server/dist/snapshotText.d.ts +47 -0
  128. package/server/dist/snapshotText.d.ts.map +1 -0
  129. package/server/dist/snapshotText.js +99 -0
  130. package/server/dist/snapshotText.js.map +1 -0
  131. package/server/dist/structureFilter.d.ts.map +1 -1
  132. package/server/dist/structureFilter.js.map +1 -1
  133. package/server/dist/surveyCatalog.d.ts.map +1 -1
  134. package/server/dist/surveyCatalog.js +1 -0
  135. package/server/dist/surveyCatalog.js.map +1 -1
  136. package/server/dist/surveyRunner.d.ts.map +1 -1
  137. package/server/dist/surveyRunner.js.map +1 -1
  138. package/server/dist/toolHelpers.d.ts +27 -2
  139. package/server/dist/toolHelpers.d.ts.map +1 -1
  140. package/server/dist/toolHelpers.js +67 -1
  141. package/server/dist/toolHelpers.js.map +1 -1
  142. package/server/dist/toolRuntime.d.ts +4 -0
  143. package/server/dist/toolRuntime.d.ts.map +1 -1
  144. package/server/dist/toolRuntime.js +12 -6
  145. package/server/dist/toolRuntime.js.map +1 -1
  146. package/server/dist/toolSchemas.d.ts +37 -1
  147. package/server/dist/toolSchemas.d.ts.map +1 -1
  148. package/server/dist/toolSchemas.js +33 -1
  149. package/server/dist/toolSchemas.js.map +1 -1
  150. package/server/dist/tools.d.ts +2 -2
  151. package/server/dist/tools.d.ts.map +1 -1
  152. package/server/dist/tools.js +65 -11
  153. package/server/dist/tools.js.map +1 -1
  154. package/server/dist/verdictExtract.d.ts.map +1 -1
  155. package/server/dist/verdictExtract.js +1 -0
  156. package/server/dist/verdictExtract.js.map +1 -1
  157. package/server/dist/workflowTools.d.ts +2 -2
  158. package/server/dist/workflowTools.d.ts.map +1 -1
  159. package/server/dist/workflowTools.js +16 -4
  160. package/server/dist/workflowTools.js.map +1 -1
  161. package/skills/browser-agent/SKILL.md +89 -0
@@ -1,10 +1,38 @@
1
+ // SPDX-License-Identifier: Apache-2.0
1
2
  /**
2
- * Browser Automation — MV3 service worker (WebSocket client to localhost bridge).
3
+ * AgentRuntime Browser Agent — MV3 service worker (WebSocket client to localhost bridge).
3
4
  */
4
5
 
5
6
  import { DEFAULT_WS_PORT, WS_PORT_FALLBACK_ATTEMPTS } from './constants.js';
7
+ import * as cdp from './cdp.js';
8
+ import * as soft from './soft.js';
9
+ import * as policy from './policy.js';
6
10
 
7
- let ws = null;
11
+ /**
12
+ * One WebSocket per MCP host (Claude, Cursor, another Claude session ...). Each host runs its own server on its own
13
+ * port; we stay connected to all of them. Replies always go back on the socket the request came in on.
14
+ * @type {Map<number, WebSocket>}
15
+ */
16
+ const sockets = new Map();
17
+ /** port -> time until which we skip it because another extension (profile) already holds that server. */
18
+ const busyUntil = new Map();
19
+ const BUSY_BACKOFF_MS = 5 * 60 * 1000;
20
+ /** Last agent that sent a request (shown in the popup). */
21
+ let lastRequest = null;
22
+
23
+ /** Identity of this extension install = this Chrome profile. Loaded from storage before any connection. */
24
+ const identity = { instanceId: '', label: '' };
25
+ const identityReady = (async () => {
26
+ const o = await chrome.storage.local.get(['instanceId', 'profileLabel']);
27
+ let id = o.instanceId;
28
+ if (!id) {
29
+ id = crypto.randomUUID();
30
+ await chrome.storage.local.set({ instanceId: id });
31
+ }
32
+ identity.instanceId = id;
33
+ identity.label = (o.profileLabel || '').trim() || `profile-${id.slice(0, 4)}`;
34
+ })();
35
+ const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
8
36
 
9
37
  /** @type {Map<string, (out: unknown) => void>} */
10
38
  const approvalWaiters = new Map();
@@ -20,6 +48,7 @@ try {
20
48
  return;
21
49
  }
22
50
  networkRing.push({
51
+ tabId: d.tabId,
23
52
  url: (d.url || '').slice(0, 800),
24
53
  statusCode: d.statusCode ?? 0,
25
54
  method: d.method || 'GET',
@@ -36,7 +65,11 @@ try {
36
65
  console.warn('[browser-automation] webRequest init failed', e);
37
66
  }
38
67
 
39
- chrome.runtime.onInstalled.addListener(() => {
68
+ chrome.runtime.onInstalled.addListener((details) => {
69
+ if (details.reason === 'install') {
70
+ // The extension does nothing without the local helper, so show the setup guide once.
71
+ void chrome.tabs.create({ url: chrome.runtime.getURL('welcome.html') });
72
+ }
40
73
  chrome.storage.local.get('wsPort', (o) => {
41
74
  if (o.wsPort == null) {
42
75
  chrome.storage.local.set({ wsPort: DEFAULT_WS_PORT });
@@ -65,7 +98,9 @@ async function ensureInject(tabId, params) {
65
98
  async function rpcToTab(tabId, method, params) {
66
99
  await ensureInject(tabId, params);
67
100
  const fid = frameIdFromParams(params);
68
- const opts = fid !== undefined ? { frameId: fid } : {};
101
+ // Always target one frame explicitly: without frameId the message is delivered to EVERY frame
102
+ // that has the content script, and a child frame's same-named ref (e2) can answer first.
103
+ const opts = { frameId: fid !== undefined ? fid : 0 };
69
104
  const raw = await chrome.tabs.sendMessage(
70
105
  tabId,
71
106
  { type: 'ba-rpc', method, params },
@@ -74,6 +109,204 @@ async function rpcToTab(tabId, method, params) {
74
109
  return raw;
75
110
  }
76
111
 
112
+ // ---- iframe support -------------------------------------------------------------------------
113
+ // Content scripts run per frame. get_page_structure/get_page_text fan out to every frame; nodes from
114
+ // child frames get refs "f<frameId>.e<n>", which dispatchRequest routes back to the right frame.
115
+
116
+ /** tabId -> Map(frameId -> {x, y}) offset of the frame's viewport inside the top viewport. */
117
+ const frameOffsetsByTab = new Map();
118
+
119
+ const CAPTCHA_FRAME_RE = [
120
+ [/google\.com\/recaptcha|recaptcha\.net/i, 'recaptcha'],
121
+ [/hcaptcha\.com/i, 'hcaptcha'],
122
+ [/challenges\.cloudflare\.com|turnstile/i, 'turnstile'],
123
+ [/arkoselabs\.com|funcaptcha/i, 'arkose'],
124
+ [/geetest\.com/i, 'geetest']
125
+ ];
126
+
127
+ function detectCaptcha(urls) {
128
+ for (const u of urls) {
129
+ for (const [re, provider] of CAPTCHA_FRAME_RE) {
130
+ if (re.test(u)) {
131
+ return { provider, frameUrl: String(u).slice(0, 200) };
132
+ }
133
+ }
134
+ }
135
+ return null;
136
+ }
137
+
138
+ /** Inject content.js into every frame; returns child frame ids (excludes top frame 0). */
139
+ async function childFrameIds(tabId) {
140
+ try {
141
+ const res = await chrome.scripting.executeScript({
142
+ target: { tabId, allFrames: true },
143
+ files: ['content.js']
144
+ });
145
+ return [...new Set(res.map((r) => r.frameId))].filter((id) => id !== 0);
146
+ } catch {
147
+ return [];
148
+ }
149
+ }
150
+
151
+ async function frameRpc(tabId, frameId, method, params) {
152
+ try {
153
+ const r = await chrome.tabs.sendMessage(tabId, { type: 'ba-rpc', method, params }, { frameId });
154
+ return r?.ok ? r.result : null;
155
+ } catch {
156
+ return null;
157
+ }
158
+ }
159
+
160
+ function sameUrlLoose(a, b) {
161
+ if (a === b) {
162
+ return true;
163
+ }
164
+ try {
165
+ const ua = new URL(a);
166
+ const ub = new URL(b);
167
+ return ua.origin === ub.origin && ua.pathname === ub.pathname;
168
+ } catch {
169
+ return false;
170
+ }
171
+ }
172
+
173
+ async function structureWithFrames(tabId, params) {
174
+ const top = wrapRpcResult(await rpcToTab(tabId, 'get_page_structure', params));
175
+ if (params.includeFrames === false || frameIdFromParams(params) !== undefined) {
176
+ return top;
177
+ }
178
+ const ids = await childFrameIds(tabId);
179
+ if (!ids.length) {
180
+ return top;
181
+ }
182
+ const maxNodes = params.maxNodes ?? 500;
183
+ const viewportOnly = params.viewportOnly !== false;
184
+ const frames = [];
185
+ for (const fid of ids) {
186
+ const r = await frameRpc(tabId, fid, 'get_page_structure', {
187
+ ...params,
188
+ viewportOnly: false,
189
+ includeRects: true,
190
+ maxNodes
191
+ });
192
+ if (r) {
193
+ frames.push({ fid, ...r });
194
+ }
195
+ }
196
+ // Locate each frame's <iframe> element (in top or another frame) by URL to learn its offset + parent.
197
+ const iframeEntries = [];
198
+ const collect = (owner, nodes) => {
199
+ for (const n of nodes) {
200
+ if (n.frameSrc) {
201
+ iframeEntries.push({ owner, src: n.frameSrc, rect: n.rect, ref: n.ref, used: false });
202
+ }
203
+ }
204
+ };
205
+ collect(0, top.nodes);
206
+ for (const f of frames) {
207
+ collect(f.fid, f.nodes);
208
+ }
209
+ const link = new Map(); // fid -> entry
210
+ for (const f of frames) {
211
+ const entry =
212
+ iframeEntries.find((e) => !e.used && e.owner !== f.fid && e.src === f.url) ||
213
+ iframeEntries.find((e) => !e.used && e.owner !== f.fid && sameUrlLoose(e.src, f.url)) ||
214
+ iframeEntries.find((e) => !e.used && e.owner !== f.fid && e.src === 'about:srcdoc' && /^about:/.test(f.url));
215
+ if (entry) {
216
+ entry.used = true;
217
+ link.set(f.fid, entry);
218
+ }
219
+ }
220
+ const offsets = new Map([[0, { x: 0, y: 0 }]]);
221
+ const offsetOf = (fid, depth = 0) => {
222
+ if (offsets.has(fid)) {
223
+ return offsets.get(fid);
224
+ }
225
+ const e = link.get(fid);
226
+ if (!e || !e.rect || depth > 8) {
227
+ return null;
228
+ }
229
+ const parent = offsetOf(e.owner, depth + 1);
230
+ if (!parent) {
231
+ return null;
232
+ }
233
+ const o = { x: parent.x + e.rect.x, y: parent.y + e.rect.y };
234
+ offsets.set(fid, o);
235
+ return o;
236
+ };
237
+ frameOffsetsByTab.set(tabId, offsets);
238
+
239
+ const usedRefs = (owner) => new Set(iframeEntries.filter((e) => e.used && e.owner === owner).map((e) => e.ref));
240
+ const nodes = top.nodes.filter((n) => !usedRefs(0).has(n.ref)); // expanded iframes are replaced by their contents
241
+ let truncated = !!top.truncated;
242
+ const frameSummaries = [];
243
+ for (const f of frames) {
244
+ const off = offsetOf(f.fid);
245
+ if (!off && viewportOnly) {
246
+ continue; // hidden/off-screen frame (no visible <iframe> found for it)
247
+ }
248
+ const skip = usedRefs(f.fid);
249
+ let added = 0;
250
+ for (const n of f.nodes) {
251
+ if (skip.has(n.ref)) {
252
+ continue;
253
+ }
254
+ if (nodes.length >= maxNodes) {
255
+ truncated = true;
256
+ break;
257
+ }
258
+ const out = { ...n, ref: `f${f.fid}.${n.ref}`, frameId: f.fid, frameUrl: f.url };
259
+ delete out.frameSrc;
260
+ if (n.rect && off) {
261
+ out.rect = { x: n.rect.x + off.x, y: n.rect.y + off.y, w: n.rect.w, h: n.rect.h };
262
+ } else if (n.rect) {
263
+ out.rectSpace = 'frame';
264
+ }
265
+ if (params.includeRects === false) {
266
+ delete out.rect;
267
+ }
268
+ nodes.push(out);
269
+ added += 1;
270
+ }
271
+ if (added) {
272
+ frameSummaries.push({ prefix: `f${f.fid}`, url: f.url, nodeCount: added });
273
+ }
274
+ }
275
+ for (const n of nodes) {
276
+ delete n.frameSrc;
277
+ }
278
+ const captcha = detectCaptcha(frames.map((f) => f.url));
279
+ return {
280
+ ...top,
281
+ nodes,
282
+ truncated,
283
+ emptyHint: nodes.length === 0 ? top.emptyHint : null,
284
+ ...(frameSummaries.length ? { frames: frameSummaries } : {}),
285
+ ...(captcha
286
+ ? { captcha, humanRequired: true, hint: `Captcha detected (${captcha.provider}); it requires a human.` }
287
+ : {})
288
+ };
289
+ }
290
+
291
+ async function pageTextWithFrames(tabId, params) {
292
+ const top = wrapRpcResult(await rpcToTab(tabId, 'get_page_text', params));
293
+ if (params.includeFrames === false || frameIdFromParams(params) !== undefined) {
294
+ return top;
295
+ }
296
+ const max = params.maxChars ?? 50000;
297
+ let text = top.text;
298
+ for (const fid of await childFrameIds(tabId)) {
299
+ if (text.length > max) {
300
+ break;
301
+ }
302
+ const r = await frameRpc(tabId, fid, 'get_page_text', params);
303
+ if (r && r.text && r.text.trim()) {
304
+ text += `\n\n[iframe ${String(r.url).slice(0, 100)}]\n${r.text.trim()}`;
305
+ }
306
+ }
307
+ return { ...top, text: text.slice(0, max), truncated: top.truncated || text.length > max, sourceLength: text.length };
308
+ }
309
+
77
310
  async function assertTabExists(tabId, { storedTarget = false } = {}) {
78
311
  try {
79
312
  await chrome.tabs.get(tabId);
@@ -115,18 +348,59 @@ async function getActiveAutomatableTabId() {
115
348
  return tab.id;
116
349
  }
117
350
 
351
+ /**
352
+ * Tab policy (see policy.js): freeform | follow | pinned, set only from the popup. Stored as `tabMode` and `pinnedTabs`.
353
+ * Older installs: tabMode 'agent' = freeform, 'front' = follow, a stored `connectedTabId` = one pinned tab.
354
+ */
355
+ async function getPolicyState() {
356
+ const o = await chrome.storage.local.get(['tabMode', 'pinnedTabs', 'connectedTabId']);
357
+ const mode = policy.normalizeMode(o.tabMode, o.connectedTabId);
358
+ const pinned = Array.isArray(o.pinnedTabs)
359
+ ? o.pinnedTabs.filter(Number.isFinite)
360
+ : o.connectedTabId != null
361
+ ? [o.connectedTabId]
362
+ : [];
363
+ return { mode, pinned };
364
+ }
365
+
366
+ async function frontTabId() {
367
+ const [t] = await chrome.tabs.query({ active: true, lastFocusedWindow: true });
368
+ return t?.id ?? null;
369
+ }
370
+
371
+ let lastUsedPinned = null;
372
+
373
+ // A closed tab leaves the pinned set.
374
+ chrome.tabs.onRemoved.addListener(async (id) => {
375
+ const o = await chrome.storage.local.get('pinnedTabs');
376
+ if (Array.isArray(o.pinnedTabs) && o.pinnedTabs.includes(id)) {
377
+ await chrome.storage.local.set({ pinnedTabs: o.pinnedTabs.filter((x) => x !== id) });
378
+ }
379
+ if (lastUsedPinned === id) {
380
+ lastUsedPinned = null;
381
+ }
382
+ });
383
+
118
384
  async function getTargetTabId(params) {
119
- if (params.tabId != null && params.tabId !== '') {
120
- return assertTabExists(Number(params.tabId), { storedTarget: false });
385
+ const { mode, pinned } = await getPolicyState();
386
+ const front = mode === 'follow' ? await frontTabId() : null;
387
+ const requested = params.tabId != null && params.tabId !== '' ? Number(params.tabId) : null;
388
+ const r = policy.resolveTarget({
389
+ mode,
390
+ pinned,
391
+ front,
392
+ lastUsedPinned,
393
+ requested,
394
+ fromAgentDefault: params.tabFromAgentDefault === true
395
+ });
396
+ if (r.error) {
397
+ throw Object.assign(new Error(r.error.message), { code: r.error.code });
121
398
  }
122
- const { connectedTabId } = await chrome.storage.local.get('connectedTabId');
123
- if (connectedTabId != null) {
124
- try {
125
- await chrome.tabs.get(connectedTabId);
126
- return connectedTabId;
127
- } catch {
128
- await chrome.storage.local.remove('connectedTabId');
129
- }
399
+ if (mode === 'pinned' && r.tabId != null) {
400
+ lastUsedPinned = r.tabId;
401
+ }
402
+ if (r.tabId != null) {
403
+ return assertTabExists(r.tabId, { storedTarget: false });
130
404
  }
131
405
  return getActiveAutomatableTabId();
132
406
  }
@@ -157,19 +431,68 @@ function wireOpName(method) {
157
431
  return s;
158
432
  }
159
433
 
434
+ /** Viewport centre (top-frame CSS px) of a snapshot ref, accounting for iframe offset. */
435
+ async function refCentre(tabId, params, rawRef) {
436
+ let ref = String(rawRef).trim();
437
+ let fid = frameIdFromParams(params);
438
+ const m = /^@?f(\d+)\.(.+)$/.exec(ref);
439
+ if (m) {
440
+ fid = Number(m[1]);
441
+ ref = m[2];
442
+ } else if (ref.startsWith('@')) {
443
+ ref = ref.slice(1);
444
+ }
445
+ const r = await rpcToTab(tabId, 'get_ref_presentation_rect', { ref, frameId: fid });
446
+ if (!r?.ok) {
447
+ return wrapRpcResult(r);
448
+ }
449
+ const off = fid !== undefined ? frameOffsetsByTab.get(tabId)?.get(fid) : { x: 0, y: 0 };
450
+ if (!off) {
451
+ throw Object.assign(new Error('Frame position unknown — take a fresh browser_snapshot, then retry.'), { code: 'stale_ref' });
452
+ }
453
+ return cdp.centreFromRect(r.result, off);
454
+ }
455
+
160
456
  /**
161
457
  * @param {string} method
162
458
  * @param {Record<string, unknown>} params
163
459
  * @param {string} requestId WebSocket request id (needed for ask_confirmation)
164
460
  */
461
+ /** Page size and pixel ratio, so a screenshot can be mapped back to page (CSS) coordinates for clicks. */
462
+ async function viewportInfo(tabId) {
463
+ try {
464
+ const [r] = await chrome.scripting.executeScript({
465
+ target: { tabId },
466
+ func: () => ({
467
+ cssWidth: window.innerWidth,
468
+ cssHeight: window.innerHeight,
469
+ dpr: window.devicePixelRatio,
470
+ scrollX: Math.round(window.scrollX),
471
+ scrollY: Math.round(window.scrollY)
472
+ })
473
+ });
474
+ return r?.result ?? null;
475
+ } catch {
476
+ return null;
477
+ }
478
+ }
479
+
165
480
  async function dispatchRequest(method, params, requestId) {
166
481
  const op = wireOpName(method);
167
482
 
483
+ // Tab policy: in follow and pinned mode some calls are refused and lists show only allowed tabs.
484
+ const pol = await getPolicyState();
485
+ const refusal = policy.blockedOp(pol.mode, op);
486
+ if (refusal) {
487
+ throw Object.assign(new Error(refusal), { code: 'tab_not_allowed' });
488
+ }
489
+ const allowed = policy.allowedSet(pol.mode, pol.pinned, pol.mode === 'follow' ? await frontTabId() : null);
490
+
168
491
  if (op === 'get_tabs') {
169
492
  const q = params.activeOnly
170
493
  ? { active: true, currentWindow: true }
171
494
  : {};
172
- const tabs = await chrome.tabs.query(q);
495
+ const tabs = (await chrome.tabs.query(q)).filter((t) => !allowed || allowed.has(t.id));
173
496
  return {
174
497
  tabs: tabs.map((t) => ({
175
498
  tabId: t.id,
@@ -193,12 +516,49 @@ async function dispatchRequest(method, params, requestId) {
193
516
  };
194
517
  }
195
518
 
519
+ if (op === 'window_list') {
520
+ const out = await cdp.windowList();
521
+ if (!allowed) {
522
+ return out;
523
+ }
524
+ const windows = out.windows
525
+ .map((w) => ({ ...w, tabs: w.tabs.filter((t) => allowed.has(t.tabId)) }))
526
+ .filter((w) => w.tabs.length > 0);
527
+ return { windows };
528
+ }
529
+ if (op === 'window_open') {
530
+ return cdp.windowOpen(params);
531
+ }
532
+ if (op === 'window_resize') {
533
+ return cdp.windowResize(Number(params.windowId), params);
534
+ }
535
+ if (op === 'window_focus') {
536
+ return cdp.windowFocus(Number(params.windowId));
537
+ }
538
+ if (op === 'get_downloads') {
539
+ return cdp.downloadsRecent(Number(params.waitMs) || 0);
540
+ }
541
+
196
542
  if (op === 'get_network_log') {
197
543
  const max = Math.min(Number(params.maxEntries) || 50, NETWORK_CAP);
198
- return { entries: networkRing.slice(-max) };
544
+ const entries = networkRing.filter((e) => !allowed || allowed.has(e.tabId));
545
+ return { entries: entries.slice(-max) };
199
546
  }
200
547
 
201
548
  const tabId = await getTargetTabId(params);
549
+ if (lastRequest) {
550
+ lastRequest.tabId = tabId;
551
+ }
552
+
553
+ // Snapshot-style refs: "@e3" -> "e3"; frame refs "f4.e2" -> frameId 4 + ref "e2".
554
+ if (typeof params.ref === 'string') {
555
+ const m = /^@?f(\d+)\.(.+)$/.exec(params.ref.trim());
556
+ if (m) {
557
+ params = { ...params, frameId: Number(m[1]), ref: m[2] };
558
+ } else if (params.ref.startsWith('@')) {
559
+ params = { ...params, ref: params.ref.slice(1) };
560
+ }
561
+ }
202
562
 
203
563
  switch (op) {
204
564
  case 'navigate': {
@@ -218,17 +578,27 @@ async function dispatchRequest(method, params, requestId) {
218
578
  };
219
579
  }
220
580
  case 'get_current_tab': {
581
+ // The orientation call: where am I, in which profile, and what am I allowed to do.
221
582
  const t = await chrome.tabs.get(tabId);
583
+ const win = await chrome.windows.get(t.windowId).catch(() => null);
222
584
  return {
223
585
  tabId: t.id,
224
586
  title: t.title,
225
587
  url: t.url,
226
588
  windowId: t.windowId,
227
- active: t.active
589
+ active: t.active,
590
+ scriptable: /^https?:/i.test(t.url || ''),
591
+ profile: identity.label,
592
+ tabMode: pol.mode,
593
+ tabRule: policy.modeSummary(pol.mode),
594
+ pinnedTabs: pol.mode === 'pinned' ? pol.pinned : undefined,
595
+ window: win ? { state: win.state, focused: win.focused } : undefined
228
596
  };
229
597
  }
230
598
  case 'get_page_text':
599
+ return pageTextWithFrames(tabId, params);
231
600
  case 'get_page_structure':
601
+ return structureWithFrames(tabId, params);
232
602
  case 'click_element':
233
603
  case 'fill_input':
234
604
  case 'press_key':
@@ -252,13 +622,22 @@ async function dispatchRequest(method, params, requestId) {
252
622
  return wrapRpcResult(presR);
253
623
  }
254
624
  const pres = presR.result;
255
- const insetsR = await rpcToTab(tabId, 'get_viewport_chrome_insets', params);
625
+ // Chrome insets must come from the TOP frame (an iframe's innerHeight is not the window's).
626
+ const insetsR = await rpcToTab(tabId, 'get_viewport_chrome_insets', { ...params, frameId: undefined });
256
627
  if (!insetsR?.ok) {
257
628
  return wrapRpcResult(insetsR);
258
629
  }
259
630
  const insets = insetsR.result;
260
- const screenX = win.left + insets.left + pres.x;
261
- const screenY = win.top + insets.top + pres.y;
631
+ const fid = frameIdFromParams(params);
632
+ const frameOff = fid !== undefined ? frameOffsetsByTab.get(tabId)?.get(fid) : { x: 0, y: 0 };
633
+ if (!frameOff) {
634
+ throw Object.assign(
635
+ new Error('Frame position unknown — take a fresh browser_snapshot, then retry.'),
636
+ { code: 'stale_ref' }
637
+ );
638
+ }
639
+ const screenX = win.left + insets.left + frameOff.x + pres.x;
640
+ const screenY = win.top + insets.top + frameOff.y + pres.y;
262
641
  const width = pres.w;
263
642
  const height = pres.h;
264
643
  return {
@@ -339,6 +718,7 @@ async function dispatchRequest(method, params, requestId) {
339
718
  fmt === 'png'
340
719
  ? { format: 'png' }
341
720
  : { format: 'jpeg', quality: params.quality ?? 85 };
721
+ await soft.paceCapture();
342
722
  const dataUrl = await chrome.tabs.captureVisibleTab(t.windowId, capOpts);
343
723
  const mimeType = fmt === 'png' ? 'image/png' : 'image/jpeg';
344
724
  const base64 = dataUrl.split(',')[1] || '';
@@ -411,6 +791,15 @@ async function dispatchRequest(method, params, requestId) {
411
791
  return inj.result;
412
792
  }
413
793
  case 'evaluate_expression': {
794
+ // Running a script string sent by an agent is the most powerful call here, so it is off unless the user
795
+ // turns it on in the popup (Advanced). The server also needs BROWSER_AUTOMATION_ALLOW_EVAL=1.
796
+ const { allowEval } = await chrome.storage.local.get('allowEval');
797
+ if (allowEval !== true) {
798
+ throw Object.assign(
799
+ new Error('Script evaluation is switched off in the extension. Turn on "Allow page script evaluation" in the popup (Advanced) to use browser_evaluate.'),
800
+ { code: 'eval_disabled' }
801
+ );
802
+ }
414
803
  const expr = String(params.expression ?? '').trim();
415
804
  if (!expr) {
416
805
  throw Object.assign(new Error('expression required'), { code: 'invalid_params' });
@@ -451,19 +840,106 @@ async function dispatchRequest(method, params, requestId) {
451
840
  return inj.result;
452
841
  }
453
842
  case 'screenshot_visible': {
843
+ // mode: 'auto' (default) uses the fast tab capture only when the tab is the active tab of a focused
844
+ // window, otherwise CDP (works hidden/covered). 'visible' forces tab capture (focuses the window).
845
+ // 'cdp' forces CDP. fullPage and clip are CDP-only.
454
846
  const t = await chrome.tabs.get(tabId);
847
+ let win = await chrome.windows.get(t.windowId);
848
+ // A minimized window never paints, so nothing can capture it. With restoreWindow:true we un-minimize it
849
+ // without focusing it, capture, and minimize it again (it flashes on screen for about half a second).
850
+ const wasMinimized = win.state === 'minimized';
851
+ if (wasMinimized) {
852
+ if (params.restoreWindow !== true) {
853
+ throw Object.assign(
854
+ new Error(
855
+ 'The tab is in a minimized window, which does not paint. Pass restoreWindow:true (briefly restores it and minimizes it again), ' +
856
+ 'or use browser_window_open for a dedicated window.'
857
+ ),
858
+ { code: 'tab_not_painting' }
859
+ );
860
+ }
861
+ await chrome.windows.update(t.windowId, { state: 'normal', focused: false });
862
+ await chrome.tabs.update(tabId, { active: true });
863
+ await sleep(450);
864
+ win = await chrome.windows.get(t.windowId);
865
+ }
866
+ try {
867
+ const mode = params.mode === 'visible' || params.mode === 'cdp' ? params.mode : 'auto';
868
+ const fmt = params.format === 'png' ? 'png' : 'jpeg';
869
+ // DevTools capture only when the call needs it (full page, clip, or mode:'cdp'). Everything else uses the
870
+ // tab capture, which shows no Chrome debugging bar.
871
+ const wantCdp = mode === 'cdp' || params.fullPage === true || params.clip != null;
872
+ if (wantCdp) {
873
+ const shot = await cdp.cdpScreenshot(tabId, {
874
+ format: fmt,
875
+ quality: params.quality ?? 80,
876
+ fullPage: params.fullPage === true,
877
+ clip: params.clip
878
+ });
879
+ return { ...shot, imageScale: 1, viewport: await viewportInfo(tabId) };
880
+ }
455
881
  await chrome.tabs.update(tabId, { active: true });
456
882
  await chrome.windows.update(t.windowId, { focused: true });
457
- const fmt = params.format === 'png' ? 'png' : 'jpeg';
458
883
  const capOpts =
459
884
  fmt === 'png'
460
885
  ? { format: 'png' }
461
886
  : { format: 'jpeg', quality: params.quality ?? 80 };
887
+ await soft.paceCapture();
462
888
  const dataUrl = await chrome.tabs.captureVisibleTab(t.windowId, capOpts);
463
889
  const mimeType = fmt === 'png' ? 'image/png' : 'image/jpeg';
464
890
  const base64 = dataUrl.split(',')[1] || '';
465
- return { mimeType, dataBase64: base64 };
891
+ const viewport = await viewportInfo(tabId);
892
+ return { mimeType, dataBase64: base64, method: 'tab', restoredWindow: wasMinimized, imageScale: viewport?.dpr ?? 1, viewport };
893
+ } finally {
894
+ if (wasMinimized) {
895
+ await chrome.windows.update(t.windowId, { state: 'minimized' }).catch(() => undefined);
896
+ }
897
+ }
466
898
  }
899
+ case 'set_viewport':
900
+ return params.engine === 'cdp' ? cdp.setViewport(tabId, params) : soft.softSetViewport(tabId, params);
901
+ case 'set_color_scheme':
902
+ return cdp.setColorScheme(tabId, String(params.scheme ?? ''));
903
+ case 'reset_emulation': {
904
+ const out = await soft.softResetViewport(tabId);
905
+ if (await cdp.enabled()) {
906
+ await cdp.resetEmulation(tabId);
907
+ }
908
+ return out;
909
+ }
910
+ case 'pointer': {
911
+ let { x, y, toX, toY } = params;
912
+ if (typeof params.ref === 'string' && (x == null || y == null)) {
913
+ const c = await refCentre(tabId, params, params.ref);
914
+ x = c.x;
915
+ y = c.y;
916
+ }
917
+ if (typeof params.toRef === 'string' && (toX == null || toY == null)) {
918
+ const c = await refCentre(tabId, { ...params, frameId: undefined }, params.toRef);
919
+ toX = c.x;
920
+ toY = c.y;
921
+ }
922
+ const pp = { ...params, x, y, toX, toY };
923
+ return params.engine === 'cdp' ? cdp.pointer(tabId, pp) : soft.softPointer(tabId, pp);
924
+ }
925
+ case 'clipboard_read':
926
+ return params.engine === 'cdp' ? cdp.clipboardRead(tabId) : soft.softClipboardRead(tabId);
927
+ case 'clipboard_write':
928
+ return params.engine === 'cdp'
929
+ ? cdp.clipboardWrite(tabId, params.text ?? '')
930
+ : soft.softClipboardWrite(tabId, params.text ?? '');
931
+ case 'record_start':
932
+ return params.engine === 'cdp' ? cdp.recordStart(tabId, params) : soft.softRecordStart(tabId, params);
933
+ case 'record_stop':
934
+ return soft.hasSoftRecording(tabId) ? soft.softRecordStop(tabId) : cdp.recordStop(tabId);
935
+ case 'record_fetch': {
936
+ const n = Math.min(Number(params.count) || 10, 25);
937
+ return soft.hasSoftRecording(tabId)
938
+ ? soft.softRecordFetch(tabId, Number(params.from) || 0, n)
939
+ : cdp.recordFetch(tabId, Number(params.from) || 0, n);
940
+ }
941
+ case 'record_discard':
942
+ return soft.hasSoftRecording(tabId) ? soft.softRecordDiscard(tabId) : cdp.recordDiscard(tabId);
467
943
  case 'wait': {
468
944
  const ms = Math.min(params.ms ?? 500, 30000);
469
945
  await new Promise((r) => setTimeout(r, ms));
@@ -553,31 +1029,65 @@ chrome.runtime.onMessage.addListener((msg, _s, sendResponse) => {
553
1029
  return;
554
1030
  }
555
1031
  if (msg?.type === 'connect') {
1032
+ void chrome.storage.session.remove('autoConnectOff');
1033
+ void chrome.storage.local.remove('autoConnect');
556
1034
  connectWs(msg.port, msg.token)
557
1035
  .then(() => sendResponse({ ok: true }))
558
1036
  .catch((e) => sendResponse({ ok: false, error: String(e) }));
559
1037
  return true;
560
1038
  }
561
1039
  if (msg?.type === 'disconnect') {
1040
+ // A deliberate disconnect turns auto-connect off until the next manual Connect or until Chrome restarts.
1041
+ void chrome.storage.session.set({ autoConnectOff: true });
562
1042
  disconnectWs();
563
1043
  sendResponse({ ok: true });
564
1044
  return;
565
1045
  }
1046
+ if (msg?.type === 'set-label') {
1047
+ const label = String(msg.label ?? '').trim().slice(0, 60);
1048
+ chrome.storage.local
1049
+ .set({ profileLabel: label })
1050
+ .then(async () => {
1051
+ await identityReady;
1052
+ identity.label = label || `profile-${identity.instanceId.slice(0, 4)}`;
1053
+ // The label is sent when a connection opens, so reconnect to apply it.
1054
+ disconnectWs();
1055
+ setTimeout(() => void autoConnectTick(), 300);
1056
+ sendResponse({ ok: true, label: identity.label });
1057
+ })
1058
+ .catch((e) => sendResponse({ ok: false, error: String(e) }));
1059
+ return true;
1060
+ }
566
1061
  if (msg?.type === 'get-ws-status') {
567
- sendResponse({ connected: ws != null && ws.readyState === 1 });
568
- return;
1062
+ void (async () => {
1063
+ sendResponse({
1064
+ label: identity.label,
1065
+ autoConnectOff: (await chrome.storage.session.get('autoConnectOff')).autoConnectOff === true,
1066
+ everConnected: (await chrome.storage.local.get('everConnected')).everConnected === true,
1067
+ lastScanAt: diag.lastTickAt,
1068
+ lastScanError: diag.lastError,
1069
+ failingSince: diag.failingSince,
1070
+ tabMode: (await getPolicyState()).mode,
1071
+ pinned: (await getPolicyState()).pinned,
1072
+ instanceId: identity.instanceId,
1073
+ connected: sockets.size > 0,
1074
+ ports: [...sockets.keys()].sort((x, y) => x - y),
1075
+ lastRequest
1076
+ });
1077
+ })();
1078
+ return true;
569
1079
  }
570
1080
  return false;
571
1081
  });
572
1082
 
573
1083
  function disconnectWs() {
574
- if (ws) {
1084
+ for (const [port, sock] of sockets) {
575
1085
  try {
576
- ws.close();
1086
+ sock.close();
577
1087
  } catch {
578
1088
  /* ignore */
579
1089
  }
580
- ws = null;
1090
+ sockets.delete(port);
581
1091
  }
582
1092
  }
583
1093
 
@@ -585,7 +1095,6 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
585
1095
  return new Promise((resolve, reject) => {
586
1096
  const url = `ws://127.0.0.1:${port}`;
587
1097
  const socket = new WebSocket(url);
588
- ws = socket;
589
1098
  let opened = false;
590
1099
  const failTimer = setTimeout(() => {
591
1100
  if (!opened) {
@@ -600,6 +1109,10 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
600
1109
  socket.onopen = () => {
601
1110
  opened = true;
602
1111
  clearTimeout(failTimer);
1112
+ sockets.set(port, socket);
1113
+ diag.failingSince = null;
1114
+ void chrome.storage.local.set({ everConnected: true });
1115
+ chrome.windows.getLastFocused().then((w) => w.focused && socket.send(JSON.stringify({ proto: 1, type: 'event', id: crypto.randomUUID(), payload: { kind: 'focus', windowId: w.id } }))).catch(() => undefined);
603
1116
  socket.send(
604
1117
  JSON.stringify({
605
1118
  proto: 1,
@@ -609,60 +1122,110 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
609
1122
  token: token || '',
610
1123
  extensionId: chrome.runtime.id,
611
1124
  client: 'browser_automation_ext',
612
- clientVersion: '0.1.0'
1125
+ clientVersion: chrome.runtime.getManifest().version,
1126
+ instanceId: identity.instanceId,
1127
+ label: identity.label
613
1128
  }
614
1129
  })
615
1130
  );
616
- resolve(undefined);
1131
+ resolve(port);
617
1132
  };
618
1133
  socket.onerror = () => {
619
1134
  clearTimeout(failTimer);
620
1135
  reject(new Error('websocket_error'));
621
1136
  };
622
- socket.onclose = () => {
623
- if (ws === socket) {
624
- ws = null;
1137
+ socket.onclose = (ev) => {
1138
+ if (ev.code === 4000) {
1139
+ // The server already has an extension (another Chrome profile). Leave it alone for a while instead of
1140
+ // reconnecting every few seconds.
1141
+ busyUntil.set(port, Date.now() + BUSY_BACKOFF_MS);
1142
+ }
1143
+ if (sockets.get(port) === socket) {
1144
+ sockets.delete(port);
1145
+ fastRetryUntil = Date.now() + 120_000;
1146
+ diag.failingSince = Date.now();
1147
+ if (ev.code !== 4000) {
1148
+ // Server restarted or went away: try again shortly (alarm is the backup if this worker sleeps).
1149
+ setTimeout(() => void autoConnectTick(), 3000);
1150
+ }
625
1151
  }
626
1152
  };
627
1153
  socket.onmessage = (ev) => {
628
- void handleServerJson(ev.data);
1154
+ void handleServerJson(ev.data, socket, port);
629
1155
  };
630
1156
  });
631
1157
  }
632
1158
 
633
- async function connectFromPort(startPort, token) {
634
- let lastErr;
635
- const attempts = Math.max(1, WS_PORT_FALLBACK_ATTEMPTS);
636
- for (let i = 0; i < attempts; i++) {
637
- const candidate = startPort + i;
638
- if (candidate >= 65536) {
639
- break;
640
- }
641
- try {
642
- await connectWsRaw(candidate, token);
643
- await chrome.storage.local.set({ wsPort: candidate });
644
- return candidate;
645
- } catch (e) {
646
- lastErr = e;
1159
+ /**
1160
+ * Ports to look at. Chrome writes one entry on the extension's Errors page for every WebSocket attempt that is refused,
1161
+ * so this is built to attempt as little as possible:
1162
+ * - the standard slots the installer and the command line use (Cursor 18791, Claude Desktop 18792, command line 18795),
1163
+ * - ports that worked before, and the port the user typed in the popup,
1164
+ * - a wider range (18791 to 18800) ONLY when the user presses Connect.
1165
+ * A port that refuses is not retried right away: the wait doubles each time (10 s, 20 s, 40 s ... up to 5 min), and a
1166
+ * longer base wait applies while something is already connected. Repeated refusals of the same port show up as one
1167
+ * entry with a count, so a stopped helper costs a handful of lines, not a wall of them.
1168
+ */
1169
+ const STANDARD_PORTS = [18791, 18792, 18795];
1170
+ const WIDE_SCAN_END = 18800;
1171
+ const BACKOFF_BASE_MS = 10_000;
1172
+ const BACKOFF_BASE_WHEN_CONNECTED_MS = 60_000;
1173
+ const BACKOFF_MAX_MS = 5 * 60_000;
1174
+ /** port -> { n: failed attempts in a row, next: earliest time to try again } */
1175
+ const portBackoff = new Map();
1176
+
1177
+ function candidatePorts(startPort, known, wide) {
1178
+ const set = new Set([...STANDARD_PORTS, ...known]);
1179
+ if (Number.isFinite(startPort)) {
1180
+ set.add(startPort);
1181
+ }
1182
+ if (wide) {
1183
+ for (let p = DEFAULT_WS_PORT; p <= WIDE_SCAN_END; p++) {
1184
+ set.add(p);
647
1185
  }
648
1186
  }
649
- throw lastErr ?? new Error('websocket_error');
1187
+ return [...set].filter((p) => p > 0 && p < 65536);
650
1188
  }
651
1189
 
652
- async function connectWs(port, token) {
653
- disconnectWs();
654
- try {
655
- await connectFromPort(port, token);
656
- } catch (firstErr) {
657
- if (port !== DEFAULT_WS_PORT) {
658
- await connectFromPort(DEFAULT_WS_PORT, token);
1190
+ /**
1191
+ * Connect to every live bridge among the candidate ports, skipping ports we already hold and ports that are waiting out a
1192
+ * back-off. Resolves to the sorted list of connected ports; throws if none are connected.
1193
+ */
1194
+ async function connectAll(startPort, token, opts = {}) {
1195
+ const { knownPorts = [] } = await chrome.storage.local.get('knownPorts');
1196
+ const manual = opts.full === true;
1197
+ if (manual) {
1198
+ portBackoff.clear();
1199
+ }
1200
+ const now = Date.now();
1201
+ const todo = candidatePorts(startPort, knownPorts, manual).filter(
1202
+ (p) => !sockets.has(p) && (busyUntil.get(p) ?? 0) < now && (portBackoff.get(p)?.next ?? 0) <= now
1203
+ );
1204
+ const results = await Promise.allSettled(todo.map((p) => connectWsRaw(p, token, 2500)));
1205
+ const base = sockets.size > 0 ? BACKOFF_BASE_WHEN_CONNECTED_MS : BACKOFF_BASE_MS;
1206
+ todo.forEach((p, i) => {
1207
+ if (results[i].status === 'fulfilled') {
1208
+ portBackoff.delete(p);
659
1209
  } else {
660
- throw firstErr;
1210
+ const n = (portBackoff.get(p)?.n ?? 0) + 1;
1211
+ portBackoff.set(p, { n, next: Date.now() + Math.min(BACKOFF_MAX_MS, base * 2 ** (n - 1)) });
661
1212
  }
1213
+ });
1214
+ if (sockets.size === 0) {
1215
+ throw new Error('websocket_error');
662
1216
  }
1217
+ const ports = [...sockets.keys()].sort((x, y) => x - y);
1218
+ await chrome.storage.local.set({ wsPort: ports[0], knownPorts: [...new Set([...knownPorts, ...ports])].slice(-8) });
1219
+ return ports;
663
1220
  }
664
1221
 
665
- async function handleServerJson(raw) {
1222
+ async function connectWs(port, token) {
1223
+ await identityReady;
1224
+ busyUntil.clear(); // a manual Connect retries everything
1225
+ return connectAll(port, token, { full: true });
1226
+ }
1227
+
1228
+ async function handleServerJson(raw, socket, port) {
666
1229
  const msg = JSON.parse(raw);
667
1230
  if (msg.type === 'session.opened') {
668
1231
  const bp = msg.payload?.bridgePort;
@@ -673,15 +1236,16 @@ async function handleServerJson(raw) {
673
1236
  return;
674
1237
  }
675
1238
  if (msg.type === 'ping' && msg.id) {
676
- ws?.send(JSON.stringify({ proto: 1, type: 'pong', id: msg.id, payload: {} }));
1239
+ socket.send(JSON.stringify({ proto: 1, type: 'pong', id: msg.id, payload: {} }));
677
1240
  return;
678
1241
  }
679
1242
  if (msg.type === 'request' && msg.id && msg.payload) {
680
1243
  const requestId = msg.id;
681
1244
  const { method, params } = msg.payload;
1245
+ lastRequest = { port, method: String(method), at: Date.now() };
682
1246
  try {
683
1247
  const result = await dispatchRequest(method, params ?? {}, requestId);
684
- ws?.send(
1248
+ socket.send(
685
1249
  JSON.stringify({
686
1250
  proto: 1,
687
1251
  type: 'response',
@@ -691,7 +1255,7 @@ async function handleServerJson(raw) {
691
1255
  );
692
1256
  } catch (e) {
693
1257
  const code = e.code || 'internal';
694
- ws?.send(
1258
+ socket.send(
695
1259
  JSON.stringify({
696
1260
  proto: 1,
697
1261
  type: 'response',
@@ -708,4 +1272,154 @@ async function handleServerJson(raw) {
708
1272
  }
709
1273
  return;
710
1274
  }
711
- }
1275
+ }
1276
+
1277
+ // ---- just-in-time DevTools grant --------------------------------------------------------------
1278
+ // When a call needs the debugger (dark-mode emulation, full-page or clipped screenshots, engine:'cdp') and the optional
1279
+ // permission is not granted yet, open a small window with one Allow button: the click is the user gesture Chrome needs.
1280
+ // Chrome remembers the grant, so this happens once. The debugger is attached only for the call (see cdp.js withCdp).
1281
+
1282
+ let grantWaiters = [];
1283
+ let grantWindowId = null;
1284
+ let grantTimer = null;
1285
+
1286
+ function finishGrant(ok) {
1287
+ const waiters = grantWaiters;
1288
+ grantWaiters = [];
1289
+ clearTimeout(grantTimer);
1290
+ for (const r of waiters) {
1291
+ r(ok);
1292
+ }
1293
+ if (grantWindowId != null) {
1294
+ const id = grantWindowId;
1295
+ grantWindowId = null;
1296
+ chrome.windows.remove(id).catch(() => undefined);
1297
+ }
1298
+ }
1299
+
1300
+ async function requestDevtoolsGrant() {
1301
+ if (await cdp.enabled()) {
1302
+ return true;
1303
+ }
1304
+ const { autoDevtools } = await chrome.storage.local.get('autoDevtools');
1305
+ if (autoDevtools === false) {
1306
+ return false;
1307
+ }
1308
+ return new Promise((resolve) => {
1309
+ grantWaiters.push(resolve);
1310
+ if (grantWaiters.length === 1) {
1311
+ chrome.windows
1312
+ .create({
1313
+ url: chrome.runtime.getURL('devtools-grant.html'),
1314
+ type: 'popup',
1315
+ width: 600,
1316
+ height: 420,
1317
+ focused: true
1318
+ })
1319
+ .then((w) => {
1320
+ grantWindowId = w.id ?? null;
1321
+ })
1322
+ .catch(() => finishGrant(false));
1323
+ grantTimer = setTimeout(() => finishGrant(false), 50000);
1324
+ }
1325
+ });
1326
+ }
1327
+ cdp.setGrantRequester(requestDevtoolsGrant);
1328
+
1329
+ chrome.runtime.onMessage.addListener((msg) => {
1330
+ if (msg?.type === 'devtools-grant-result') {
1331
+ finishGrant(msg.granted === true);
1332
+ }
1333
+ return false;
1334
+ });
1335
+ chrome.windows.onRemoved.addListener((id) => {
1336
+ if (id === grantWindowId) {
1337
+ grantWindowId = null;
1338
+ finishGrant(false);
1339
+ }
1340
+ });
1341
+
1342
+ // ---- focus: tell every server when a window of this profile gets focus ----------------------------
1343
+ // A server uses it to send commands that name no profile to the profile the user used last.
1344
+ function announceFocus(windowId) {
1345
+ if (windowId === chrome.windows.WINDOW_ID_NONE) {
1346
+ return;
1347
+ }
1348
+ const frame = JSON.stringify({ proto: 1, type: 'event', id: crypto.randomUUID(), payload: { kind: 'focus', windowId } });
1349
+ for (const sock of sockets.values()) {
1350
+ try {
1351
+ sock.send(frame);
1352
+ } catch {
1353
+ /* closing */
1354
+ }
1355
+ }
1356
+ }
1357
+ chrome.windows.onFocusChanged.addListener(announceFocus);
1358
+
1359
+ // ---- auto connect -----------------------------------------------------------------------------
1360
+ // Find the local bridge without the popup: try the last-used port (then the default), scanning upward
1361
+ // through WS_PORT_FALLBACK_ATTEMPTS ports. Runs when Chrome starts, when this worker wakes, after a drop,
1362
+ // and every 30 s while disconnected. Turned off by a manual Disconnect.
1363
+
1364
+ let autoConnecting = false;
1365
+ /** Shown in the popup so a missing connection can be explained. */
1366
+ const diag = { lastTickAt: 0, lastError: null, failingSince: null };
1367
+ let fastRetryUntil = Date.now() + 120_000;
1368
+ let lastScanAt = 0;
1369
+ /** While at least one server is connected, only look for more every 5 minutes (each refused port is logged by Chrome). */
1370
+ const RESCAN_WHEN_CONNECTED_MS = 60 * 1000;
1371
+
1372
+ async function autoConnectTick() {
1373
+ await identityReady;
1374
+ if (autoConnecting) {
1375
+ return;
1376
+ }
1377
+ if (sockets.size > 0 && Date.now() - lastScanAt < RESCAN_WHEN_CONNECTED_MS) {
1378
+ return;
1379
+ }
1380
+ const o = await chrome.storage.local.get(['wsPort', 'sessionToken']);
1381
+ if ((await chrome.storage.session.get('autoConnectOff')).autoConnectOff === true) {
1382
+ return;
1383
+ }
1384
+ autoConnecting = true;
1385
+ lastScanAt = Date.now();
1386
+ diag.lastTickAt = Date.now();
1387
+ try {
1388
+ const before = sockets.size;
1389
+ await connectAll(Number(o.wsPort) || DEFAULT_WS_PORT, o.sessionToken || '');
1390
+ if (sockets.size !== before) {
1391
+ console.info('[browser-automation] connected to ports', [...sockets.keys()]);
1392
+ }
1393
+ diag.lastError = null;
1394
+ if (sockets.size > 0) {
1395
+ diag.failingSince = null;
1396
+ } else if (diag.failingSince == null) {
1397
+ diag.failingSince = Date.now();
1398
+ }
1399
+ } catch (e) {
1400
+ /* no server yet: the alarm tries again */
1401
+ diag.lastError = String(e?.message ?? e);
1402
+ if (diag.failingSince == null) {
1403
+ diag.failingSince = Date.now();
1404
+ }
1405
+ } finally {
1406
+ autoConnecting = false;
1407
+ // While nothing is connected, look again every 10 s for the first two minutes (the 30 s alarm is the backup).
1408
+ if (sockets.size === 0 && Date.now() < fastRetryUntil) {
1409
+ setTimeout(() => void autoConnectTick(), 10_000);
1410
+ }
1411
+ }
1412
+ }
1413
+
1414
+ try {
1415
+ chrome.alarms.create('ba-reconnect', { periodInMinutes: 0.5 });
1416
+ chrome.alarms.onAlarm.addListener((a) => {
1417
+ if (a.name === 'ba-reconnect') {
1418
+ void autoConnectTick();
1419
+ }
1420
+ });
1421
+ } catch (e) {
1422
+ console.warn('[browser-automation] alarms unavailable', e);
1423
+ }
1424
+ chrome.runtime.onStartup.addListener(() => void autoConnectTick());
1425
+ void autoConnectTick();