@agentruntime-labs/browser-agent 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +64 -34
- package/bin/browser-agent.js +1 -0
- package/cli/control.mjs +477 -0
- package/cli/daemon.mjs +143 -0
- package/cli/hosts.mjs +1 -0
- package/cli/main.mjs +24 -0
- package/cli/mcp.mjs +1 -0
- package/docs/ARCHITECTURE.md +73 -0
- package/docs/DESKTOP_MCP.md +41 -0
- package/docs/EXTENSION_MANIFEST.md +163 -0
- package/docs/HYBRID.md +12 -0
- package/docs/MANAGED_BROWSER.md +111 -0
- package/docs/MCP_TOOLS.md +486 -0
- package/docs/MESSAGES.md +272 -0
- package/docs/MVP.md +76 -0
- package/docs/PRIVACY_POLICY.md +50 -0
- package/docs/README.md +27 -0
- package/docs/REFERENCE_PROJECTS.md +24 -0
- package/docs/RELEASE_CHECKLIST.md +49 -0
- package/docs/REPEATED_DOCS.md +27 -0
- package/docs/SAFETY.md +92 -0
- package/docs/SHELL_QUOTING.md +30 -0
- package/docs/SPLIT_REPOS.md +46 -0
- package/docs/THIRD_PARTY_LICENSES.md +182 -0
- package/docs/VISION.md +38 -0
- package/docs/WEB_STORE_SUBMISSION.md +117 -0
- package/extension/approval.js +1 -0
- package/extension/assets/agentruntime-logo.svg +77 -0
- package/extension/assets/agentruntime-mark-512.png +0 -0
- package/extension/background.js +776 -62
- package/extension/cdp.js +577 -0
- package/extension/constants.js +1 -0
- package/extension/content.js +69 -9
- package/extension/devtools-grant.html +31 -0
- package/extension/devtools-grant.js +29 -0
- package/extension/icons/icon128.png +0 -0
- package/extension/icons/icon16.png +0 -0
- package/extension/icons/icon32.png +0 -0
- package/extension/icons/icon48.png +0 -0
- package/extension/manifest.json +50 -23
- package/extension/policy.js +102 -0
- package/extension/popup.html +190 -56
- package/extension/popup.js +376 -166
- package/extension/soft.js +301 -0
- package/extension/welcome.html +91 -0
- package/extension/welcome.js +13 -0
- package/package.json +6 -2
- package/server/dist/arBrowserPaths.d.ts.map +1 -1
- package/server/dist/arBrowserPaths.js +1 -0
- package/server/dist/arBrowserPaths.js.map +1 -1
- package/server/dist/automationBackend.d.ts +46 -0
- package/server/dist/automationBackend.d.ts.map +1 -0
- package/server/dist/automationBackend.js +20 -0
- package/server/dist/automationBackend.js.map +1 -0
- package/server/dist/automationSession.d.ts.map +1 -1
- package/server/dist/automationSession.js +1 -0
- package/server/dist/automationSession.js.map +1 -1
- package/server/dist/bridge.d.ts +17 -4
- package/server/dist/bridge.d.ts.map +1 -1
- package/server/dist/bridge.js +132 -33
- package/server/dist/bridge.js.map +1 -1
- package/server/dist/browserToolsExtras.d.ts +2 -2
- package/server/dist/browserToolsExtras.d.ts.map +1 -1
- package/server/dist/browserToolsExtras.js +70 -1
- package/server/dist/browserToolsExtras.js.map +1 -1
- package/server/dist/cdpTools.d.ts +5 -0
- package/server/dist/cdpTools.d.ts.map +1 -0
- package/server/dist/cdpTools.js +411 -0
- package/server/dist/cdpTools.js.map +1 -0
- package/server/dist/compositeTools.d.ts +2 -2
- package/server/dist/compositeTools.d.ts.map +1 -1
- package/server/dist/compositeTools.js.map +1 -1
- package/server/dist/debugSnapshot.d.ts +2 -2
- package/server/dist/debugSnapshot.d.ts.map +1 -1
- package/server/dist/debugSnapshot.js.map +1 -1
- package/server/dist/guardrails.d.ts.map +1 -1
- package/server/dist/guardrails.js +1 -0
- package/server/dist/guardrails.js.map +1 -1
- package/server/dist/humanLike.d.ts.map +1 -1
- package/server/dist/humanLike.js +1 -0
- package/server/dist/humanLike.js.map +1 -1
- package/server/dist/main.js +56 -15
- package/server/dist/main.js.map +1 -1
- package/server/dist/managedBridge.d.ts +58 -0
- package/server/dist/managedBridge.d.ts.map +1 -0
- package/server/dist/managedBridge.js +619 -0
- package/server/dist/managedBridge.js.map +1 -0
- package/server/dist/managedDom.d.ts +58 -0
- package/server/dist/managedDom.d.ts.map +1 -0
- package/server/dist/managedDom.js +310 -0
- package/server/dist/managedDom.js.map +1 -0
- package/server/dist/policyTiers.d.ts.map +1 -1
- package/server/dist/policyTiers.js +3 -1
- package/server/dist/policyTiers.js.map +1 -1
- package/server/dist/protocol.d.ts +4 -0
- package/server/dist/protocol.d.ts.map +1 -1
- package/server/dist/protocol.js +1 -0
- package/server/dist/protocol.js.map +1 -1
- package/server/dist/recorder.d.ts +22 -0
- package/server/dist/recorder.d.ts.map +1 -0
- package/server/dist/recorder.js +82 -0
- package/server/dist/recorder.js.map +1 -0
- package/server/dist/richEditorFill.d.ts.map +1 -1
- package/server/dist/richEditorFill.js +1 -0
- package/server/dist/richEditorFill.js.map +1 -1
- package/server/dist/screenshotCrop.d.ts.map +1 -1
- package/server/dist/screenshotCrop.js +1 -0
- package/server/dist/screenshotCrop.js.map +1 -1
- package/server/dist/sessionLog.d.ts +1 -0
- package/server/dist/sessionLog.d.ts.map +1 -1
- package/server/dist/sessionLog.js +1 -0
- package/server/dist/sessionLog.js.map +1 -1
- package/server/dist/sessionPhase.d.ts.map +1 -1
- package/server/dist/sessionPhase.js.map +1 -1
- package/server/dist/sessionSummary.d.ts.map +1 -1
- package/server/dist/sessionSummary.js +1 -0
- package/server/dist/sessionSummary.js.map +1 -1
- package/server/dist/shotUtil.d.ts +44 -0
- package/server/dist/shotUtil.d.ts.map +1 -0
- package/server/dist/shotUtil.js +89 -0
- package/server/dist/shotUtil.js.map +1 -0
- package/server/dist/skillsInstructions.d.ts.map +1 -1
- package/server/dist/skillsInstructions.js +1 -0
- package/server/dist/skillsInstructions.js.map +1 -1
- package/server/dist/snapshotText.d.ts +47 -0
- package/server/dist/snapshotText.d.ts.map +1 -0
- package/server/dist/snapshotText.js +99 -0
- package/server/dist/snapshotText.js.map +1 -0
- package/server/dist/structureFilter.d.ts.map +1 -1
- package/server/dist/structureFilter.js.map +1 -1
- package/server/dist/surveyCatalog.d.ts.map +1 -1
- package/server/dist/surveyCatalog.js +1 -0
- package/server/dist/surveyCatalog.js.map +1 -1
- package/server/dist/surveyRunner.d.ts.map +1 -1
- package/server/dist/surveyRunner.js.map +1 -1
- package/server/dist/toolHelpers.d.ts +27 -2
- package/server/dist/toolHelpers.d.ts.map +1 -1
- package/server/dist/toolHelpers.js +67 -1
- package/server/dist/toolHelpers.js.map +1 -1
- package/server/dist/toolRuntime.d.ts +4 -0
- package/server/dist/toolRuntime.d.ts.map +1 -1
- package/server/dist/toolRuntime.js +12 -6
- package/server/dist/toolRuntime.js.map +1 -1
- package/server/dist/toolSchemas.d.ts +37 -1
- package/server/dist/toolSchemas.d.ts.map +1 -1
- package/server/dist/toolSchemas.js +33 -1
- package/server/dist/toolSchemas.js.map +1 -1
- package/server/dist/tools.d.ts +2 -2
- package/server/dist/tools.d.ts.map +1 -1
- package/server/dist/tools.js +65 -11
- package/server/dist/tools.js.map +1 -1
- package/server/dist/verdictExtract.d.ts.map +1 -1
- package/server/dist/verdictExtract.js +1 -0
- package/server/dist/verdictExtract.js.map +1 -1
- package/server/dist/workflowTools.d.ts +2 -2
- package/server/dist/workflowTools.d.ts.map +1 -1
- package/server/dist/workflowTools.js +16 -4
- package/server/dist/workflowTools.js.map +1 -1
- package/skills/browser-agent/SKILL.md +89 -0
package/extension/background.js
CHANGED
|
@@ -1,10 +1,38 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
1
2
|
/**
|
|
2
|
-
* Browser
|
|
3
|
+
* AgentRuntime Browser Agent — MV3 service worker (WebSocket client to localhost bridge).
|
|
3
4
|
*/
|
|
4
5
|
|
|
5
6
|
import { DEFAULT_WS_PORT, WS_PORT_FALLBACK_ATTEMPTS } from './constants.js';
|
|
7
|
+
import * as cdp from './cdp.js';
|
|
8
|
+
import * as soft from './soft.js';
|
|
9
|
+
import * as policy from './policy.js';
|
|
6
10
|
|
|
7
|
-
|
|
11
|
+
/**
|
|
12
|
+
* One WebSocket per MCP host (Claude, Cursor, another Claude session ...). Each host runs its own server on its own
|
|
13
|
+
* port; we stay connected to all of them. Replies always go back on the socket the request came in on.
|
|
14
|
+
* @type {Map<number, WebSocket>}
|
|
15
|
+
*/
|
|
16
|
+
const sockets = new Map();
|
|
17
|
+
/** port -> time until which we skip it because another extension (profile) already holds that server. */
|
|
18
|
+
const busyUntil = new Map();
|
|
19
|
+
const BUSY_BACKOFF_MS = 5 * 60 * 1000;
|
|
20
|
+
/** Last agent that sent a request (shown in the popup). */
|
|
21
|
+
let lastRequest = null;
|
|
22
|
+
|
|
23
|
+
/** Identity of this extension install = this Chrome profile. Loaded from storage before any connection. */
|
|
24
|
+
const identity = { instanceId: '', label: '' };
|
|
25
|
+
const identityReady = (async () => {
|
|
26
|
+
const o = await chrome.storage.local.get(['instanceId', 'profileLabel']);
|
|
27
|
+
let id = o.instanceId;
|
|
28
|
+
if (!id) {
|
|
29
|
+
id = crypto.randomUUID();
|
|
30
|
+
await chrome.storage.local.set({ instanceId: id });
|
|
31
|
+
}
|
|
32
|
+
identity.instanceId = id;
|
|
33
|
+
identity.label = (o.profileLabel || '').trim() || `profile-${id.slice(0, 4)}`;
|
|
34
|
+
})();
|
|
35
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
8
36
|
|
|
9
37
|
/** @type {Map<string, (out: unknown) => void>} */
|
|
10
38
|
const approvalWaiters = new Map();
|
|
@@ -20,6 +48,7 @@ try {
|
|
|
20
48
|
return;
|
|
21
49
|
}
|
|
22
50
|
networkRing.push({
|
|
51
|
+
tabId: d.tabId,
|
|
23
52
|
url: (d.url || '').slice(0, 800),
|
|
24
53
|
statusCode: d.statusCode ?? 0,
|
|
25
54
|
method: d.method || 'GET',
|
|
@@ -36,7 +65,11 @@ try {
|
|
|
36
65
|
console.warn('[browser-automation] webRequest init failed', e);
|
|
37
66
|
}
|
|
38
67
|
|
|
39
|
-
chrome.runtime.onInstalled.addListener(() => {
|
|
68
|
+
chrome.runtime.onInstalled.addListener((details) => {
|
|
69
|
+
if (details.reason === 'install') {
|
|
70
|
+
// The extension does nothing without the local helper, so show the setup guide once.
|
|
71
|
+
void chrome.tabs.create({ url: chrome.runtime.getURL('welcome.html') });
|
|
72
|
+
}
|
|
40
73
|
chrome.storage.local.get('wsPort', (o) => {
|
|
41
74
|
if (o.wsPort == null) {
|
|
42
75
|
chrome.storage.local.set({ wsPort: DEFAULT_WS_PORT });
|
|
@@ -65,7 +98,9 @@ async function ensureInject(tabId, params) {
|
|
|
65
98
|
async function rpcToTab(tabId, method, params) {
|
|
66
99
|
await ensureInject(tabId, params);
|
|
67
100
|
const fid = frameIdFromParams(params);
|
|
68
|
-
|
|
101
|
+
// Always target one frame explicitly: without frameId the message is delivered to EVERY frame
|
|
102
|
+
// that has the content script, and a child frame's same-named ref (e2) can answer first.
|
|
103
|
+
const opts = { frameId: fid !== undefined ? fid : 0 };
|
|
69
104
|
const raw = await chrome.tabs.sendMessage(
|
|
70
105
|
tabId,
|
|
71
106
|
{ type: 'ba-rpc', method, params },
|
|
@@ -74,6 +109,204 @@ async function rpcToTab(tabId, method, params) {
|
|
|
74
109
|
return raw;
|
|
75
110
|
}
|
|
76
111
|
|
|
112
|
+
// ---- iframe support -------------------------------------------------------------------------
|
|
113
|
+
// Content scripts run per frame. get_page_structure/get_page_text fan out to every frame; nodes from
|
|
114
|
+
// child frames get refs "f<frameId>.e<n>", which dispatchRequest routes back to the right frame.
|
|
115
|
+
|
|
116
|
+
/** tabId -> Map(frameId -> {x, y}) offset of the frame's viewport inside the top viewport. */
|
|
117
|
+
const frameOffsetsByTab = new Map();
|
|
118
|
+
|
|
119
|
+
const CAPTCHA_FRAME_RE = [
|
|
120
|
+
[/google\.com\/recaptcha|recaptcha\.net/i, 'recaptcha'],
|
|
121
|
+
[/hcaptcha\.com/i, 'hcaptcha'],
|
|
122
|
+
[/challenges\.cloudflare\.com|turnstile/i, 'turnstile'],
|
|
123
|
+
[/arkoselabs\.com|funcaptcha/i, 'arkose'],
|
|
124
|
+
[/geetest\.com/i, 'geetest']
|
|
125
|
+
];
|
|
126
|
+
|
|
127
|
+
function detectCaptcha(urls) {
|
|
128
|
+
for (const u of urls) {
|
|
129
|
+
for (const [re, provider] of CAPTCHA_FRAME_RE) {
|
|
130
|
+
if (re.test(u)) {
|
|
131
|
+
return { provider, frameUrl: String(u).slice(0, 200) };
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return null;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Inject content.js into every frame; returns child frame ids (excludes top frame 0). */
|
|
139
|
+
async function childFrameIds(tabId) {
|
|
140
|
+
try {
|
|
141
|
+
const res = await chrome.scripting.executeScript({
|
|
142
|
+
target: { tabId, allFrames: true },
|
|
143
|
+
files: ['content.js']
|
|
144
|
+
});
|
|
145
|
+
return [...new Set(res.map((r) => r.frameId))].filter((id) => id !== 0);
|
|
146
|
+
} catch {
|
|
147
|
+
return [];
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function frameRpc(tabId, frameId, method, params) {
|
|
152
|
+
try {
|
|
153
|
+
const r = await chrome.tabs.sendMessage(tabId, { type: 'ba-rpc', method, params }, { frameId });
|
|
154
|
+
return r?.ok ? r.result : null;
|
|
155
|
+
} catch {
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function sameUrlLoose(a, b) {
|
|
161
|
+
if (a === b) {
|
|
162
|
+
return true;
|
|
163
|
+
}
|
|
164
|
+
try {
|
|
165
|
+
const ua = new URL(a);
|
|
166
|
+
const ub = new URL(b);
|
|
167
|
+
return ua.origin === ub.origin && ua.pathname === ub.pathname;
|
|
168
|
+
} catch {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
async function structureWithFrames(tabId, params) {
|
|
174
|
+
const top = wrapRpcResult(await rpcToTab(tabId, 'get_page_structure', params));
|
|
175
|
+
if (params.includeFrames === false || frameIdFromParams(params) !== undefined) {
|
|
176
|
+
return top;
|
|
177
|
+
}
|
|
178
|
+
const ids = await childFrameIds(tabId);
|
|
179
|
+
if (!ids.length) {
|
|
180
|
+
return top;
|
|
181
|
+
}
|
|
182
|
+
const maxNodes = params.maxNodes ?? 500;
|
|
183
|
+
const viewportOnly = params.viewportOnly !== false;
|
|
184
|
+
const frames = [];
|
|
185
|
+
for (const fid of ids) {
|
|
186
|
+
const r = await frameRpc(tabId, fid, 'get_page_structure', {
|
|
187
|
+
...params,
|
|
188
|
+
viewportOnly: false,
|
|
189
|
+
includeRects: true,
|
|
190
|
+
maxNodes
|
|
191
|
+
});
|
|
192
|
+
if (r) {
|
|
193
|
+
frames.push({ fid, ...r });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Locate each frame's <iframe> element (in top or another frame) by URL to learn its offset + parent.
|
|
197
|
+
const iframeEntries = [];
|
|
198
|
+
const collect = (owner, nodes) => {
|
|
199
|
+
for (const n of nodes) {
|
|
200
|
+
if (n.frameSrc) {
|
|
201
|
+
iframeEntries.push({ owner, src: n.frameSrc, rect: n.rect, ref: n.ref, used: false });
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
collect(0, top.nodes);
|
|
206
|
+
for (const f of frames) {
|
|
207
|
+
collect(f.fid, f.nodes);
|
|
208
|
+
}
|
|
209
|
+
const link = new Map(); // fid -> entry
|
|
210
|
+
for (const f of frames) {
|
|
211
|
+
const entry =
|
|
212
|
+
iframeEntries.find((e) => !e.used && e.owner !== f.fid && e.src === f.url) ||
|
|
213
|
+
iframeEntries.find((e) => !e.used && e.owner !== f.fid && sameUrlLoose(e.src, f.url)) ||
|
|
214
|
+
iframeEntries.find((e) => !e.used && e.owner !== f.fid && e.src === 'about:srcdoc' && /^about:/.test(f.url));
|
|
215
|
+
if (entry) {
|
|
216
|
+
entry.used = true;
|
|
217
|
+
link.set(f.fid, entry);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
const offsets = new Map([[0, { x: 0, y: 0 }]]);
|
|
221
|
+
const offsetOf = (fid, depth = 0) => {
|
|
222
|
+
if (offsets.has(fid)) {
|
|
223
|
+
return offsets.get(fid);
|
|
224
|
+
}
|
|
225
|
+
const e = link.get(fid);
|
|
226
|
+
if (!e || !e.rect || depth > 8) {
|
|
227
|
+
return null;
|
|
228
|
+
}
|
|
229
|
+
const parent = offsetOf(e.owner, depth + 1);
|
|
230
|
+
if (!parent) {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
const o = { x: parent.x + e.rect.x, y: parent.y + e.rect.y };
|
|
234
|
+
offsets.set(fid, o);
|
|
235
|
+
return o;
|
|
236
|
+
};
|
|
237
|
+
frameOffsetsByTab.set(tabId, offsets);
|
|
238
|
+
|
|
239
|
+
const usedRefs = (owner) => new Set(iframeEntries.filter((e) => e.used && e.owner === owner).map((e) => e.ref));
|
|
240
|
+
const nodes = top.nodes.filter((n) => !usedRefs(0).has(n.ref)); // expanded iframes are replaced by their contents
|
|
241
|
+
let truncated = !!top.truncated;
|
|
242
|
+
const frameSummaries = [];
|
|
243
|
+
for (const f of frames) {
|
|
244
|
+
const off = offsetOf(f.fid);
|
|
245
|
+
if (!off && viewportOnly) {
|
|
246
|
+
continue; // hidden/off-screen frame (no visible <iframe> found for it)
|
|
247
|
+
}
|
|
248
|
+
const skip = usedRefs(f.fid);
|
|
249
|
+
let added = 0;
|
|
250
|
+
for (const n of f.nodes) {
|
|
251
|
+
if (skip.has(n.ref)) {
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (nodes.length >= maxNodes) {
|
|
255
|
+
truncated = true;
|
|
256
|
+
break;
|
|
257
|
+
}
|
|
258
|
+
const out = { ...n, ref: `f${f.fid}.${n.ref}`, frameId: f.fid, frameUrl: f.url };
|
|
259
|
+
delete out.frameSrc;
|
|
260
|
+
if (n.rect && off) {
|
|
261
|
+
out.rect = { x: n.rect.x + off.x, y: n.rect.y + off.y, w: n.rect.w, h: n.rect.h };
|
|
262
|
+
} else if (n.rect) {
|
|
263
|
+
out.rectSpace = 'frame';
|
|
264
|
+
}
|
|
265
|
+
if (params.includeRects === false) {
|
|
266
|
+
delete out.rect;
|
|
267
|
+
}
|
|
268
|
+
nodes.push(out);
|
|
269
|
+
added += 1;
|
|
270
|
+
}
|
|
271
|
+
if (added) {
|
|
272
|
+
frameSummaries.push({ prefix: `f${f.fid}`, url: f.url, nodeCount: added });
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
for (const n of nodes) {
|
|
276
|
+
delete n.frameSrc;
|
|
277
|
+
}
|
|
278
|
+
const captcha = detectCaptcha(frames.map((f) => f.url));
|
|
279
|
+
return {
|
|
280
|
+
...top,
|
|
281
|
+
nodes,
|
|
282
|
+
truncated,
|
|
283
|
+
emptyHint: nodes.length === 0 ? top.emptyHint : null,
|
|
284
|
+
...(frameSummaries.length ? { frames: frameSummaries } : {}),
|
|
285
|
+
...(captcha
|
|
286
|
+
? { captcha, humanRequired: true, hint: `Captcha detected (${captcha.provider}); it requires a human.` }
|
|
287
|
+
: {})
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function pageTextWithFrames(tabId, params) {
|
|
292
|
+
const top = wrapRpcResult(await rpcToTab(tabId, 'get_page_text', params));
|
|
293
|
+
if (params.includeFrames === false || frameIdFromParams(params) !== undefined) {
|
|
294
|
+
return top;
|
|
295
|
+
}
|
|
296
|
+
const max = params.maxChars ?? 50000;
|
|
297
|
+
let text = top.text;
|
|
298
|
+
for (const fid of await childFrameIds(tabId)) {
|
|
299
|
+
if (text.length > max) {
|
|
300
|
+
break;
|
|
301
|
+
}
|
|
302
|
+
const r = await frameRpc(tabId, fid, 'get_page_text', params);
|
|
303
|
+
if (r && r.text && r.text.trim()) {
|
|
304
|
+
text += `\n\n[iframe ${String(r.url).slice(0, 100)}]\n${r.text.trim()}`;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
return { ...top, text: text.slice(0, max), truncated: top.truncated || text.length > max, sourceLength: text.length };
|
|
308
|
+
}
|
|
309
|
+
|
|
77
310
|
async function assertTabExists(tabId, { storedTarget = false } = {}) {
|
|
78
311
|
try {
|
|
79
312
|
await chrome.tabs.get(tabId);
|
|
@@ -115,18 +348,59 @@ async function getActiveAutomatableTabId() {
|
|
|
115
348
|
return tab.id;
|
|
116
349
|
}
|
|
117
350
|
|
|
351
|
+
/**
|
|
352
|
+
* Tab policy (see policy.js): freeform | follow | pinned, set only from the popup. Stored as `tabMode` and `pinnedTabs`.
|
|
353
|
+
* Older installs: tabMode 'agent' = freeform, 'front' = follow, a stored `connectedTabId` = one pinned tab.
|
|
354
|
+
*/
|
|
355
|
+
async function getPolicyState() {
|
|
356
|
+
const o = await chrome.storage.local.get(['tabMode', 'pinnedTabs', 'connectedTabId']);
|
|
357
|
+
const mode = policy.normalizeMode(o.tabMode, o.connectedTabId);
|
|
358
|
+
const pinned = Array.isArray(o.pinnedTabs)
|
|
359
|
+
? o.pinnedTabs.filter(Number.isFinite)
|
|
360
|
+
: o.connectedTabId != null
|
|
361
|
+
? [o.connectedTabId]
|
|
362
|
+
: [];
|
|
363
|
+
return { mode, pinned };
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
async function frontTabId() {
|
|
367
|
+
const [t] = await chrome.tabs.query({ active: true, lastFocusedWindow: true });
|
|
368
|
+
return t?.id ?? null;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
let lastUsedPinned = null;
|
|
372
|
+
|
|
373
|
+
// A closed tab leaves the pinned set.
|
|
374
|
+
chrome.tabs.onRemoved.addListener(async (id) => {
|
|
375
|
+
const o = await chrome.storage.local.get('pinnedTabs');
|
|
376
|
+
if (Array.isArray(o.pinnedTabs) && o.pinnedTabs.includes(id)) {
|
|
377
|
+
await chrome.storage.local.set({ pinnedTabs: o.pinnedTabs.filter((x) => x !== id) });
|
|
378
|
+
}
|
|
379
|
+
if (lastUsedPinned === id) {
|
|
380
|
+
lastUsedPinned = null;
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
|
|
118
384
|
async function getTargetTabId(params) {
|
|
119
|
-
|
|
120
|
-
|
|
385
|
+
const { mode, pinned } = await getPolicyState();
|
|
386
|
+
const front = mode === 'follow' ? await frontTabId() : null;
|
|
387
|
+
const requested = params.tabId != null && params.tabId !== '' ? Number(params.tabId) : null;
|
|
388
|
+
const r = policy.resolveTarget({
|
|
389
|
+
mode,
|
|
390
|
+
pinned,
|
|
391
|
+
front,
|
|
392
|
+
lastUsedPinned,
|
|
393
|
+
requested,
|
|
394
|
+
fromAgentDefault: params.tabFromAgentDefault === true
|
|
395
|
+
});
|
|
396
|
+
if (r.error) {
|
|
397
|
+
throw Object.assign(new Error(r.error.message), { code: r.error.code });
|
|
121
398
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
} catch {
|
|
128
|
-
await chrome.storage.local.remove('connectedTabId');
|
|
129
|
-
}
|
|
399
|
+
if (mode === 'pinned' && r.tabId != null) {
|
|
400
|
+
lastUsedPinned = r.tabId;
|
|
401
|
+
}
|
|
402
|
+
if (r.tabId != null) {
|
|
403
|
+
return assertTabExists(r.tabId, { storedTarget: false });
|
|
130
404
|
}
|
|
131
405
|
return getActiveAutomatableTabId();
|
|
132
406
|
}
|
|
@@ -157,19 +431,68 @@ function wireOpName(method) {
|
|
|
157
431
|
return s;
|
|
158
432
|
}
|
|
159
433
|
|
|
434
|
+
/** Viewport centre (top-frame CSS px) of a snapshot ref, accounting for iframe offset. */
|
|
435
|
+
async function refCentre(tabId, params, rawRef) {
|
|
436
|
+
let ref = String(rawRef).trim();
|
|
437
|
+
let fid = frameIdFromParams(params);
|
|
438
|
+
const m = /^@?f(\d+)\.(.+)$/.exec(ref);
|
|
439
|
+
if (m) {
|
|
440
|
+
fid = Number(m[1]);
|
|
441
|
+
ref = m[2];
|
|
442
|
+
} else if (ref.startsWith('@')) {
|
|
443
|
+
ref = ref.slice(1);
|
|
444
|
+
}
|
|
445
|
+
const r = await rpcToTab(tabId, 'get_ref_presentation_rect', { ref, frameId: fid });
|
|
446
|
+
if (!r?.ok) {
|
|
447
|
+
return wrapRpcResult(r);
|
|
448
|
+
}
|
|
449
|
+
const off = fid !== undefined ? frameOffsetsByTab.get(tabId)?.get(fid) : { x: 0, y: 0 };
|
|
450
|
+
if (!off) {
|
|
451
|
+
throw Object.assign(new Error('Frame position unknown — take a fresh browser_snapshot, then retry.'), { code: 'stale_ref' });
|
|
452
|
+
}
|
|
453
|
+
return cdp.centreFromRect(r.result, off);
|
|
454
|
+
}
|
|
455
|
+
|
|
160
456
|
/**
|
|
161
457
|
* @param {string} method
|
|
162
458
|
* @param {Record<string, unknown>} params
|
|
163
459
|
* @param {string} requestId WebSocket request id (needed for ask_confirmation)
|
|
164
460
|
*/
|
|
461
|
+
/** Page size and pixel ratio, so a screenshot can be mapped back to page (CSS) coordinates for clicks. */
|
|
462
|
+
async function viewportInfo(tabId) {
|
|
463
|
+
try {
|
|
464
|
+
const [r] = await chrome.scripting.executeScript({
|
|
465
|
+
target: { tabId },
|
|
466
|
+
func: () => ({
|
|
467
|
+
cssWidth: window.innerWidth,
|
|
468
|
+
cssHeight: window.innerHeight,
|
|
469
|
+
dpr: window.devicePixelRatio,
|
|
470
|
+
scrollX: Math.round(window.scrollX),
|
|
471
|
+
scrollY: Math.round(window.scrollY)
|
|
472
|
+
})
|
|
473
|
+
});
|
|
474
|
+
return r?.result ?? null;
|
|
475
|
+
} catch {
|
|
476
|
+
return null;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
|
|
165
480
|
async function dispatchRequest(method, params, requestId) {
|
|
166
481
|
const op = wireOpName(method);
|
|
167
482
|
|
|
483
|
+
// Tab policy: in follow and pinned mode some calls are refused and lists show only allowed tabs.
|
|
484
|
+
const pol = await getPolicyState();
|
|
485
|
+
const refusal = policy.blockedOp(pol.mode, op);
|
|
486
|
+
if (refusal) {
|
|
487
|
+
throw Object.assign(new Error(refusal), { code: 'tab_not_allowed' });
|
|
488
|
+
}
|
|
489
|
+
const allowed = policy.allowedSet(pol.mode, pol.pinned, pol.mode === 'follow' ? await frontTabId() : null);
|
|
490
|
+
|
|
168
491
|
if (op === 'get_tabs') {
|
|
169
492
|
const q = params.activeOnly
|
|
170
493
|
? { active: true, currentWindow: true }
|
|
171
494
|
: {};
|
|
172
|
-
const tabs = await chrome.tabs.query(q);
|
|
495
|
+
const tabs = (await chrome.tabs.query(q)).filter((t) => !allowed || allowed.has(t.id));
|
|
173
496
|
return {
|
|
174
497
|
tabs: tabs.map((t) => ({
|
|
175
498
|
tabId: t.id,
|
|
@@ -193,12 +516,49 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
193
516
|
};
|
|
194
517
|
}
|
|
195
518
|
|
|
519
|
+
if (op === 'window_list') {
|
|
520
|
+
const out = await cdp.windowList();
|
|
521
|
+
if (!allowed) {
|
|
522
|
+
return out;
|
|
523
|
+
}
|
|
524
|
+
const windows = out.windows
|
|
525
|
+
.map((w) => ({ ...w, tabs: w.tabs.filter((t) => allowed.has(t.tabId)) }))
|
|
526
|
+
.filter((w) => w.tabs.length > 0);
|
|
527
|
+
return { windows };
|
|
528
|
+
}
|
|
529
|
+
if (op === 'window_open') {
|
|
530
|
+
return cdp.windowOpen(params);
|
|
531
|
+
}
|
|
532
|
+
if (op === 'window_resize') {
|
|
533
|
+
return cdp.windowResize(Number(params.windowId), params);
|
|
534
|
+
}
|
|
535
|
+
if (op === 'window_focus') {
|
|
536
|
+
return cdp.windowFocus(Number(params.windowId));
|
|
537
|
+
}
|
|
538
|
+
if (op === 'get_downloads') {
|
|
539
|
+
return cdp.downloadsRecent(Number(params.waitMs) || 0);
|
|
540
|
+
}
|
|
541
|
+
|
|
196
542
|
if (op === 'get_network_log') {
|
|
197
543
|
const max = Math.min(Number(params.maxEntries) || 50, NETWORK_CAP);
|
|
198
|
-
|
|
544
|
+
const entries = networkRing.filter((e) => !allowed || allowed.has(e.tabId));
|
|
545
|
+
return { entries: entries.slice(-max) };
|
|
199
546
|
}
|
|
200
547
|
|
|
201
548
|
const tabId = await getTargetTabId(params);
|
|
549
|
+
if (lastRequest) {
|
|
550
|
+
lastRequest.tabId = tabId;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
// Snapshot-style refs: "@e3" -> "e3"; frame refs "f4.e2" -> frameId 4 + ref "e2".
|
|
554
|
+
if (typeof params.ref === 'string') {
|
|
555
|
+
const m = /^@?f(\d+)\.(.+)$/.exec(params.ref.trim());
|
|
556
|
+
if (m) {
|
|
557
|
+
params = { ...params, frameId: Number(m[1]), ref: m[2] };
|
|
558
|
+
} else if (params.ref.startsWith('@')) {
|
|
559
|
+
params = { ...params, ref: params.ref.slice(1) };
|
|
560
|
+
}
|
|
561
|
+
}
|
|
202
562
|
|
|
203
563
|
switch (op) {
|
|
204
564
|
case 'navigate': {
|
|
@@ -218,17 +578,27 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
218
578
|
};
|
|
219
579
|
}
|
|
220
580
|
case 'get_current_tab': {
|
|
581
|
+
// The orientation call: where am I, in which profile, and what am I allowed to do.
|
|
221
582
|
const t = await chrome.tabs.get(tabId);
|
|
583
|
+
const win = await chrome.windows.get(t.windowId).catch(() => null);
|
|
222
584
|
return {
|
|
223
585
|
tabId: t.id,
|
|
224
586
|
title: t.title,
|
|
225
587
|
url: t.url,
|
|
226
588
|
windowId: t.windowId,
|
|
227
|
-
active: t.active
|
|
589
|
+
active: t.active,
|
|
590
|
+
scriptable: /^https?:/i.test(t.url || ''),
|
|
591
|
+
profile: identity.label,
|
|
592
|
+
tabMode: pol.mode,
|
|
593
|
+
tabRule: policy.modeSummary(pol.mode),
|
|
594
|
+
pinnedTabs: pol.mode === 'pinned' ? pol.pinned : undefined,
|
|
595
|
+
window: win ? { state: win.state, focused: win.focused } : undefined
|
|
228
596
|
};
|
|
229
597
|
}
|
|
230
598
|
case 'get_page_text':
|
|
599
|
+
return pageTextWithFrames(tabId, params);
|
|
231
600
|
case 'get_page_structure':
|
|
601
|
+
return structureWithFrames(tabId, params);
|
|
232
602
|
case 'click_element':
|
|
233
603
|
case 'fill_input':
|
|
234
604
|
case 'press_key':
|
|
@@ -252,13 +622,22 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
252
622
|
return wrapRpcResult(presR);
|
|
253
623
|
}
|
|
254
624
|
const pres = presR.result;
|
|
255
|
-
|
|
625
|
+
// Chrome insets must come from the TOP frame (an iframe's innerHeight is not the window's).
|
|
626
|
+
const insetsR = await rpcToTab(tabId, 'get_viewport_chrome_insets', { ...params, frameId: undefined });
|
|
256
627
|
if (!insetsR?.ok) {
|
|
257
628
|
return wrapRpcResult(insetsR);
|
|
258
629
|
}
|
|
259
630
|
const insets = insetsR.result;
|
|
260
|
-
const
|
|
261
|
-
const
|
|
631
|
+
const fid = frameIdFromParams(params);
|
|
632
|
+
const frameOff = fid !== undefined ? frameOffsetsByTab.get(tabId)?.get(fid) : { x: 0, y: 0 };
|
|
633
|
+
if (!frameOff) {
|
|
634
|
+
throw Object.assign(
|
|
635
|
+
new Error('Frame position unknown — take a fresh browser_snapshot, then retry.'),
|
|
636
|
+
{ code: 'stale_ref' }
|
|
637
|
+
);
|
|
638
|
+
}
|
|
639
|
+
const screenX = win.left + insets.left + frameOff.x + pres.x;
|
|
640
|
+
const screenY = win.top + insets.top + frameOff.y + pres.y;
|
|
262
641
|
const width = pres.w;
|
|
263
642
|
const height = pres.h;
|
|
264
643
|
return {
|
|
@@ -339,6 +718,7 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
339
718
|
fmt === 'png'
|
|
340
719
|
? { format: 'png' }
|
|
341
720
|
: { format: 'jpeg', quality: params.quality ?? 85 };
|
|
721
|
+
await soft.paceCapture();
|
|
342
722
|
const dataUrl = await chrome.tabs.captureVisibleTab(t.windowId, capOpts);
|
|
343
723
|
const mimeType = fmt === 'png' ? 'image/png' : 'image/jpeg';
|
|
344
724
|
const base64 = dataUrl.split(',')[1] || '';
|
|
@@ -411,6 +791,15 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
411
791
|
return inj.result;
|
|
412
792
|
}
|
|
413
793
|
case 'evaluate_expression': {
|
|
794
|
+
// Running a script string sent by an agent is the most powerful call here, so it is off unless the user
|
|
795
|
+
// turns it on in the popup (Advanced). The server also needs BROWSER_AUTOMATION_ALLOW_EVAL=1.
|
|
796
|
+
const { allowEval } = await chrome.storage.local.get('allowEval');
|
|
797
|
+
if (allowEval !== true) {
|
|
798
|
+
throw Object.assign(
|
|
799
|
+
new Error('Script evaluation is switched off in the extension. Turn on "Allow page script evaluation" in the popup (Advanced) to use browser_evaluate.'),
|
|
800
|
+
{ code: 'eval_disabled' }
|
|
801
|
+
);
|
|
802
|
+
}
|
|
414
803
|
const expr = String(params.expression ?? '').trim();
|
|
415
804
|
if (!expr) {
|
|
416
805
|
throw Object.assign(new Error('expression required'), { code: 'invalid_params' });
|
|
@@ -451,19 +840,106 @@ async function dispatchRequest(method, params, requestId) {
|
|
|
451
840
|
return inj.result;
|
|
452
841
|
}
|
|
453
842
|
case 'screenshot_visible': {
|
|
843
|
+
// mode: 'auto' (default) uses the fast tab capture only when the tab is the active tab of a focused
|
|
844
|
+
// window, otherwise CDP (works hidden/covered). 'visible' forces tab capture (focuses the window).
|
|
845
|
+
// 'cdp' forces CDP. fullPage and clip are CDP-only.
|
|
454
846
|
const t = await chrome.tabs.get(tabId);
|
|
847
|
+
let win = await chrome.windows.get(t.windowId);
|
|
848
|
+
// A minimized window never paints, so nothing can capture it. With restoreWindow:true we un-minimize it
|
|
849
|
+
// without focusing it, capture, and minimize it again (it flashes on screen for about half a second).
|
|
850
|
+
const wasMinimized = win.state === 'minimized';
|
|
851
|
+
if (wasMinimized) {
|
|
852
|
+
if (params.restoreWindow !== true) {
|
|
853
|
+
throw Object.assign(
|
|
854
|
+
new Error(
|
|
855
|
+
'The tab is in a minimized window, which does not paint. Pass restoreWindow:true (briefly restores it and minimizes it again), ' +
|
|
856
|
+
'or use browser_window_open for a dedicated window.'
|
|
857
|
+
),
|
|
858
|
+
{ code: 'tab_not_painting' }
|
|
859
|
+
);
|
|
860
|
+
}
|
|
861
|
+
await chrome.windows.update(t.windowId, { state: 'normal', focused: false });
|
|
862
|
+
await chrome.tabs.update(tabId, { active: true });
|
|
863
|
+
await sleep(450);
|
|
864
|
+
win = await chrome.windows.get(t.windowId);
|
|
865
|
+
}
|
|
866
|
+
try {
|
|
867
|
+
const mode = params.mode === 'visible' || params.mode === 'cdp' ? params.mode : 'auto';
|
|
868
|
+
const fmt = params.format === 'png' ? 'png' : 'jpeg';
|
|
869
|
+
// DevTools capture only when the call needs it (full page, clip, or mode:'cdp'). Everything else uses the
|
|
870
|
+
// tab capture, which shows no Chrome debugging bar.
|
|
871
|
+
const wantCdp = mode === 'cdp' || params.fullPage === true || params.clip != null;
|
|
872
|
+
if (wantCdp) {
|
|
873
|
+
const shot = await cdp.cdpScreenshot(tabId, {
|
|
874
|
+
format: fmt,
|
|
875
|
+
quality: params.quality ?? 80,
|
|
876
|
+
fullPage: params.fullPage === true,
|
|
877
|
+
clip: params.clip
|
|
878
|
+
});
|
|
879
|
+
return { ...shot, imageScale: 1, viewport: await viewportInfo(tabId) };
|
|
880
|
+
}
|
|
455
881
|
await chrome.tabs.update(tabId, { active: true });
|
|
456
882
|
await chrome.windows.update(t.windowId, { focused: true });
|
|
457
|
-
const fmt = params.format === 'png' ? 'png' : 'jpeg';
|
|
458
883
|
const capOpts =
|
|
459
884
|
fmt === 'png'
|
|
460
885
|
? { format: 'png' }
|
|
461
886
|
: { format: 'jpeg', quality: params.quality ?? 80 };
|
|
887
|
+
await soft.paceCapture();
|
|
462
888
|
const dataUrl = await chrome.tabs.captureVisibleTab(t.windowId, capOpts);
|
|
463
889
|
const mimeType = fmt === 'png' ? 'image/png' : 'image/jpeg';
|
|
464
890
|
const base64 = dataUrl.split(',')[1] || '';
|
|
465
|
-
|
|
891
|
+
const viewport = await viewportInfo(tabId);
|
|
892
|
+
return { mimeType, dataBase64: base64, method: 'tab', restoredWindow: wasMinimized, imageScale: viewport?.dpr ?? 1, viewport };
|
|
893
|
+
} finally {
|
|
894
|
+
if (wasMinimized) {
|
|
895
|
+
await chrome.windows.update(t.windowId, { state: 'minimized' }).catch(() => undefined);
|
|
896
|
+
}
|
|
897
|
+
}
|
|
466
898
|
}
|
|
899
|
+
case 'set_viewport':
|
|
900
|
+
return params.engine === 'cdp' ? cdp.setViewport(tabId, params) : soft.softSetViewport(tabId, params);
|
|
901
|
+
case 'set_color_scheme':
|
|
902
|
+
return cdp.setColorScheme(tabId, String(params.scheme ?? ''));
|
|
903
|
+
case 'reset_emulation': {
|
|
904
|
+
const out = await soft.softResetViewport(tabId);
|
|
905
|
+
if (await cdp.enabled()) {
|
|
906
|
+
await cdp.resetEmulation(tabId);
|
|
907
|
+
}
|
|
908
|
+
return out;
|
|
909
|
+
}
|
|
910
|
+
case 'pointer': {
|
|
911
|
+
let { x, y, toX, toY } = params;
|
|
912
|
+
if (typeof params.ref === 'string' && (x == null || y == null)) {
|
|
913
|
+
const c = await refCentre(tabId, params, params.ref);
|
|
914
|
+
x = c.x;
|
|
915
|
+
y = c.y;
|
|
916
|
+
}
|
|
917
|
+
if (typeof params.toRef === 'string' && (toX == null || toY == null)) {
|
|
918
|
+
const c = await refCentre(tabId, { ...params, frameId: undefined }, params.toRef);
|
|
919
|
+
toX = c.x;
|
|
920
|
+
toY = c.y;
|
|
921
|
+
}
|
|
922
|
+
const pp = { ...params, x, y, toX, toY };
|
|
923
|
+
return params.engine === 'cdp' ? cdp.pointer(tabId, pp) : soft.softPointer(tabId, pp);
|
|
924
|
+
}
|
|
925
|
+
case 'clipboard_read':
|
|
926
|
+
return params.engine === 'cdp' ? cdp.clipboardRead(tabId) : soft.softClipboardRead(tabId);
|
|
927
|
+
case 'clipboard_write':
|
|
928
|
+
return params.engine === 'cdp'
|
|
929
|
+
? cdp.clipboardWrite(tabId, params.text ?? '')
|
|
930
|
+
: soft.softClipboardWrite(tabId, params.text ?? '');
|
|
931
|
+
case 'record_start':
|
|
932
|
+
return params.engine === 'cdp' ? cdp.recordStart(tabId, params) : soft.softRecordStart(tabId, params);
|
|
933
|
+
case 'record_stop':
|
|
934
|
+
return soft.hasSoftRecording(tabId) ? soft.softRecordStop(tabId) : cdp.recordStop(tabId);
|
|
935
|
+
case 'record_fetch': {
|
|
936
|
+
const n = Math.min(Number(params.count) || 10, 25);
|
|
937
|
+
return soft.hasSoftRecording(tabId)
|
|
938
|
+
? soft.softRecordFetch(tabId, Number(params.from) || 0, n)
|
|
939
|
+
: cdp.recordFetch(tabId, Number(params.from) || 0, n);
|
|
940
|
+
}
|
|
941
|
+
case 'record_discard':
|
|
942
|
+
return soft.hasSoftRecording(tabId) ? soft.softRecordDiscard(tabId) : cdp.recordDiscard(tabId);
|
|
467
943
|
case 'wait': {
|
|
468
944
|
const ms = Math.min(params.ms ?? 500, 30000);
|
|
469
945
|
await new Promise((r) => setTimeout(r, ms));
|
|
@@ -553,31 +1029,65 @@ chrome.runtime.onMessage.addListener((msg, _s, sendResponse) => {
|
|
|
553
1029
|
return;
|
|
554
1030
|
}
|
|
555
1031
|
if (msg?.type === 'connect') {
|
|
1032
|
+
void chrome.storage.session.remove('autoConnectOff');
|
|
1033
|
+
void chrome.storage.local.remove('autoConnect');
|
|
556
1034
|
connectWs(msg.port, msg.token)
|
|
557
1035
|
.then(() => sendResponse({ ok: true }))
|
|
558
1036
|
.catch((e) => sendResponse({ ok: false, error: String(e) }));
|
|
559
1037
|
return true;
|
|
560
1038
|
}
|
|
561
1039
|
if (msg?.type === 'disconnect') {
|
|
1040
|
+
// A deliberate disconnect turns auto-connect off until the next manual Connect or until Chrome restarts.
|
|
1041
|
+
void chrome.storage.session.set({ autoConnectOff: true });
|
|
562
1042
|
disconnectWs();
|
|
563
1043
|
sendResponse({ ok: true });
|
|
564
1044
|
return;
|
|
565
1045
|
}
|
|
1046
|
+
if (msg?.type === 'set-label') {
|
|
1047
|
+
const label = String(msg.label ?? '').trim().slice(0, 60);
|
|
1048
|
+
chrome.storage.local
|
|
1049
|
+
.set({ profileLabel: label })
|
|
1050
|
+
.then(async () => {
|
|
1051
|
+
await identityReady;
|
|
1052
|
+
identity.label = label || `profile-${identity.instanceId.slice(0, 4)}`;
|
|
1053
|
+
// The label is sent when a connection opens, so reconnect to apply it.
|
|
1054
|
+
disconnectWs();
|
|
1055
|
+
setTimeout(() => void autoConnectTick(), 300);
|
|
1056
|
+
sendResponse({ ok: true, label: identity.label });
|
|
1057
|
+
})
|
|
1058
|
+
.catch((e) => sendResponse({ ok: false, error: String(e) }));
|
|
1059
|
+
return true;
|
|
1060
|
+
}
|
|
566
1061
|
if (msg?.type === 'get-ws-status') {
|
|
567
|
-
|
|
568
|
-
|
|
1062
|
+
void (async () => {
|
|
1063
|
+
sendResponse({
|
|
1064
|
+
label: identity.label,
|
|
1065
|
+
autoConnectOff: (await chrome.storage.session.get('autoConnectOff')).autoConnectOff === true,
|
|
1066
|
+
everConnected: (await chrome.storage.local.get('everConnected')).everConnected === true,
|
|
1067
|
+
lastScanAt: diag.lastTickAt,
|
|
1068
|
+
lastScanError: diag.lastError,
|
|
1069
|
+
failingSince: diag.failingSince,
|
|
1070
|
+
tabMode: (await getPolicyState()).mode,
|
|
1071
|
+
pinned: (await getPolicyState()).pinned,
|
|
1072
|
+
instanceId: identity.instanceId,
|
|
1073
|
+
connected: sockets.size > 0,
|
|
1074
|
+
ports: [...sockets.keys()].sort((x, y) => x - y),
|
|
1075
|
+
lastRequest
|
|
1076
|
+
});
|
|
1077
|
+
})();
|
|
1078
|
+
return true;
|
|
569
1079
|
}
|
|
570
1080
|
return false;
|
|
571
1081
|
});
|
|
572
1082
|
|
|
573
1083
|
function disconnectWs() {
|
|
574
|
-
|
|
1084
|
+
for (const [port, sock] of sockets) {
|
|
575
1085
|
try {
|
|
576
|
-
|
|
1086
|
+
sock.close();
|
|
577
1087
|
} catch {
|
|
578
1088
|
/* ignore */
|
|
579
1089
|
}
|
|
580
|
-
|
|
1090
|
+
sockets.delete(port);
|
|
581
1091
|
}
|
|
582
1092
|
}
|
|
583
1093
|
|
|
@@ -585,7 +1095,6 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
|
|
|
585
1095
|
return new Promise((resolve, reject) => {
|
|
586
1096
|
const url = `ws://127.0.0.1:${port}`;
|
|
587
1097
|
const socket = new WebSocket(url);
|
|
588
|
-
ws = socket;
|
|
589
1098
|
let opened = false;
|
|
590
1099
|
const failTimer = setTimeout(() => {
|
|
591
1100
|
if (!opened) {
|
|
@@ -600,6 +1109,10 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
|
|
|
600
1109
|
socket.onopen = () => {
|
|
601
1110
|
opened = true;
|
|
602
1111
|
clearTimeout(failTimer);
|
|
1112
|
+
sockets.set(port, socket);
|
|
1113
|
+
diag.failingSince = null;
|
|
1114
|
+
void chrome.storage.local.set({ everConnected: true });
|
|
1115
|
+
chrome.windows.getLastFocused().then((w) => w.focused && socket.send(JSON.stringify({ proto: 1, type: 'event', id: crypto.randomUUID(), payload: { kind: 'focus', windowId: w.id } }))).catch(() => undefined);
|
|
603
1116
|
socket.send(
|
|
604
1117
|
JSON.stringify({
|
|
605
1118
|
proto: 1,
|
|
@@ -609,60 +1122,110 @@ function connectWsRaw(port, token, failTimeMs = 10000) {
|
|
|
609
1122
|
token: token || '',
|
|
610
1123
|
extensionId: chrome.runtime.id,
|
|
611
1124
|
client: 'browser_automation_ext',
|
|
612
|
-
clientVersion:
|
|
1125
|
+
clientVersion: chrome.runtime.getManifest().version,
|
|
1126
|
+
instanceId: identity.instanceId,
|
|
1127
|
+
label: identity.label
|
|
613
1128
|
}
|
|
614
1129
|
})
|
|
615
1130
|
);
|
|
616
|
-
resolve(
|
|
1131
|
+
resolve(port);
|
|
617
1132
|
};
|
|
618
1133
|
socket.onerror = () => {
|
|
619
1134
|
clearTimeout(failTimer);
|
|
620
1135
|
reject(new Error('websocket_error'));
|
|
621
1136
|
};
|
|
622
|
-
socket.onclose = () => {
|
|
623
|
-
if (
|
|
624
|
-
|
|
1137
|
+
socket.onclose = (ev) => {
|
|
1138
|
+
if (ev.code === 4000) {
|
|
1139
|
+
// The server already has an extension (another Chrome profile). Leave it alone for a while instead of
|
|
1140
|
+
// reconnecting every few seconds.
|
|
1141
|
+
busyUntil.set(port, Date.now() + BUSY_BACKOFF_MS);
|
|
1142
|
+
}
|
|
1143
|
+
if (sockets.get(port) === socket) {
|
|
1144
|
+
sockets.delete(port);
|
|
1145
|
+
fastRetryUntil = Date.now() + 120_000;
|
|
1146
|
+
diag.failingSince = Date.now();
|
|
1147
|
+
if (ev.code !== 4000) {
|
|
1148
|
+
// Server restarted or went away: try again shortly (alarm is the backup if this worker sleeps).
|
|
1149
|
+
setTimeout(() => void autoConnectTick(), 3000);
|
|
1150
|
+
}
|
|
625
1151
|
}
|
|
626
1152
|
};
|
|
627
1153
|
socket.onmessage = (ev) => {
|
|
628
|
-
void handleServerJson(ev.data);
|
|
1154
|
+
void handleServerJson(ev.data, socket, port);
|
|
629
1155
|
};
|
|
630
1156
|
});
|
|
631
1157
|
}
|
|
632
1158
|
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
1159
|
+
/**
|
|
1160
|
+
* Ports to look at. Chrome writes one entry on the extension's Errors page for every WebSocket attempt that is refused,
|
|
1161
|
+
* so this is built to attempt as little as possible:
|
|
1162
|
+
* - the standard slots the installer and the command line use (Cursor 18791, Claude Desktop 18792, command line 18795),
|
|
1163
|
+
* - ports that worked before, and the port the user typed in the popup,
|
|
1164
|
+
* - a wider range (18791 to 18800) ONLY when the user presses Connect.
|
|
1165
|
+
* A port that refuses is not retried right away: the wait doubles each time (10 s, 20 s, 40 s ... up to 5 min), and a
|
|
1166
|
+
* longer base wait applies while something is already connected. Repeated refusals of the same port show up as one
|
|
1167
|
+
* entry with a count, so a stopped helper costs a handful of lines, not a wall of them.
|
|
1168
|
+
*/
|
|
1169
|
+
const STANDARD_PORTS = [18791, 18792, 18795];
|
|
1170
|
+
const WIDE_SCAN_END = 18800;
|
|
1171
|
+
const BACKOFF_BASE_MS = 10_000;
|
|
1172
|
+
const BACKOFF_BASE_WHEN_CONNECTED_MS = 60_000;
|
|
1173
|
+
const BACKOFF_MAX_MS = 5 * 60_000;
|
|
1174
|
+
/** port -> { n: failed attempts in a row, next: earliest time to try again } */
|
|
1175
|
+
const portBackoff = new Map();
|
|
1176
|
+
|
|
1177
|
+
function candidatePorts(startPort, known, wide) {
|
|
1178
|
+
const set = new Set([...STANDARD_PORTS, ...known]);
|
|
1179
|
+
if (Number.isFinite(startPort)) {
|
|
1180
|
+
set.add(startPort);
|
|
1181
|
+
}
|
|
1182
|
+
if (wide) {
|
|
1183
|
+
for (let p = DEFAULT_WS_PORT; p <= WIDE_SCAN_END; p++) {
|
|
1184
|
+
set.add(p);
|
|
647
1185
|
}
|
|
648
1186
|
}
|
|
649
|
-
|
|
1187
|
+
return [...set].filter((p) => p > 0 && p < 65536);
|
|
650
1188
|
}
|
|
651
1189
|
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
1190
|
+
/**
|
|
1191
|
+
* Connect to every live bridge among the candidate ports, skipping ports we already hold and ports that are waiting out a
|
|
1192
|
+
* back-off. Resolves to the sorted list of connected ports; throws if none are connected.
|
|
1193
|
+
*/
|
|
1194
|
+
async function connectAll(startPort, token, opts = {}) {
|
|
1195
|
+
const { knownPorts = [] } = await chrome.storage.local.get('knownPorts');
|
|
1196
|
+
const manual = opts.full === true;
|
|
1197
|
+
if (manual) {
|
|
1198
|
+
portBackoff.clear();
|
|
1199
|
+
}
|
|
1200
|
+
const now = Date.now();
|
|
1201
|
+
const todo = candidatePorts(startPort, knownPorts, manual).filter(
|
|
1202
|
+
(p) => !sockets.has(p) && (busyUntil.get(p) ?? 0) < now && (portBackoff.get(p)?.next ?? 0) <= now
|
|
1203
|
+
);
|
|
1204
|
+
const results = await Promise.allSettled(todo.map((p) => connectWsRaw(p, token, 2500)));
|
|
1205
|
+
const base = sockets.size > 0 ? BACKOFF_BASE_WHEN_CONNECTED_MS : BACKOFF_BASE_MS;
|
|
1206
|
+
todo.forEach((p, i) => {
|
|
1207
|
+
if (results[i].status === 'fulfilled') {
|
|
1208
|
+
portBackoff.delete(p);
|
|
659
1209
|
} else {
|
|
660
|
-
|
|
1210
|
+
const n = (portBackoff.get(p)?.n ?? 0) + 1;
|
|
1211
|
+
portBackoff.set(p, { n, next: Date.now() + Math.min(BACKOFF_MAX_MS, base * 2 ** (n - 1)) });
|
|
661
1212
|
}
|
|
1213
|
+
});
|
|
1214
|
+
if (sockets.size === 0) {
|
|
1215
|
+
throw new Error('websocket_error');
|
|
662
1216
|
}
|
|
1217
|
+
const ports = [...sockets.keys()].sort((x, y) => x - y);
|
|
1218
|
+
await chrome.storage.local.set({ wsPort: ports[0], knownPorts: [...new Set([...knownPorts, ...ports])].slice(-8) });
|
|
1219
|
+
return ports;
|
|
663
1220
|
}
|
|
664
1221
|
|
|
665
|
-
async function
|
|
1222
|
+
async function connectWs(port, token) {
|
|
1223
|
+
await identityReady;
|
|
1224
|
+
busyUntil.clear(); // a manual Connect retries everything
|
|
1225
|
+
return connectAll(port, token, { full: true });
|
|
1226
|
+
}
|
|
1227
|
+
|
|
1228
|
+
async function handleServerJson(raw, socket, port) {
|
|
666
1229
|
const msg = JSON.parse(raw);
|
|
667
1230
|
if (msg.type === 'session.opened') {
|
|
668
1231
|
const bp = msg.payload?.bridgePort;
|
|
@@ -673,15 +1236,16 @@ async function handleServerJson(raw) {
|
|
|
673
1236
|
return;
|
|
674
1237
|
}
|
|
675
1238
|
if (msg.type === 'ping' && msg.id) {
|
|
676
|
-
|
|
1239
|
+
socket.send(JSON.stringify({ proto: 1, type: 'pong', id: msg.id, payload: {} }));
|
|
677
1240
|
return;
|
|
678
1241
|
}
|
|
679
1242
|
if (msg.type === 'request' && msg.id && msg.payload) {
|
|
680
1243
|
const requestId = msg.id;
|
|
681
1244
|
const { method, params } = msg.payload;
|
|
1245
|
+
lastRequest = { port, method: String(method), at: Date.now() };
|
|
682
1246
|
try {
|
|
683
1247
|
const result = await dispatchRequest(method, params ?? {}, requestId);
|
|
684
|
-
|
|
1248
|
+
socket.send(
|
|
685
1249
|
JSON.stringify({
|
|
686
1250
|
proto: 1,
|
|
687
1251
|
type: 'response',
|
|
@@ -691,7 +1255,7 @@ async function handleServerJson(raw) {
|
|
|
691
1255
|
);
|
|
692
1256
|
} catch (e) {
|
|
693
1257
|
const code = e.code || 'internal';
|
|
694
|
-
|
|
1258
|
+
socket.send(
|
|
695
1259
|
JSON.stringify({
|
|
696
1260
|
proto: 1,
|
|
697
1261
|
type: 'response',
|
|
@@ -708,4 +1272,154 @@ async function handleServerJson(raw) {
|
|
|
708
1272
|
}
|
|
709
1273
|
return;
|
|
710
1274
|
}
|
|
711
|
-
}
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
// ---- just-in-time DevTools grant --------------------------------------------------------------
|
|
1278
|
+
// When a call needs the debugger (dark-mode emulation, full-page or clipped screenshots, engine:'cdp') and the optional
|
|
1279
|
+
// permission is not granted yet, open a small window with one Allow button: the click is the user gesture Chrome needs.
|
|
1280
|
+
// Chrome remembers the grant, so this happens once. The debugger is attached only for the call (see cdp.js withCdp).
|
|
1281
|
+
|
|
1282
|
+
let grantWaiters = [];
|
|
1283
|
+
let grantWindowId = null;
|
|
1284
|
+
let grantTimer = null;
|
|
1285
|
+
|
|
1286
|
+
function finishGrant(ok) {
|
|
1287
|
+
const waiters = grantWaiters;
|
|
1288
|
+
grantWaiters = [];
|
|
1289
|
+
clearTimeout(grantTimer);
|
|
1290
|
+
for (const r of waiters) {
|
|
1291
|
+
r(ok);
|
|
1292
|
+
}
|
|
1293
|
+
if (grantWindowId != null) {
|
|
1294
|
+
const id = grantWindowId;
|
|
1295
|
+
grantWindowId = null;
|
|
1296
|
+
chrome.windows.remove(id).catch(() => undefined);
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1300
|
+
async function requestDevtoolsGrant() {
|
|
1301
|
+
if (await cdp.enabled()) {
|
|
1302
|
+
return true;
|
|
1303
|
+
}
|
|
1304
|
+
const { autoDevtools } = await chrome.storage.local.get('autoDevtools');
|
|
1305
|
+
if (autoDevtools === false) {
|
|
1306
|
+
return false;
|
|
1307
|
+
}
|
|
1308
|
+
return new Promise((resolve) => {
|
|
1309
|
+
grantWaiters.push(resolve);
|
|
1310
|
+
if (grantWaiters.length === 1) {
|
|
1311
|
+
chrome.windows
|
|
1312
|
+
.create({
|
|
1313
|
+
url: chrome.runtime.getURL('devtools-grant.html'),
|
|
1314
|
+
type: 'popup',
|
|
1315
|
+
width: 600,
|
|
1316
|
+
height: 420,
|
|
1317
|
+
focused: true
|
|
1318
|
+
})
|
|
1319
|
+
.then((w) => {
|
|
1320
|
+
grantWindowId = w.id ?? null;
|
|
1321
|
+
})
|
|
1322
|
+
.catch(() => finishGrant(false));
|
|
1323
|
+
grantTimer = setTimeout(() => finishGrant(false), 50000);
|
|
1324
|
+
}
|
|
1325
|
+
});
|
|
1326
|
+
}
|
|
1327
|
+
cdp.setGrantRequester(requestDevtoolsGrant);
|
|
1328
|
+
|
|
1329
|
+
chrome.runtime.onMessage.addListener((msg) => {
|
|
1330
|
+
if (msg?.type === 'devtools-grant-result') {
|
|
1331
|
+
finishGrant(msg.granted === true);
|
|
1332
|
+
}
|
|
1333
|
+
return false;
|
|
1334
|
+
});
|
|
1335
|
+
chrome.windows.onRemoved.addListener((id) => {
|
|
1336
|
+
if (id === grantWindowId) {
|
|
1337
|
+
grantWindowId = null;
|
|
1338
|
+
finishGrant(false);
|
|
1339
|
+
}
|
|
1340
|
+
});
|
|
1341
|
+
|
|
1342
|
+
// ---- focus: tell every server when a window of this profile gets focus ----------------------------
|
|
1343
|
+
// A server uses it to send commands that name no profile to the profile the user used last.
|
|
1344
|
+
function announceFocus(windowId) {
|
|
1345
|
+
if (windowId === chrome.windows.WINDOW_ID_NONE) {
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
1348
|
+
const frame = JSON.stringify({ proto: 1, type: 'event', id: crypto.randomUUID(), payload: { kind: 'focus', windowId } });
|
|
1349
|
+
for (const sock of sockets.values()) {
|
|
1350
|
+
try {
|
|
1351
|
+
sock.send(frame);
|
|
1352
|
+
} catch {
|
|
1353
|
+
/* closing */
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
chrome.windows.onFocusChanged.addListener(announceFocus);
|
|
1358
|
+
|
|
1359
|
+
// ---- auto connect -----------------------------------------------------------------------------
|
|
1360
|
+
// Find the local bridge without the popup: try the last-used port (then the default), scanning upward
|
|
1361
|
+
// through WS_PORT_FALLBACK_ATTEMPTS ports. Runs when Chrome starts, when this worker wakes, after a drop,
|
|
1362
|
+
// and every 30 s while disconnected. Turned off by a manual Disconnect.
|
|
1363
|
+
|
|
1364
|
+
let autoConnecting = false;
|
|
1365
|
+
/** Shown in the popup so a missing connection can be explained. */
|
|
1366
|
+
const diag = { lastTickAt: 0, lastError: null, failingSince: null };
|
|
1367
|
+
let fastRetryUntil = Date.now() + 120_000;
|
|
1368
|
+
let lastScanAt = 0;
|
|
1369
|
+
/** While at least one server is connected, only look for more every 5 minutes (each refused port is logged by Chrome). */
|
|
1370
|
+
const RESCAN_WHEN_CONNECTED_MS = 60 * 1000;
|
|
1371
|
+
|
|
1372
|
+
async function autoConnectTick() {
|
|
1373
|
+
await identityReady;
|
|
1374
|
+
if (autoConnecting) {
|
|
1375
|
+
return;
|
|
1376
|
+
}
|
|
1377
|
+
if (sockets.size > 0 && Date.now() - lastScanAt < RESCAN_WHEN_CONNECTED_MS) {
|
|
1378
|
+
return;
|
|
1379
|
+
}
|
|
1380
|
+
const o = await chrome.storage.local.get(['wsPort', 'sessionToken']);
|
|
1381
|
+
if ((await chrome.storage.session.get('autoConnectOff')).autoConnectOff === true) {
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
autoConnecting = true;
|
|
1385
|
+
lastScanAt = Date.now();
|
|
1386
|
+
diag.lastTickAt = Date.now();
|
|
1387
|
+
try {
|
|
1388
|
+
const before = sockets.size;
|
|
1389
|
+
await connectAll(Number(o.wsPort) || DEFAULT_WS_PORT, o.sessionToken || '');
|
|
1390
|
+
if (sockets.size !== before) {
|
|
1391
|
+
console.info('[browser-automation] connected to ports', [...sockets.keys()]);
|
|
1392
|
+
}
|
|
1393
|
+
diag.lastError = null;
|
|
1394
|
+
if (sockets.size > 0) {
|
|
1395
|
+
diag.failingSince = null;
|
|
1396
|
+
} else if (diag.failingSince == null) {
|
|
1397
|
+
diag.failingSince = Date.now();
|
|
1398
|
+
}
|
|
1399
|
+
} catch (e) {
|
|
1400
|
+
/* no server yet: the alarm tries again */
|
|
1401
|
+
diag.lastError = String(e?.message ?? e);
|
|
1402
|
+
if (diag.failingSince == null) {
|
|
1403
|
+
diag.failingSince = Date.now();
|
|
1404
|
+
}
|
|
1405
|
+
} finally {
|
|
1406
|
+
autoConnecting = false;
|
|
1407
|
+
// While nothing is connected, look again every 10 s for the first two minutes (the 30 s alarm is the backup).
|
|
1408
|
+
if (sockets.size === 0 && Date.now() < fastRetryUntil) {
|
|
1409
|
+
setTimeout(() => void autoConnectTick(), 10_000);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
|
|
1414
|
+
try {
|
|
1415
|
+
chrome.alarms.create('ba-reconnect', { periodInMinutes: 0.5 });
|
|
1416
|
+
chrome.alarms.onAlarm.addListener((a) => {
|
|
1417
|
+
if (a.name === 'ba-reconnect') {
|
|
1418
|
+
void autoConnectTick();
|
|
1419
|
+
}
|
|
1420
|
+
});
|
|
1421
|
+
} catch (e) {
|
|
1422
|
+
console.warn('[browser-automation] alarms unavailable', e);
|
|
1423
|
+
}
|
|
1424
|
+
chrome.runtime.onStartup.addListener(() => void autoConnectTick());
|
|
1425
|
+
void autoConnectTick();
|