@agentruntime-labs/browser-agent 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +6 -0
- package/README.md +64 -34
- package/bin/browser-agent.js +1 -0
- package/cli/control.mjs +477 -0
- package/cli/daemon.mjs +143 -0
- package/cli/hosts.mjs +1 -0
- package/cli/main.mjs +24 -0
- package/cli/mcp.mjs +1 -0
- package/docs/ARCHITECTURE.md +73 -0
- package/docs/DESKTOP_MCP.md +41 -0
- package/docs/EXTENSION_MANIFEST.md +163 -0
- package/docs/HYBRID.md +12 -0
- package/docs/MANAGED_BROWSER.md +111 -0
- package/docs/MCP_TOOLS.md +486 -0
- package/docs/MESSAGES.md +272 -0
- package/docs/MVP.md +76 -0
- package/docs/PRIVACY_POLICY.md +50 -0
- package/docs/README.md +27 -0
- package/docs/REFERENCE_PROJECTS.md +24 -0
- package/docs/RELEASE_CHECKLIST.md +49 -0
- package/docs/REPEATED_DOCS.md +27 -0
- package/docs/SAFETY.md +92 -0
- package/docs/SHELL_QUOTING.md +30 -0
- package/docs/SPLIT_REPOS.md +46 -0
- package/docs/THIRD_PARTY_LICENSES.md +182 -0
- package/docs/VISION.md +38 -0
- package/docs/WEB_STORE_SUBMISSION.md +117 -0
- package/extension/approval.js +1 -0
- package/extension/assets/agentruntime-logo.svg +77 -0
- package/extension/assets/agentruntime-mark-512.png +0 -0
- package/extension/background.js +776 -62
- package/extension/cdp.js +577 -0
- package/extension/constants.js +1 -0
- package/extension/content.js +69 -9
- package/extension/devtools-grant.html +31 -0
- package/extension/devtools-grant.js +29 -0
- package/extension/icons/icon128.png +0 -0
- package/extension/icons/icon16.png +0 -0
- package/extension/icons/icon32.png +0 -0
- package/extension/icons/icon48.png +0 -0
- package/extension/manifest.json +50 -23
- package/extension/policy.js +102 -0
- package/extension/popup.html +190 -56
- package/extension/popup.js +376 -166
- package/extension/soft.js +301 -0
- package/extension/welcome.html +91 -0
- package/extension/welcome.js +13 -0
- package/package.json +6 -2
- package/server/dist/arBrowserPaths.d.ts.map +1 -1
- package/server/dist/arBrowserPaths.js +1 -0
- package/server/dist/arBrowserPaths.js.map +1 -1
- package/server/dist/automationBackend.d.ts +46 -0
- package/server/dist/automationBackend.d.ts.map +1 -0
- package/server/dist/automationBackend.js +20 -0
- package/server/dist/automationBackend.js.map +1 -0
- package/server/dist/automationSession.d.ts.map +1 -1
- package/server/dist/automationSession.js +1 -0
- package/server/dist/automationSession.js.map +1 -1
- package/server/dist/bridge.d.ts +17 -4
- package/server/dist/bridge.d.ts.map +1 -1
- package/server/dist/bridge.js +132 -33
- package/server/dist/bridge.js.map +1 -1
- package/server/dist/browserToolsExtras.d.ts +2 -2
- package/server/dist/browserToolsExtras.d.ts.map +1 -1
- package/server/dist/browserToolsExtras.js +70 -1
- package/server/dist/browserToolsExtras.js.map +1 -1
- package/server/dist/cdpTools.d.ts +5 -0
- package/server/dist/cdpTools.d.ts.map +1 -0
- package/server/dist/cdpTools.js +411 -0
- package/server/dist/cdpTools.js.map +1 -0
- package/server/dist/compositeTools.d.ts +2 -2
- package/server/dist/compositeTools.d.ts.map +1 -1
- package/server/dist/compositeTools.js.map +1 -1
- package/server/dist/debugSnapshot.d.ts +2 -2
- package/server/dist/debugSnapshot.d.ts.map +1 -1
- package/server/dist/debugSnapshot.js.map +1 -1
- package/server/dist/guardrails.d.ts.map +1 -1
- package/server/dist/guardrails.js +1 -0
- package/server/dist/guardrails.js.map +1 -1
- package/server/dist/humanLike.d.ts.map +1 -1
- package/server/dist/humanLike.js +1 -0
- package/server/dist/humanLike.js.map +1 -1
- package/server/dist/main.js +56 -15
- package/server/dist/main.js.map +1 -1
- package/server/dist/managedBridge.d.ts +58 -0
- package/server/dist/managedBridge.d.ts.map +1 -0
- package/server/dist/managedBridge.js +619 -0
- package/server/dist/managedBridge.js.map +1 -0
- package/server/dist/managedDom.d.ts +58 -0
- package/server/dist/managedDom.d.ts.map +1 -0
- package/server/dist/managedDom.js +310 -0
- package/server/dist/managedDom.js.map +1 -0
- package/server/dist/policyTiers.d.ts.map +1 -1
- package/server/dist/policyTiers.js +3 -1
- package/server/dist/policyTiers.js.map +1 -1
- package/server/dist/protocol.d.ts +4 -0
- package/server/dist/protocol.d.ts.map +1 -1
- package/server/dist/protocol.js +1 -0
- package/server/dist/protocol.js.map +1 -1
- package/server/dist/recorder.d.ts +22 -0
- package/server/dist/recorder.d.ts.map +1 -0
- package/server/dist/recorder.js +82 -0
- package/server/dist/recorder.js.map +1 -0
- package/server/dist/richEditorFill.d.ts.map +1 -1
- package/server/dist/richEditorFill.js +1 -0
- package/server/dist/richEditorFill.js.map +1 -1
- package/server/dist/screenshotCrop.d.ts.map +1 -1
- package/server/dist/screenshotCrop.js +1 -0
- package/server/dist/screenshotCrop.js.map +1 -1
- package/server/dist/sessionLog.d.ts +1 -0
- package/server/dist/sessionLog.d.ts.map +1 -1
- package/server/dist/sessionLog.js +1 -0
- package/server/dist/sessionLog.js.map +1 -1
- package/server/dist/sessionPhase.d.ts.map +1 -1
- package/server/dist/sessionPhase.js.map +1 -1
- package/server/dist/sessionSummary.d.ts.map +1 -1
- package/server/dist/sessionSummary.js +1 -0
- package/server/dist/sessionSummary.js.map +1 -1
- package/server/dist/shotUtil.d.ts +44 -0
- package/server/dist/shotUtil.d.ts.map +1 -0
- package/server/dist/shotUtil.js +89 -0
- package/server/dist/shotUtil.js.map +1 -0
- package/server/dist/skillsInstructions.d.ts.map +1 -1
- package/server/dist/skillsInstructions.js +1 -0
- package/server/dist/skillsInstructions.js.map +1 -1
- package/server/dist/snapshotText.d.ts +47 -0
- package/server/dist/snapshotText.d.ts.map +1 -0
- package/server/dist/snapshotText.js +99 -0
- package/server/dist/snapshotText.js.map +1 -0
- package/server/dist/structureFilter.d.ts.map +1 -1
- package/server/dist/structureFilter.js.map +1 -1
- package/server/dist/surveyCatalog.d.ts.map +1 -1
- package/server/dist/surveyCatalog.js +1 -0
- package/server/dist/surveyCatalog.js.map +1 -1
- package/server/dist/surveyRunner.d.ts.map +1 -1
- package/server/dist/surveyRunner.js.map +1 -1
- package/server/dist/toolHelpers.d.ts +27 -2
- package/server/dist/toolHelpers.d.ts.map +1 -1
- package/server/dist/toolHelpers.js +67 -1
- package/server/dist/toolHelpers.js.map +1 -1
- package/server/dist/toolRuntime.d.ts +4 -0
- package/server/dist/toolRuntime.d.ts.map +1 -1
- package/server/dist/toolRuntime.js +12 -6
- package/server/dist/toolRuntime.js.map +1 -1
- package/server/dist/toolSchemas.d.ts +37 -1
- package/server/dist/toolSchemas.d.ts.map +1 -1
- package/server/dist/toolSchemas.js +33 -1
- package/server/dist/toolSchemas.js.map +1 -1
- package/server/dist/tools.d.ts +2 -2
- package/server/dist/tools.d.ts.map +1 -1
- package/server/dist/tools.js +65 -11
- package/server/dist/tools.js.map +1 -1
- package/server/dist/verdictExtract.d.ts.map +1 -1
- package/server/dist/verdictExtract.js +1 -0
- package/server/dist/verdictExtract.js.map +1 -1
- package/server/dist/workflowTools.d.ts +2 -2
- package/server/dist/workflowTools.d.ts.map +1 -1
- package/server/dist/workflowTools.js +16 -4
- package/server/dist/workflowTools.js.map +1 -1
- package/skills/browser-agent/SKILL.md +89 -0
package/extension/cdp.js
ADDED
|
@@ -0,0 +1,577 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
/**
|
|
3
|
+
* Chrome DevTools Protocol helpers (chrome.debugger) for the AgentRuntime Browser Agent extension.
|
|
4
|
+
*
|
|
5
|
+
* Why: chrome.tabs.captureVisibleTab needs a visible, focused window. CDP can screenshot a background
|
|
6
|
+
* or covered tab, emulate a phone/tablet viewport and dark mode, send real mouse events (double click,
|
|
7
|
+
* right click, drag), stream a screencast for recordings, and use the clipboard.
|
|
8
|
+
*
|
|
9
|
+
* Chrome shows its "started debugging this browser" bar while a tab is attached. We attach per call and
|
|
10
|
+
* detach again, unless something sticky (emulation, recording) needs the session to stay.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
const PROTOCOL = '1.3';
|
|
14
|
+
|
|
15
|
+
/** tabId -> { emulation: boolean, recording: boolean, perms: boolean } */
|
|
16
|
+
const sticky = new Map();
|
|
17
|
+
/** tabIds we currently hold. */
|
|
18
|
+
const attached = new Set();
|
|
19
|
+
|
|
20
|
+
let listenersReady = false;
|
|
21
|
+
function ensureListeners() {
|
|
22
|
+
if (listenersReady || !chrome.debugger) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
listenersReady = true;
|
|
26
|
+
chrome.debugger.onDetach.addListener((source) => {
|
|
27
|
+
if (source.tabId != null) {
|
|
28
|
+
attached.delete(source.tabId);
|
|
29
|
+
const st = sticky.get(source.tabId);
|
|
30
|
+
if (st) {
|
|
31
|
+
st.emulation = false;
|
|
32
|
+
st.recording = false;
|
|
33
|
+
st.perms = false;
|
|
34
|
+
}
|
|
35
|
+
recordings.delete(source.tabId);
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
chrome.debugger.onEvent.addListener((source, method, params) => {
|
|
39
|
+
if (method === 'Page.screencastFrame' && source.tabId != null) {
|
|
40
|
+
onScreencastFrame(source.tabId, params);
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Chrome does not allow `debugger` as an optional permission, so the permission is declared in the manifest and this
|
|
47
|
+
* extension gates it itself: the debugger is never attached unless the user allowed DevTools features, either until
|
|
48
|
+
* they switch it off ('always') or for a limited time (`devtoolsUntil`). The Chrome debugging bar only shows while a
|
|
49
|
+
* call that needs it has the debugger attached.
|
|
50
|
+
*/
|
|
51
|
+
export async function enabled() {
|
|
52
|
+
try {
|
|
53
|
+
if (!chrome.debugger) {
|
|
54
|
+
return false;
|
|
55
|
+
}
|
|
56
|
+
const o = await chrome.storage.local.get(['devtoolsMode', 'devtoolsUntil']);
|
|
57
|
+
return o.devtoolsMode === 'always' || (typeof o.devtoolsUntil === 'number' && o.devtoolsUntil > Date.now());
|
|
58
|
+
} catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function stickyOf(tabId) {
|
|
64
|
+
let s = sticky.get(tabId);
|
|
65
|
+
if (!s) {
|
|
66
|
+
s = { emulation: false, recording: false, perms: false };
|
|
67
|
+
sticky.set(tabId, s);
|
|
68
|
+
}
|
|
69
|
+
return s;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function cdpError(message, code = 'cdp_error') {
|
|
73
|
+
return Object.assign(new Error(message), { code });
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Set by background.js: resolves true once the user has granted the debugger permission (opens a one-click window). */
|
|
77
|
+
let grantRequester = null;
|
|
78
|
+
export function setGrantRequester(fn) {
|
|
79
|
+
grantRequester = fn;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async function attach(tabId) {
|
|
83
|
+
if (attached.has(tabId)) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
if (!(await enabled()) && grantRequester) {
|
|
87
|
+
await grantRequester();
|
|
88
|
+
}
|
|
89
|
+
if (!(await enabled()) || !chrome.debugger) {
|
|
90
|
+
throw cdpError(
|
|
91
|
+
'DevTools features are off. They make Chrome show a "started debugging this browser" bar. ' +
|
|
92
|
+
'Allow them when asked, or turn them on in the extension popup (Enable DevTools features).',
|
|
93
|
+
'cdp_not_enabled'
|
|
94
|
+
);
|
|
95
|
+
}
|
|
96
|
+
ensureListeners();
|
|
97
|
+
try {
|
|
98
|
+
await chrome.debugger.attach({ tabId }, PROTOCOL);
|
|
99
|
+
} catch (e) {
|
|
100
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
101
|
+
if (/already attached/i.test(msg)) {
|
|
102
|
+
attached.add(tabId);
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (/cannot (access|attach)|chrome:\/\//i.test(msg)) {
|
|
106
|
+
throw cdpError(`Cannot debug this page (${msg}). chrome:// and Web Store pages are not allowed.`, 'cdp_not_allowed');
|
|
107
|
+
}
|
|
108
|
+
throw cdpError(`debugger attach failed: ${msg}`);
|
|
109
|
+
}
|
|
110
|
+
attached.add(tabId);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function detachIfIdle(tabId) {
|
|
114
|
+
const s = sticky.get(tabId);
|
|
115
|
+
if (s && (s.emulation || s.recording || s.perms)) {
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (!attached.has(tabId)) {
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
try {
|
|
122
|
+
await chrome.debugger.detach({ tabId });
|
|
123
|
+
} catch {
|
|
124
|
+
/* already gone */
|
|
125
|
+
}
|
|
126
|
+
attached.delete(tabId);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Send one CDP command (caller must have attached). */
|
|
130
|
+
function send(tabId, method, params = {}) {
|
|
131
|
+
return chrome.debugger.sendCommand({ tabId }, method, params);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function withTimeout(promise, ms, what) {
|
|
135
|
+
let t;
|
|
136
|
+
return Promise.race([
|
|
137
|
+
promise,
|
|
138
|
+
new Promise((_, reject) => {
|
|
139
|
+
t = setTimeout(() => reject(cdpError(`${what} timed out after ${ms} ms`, 'cdp_timeout')), ms);
|
|
140
|
+
})
|
|
141
|
+
]).finally(() => clearTimeout(t));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Attach, run fn(send), detach if idle. Re-attaches once if Chrome dropped the debugger mid-call. */
|
|
145
|
+
export async function withCdp(tabId, fn) {
|
|
146
|
+
for (let attempt = 0; ; attempt++) {
|
|
147
|
+
await attach(tabId);
|
|
148
|
+
try {
|
|
149
|
+
return await fn((method, params) => send(tabId, method, params));
|
|
150
|
+
} catch (e) {
|
|
151
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
152
|
+
if (attempt === 0 && /detached|not attached|target closed/i.test(msg)) {
|
|
153
|
+
attached.delete(tabId);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
throw e;
|
|
157
|
+
} finally {
|
|
158
|
+
await detachIfIdle(tabId);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// ---- screenshots -----------------------------------------------------------------------------
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Screenshot without needing the window to be visible.
|
|
167
|
+
* @param {number} tabId
|
|
168
|
+
* @param {{format?: 'png'|'jpeg', quality?: number, fullPage?: boolean, clip?: {x:number,y:number,width:number,height:number}}} o
|
|
169
|
+
*/
|
|
170
|
+
export async function cdpScreenshot(tabId, o = {}) {
|
|
171
|
+
const format = o.format === 'png' ? 'png' : 'jpeg';
|
|
172
|
+
return withCdp(tabId, async (cmd) => {
|
|
173
|
+
const params = { format, fromSurface: true, captureBeyondViewport: o.fullPage === true };
|
|
174
|
+
if (format === 'jpeg') {
|
|
175
|
+
params.quality = o.quality ?? 80;
|
|
176
|
+
}
|
|
177
|
+
if (o.clip) {
|
|
178
|
+
params.clip = { ...o.clip, scale: 1 };
|
|
179
|
+
} else if (!o.fullPage) {
|
|
180
|
+
// Always clip to the visual viewport: without an explicit clip a hidden tab under device-scale emulation
|
|
181
|
+
// can come back tiled (the page repeated dsf x dsf times).
|
|
182
|
+
const m = await cmd('Page.getLayoutMetrics');
|
|
183
|
+
const vp = m.cssVisualViewport || m.visualViewport;
|
|
184
|
+
if (vp?.clientWidth && vp?.clientHeight) {
|
|
185
|
+
params.clip = {
|
|
186
|
+
x: vp.pageX ?? 0,
|
|
187
|
+
y: vp.pageY ?? 0,
|
|
188
|
+
width: vp.clientWidth,
|
|
189
|
+
height: vp.clientHeight,
|
|
190
|
+
scale: 1
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
} else if (o.fullPage) {
|
|
194
|
+
const m = await cmd('Page.getLayoutMetrics');
|
|
195
|
+
const cs = m.cssContentSize || m.contentSize;
|
|
196
|
+
const h = Math.min(Math.ceil(cs.height), 16000);
|
|
197
|
+
params.clip = { x: 0, y: 0, width: Math.ceil(cs.width), height: h, scale: 1 };
|
|
198
|
+
}
|
|
199
|
+
// A tab that is hidden or in a minimized window may never paint a frame, so the first capture can hang.
|
|
200
|
+
// Ladder: capture (6 s) -> focus emulation + retry (6 s) -> clear error.
|
|
201
|
+
let shot;
|
|
202
|
+
try {
|
|
203
|
+
shot = await withTimeout(cmd('Page.captureScreenshot', params), 6000, 'Page.captureScreenshot');
|
|
204
|
+
} catch (e) {
|
|
205
|
+
if (e?.code !== 'cdp_timeout') {
|
|
206
|
+
throw e;
|
|
207
|
+
}
|
|
208
|
+
try {
|
|
209
|
+
await cmd('Emulation.setFocusEmulationEnabled', { enabled: true });
|
|
210
|
+
// Chrome only allows surface captures now (fromSurface:false is rejected), so retry the same call.
|
|
211
|
+
shot = await withTimeout(cmd('Page.captureScreenshot', params), 6000, 'Page.captureScreenshot (retry)');
|
|
212
|
+
} catch (e2) {
|
|
213
|
+
if (e2?.code === 'cdp_timeout') {
|
|
214
|
+
throw cdpError(
|
|
215
|
+
'The tab is not painting (its window is probably minimized or covered). ' +
|
|
216
|
+
'Use browser_window_resize with state "normal" (or browser_window_focus), or browser_window_open for a clean window, then retry.',
|
|
217
|
+
'tab_not_painting'
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
throw e2;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return { mimeType: format === 'png' ? 'image/png' : 'image/jpeg', dataBase64: shot.data, method: 'cdp' };
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ---- emulation -------------------------------------------------------------------------------
|
|
228
|
+
|
|
229
|
+
export const VIEWPORT_PRESETS = {
|
|
230
|
+
mobile: { width: 375, height: 812, deviceScaleFactor: 3, mobile: true },
|
|
231
|
+
'mobile-small': { width: 320, height: 568, deviceScaleFactor: 2, mobile: true },
|
|
232
|
+
tablet: { width: 768, height: 1024, deviceScaleFactor: 2, mobile: true },
|
|
233
|
+
laptop: { width: 1366, height: 768, deviceScaleFactor: 1, mobile: false },
|
|
234
|
+
desktop: { width: 1568, height: 772, deviceScaleFactor: 1, mobile: false }
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const MOBILE_UA =
|
|
238
|
+
'Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Mobile Safari/537.36';
|
|
239
|
+
|
|
240
|
+
/**
|
|
241
|
+
* @param {number} tabId
|
|
242
|
+
* @param {{preset?: string, width?: number, height?: number, deviceScaleFactor?: number, mobile?: boolean, userAgent?: string}} o
|
|
243
|
+
*/
|
|
244
|
+
export async function setViewport(tabId, o = {}) {
|
|
245
|
+
const base = o.preset ? VIEWPORT_PRESETS[o.preset] : undefined;
|
|
246
|
+
if (o.preset && !base) {
|
|
247
|
+
throw cdpError(`Unknown preset "${o.preset}". Use: ${Object.keys(VIEWPORT_PRESETS).join(', ')}.`, 'invalid_params');
|
|
248
|
+
}
|
|
249
|
+
const width = o.width ?? base?.width;
|
|
250
|
+
const height = o.height ?? base?.height;
|
|
251
|
+
if (!width || !height) {
|
|
252
|
+
throw cdpError('Give a preset, or width and height.', 'invalid_params');
|
|
253
|
+
}
|
|
254
|
+
const mobile = o.mobile ?? base?.mobile ?? false;
|
|
255
|
+
const dsf = o.deviceScaleFactor ?? base?.deviceScaleFactor ?? 1;
|
|
256
|
+
await attach(tabId);
|
|
257
|
+
stickyOf(tabId).emulation = true;
|
|
258
|
+
await send(tabId, 'Emulation.setDeviceMetricsOverride', {
|
|
259
|
+
width,
|
|
260
|
+
height,
|
|
261
|
+
deviceScaleFactor: dsf,
|
|
262
|
+
mobile,
|
|
263
|
+
screenWidth: width,
|
|
264
|
+
screenHeight: height
|
|
265
|
+
});
|
|
266
|
+
await send(tabId, 'Emulation.setTouchEmulationEnabled', { enabled: mobile, maxTouchPoints: mobile ? 5 : 0 });
|
|
267
|
+
if (mobile || o.userAgent) {
|
|
268
|
+
await send(tabId, 'Emulation.setUserAgentOverride', { userAgent: o.userAgent ?? MOBILE_UA });
|
|
269
|
+
}
|
|
270
|
+
return { tabId, width, height, deviceScaleFactor: dsf, mobile, preset: o.preset ?? null };
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export async function setColorScheme(tabId, scheme) {
|
|
274
|
+
if (!['light', 'dark', 'none'].includes(scheme)) {
|
|
275
|
+
throw cdpError('scheme must be light, dark or none.', 'invalid_params');
|
|
276
|
+
}
|
|
277
|
+
await attach(tabId);
|
|
278
|
+
stickyOf(tabId).emulation = scheme !== 'none' || stickyOf(tabId).emulation;
|
|
279
|
+
await send(tabId, 'Emulation.setEmulatedMedia', {
|
|
280
|
+
features: scheme === 'none' ? [] : [{ name: 'prefers-color-scheme', value: scheme }]
|
|
281
|
+
});
|
|
282
|
+
return { tabId, colorScheme: scheme };
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/** Clear viewport, touch, user agent and color-scheme overrides and release the debugger. */
|
|
286
|
+
export async function resetEmulation(tabId) {
|
|
287
|
+
if (!attached.has(tabId)) {
|
|
288
|
+
return { tabId, reset: true, wasActive: false };
|
|
289
|
+
}
|
|
290
|
+
try {
|
|
291
|
+
await send(tabId, 'Emulation.clearDeviceMetricsOverride');
|
|
292
|
+
await send(tabId, 'Emulation.setTouchEmulationEnabled', { enabled: false });
|
|
293
|
+
await send(tabId, 'Emulation.setUserAgentOverride', { userAgent: '' });
|
|
294
|
+
await send(tabId, 'Emulation.setEmulatedMedia', { features: [] });
|
|
295
|
+
} finally {
|
|
296
|
+
stickyOf(tabId).emulation = false;
|
|
297
|
+
await detachIfIdle(tabId);
|
|
298
|
+
}
|
|
299
|
+
return { tabId, reset: true, wasActive: true };
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// ---- pointer ---------------------------------------------------------------------------------
|
|
303
|
+
|
|
304
|
+
const BUTTONS = { left: 1, right: 2, middle: 4 };
|
|
305
|
+
|
|
306
|
+
async function mouse(cmd, type, x, y, button = 'none', clickCount = 0, buttons = 0) {
|
|
307
|
+
await cmd('Input.dispatchMouseEvent', { type, x, y, button, buttons, clickCount, pointerType: 'mouse' });
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Real mouse input at viewport (CSS px) coordinates.
|
|
312
|
+
* @param {number} tabId
|
|
313
|
+
* @param {{action: 'move'|'click'|'double_click'|'right_click'|'drag', x:number, y:number, toX?:number, toY?:number,
|
|
314
|
+
* holdMs?: number, steps?: number}} o
|
|
315
|
+
*/
|
|
316
|
+
export async function pointer(tabId, o) {
|
|
317
|
+
const { action, x, y } = o;
|
|
318
|
+
if (![x, y].every(Number.isFinite)) {
|
|
319
|
+
throw cdpError('x and y (viewport CSS pixels) are required.', 'invalid_params');
|
|
320
|
+
}
|
|
321
|
+
return withCdp(tabId, async (cmd) => {
|
|
322
|
+
const hold = Math.min(Math.max(o.holdMs ?? 0, 0), 30000);
|
|
323
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
324
|
+
await mouse(cmd, 'mouseMoved', x, y);
|
|
325
|
+
if (action === 'move') {
|
|
326
|
+
if (hold) {
|
|
327
|
+
await sleep(hold);
|
|
328
|
+
}
|
|
329
|
+
} else if (action === 'wheel') {
|
|
330
|
+
await cmd('Input.dispatchMouseEvent', {
|
|
331
|
+
type: 'mouseWheel', x, y, deltaX: Number(o.deltaX) || 0, deltaY: Number(o.deltaY) || 0, pointerType: 'mouse'
|
|
332
|
+
});
|
|
333
|
+
} else if (action === 'click' || action === 'double_click' || action === 'right_click') {
|
|
334
|
+
const button = action === 'right_click' ? 'right' : 'left';
|
|
335
|
+
const n = action === 'double_click' ? 2 : 1;
|
|
336
|
+
for (let i = 1; i <= n; i++) {
|
|
337
|
+
await mouse(cmd, 'mousePressed', x, y, button, i, BUTTONS[button]);
|
|
338
|
+
if (hold && i === n) {
|
|
339
|
+
await sleep(hold);
|
|
340
|
+
}
|
|
341
|
+
await mouse(cmd, 'mouseReleased', x, y, button, i, 0);
|
|
342
|
+
}
|
|
343
|
+
} else if (action === 'drag') {
|
|
344
|
+
const { toX, toY } = o;
|
|
345
|
+
if (![toX, toY].every(Number.isFinite)) {
|
|
346
|
+
throw cdpError('drag needs toX and toY.', 'invalid_params');
|
|
347
|
+
}
|
|
348
|
+
const steps = Math.min(Math.max(o.steps ?? 12, 2), 60);
|
|
349
|
+
await mouse(cmd, 'mousePressed', x, y, 'left', 1, 1);
|
|
350
|
+
for (let i = 1; i <= steps; i++) {
|
|
351
|
+
const px = x + ((toX - x) * i) / steps;
|
|
352
|
+
const py = y + ((toY - y) * i) / steps;
|
|
353
|
+
await mouse(cmd, 'mouseMoved', px, py, 'left', 0, 1);
|
|
354
|
+
await sleep(16);
|
|
355
|
+
}
|
|
356
|
+
if (hold) {
|
|
357
|
+
await sleep(hold);
|
|
358
|
+
}
|
|
359
|
+
await mouse(cmd, 'mouseReleased', toX, toY, 'left', 1, 0);
|
|
360
|
+
} else {
|
|
361
|
+
throw cdpError(`Unknown pointer action "${action}".`, 'invalid_params');
|
|
362
|
+
}
|
|
363
|
+
return { ok: true, action, x, y, toX: o.toX ?? null, toY: o.toY ?? null };
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// ---- clipboard -------------------------------------------------------------------------------
|
|
368
|
+
|
|
369
|
+
async function grantClipboard(tabId) {
|
|
370
|
+
await attach(tabId);
|
|
371
|
+
try {
|
|
372
|
+
await send(tabId, 'Browser.grantPermissions', {
|
|
373
|
+
permissions: ['clipboardReadWrite', 'clipboardSanitizedWrite']
|
|
374
|
+
});
|
|
375
|
+
} catch {
|
|
376
|
+
/* older Chrome: permission may already be implied for user-gesture evaluate */
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
export async function clipboardRead(tabId) {
|
|
381
|
+
await grantClipboard(tabId);
|
|
382
|
+
try {
|
|
383
|
+
const r = await send(tabId, 'Runtime.evaluate', {
|
|
384
|
+
expression: 'navigator.clipboard.readText()',
|
|
385
|
+
awaitPromise: true,
|
|
386
|
+
userGesture: true,
|
|
387
|
+
returnByValue: true
|
|
388
|
+
});
|
|
389
|
+
if (r.exceptionDetails) {
|
|
390
|
+
throw cdpError(r.exceptionDetails.exception?.description || 'clipboard read failed', 'clipboard_denied');
|
|
391
|
+
}
|
|
392
|
+
return { text: String(r.result?.value ?? '') };
|
|
393
|
+
} finally {
|
|
394
|
+
await detachIfIdle(tabId);
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export async function clipboardWrite(tabId, text) {
|
|
399
|
+
await grantClipboard(tabId);
|
|
400
|
+
try {
|
|
401
|
+
const r = await send(tabId, 'Runtime.evaluate', {
|
|
402
|
+
expression: `navigator.clipboard.writeText(${JSON.stringify(String(text))})`,
|
|
403
|
+
awaitPromise: true,
|
|
404
|
+
userGesture: true,
|
|
405
|
+
returnByValue: true
|
|
406
|
+
});
|
|
407
|
+
if (r.exceptionDetails) {
|
|
408
|
+
throw cdpError(r.exceptionDetails.exception?.description || 'clipboard write failed', 'clipboard_denied');
|
|
409
|
+
}
|
|
410
|
+
return { written: true, length: String(text).length };
|
|
411
|
+
} finally {
|
|
412
|
+
await detachIfIdle(tabId);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// ---- recording (screencast) ------------------------------------------------------------------
|
|
417
|
+
|
|
418
|
+
/** tabId -> { frames: {t:number, data:string}[], startedAt:number, maxFrames:number } */
|
|
419
|
+
const recordings = new Map();
|
|
420
|
+
|
|
421
|
+
function onScreencastFrame(tabId, p) {
|
|
422
|
+
const rec = recordings.get(tabId);
|
|
423
|
+
// Always ack, or Chrome stops sending frames.
|
|
424
|
+
chrome.debugger.sendCommand({ tabId }, 'Page.screencastFrameAck', { sessionId: p.sessionId }).catch(() => {});
|
|
425
|
+
if (!rec) {
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
if (rec.frames.length >= rec.maxFrames) {
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
rec.frames.push({ t: (p.metadata?.timestamp ?? Date.now() / 1000) * 1000, data: p.data });
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
export async function recordStart(tabId, o = {}) {
|
|
435
|
+
if (recordings.has(tabId)) {
|
|
436
|
+
throw cdpError('A recording is already running on this tab. Call browser_record_stop first.', 'already_recording');
|
|
437
|
+
}
|
|
438
|
+
await attach(tabId);
|
|
439
|
+
stickyOf(tabId).recording = true;
|
|
440
|
+
recordings.set(tabId, { frames: [], startedAt: Date.now(), maxFrames: o.maxFrames ?? 900 });
|
|
441
|
+
await send(tabId, 'Page.startScreencast', {
|
|
442
|
+
format: 'jpeg',
|
|
443
|
+
quality: o.quality ?? 70,
|
|
444
|
+
maxWidth: o.maxWidth ?? 1280,
|
|
445
|
+
maxHeight: o.maxHeight ?? 1280,
|
|
446
|
+
everyNthFrame: o.everyNthFrame ?? 1
|
|
447
|
+
});
|
|
448
|
+
return { tabId, recording: true };
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
export async function recordStop(tabId) {
|
|
452
|
+
const rec = recordings.get(tabId);
|
|
453
|
+
if (!rec) {
|
|
454
|
+
throw cdpError('No recording is running on this tab.', 'not_recording');
|
|
455
|
+
}
|
|
456
|
+
try {
|
|
457
|
+
await send(tabId, 'Page.stopScreencast');
|
|
458
|
+
} catch {
|
|
459
|
+
/* tab may have navigated away */
|
|
460
|
+
}
|
|
461
|
+
stickyOf(tabId).recording = false;
|
|
462
|
+
await detachIfIdle(tabId);
|
|
463
|
+
// Keep frames for record_fetch; the server drains them in chunks.
|
|
464
|
+
const t0 = rec.frames[0]?.t ?? 0;
|
|
465
|
+
const t1 = rec.frames.at(-1)?.t ?? 0;
|
|
466
|
+
return { tabId, frameCount: rec.frames.length, durationMs: Math.round(t1 - t0) };
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/** Return frames [from, from+count) with timestamps relative to the first frame. */
|
|
470
|
+
export function recordFetch(tabId, from, count) {
|
|
471
|
+
const rec = recordings.get(tabId);
|
|
472
|
+
if (!rec) {
|
|
473
|
+
throw cdpError('No recording buffered for this tab.', 'not_recording');
|
|
474
|
+
}
|
|
475
|
+
const t0 = rec.frames[0]?.t ?? 0;
|
|
476
|
+
const slice = rec.frames.slice(from, from + count).map((f) => ({ tMs: Math.round(f.t - t0), data: f.data }));
|
|
477
|
+
return { frames: slice, total: rec.frames.length };
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
export function recordDiscard(tabId) {
|
|
481
|
+
recordings.delete(tabId);
|
|
482
|
+
return { discarded: true };
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
// ---- windows ---------------------------------------------------------------------------------
|
|
486
|
+
|
|
487
|
+
export async function windowList() {
|
|
488
|
+
const wins = await chrome.windows.getAll({ populate: true });
|
|
489
|
+
return {
|
|
490
|
+
windows: wins.map((w) => ({
|
|
491
|
+
windowId: w.id,
|
|
492
|
+
focused: w.focused,
|
|
493
|
+
state: w.state,
|
|
494
|
+
type: w.type,
|
|
495
|
+
left: w.left,
|
|
496
|
+
top: w.top,
|
|
497
|
+
width: w.width,
|
|
498
|
+
height: w.height,
|
|
499
|
+
tabs: (w.tabs ?? []).map((t) => ({ tabId: t.id, active: t.active, url: (t.url || '').slice(0, 200), title: (t.title || '').slice(0, 100) }))
|
|
500
|
+
}))
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
export async function windowOpen(o = {}) {
|
|
505
|
+
const w = await chrome.windows.create({
|
|
506
|
+
url: o.url || 'about:blank',
|
|
507
|
+
type: o.popup ? 'popup' : 'normal',
|
|
508
|
+
focused: o.focused !== false,
|
|
509
|
+
width: o.width,
|
|
510
|
+
height: o.height,
|
|
511
|
+
left: o.left,
|
|
512
|
+
top: o.top,
|
|
513
|
+
incognito: false
|
|
514
|
+
});
|
|
515
|
+
const tab = w.tabs?.[0];
|
|
516
|
+
return { windowId: w.id, tabId: tab?.id ?? null, width: w.width, height: w.height };
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
export async function windowResize(windowId, o = {}) {
|
|
520
|
+
const upd = {};
|
|
521
|
+
for (const k of ['width', 'height', 'left', 'top']) {
|
|
522
|
+
if (o[k] != null) {
|
|
523
|
+
upd[k] = o[k];
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
if (o.state) {
|
|
527
|
+
upd.state = o.state;
|
|
528
|
+
}
|
|
529
|
+
if (o.focused != null) {
|
|
530
|
+
upd.focused = o.focused;
|
|
531
|
+
}
|
|
532
|
+
if (upd.width != null || upd.height != null || upd.left != null || upd.top != null) {
|
|
533
|
+
upd.state = upd.state ?? 'normal';
|
|
534
|
+
}
|
|
535
|
+
const w = await chrome.windows.update(windowId, upd);
|
|
536
|
+
return { windowId: w.id, state: w.state, left: w.left, top: w.top, width: w.width, height: w.height, focused: w.focused };
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
export async function windowFocus(windowId) {
|
|
540
|
+
const w = await chrome.windows.update(windowId, { focused: true, drawAttention: true });
|
|
541
|
+
return { windowId: w.id, focused: w.focused };
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// ---- downloads -------------------------------------------------------------------------------
|
|
545
|
+
|
|
546
|
+
/** Recent downloads from chrome.downloads, optionally waiting for in-progress ones. */
|
|
547
|
+
export async function downloadsRecent(waitMs = 0, limit = 15) {
|
|
548
|
+
const deadline = Date.now() + Math.min(Math.max(waitMs, 0), 30000);
|
|
549
|
+
let items = [];
|
|
550
|
+
for (;;) {
|
|
551
|
+
items = await chrome.downloads.search({ orderBy: ['-startTime'], limit });
|
|
552
|
+
if (!items.some((d) => d.state === 'in_progress') || Date.now() >= deadline) {
|
|
553
|
+
break;
|
|
554
|
+
}
|
|
555
|
+
await new Promise((r) => setTimeout(r, 300));
|
|
556
|
+
}
|
|
557
|
+
const dir = items.find((d) => d.filename)?.filename?.replace(/[\\/][^\\/]*$/, '') ?? '';
|
|
558
|
+
return {
|
|
559
|
+
downloads: items.map((d) => ({
|
|
560
|
+
id: d.id,
|
|
561
|
+
suggestedFilename: (d.filename || '').split(/[\\/]/).pop() || '',
|
|
562
|
+
url: (d.finalUrl || d.url || '').slice(0, 500),
|
|
563
|
+
state: d.state === 'complete' ? 'completed' : d.state === 'interrupted' ? 'failed' : 'in_progress',
|
|
564
|
+
path: d.filename || undefined,
|
|
565
|
+
sizeBytes: d.fileSize > 0 ? d.fileSize : d.totalBytes > 0 ? d.totalBytes : undefined,
|
|
566
|
+
error: d.error || undefined
|
|
567
|
+
})),
|
|
568
|
+
downloadDir: dir
|
|
569
|
+
};
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
// ---- pure helpers (unit-tested in the server package via a copy-free dynamic import) ----------
|
|
573
|
+
|
|
574
|
+
/** Viewport (top-frame CSS px) centre of a ref's rect given the frame offset. */
|
|
575
|
+
export function centreFromRect(rect, frameOff = { x: 0, y: 0 }) {
|
|
576
|
+
return { x: frameOff.x + rect.x + rect.w / 2, y: frameOff.y + rect.y + rect.h / 2 };
|
|
577
|
+
}
|
package/extension/constants.js
CHANGED