@evan_zhong/browser-attach 0.0.0-stage → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +27 -0
- package/LICENSE +21 -0
- package/README.md +169 -3
- package/bin/browser-attach.mjs +43 -0
- package/docs/ARCHITECTURE.md +131 -0
- package/docs/NOSIS-INTEGRATION.md +53 -0
- package/docs/TEST-REPORT.md +39 -0
- package/docs/USAGE.md +372 -0
- package/extension/actionability.js +66 -0
- package/extension/background.js +98 -0
- package/extension/batch.js +32 -0
- package/extension/click-visuals.js +82 -0
- package/extension/click.js +308 -0
- package/extension/controller.js +812 -0
- package/extension/diagnostics.js +205 -0
- package/extension/drag.js +163 -0
- package/extension/evaluate.js +131 -0
- package/extension/icons/BrowserAttach.png +0 -0
- package/extension/input.js +318 -0
- package/extension/limits.js +6 -0
- package/extension/locator.js +115 -0
- package/extension/manifest.json +34 -0
- package/extension/navigation.js +65 -0
- package/extension/options.html +6 -0
- package/extension/options.js +15 -0
- package/extension/outcomes.js +15 -0
- package/extension/popup.html +5 -0
- package/extension/popup.js +10 -0
- package/extension/scroll.js +29 -0
- package/extension/sessions.js +100 -0
- package/extension/snapshot.js +75 -0
- package/extension/text.js +37 -0
- package/extension/ui.css +12 -0
- package/extension/visuals.js +93 -0
- package/extension/wait.js +81 -0
- package/mcp-config.json +8 -0
- package/package.json +54 -6
- package/scripts/check-package.mjs +20 -0
- package/src/client.mjs +91 -0
- package/src/config.mjs +36 -0
- package/src/hub.mjs +197 -0
- package/src/mcp.mjs +179 -0
- package/src/setup.mjs +59 -0
- package/src/upload.mjs +15 -0
|
@@ -0,0 +1,812 @@
|
|
|
1
|
+
// Browser operations live here; the WebSocket transport never exposes raw CDP.
|
|
2
|
+
import { createVisuals } from './visuals.js';
|
|
3
|
+
import { createClickVisuals } from './click-visuals.js';
|
|
4
|
+
import { renderAX, snapshotPage } from './snapshot.js';
|
|
5
|
+
import { createNavigation } from './navigation.js';
|
|
6
|
+
import { createClick, clickOptions, framePoint } from './click.js';
|
|
7
|
+
import { createInput, inputMode, selectOptions, typeOptions } from './input.js';
|
|
8
|
+
import { createEvaluations } from './evaluate.js';
|
|
9
|
+
import { targetOptions, resolveSelector, resolveFrame } from './locator.js';
|
|
10
|
+
import { readText } from './text.js';
|
|
11
|
+
import { createDiagnostics } from './diagnostics.js';
|
|
12
|
+
import { createSessions } from './sessions.js';
|
|
13
|
+
import { NAVIGATION_OBSERVATION_TIMEOUT_MS } from './limits.js';
|
|
14
|
+
import { deadline } from './actionability.js';
|
|
15
|
+
import { scrollOptions, scrollElement, findScrollText } from './scroll.js';
|
|
16
|
+
import { validateSteps } from './batch.js';
|
|
17
|
+
import { waitFor } from './wait.js';
|
|
18
|
+
import { boundedRead, installOutcomeObserver } from './outcomes.js';
|
|
19
|
+
import { createDrag, dragOptions } from './drag.js';
|
|
20
|
+
|
|
21
|
+
export function createController(api, notify = () => {}) {
|
|
22
|
+
const claims = new Map();
|
|
23
|
+
const snapshots = new Map();
|
|
24
|
+
const documents = new Map();
|
|
25
|
+
const pageContexts = new Map();
|
|
26
|
+
const frameStates = new Map();
|
|
27
|
+
const queues = new Map();
|
|
28
|
+
const dialogs = new Map();
|
|
29
|
+
const dialogPolicies = new Map();
|
|
30
|
+
const fileChoosers = new Map();
|
|
31
|
+
const dialogWaiters = new Map();
|
|
32
|
+
const visuals = createVisuals(api);
|
|
33
|
+
const clickVisuals = createClickVisuals(api);
|
|
34
|
+
const navigation = createNavigation(cdp);
|
|
35
|
+
const diagnostics = createDiagnostics(cdp);
|
|
36
|
+
const sessions = createSessions(cdp, invalidateFrame, (tabId, sessionId) => {
|
|
37
|
+
diagnostics.detach(tabId, sessionId);
|
|
38
|
+
for (const key of pageContexts.keys()) { const [tab, session] = JSON.parse(key); if (tab === tabId && session === sessionId) pageContexts.delete(key); }
|
|
39
|
+
}, tabId => diagnostics.networkOptions(tabId));
|
|
40
|
+
const mouse = createClick(cdp, value, viewport, position, sessions, tabId => api.tabs.get(tabId), windowId => api.windows.get(windowId));
|
|
41
|
+
const gestures = createDrag(cdp, value, mouse);
|
|
42
|
+
const input = createInput(cdp, value, sessions);
|
|
43
|
+
const evaluations = createEvaluations(cdp);
|
|
44
|
+
|
|
45
|
+
function queued(tabId, operation) {
|
|
46
|
+
const result = (queues.get(tabId) || Promise.resolve()).catch(() => {}).then(operation);
|
|
47
|
+
queues.set(tabId, result);
|
|
48
|
+
const done = () => { if (queues.get(tabId) === result) queues.delete(tabId); };
|
|
49
|
+
result.then(done, done);
|
|
50
|
+
return result;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function httpUrl(url) {
|
|
54
|
+
const parsed = new URL(url);
|
|
55
|
+
if (!['http:', 'https:'].includes(parsed.protocol) || parsed.username || parsed.password) {
|
|
56
|
+
throw new Error('Only http(s) pages without URL credentials can be controlled');
|
|
57
|
+
}
|
|
58
|
+
return parsed.href;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function cdp(target, method, params = {}) {
|
|
62
|
+
if (typeof target === 'number') target = { tabId: target };
|
|
63
|
+
const { tabId } = target;
|
|
64
|
+
if (!method.startsWith('Input.') && method !== 'Runtime.callFunctionOn' && method !== 'Runtime.evaluate' && method !== 'Page.navigate' && method !== 'DOM.setFileInputFiles') {
|
|
65
|
+
return await api.debugger.sendCommand(target, method, params);
|
|
66
|
+
}
|
|
67
|
+
// Input handlers may open a modal and prevent the CDP response from returning.
|
|
68
|
+
// Free the tab queue so the next tool can observe and resolve that modal.
|
|
69
|
+
let rejectDialog;
|
|
70
|
+
const dialog = new Promise((resolve, reject) => { rejectDialog = reject; });
|
|
71
|
+
const waiters = dialogWaiters.get(tabId) || new Set();
|
|
72
|
+
dialogWaiters.set(tabId, waiters);
|
|
73
|
+
waiters.add(rejectDialog);
|
|
74
|
+
try {
|
|
75
|
+
return await Promise.race([api.debugger.sendCommand(target, method, params), dialog]);
|
|
76
|
+
} finally {
|
|
77
|
+
waiters.delete(rejectDialog);
|
|
78
|
+
if (waiters.size === 0) dialogWaiters.delete(tabId);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function value(result) {
|
|
83
|
+
if (result.exceptionDetails) throw new Error(result.exceptionDetails.exception?.description || result.exceptionDetails.text);
|
|
84
|
+
return result.result?.value;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function getObject(tabId, ref, budget) {
|
|
88
|
+
const node = snapshots.get(tabId)?.refs.get(ref);
|
|
89
|
+
if (!node) throw new Error('Element ref is stale or unknown. Take a new browser_snapshot.');
|
|
90
|
+
const target = sessions.target(tabId, node.frameId);
|
|
91
|
+
if (target.sessionId !== node.sessionId) throw new Error('Frame session changed; take a new browser_snapshot.');
|
|
92
|
+
const resolved = await budget.run(() => cdp(target, 'DOM.resolveNode', { backendNodeId: node.backendNodeId, objectGroup: 'browser-attach' }));
|
|
93
|
+
if (!resolved.object?.objectId) throw new Error('Element no longer exists. Take a new browser_snapshot.');
|
|
94
|
+
const { nodes } = await budget.run(() => cdp(target, 'Accessibility.getPartialAXTree', { backendNodeId: node.backendNodeId, fetchRelatives: false }));
|
|
95
|
+
const current = nodes?.find(item => item.backendDOMNodeId === node.backendNodeId);
|
|
96
|
+
if (!current || !!current.ignored !== !!node.ignored || current.role?.value !== node.role || String(current.name?.value || '') !== node.name) {
|
|
97
|
+
throw new Error('Element ref identity changed (role/name or accessibility); take a new browser_snapshot. No input was dispatched.');
|
|
98
|
+
}
|
|
99
|
+
return { ...node, target, objectId: resolved.object.objectId };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function rememberElement(tabId, target, objectId, frameId, budget) {
|
|
103
|
+
const version = documents.get(tabId), claim = claims.get(tabId), frameVersion = frameStates.get(tabId)?.versions.get(frameId);
|
|
104
|
+
const send = (method, params) => budget.run(() => cdp(target, method, params));
|
|
105
|
+
const { node } = await send('DOM.describeNode', { objectId, depth: 0 });
|
|
106
|
+
if (node.nodeType !== 1) throw new Error('Result must be a DOM element');
|
|
107
|
+
const { nodes } = await send('Accessibility.getPartialAXTree', { backendNodeId: node.backendNodeId, fetchRelatives: false });
|
|
108
|
+
const ax = nodes?.find(item => item.backendDOMNodeId === node.backendNodeId);
|
|
109
|
+
if (!ax) throw new Error('Element accessibility identity is unavailable; observe again');
|
|
110
|
+
const connected = value(await send('Runtime.callFunctionOn', { objectId, functionDeclaration: 'function() { return this.isConnected; }', returnByValue: true }));
|
|
111
|
+
if (!connected) throw new Error('Element is detached; no ref was created');
|
|
112
|
+
if (documents.get(tabId) !== version || claims.get(tabId) !== claim || frameStates.get(tabId)?.versions.get(frameId) !== frameVersion) throw new Error('Element document/frame/control changed; no ref was created');
|
|
113
|
+
let state = snapshots.get(tabId);
|
|
114
|
+
if (!state) {
|
|
115
|
+
state = { id: crypto.randomUUID().slice(0, 8), refs: new Map(), entries: [], invalidFrames: new Set(), totalNodes: 0, nextCursor: null, metadata: { omitted_frames: [] } };
|
|
116
|
+
snapshots.set(tabId, state);
|
|
117
|
+
}
|
|
118
|
+
if ((state.extraRefs || 0) >= 500) throw new Error('Element ref budget exhausted; take a new snapshot');
|
|
119
|
+
state.extraRefs = (state.extraRefs || 0) + 1;
|
|
120
|
+
const ref = `${state.id}:element:${state.extraRefs}`;
|
|
121
|
+
state.refs.set(ref, { backendNodeId: node.backendNodeId, frameId, sessionId: target.sessionId, role: ax.role?.value, name: String(ax.name?.value || ''), ignored: !!ax.ignored });
|
|
122
|
+
return { ref, frame_id: frameId, role: ax.role?.value, name: String(ax.name?.value || '') };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function viewport(tabId) {
|
|
126
|
+
return value(await cdp(tabId, 'Runtime.evaluate', {
|
|
127
|
+
expression: '({width: innerWidth, height: innerHeight, scroll_x: scrollX, scroll_y: scrollY, device_pixel_ratio: devicePixelRatio})', returnByValue: true,
|
|
128
|
+
}));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function position(view, x, y) {
|
|
132
|
+
if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0 || x >= view.width || y >= view.height) {
|
|
133
|
+
throw new Error(`Position must be inside the viewport (${view.width} x ${view.height} CSS pixels)`);
|
|
134
|
+
}
|
|
135
|
+
return { x, y };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function release(tabId) {
|
|
139
|
+
if (!claims.has(tabId)) return;
|
|
140
|
+
// Forget only after detach; a failed detach must not silently grant another owner.
|
|
141
|
+
try { await clickVisuals.remove(tabId, dialogs.has(tabId)); await visuals.remove(tabId); }
|
|
142
|
+
finally { await api.debugger.detach({ tabId }); }
|
|
143
|
+
claims.delete(tabId);
|
|
144
|
+
evaluations.forget(tabId);
|
|
145
|
+
snapshots.delete(tabId);
|
|
146
|
+
documents.delete(tabId);
|
|
147
|
+
for (const key of pageContexts.keys()) if (JSON.parse(key)[0] === tabId) pageContexts.delete(key);
|
|
148
|
+
frameStates.delete(tabId);
|
|
149
|
+
dialogs.delete(tabId);
|
|
150
|
+
dialogPolicies.delete(tabId);
|
|
151
|
+
fileChoosers.delete(tabId);
|
|
152
|
+
diagnostics.forget(tabId);
|
|
153
|
+
sessions.forget(tabId);
|
|
154
|
+
notify({ type: 'detached', tab_id: tabId });
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async function releaseMatching(predicate) {
|
|
158
|
+
const targets = [...claims.entries()].filter(([, claim]) => predicate(claim));
|
|
159
|
+
const outcomes = await Promise.allSettled(targets.map(([tabId]) => queued(tabId, () => release(tabId))));
|
|
160
|
+
return { released: outcomes.filter(item => item.status === 'fulfilled').length,
|
|
161
|
+
errors: outcomes.filter(item => item.status === 'rejected').map(item => String(item.reason)) };
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function forget(tabId) {
|
|
165
|
+
evaluations.forget(tabId);
|
|
166
|
+
navigation.cancel(tabId);
|
|
167
|
+
mouse.cancel(tabId);
|
|
168
|
+
gestures.cancel(tabId);
|
|
169
|
+
input.cancel(tabId);
|
|
170
|
+
visuals.forget(tabId);
|
|
171
|
+
clickVisuals.forget(tabId);
|
|
172
|
+
claims.delete(tabId);
|
|
173
|
+
snapshots.delete(tabId);
|
|
174
|
+
documents.delete(tabId);
|
|
175
|
+
for (const key of pageContexts.keys()) if (JSON.parse(key)[0] === tabId) pageContexts.delete(key);
|
|
176
|
+
frameStates.delete(tabId);
|
|
177
|
+
dialogs.delete(tabId);
|
|
178
|
+
dialogPolicies.delete(tabId);
|
|
179
|
+
fileChoosers.delete(tabId);
|
|
180
|
+
diagnostics.forget(tabId);
|
|
181
|
+
sessions.forget(tabId);
|
|
182
|
+
notify({ type: 'detached', tab_id: tabId });
|
|
183
|
+
}
|
|
184
|
+
api.debugger.onDetach.addListener(source => { if (!source.sessionId) forget(source.tabId); });
|
|
185
|
+
api.debugger.onEvent.addListener((source, method, params = {}) => {
|
|
186
|
+
sessions.event(source, method, params);
|
|
187
|
+
const contextKey = JSON.stringify([source.tabId, source.sessionId]);
|
|
188
|
+
if (method === 'Runtime.executionContextCreated' && params.context.auxData?.isDefault) pageContexts.set(JSON.stringify([source.tabId, source.sessionId, params.context.auxData.frameId]), params.context.id);
|
|
189
|
+
if (method === 'Runtime.executionContextDestroyed' || method === 'Runtime.executionContextsCleared') {
|
|
190
|
+
for (const [key, id] of pageContexts) if (key.startsWith(contextKey.slice(0, -1) + ',') && (method.endsWith('Cleared') || id === params.executionContextId)) pageContexts.delete(key);
|
|
191
|
+
}
|
|
192
|
+
if (!sessions.has(source)) return;
|
|
193
|
+
diagnostics.event(source, method, params);
|
|
194
|
+
if (!source.sessionId) navigation.event(source.tabId, method, params);
|
|
195
|
+
const policy = dialogPolicies.get(source.tabId);
|
|
196
|
+
const autoDialog = method === 'Page.javascriptDialogOpening' && policy && policy.policy !== 'ask' && policy.messages.length < 10;
|
|
197
|
+
if (!autoDialog) {
|
|
198
|
+
mouse.event(source.tabId, method, params, source.sessionId);
|
|
199
|
+
gestures.event(source.tabId, method, params, source.sessionId);
|
|
200
|
+
input.event(source.tabId, method, params, source.sessionId);
|
|
201
|
+
}
|
|
202
|
+
const frames = frameStates.get(source.tabId);
|
|
203
|
+
if (method === 'Page.frameAttached') frames?.parents.set(params.frameId, params.parentFrameId);
|
|
204
|
+
const mainNavigation = !source.sessionId && method === 'Page.frameNavigated' && params.frame && !params.frame.parentId;
|
|
205
|
+
if (mainNavigation && frames) {
|
|
206
|
+
frames.mainId = params.frame.id;
|
|
207
|
+
frames.parents.clear(); frames.versions.clear(); frames.sessions.clear();
|
|
208
|
+
}
|
|
209
|
+
if (method === 'Page.frameNavigated' && params.frame?.parentId) frames?.parents.set(params.frame.id, params.frame.parentId);
|
|
210
|
+
const sameDocumentMain = method === 'Page.navigatedWithinDocument' && params.frameId === frames?.mainId;
|
|
211
|
+
if ((!source.sessionId && method === 'DOM.documentUpdated') || mainNavigation || sameDocumentMain) {
|
|
212
|
+
evaluations.forget(source.tabId);
|
|
213
|
+
documents.set(source.tabId, (documents.get(source.tabId) || 0) + 1);
|
|
214
|
+
snapshots.delete(source.tabId);
|
|
215
|
+
fileChoosers.delete(source.tabId);
|
|
216
|
+
} else if (source.sessionId && method === 'DOM.documentUpdated') {
|
|
217
|
+
for (const [frame, session] of frameStates.get(source.tabId)?.sessions || []) if (session === source.sessionId) invalidateFrame(source.tabId, frame);
|
|
218
|
+
} else if (method === 'Page.frameNavigated' || method === 'Page.navigatedWithinDocument' || method === 'Page.frameDetached') {
|
|
219
|
+
invalidateFrame(source.tabId, params.frame?.id || params.frameId);
|
|
220
|
+
}
|
|
221
|
+
if (method === 'Page.fileChooserOpened' && claims.has(source.tabId)) {
|
|
222
|
+
if (params.backendNodeId) fileChoosers.set(source.tabId, {
|
|
223
|
+
sessionId: source.sessionId, backendNodeId: params.backendNodeId, mode: params.mode, frame_id: params.frameId,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
if (mainNavigation) clickVisuals.navigated(source.tabId);
|
|
227
|
+
if (method === 'Page.javascriptDialogOpening') {
|
|
228
|
+
dialogs.set(source.tabId, { sessionId: source.sessionId, type: params.type, message: params.message, default_prompt: params.defaultPrompt, default_button: null });
|
|
229
|
+
snapshots.delete(source.tabId);
|
|
230
|
+
const openedDialog = dialogs.get(source.tabId);
|
|
231
|
+
const message = { ...openedDialog, policy: policy?.policy || 'ask', accepted: autoDialog ? policy.policy === 'accept' : null };
|
|
232
|
+
for (let item = policy; item; item = item.parent) item.messages.push(message);
|
|
233
|
+
if (autoDialog) {
|
|
234
|
+
const handling = cdp(source, 'Page.handleJavaScriptDialog', { accept: policy.policy === 'accept', ...(params.type === 'prompt' ? { promptText: params.defaultPrompt || '' } : {}) });
|
|
235
|
+
policy.handling = handling;
|
|
236
|
+
handling.then(() => { message.handled = true; if (dialogs.get(source.tabId) === openedDialog) dialogs.delete(source.tabId); if (params.type === 'beforeunload' && policy.policy === 'dismiss') navigation.cancel(source.tabId, new Error('Navigation cancelled by dismissed beforeunload')); }, error => { message.handled = false; message.error = error.message; for (const reject of dialogWaiters.get(source.tabId) || []) reject(new Error(`Automatic dialog handling failed: ${error.message}. Outcome is unknown.`)); });
|
|
237
|
+
}
|
|
238
|
+
if (!autoDialog) for (const reject of dialogWaiters.get(source.tabId) || []) {
|
|
239
|
+
reject(new Error('The action opened a JavaScript dialog. It may already have executed; use browser_snapshot and browser_handle_dialog before continuing.'));
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (method === 'Page.javascriptDialogClosed') dialogs.delete(source.tabId);
|
|
243
|
+
});
|
|
244
|
+
api.tabs.onRemoved.addListener(forget);
|
|
245
|
+
api.tabs.onUpdated.addListener((tabId, change) => {
|
|
246
|
+
// A subframe reload can also set the TAB status to loading. CDP main-frame
|
|
247
|
+
// events identify document changes; a status-only update carries no identity.
|
|
248
|
+
if (change.url) {
|
|
249
|
+
evaluations.forget(tabId);
|
|
250
|
+
documents.set(tabId, (documents.get(tabId) || 0) + 1);
|
|
251
|
+
snapshots.delete(tabId); fileChoosers.delete(tabId);
|
|
252
|
+
}
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
function invalidateFrame(tabId, frameId) {
|
|
256
|
+
const frames = frameStates.get(tabId);
|
|
257
|
+
if (!frames || !frameId) return;
|
|
258
|
+
const affected = new Set([frameId]);
|
|
259
|
+
// Descendant documents disappear with their parent, even without individual events.
|
|
260
|
+
let size;
|
|
261
|
+
do {
|
|
262
|
+
size = affected.size;
|
|
263
|
+
for (const [id, parent] of frames.parents) if (affected.has(parent)) affected.add(id);
|
|
264
|
+
} while (size !== affected.size);
|
|
265
|
+
const state = snapshots.get(tabId);
|
|
266
|
+
for (const id of affected) {
|
|
267
|
+
evaluations.forgetFrame(tabId, id);
|
|
268
|
+
frames.versions.set(id, (frames.versions.get(id) || 0) + 1);
|
|
269
|
+
if (state) omitFrame(state, id, 'Frame navigated or detached after observation');
|
|
270
|
+
}
|
|
271
|
+
if (affected.has(fileChoosers.get(tabId)?.frame_id)) fileChoosers.delete(tabId);
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function omitFrame(state, frameId, reason) {
|
|
275
|
+
if (state.invalidFrames.has(frameId)) return;
|
|
276
|
+
state.invalidFrames.add(frameId);
|
|
277
|
+
for (const [ref, node] of state.refs) if (node.frameId === frameId) state.refs.delete(ref);
|
|
278
|
+
state.totalNodes -= state.entries.filter(entry => entry.frameId === frameId && entry.node).length;
|
|
279
|
+
state.metadata.omitted_frames.push({ frame_id: frameId, reason });
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
function expireCursor(tabId) {
|
|
283
|
+
const state = snapshots.get(tabId);
|
|
284
|
+
if (state) state.nextCursor = null;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
async function snapshot(tabId, args = {}, budget) {
|
|
288
|
+
const send = (...params) => budget ? budget.run(() => cdp(...params)) : cdp(...params);
|
|
289
|
+
if (args.cursor != null) {
|
|
290
|
+
const state = snapshots.get(tabId);
|
|
291
|
+
if (!state) throw new Error('Snapshot cursor is stale or unknown. Take a new browser_snapshot.');
|
|
292
|
+
return snapshotPage(state, args);
|
|
293
|
+
}
|
|
294
|
+
snapshots.delete(tabId);
|
|
295
|
+
const documentVersion = documents.get(tabId);
|
|
296
|
+
const versions = new Map(frameStates.get(tabId)?.versions);
|
|
297
|
+
await (budget ? budget.run(() => sessions.settle(tabId)) : sessions.settle(tabId));
|
|
298
|
+
for (const target of sessions.targets(tabId)) await send(target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach' });
|
|
299
|
+
const frames = await (budget ? budget.run(() => sessions.frames(tabId, send)) : sessions.frames(tabId, send));
|
|
300
|
+
const mainId = frameStates.get(tabId).mainId;
|
|
301
|
+
for (const frame of frames) if (frame.parentId) frameStates.get(tabId).parents.set(frame.id, frame.parentId);
|
|
302
|
+
const knownFrames = new Set(frames.map(frame => frame.id));
|
|
303
|
+
const token = crypto.randomUUID().slice(0, 8);
|
|
304
|
+
const refs = new Map(), entries = [], omittedFrames = [];
|
|
305
|
+
for (const frame of frames) {
|
|
306
|
+
let nodes;
|
|
307
|
+
let target;
|
|
308
|
+
try {
|
|
309
|
+
target = sessions.target(tabId, frame.id);
|
|
310
|
+
frameStates.get(tabId).sessions.set(frame.id, target.sessionId);
|
|
311
|
+
({ nodes } = await send(target, 'Accessibility.getFullAXTree', { frameId: frame.id }));
|
|
312
|
+
}
|
|
313
|
+
catch (error) {
|
|
314
|
+
if (frame.id === mainId) throw error;
|
|
315
|
+
omittedFrames.push({ frame_id: frame.id, url: frame.url, reason: error.message });
|
|
316
|
+
continue;
|
|
317
|
+
}
|
|
318
|
+
// Report cross-process frames that are missing from Page.getFrameTree.
|
|
319
|
+
for (const node of nodes) {
|
|
320
|
+
if (!/^iframe$/i.test(node.role?.value || '') || !node.backendDOMNodeId) continue;
|
|
321
|
+
const { node: owner } = await send(target, 'DOM.describeNode', { backendNodeId: node.backendDOMNodeId, depth: 0 });
|
|
322
|
+
const childId = owner.frameId || owner.contentDocument?.frameId;
|
|
323
|
+
if (!childId) continue;
|
|
324
|
+
frameStates.get(tabId)?.parents.set(childId, frame.id);
|
|
325
|
+
sessions.rememberParent(tabId, childId, frame.id);
|
|
326
|
+
if (knownFrames.has(childId)) continue;
|
|
327
|
+
const attributes = owner.attributes || [], srcIndex = attributes.indexOf('src');
|
|
328
|
+
const url = srcIndex >= 0 ? new URL(attributes[srcIndex + 1], frame.url).href : 'about:blank';
|
|
329
|
+
knownFrames.add(childId); frames.push({ id: childId, url });
|
|
330
|
+
}
|
|
331
|
+
entries.push({ node: false, frameId: frame.id, line: `Frame ${frame.id} ${frame.url}` });
|
|
332
|
+
entries.push(...renderAX(nodes, token, refs, frame.id, target.sessionId));
|
|
333
|
+
}
|
|
334
|
+
const metadata = value(await send(tabId, 'Runtime.evaluate', {
|
|
335
|
+
expression: '({title: document.title, url: location.href})', returnByValue: true,
|
|
336
|
+
}));
|
|
337
|
+
const state = { id: token, refs, entries, invalidFrames: new Set(), totalNodes: entries.filter(entry => entry.node).length,
|
|
338
|
+
metadata: { title: metadata.title, url: metadata.url, viewport: await (budget ? budget.run(() => viewport(tabId)) : viewport(tabId)), omitted_frames: omittedFrames } };
|
|
339
|
+
if (documentVersion !== documents.get(tabId)) throw new Error('Document changed while reading the snapshot. Take a new browser_snapshot.');
|
|
340
|
+
for (const frame of frames) if (versions.get(frame.id) !== frameStates.get(tabId)?.versions.get(frame.id)) {
|
|
341
|
+
omitFrame(state, frame.id, 'Frame changed while reading the snapshot; take a new snapshot to observe it');
|
|
342
|
+
}
|
|
343
|
+
const page = snapshotPage(state, args);
|
|
344
|
+
snapshots.set(tabId, state);
|
|
345
|
+
return page;
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
async function action(tabId, command, args) {
|
|
349
|
+
// Reading buffered diagnostics does not require a responsive renderer.
|
|
350
|
+
if (command === 'logs') return await diagnostics.readDetails(tabId, args, sessions.targets(tabId));
|
|
351
|
+
if (command === 'handle_dialog') {
|
|
352
|
+
if (typeof args.accept !== 'boolean') throw new Error('accept must be a boolean');
|
|
353
|
+
const dialog = dialogs.get(tabId);
|
|
354
|
+
await cdp({ tabId, ...(dialogs.get(tabId)?.sessionId ? { sessionId: dialogs.get(tabId).sessionId } : {}) }, 'Page.handleJavaScriptDialog', { accept: args.accept,
|
|
355
|
+
...(args.prompt_text != null ? { promptText: String(args.prompt_text) } : {}) });
|
|
356
|
+
dialogs.delete(tabId);
|
|
357
|
+
snapshots.delete(tabId);
|
|
358
|
+
return { handled: true, dialog: dialog ?? null, accepted: args.accept, observe_again: true };
|
|
359
|
+
}
|
|
360
|
+
if (dialogs.has(tabId)) {
|
|
361
|
+
if (command === 'snapshot') return { dialog: dialogs.get(tabId), note: 'Use browser_handle_dialog to accept or dismiss this dialog.' };
|
|
362
|
+
if (command !== 'release_tab') throw new Error('A JavaScript dialog is open. Observe browser_snapshot and use browser_handle_dialog.');
|
|
363
|
+
}
|
|
364
|
+
if (command === 'snapshot') return await snapshot(tabId, args);
|
|
365
|
+
if (command === 'batch') {
|
|
366
|
+
validateSteps(args.steps, args.start_at ?? 0);
|
|
367
|
+
const budget = deadline(args.timeout_ms ?? 15000), results = [];
|
|
368
|
+
const claim = claims.get(tabId), documentVersion = documents.get(tabId);
|
|
369
|
+
for (let index = args.start_at ?? 0; index < args.steps.length; index++) {
|
|
370
|
+
const step = args.steps[index];
|
|
371
|
+
try {
|
|
372
|
+
if (claims.get(tabId) !== claim) throw new Error('Tab control ended during batch; observe before continuing');
|
|
373
|
+
if (documents.get(tabId) !== documentVersion) throw new Error('Document changed during batch; observe before continuing');
|
|
374
|
+
httpUrl((await budget.run(() => api.tabs.get(tabId))).url);
|
|
375
|
+
const { action: command, ...inputArgs } = step;
|
|
376
|
+
const stepBudget = budget.child(step.timeout_ms ?? 30000, command === 'wait' ? 'wait' : 'input');
|
|
377
|
+
results.push(await perform(tabId, command, { ...inputArgs, dialog_policy: step.dialog_policy ?? args.dialog_policy ?? 'ask', budget: stepBudget }));
|
|
378
|
+
} catch (error) {
|
|
379
|
+
return { completed: false, results, started_at: args.start_at ?? 0, failed_step: index, next_unattempted_step: index + 1 < args.steps.length ? index + 1 : null, retry_failed_step_safe: error.input_dispatched === false, error: error.message, ...(error.post_action ? { post_action: error.post_action } : {}),
|
|
380
|
+
application_result: 'not_checked', observe_again: true };
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return { completed: true, results, application_result: 'not_checked', observe_again: true };
|
|
384
|
+
}
|
|
385
|
+
if (command === 'evaluate') {
|
|
386
|
+
await args.observe_start?.();
|
|
387
|
+
if (args.result_type != null && !['json', 'element'].includes(args.result_type)) throw new Error('result_type must be json or element');
|
|
388
|
+
if (args.cursor == null) expireCursor(tabId);
|
|
389
|
+
if (args.cursor != null) return await evaluations.read(tabId, args);
|
|
390
|
+
const budget = deadline(args.timeout_ms ?? 10000, 'evaluation'), version = documents.get(tabId), claim = claims.get(tabId);
|
|
391
|
+
const frameId = await resolveFrame(cdp, value, sessions, tabId, args, budget), target = sessions.target(tabId, frameId);
|
|
392
|
+
const contextId = args.frame || args.frame_id ? pageContexts.get(JSON.stringify([tabId, target.sessionId, frameId])) : undefined;
|
|
393
|
+
if ((args.frame || args.frame_id) && contextId == null) throw new Error('Page frame context is unavailable; observe again');
|
|
394
|
+
let result;
|
|
395
|
+
try { result = await budget.run(() => { budget.dispatch(); return evaluations.read(tabId, { ...args, evaluation_frame: frameId, target, context_id: contextId, timeout_ms: Math.max(100, Math.min(args.timeout_ms ?? 10000, budget.remaining())) }); }); }
|
|
396
|
+
catch (error) { evaluations.forget(tabId); if (args.result_type === 'element') await boundedRead(() => cdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach-evaluation' }), 100).catch(() => {}); throw error; }
|
|
397
|
+
if (!result.element) return { ...result, frame_id: frameId, scope: args.frame || args.frame_id ? 'frame' : 'main_frame' };
|
|
398
|
+
try {
|
|
399
|
+
if (documents.get(tabId) !== version || claims.get(tabId) !== claim) throw new Error('Document or claim changed during evaluation; no element ref was created');
|
|
400
|
+
const owner = value(await budget.run(() => cdp(target, 'Runtime.callFunctionOn', { objectId: result.element.objectId,
|
|
401
|
+
functionDeclaration: 'function() { return this.nodeType === 1 && this.isConnected && this.ownerDocument === document; }', returnByValue: true })));
|
|
402
|
+
if (!owner) throw new Error('Element must belong to the selected frame; use a frame locator');
|
|
403
|
+
return { result_type: 'element', ...await rememberElement(tabId, target, result.element.objectId, frameId, budget), observe_again: true };
|
|
404
|
+
} finally { await boundedRead(() => cdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach-evaluation' }), 100).catch(() => {}); }
|
|
405
|
+
}
|
|
406
|
+
if (command === 'wait') return await waitFor(cdp, value, sessions, diagnostics, mouse, tabId, args);
|
|
407
|
+
if (command === 'drag') {
|
|
408
|
+
dragOptions(args);
|
|
409
|
+
args = { ...args, budget: deadline(args.timeout_ms ?? 5000) };
|
|
410
|
+
await mouse.assertReady(tabId, args.budget);
|
|
411
|
+
await args.observe_start?.();
|
|
412
|
+
const claim = claims.get(tabId), documentVersion = documents.get(tabId);
|
|
413
|
+
for (const target of sessions.targets(tabId)) await args.budget.run(() => cdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach' }));
|
|
414
|
+
expireCursor(tabId);
|
|
415
|
+
return await gestures.drag(tabId, args, (target, budget) => target.selector ? resolveSelector(cdp, value, sessions, tabId, target, budget) : target.ref ? getObject(tabId, target.ref, budget) : null, () => {
|
|
416
|
+
if (claims.get(tabId) !== claim || documents.get(tabId) !== documentVersion) throw new Error('Tab control or document changed during drag; outcome is unknown. Observe before continuing.');
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
if (command === 'read_text') {
|
|
420
|
+
if (!args.frame && !args.frame_id) return await readText(cdp, value, tabId, args);
|
|
421
|
+
const budget = deadline(5000), frameId = await resolveFrame(cdp, value, sessions, tabId, args, budget), target = sessions.target(tabId, frameId);
|
|
422
|
+
const contextId = pageContexts.get(JSON.stringify([tabId, target.sessionId, frameId]));
|
|
423
|
+
if (contextId == null) throw new Error('Page frame context is unavailable; observe again');
|
|
424
|
+
return { ...await readText(cdp, value, target, { ...args, context_id: contextId }), scope: 'frame', frame_id: frameId };
|
|
425
|
+
}
|
|
426
|
+
if (command === 'screenshot') {
|
|
427
|
+
const view = await viewport(tabId);
|
|
428
|
+
const hasTarget = args.ref != null || args.selector != null;
|
|
429
|
+
if (!hasTarget && (args.frame || args.frame_id || args.nth != null || args.selector_type)) throw new Error('Element screenshot frame/nth/type requires an element selector');
|
|
430
|
+
if (hasTarget && args.full_page) throw new Error('Element target and full_page are mutually exclusive');
|
|
431
|
+
const clip = hasTarget ? await elementClip(tabId, args, view) : args.full_page ? await fullPageClip(tabId) : null;
|
|
432
|
+
const includeIndicators = args.include_indicators === true;
|
|
433
|
+
const hiddenError = includeIndicators ? null : await clickVisuals.visible(tabId, false);
|
|
434
|
+
if (hiddenError) throw new Error(`Could not hide click feedback for a clean screenshot: ${hiddenError}`);
|
|
435
|
+
try {
|
|
436
|
+
// Render an explicit target surface without activating its tab or window.
|
|
437
|
+
const captureClip = clip || { x: view.scroll_x, y: view.scroll_y, width: view.width, height: view.height, scale: 1 };
|
|
438
|
+
let timer;
|
|
439
|
+
let result;
|
|
440
|
+
try {
|
|
441
|
+
result = await Promise.race([
|
|
442
|
+
cdp(tabId, 'Page.captureScreenshot', { format: 'png', fromSurface: true,
|
|
443
|
+
captureBeyondViewport: true, optimizeForSpeed: true, clip: captureClip }),
|
|
444
|
+
new Promise((resolve, reject) => { timer = setTimeout(() => reject(new Error(
|
|
445
|
+
'Screenshot did not finish within 5 seconds. An inactive or occluded Chromium page may not render. Use browser_tab_control action=activate before requesting another screenshot, or use DOM/AX.'
|
|
446
|
+
)), 5000); }),
|
|
447
|
+
]);
|
|
448
|
+
} finally { clearTimeout(timer); }
|
|
449
|
+
return { ...result, capture: { scope: hasTarget ? 'element' : clip ? 'full_page' : 'viewport', viewport: view,
|
|
450
|
+
css_width: clip?.width ?? view.width, css_height: clip?.height ?? view.height,
|
|
451
|
+
image_origin_css: { x: clip?.x ?? view.scroll_x, y: clip?.y ?? view.scroll_y },
|
|
452
|
+
click_coordinate_space: 'viewport_css_pixels', agent_indicators_included: includeIndicators } };
|
|
453
|
+
} finally { await clickVisuals.visible(tabId, true); }
|
|
454
|
+
}
|
|
455
|
+
if (command === 'release_tab') { await release(tabId); return { released: true }; }
|
|
456
|
+
if (command === 'navigate') {
|
|
457
|
+
await args.observe_start?.();
|
|
458
|
+
const url = args.direction != null ? undefined : httpUrl(args.url);
|
|
459
|
+
if (args.direction != null && args.url != null) throw new Error('Provide url or direction, never both');
|
|
460
|
+
evaluations.forget(tabId);
|
|
461
|
+
snapshots.delete(tabId);
|
|
462
|
+
fileChoosers.delete(tabId);
|
|
463
|
+
const result = await navigation.navigate(tabId, url, args);
|
|
464
|
+
const budget = deadline(NAVIGATION_OBSERVATION_TIMEOUT_MS, 'observation');
|
|
465
|
+
for (let attempt = 0; attempt < 2; attempt++) try {
|
|
466
|
+
if (args.include_snapshot === false) {
|
|
467
|
+
const metadata = value(await budget.run(() => cdp(tabId, 'Runtime.evaluate', {
|
|
468
|
+
expression: '({title: document.title, url: location.href})', returnByValue: true,
|
|
469
|
+
})));
|
|
470
|
+
return { ...result, ...metadata };
|
|
471
|
+
}
|
|
472
|
+
return { ...result, ...await snapshot(tabId, {}, budget) };
|
|
473
|
+
} catch (error) {
|
|
474
|
+
snapshots.delete(tabId);
|
|
475
|
+
if (attempt === 0 && budget.remaining() > 100 && /context.*(destroyed|not found)|cannot find context|document (changed|updated)|inspected target navigated|no frame with given id/i.test(error.message)) {
|
|
476
|
+
await new Promise(resolve => setTimeout(resolve, 50));
|
|
477
|
+
continue;
|
|
478
|
+
}
|
|
479
|
+
return { ...result, observation_error: error.message, observe_again: true };
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
// Re-resolve backend identity for every action; document/frame events expire refs.
|
|
483
|
+
if (command === 'click' || command === 'fill') inputMode(args);
|
|
484
|
+
if (command === 'select_option') selectOptions(args);
|
|
485
|
+
if (command === 'type') typeOptions(args);
|
|
486
|
+
if (command === 'scroll') scrollOptions(args);
|
|
487
|
+
if (['scroll', 'upload_file'].includes(command) && ['ref', 'selector', 'frame', 'frame_id', 'selector_type', 'nth'].some(key => args[key] != null)) targetOptions(args);
|
|
488
|
+
if (['click', 'fill', 'type', 'hover', 'select_option'].includes(command)) targetOptions(args, { coordinates: command === 'hover' || command === 'click' && inputMode(args) !== 'dom' });
|
|
489
|
+
if (command === 'click') clickOptions(args);
|
|
490
|
+
if (['click', 'fill', 'type', 'press_key', 'scroll', 'hover', 'select_option', 'upload_file'].includes(command)) args = { ...args, budget: args.budget || deadline(args.timeout_ms) };
|
|
491
|
+
// Check before any renderer preparation or ref invalidation. A rejected
|
|
492
|
+
// inactive native click is known not to have dispatched, unlike a lost
|
|
493
|
+
// receipt after input. DOM mode remains an explicit separate operation.
|
|
494
|
+
if (command === 'hover' || command === 'click' && inputMode(args) === 'native') await mouse.assertReady(tabId, args.budget);
|
|
495
|
+
await args.observe_start?.();
|
|
496
|
+
// Reusable selectors permit many actions without a snapshot. Release the
|
|
497
|
+
// previous operation's remote wrappers before resolving this one. Refs store
|
|
498
|
+
// backend ids, not object handles; this cannot change their identity.
|
|
499
|
+
if (args.budget) for (const target of sessions.targets(tabId)) {
|
|
500
|
+
await args.budget.run(() => cdp(target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach' }));
|
|
501
|
+
}
|
|
502
|
+
const node = args.selector ? await resolveSelector(cdp, value, sessions, tabId, args, args.budget || deadline(30000)) :
|
|
503
|
+
args.ref ? await getObject(tabId, args.ref, args.budget || deadline(30000)) : null;
|
|
504
|
+
expireCursor(tabId);
|
|
505
|
+
if (command === 'upload_file') {
|
|
506
|
+
const send = (target, method, params) => args.budget.run(() => cdp(target, method, params));
|
|
507
|
+
if (!Array.isArray(args.files) || args.files.length > 100 || args.files.some(file => typeof file !== 'string' || !file || file.includes('\0'))) {
|
|
508
|
+
throw new Error('files must contain at most 100 valid absolute file paths; [] clears selection');
|
|
509
|
+
}
|
|
510
|
+
const chooser = fileChoosers.get(tabId);
|
|
511
|
+
let target = node;
|
|
512
|
+
if (!target) {
|
|
513
|
+
if (!chooser) throw new Error('No pending file chooser. Supply a fresh file-input/label ref, or click the upload button first.');
|
|
514
|
+
const resolved = await send({ tabId, ...(chooser.sessionId ? { sessionId: chooser.sessionId } : {}) }, 'DOM.resolveNode', { backendNodeId: chooser.backendNodeId, objectGroup: 'browser-attach' });
|
|
515
|
+
if (!resolved.object?.objectId) throw new Error('File chooser input no longer exists. Observe again.');
|
|
516
|
+
target = { objectId: resolved.object.objectId, target: { tabId, ...(chooser.sessionId ? { sessionId: chooser.sessionId } : {}) } };
|
|
517
|
+
}
|
|
518
|
+
const resolved = await send(target.target, 'Runtime.callFunctionOn', {
|
|
519
|
+
objectId: target.objectId, arguments: [{ value: args.files.length }],
|
|
520
|
+
functionDeclaration: `function(count) {
|
|
521
|
+
const input = this.tagName === 'LABEL' ? this.control : this;
|
|
522
|
+
if (!input || input.tagName !== 'INPUT' || input.type !== 'file') throw new Error('Ref is not a file input or associated label');
|
|
523
|
+
if (!input.isConnected) throw new Error('File input is detached');
|
|
524
|
+
if (input.matches(':disabled')) throw new Error('File input is disabled');
|
|
525
|
+
if (input.webkitdirectory) throw new Error('Directory upload is unsupported; use a regular file input');
|
|
526
|
+
if (count > 1 && !input.multiple) throw new Error('Single-file input cannot accept multiple files');
|
|
527
|
+
return input;
|
|
528
|
+
}`, objectGroup: 'browser-attach',
|
|
529
|
+
});
|
|
530
|
+
if (resolved.exceptionDetails) value(resolved);
|
|
531
|
+
if (!resolved.result?.objectId) throw new Error('Could not resolve the file input');
|
|
532
|
+
args.budget.dispatch();
|
|
533
|
+
if (args.files.length === 0) {
|
|
534
|
+
// Chromium's native empty-path assignment does not emit input/change.
|
|
535
|
+
// Like Playwright's empty payload path, explicitly clear and notify the app.
|
|
536
|
+
value(await send(target.target, 'Runtime.callFunctionOn', { objectId: resolved.result.objectId, returnByValue: true,
|
|
537
|
+
functionDeclaration: `function() {
|
|
538
|
+
this.value = '';
|
|
539
|
+
const Event = this.ownerDocument.defaultView.Event;
|
|
540
|
+
this.dispatchEvent(new Event('input', { bubbles: true, composed: true }));
|
|
541
|
+
this.dispatchEvent(new Event('change', { bubbles: true }));
|
|
542
|
+
return true;
|
|
543
|
+
}` }));
|
|
544
|
+
} else {
|
|
545
|
+
// Native CDP assignment emits input/change and preserves actual file metadata.
|
|
546
|
+
await send(target.target, 'DOM.setFileInputFiles', { objectId: resolved.result.objectId, files: args.files });
|
|
547
|
+
}
|
|
548
|
+
fileChoosers.delete(tabId);
|
|
549
|
+
return { files_set: args.files.length, cleared: args.files.length === 0, events_trusted: args.files.length > 0, observe_again: true };
|
|
550
|
+
}
|
|
551
|
+
if (command === 'click') {
|
|
552
|
+
if (inputMode(args) === 'dom') return await input.domClick(tabId, node, args);
|
|
553
|
+
const result = await mouse.click(tabId, node, args);
|
|
554
|
+
// Keep click feedback out of the input dispatch/verification path.
|
|
555
|
+
void clickVisuals.click(tabId, result.clicked.x, result.clicked.y).catch(() => {});
|
|
556
|
+
return result;
|
|
557
|
+
}
|
|
558
|
+
if (command === 'hover') return await mouse.click(tabId, node, { ...args, hover: true });
|
|
559
|
+
if (command === 'select_option') return await input.selectOption(tabId, node, args);
|
|
560
|
+
if (command === 'type') return await input.typeText(tabId, node, args);
|
|
561
|
+
if (command === 'fill') return await input.fill(tabId, node, args);
|
|
562
|
+
if (command === 'press_key') return await input.pressKey(tabId, args.key, args);
|
|
563
|
+
if (command === 'scroll' && args.until != null) {
|
|
564
|
+
const { text, selector } = args.until;
|
|
565
|
+
const maxSteps = args.max_steps ?? 100;
|
|
566
|
+
const root = node || await resolveSelector(cdp, value, sessions, tabId, { selector: 'html' }, args.budget);
|
|
567
|
+
const claim = claims.get(tabId), version = documents.get(tabId), frameVersion = frameStates.get(tabId)?.versions.get(root.frameId);
|
|
568
|
+
let last;
|
|
569
|
+
for (let step = 0; step <= maxSteps; step++) {
|
|
570
|
+
if (claims.get(tabId) !== claim || documents.get(tabId) !== version || frameStates.get(tabId)?.versions.get(root.frameId) !== frameVersion) throw new Error('Document/frame/control changed during scroll search; observe before continuing');
|
|
571
|
+
const found = await args.budget.run(() => cdp(root.target, 'Runtime.callFunctionOn', { objectId: root.objectId,
|
|
572
|
+
functionDeclaration: findScrollText.toString(), arguments: [{ value: text }, { value: selector }], objectGroup: 'browser-attach-search' }));
|
|
573
|
+
value(found);
|
|
574
|
+
if (found.result?.objectId) {
|
|
575
|
+
const result = await rememberElement(tabId, root.target, found.result.objectId, root.frameId, args.budget);
|
|
576
|
+
await args.budget.run(() => cdp(root.target, 'Runtime.releaseObjectGroup', { objectGroup: 'browser-attach-search' }));
|
|
577
|
+
return { found: true, ...result, scroll_steps: step, application_result: 'not_checked', observe_again: true };
|
|
578
|
+
}
|
|
579
|
+
if (step === maxSteps || last?.scrolled === false) break;
|
|
580
|
+
args.budget.dispatch();
|
|
581
|
+
last = value(await args.budget.run(() => cdp(root.target, 'Runtime.callFunctionOn', { objectId: root.objectId,
|
|
582
|
+
functionDeclaration: scrollElement.toString(), arguments: [{ value: args.delta_x ?? 0 }, { value: args.delta_y ?? null }], returnByValue: true })));
|
|
583
|
+
await args.budget.run(() => new Promise(resolve => setTimeout(resolve, 50)));
|
|
584
|
+
}
|
|
585
|
+
throw new Error(`Scroll search did not find ${JSON.stringify(text)} within ${maxSteps} steps or reached the scroll boundary; current=${JSON.stringify(last)}. Scrolling may already have occurred.`);
|
|
586
|
+
}
|
|
587
|
+
if (command === 'scroll') {
|
|
588
|
+
const mode = args.mode ?? 'dom';
|
|
589
|
+
const budget = args.budget;
|
|
590
|
+
const view = await budget.run(() => viewport(tabId));
|
|
591
|
+
if (node && mode === 'dom') {
|
|
592
|
+
budget.dispatch();
|
|
593
|
+
return { ...value(await budget.run(() => cdp(node.target, 'Runtime.callFunctionOn', { objectId: node.objectId,
|
|
594
|
+
functionDeclaration: scrollElement.toString(), arguments: [{ value: args.delta_x ?? 0 }, { value: args.delta_y }], returnByValue: true }))), observe_again: true };
|
|
595
|
+
}
|
|
596
|
+
const point = node ? (await mouse.prepare(tabId, node, { budget })).point : position(view, args.x ?? view.width / 2, args.y ?? view.height / 2);
|
|
597
|
+
budget.dispatch();
|
|
598
|
+
if (mode === 'wheel') {
|
|
599
|
+
// Root-widget routing respects ancestor overlays and OOPIF boundaries.
|
|
600
|
+
// A protocol acknowledgement does not prove a page handler ran or scrolled.
|
|
601
|
+
await budget.run(() => cdp(tabId, 'Input.dispatchMouseEvent', {
|
|
602
|
+
type: 'mouseWheel', ...point, deltaX: args.delta_x ?? 0, deltaY: args.delta_y,
|
|
603
|
+
button: 'none', buttons: 0, modifiers: 0,
|
|
604
|
+
}));
|
|
605
|
+
return { dispatched: true, method: 'native_wheel', at: point, coordinate_space: 'viewport_css_pixels',
|
|
606
|
+
application_result: 'not_checked', observe_again: true };
|
|
607
|
+
}
|
|
608
|
+
const result = value(await budget.run(() => cdp(tabId, 'Runtime.evaluate', { returnByValue: true,
|
|
609
|
+
expression: `(${scrollAtPoint.toString()})(${point.x},${point.y},${args.delta_x ?? 0},${args.delta_y})` })));
|
|
610
|
+
return { ...result, observe_again: true };
|
|
611
|
+
}
|
|
612
|
+
throw new Error(`Unknown operation: ${command}`);
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
async function perform(tabId, command, args) {
|
|
616
|
+
if (args.dialog_policy != null && !['ask', 'accept', 'dismiss'].includes(args.dialog_policy)) throw new Error('dialog_policy must be ask, accept or dismiss');
|
|
617
|
+
const parent = dialogPolicies.get(tabId), state = { policy: args.dialog_policy ?? parent?.policy ?? 'ask', messages: [], parent };
|
|
618
|
+
dialogPolicies.set(tabId, state);
|
|
619
|
+
const version = documents.get(tabId), startTab = await api.tabs.get(tabId);
|
|
620
|
+
const watch = ['click', 'fill', 'type', 'press_key', 'hover', 'scroll', 'select_option', 'upload_file', 'evaluate', 'navigate'].includes(command) && !(command === 'evaluate' && args.cursor != null);
|
|
621
|
+
let observer, observationFrame = frameStates.get(tabId)?.mainId, observationTarget = { tabId }, observedVersion = frameStates.get(tabId)?.versions.get(observationFrame);
|
|
622
|
+
const startObservation = async () => { if (!watch || observer || dialogs.has(tabId)) return; try {
|
|
623
|
+
observer = await boundedRead(async () => {
|
|
624
|
+
if (args.ref) observationFrame = snapshots.get(tabId)?.refs.get(args.ref)?.frameId || observationFrame;
|
|
625
|
+
else if (args.frame || args.frame_id) observationFrame = await resolveFrame(cdp, value, sessions, tabId, args, deadline(Math.max(100, Math.min(200, args.budget?.remaining() ?? 200))));
|
|
626
|
+
observationTarget = sessions.target(tabId, observationFrame);
|
|
627
|
+
observedVersion = frameStates.get(tabId)?.versions.get(observationFrame);
|
|
628
|
+
const { executionContextId } = await cdp(observationTarget, 'Page.createIsolatedWorld', { frameId: observationFrame, worldName: 'browser-attach-outcome' });
|
|
629
|
+
const result = await cdp(observationTarget, 'Runtime.evaluate', { contextId: executionContextId, expression: `(${installOutcomeObserver.toString()})()`, objectGroup: 'browser-attach-outcome' });
|
|
630
|
+
return result.result?.objectId;
|
|
631
|
+
}, Math.min(200, args.budget?.remaining() ?? 200));
|
|
632
|
+
} catch {} };
|
|
633
|
+
try {
|
|
634
|
+
const result = await action(tabId, command, { ...args, observe_start: startObservation });
|
|
635
|
+
if (state.handling) await boundedRead(() => state.handling, 500);
|
|
636
|
+
return { ...result, ...(state.messages.length ? { dialogs: state.messages } : {}) };
|
|
637
|
+
} catch (error) {
|
|
638
|
+
if (error.input_dispatched == null && args.budget?.has_dispatched) error.input_dispatched = args.budget.has_dispatched();
|
|
639
|
+
if (state.handling) await boundedRead(() => state.handling, 500).catch(() => {});
|
|
640
|
+
if (state.messages.some(d => d.type === 'beforeunload' && d.accepted === false && d.handled) && /ERR_ABORTED|cancel/i.test(error.message)) return { navigated: false, cancelled_by_dialog: true, dialogs: state.messages, observe_again: true };
|
|
641
|
+
if (watch && error.input_dispatched !== false && /outcome is unknown|may already|effects may|may have been|timed out|did not reach|may still/i.test(error.message)) {
|
|
642
|
+
let current, dom;
|
|
643
|
+
await Promise.allSettled([
|
|
644
|
+
boundedRead(async () => { current = await api.tabs.get(tabId); }),
|
|
645
|
+
observer ? boundedRead(async () => { dom = value(await cdp(observationTarget, 'Runtime.callFunctionOn', { objectId: observer, functionDeclaration: 'function() { return this.read(); }', returnByValue: true })); }) : Promise.resolve(),
|
|
646
|
+
]);
|
|
647
|
+
const frameChanged = frameStates.get(tabId)?.versions.get(observationFrame) !== observedVersion;
|
|
648
|
+
error.post_action = { delivery: 'unknown', frame_id: observationFrame, frame_navigation_observed: frameChanged, dialog: dialogs.get(tabId) || null, dialogs: state.messages,
|
|
649
|
+
navigation_observed: documents.get(tabId) !== version || !!current && current.url !== startTab.url,
|
|
650
|
+
current_url: current?.url ?? null, dom_changed: dom?.dom_changed ?? null,
|
|
651
|
+
assessment: dialogs.has(tabId) || state.messages.length || documents.get(tabId) !== version || frameChanged || dom?.dom_changed ? 'observable_effects' : 'no_effect_confirmed',
|
|
652
|
+
note: 'These observations do not prove business success or that an unchanged page received no input. Do not replay automatically.' };
|
|
653
|
+
error.message += ` Post-action observation: ${JSON.stringify(error.post_action)}`;
|
|
654
|
+
} else if (state.messages.length) error.message += ` Dialogs: ${JSON.stringify(state.messages)}`;
|
|
655
|
+
throw error;
|
|
656
|
+
} finally {
|
|
657
|
+
if (parent && claims.has(tabId)) dialogPolicies.set(tabId, parent); else dialogPolicies.delete(tabId);
|
|
658
|
+
if (observer) await boundedRead(async () => {
|
|
659
|
+
await cdp(observationTarget, 'Runtime.callFunctionOn', { objectId: observer, functionDeclaration: 'function() { this.stop(); }' });
|
|
660
|
+
await cdp(observationTarget, 'Runtime.releaseObject', { objectId: observer });
|
|
661
|
+
}, 100).catch(() => {});
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
async function elementClip(tabId, args, view) {
|
|
666
|
+
targetOptions(args);
|
|
667
|
+
const budget = deadline(5000);
|
|
668
|
+
const node = args.selector ? await resolveSelector(cdp, value, sessions, tabId, args, budget) : await getObject(tabId, args.ref, budget);
|
|
669
|
+
const { model } = await budget.run(() => cdp(node.target, 'DOM.getBoxModel', { backendNodeId: node.backendNodeId }));
|
|
670
|
+
if (!model?.border) throw new Error('Element has no rendered box');
|
|
671
|
+
let points = Array.from({ length: 4 }, (_, i) => ({ x: model.border[i * 2], y: model.border[i * 2 + 1] })), target = node.target;
|
|
672
|
+
while (target.sessionId) {
|
|
673
|
+
const parent = sessions.parent(tabId, target.sessionId);
|
|
674
|
+
const owner = await budget.run(() => cdp(parent.target, 'DOM.getFrameOwner', { frameId: parent.frameId }));
|
|
675
|
+
const box = await budget.run(() => cdp(parent.target, 'DOM.getBoxModel', { backendNodeId: owner.backendNodeId }));
|
|
676
|
+
const childView = await budget.run(() => viewport(target));
|
|
677
|
+
points = points.map(p => framePoint(p, box.model.content, childView));
|
|
678
|
+
target = parent.target;
|
|
679
|
+
}
|
|
680
|
+
const x = Math.min(...points.map(p => p.x)) + view.scroll_x, y = Math.min(...points.map(p => p.y)) + view.scroll_y;
|
|
681
|
+
const width = Math.max(...points.map(p => p.x)) + view.scroll_x - x, height = Math.max(...points.map(p => p.y)) + view.scroll_y - y;
|
|
682
|
+
if (width <= 0 || height <= 0 || width * height > 40000000 || x < 0 || y < 0) throw new Error('Element capture requires a positive box within the document and at most 40 million pixels');
|
|
683
|
+
return { x, y, width, height, scale: 1 };
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
async function fullPageClip(tabId) {
|
|
687
|
+
const { cssContentSize } = await cdp(tabId, 'Page.getLayoutMetrics');
|
|
688
|
+
if (cssContentSize.width * cssContentSize.height > 40000000) throw new Error('Full-page capture exceeds 40 million pixels; use viewport capture');
|
|
689
|
+
return { x: 0, y: 0, width: cssContentSize.width, height: cssContentSize.height, scale: 1 };
|
|
690
|
+
}
|
|
691
|
+
|
|
692
|
+
async function execute(owner, command, args) {
|
|
693
|
+
if (command === 'open_tab') {
|
|
694
|
+
const url = httpUrl(args.url);
|
|
695
|
+
if (args.active != null && typeof args.active !== 'boolean') throw new Error('active must be a boolean');
|
|
696
|
+
const tab = await api.tabs.create({ url, active: args.active ?? false });
|
|
697
|
+
return { opened: true, tab_id: tab.id, window_id: tab.windowId, active: tab.active,
|
|
698
|
+
note: 'List tabs after loading, then claim tab_id with optional exact URL/title guards before controlling this tab.' };
|
|
699
|
+
}
|
|
700
|
+
if (command === 'list_tabs') {
|
|
701
|
+
const tabs = await api.tabs.query({});
|
|
702
|
+
return { tabs: tabs.map(tab => ({ tab_id: tab.id, window_id: tab.windowId, title: tab.title || '', url: tab.url || '',
|
|
703
|
+
active: tab.active, group_id: tab.groupId, controllable: /^https?:\/\//.test(tab.url || '') })) };
|
|
704
|
+
}
|
|
705
|
+
if (command === 'release_owner') return await releaseMatching(claim => claim.owner === owner);
|
|
706
|
+
const tabId = args.tab_id;
|
|
707
|
+
if (!Number.isInteger(tabId)) throw new Error('Invalid tab ID');
|
|
708
|
+
return await queued(tabId, async () => {
|
|
709
|
+
if (command === 'claim_tab') {
|
|
710
|
+
const tab = await api.tabs.get(tabId);
|
|
711
|
+
if ((args.expected_url != null && tab.url !== args.expected_url) || (args.expected_title != null && tab.title !== args.expected_title)) throw new Error('Tab title/URL changed. List tabs again or omit optional identity guards to claim by tab_id.');
|
|
712
|
+
httpUrl(tab.url);
|
|
713
|
+
const existing = claims.get(tabId);
|
|
714
|
+
if (existing) {
|
|
715
|
+
if (existing.owner === owner && existing.handle === args.tab_handle) return { claimed: true };
|
|
716
|
+
throw new Error('Tab is already controlled by another session');
|
|
717
|
+
}
|
|
718
|
+
await api.debugger.attach({ tabId }, '1.3');
|
|
719
|
+
try {
|
|
720
|
+
diagnostics.start(tabId);
|
|
721
|
+
await sessions.start(tabId);
|
|
722
|
+
const { frameTree } = await cdp(tabId, 'Page.getFrameTree');
|
|
723
|
+
const state = { mainId: frameTree.frame.id, parents: new Map(), versions: new Map(), sessions: new Map() };
|
|
724
|
+
const remember = tree => {
|
|
725
|
+
if (tree.frame.parentId) state.parents.set(tree.frame.id, tree.frame.parentId);
|
|
726
|
+
(tree.childFrames || []).forEach(remember);
|
|
727
|
+
};
|
|
728
|
+
remember(frameTree); frameStates.set(tabId, state);
|
|
729
|
+
claims.set(tabId, { owner, handle: args.tab_handle });
|
|
730
|
+
let visualWarning;
|
|
731
|
+
try { await visuals.install(tabId); }
|
|
732
|
+
catch (error) { visualWarning = error.message; }
|
|
733
|
+
return { claimed: true, activation_requested: false, title: tab.title, url: tab.url,
|
|
734
|
+
...(visualWarning ? { visual_warning: visualWarning } : {}) };
|
|
735
|
+
} catch (error) {
|
|
736
|
+
diagnostics.forget(tabId);
|
|
737
|
+
sessions.forget(tabId);
|
|
738
|
+
await api.debugger.detach({ tabId });
|
|
739
|
+
throw error;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
const claim = claims.get(tabId);
|
|
743
|
+
if (!claim || claim.owner !== owner || claim.handle !== args.tab_handle) throw new Error('Tab is not claimed by this connection');
|
|
744
|
+
if (command === 'activate_tab') {
|
|
745
|
+
if (args.focus_window != null && typeof args.focus_window !== 'boolean') throw new Error('focus_window must be a boolean');
|
|
746
|
+
const tab = await api.tabs.update(tabId, { active: true });
|
|
747
|
+
if (args.focus_window ?? true) {
|
|
748
|
+
await api.windows.update(tab.windowId, { focused: true });
|
|
749
|
+
// Explicit activation also restores the renderer focus, which can be
|
|
750
|
+
// lost when a focused iframe navigates even if its tab stays active.
|
|
751
|
+
await cdp(tabId, 'Page.bringToFront');
|
|
752
|
+
}
|
|
753
|
+
return { activated: true, window_focus_requested: args.focus_window ?? true, tab_id: tabId, window_id: tab.windowId };
|
|
754
|
+
}
|
|
755
|
+
if (command === 'close_tab') {
|
|
756
|
+
await api.tabs.remove(tabId);
|
|
757
|
+
if (claims.has(tabId)) forget(tabId);
|
|
758
|
+
return { closed: true, tab_id: tabId };
|
|
759
|
+
}
|
|
760
|
+
if (command === 'release_tab') return await action(tabId, command, args);
|
|
761
|
+
httpUrl((await api.tabs.get(tabId)).url);
|
|
762
|
+
const result = await perform(tabId, command, args);
|
|
763
|
+
// A batch can stop because its tab closed. Preserve the partial results
|
|
764
|
+
// instead of losing them to a metadata lookup on the vanished tab.
|
|
765
|
+
if (command === 'batch') return { ...result, tab_id: tabId };
|
|
766
|
+
const tab = await api.tabs.get(tabId);
|
|
767
|
+
const chooser = fileChoosers.get(tabId);
|
|
768
|
+
return { ...result, ...(chooser ? { file_chooser: { pending: true, mode: chooser.mode, frame_id: chooser.frame_id,
|
|
769
|
+
note: 'Use browser_upload_file without ref; files: [] cancels the selection.' } } : {}),
|
|
770
|
+
tab_id: tabId, title: result.title ?? tab.title, url: result.url ?? tab.url, active: tab.active };
|
|
771
|
+
});
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
return { execute, releaseAll: () => releaseMatching(() => true), claimedCount: () => claims.size };
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
function scrollAtPoint(x, y, dx, dy) {
|
|
778
|
+
let doc = document;
|
|
779
|
+
let element = doc.elementFromPoint(x, y);
|
|
780
|
+
const enteredShadowHosts = new Set();
|
|
781
|
+
while (element) {
|
|
782
|
+
if (element.tagName === 'IFRAME') {
|
|
783
|
+
const frame = element;
|
|
784
|
+
const rect = frame.getBoundingClientRect();
|
|
785
|
+
try { doc = frame.contentDocument; } catch { doc = null; }
|
|
786
|
+
if (!doc) throw new Error('Cross-origin iframe scrolling is unavailable; scroll the outer page or use the website controls.');
|
|
787
|
+
x -= rect.left + frame.clientLeft; y -= rect.top + frame.clientTop;
|
|
788
|
+
element = doc.elementFromPoint(x, y);
|
|
789
|
+
continue;
|
|
790
|
+
}
|
|
791
|
+
if (element.shadowRoot && !enteredShadowHosts.has(element)) {
|
|
792
|
+
enteredShadowHosts.add(element);
|
|
793
|
+
const inner = element.shadowRoot.elementFromPoint(x, y);
|
|
794
|
+
if (inner && inner !== element) { element = inner; continue; }
|
|
795
|
+
}
|
|
796
|
+
const style = doc.defaultView.getComputedStyle(element);
|
|
797
|
+
const canX = /auto|scroll|overlay/.test(style.overflowX) && element.scrollWidth > element.clientWidth &&
|
|
798
|
+
((dx > 0 && element.scrollLeft < element.scrollWidth - element.clientWidth) || (dx < 0 && element.scrollLeft > 0));
|
|
799
|
+
const canY = /auto|scroll|overlay/.test(style.overflowY) && element.scrollHeight > element.clientHeight &&
|
|
800
|
+
((dy > 0 && element.scrollTop < element.scrollHeight - element.clientHeight) || (dy < 0 && element.scrollTop > 0));
|
|
801
|
+
if (canX || canY) break;
|
|
802
|
+
const parent = element.parentElement || element.getRootNode().host;
|
|
803
|
+
if (!parent) { element = doc.scrollingElement; break; }
|
|
804
|
+
element = parent;
|
|
805
|
+
}
|
|
806
|
+
if (!element) element = doc.scrollingElement;
|
|
807
|
+
const before = { x: element.scrollLeft, y: element.scrollTop };
|
|
808
|
+
element.scrollBy({ left: dx, top: dy, behavior: 'instant' });
|
|
809
|
+
const after = { x: element.scrollLeft, y: element.scrollTop };
|
|
810
|
+
return { scrolled: before.x !== after.x || before.y !== after.y, before, after,
|
|
811
|
+
method: 'dom_scroll', scope: element === doc.scrollingElement ? 'document' : 'element' };
|
|
812
|
+
}
|