@evan_zhong/browser-attach 0.21.0 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/browser-attach.mjs +44 -43
- package/docs/ARCHITECTURE.md +18 -9
- package/docs/TEST-REPORT.md +31 -1
- package/docs/USAGE.md +18 -2
- package/extension/actionability.js +99 -95
- package/extension/click.js +143 -377
- package/extension/controller.js +430 -1018
- package/extension/diagnostics.js +167 -241
- package/extension/dialogs.js +73 -0
- package/extension/drag.js +195 -195
- package/extension/errors.js +18 -0
- package/extension/evaluate.js +133 -133
- package/extension/fetch-page.js +131 -0
- package/extension/input.js +344 -339
- package/extension/locator.js +43 -22
- package/extension/log-data.js +29 -0
- package/extension/manifest.json +1 -1
- package/extension/navigation.js +74 -74
- package/extension/network.js +79 -0
- package/extension/observation.js +232 -0
- package/extension/outcomes.js +71 -15
- package/extension/page-url.js +8 -0
- package/extension/pointer.js +243 -0
- package/extension/screenshot.js +64 -0
- package/extension/scroll.js +142 -42
- package/extension/text.js +4 -2
- package/extension/upload.js +57 -0
- package/extension/wait.js +119 -117
- package/package.json +1 -1
- package/scripts/check-package.mjs +1 -0
- package/src/client.mjs +50 -135
- package/src/config.mjs +1 -1
- package/src/hub-runtime.mjs +87 -0
- package/src/mcp.mjs +216 -215
package/extension/evaluate.js
CHANGED
|
@@ -1,133 +1,133 @@
|
|
|
1
|
-
// Deliberately a page expression API, not a raw CDP transport.
|
|
2
|
-
const CAPTURE_CHARS = 1000000, CAPTURE_NODES = 50000;
|
|
3
|
-
async function evaluatePage(cdp, tabId, args) {
|
|
4
|
-
const timeout = args.timeout_ms ?? 10000, maxChars = args.max_chars ?? 20000;
|
|
5
|
-
if (typeof args.expression !== 'string' || !args.expression.trim() || args.expression.length > 100000)
|
|
6
|
-
throw new Error('expression must contain 1..100000 characters of JavaScript');
|
|
7
|
-
if (!Number.isInteger(timeout) || timeout < 100 || timeout > 30000) throw new Error('timeout_ms must be 100..30000');
|
|
8
|
-
if (!Number.isInteger(maxChars) || maxChars < 1 || maxChars > 100000) throw new Error('max_chars must be 1..100000');
|
|
9
|
-
let timer;
|
|
10
|
-
try {
|
|
11
|
-
const response = await Promise.race([
|
|
12
|
-
cdp(tabId, 'Runtime.evaluate', {
|
|
13
|
-
expression: `(async () => { let result = await (\n${args.expression}\n); if (typeof result === "function") result = await result(); return ${args.result_type === 'element' ? 'result' : `(${serializeEvaluation.toString()})(result, ${CAPTURE_CHARS}, ${CAPTURE_NODES})`}; })()`,
|
|
14
|
-
...(args.result_type === 'element' ? { objectGroup: 'browser-attach-evaluation' } : {}), ...(args.context_id != null ? { contextId: args.context_id } : {}), awaitPromise: true, returnByValue: args.result_type !== 'element', userGesture: false, timeout,
|
|
15
|
-
}),
|
|
16
|
-
new Promise((resolve, reject) => { timer = setTimeout(() => reject(new Error(
|
|
17
|
-
'JavaScript evaluation timed out. It may still run or have changed the page; observe before continuing. Do not replay automatically.'
|
|
18
|
-
)), timeout); }),
|
|
19
|
-
]);
|
|
20
|
-
if (response.exceptionDetails) throw new Error(response.exceptionDetails.exception?.description || response.exceptionDetails.text);
|
|
21
|
-
if (args.result_type === 'element') {
|
|
22
|
-
if (response.result?.subtype !== 'node' || !response.result.objectId) throw new Error('Element result must be one connected DOM element, not JSON/array/null');
|
|
23
|
-
return { element: response.result };
|
|
24
|
-
}
|
|
25
|
-
if (!response.result?.value || typeof response.result.value.truncated !== 'boolean') throw new Error('Invalid evaluation result');
|
|
26
|
-
return { ...response.result.value, scope: args.frame || args.frame_id ? 'frame' : 'main_frame', ...(args.evaluation_frame ? { frame_id: args.evaluation_frame } : {}), observe_again: true };
|
|
27
|
-
} catch (error) {
|
|
28
|
-
throw new Error(`Evaluation failed; page effects may already have occurred. ${String(error.message).slice(0, 12000)}`);
|
|
29
|
-
} finally { clearTimeout(timer); }
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
// Runs in the page. Bound traversal before transferring the result across CDP.
|
|
33
|
-
// Preserve ordinary JSON; explicitly mark values that JSON cannot represent.
|
|
34
|
-
export function serializeEvaluation(value, maxChars, maxNodes = 1000) {
|
|
35
|
-
const seen = new WeakSet();
|
|
36
|
-
let nodes = 0, chars = 0, truncated = false;
|
|
37
|
-
const mark = type => ({ $type: type });
|
|
38
|
-
function visit(item, depth) {
|
|
39
|
-
if (++nodes > maxNodes || depth > 8 || chars > maxChars) { truncated = true; return mark('truncated'); }
|
|
40
|
-
if (item === null || typeof item === 'boolean') return item;
|
|
41
|
-
if (typeof item === 'number') return Number.isFinite(item) && !Object.is(item, -0) ? item : { $type: 'number', value: String(Object.is(item, -0) ? '-0' : item) };
|
|
42
|
-
if (typeof item === 'string') {
|
|
43
|
-
const text = item.slice(0, Math.max(0, maxChars - chars));
|
|
44
|
-
chars += text.length;
|
|
45
|
-
if (text.length !== item.length) truncated = true;
|
|
46
|
-
return text;
|
|
47
|
-
}
|
|
48
|
-
if (typeof item === 'bigint') return { $type: 'bigint', value: visit(String(item), depth + 1) };
|
|
49
|
-
if (typeof item !== 'object') return mark(typeof item);
|
|
50
|
-
if (seen.has(item)) return mark('circular_or_repeated');
|
|
51
|
-
seen.add(item);
|
|
52
|
-
if (item instanceof Date) return { $type: 'date', value: String(item) };
|
|
53
|
-
if (item instanceof Error) return { $type: 'error', name: visit(item.name, depth + 1), message: visit(item.message, depth + 1) };
|
|
54
|
-
if (Array.isArray(item)) {
|
|
55
|
-
const output = [];
|
|
56
|
-
// Do not copy a sparse index into an array with an unbounded length.
|
|
57
|
-
for (let i = 0; i < item.length && nodes < maxNodes && chars < maxChars && depth < 8; i++) {
|
|
58
|
-
const property = Object.getOwnPropertyDescriptor(item, String(i));
|
|
59
|
-
output.push(!property ? visit(undefined, depth + 1) : 'value' in property ? visit(property.value, depth + 1) : (nodes++, mark('accessor')));
|
|
60
|
-
}
|
|
61
|
-
if (output.length < item.length) truncated = true;
|
|
62
|
-
return output;
|
|
63
|
-
}
|
|
64
|
-
const output = Object.create(null);
|
|
65
|
-
for (const key in item) {
|
|
66
|
-
if (!Object.prototype.hasOwnProperty.call(item, key)) continue;
|
|
67
|
-
if (nodes >= maxNodes || chars + key.length > maxChars || depth >= 8) { truncated = true; break; }
|
|
68
|
-
chars += key.length;
|
|
69
|
-
const property = Object.getOwnPropertyDescriptor(item, key);
|
|
70
|
-
const result = property && 'value' in property ? visit(property.value, depth + 1) : (nodes++, mark('accessor'));
|
|
71
|
-
Object.defineProperty(output, key, { value: result, enumerable: true, configurable: true, writable: true });
|
|
72
|
-
}
|
|
73
|
-
return output;
|
|
74
|
-
}
|
|
75
|
-
const result = visit(value, 0);
|
|
76
|
-
const json = JSON.stringify(result);
|
|
77
|
-
// Escaped strings and property names also count toward the transfer budget.
|
|
78
|
-
// Never emit half a serialized JSON document. Keep the payload structured,
|
|
79
|
-
// and state explicitly when the capture budget lost information.
|
|
80
|
-
if (json.length > maxChars) return { value: { $type: 'truncated', reason: 'serialized_size', required_chars: json.length }, truncated: true };
|
|
81
|
-
return { value: result, truncated };
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
// One bounded immutable result per claimed tab. Continuations never execute JS.
|
|
85
|
-
export function createEvaluations(cdp) {
|
|
86
|
-
const results = new Map();
|
|
87
|
-
async function read(tabId, args) {
|
|
88
|
-
const max = args.max_chars ?? 20000;
|
|
89
|
-
if (!Number.isInteger(max) || max < 1 || max > 100000) throw new Error('max_chars must be 1..100000');
|
|
90
|
-
let state;
|
|
91
|
-
if (args.cursor != null) {
|
|
92
|
-
state = results.get(tabId);
|
|
93
|
-
if (!state || state.cursor !== args.cursor) throw new Error('Evaluation cursor is stale or unknown; results expire on navigation, release or a new evaluation.');
|
|
94
|
-
if (args.expression != null) throw new Error('Provide cursor or expression, never both');
|
|
95
|
-
} else {
|
|
96
|
-
// Navigation/detach may arrive while the CDP promise is pending. A late
|
|
97
|
-
// reply must not resurrect cached page data after cleanup.
|
|
98
|
-
const pending = { frameId: args.evaluation_frame };
|
|
99
|
-
results.set(tabId, pending);
|
|
100
|
-
let result;
|
|
101
|
-
try { result = await evaluatePage(cdp, args.target || tabId, args); }
|
|
102
|
-
catch (error) { if (results.get(tabId) === pending) results.delete(tabId); throw error; }
|
|
103
|
-
if (results.get(tabId) !== pending) throw new Error('Document or claim changed during evaluation; effects may already have occurred. Result was discarded. Observe before continuing; do not replay automatically.');
|
|
104
|
-
if (result.element) { results.delete(tabId); return result; }
|
|
105
|
-
const kind = Array.isArray(result.value) ? 'array' : typeof result.value === 'string' ? 'string' :
|
|
106
|
-
result.value && typeof result.value === 'object' ? 'object' : 'scalar';
|
|
107
|
-
state = Object.assign(pending, { result, kind, offset: 0, token: crypto.randomUUID(), cursor: null,
|
|
108
|
-
items: kind === 'object' ? Object.entries(result.value) : result.value });
|
|
109
|
-
}
|
|
110
|
-
const { kind, items, offset, result } = state;
|
|
111
|
-
const length = kind === 'scalar' ? 1 : items.length;
|
|
112
|
-
const make = (end, start = offset) => kind === 'string' || kind === 'array' ? items.slice(start, end) :
|
|
113
|
-
kind === 'object' ? Object.fromEntries(items.slice(start, end)) : items;
|
|
114
|
-
let low = offset, high = length;
|
|
115
|
-
while (low < high) {
|
|
116
|
-
const mid = Math.ceil((low + high) / 2);
|
|
117
|
-
if (JSON.stringify(make(mid)).length <= max) low = mid; else high = mid - 1;
|
|
118
|
-
}
|
|
119
|
-
if ((length > offset && low === offset) || JSON.stringify(make(low)).length > max) {
|
|
120
|
-
state.cursor ??= `${state.token}@${offset}`;
|
|
121
|
-
return { scope: result.scope, ...(result.frame_id ? { frame_id: result.frame_id } : {}), result_kind: kind, budget_too_small: true, next_cursor: state.cursor,
|
|
122
|
-
required_chars: JSON.stringify(make(Math.min(length, offset + 1))).length,
|
|
123
|
-
note: 'Increase max_chars (allowed 1..100000) and continue with this cursor. Select smaller fields if an individual item exceeds 100000 characters.' };
|
|
124
|
-
}
|
|
125
|
-
state.offset = low;
|
|
126
|
-
state.cursor = low < length ? `${state.token}@${low}` : null;
|
|
127
|
-
const nextRequired = low < length ? JSON.stringify(make(low + 1, low)).length : null;
|
|
128
|
-
return { ...result, value: make(low), result_kind: kind, has_more: low < length,
|
|
129
|
-
...(nextRequired > max ? { next_item: { ...(kind === 'object' ? { key: items[low][0] } : { index: low }), required_chars: nextRequired, exceeds_max_chars: nextRequired > 100000 } } : {}),
|
|
130
|
-
next_cursor: state.cursor, capture_truncated: result.truncated, truncated: result.truncated || low < length };
|
|
131
|
-
}
|
|
132
|
-
return { read, forgetFrame(tabId, frameId) { if (results.get(tabId)?.frameId === frameId) results.delete(tabId); }, forget: tabId => results.delete(tabId) };
|
|
133
|
-
}
|
|
1
|
+
// Deliberately a page expression API, not a raw CDP transport.
|
|
2
|
+
const CAPTURE_CHARS = 1000000, CAPTURE_NODES = 50000;
|
|
3
|
+
async function evaluatePage(cdp, tabId, args) {
|
|
4
|
+
const timeout = args.timeout_ms ?? 10000, maxChars = args.max_chars ?? 20000;
|
|
5
|
+
if (typeof args.expression !== 'string' || !args.expression.trim() || args.expression.length > 100000)
|
|
6
|
+
throw new Error('expression must contain 1..100000 characters of JavaScript');
|
|
7
|
+
if (!Number.isInteger(timeout) || timeout < 100 || timeout > 30000) throw new Error('timeout_ms must be 100..30000');
|
|
8
|
+
if (!Number.isInteger(maxChars) || maxChars < 1 || maxChars > 100000) throw new Error('max_chars must be 1..100000');
|
|
9
|
+
let timer;
|
|
10
|
+
try {
|
|
11
|
+
const response = await Promise.race([
|
|
12
|
+
cdp(tabId, 'Runtime.evaluate', {
|
|
13
|
+
expression: `(async () => { let result = await (\n${args.expression}\n); if (typeof result === "function") result = await result(); return ${args.result_type === 'element' ? 'result' : `(${serializeEvaluation.toString()})(result, ${CAPTURE_CHARS}, ${CAPTURE_NODES})`}; })()`,
|
|
14
|
+
...(args.result_type === 'element' ? { objectGroup: 'browser-attach-evaluation' } : {}), ...(args.context_id != null ? { contextId: args.context_id } : {}), awaitPromise: true, returnByValue: args.result_type !== 'element', userGesture: false, timeout,
|
|
15
|
+
}),
|
|
16
|
+
new Promise((resolve, reject) => { timer = setTimeout(() => reject(Object.assign(new Error(
|
|
17
|
+
'JavaScript evaluation timed out. It may still run or have changed the page; observe before continuing. Do not replay automatically.'
|
|
18
|
+
), { outcome_unknown: true })), timeout); }),
|
|
19
|
+
]);
|
|
20
|
+
if (response.exceptionDetails) throw new Error(response.exceptionDetails.exception?.description || response.exceptionDetails.text);
|
|
21
|
+
if (args.result_type === 'element') {
|
|
22
|
+
if (response.result?.subtype !== 'node' || !response.result.objectId) throw new Error('Element result must be one connected DOM element, not JSON/array/null');
|
|
23
|
+
return { element: response.result };
|
|
24
|
+
}
|
|
25
|
+
if (!response.result?.value || typeof response.result.value.truncated !== 'boolean') throw new Error('Invalid evaluation result');
|
|
26
|
+
return { ...response.result.value, scope: args.frame || args.frame_id ? 'frame' : 'main_frame', ...(args.evaluation_frame ? { frame_id: args.evaluation_frame } : {}), observe_again: true };
|
|
27
|
+
} catch (error) {
|
|
28
|
+
throw Object.assign(new Error(`Evaluation failed; page effects may already have occurred. ${String(error.message).slice(0, 12000)}`), { outcome_unknown: true });
|
|
29
|
+
} finally { clearTimeout(timer); }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Runs in the page. Bound traversal before transferring the result across CDP.
|
|
33
|
+
// Preserve ordinary JSON; explicitly mark values that JSON cannot represent.
|
|
34
|
+
export function serializeEvaluation(value, maxChars, maxNodes = 1000) {
|
|
35
|
+
const seen = new WeakSet();
|
|
36
|
+
let nodes = 0, chars = 0, truncated = false;
|
|
37
|
+
const mark = type => ({ $type: type });
|
|
38
|
+
function visit(item, depth) {
|
|
39
|
+
if (++nodes > maxNodes || depth > 8 || chars > maxChars) { truncated = true; return mark('truncated'); }
|
|
40
|
+
if (item === null || typeof item === 'boolean') return item;
|
|
41
|
+
if (typeof item === 'number') return Number.isFinite(item) && !Object.is(item, -0) ? item : { $type: 'number', value: String(Object.is(item, -0) ? '-0' : item) };
|
|
42
|
+
if (typeof item === 'string') {
|
|
43
|
+
const text = item.slice(0, Math.max(0, maxChars - chars));
|
|
44
|
+
chars += text.length;
|
|
45
|
+
if (text.length !== item.length) truncated = true;
|
|
46
|
+
return text;
|
|
47
|
+
}
|
|
48
|
+
if (typeof item === 'bigint') return { $type: 'bigint', value: visit(String(item), depth + 1) };
|
|
49
|
+
if (typeof item !== 'object') return mark(typeof item);
|
|
50
|
+
if (seen.has(item)) return mark('circular_or_repeated');
|
|
51
|
+
seen.add(item);
|
|
52
|
+
if (item instanceof Date) return { $type: 'date', value: String(item) };
|
|
53
|
+
if (item instanceof Error) return { $type: 'error', name: visit(item.name, depth + 1), message: visit(item.message, depth + 1) };
|
|
54
|
+
if (Array.isArray(item)) {
|
|
55
|
+
const output = [];
|
|
56
|
+
// Do not copy a sparse index into an array with an unbounded length.
|
|
57
|
+
for (let i = 0; i < item.length && nodes < maxNodes && chars < maxChars && depth < 8; i++) {
|
|
58
|
+
const property = Object.getOwnPropertyDescriptor(item, String(i));
|
|
59
|
+
output.push(!property ? visit(undefined, depth + 1) : 'value' in property ? visit(property.value, depth + 1) : (nodes++, mark('accessor')));
|
|
60
|
+
}
|
|
61
|
+
if (output.length < item.length) truncated = true;
|
|
62
|
+
return output;
|
|
63
|
+
}
|
|
64
|
+
const output = Object.create(null);
|
|
65
|
+
for (const key in item) {
|
|
66
|
+
if (!Object.prototype.hasOwnProperty.call(item, key)) continue;
|
|
67
|
+
if (nodes >= maxNodes || chars + key.length > maxChars || depth >= 8) { truncated = true; break; }
|
|
68
|
+
chars += key.length;
|
|
69
|
+
const property = Object.getOwnPropertyDescriptor(item, key);
|
|
70
|
+
const result = property && 'value' in property ? visit(property.value, depth + 1) : (nodes++, mark('accessor'));
|
|
71
|
+
Object.defineProperty(output, key, { value: result, enumerable: true, configurable: true, writable: true });
|
|
72
|
+
}
|
|
73
|
+
return output;
|
|
74
|
+
}
|
|
75
|
+
const result = visit(value, 0);
|
|
76
|
+
const json = JSON.stringify(result);
|
|
77
|
+
// Escaped strings and property names also count toward the transfer budget.
|
|
78
|
+
// Never emit half a serialized JSON document. Keep the payload structured,
|
|
79
|
+
// and state explicitly when the capture budget lost information.
|
|
80
|
+
if (json.length > maxChars) return { value: { $type: 'truncated', reason: 'serialized_size', required_chars: json.length }, truncated: true };
|
|
81
|
+
return { value: result, truncated };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// One bounded immutable result per claimed tab. Continuations never execute JS.
|
|
85
|
+
export function createEvaluations(cdp) {
|
|
86
|
+
const results = new Map();
|
|
87
|
+
async function read(tabId, args) {
|
|
88
|
+
const max = args.max_chars ?? 20000;
|
|
89
|
+
if (!Number.isInteger(max) || max < 1 || max > 100000) throw new Error('max_chars must be 1..100000');
|
|
90
|
+
let state;
|
|
91
|
+
if (args.cursor != null) {
|
|
92
|
+
state = results.get(tabId);
|
|
93
|
+
if (!state || state.cursor !== args.cursor) throw new Error('Evaluation cursor is stale or unknown; results expire on navigation, release or a new evaluation.');
|
|
94
|
+
if (args.expression != null) throw new Error('Provide cursor or expression, never both');
|
|
95
|
+
} else {
|
|
96
|
+
// Navigation/detach may arrive while the CDP promise is pending. A late
|
|
97
|
+
// reply must not resurrect cached page data after cleanup.
|
|
98
|
+
const pending = { frameId: args.evaluation_frame };
|
|
99
|
+
results.set(tabId, pending);
|
|
100
|
+
let result;
|
|
101
|
+
try { result = await evaluatePage(cdp, args.target || tabId, args); }
|
|
102
|
+
catch (error) { if (results.get(tabId) === pending) results.delete(tabId); throw error; }
|
|
103
|
+
if (results.get(tabId) !== pending) throw Object.assign(new Error('Document or claim changed during evaluation; effects may already have occurred. Result was discarded. Observe before continuing; do not replay automatically.'), { outcome_unknown: true });
|
|
104
|
+
if (result.element) { results.delete(tabId); return result; }
|
|
105
|
+
const kind = Array.isArray(result.value) ? 'array' : typeof result.value === 'string' ? 'string' :
|
|
106
|
+
result.value && typeof result.value === 'object' ? 'object' : 'scalar';
|
|
107
|
+
state = Object.assign(pending, { result, kind, offset: 0, token: crypto.randomUUID(), cursor: null,
|
|
108
|
+
items: kind === 'object' ? Object.entries(result.value) : result.value });
|
|
109
|
+
}
|
|
110
|
+
const { kind, items, offset, result } = state;
|
|
111
|
+
const length = kind === 'scalar' ? 1 : items.length;
|
|
112
|
+
const make = (end, start = offset) => kind === 'string' || kind === 'array' ? items.slice(start, end) :
|
|
113
|
+
kind === 'object' ? Object.fromEntries(items.slice(start, end)) : items;
|
|
114
|
+
let low = offset, high = length;
|
|
115
|
+
while (low < high) {
|
|
116
|
+
const mid = Math.ceil((low + high) / 2);
|
|
117
|
+
if (JSON.stringify(make(mid)).length <= max) low = mid; else high = mid - 1;
|
|
118
|
+
}
|
|
119
|
+
if ((length > offset && low === offset) || JSON.stringify(make(low)).length > max) {
|
|
120
|
+
state.cursor ??= `${state.token}@${offset}`;
|
|
121
|
+
return { scope: result.scope, ...(result.frame_id ? { frame_id: result.frame_id } : {}), result_kind: kind, budget_too_small: true, next_cursor: state.cursor,
|
|
122
|
+
required_chars: JSON.stringify(make(Math.min(length, offset + 1))).length,
|
|
123
|
+
note: 'Increase max_chars (allowed 1..100000) and continue with this cursor. Select smaller fields if an individual item exceeds 100000 characters.' };
|
|
124
|
+
}
|
|
125
|
+
state.offset = low;
|
|
126
|
+
state.cursor = low < length ? `${state.token}@${low}` : null;
|
|
127
|
+
const nextRequired = low < length ? JSON.stringify(make(low + 1, low)).length : null;
|
|
128
|
+
return { ...result, value: make(low), result_kind: kind, has_more: low < length,
|
|
129
|
+
...(nextRequired > max ? { next_item: { ...(kind === 'object' ? { key: items[low][0] } : { index: low }), required_chars: nextRequired, exceeds_max_chars: nextRequired > 100000 } } : {}),
|
|
130
|
+
next_cursor: state.cursor, capture_truncated: result.truncated, truncated: result.truncated || low < length };
|
|
131
|
+
}
|
|
132
|
+
return { read, forgetFrame(tabId, frameId) { if (results.get(tabId)?.frameId === frameId) results.delete(tabId); }, forget: tabId => results.delete(tabId) };
|
|
133
|
+
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import { deadline } from './actionability.js';
|
|
2
|
+
import { boundedRead } from './outcomes.js';
|
|
3
|
+
import { capturePage } from './fetch.js';
|
|
4
|
+
import { waitFor } from './wait.js';
|
|
5
|
+
import { httpUrl } from './page-url.js';
|
|
6
|
+
|
|
7
|
+
export function createFetchPages(api, cdp, value, navigation, network, sessions, pointer, observations, { queued, attach, currentClaim, forget }) {
|
|
8
|
+
const temporaryTabs = new Map();
|
|
9
|
+
async function closeTemporary(tabId) {
|
|
10
|
+
try { await api.tabs.remove(tabId); }
|
|
11
|
+
catch (error) {
|
|
12
|
+
// A closed tab is already clean; otherwise retain ownership for later cleanup.
|
|
13
|
+
let tab;
|
|
14
|
+
try { tab = await api.tabs.get(tabId); }
|
|
15
|
+
catch (lookup) { if (!/No tab with id|Invalid tab ID/i.test(lookup.message)) throw error; }
|
|
16
|
+
if (tab) throw error;
|
|
17
|
+
}
|
|
18
|
+
temporaryTabs.delete(tabId);
|
|
19
|
+
if (currentClaim(tabId)) forget(tabId);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
async function fetchPage(owner, args, cap, checkConnection) {
|
|
23
|
+
const url = httpUrl(args.url), timeout = args.timeout_ms ?? 15000, waitUntil = args.wait_until ?? 'load';
|
|
24
|
+
const end = Date.now() + timeout, cleanupReserve = Math.min(250, Math.floor(timeout / 10));
|
|
25
|
+
const budget = deadline(timeout, 'fetch', end - cleanupReserve);
|
|
26
|
+
const handle = crypto.randomUUID(), temporary = { owner, handle };
|
|
27
|
+
const creation = api.tabs.create({ url: 'about:blank', active: false }).then(tab => {
|
|
28
|
+
temporaryTabs.set(tab.id, temporary);
|
|
29
|
+
return tab;
|
|
30
|
+
});
|
|
31
|
+
let tab;
|
|
32
|
+
try { tab = await budget.run(() => creation); }
|
|
33
|
+
catch (error) {
|
|
34
|
+
// A late tabs.create response must not start navigation or leak its tab.
|
|
35
|
+
creation.then(created => queued(created.id, () => closeTemporary(created.id))).catch(() => {});
|
|
36
|
+
throw error;
|
|
37
|
+
}
|
|
38
|
+
return await queued(tab.id, async () => {
|
|
39
|
+
let failure;
|
|
40
|
+
try {
|
|
41
|
+
checkConnection();
|
|
42
|
+
await attach(tab.id, owner, { tab_handle: handle }, tab, budget);
|
|
43
|
+
const claim = currentClaim(tab.id);
|
|
44
|
+
const checkControl = () => {
|
|
45
|
+
checkConnection();
|
|
46
|
+
if (currentClaim(tab.id) !== claim) throw new Error('Tab control ended during fetch; capture discarded');
|
|
47
|
+
};
|
|
48
|
+
const metadata = (satisfied, reason, readyState) => ({ wait_until: waitUntil, wait_satisfied: satisfied,
|
|
49
|
+
degraded: !satisfied, degraded_reason: reason, ready_state: readyState, document_changed: !!temporary.document_changed });
|
|
50
|
+
const httpFailure = response => {
|
|
51
|
+
if ((args.format ?? 'text') === 'json') throw new Error(`HTTP ${response.status}: navigation produced an error page without a readable JSON document`);
|
|
52
|
+
return { url: httpUrl(response.url), contentType: response.contentType, format: args.format ?? 'text', body: '',
|
|
53
|
+
body_source: response.empty_body ? 'http_response' : 'unavailable', ...(!response.empty_body ? { body_unavailable: true } : {}),
|
|
54
|
+
capture_truncated: false, status: response.status, ok: response.status >= 200 && response.status < 300,
|
|
55
|
+
...metadata(false, 'http_error_response', null), navigation_error: response.error };
|
|
56
|
+
};
|
|
57
|
+
checkControl();
|
|
58
|
+
if (budget.remaining() < 100) throw new Error('Fetch preparation timed out; no navigation was dispatched');
|
|
59
|
+
const readReserve = Math.min(2000, Math.floor(budget.remaining() / 3));
|
|
60
|
+
const waitEnd = Date.now() + budget.remaining() - readReserve;
|
|
61
|
+
const waitBudget = deadline(timeout, 'wait', waitEnd);
|
|
62
|
+
let satisfied = false, reason = null, waitedDocument;
|
|
63
|
+
try {
|
|
64
|
+
const navigationResult = await navigation.navigate(tab.id, url, { wait_until: waitUntil === 'network_idle' ? 'load' : waitUntil,
|
|
65
|
+
timeout_ms: Math.max(100, waitBudget.remaining()) });
|
|
66
|
+
waitedDocument = JSON.stringify([navigationResult.document.frameId, navigationResult.document.loaderId]);
|
|
67
|
+
checkControl();
|
|
68
|
+
if (waitUntil === 'network_idle') await waitFor(cdp, value, sessions, network, pointer, tab.id, { condition: 'network_idle', budget: waitBudget });
|
|
69
|
+
satisfied = true;
|
|
70
|
+
} catch (error) {
|
|
71
|
+
checkControl();
|
|
72
|
+
if (error.navigation_error === 'net::ERR_HTTP_RESPONSE_CODE_FAILURE') {
|
|
73
|
+
const response = network.failedDocument(tab.id, observations.mainFrame(tab.id));
|
|
74
|
+
if (response) return httpFailure(response);
|
|
75
|
+
throw error;
|
|
76
|
+
}
|
|
77
|
+
if (error.code === 'NAVIGATION_TIMEOUT' || (waitUntil === 'network_idle' && waitBudget.remaining() === 0)) reason = 'wait_timeout';
|
|
78
|
+
else if (error.navigation_error === 'net::ERR_ABORTED' && temporary.document) reason = 'navigation_interrupted';
|
|
79
|
+
else throw error;
|
|
80
|
+
}
|
|
81
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
82
|
+
checkControl();
|
|
83
|
+
const version = observations.version(tab.id);
|
|
84
|
+
try {
|
|
85
|
+
const { frameTree } = await budget.run(() => cdp(tab.id, 'Page.getFrameTree'));
|
|
86
|
+
const response = frameTree.frame.unreachableUrl && network.failedDocument(tab.id, frameTree.frame.id);
|
|
87
|
+
if (response && response.url === frameTree.frame.unreachableUrl) return httpFailure(response);
|
|
88
|
+
httpUrl(frameTree.frame.url);
|
|
89
|
+
const { executionContextId } = await budget.run(() => cdp(tab.id, 'Page.createIsolatedWorld', { frameId: frameTree.frame.id, worldName: 'browser-attach-fetch' }));
|
|
90
|
+
const result = value(await budget.run(() => cdp(tab.id, 'Runtime.evaluate', { contextId: executionContextId, returnByValue: true, userGesture: false,
|
|
91
|
+
expression: `(${capturePage.toString()})(${JSON.stringify(args.format ?? 'text')},${cap})` })));
|
|
92
|
+
checkControl();
|
|
93
|
+
if (observations.version(tab.id) !== version) throw Object.assign(new Error('Document changed during fetch capture'), { code: 'FETCH_DOCUMENT_CHANGED' });
|
|
94
|
+
httpUrl(result.url);
|
|
95
|
+
const status = network.documentStatus(tab.id, frameTree.frame);
|
|
96
|
+
if (satisfied && waitedDocument !== JSON.stringify([frameTree.frame.id, frameTree.frame.loaderId])) { satisfied = false; reason = 'document_changed'; }
|
|
97
|
+
return { ...result, status, ok: status == null ? null : status >= 200 && status < 300,
|
|
98
|
+
...metadata(satisfied, reason, result.ready_state) };
|
|
99
|
+
} catch (error) {
|
|
100
|
+
checkControl();
|
|
101
|
+
const changing = observations.version(tab.id) !== version || error.code === 'FETCH_DOCUMENT_CHANGED' || /context.*(destroyed|not found)|cannot find context|no frame with given id|inspected target navigated/i.test(error.message);
|
|
102
|
+
if (!changing || attempt === 2) throw error;
|
|
103
|
+
await budget.run(() => new Promise(resolve => setTimeout(resolve, 25)));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
} catch (error) { failure = error; throw error; }
|
|
107
|
+
finally {
|
|
108
|
+
try { await boundedRead(() => closeTemporary(tab.id), Math.max(1, end - Date.now())); }
|
|
109
|
+
catch (error) { throw new Error(`${failure ? failure.message + '; ' : ''}Fetch temporary tab ${tab.id} could not be closed: ${error.message}. Cleanup will be retried on disconnect.`); }
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
function event(source, method, params) {
|
|
116
|
+
const mainNavigation = !source.sessionId && method === 'Page.frameNavigated' && params.frame && !params.frame.parentId;
|
|
117
|
+
const temporary = temporaryTabs.get(source.tabId);
|
|
118
|
+
if (mainNavigation && temporary && params.frame.loaderId) {
|
|
119
|
+
const document = JSON.stringify([params.frame.id, params.frame.loaderId]);
|
|
120
|
+
if (!temporary.firstDocument && /^https?:/.test(params.frame.url || '') && !params.frame.unreachableUrl) temporary.firstDocument = document;
|
|
121
|
+
if (temporary.firstDocument) {
|
|
122
|
+
temporary.document = document;
|
|
123
|
+
temporary.document_changed ||= document !== temporary.firstDocument;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return { read: fetchPage, close: closeTemporary, event,
|
|
128
|
+
has: tabId => temporaryTabs.has(tabId), entries: () => temporaryTabs.entries(),
|
|
129
|
+
removed: tabId => temporaryTabs.delete(tabId),
|
|
130
|
+
};
|
|
131
|
+
}
|