@agenttool/browser 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +31 -21
- package/README.md +77 -24
- package/dist/src/browser.d.ts +18 -2
- package/dist/src/browser.d.ts.map +1 -1
- package/dist/src/browser.js +371 -82
- package/dist/src/browser.js.map +1 -1
- package/dist/src/capabilities.d.ts +6 -2
- package/dist/src/capabilities.d.ts.map +1 -1
- package/dist/src/capabilities.js +8 -4
- package/dist/src/capabilities.js.map +1 -1
- package/dist/src/cli.d.ts.map +1 -1
- package/dist/src/cli.js +3 -2
- package/dist/src/cli.js.map +1 -1
- package/dist/src/mcp.d.ts +14 -7
- package/dist/src/mcp.d.ts.map +1 -1
- package/dist/src/mcp.js +71 -63
- package/dist/src/mcp.js.map +1 -1
- package/dist/src/planning.d.ts.map +1 -1
- package/dist/src/planning.js +1 -3
- package/dist/src/planning.js.map +1 -1
- package/dist/src/policy.d.ts +5 -1
- package/dist/src/policy.d.ts.map +1 -1
- package/dist/src/policy.js +7 -2
- package/dist/src/policy.js.map +1 -1
- package/dist/src/snapshot.d.ts +6 -0
- package/dist/src/snapshot.d.ts.map +1 -1
- package/dist/src/snapshot.js +116 -23
- package/dist/src/snapshot.js.map +1 -1
- package/dist/src/types.d.ts +12 -4
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/version.d.ts +1 -1
- package/dist/src/version.js +1 -1
- package/package.json +1 -1
package/dist/src/browser.js
CHANGED
|
@@ -3,11 +3,12 @@ import { existsSync } from "node:fs";
|
|
|
3
3
|
import { chmod, lstat, mkdir, realpath } from "node:fs/promises";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
6
|
+
import { performance } from "node:perf_hooks";
|
|
6
7
|
import { resolveBrowserCapabilities, } from "./capabilities.js";
|
|
7
8
|
import { asBrowserError, BrowserError } from "./errors.js";
|
|
8
9
|
import { planBrowserAction, } from "./planning.js";
|
|
9
10
|
import { BrowserNetworkPolicy, redactHtmlUrlAttributes, redactUrlReferenceForOutput, redactUrlForOutput, redactUrlsInText, } from "./policy.js";
|
|
10
|
-
import { boundText, compactAriaSnapshot, intersectsViewport, looksLikeSensitiveControl, parseAriaCandidates, redactAriaSecrets, redactSensitiveInputValues, } from "./snapshot.js";
|
|
11
|
+
import { boundText, compactAriaSnapshot, intersectsViewport, looksLikeSensitiveControl, parseAriaCandidates, parseStructuralAriaCandidates, redactAriaSecrets, redactSensitiveInputValues, } from "./snapshot.js";
|
|
11
12
|
import { OBSERVATION_SCHEMA, } from "./types.js";
|
|
12
13
|
export const DEFAULT_BROWSER_LIMITS = Object.freeze({
|
|
13
14
|
maxSnapshotChars: 24_000,
|
|
@@ -22,6 +23,12 @@ const DEFAULT_VIEWPORT = Object.freeze({ width: 1280, height: 720 });
|
|
|
22
23
|
const DEFAULT_ACTION_TIMEOUT_MS = 10_000;
|
|
23
24
|
const DEFAULT_NAVIGATION_TIMEOUT_MS = 30_000;
|
|
24
25
|
const MAX_RESPONSE_HINT_CHARS = 4_096;
|
|
26
|
+
const MAX_RETAINED_SNAPSHOTS_PER_TAB = 8;
|
|
27
|
+
const MAX_STRUCTURAL_CONTEXT_ELEMENTS = 32;
|
|
28
|
+
const VIEWPORT_SETTLE_INTERVAL_MS = 25;
|
|
29
|
+
const VIEWPORT_SETTLE_MAX_MS = 1_000;
|
|
30
|
+
const VIEWPORT_SETTLE_STABLE_INTERVALS = 2;
|
|
31
|
+
const VIEWPORT_SETTLE_TOLERANCE_PX = 0.5;
|
|
25
32
|
const RESPONSE_HINT_HEADERS = Object.freeze([
|
|
26
33
|
"link",
|
|
27
34
|
"content-location",
|
|
@@ -41,9 +48,12 @@ export class AgentBrowser {
|
|
|
41
48
|
browser;
|
|
42
49
|
states = new Map();
|
|
43
50
|
pageStates = new Map();
|
|
51
|
+
requestPolicyDenials = new Map();
|
|
44
52
|
activeTabId = null;
|
|
45
53
|
nextTabNumber = 1;
|
|
54
|
+
requestPolicyDenialSequence = 0;
|
|
46
55
|
closed = false;
|
|
56
|
+
closePromise = null;
|
|
47
57
|
operationTail = Promise.resolve();
|
|
48
58
|
constructor(options, context, browser) {
|
|
49
59
|
this.sessionId = `session_${randomUUID()}`;
|
|
@@ -102,13 +112,7 @@ export class AgentBrowser {
|
|
|
102
112
|
}
|
|
103
113
|
catch (error) {
|
|
104
114
|
try {
|
|
105
|
-
await context
|
|
106
|
-
}
|
|
107
|
-
catch {
|
|
108
|
-
// Preserve the launch error; cleanup failure is secondary.
|
|
109
|
-
}
|
|
110
|
-
try {
|
|
111
|
-
await browser?.close();
|
|
115
|
+
await closeRuntime(browser, context);
|
|
112
116
|
}
|
|
113
117
|
catch {
|
|
114
118
|
// Preserve the launch error; cleanup failure is secondary.
|
|
@@ -123,9 +127,12 @@ export class AgentBrowser {
|
|
|
123
127
|
async openUnlocked(url) {
|
|
124
128
|
this.assertOpen();
|
|
125
129
|
const destination = await this.policy.assertAllowed(url);
|
|
130
|
+
this.assertOpen();
|
|
126
131
|
const page = await this.context.newPage();
|
|
132
|
+
this.assertOpen();
|
|
127
133
|
const state = this.registerPage(page);
|
|
128
134
|
this.activeTabId = state.id;
|
|
135
|
+
const denialSequence = this.requestPolicyDenialSequence;
|
|
129
136
|
try {
|
|
130
137
|
this.clearMainDocumentResponse(state);
|
|
131
138
|
const response = await page.goto(destination.href, {
|
|
@@ -133,9 +140,15 @@ export class AgentBrowser {
|
|
|
133
140
|
timeout: this.options.navigationTimeoutMs,
|
|
134
141
|
});
|
|
135
142
|
this.queueMainDocumentResponse(state, response);
|
|
143
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
144
|
+
if (denial)
|
|
145
|
+
throw denial;
|
|
136
146
|
}
|
|
137
147
|
catch (error) {
|
|
138
148
|
this.invalidate(state);
|
|
149
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
150
|
+
if (denial)
|
|
151
|
+
throw denial;
|
|
139
152
|
throw asBrowserError(error, "action_failed", "Navigation was attempted once and did not complete.");
|
|
140
153
|
}
|
|
141
154
|
return this.observeUnlocked({ tabId: state.id });
|
|
@@ -152,8 +165,12 @@ export class AgentBrowser {
|
|
|
152
165
|
}
|
|
153
166
|
const state = this.getState(options.tabId);
|
|
154
167
|
this.activeTabId = state.id;
|
|
155
|
-
this.
|
|
156
|
-
|
|
168
|
+
await this.awaitWindowViewportSettled(state.page);
|
|
169
|
+
this.assertOpen();
|
|
170
|
+
state.revision += 1;
|
|
171
|
+
const observationRevision = state.revision;
|
|
172
|
+
const observationNavigationEpoch = state.navigationEpoch;
|
|
173
|
+
const snapshotId = `${this.sessionId}:${state.id}:${observationRevision}`;
|
|
157
174
|
const viewport = state.page.viewportSize() ?? this.options.viewport;
|
|
158
175
|
const includeText = options.includeText ?? true;
|
|
159
176
|
const maxTextChars = boundedPositiveInteger(options.maxTextChars, this.options.limits.maxTextChars, "maxTextChars", this.options.limits.maxTextChars);
|
|
@@ -164,39 +181,63 @@ export class AgentBrowser {
|
|
|
164
181
|
timeout: this.options.actionTimeoutMs,
|
|
165
182
|
});
|
|
166
183
|
const candidates = parseAriaCandidates(raw).slice(0, this.options.limits.maxSnapshotElements * 3);
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
184
|
+
const structuralCandidateScanLimit = MAX_STRUCTURAL_CONTEXT_ELEMENTS * 3;
|
|
185
|
+
const allStructuralCandidates = parseStructuralAriaCandidates(raw);
|
|
186
|
+
const structuralCandidates = allStructuralCandidates.slice(0, structuralCandidateScanLimit);
|
|
187
|
+
const [inspected, inspectedStructural] = await Promise.all([
|
|
188
|
+
Promise.all(candidates.map(async (candidate) => {
|
|
189
|
+
const locator = state.page.locator(`aria-ref=${candidate.nativeRef}`);
|
|
190
|
+
try {
|
|
191
|
+
if ((await locator.count()) !== 1 || !(await locator.isVisible())) {
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
if (!intersectsViewport(await locator.boundingBox(), viewport)) {
|
|
195
|
+
return null;
|
|
196
|
+
}
|
|
197
|
+
const attributeNames = [
|
|
198
|
+
"type",
|
|
199
|
+
"autocomplete",
|
|
200
|
+
"name",
|
|
201
|
+
"id",
|
|
202
|
+
"placeholder",
|
|
203
|
+
"aria-label",
|
|
204
|
+
];
|
|
205
|
+
const values = await Promise.all(attributeNames.map((name) => locator.getAttribute(name)));
|
|
206
|
+
const attributes = {};
|
|
207
|
+
attributeNames.forEach((name, index) => {
|
|
208
|
+
attributes[name] = values[index] ?? null;
|
|
209
|
+
});
|
|
210
|
+
return {
|
|
211
|
+
candidate,
|
|
212
|
+
secret: looksLikeSensitiveControl(attributes, candidate.name),
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
catch {
|
|
216
|
+
// The page changed while it was being observed. Omit that target;
|
|
217
|
+
// never guess a replacement ref.
|
|
171
218
|
return null;
|
|
172
219
|
}
|
|
173
|
-
|
|
220
|
+
})),
|
|
221
|
+
Promise.all(structuralCandidates.map(async (candidate) => {
|
|
222
|
+
const locator = state.page.locator(`aria-ref=${candidate.nativeRef}`);
|
|
223
|
+
try {
|
|
224
|
+
if ((await locator.count()) !== 1 || !(await locator.isVisible())) {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
227
|
+
if (!intersectsViewport(await locator.boundingBox(), viewport)) {
|
|
228
|
+
return null;
|
|
229
|
+
}
|
|
230
|
+
return candidate.nativeRef;
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// Structural context is optional. Omit a node whose geometry
|
|
234
|
+
// changed rather than exposing an unbound or guessed context.
|
|
174
235
|
return null;
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
"name",
|
|
179
|
-
"id",
|
|
180
|
-
"placeholder",
|
|
181
|
-
"aria-label",
|
|
182
|
-
];
|
|
183
|
-
const values = await Promise.all(attributeNames.map((name) => locator.getAttribute(name)));
|
|
184
|
-
const attributes = {};
|
|
185
|
-
attributeNames.forEach((name, index) => {
|
|
186
|
-
attributes[name] = values[index] ?? null;
|
|
187
|
-
});
|
|
188
|
-
return {
|
|
189
|
-
candidate,
|
|
190
|
-
secret: looksLikeSensitiveControl(attributes, candidate.name),
|
|
191
|
-
};
|
|
192
|
-
}
|
|
193
|
-
catch {
|
|
194
|
-
// The page changed while it was being observed. Omit that target;
|
|
195
|
-
// never guess a replacement ref.
|
|
196
|
-
return null;
|
|
197
|
-
}
|
|
198
|
-
}));
|
|
236
|
+
}
|
|
237
|
+
})),
|
|
238
|
+
]);
|
|
199
239
|
const visibleRefs = new Set();
|
|
240
|
+
const visibleStructuralRefs = new Set();
|
|
200
241
|
const secretRefs = new Set();
|
|
201
242
|
const publicRefs = new Map();
|
|
202
243
|
for (const item of inspected) {
|
|
@@ -208,16 +249,21 @@ export class AgentBrowser {
|
|
|
208
249
|
if (item.secret)
|
|
209
250
|
secretRefs.add(item.candidate.nativeRef);
|
|
210
251
|
}
|
|
252
|
+
for (const nativeRef of inspectedStructural) {
|
|
253
|
+
if (nativeRef)
|
|
254
|
+
visibleStructuralRefs.add(nativeRef);
|
|
255
|
+
}
|
|
211
256
|
const sanitizedRaw = redactUrlsInText(redactAriaSecrets(raw, secretRefs));
|
|
212
257
|
const compact = compactAriaSnapshot(sanitizedRaw, {
|
|
213
258
|
publicRefs,
|
|
214
259
|
visibleRefs,
|
|
260
|
+
visibleStructuralRefs,
|
|
215
261
|
secretRefs,
|
|
216
262
|
maxChars: this.options.limits.maxSnapshotChars,
|
|
217
263
|
maxElements: this.options.limits.maxSnapshotElements,
|
|
264
|
+
maxStructuralElements: MAX_STRUCTURAL_CONTEXT_ELEMENTS,
|
|
218
265
|
});
|
|
219
|
-
|
|
220
|
-
state.refs = new Map(compact.refs.map((ref) => {
|
|
266
|
+
const snapshotRefs = new Map(compact.refs.map((ref) => {
|
|
221
267
|
const nativeRef = publicRefsEntries(publicRefs, ref.ref);
|
|
222
268
|
return [ref.ref, nativeRef];
|
|
223
269
|
}));
|
|
@@ -241,13 +287,15 @@ export class AgentBrowser {
|
|
|
241
287
|
: null;
|
|
242
288
|
if (responseSequence !== state.responseSequence)
|
|
243
289
|
response = null;
|
|
290
|
+
this.assertObservationCurrent(state, observationRevision, observationNavigationEpoch);
|
|
291
|
+
this.rememberSnapshot(state, snapshotId, snapshotRefs, observationNavigationEpoch);
|
|
244
292
|
return {
|
|
245
293
|
schema: OBSERVATION_SCHEMA,
|
|
246
294
|
sessionId: this.sessionId,
|
|
247
295
|
snapshotId,
|
|
248
296
|
tabId: state.id,
|
|
249
297
|
pageId: state.id,
|
|
250
|
-
revision:
|
|
298
|
+
revision: observationRevision,
|
|
251
299
|
url,
|
|
252
300
|
title,
|
|
253
301
|
snapshot: compact.snapshot,
|
|
@@ -255,7 +303,8 @@ export class AgentBrowser {
|
|
|
255
303
|
refs: compact.refs,
|
|
256
304
|
response,
|
|
257
305
|
truncated: {
|
|
258
|
-
snapshot: compact.truncated.snapshot
|
|
306
|
+
snapshot: compact.truncated.snapshot
|
|
307
|
+
|| allStructuralCandidates.length > structuralCandidateScanLimit,
|
|
259
308
|
text: textTruncated,
|
|
260
309
|
elements: compact.truncated.elements
|
|
261
310
|
|| candidates.length
|
|
@@ -266,13 +315,12 @@ export class AgentBrowser {
|
|
|
266
315
|
};
|
|
267
316
|
}
|
|
268
317
|
catch (error) {
|
|
269
|
-
state.snapshotId = null;
|
|
270
|
-
state.refs.clear();
|
|
271
318
|
throw asBrowserError(error, "action_failed", "Could not create a bounded browser observation.");
|
|
272
319
|
}
|
|
273
320
|
}
|
|
274
321
|
async act(action) {
|
|
275
|
-
|
|
322
|
+
const capturedAction = captureBrowserAction(action);
|
|
323
|
+
return this.withLock(() => this.actUnlocked(capturedAction));
|
|
276
324
|
}
|
|
277
325
|
/** Return the immutable authority manifest selected when this session launched. */
|
|
278
326
|
capabilities() {
|
|
@@ -292,11 +340,12 @@ export class AgentBrowser {
|
|
|
292
340
|
* interleave between the one action attempt and its read-only observation.
|
|
293
341
|
*/
|
|
294
342
|
async actAndObserve(action) {
|
|
343
|
+
const capturedAction = captureBrowserAction(action);
|
|
295
344
|
return this.withLock(async () => {
|
|
296
|
-
const actionResult = await this.actUnlocked(
|
|
345
|
+
const actionResult = await this.actUnlocked(capturedAction);
|
|
297
346
|
try {
|
|
298
347
|
const observation = await this.observeUnlocked({
|
|
299
|
-
...(
|
|
348
|
+
...(capturedAction.kind !== "close_tab" && actionResult.tabId
|
|
300
349
|
? { tabId: actionResult.tabId }
|
|
301
350
|
: {}),
|
|
302
351
|
});
|
|
@@ -330,15 +379,20 @@ export class AgentBrowser {
|
|
|
330
379
|
return this.closeTab(state);
|
|
331
380
|
let resolved = null;
|
|
332
381
|
if ("ref" in action && typeof action.ref === "string") {
|
|
333
|
-
resolved = await this.resolveRef(state, action.ref, action.snapshotId, action.kind !== "scroll");
|
|
382
|
+
resolved = await this.resolveRef(state, action.ref, action.snapshotId, action.kind !== "scroll", true);
|
|
334
383
|
}
|
|
335
|
-
|
|
336
|
-
await this.policy.assertAllowed(action.url)
|
|
384
|
+
const navigationDestination = action.kind === "navigate"
|
|
385
|
+
? await this.policy.assertAllowed(action.url)
|
|
386
|
+
: null;
|
|
387
|
+
this.assertOpen();
|
|
388
|
+
const denialSequence = this.requestPolicyDenialSequence;
|
|
337
389
|
try {
|
|
390
|
+
if (resolved)
|
|
391
|
+
this.assertResolvedRefCurrent(resolved);
|
|
338
392
|
switch (action.kind) {
|
|
339
393
|
case "navigate":
|
|
340
394
|
this.clearMainDocumentResponse(state);
|
|
341
|
-
this.queueMainDocumentResponse(state, await state.page.goto(
|
|
395
|
+
this.queueMainDocumentResponse(state, await state.page.goto(navigationDestination.href, {
|
|
342
396
|
waitUntil: "domcontentloaded",
|
|
343
397
|
timeout: this.options.navigationTimeoutMs,
|
|
344
398
|
}));
|
|
@@ -391,8 +445,14 @@ export class AgentBrowser {
|
|
|
391
445
|
default:
|
|
392
446
|
throw new BrowserError("invalid_action", "Unsupported browser action.");
|
|
393
447
|
}
|
|
448
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
449
|
+
if (denial)
|
|
450
|
+
throw denial;
|
|
394
451
|
}
|
|
395
452
|
catch (error) {
|
|
453
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
454
|
+
if (denial)
|
|
455
|
+
throw denial;
|
|
396
456
|
throw asBrowserError(error, "action_failed", "Browser action was attempted once and did not complete.");
|
|
397
457
|
}
|
|
398
458
|
finally {
|
|
@@ -407,9 +467,12 @@ export class AgentBrowser {
|
|
|
407
467
|
this.assertOpen();
|
|
408
468
|
validateExtractInput(input, this.options.limits.maxExtractChars);
|
|
409
469
|
const state = this.getState(input.tabId);
|
|
470
|
+
const refTargeted = "ref" in input && typeof input.ref === "string";
|
|
471
|
+
let resolved = null;
|
|
410
472
|
let locator;
|
|
411
|
-
if (
|
|
412
|
-
|
|
473
|
+
if (refTargeted) {
|
|
474
|
+
resolved = await this.resolveRef(state, input.ref, input.snapshotId, false, false);
|
|
475
|
+
locator = resolved.locator;
|
|
413
476
|
}
|
|
414
477
|
else {
|
|
415
478
|
locator = state.page.locator(input.selector ?? "body");
|
|
@@ -418,16 +481,25 @@ export class AgentBrowser {
|
|
|
418
481
|
const rawUrl = state.page.url();
|
|
419
482
|
try {
|
|
420
483
|
if (input.format === "links") {
|
|
421
|
-
const
|
|
422
|
-
|
|
484
|
+
const selfLinkLocator = refTargeted
|
|
485
|
+
? locator.locator("xpath=self::*[local-name()='a'][@href]")
|
|
486
|
+
: null;
|
|
487
|
+
const selfLinkCount = selfLinkLocator
|
|
488
|
+
? Math.min(await selfLinkLocator.count(), 1)
|
|
489
|
+
: 0;
|
|
490
|
+
const descendantLinkLocator = locator.locator("a[href]");
|
|
491
|
+
const descendantLinkCount = await descendantLinkLocator.count();
|
|
492
|
+
const count = selfLinkCount + descendantLinkCount;
|
|
423
493
|
const links = [];
|
|
424
494
|
let usedChars = 0;
|
|
425
495
|
let truncated = count > this.options.limits.maxExtractLinks;
|
|
426
496
|
const limit = Math.min(count, this.options.limits.maxExtractLinks);
|
|
427
497
|
for (let index = 0; index < limit; index += 1) {
|
|
428
|
-
const link =
|
|
498
|
+
const link = index < selfLinkCount
|
|
499
|
+
? selfLinkLocator.nth(index)
|
|
500
|
+
: descendantLinkLocator.nth(index - selfLinkCount);
|
|
429
501
|
const href = await link.getAttribute("href");
|
|
430
|
-
if (
|
|
502
|
+
if (href === null)
|
|
431
503
|
continue;
|
|
432
504
|
const text = redactUrlsInText((await link.textContent())?.trim() ?? "");
|
|
433
505
|
const resolvedHref = redactUrlForOutput(safeResolveUrl(href, rawUrl));
|
|
@@ -439,6 +511,9 @@ export class AgentBrowser {
|
|
|
439
511
|
usedChars += chars;
|
|
440
512
|
links.push({ text, href: resolvedHref });
|
|
441
513
|
}
|
|
514
|
+
if (resolved)
|
|
515
|
+
this.assertResolvedRefCurrent(resolved);
|
|
516
|
+
this.assertOpen();
|
|
442
517
|
return {
|
|
443
518
|
format: input.format,
|
|
444
519
|
sessionId: this.sessionId,
|
|
@@ -456,6 +531,9 @@ export class AgentBrowser {
|
|
|
456
531
|
? redactHtmlUrlAttributes(redactSensitiveInputValues(await locator.innerHTML()))
|
|
457
532
|
: await locator.innerText();
|
|
458
533
|
const bounded = boundText(redactUrlsInText(rawContent), maxChars);
|
|
534
|
+
if (resolved)
|
|
535
|
+
this.assertResolvedRefCurrent(resolved);
|
|
536
|
+
this.assertOpen();
|
|
459
537
|
return {
|
|
460
538
|
format: input.format,
|
|
461
539
|
sessionId: this.sessionId,
|
|
@@ -489,8 +567,11 @@ export class AgentBrowser {
|
|
|
489
567
|
const artifactPath = join(this.options.outputDir, `${timestamp}-${state.id}-${randomUUID()}.png`);
|
|
490
568
|
try {
|
|
491
569
|
await validateCanonicalStoragePaths(this.options.profile, this.options.outputDir);
|
|
570
|
+
this.assertOpen();
|
|
492
571
|
await ensurePrivateDirectory(this.options.outputDir, "artifact");
|
|
572
|
+
this.assertOpen();
|
|
493
573
|
await validateCanonicalStoragePaths(this.options.profile, this.options.outputDir);
|
|
574
|
+
this.assertOpen();
|
|
494
575
|
const bytes = await state.page.screenshot({
|
|
495
576
|
path: artifactPath,
|
|
496
577
|
fullPage,
|
|
@@ -498,6 +579,7 @@ export class AgentBrowser {
|
|
|
498
579
|
});
|
|
499
580
|
if (process.platform !== "win32")
|
|
500
581
|
await chmod(artifactPath, 0o600);
|
|
582
|
+
this.assertOpen();
|
|
501
583
|
const rawUrl = state.page.url();
|
|
502
584
|
return {
|
|
503
585
|
sessionId: this.sessionId,
|
|
@@ -533,29 +615,33 @@ export class AgentBrowser {
|
|
|
533
615
|
title: boundText(redactUrlsInText(await state.page.title()), 512).value,
|
|
534
616
|
active: state.id === this.activeTabId,
|
|
535
617
|
});
|
|
618
|
+
this.assertOpen();
|
|
536
619
|
}
|
|
537
620
|
return summaries;
|
|
538
621
|
}
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
async closeUnlocked() {
|
|
543
|
-
if (this.closed)
|
|
544
|
-
return;
|
|
622
|
+
close() {
|
|
623
|
+
if (this.closePromise)
|
|
624
|
+
return this.closePromise;
|
|
545
625
|
this.closed = true;
|
|
546
626
|
this.states.clear();
|
|
547
627
|
this.pageStates.clear();
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
await this.browser?.close();
|
|
553
|
-
}
|
|
628
|
+
this.requestPolicyDenials.clear();
|
|
629
|
+
const closing = Promise.resolve().then(() => closeRuntime(this.browser, this.context));
|
|
630
|
+
this.closePromise = closing;
|
|
631
|
+
return closing;
|
|
554
632
|
}
|
|
555
633
|
async installRequestPolicy() {
|
|
556
634
|
await this.context.route("**/*", async (route) => {
|
|
635
|
+
const request = route.request();
|
|
636
|
+
try {
|
|
637
|
+
await this.policy.assertAllowed(request.url());
|
|
638
|
+
}
|
|
639
|
+
catch (error) {
|
|
640
|
+
this.recordMainFrameRequestPolicyDenial(request, asBrowserError(error, "network_blocked", "Browser request was denied by the launch-time network policy."));
|
|
641
|
+
await route.abort("blockedbyclient");
|
|
642
|
+
return;
|
|
643
|
+
}
|
|
557
644
|
try {
|
|
558
|
-
await this.policy.assertAllowed(route.request().url());
|
|
559
645
|
await route.continue();
|
|
560
646
|
}
|
|
561
647
|
catch {
|
|
@@ -583,6 +669,7 @@ export class AgentBrowser {
|
|
|
583
669
|
if (state.page.isClosed()) {
|
|
584
670
|
this.states.delete(state.id);
|
|
585
671
|
this.pageStates.delete(state.page);
|
|
672
|
+
this.requestPolicyDenials.delete(state.id);
|
|
586
673
|
}
|
|
587
674
|
}
|
|
588
675
|
for (const page of this.context.pages()) {
|
|
@@ -602,24 +689,28 @@ export class AgentBrowser {
|
|
|
602
689
|
id: `tab_${this.nextTabNumber++}`,
|
|
603
690
|
page,
|
|
604
691
|
revision: 0,
|
|
605
|
-
|
|
606
|
-
|
|
692
|
+
navigationEpoch: 0,
|
|
693
|
+
snapshots: new Map(),
|
|
607
694
|
response: null,
|
|
608
695
|
responseDocumentUrl: null,
|
|
609
696
|
responseCapture: Promise.resolve(),
|
|
610
697
|
responseSequence: 0,
|
|
611
698
|
};
|
|
699
|
+
this.watchPageEvents(state);
|
|
612
700
|
this.pageStates.set(page, state);
|
|
613
701
|
this.states.set(state.id, state);
|
|
614
702
|
this.activeTabId = state.id;
|
|
615
|
-
this.watchMainDocumentResponses(state);
|
|
616
703
|
return state;
|
|
617
704
|
}
|
|
618
|
-
|
|
705
|
+
watchPageEvents(state) {
|
|
619
706
|
if (typeof state.page.on !== "function"
|
|
620
707
|
|| typeof state.page.mainFrame !== "function") {
|
|
621
|
-
|
|
708
|
+
throw new BrowserError("invalid_options", "Browser runtime must support page frame-navigation events and main-frame identity.");
|
|
622
709
|
}
|
|
710
|
+
state.page.on("framenavigated", () => {
|
|
711
|
+
state.navigationEpoch += 1;
|
|
712
|
+
this.invalidate(state);
|
|
713
|
+
});
|
|
623
714
|
state.page.on("response", (response) => {
|
|
624
715
|
try {
|
|
625
716
|
const request = response.request();
|
|
@@ -738,17 +829,34 @@ export class AgentBrowser {
|
|
|
738
829
|
}
|
|
739
830
|
invalidate(state) {
|
|
740
831
|
state.revision += 1;
|
|
741
|
-
state.
|
|
742
|
-
|
|
832
|
+
state.snapshots.clear();
|
|
833
|
+
}
|
|
834
|
+
rememberSnapshot(state, snapshotId, refs, navigationEpoch) {
|
|
835
|
+
state.snapshots.set(snapshotId, { navigationEpoch, refs });
|
|
836
|
+
while (state.snapshots.size > MAX_RETAINED_SNAPSHOTS_PER_TAB) {
|
|
837
|
+
const oldest = state.snapshots.keys().next().value;
|
|
838
|
+
if (oldest === undefined)
|
|
839
|
+
break;
|
|
840
|
+
state.snapshots.delete(oldest);
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
assertObservationCurrent(state, revision, navigationEpoch) {
|
|
844
|
+
this.assertOpen();
|
|
845
|
+
if (state.revision !== revision
|
|
846
|
+
|| state.navigationEpoch !== navigationEpoch) {
|
|
847
|
+
throw new BrowserError("action_failed", "The page navigated while it was being observed; observe the tab again.");
|
|
848
|
+
}
|
|
743
849
|
}
|
|
744
|
-
async resolveRef(state, ref, snapshotId, requireEnabled) {
|
|
850
|
+
async resolveRef(state, ref, snapshotId, requireEnabled, requireCurrentViewport) {
|
|
745
851
|
if (!snapshotId) {
|
|
746
852
|
throw new BrowserError("snapshot_required", "A snapshotId is required for every ref-targeted operation.");
|
|
747
853
|
}
|
|
748
|
-
|
|
854
|
+
const snapshot = state.snapshots.get(snapshotId);
|
|
855
|
+
if (!snapshot
|
|
856
|
+
|| snapshot.navigationEpoch !== state.navigationEpoch) {
|
|
749
857
|
throw new BrowserError("stale_snapshot", "The snapshot is stale; observe the tab again before acting.");
|
|
750
858
|
}
|
|
751
|
-
const nativeRef =
|
|
859
|
+
const nativeRef = snapshot.refs.get(ref);
|
|
752
860
|
if (!nativeRef) {
|
|
753
861
|
throw new BrowserError("ref_not_found", "The ref does not belong to this snapshot.");
|
|
754
862
|
}
|
|
@@ -763,16 +871,168 @@ export class AgentBrowser {
|
|
|
763
871
|
if (!(await locator.isVisible())) {
|
|
764
872
|
throw new BrowserError("ref_hidden", "The snapshot ref is no longer visible.");
|
|
765
873
|
}
|
|
874
|
+
if (requireCurrentViewport
|
|
875
|
+
&& !intersectsViewport(await locator.boundingBox(), state.page.viewportSize() ?? this.options.viewport)) {
|
|
876
|
+
throw new BrowserError("stale_snapshot", "The snapshot ref moved outside the current viewport; observe the tab again before acting.");
|
|
877
|
+
}
|
|
766
878
|
if (requireEnabled && !(await locator.isEnabled())) {
|
|
767
879
|
throw new BrowserError("ref_disabled", "The snapshot ref is disabled.");
|
|
768
880
|
}
|
|
769
|
-
|
|
881
|
+
const resolved = { state, locator, snapshotId, snapshot };
|
|
882
|
+
this.assertResolvedRefCurrent(resolved);
|
|
883
|
+
return resolved;
|
|
884
|
+
}
|
|
885
|
+
assertResolvedRefCurrent(resolved) {
|
|
886
|
+
this.assertOpen();
|
|
887
|
+
if (resolved.state.navigationEpoch !== resolved.snapshot.navigationEpoch
|
|
888
|
+
|| resolved.state.snapshots.get(resolved.snapshotId) !== resolved.snapshot) {
|
|
889
|
+
throw new BrowserError("stale_snapshot", "The snapshot is stale; observe the tab again before acting.");
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* Playwright's wheel dispatch and a page's own smooth scrolling can outlive
|
|
894
|
+
* the action promise. Sample top-level viewport geometry before issuing refs
|
|
895
|
+
* so the observation is less likely to describe an in-flight window scroll.
|
|
896
|
+
* This is deliberately best-effort: a probe failure must not turn an already
|
|
897
|
+
* executed action into a reported action failure.
|
|
898
|
+
*/
|
|
899
|
+
async awaitWindowViewportSettled(page) {
|
|
900
|
+
try {
|
|
901
|
+
const documentElement = page.locator("html");
|
|
902
|
+
const deadline = performance.now() + VIEWPORT_SETTLE_MAX_MS;
|
|
903
|
+
const remainingTimeout = () => {
|
|
904
|
+
const remaining = deadline - performance.now();
|
|
905
|
+
return remaining > 0 ? Math.max(1, Math.ceil(remaining)) : null;
|
|
906
|
+
};
|
|
907
|
+
const initialTimeout = remainingTimeout();
|
|
908
|
+
if (initialTimeout === null)
|
|
909
|
+
return;
|
|
910
|
+
let previous = await documentElement.boundingBox({
|
|
911
|
+
timeout: initialTimeout,
|
|
912
|
+
});
|
|
913
|
+
if (!previous)
|
|
914
|
+
return;
|
|
915
|
+
let stableIntervals = 0;
|
|
916
|
+
while (true) {
|
|
917
|
+
const remainingBeforeWait = deadline - performance.now();
|
|
918
|
+
if (remainingBeforeWait <= 0)
|
|
919
|
+
return;
|
|
920
|
+
await page.waitForTimeout(Math.min(VIEWPORT_SETTLE_INTERVAL_MS, remainingBeforeWait));
|
|
921
|
+
const probeTimeout = remainingTimeout();
|
|
922
|
+
if (probeTimeout === null)
|
|
923
|
+
return;
|
|
924
|
+
const current = await documentElement.boundingBox({
|
|
925
|
+
timeout: probeTimeout,
|
|
926
|
+
});
|
|
927
|
+
if (!current)
|
|
928
|
+
return;
|
|
929
|
+
if (Math.abs(current.x - previous.x) <= VIEWPORT_SETTLE_TOLERANCE_PX
|
|
930
|
+
&& Math.abs(current.y - previous.y) <= VIEWPORT_SETTLE_TOLERANCE_PX) {
|
|
931
|
+
stableIntervals += 1;
|
|
932
|
+
if (stableIntervals >= VIEWPORT_SETTLE_STABLE_INTERVALS)
|
|
933
|
+
return;
|
|
934
|
+
}
|
|
935
|
+
else {
|
|
936
|
+
stableIntervals = 0;
|
|
937
|
+
}
|
|
938
|
+
previous = current;
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
catch {
|
|
942
|
+
// Observation remains available when a custom runtime cannot expose
|
|
943
|
+
// stable document geometry or the page changes during this probe.
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
recordMainFrameRequestPolicyDenial(request, error) {
|
|
947
|
+
try {
|
|
948
|
+
if (!request.isNavigationRequest())
|
|
949
|
+
return;
|
|
950
|
+
}
|
|
951
|
+
catch {
|
|
952
|
+
return;
|
|
953
|
+
}
|
|
954
|
+
// A popup's initial navigation can be routed before Playwright has created
|
|
955
|
+
// a Frame object or exposed the Page through context.pages(). Such a
|
|
956
|
+
// request is still a navigation request. Keep only this genuinely
|
|
957
|
+
// unframed race session-ambiguous so a concurrent action can surface
|
|
958
|
+
// uncertainty without guessing which tab created it.
|
|
959
|
+
let frame;
|
|
960
|
+
try {
|
|
961
|
+
frame = request.frame();
|
|
962
|
+
}
|
|
963
|
+
catch {
|
|
964
|
+
this.recordRequestPolicyDenial(error, null);
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
// A denied subframe navigation is real policy enforcement, but it is not
|
|
968
|
+
// the result of the top-level action. Recording it against the tab would
|
|
969
|
+
// falsely turn a successful click into a failed action.
|
|
970
|
+
try {
|
|
971
|
+
if (frame.parentFrame() !== null)
|
|
972
|
+
return;
|
|
973
|
+
}
|
|
974
|
+
catch {
|
|
975
|
+
// A runtime that cannot classify a returned frame has crossed the same
|
|
976
|
+
// attribution boundary as an unregistered popup.
|
|
977
|
+
this.recordRequestPolicyDenial(error, null);
|
|
978
|
+
return;
|
|
979
|
+
}
|
|
980
|
+
try {
|
|
981
|
+
this.refreshPages();
|
|
982
|
+
}
|
|
983
|
+
catch {
|
|
984
|
+
// The policy decision must still be surfaced and the route aborted even
|
|
985
|
+
// if a custom runtime cannot enumerate/register the new page in time.
|
|
986
|
+
this.recordRequestPolicyDenial(error, null);
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
for (const state of this.states.values()) {
|
|
990
|
+
try {
|
|
991
|
+
if (state.page.mainFrame() === frame) {
|
|
992
|
+
this.recordRequestPolicyDenial(error, state.id);
|
|
993
|
+
return;
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
catch {
|
|
997
|
+
// A custom runtime may not expose a stable frame identity.
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
// A navigation request can race page registration (notably for popups).
|
|
1001
|
+
// Preserve the policy denial at session scope and report an uncertain
|
|
1002
|
+
// action outcome rather than inventing an attribution.
|
|
1003
|
+
this.recordRequestPolicyDenial(error, null);
|
|
1004
|
+
}
|
|
1005
|
+
recordRequestPolicyDenial(error, tabId) {
|
|
1006
|
+
this.requestPolicyDenialSequence += 1;
|
|
1007
|
+
this.requestPolicyDenials.set(tabId, {
|
|
1008
|
+
sequence: this.requestPolicyDenialSequence,
|
|
1009
|
+
error: tabId === null
|
|
1010
|
+
? new BrowserError("action_failed", "A navigation request was policy-blocked while an action was pending, but its tab could not be attributed; the current action outcome is uncertain.", { cause: error })
|
|
1011
|
+
: error,
|
|
1012
|
+
tabId,
|
|
1013
|
+
});
|
|
1014
|
+
}
|
|
1015
|
+
requestPolicyDenialAfter(sequence, tabId) {
|
|
1016
|
+
const scoped = this.requestPolicyDenials.get(tabId);
|
|
1017
|
+
const ambiguous = this.requestPolicyDenials.get(null);
|
|
1018
|
+
let latest = null;
|
|
1019
|
+
for (const denial of [scoped, ambiguous]) {
|
|
1020
|
+
if (denial
|
|
1021
|
+
&& denial.sequence > sequence
|
|
1022
|
+
&& (!latest || denial.sequence > latest.sequence)) {
|
|
1023
|
+
latest = denial;
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
return latest?.error ?? null;
|
|
770
1027
|
}
|
|
771
1028
|
async newTab(url) {
|
|
772
1029
|
const destination = url ? await this.policy.assertAllowed(url) : null;
|
|
1030
|
+
this.assertOpen();
|
|
773
1031
|
const page = await this.context.newPage();
|
|
1032
|
+
this.assertOpen();
|
|
774
1033
|
const state = this.registerPage(page);
|
|
775
1034
|
if (destination) {
|
|
1035
|
+
const denialSequence = this.requestPolicyDenialSequence;
|
|
776
1036
|
try {
|
|
777
1037
|
this.clearMainDocumentResponse(state);
|
|
778
1038
|
const response = await page.goto(destination.href, {
|
|
@@ -780,9 +1040,15 @@ export class AgentBrowser {
|
|
|
780
1040
|
timeout: this.options.navigationTimeoutMs,
|
|
781
1041
|
});
|
|
782
1042
|
this.queueMainDocumentResponse(state, response);
|
|
1043
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
1044
|
+
if (denial)
|
|
1045
|
+
throw denial;
|
|
783
1046
|
}
|
|
784
1047
|
catch (error) {
|
|
785
1048
|
this.invalidate(state);
|
|
1049
|
+
const denial = this.requestPolicyDenialAfter(denialSequence, state.id);
|
|
1050
|
+
if (denial)
|
|
1051
|
+
throw denial;
|
|
786
1052
|
throw asBrowserError(error, "action_failed", "New-tab navigation was attempted once and did not complete.");
|
|
787
1053
|
}
|
|
788
1054
|
this.invalidate(state);
|
|
@@ -801,6 +1067,7 @@ export class AgentBrowser {
|
|
|
801
1067
|
}
|
|
802
1068
|
this.states.delete(state.id);
|
|
803
1069
|
this.pageStates.delete(state.page);
|
|
1070
|
+
this.requestPolicyDenials.delete(state.id);
|
|
804
1071
|
this.refreshPages();
|
|
805
1072
|
return {
|
|
806
1073
|
ok: true,
|
|
@@ -838,6 +1105,9 @@ export class AgentBrowser {
|
|
|
838
1105
|
}
|
|
839
1106
|
}
|
|
840
1107
|
withLock(operation) {
|
|
1108
|
+
if (this.closed) {
|
|
1109
|
+
return Promise.reject(new BrowserError("browser_closed", "Browser session is closed."));
|
|
1110
|
+
}
|
|
841
1111
|
const previous = this.operationTail;
|
|
842
1112
|
let release;
|
|
843
1113
|
this.operationTail = new Promise((resolveTurn) => {
|
|
@@ -940,10 +1210,29 @@ function authorityContainsUserinfo(reference) {
|
|
|
940
1210
|
const authority = authorityEnd < 0 ? remainder : remainder.slice(0, authorityEnd);
|
|
941
1211
|
return authority.includes("@");
|
|
942
1212
|
}
|
|
1213
|
+
function captureBrowserAction(action) {
|
|
1214
|
+
const captured = {
|
|
1215
|
+
...action,
|
|
1216
|
+
};
|
|
1217
|
+
if (Array.isArray(captured.values)) {
|
|
1218
|
+
captured.values = Object.freeze([...captured.values]);
|
|
1219
|
+
}
|
|
1220
|
+
return Object.freeze(captured);
|
|
1221
|
+
}
|
|
943
1222
|
async function loadDefaultRuntime() {
|
|
944
1223
|
const playwright = await import("playwright-core");
|
|
945
1224
|
return playwright.chromium;
|
|
946
1225
|
}
|
|
1226
|
+
async function closeRuntime(browser, context) {
|
|
1227
|
+
// An ephemeral session owns the Browser process, and Browser.close() closes
|
|
1228
|
+
// its contexts while also terminating a child that a stuck Context.close()
|
|
1229
|
+
// could otherwise orphan. A persistent launch returns only its context.
|
|
1230
|
+
if (browser) {
|
|
1231
|
+
await browser.close();
|
|
1232
|
+
return;
|
|
1233
|
+
}
|
|
1234
|
+
await context?.close();
|
|
1235
|
+
}
|
|
947
1236
|
function normalizeOptions(options) {
|
|
948
1237
|
for (const [name, value] of [["headless", options.headless]]) {
|
|
949
1238
|
if (value !== undefined && typeof value !== "boolean") {
|