@parcha/pi-browser 0.1.0-beta.35 → 0.1.0-beta.37
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +20 -25
- package/dist/core/driving-guard.d.ts +1 -1
- package/dist/core/driving-guard.js +1 -1
- package/dist/core/host.d.ts +1 -12
- package/dist/core/images.d.ts +32 -4
- package/dist/core/images.js +81 -16
- package/dist/core/page-scan.d.ts +69 -0
- package/dist/core/page-scan.js +467 -10
- package/dist/core/public-fetch.js +3 -0
- package/dist/core/read-page.js +20 -4
- package/dist/core/tools.js +4 -1
- package/dist/core/vault.d.ts +0 -10
- package/dist/core/vault.js +0 -13
- package/dist/driver/cdp.js +3 -3
- package/dist/driver/page-driver.js +7 -3
- package/dist/durable/store.d.ts +1 -1
- package/dist/durable.d.ts +6 -1
- package/dist/durable.js +24 -5
- package/dist/index.d.ts +1 -0
- package/dist/index.js +3 -0
- package/dist/live-view/index.d.ts +2 -0
- package/dist/live-view/index.js +4 -0
- package/dist/live-view/server.d.ts +11 -0
- package/dist/live-view/server.js +13 -2
- package/dist/providers/cdp.js +2 -2
- package/dist/providers/profiles.d.ts +1 -15
- package/dist/providers/profiles.js +1 -64
- package/dist/providers/shared.d.ts +1 -47
- package/dist/providers/shared.js +1 -82
- package/dist/providers/worker-extension.js +1 -1
- package/dist/testing/backend.d.ts +4 -3
- package/dist/testing/backend.js +6 -5
- package/dist/vault-session.d.ts +4 -7
- package/dist/vault-session.js +26 -81
- package/dist/vendor/browser-use-pi/page.d.ts +3 -12
- package/dist/vendor/browser-use-pi/page.js +6 -19
- package/package.json +7 -20
- package/dist/providers/browserbase-net.d.ts +0 -12
- package/dist/providers/browserbase-net.js +0 -95
- package/dist/providers/browserbase.d.ts +0 -138
- package/dist/providers/browserbase.js +0 -294
- package/dist/providers/kernel.d.ts +0 -58
- package/dist/providers/kernel.js +0 -291
- /package/dist/{providers → core}/cdp-call.d.ts +0 -0
- /package/dist/{providers → core}/cdp-call.js +0 -0
package/dist/core/page-scan.js
CHANGED
|
@@ -26,6 +26,7 @@ export class InertDrop {
|
|
|
26
26
|
#next;
|
|
27
27
|
#unclosed = new Map();
|
|
28
28
|
#kept = [];
|
|
29
|
+
#styles = [];
|
|
29
30
|
/** Where the scan is, and where the HTML kept since the last block dropped starts. */
|
|
30
31
|
#at = 0;
|
|
31
32
|
#from = 0;
|
|
@@ -84,8 +85,12 @@ export class InertDrop {
|
|
|
84
85
|
end = close >= 0 ? close : html.length;
|
|
85
86
|
}
|
|
86
87
|
else {
|
|
87
|
-
const close = this.#
|
|
88
|
-
|
|
88
|
+
const close = this.#endTag(name, end);
|
|
89
|
+
// A style element of the page itself: never one inside a comment, a script, a template, a noscript or an svg,
|
|
90
|
+
// since each of those goes whole, its insides unread.
|
|
91
|
+
if (name === "style")
|
|
92
|
+
this.#styles.push({ inside: insideOf(html, name, tag.lt, tag.gt), css: html.slice(end, close ? close.start : html.length) });
|
|
93
|
+
end = close ? close.end : name === "svg" ? end : html.length;
|
|
89
94
|
}
|
|
90
95
|
}
|
|
91
96
|
this.#kept.push(html.slice(this.#from, tag.lt));
|
|
@@ -166,6 +171,8 @@ export class InertDrop {
|
|
|
166
171
|
}
|
|
167
172
|
/** The page as dropped so far (the whole of it once step has answered true). */
|
|
168
173
|
result() { return this.#kept.length === 1 ? this.#kept[0] : this.#kept.join(""); }
|
|
174
|
+
/** The page's own style elements dropped so far, in page order: what each start tag holds (insideOf) and its text. */
|
|
175
|
+
styles() { return this.#styles; }
|
|
169
176
|
}
|
|
170
177
|
/** `html` without its inert blocks, all at once (see InertDrop). */
|
|
171
178
|
export function dropInert(html) {
|
|
@@ -173,6 +180,447 @@ export function dropInert(html) {
|
|
|
173
180
|
drop.step(Number.POSITIVE_INFINITY);
|
|
174
181
|
return drop.result();
|
|
175
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* The classes common frameworks hide an element from everyone with, whatever the page's own style says (it is often an
|
|
185
|
+
* external stylesheet a page read never fetches): mfp-hide (Magnific Popup: a pop-up's content until it opens, as on
|
|
186
|
+
* Canada.ca's record pages), modal (Bootstrap: a modal until it opens), d-none (Bootstrap 4+), hidden (Tailwind, Bootstrap
|
|
187
|
+
* 3), hide (Bootstrap 2), invisible (Tailwind, Bootstrap: visibility), is-hidden (Bulma), u-hidden (SUIT utilities). The
|
|
188
|
+
* classes that hide text from the screen only (sr-only, visually-hidden, wb-inv, screen-reader-text) are not here: a
|
|
189
|
+
* screen reader reads them, and they are often a record's field labels and table captions, the words its values hang
|
|
190
|
+
* on. aria-hidden is read for the same reason the other way: it hides from a screen reader only, and is usually shown.
|
|
191
|
+
*/
|
|
192
|
+
export const HIDING_CLASSES = new Set(["mfp-hide", "modal", "d-none", "hidden", "hide", "invisible", "is-hidden", "u-hidden"]);
|
|
193
|
+
/** The most of a page's style text read for what it hides (pageHidingOf). */
|
|
194
|
+
const STYLE_READ_CHARS = 1_000_000;
|
|
195
|
+
/** A selector of one class alone (`.x`), and every class a selector names. */
|
|
196
|
+
const ONE_CLASS = /^\s*\.(-?[_a-zA-Z][_a-zA-Z0-9-]*)\s*$/;
|
|
197
|
+
const CLASSES_IN = /\.(-?[_a-zA-Z][_a-zA-Z0-9-]*)/g;
|
|
198
|
+
/** Elements that never hold content: hidden, only the tag itself goes. */
|
|
199
|
+
const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
|
|
200
|
+
/** The media a style element counts for here: none named, or the screen among them. */
|
|
201
|
+
const SCREEN_MEDIA = new Set(["", "all", "screen", "only screen", "only all"]);
|
|
202
|
+
const DEFERRING = new Set(["inherit", "unset", "revert", "revert-layer"]);
|
|
203
|
+
/** What `value` of `property` comes to. */
|
|
204
|
+
function outcomeOf(property, value) {
|
|
205
|
+
if (!value || DEFERRING.has(value))
|
|
206
|
+
return "doubt";
|
|
207
|
+
if (property === "display")
|
|
208
|
+
return value === "none" ? "hide" : "show";
|
|
209
|
+
return value === "hidden" || value === "collapse" ? "hide" : value === "visible" ? "show" : "doubt";
|
|
210
|
+
}
|
|
211
|
+
/** `next` declared after `prior` of one property, as the cascade takes it among declarations of equal weight: the later
|
|
212
|
+
* wins, but an earlier !important one that a later plain one contradicts makes it doubt (that mix is not modelled). */
|
|
213
|
+
function after(prior, next) {
|
|
214
|
+
if (!prior)
|
|
215
|
+
return next;
|
|
216
|
+
if (prior.outcome === "doubt" || next.outcome === "doubt")
|
|
217
|
+
return { outcome: "doubt", important: prior.important || next.important };
|
|
218
|
+
if (prior.important && !next.important && prior.outcome !== next.outcome)
|
|
219
|
+
return { outcome: "doubt", important: true };
|
|
220
|
+
return next;
|
|
221
|
+
}
|
|
222
|
+
/** `text` split at the semicolons that end a declaration: one inside a quoted string (either quote, backslash escapes), a
|
|
223
|
+
* parenthesis (url(...)) or a comment does not end it, and a comment's text is dropped. */
|
|
224
|
+
function splitDeclarations(text) {
|
|
225
|
+
const parts = [];
|
|
226
|
+
let current = "";
|
|
227
|
+
let quote = null;
|
|
228
|
+
let depth = 0;
|
|
229
|
+
for (let i = 0; i < text.length; i++) {
|
|
230
|
+
const ch = text[i];
|
|
231
|
+
if (quote) {
|
|
232
|
+
current += ch;
|
|
233
|
+
if (ch === "\\" && i + 1 < text.length)
|
|
234
|
+
current += text[++i];
|
|
235
|
+
else if (ch === quote)
|
|
236
|
+
quote = null;
|
|
237
|
+
}
|
|
238
|
+
else if (ch === "/" && text[i + 1] === "*") {
|
|
239
|
+
const end = text.indexOf("*/", i + 2);
|
|
240
|
+
i = end < 0 ? text.length : end + 1;
|
|
241
|
+
current += " "; // a comment is a boundary: display:n/**/one is two words, not none
|
|
242
|
+
}
|
|
243
|
+
else if (ch === '"' || ch === "'") {
|
|
244
|
+
quote = ch;
|
|
245
|
+
current += ch;
|
|
246
|
+
}
|
|
247
|
+
else if (ch === "\\" && i + 1 < text.length) {
|
|
248
|
+
current += ch + text[++i];
|
|
249
|
+
}
|
|
250
|
+
else if (ch === "(") {
|
|
251
|
+
depth++;
|
|
252
|
+
current += ch;
|
|
253
|
+
}
|
|
254
|
+
else if (ch === ")") {
|
|
255
|
+
if (depth > 0)
|
|
256
|
+
depth--;
|
|
257
|
+
current += ch;
|
|
258
|
+
}
|
|
259
|
+
else if (ch === ";" && depth === 0) {
|
|
260
|
+
parts.push(current);
|
|
261
|
+
current = "";
|
|
262
|
+
}
|
|
263
|
+
else
|
|
264
|
+
current += ch;
|
|
265
|
+
}
|
|
266
|
+
parts.push(current);
|
|
267
|
+
return parts;
|
|
268
|
+
}
|
|
269
|
+
/** What a declaration list (a style attribute's value, a rule's block) declares of display and visibility, each
|
|
270
|
+
* property's declarations taken in order (after). One pass over `text`. */
|
|
271
|
+
function declarationsOf(text) {
|
|
272
|
+
const declared = {};
|
|
273
|
+
for (const part of splitDeclarations(text)) {
|
|
274
|
+
const colon = part.indexOf(":");
|
|
275
|
+
if (colon < 0)
|
|
276
|
+
continue;
|
|
277
|
+
const property = part.slice(0, colon).trim().toLowerCase();
|
|
278
|
+
if (property !== "display" && property !== "visibility")
|
|
279
|
+
continue;
|
|
280
|
+
let value = part.slice(colon + 1).trim().toLowerCase();
|
|
281
|
+
const important = /!\s*important$/.test(value);
|
|
282
|
+
if (important)
|
|
283
|
+
value = value.replace(/!\s*important$/, "").trim();
|
|
284
|
+
declared[property] = after(declared[property], { outcome: outcomeOf(property, value), important });
|
|
285
|
+
}
|
|
286
|
+
return declared;
|
|
287
|
+
}
|
|
288
|
+
/** Whether a style element whose start tag holds `inside` applies to the screen: no media named, or one of them all or
|
|
289
|
+
* screen (a media query, or a value written with references, does not count: in doubt its rules hide nothing). */
|
|
290
|
+
function forScreen(inside) {
|
|
291
|
+
let media = null;
|
|
292
|
+
eachAttribute(inside, (name, value) => {
|
|
293
|
+
if (name.toLowerCase() !== "media")
|
|
294
|
+
return false;
|
|
295
|
+
media = value;
|
|
296
|
+
return true;
|
|
297
|
+
});
|
|
298
|
+
if (media === null)
|
|
299
|
+
return true;
|
|
300
|
+
return media.split(",").some((one) => SCREEN_MEDIA.has(one.trim().toLowerCase().replace(/\s+/g, " ")));
|
|
301
|
+
}
|
|
302
|
+
/**
|
|
303
|
+
* What `styles` (a page's own live style elements, InertDrop's styles) declare of each class's display and visibility:
|
|
304
|
+
* a rule at the top level whose selector is that one class (`.x` in `.x, .y { … }`) declares for it, a later rule's
|
|
305
|
+
* declaration after an earlier one's (after); a class a selector names otherwise (`.open .x`, `.x:hover`, `div.x`) is in
|
|
306
|
+
* doubt for each property that rule sets, since when it applies is not known here. A rule inside an at-rule (`@media`)
|
|
307
|
+
* is passed, and a style element not for the screen (forScreen) counts for nothing. Its style text is read once, at
|
|
308
|
+
* most STYLE_READ_CHARS of it, each character a bounded number of times.
|
|
309
|
+
*/
|
|
310
|
+
export function pageHidingOf(styles) {
|
|
311
|
+
const hiding = new Map();
|
|
312
|
+
const declare = (name, declared) => {
|
|
313
|
+
const known = hiding.get(name) ?? {};
|
|
314
|
+
for (const property of ["display", "visibility"]) {
|
|
315
|
+
const next = declared[property];
|
|
316
|
+
if (next)
|
|
317
|
+
known[property] = after(known[property], next);
|
|
318
|
+
}
|
|
319
|
+
hiding.set(name, known);
|
|
320
|
+
};
|
|
321
|
+
let read = 0;
|
|
322
|
+
for (const { inside, css } of styles) {
|
|
323
|
+
if (read >= STYLE_READ_CHARS)
|
|
324
|
+
break;
|
|
325
|
+
const text = css.slice(0, STYLE_READ_CHARS - read);
|
|
326
|
+
read += text.length;
|
|
327
|
+
if (!forScreen(inside))
|
|
328
|
+
continue;
|
|
329
|
+
rulesOf(text, (selectors, block) => {
|
|
330
|
+
const declared = declarationsOf(block);
|
|
331
|
+
if (!declared.display && !declared.visibility)
|
|
332
|
+
return;
|
|
333
|
+
for (const selector of selectors.split(",")) {
|
|
334
|
+
const one = ONE_CLASS.exec(selector);
|
|
335
|
+
if (one) {
|
|
336
|
+
declare(one[1], declared);
|
|
337
|
+
continue;
|
|
338
|
+
}
|
|
339
|
+
const doubt = {};
|
|
340
|
+
for (const property of ["display", "visibility"])
|
|
341
|
+
if (declared[property])
|
|
342
|
+
doubt[property] = { outcome: "doubt", important: false };
|
|
343
|
+
for (const named of selector.matchAll(CLASSES_IN))
|
|
344
|
+
declare(named[1], doubt);
|
|
345
|
+
}
|
|
346
|
+
});
|
|
347
|
+
}
|
|
348
|
+
return hiding;
|
|
349
|
+
}
|
|
350
|
+
/** Each rule at the top level of `css` (its comments dropped): its selector text and its declaration block; an at-rule
|
|
351
|
+
* (a statement to its ";", or its block whole) is passed. One forward pass: each character is read once. */
|
|
352
|
+
function rulesOf(css, rule) {
|
|
353
|
+
const text = css.includes("/*") ? css.replace(/\/\*[\s\S]*?(?:\*\/|$)/g, " ") : css;
|
|
354
|
+
const { length } = text;
|
|
355
|
+
for (let at = 0; at < length;) {
|
|
356
|
+
const code = text.charCodeAt(at);
|
|
357
|
+
// Between rules: spaces, and a stray ";" or "}".
|
|
358
|
+
if (isSpace(code) || code === 59 || code === 125) {
|
|
359
|
+
at += 1;
|
|
360
|
+
continue;
|
|
361
|
+
}
|
|
362
|
+
const start = at;
|
|
363
|
+
let stop = at;
|
|
364
|
+
while (stop < length && text.charCodeAt(stop) !== 123 && text.charCodeAt(stop) !== 59)
|
|
365
|
+
stop += 1;
|
|
366
|
+
if (stop >= length)
|
|
367
|
+
break;
|
|
368
|
+
// A ";" before any "{": an at-rule statement ("@import …;"), or text that is no rule: passed.
|
|
369
|
+
if (text.charCodeAt(stop) === 59) {
|
|
370
|
+
at = stop + 1;
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
let depth = 1;
|
|
374
|
+
let end = stop + 1;
|
|
375
|
+
for (; end < length && depth > 0; end += 1) {
|
|
376
|
+
const c = text.charCodeAt(end);
|
|
377
|
+
if (c === 123)
|
|
378
|
+
depth += 1;
|
|
379
|
+
else if (c === 125)
|
|
380
|
+
depth -= 1;
|
|
381
|
+
}
|
|
382
|
+
if (code !== 64)
|
|
383
|
+
rule(text.slice(start, stop), text.slice(stop + 1, depth === 0 ? end - 1 : end));
|
|
384
|
+
at = end;
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/** One property's outcome among an element's classes: theirs where every class that declares it agrees and none is in
|
|
388
|
+
* doubt; doubt where they disagree (which rule comes last across classes is not modelled), !important if any is. */
|
|
389
|
+
function amongClasses(declared) {
|
|
390
|
+
if (declared.length === 0)
|
|
391
|
+
return undefined;
|
|
392
|
+
const important = declared.some((one) => one.important);
|
|
393
|
+
const outcomes = new Set(declared.map((one) => one.outcome));
|
|
394
|
+
return { outcome: outcomes.size === 1 ? declared[0].outcome : "doubt", important };
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Whether a start tag named `name`, holding `inside` (insideOf), hides its element from everyone. Each property is
|
|
398
|
+
* decided on its own: an inline declaration over its classes' (unless a class's !important one it contradicts, then
|
|
399
|
+
* doubt), the classes' over the browser's own (the hidden attribute and a dialog not open, display none). Its classes
|
|
400
|
+
* declare by the page's one-class rules (`page`) and HIDING_CLASSES. `why`: "display" when display comes to "hide",
|
|
401
|
+
* else "visibility" when visibility does, else null; doubt hides nothing. `shows`: the element sets visibility itself
|
|
402
|
+
* to anything but hidden (or in doubt), so a visibility-hidden ancestor's hiding may not reach it. A class or style
|
|
403
|
+
* value's character references are decoded first (`decode`).
|
|
404
|
+
*/
|
|
405
|
+
function hides(name, inside, page, decode) {
|
|
406
|
+
let hiddenAttribute = false;
|
|
407
|
+
let open = false;
|
|
408
|
+
let style = "";
|
|
409
|
+
let classes = "";
|
|
410
|
+
eachAttribute(inside, (named, value) => {
|
|
411
|
+
const attribute = named.toLowerCase();
|
|
412
|
+
if (attribute === "hidden")
|
|
413
|
+
hiddenAttribute = true;
|
|
414
|
+
else if (attribute === "open")
|
|
415
|
+
open = true;
|
|
416
|
+
else if (attribute === "style" && !style)
|
|
417
|
+
style = value.includes("&") ? decode(value) : value;
|
|
418
|
+
else if (attribute === "class" && !classes)
|
|
419
|
+
classes = value.includes("&") ? decode(value) : value;
|
|
420
|
+
return false;
|
|
421
|
+
});
|
|
422
|
+
const inline = style ? declarationsOf(style) : {};
|
|
423
|
+
const byClass = { display: [], visibility: [] };
|
|
424
|
+
for (const one of classes.split(/\s+/)) {
|
|
425
|
+
if (!one)
|
|
426
|
+
continue;
|
|
427
|
+
if (HIDING_CLASSES.has(one))
|
|
428
|
+
byClass[one === "invisible" ? "visibility" : "display"].push({ outcome: "hide", important: false });
|
|
429
|
+
const declared = page.get(one);
|
|
430
|
+
if (declared?.display)
|
|
431
|
+
byClass.display.push(declared.display);
|
|
432
|
+
if (declared?.visibility)
|
|
433
|
+
byClass.visibility.push(declared.visibility);
|
|
434
|
+
}
|
|
435
|
+
const browser = hiddenAttribute || (name === "dialog" && !open);
|
|
436
|
+
const decided = (property) => {
|
|
437
|
+
const own = inline[property];
|
|
438
|
+
const fromClasses = amongClasses(byClass[property]);
|
|
439
|
+
if (own)
|
|
440
|
+
return fromClasses?.important && !own.important && fromClasses.outcome !== own.outcome ? "doubt" : own.outcome;
|
|
441
|
+
if (fromClasses)
|
|
442
|
+
return fromClasses.outcome;
|
|
443
|
+
return property === "display" && browser ? "hide" : undefined;
|
|
444
|
+
};
|
|
445
|
+
const visibility = decided("visibility");
|
|
446
|
+
return { why: decided("display") === "hide" ? "display" : visibility === "hide" ? "visibility" : null, shows: visibility === "show" || visibility === "doubt" };
|
|
447
|
+
}
|
|
448
|
+
/**
|
|
449
|
+
* A page without the elements it hides from everyone (hides), in steps so a caller on the event loop can yield between
|
|
450
|
+
* them (dropHidden, all at once). Run on a page its inert blocks are dropped from (InertDrop), with what its own live
|
|
451
|
+
* styles declare (pageHidingOf). A hidden element goes from its start tag to the end tag that closes it, a same-name
|
|
452
|
+
* element inside it counted; one that no end tag closes hides nothing, since where it ends is not known without a full
|
|
453
|
+
* tree. A hidden element that holds no content (VOID) goes alone. Visibility is inherited and a descendant can set it
|
|
454
|
+
* back: an element hidden by visibility alone goes only if nothing inside it shows itself again (hides' `shows`), and is
|
|
455
|
+
* kept whole otherwise. Only a descendant that is shown counts: one display-hidden itself, or inside a display-hidden
|
|
456
|
+
* element within the visibility-hidden one, shows nothing. display:none is not undone by a descendant, so its element
|
|
457
|
+
* always goes. Linear: one pass pairs each start tag with its end tag (an open-tag stack per name), marking the
|
|
458
|
+
* innermost open visibility-hidden element when a shown start tag inside it shows itself, and handing the mark to the
|
|
459
|
+
* element around it (its `outer`) as each closes; each tag costs a bounded amount of work, however the page nests or
|
|
460
|
+
* fails to (an element closed out of order is kept, and its stack entry left to be popped when it reaches the top).
|
|
461
|
+
* Then the hidden ranges are cut, the outermost of each nest.
|
|
462
|
+
*/
|
|
463
|
+
export class HiddenDrop {
|
|
464
|
+
html;
|
|
465
|
+
page;
|
|
466
|
+
decode;
|
|
467
|
+
#next;
|
|
468
|
+
#open = new Map();
|
|
469
|
+
/** The visibility-hidden elements open now, outermost first; one closed out of order stays until it reaches the top. */
|
|
470
|
+
#invisible = [];
|
|
471
|
+
/** How many display-hidden elements are open now. */
|
|
472
|
+
#displayOpen = 0;
|
|
473
|
+
#ranges = [];
|
|
474
|
+
#at = 0;
|
|
475
|
+
#done = false;
|
|
476
|
+
constructor(html, page, decode) {
|
|
477
|
+
this.html = html;
|
|
478
|
+
this.page = page;
|
|
479
|
+
this.decode = decode;
|
|
480
|
+
this.#next = tagsOf(html);
|
|
481
|
+
}
|
|
482
|
+
/** Pairs the tags found in the next `chars` characters or so; true once the whole page is read. */
|
|
483
|
+
step(chars) {
|
|
484
|
+
const { html } = this;
|
|
485
|
+
const until = this.#at + chars;
|
|
486
|
+
while (!this.#done && this.#at < until) {
|
|
487
|
+
const tag = this.#next(this.#at);
|
|
488
|
+
if (!tag || tag.gt < 0) {
|
|
489
|
+
this.#done = true;
|
|
490
|
+
break;
|
|
491
|
+
}
|
|
492
|
+
this.#at = tag.gt + 1;
|
|
493
|
+
const { name } = tag;
|
|
494
|
+
if (!name)
|
|
495
|
+
continue;
|
|
496
|
+
if (tag.end) {
|
|
497
|
+
const open = this.#open.get(name)?.pop();
|
|
498
|
+
if (open) {
|
|
499
|
+
// An end tag closes what its element left open: a p or li opened after it ends here (HTML's optional end tags).
|
|
500
|
+
for (const implied of IMPLIED_END) {
|
|
501
|
+
const stack = this.#open.get(implied);
|
|
502
|
+
while (stack && stack.length > 0 && stack[stack.length - 1].lt > open.lt)
|
|
503
|
+
this.#close(stack.pop(), tag.lt);
|
|
504
|
+
}
|
|
505
|
+
this.#close(open, tag.gt + 1);
|
|
506
|
+
}
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
// A start tag that closes an open p or li (optional end tags): that element ends here, so its hiding stops here and
|
|
510
|
+
// never reaches text a browser shows.
|
|
511
|
+
if (P_CLOSERS.has(name))
|
|
512
|
+
this.#closeImplied("p", tag.lt);
|
|
513
|
+
if (name === "li") {
|
|
514
|
+
const last = this.#open.get("li")?.at(-1);
|
|
515
|
+
// Only an li not inside a list opened after it.
|
|
516
|
+
if (last && !LISTS.some((list) => (this.#open.get(list)?.at(-1)?.lt ?? -1) > last.lt))
|
|
517
|
+
this.#closeImplied("li", tag.lt);
|
|
518
|
+
}
|
|
519
|
+
const { why, shows } = hides(name, insideOf(html, name, tag.lt, tag.gt), this.page, this.decode);
|
|
520
|
+
// It shows itself inside a visibility-hidden element, and is shown (not display-hidden itself, and no display-hidden
|
|
521
|
+
// element open between them): that element is kept (and, as it closes, those around it).
|
|
522
|
+
const inner = this.#invisible.at(-1);
|
|
523
|
+
if (shows && why !== "display" && inner && this.#displayOpen === inner.displayOpen)
|
|
524
|
+
inner.shown = true;
|
|
525
|
+
const hidden = why !== null;
|
|
526
|
+
if (VOID.has(name)) {
|
|
527
|
+
if (hidden)
|
|
528
|
+
this.#ranges.push([tag.lt, tag.gt + 1]);
|
|
529
|
+
continue;
|
|
530
|
+
}
|
|
531
|
+
// Text, not markup, to its end tag (an iframe's fallback, an xmp): its "<" are not tags.
|
|
532
|
+
if (TEXT_ONLY.has(name)) {
|
|
533
|
+
const closer = CLOSER[name];
|
|
534
|
+
closer.lastIndex = tag.gt + 1;
|
|
535
|
+
const close = closer.exec(html);
|
|
536
|
+
if (hidden && close)
|
|
537
|
+
this.#ranges.push([tag.lt, close.index + close[0].length]);
|
|
538
|
+
if (!close) {
|
|
539
|
+
this.#done = true;
|
|
540
|
+
break;
|
|
541
|
+
}
|
|
542
|
+
this.#at = close.index + close[0].length;
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
let open = this.#open.get(name);
|
|
546
|
+
if (!open)
|
|
547
|
+
this.#open.set(name, (open = []));
|
|
548
|
+
const opened = { lt: tag.lt, why, shown: false, closed: false, outer: inner, displayOpen: this.#displayOpen };
|
|
549
|
+
open.push(opened);
|
|
550
|
+
if (why === "visibility")
|
|
551
|
+
this.#invisible.push(opened);
|
|
552
|
+
if (why === "display")
|
|
553
|
+
this.#displayOpen += 1;
|
|
554
|
+
}
|
|
555
|
+
return this.#done;
|
|
556
|
+
}
|
|
557
|
+
/** The innermost open `name` element, ended by a start tag at `at` that implies its end tag. */
|
|
558
|
+
#closeImplied(name, at) {
|
|
559
|
+
const open = this.#open.get(name)?.pop();
|
|
560
|
+
if (open)
|
|
561
|
+
this.#close(open, at);
|
|
562
|
+
}
|
|
563
|
+
/** `opened` closed by the end tag that ends at `end`: a display-hidden element goes; a visibility-hidden one goes
|
|
564
|
+
* unless something shown inside it showed itself again, and is then kept with its mark handed to the one around it
|
|
565
|
+
* (`outer`). One closed out of nesting order (by a stray end tag) is kept, since where its hiding reaches is in doubt,
|
|
566
|
+
* and its stack entry is left, marked closed, to be popped when it reaches the top: no search, no shift. */
|
|
567
|
+
#close(opened, end) {
|
|
568
|
+
if (opened.why === "display") {
|
|
569
|
+
this.#displayOpen = Math.max(0, this.#displayOpen - 1);
|
|
570
|
+
this.#ranges.push([opened.lt, end]);
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
if (opened.why !== "visibility")
|
|
574
|
+
return;
|
|
575
|
+
const invisible = this.#invisible;
|
|
576
|
+
opened.closed = true;
|
|
577
|
+
if (invisible.at(-1) === opened) {
|
|
578
|
+
invisible.pop();
|
|
579
|
+
while (invisible.at(-1)?.closed)
|
|
580
|
+
invisible.pop();
|
|
581
|
+
}
|
|
582
|
+
else
|
|
583
|
+
opened.shown = true;
|
|
584
|
+
if (!opened.shown) {
|
|
585
|
+
this.#ranges.push([opened.lt, end]);
|
|
586
|
+
return;
|
|
587
|
+
}
|
|
588
|
+
// Kept, it shows in the one around it only if no display-hidden element was open between them.
|
|
589
|
+
if (opened.outer && opened.displayOpen === opened.outer.displayOpen)
|
|
590
|
+
opened.outer.shown = true;
|
|
591
|
+
}
|
|
592
|
+
/** The page without its hidden ranges (once step has answered true; the page as it is before that). */
|
|
593
|
+
result() {
|
|
594
|
+
if (!this.#done || this.#ranges.length === 0)
|
|
595
|
+
return this.html;
|
|
596
|
+
const ranges = this.#ranges.sort((a, b) => a[0] - b[0]);
|
|
597
|
+
const kept = [];
|
|
598
|
+
let from = 0;
|
|
599
|
+
for (const [start, end] of ranges) {
|
|
600
|
+
if (end <= from)
|
|
601
|
+
continue;
|
|
602
|
+
if (start > from)
|
|
603
|
+
kept.push(this.html.slice(from, start));
|
|
604
|
+
from = Math.max(from, end);
|
|
605
|
+
}
|
|
606
|
+
kept.push(this.html.slice(from));
|
|
607
|
+
return kept.join("");
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
/** Elements whose end tag HTML lets a page omit; an end tag of an element around one ends it too. */
|
|
611
|
+
const IMPLIED_END = ["p", "li"];
|
|
612
|
+
const LISTS = ["ul", "ol", "menu"];
|
|
613
|
+
/** Start tags that close an open p. */
|
|
614
|
+
const P_CLOSERS = new Set([
|
|
615
|
+
"address", "article", "aside", "blockquote", "details", "div", "dl", "fieldset", "figcaption", "figure", "footer", "form",
|
|
616
|
+
"h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "ul",
|
|
617
|
+
]);
|
|
618
|
+
/** `html` without the elements it hides from everyone, all at once (see HiddenDrop); `page` and `decode` as there. */
|
|
619
|
+
export function dropHidden(html, page, decode) {
|
|
620
|
+
const drop = new HiddenDrop(html, page, decode);
|
|
621
|
+
drop.step(Number.POSITIVE_INFINITY);
|
|
622
|
+
return drop.result();
|
|
623
|
+
}
|
|
176
624
|
/** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
|
|
177
625
|
* `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
|
|
178
626
|
* falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
|
|
@@ -275,6 +723,18 @@ const TITLE_END = /<\/title(?=[\s/>])/gi;
|
|
|
275
723
|
* tagsOf ends a tag by: a quote right after the "=" and its spaces; one that no quote closes runs to the tag's end.
|
|
276
724
|
*/
|
|
277
725
|
function attrOf(inside, name, decode) {
|
|
726
|
+
let found = null;
|
|
727
|
+
eachAttribute(inside, (named, value) => {
|
|
728
|
+
if (named.length !== name.length || named.toLowerCase() !== name)
|
|
729
|
+
return false;
|
|
730
|
+
found = value.includes("&") ? decode(value) : value;
|
|
731
|
+
return true;
|
|
732
|
+
});
|
|
733
|
+
return found;
|
|
734
|
+
}
|
|
735
|
+
/** Each attribute of a tag's `inside` (insideOf), in order, read as attrOf says: `visit` gets its name as written and its
|
|
736
|
+
* raw value ("" for one with no value), and stops the walk by answering true. */
|
|
737
|
+
function eachAttribute(inside, visit) {
|
|
278
738
|
const { length } = inside;
|
|
279
739
|
for (let at = 0; at < length;) {
|
|
280
740
|
const code = inside.charCodeAt(at);
|
|
@@ -286,12 +746,12 @@ function attrOf(inside, name, decode) {
|
|
|
286
746
|
do
|
|
287
747
|
at += 1;
|
|
288
748
|
while (at < length && !endsName(inside.charCodeAt(at)));
|
|
289
|
-
const
|
|
749
|
+
const name = inside.slice(from, at);
|
|
290
750
|
while (isSpace(inside.charCodeAt(at)))
|
|
291
751
|
at += 1;
|
|
292
752
|
if (inside.charCodeAt(at) !== EQUALS) {
|
|
293
|
-
if (
|
|
294
|
-
return
|
|
753
|
+
if (visit(name, ""))
|
|
754
|
+
return;
|
|
295
755
|
continue;
|
|
296
756
|
}
|
|
297
757
|
at += 1;
|
|
@@ -311,12 +771,9 @@ function attrOf(inside, name, decode) {
|
|
|
311
771
|
at += 1;
|
|
312
772
|
end = at;
|
|
313
773
|
}
|
|
314
|
-
if (
|
|
315
|
-
|
|
316
|
-
return value.includes("&") ? decode(value) : value;
|
|
317
|
-
}
|
|
774
|
+
if (visit(name, inside.slice(start, end)))
|
|
775
|
+
return;
|
|
318
776
|
}
|
|
319
|
-
return null;
|
|
320
777
|
}
|
|
321
778
|
/** What a tag at `lt` named `name` holds between its name and its ">" at `gt`. */
|
|
322
779
|
const insideOf = (html, name, lt, gt) => html.slice(lt + 1 + name.length, gt);
|
|
@@ -41,6 +41,9 @@ export function isPrivateAddress(ip) {
|
|
|
41
41
|
return true;
|
|
42
42
|
if (groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff)
|
|
43
43
|
return isPrivateAddress([groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."));
|
|
44
|
+
// 6to4 (2002::/16, RFC 3056) carries an IPv4 address in its next 32 bits, which a relay delivers to: judged as that one.
|
|
45
|
+
if (groups[0] === 0x2002)
|
|
46
|
+
return isPrivateAddress([groups[1] >> 8, groups[1] & 0xff, groups[2] >> 8, groups[2] & 0xff].join("."));
|
|
44
47
|
return V6_PRIVATE.some(([base, bits]) => inV6(groups, v6Groups(base), bits));
|
|
45
48
|
}
|
|
46
49
|
/** Answers `address` for any name: the address a request was checked against is the one it connects to. */
|
package/dist/core/read-page.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// A fetched HTML page read as its title, its text and its links, without ever holding the event loop. The host's tools
|
|
2
2
|
// run in the run's own process, so a read that holds the loop holds the run's heartbeat with it, and a run whose
|
|
3
3
|
// heartbeat lapses is fenced: go.dev/dl, 3.15 MB of tables, held it past 240 s in one markdown conversion. So one linear
|
|
4
|
-
// pass drops the blocks that hold no readable text
|
|
4
|
+
// pass drops the blocks that hold no readable text, and another the elements the page hides from everyone (page-scan.ts:
|
|
5
|
+
// a help pop-up until its link is clicked is not the page's text); what is left is cut at READ_MAX_CHARS (the page is
|
|
5
6
|
// then `truncated`, and `html_chars` is its whole length); and a worker thread converts that to markdown, in pieces
|
|
6
7
|
// small enough that the converter stays linear (page-markdown.ts), within a deadline. A worker that has not answered by
|
|
7
8
|
// then, or that fails, is stopped, and the text is the page's plain text, read on the loop in steps that yield it
|
|
@@ -10,7 +11,7 @@
|
|
|
10
11
|
import { setImmediate as yieldToLoop } from "node:timers/promises";
|
|
11
12
|
import { Worker } from "node:worker_threads";
|
|
12
13
|
import { decodeEntities } from "./entities.js";
|
|
13
|
-
import { cutHtml, InertDrop, PageScan } from "./page-scan.js";
|
|
14
|
+
import { cutHtml, HiddenDrop, InertDrop, pageHidingOf, PageScan } from "./page-scan.js";
|
|
14
15
|
/** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
|
|
15
16
|
* comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
|
|
16
17
|
* 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
|
|
@@ -28,7 +29,11 @@ const STEP_CHARS = 32_768;
|
|
|
28
29
|
*/
|
|
29
30
|
export async function readPage(html, url, { deadlineMs = READ_DEADLINE_MS, signal } = {}) {
|
|
30
31
|
signal?.throwIfAborted();
|
|
31
|
-
|
|
32
|
+
// What the page hides from everyone who reads it (a help pop-up until its link is clicked, a closed dialog) is not
|
|
33
|
+
// its text: what its own live style elements declare is read from those the inert drop took out.
|
|
34
|
+
const inert = new InertDrop(html);
|
|
35
|
+
await stepsOnLoop(inert, signal);
|
|
36
|
+
const kept = await dropHiddenOnLoop(inert.result(), pageHidingOf(inert.styles()), signal);
|
|
32
37
|
const read = cutHtml(kept, READ_MAX_CHARS);
|
|
33
38
|
const cut = read.length < kept.length ? { truncated: true, html_chars: html.length } : {};
|
|
34
39
|
await yieldToLoop();
|
|
@@ -78,10 +83,21 @@ function inWorker(job, deadlineMs, signal) {
|
|
|
78
83
|
/** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
|
|
79
84
|
export async function dropOnLoop(html, signal) {
|
|
80
85
|
const drop = new InertDrop(html);
|
|
81
|
-
|
|
86
|
+
await stepsOnLoop(drop, signal);
|
|
87
|
+
return drop.result();
|
|
88
|
+
}
|
|
89
|
+
/** Runs `pass` STEP_CHARS at a time, the loop yielded between steps, to its end. */
|
|
90
|
+
async function stepsOnLoop(pass, signal) {
|
|
91
|
+
while (!pass.step(STEP_CHARS)) {
|
|
82
92
|
await yieldToLoop();
|
|
83
93
|
signal?.throwIfAborted();
|
|
84
94
|
}
|
|
95
|
+
}
|
|
96
|
+
/** `html` without the elements it hides from everyone (HiddenDrop), with `page` what its own live styles declare
|
|
97
|
+
* (pageHidingOf), STEP_CHARS at a time with the loop yielded between steps. */
|
|
98
|
+
async function dropHiddenOnLoop(html, page, signal) {
|
|
99
|
+
const drop = new HiddenDrop(html, page, decodeEntities);
|
|
100
|
+
await stepsOnLoop(drop, signal);
|
|
85
101
|
return drop.result();
|
|
86
102
|
}
|
|
87
103
|
/** `html`'s plain text, title and links, scanned STEP_CHARS at a time with the loop yielded between steps. */
|
package/dist/core/tools.js
CHANGED
|
@@ -421,7 +421,10 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT,
|
|
|
421
421
|
const url = pageUrl(call, await s.driver.url());
|
|
422
422
|
const receipt = await file(s, call, { tool: "screenshot", args: region ? { ...requested, clip: region.clip, ...region.of } : requested, status: "ok", facts: { final_url: url }, body: Buffer.from(shot.data, "base64"), ext: shot.size.type === "png" ? "png" : "jpg" });
|
|
423
423
|
const first = shot.adjusted?.first;
|
|
424
|
-
const
|
|
424
|
+
const over = shot.adjusted?.why === "long_edge" ? "was over the 2,000 px long edge budget" : shot.adjusted?.why === "base64_bytes" ? "was over the size budget" : "could not be measured";
|
|
425
|
+
const taken = shot.options.clip ? "the clip" : shot.options.fullPage ? "the full page" : "the viewport";
|
|
426
|
+
const scaled = shot.options.scale !== undefined && shot.options.scale < 1 ? ", scaled down" : "";
|
|
427
|
+
const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} ${over}; this is ${taken} at ${shot.size.width}x${shot.size.height}${scaled}`;
|
|
425
428
|
const lines = [news, `url: ${url}`, `evidence: ${receipt.path ?? "(not filed)"}`, adjusted].filter(Boolean).join("\n");
|
|
426
429
|
const media = { kind: "screenshot", path: receipt.bodyPath, mime: shot.mimeType, bytes: Buffer.from(shot.data, "base64").byteLength, width: shot.size.width, height: shot.size.height, ...(region ? { clip: region.clip, ...region.of } : {}), ...(ask ? { ask } : {}) };
|
|
427
430
|
return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256, media } };
|
package/dist/core/vault.d.ts
CHANGED
|
@@ -19,17 +19,7 @@ export type CapturedState = {
|
|
|
19
19
|
/** Only cookies the allowed origins would be sent (see `cookieKept`). Partitioned cookies are not kept. */
|
|
20
20
|
cookies: VaultCookie[];
|
|
21
21
|
localStorage: Record<string, Record<string, string>>;
|
|
22
|
-
}
|
|
23
|
-
/** `origins`: the origins the context was captured for. A context holds a whole browser profile, so it is restored only for a request that covers EVERY one of them (`coversEveryOrigin`); one that records none is refused. */
|
|
24
|
-
| {
|
|
25
|
-
kind: "provider-context";
|
|
26
|
-
capturedAt: string;
|
|
27
|
-
provider: "browserbase" | "kernel";
|
|
28
|
-
contextId: string;
|
|
29
|
-
origins?: string[];
|
|
30
22
|
};
|
|
31
|
-
/** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
|
|
32
|
-
export declare function coversEveryOrigin(requested: readonly string[], saved: readonly string[] | undefined): boolean;
|
|
33
23
|
export interface SessionVaultPort {
|
|
34
24
|
/** At release and at the end of a handoff, and, for a context a new session made, right after the create (so a restart still finds it). The host supplied the origins; nothing outside them was captured.
|
|
35
25
|
* A capture has a deadline (20 s) after which nothing more is read or written, but a write already handed to this port cannot be recalled: ignore one whose `capturedAt` is older than the one you hold. */
|
package/dist/core/vault.js
CHANGED
|
@@ -11,19 +11,6 @@ export function clampSessionCookieAgeMs(requested) {
|
|
|
11
11
|
throw new RangeError("the session-cookie age cap is a positive number of milliseconds");
|
|
12
12
|
return Math.min(requested, MAX_SESSION_COOKIE_AGE_MS);
|
|
13
13
|
}
|
|
14
|
-
/** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
|
|
15
|
-
export function coversEveryOrigin(requested, saved) {
|
|
16
|
-
if (!saved || saved.length === 0)
|
|
17
|
-
return false;
|
|
18
|
-
const origin = (value) => { try {
|
|
19
|
-
return new URL(value).origin;
|
|
20
|
-
}
|
|
21
|
-
catch {
|
|
22
|
-
return null;
|
|
23
|
-
} };
|
|
24
|
-
const have = new Set(requested.map(origin));
|
|
25
|
-
return saved.every((value) => { const o = origin(value); return o !== null && have.has(o); });
|
|
26
|
-
}
|
|
27
14
|
const SESSION_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
|
|
28
15
|
/** The host's allowed origins, each `scheme://host[:port]` of http or https, normalised and de-duplicated; anything else is refused. */
|
|
29
16
|
export function vaultOrigins(origins) {
|
package/dist/driver/cdp.js
CHANGED
|
@@ -224,9 +224,9 @@ export function cdpDriver() {
|
|
|
224
224
|
const selector = place?.selector ?? null;
|
|
225
225
|
return page.evaluate(`(${LOCATE_JS}).call(document, ${JSON.stringify(selector)})`);
|
|
226
226
|
},
|
|
227
|
-
screenshot: async (
|
|
228
|
-
const data = await page.screenshot(
|
|
229
|
-
return { data, mimeType: type === "jpeg" ? "image/jpeg" : "image/png" };
|
|
227
|
+
screenshot: async (options) => {
|
|
228
|
+
const data = await page.screenshot(options);
|
|
229
|
+
return { data, mimeType: options.type === "jpeg" ? "image/jpeg" : "image/png" };
|
|
230
230
|
},
|
|
231
231
|
// One evaluate reads all four, so they are one document's.
|
|
232
232
|
page: () => page.evaluate(() => ({
|