@parcha/pi-browser 0.1.0-beta.35 → 0.1.0-beta.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/README.md +20 -25
  3. package/dist/core/driving-guard.d.ts +1 -1
  4. package/dist/core/driving-guard.js +1 -1
  5. package/dist/core/host.d.ts +1 -12
  6. package/dist/core/images.d.ts +32 -4
  7. package/dist/core/images.js +81 -16
  8. package/dist/core/page-scan.d.ts +69 -0
  9. package/dist/core/page-scan.js +467 -10
  10. package/dist/core/public-fetch.js +3 -0
  11. package/dist/core/read-page.js +20 -4
  12. package/dist/core/tools.js +4 -1
  13. package/dist/core/vault.d.ts +0 -10
  14. package/dist/core/vault.js +0 -13
  15. package/dist/driver/cdp.js +3 -3
  16. package/dist/driver/page-driver.js +7 -3
  17. package/dist/durable/store.d.ts +1 -1
  18. package/dist/durable.d.ts +6 -1
  19. package/dist/durable.js +24 -5
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.js +3 -0
  22. package/dist/live-view/index.d.ts +2 -0
  23. package/dist/live-view/index.js +4 -0
  24. package/dist/live-view/server.d.ts +11 -0
  25. package/dist/live-view/server.js +13 -2
  26. package/dist/providers/cdp.js +2 -2
  27. package/dist/providers/profiles.d.ts +1 -15
  28. package/dist/providers/profiles.js +1 -64
  29. package/dist/providers/shared.d.ts +1 -47
  30. package/dist/providers/shared.js +1 -82
  31. package/dist/providers/worker-extension.js +1 -1
  32. package/dist/testing/backend.d.ts +4 -3
  33. package/dist/testing/backend.js +6 -5
  34. package/dist/vault-session.d.ts +4 -7
  35. package/dist/vault-session.js +26 -81
  36. package/dist/vendor/browser-use-pi/page.d.ts +3 -12
  37. package/dist/vendor/browser-use-pi/page.js +6 -19
  38. package/package.json +7 -20
  39. package/dist/providers/browserbase-net.d.ts +0 -12
  40. package/dist/providers/browserbase-net.js +0 -95
  41. package/dist/providers/browserbase.d.ts +0 -138
  42. package/dist/providers/browserbase.js +0 -294
  43. package/dist/providers/kernel.d.ts +0 -58
  44. package/dist/providers/kernel.js +0 -291
  45. /package/dist/{providers → core}/cdp-call.d.ts +0 -0
  46. /package/dist/{providers → core}/cdp-call.js +0 -0
@@ -26,6 +26,7 @@ export class InertDrop {
26
26
  #next;
27
27
  #unclosed = new Map();
28
28
  #kept = [];
29
+ #styles = [];
29
30
  /** Where the scan is, and where the HTML kept since the last block dropped starts. */
30
31
  #at = 0;
31
32
  #from = 0;
@@ -84,8 +85,12 @@ export class InertDrop {
84
85
  end = close >= 0 ? close : html.length;
85
86
  }
86
87
  else {
87
- const close = this.#endTagEnd(name, end);
88
- end = close >= 0 ? close : name === "svg" ? end : html.length;
88
+ const close = this.#endTag(name, end);
89
+ // A style element of the page itself: never one inside a comment, a script, a template, a noscript or an svg,
90
+ // since each of those goes whole, its insides unread.
91
+ if (name === "style")
92
+ this.#styles.push({ inside: insideOf(html, name, tag.lt, tag.gt), css: html.slice(end, close ? close.start : html.length) });
93
+ end = close ? close.end : name === "svg" ? end : html.length;
89
94
  }
90
95
  }
91
96
  this.#kept.push(html.slice(this.#from, tag.lt));
@@ -166,6 +171,8 @@ export class InertDrop {
166
171
  }
167
172
  /** The page as dropped so far (the whole of it once step has answered true). */
168
173
  result() { return this.#kept.length === 1 ? this.#kept[0] : this.#kept.join(""); }
174
+ /** The page's own style elements dropped so far, in page order: what each start tag holds (insideOf) and its text. */
175
+ styles() { return this.#styles; }
169
176
  }
170
177
  /** `html` without its inert blocks, all at once (see InertDrop). */
171
178
  export function dropInert(html) {
@@ -173,6 +180,447 @@ export function dropInert(html) {
173
180
  drop.step(Number.POSITIVE_INFINITY);
174
181
  return drop.result();
175
182
  }
183
+ /**
184
+ * The classes common frameworks hide an element from everyone with, whatever the page's own style says (it is often an
185
+ * external stylesheet a page read never fetches): mfp-hide (Magnific Popup: a pop-up's content until it opens, as on
186
+ * Canada.ca's record pages), modal (Bootstrap: a modal until it opens), d-none (Bootstrap 4+), hidden (Tailwind, Bootstrap
187
+ * 3), hide (Bootstrap 2), invisible (Tailwind, Bootstrap: visibility), is-hidden (Bulma), u-hidden (SUIT utilities). The
188
+ * classes that hide text from the screen only (sr-only, visually-hidden, wb-inv, screen-reader-text) are not here: a
189
+ * screen reader reads them, and they are often a record's field labels and table captions, the words its values hang
190
+ * on. aria-hidden is read for the same reason the other way: it hides from a screen reader only, and is usually shown.
191
+ */
192
+ export const HIDING_CLASSES = new Set(["mfp-hide", "modal", "d-none", "hidden", "hide", "invisible", "is-hidden", "u-hidden"]);
193
+ /** The most of a page's style text read for what it hides (pageHidingOf). */
194
+ const STYLE_READ_CHARS = 1_000_000;
195
+ /** A selector of one class alone (`.x`), and every class a selector names. */
196
+ const ONE_CLASS = /^\s*\.(-?[_a-zA-Z][_a-zA-Z0-9-]*)\s*$/;
197
+ const CLASSES_IN = /\.(-?[_a-zA-Z][_a-zA-Z0-9-]*)/g;
198
+ /** Elements that never hold content: hidden, only the tag itself goes. */
199
+ const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);
200
+ /** The media a style element counts for here: none named, or the screen among them. */
201
+ const SCREEN_MEDIA = new Set(["", "all", "screen", "only screen", "only all"]);
202
+ const DEFERRING = new Set(["inherit", "unset", "revert", "revert-layer"]);
203
+ /** What `value` of `property` comes to. */
204
+ function outcomeOf(property, value) {
205
+ if (!value || DEFERRING.has(value))
206
+ return "doubt";
207
+ if (property === "display")
208
+ return value === "none" ? "hide" : "show";
209
+ return value === "hidden" || value === "collapse" ? "hide" : value === "visible" ? "show" : "doubt";
210
+ }
211
+ /** `next` declared after `prior` of one property, as the cascade takes it among declarations of equal weight: the later
212
+ * wins, but an earlier !important one that a later plain one contradicts makes it doubt (that mix is not modelled). */
213
+ function after(prior, next) {
214
+ if (!prior)
215
+ return next;
216
+ if (prior.outcome === "doubt" || next.outcome === "doubt")
217
+ return { outcome: "doubt", important: prior.important || next.important };
218
+ if (prior.important && !next.important && prior.outcome !== next.outcome)
219
+ return { outcome: "doubt", important: true };
220
+ return next;
221
+ }
222
+ /** `text` split at the semicolons that end a declaration: one inside a quoted string (either quote, backslash escapes), a
223
+ * parenthesis (url(...)) or a comment does not end it, and a comment's text is dropped. */
224
+ function splitDeclarations(text) {
225
+ const parts = [];
226
+ let current = "";
227
+ let quote = null;
228
+ let depth = 0;
229
+ for (let i = 0; i < text.length; i++) {
230
+ const ch = text[i];
231
+ if (quote) {
232
+ current += ch;
233
+ if (ch === "\\" && i + 1 < text.length)
234
+ current += text[++i];
235
+ else if (ch === quote)
236
+ quote = null;
237
+ }
238
+ else if (ch === "/" && text[i + 1] === "*") {
239
+ const end = text.indexOf("*/", i + 2);
240
+ i = end < 0 ? text.length : end + 1;
241
+ current += " "; // a comment is a boundary: display:n/**/one is two words, not none
242
+ }
243
+ else if (ch === '"' || ch === "'") {
244
+ quote = ch;
245
+ current += ch;
246
+ }
247
+ else if (ch === "\\" && i + 1 < text.length) {
248
+ current += ch + text[++i];
249
+ }
250
+ else if (ch === "(") {
251
+ depth++;
252
+ current += ch;
253
+ }
254
+ else if (ch === ")") {
255
+ if (depth > 0)
256
+ depth--;
257
+ current += ch;
258
+ }
259
+ else if (ch === ";" && depth === 0) {
260
+ parts.push(current);
261
+ current = "";
262
+ }
263
+ else
264
+ current += ch;
265
+ }
266
+ parts.push(current);
267
+ return parts;
268
+ }
269
+ /** What a declaration list (a style attribute's value, a rule's block) declares of display and visibility, each
270
+ * property's declarations taken in order (after). One pass over `text`. */
271
+ function declarationsOf(text) {
272
+ const declared = {};
273
+ for (const part of splitDeclarations(text)) {
274
+ const colon = part.indexOf(":");
275
+ if (colon < 0)
276
+ continue;
277
+ const property = part.slice(0, colon).trim().toLowerCase();
278
+ if (property !== "display" && property !== "visibility")
279
+ continue;
280
+ let value = part.slice(colon + 1).trim().toLowerCase();
281
+ const important = /!\s*important$/.test(value);
282
+ if (important)
283
+ value = value.replace(/!\s*important$/, "").trim();
284
+ declared[property] = after(declared[property], { outcome: outcomeOf(property, value), important });
285
+ }
286
+ return declared;
287
+ }
288
+ /** Whether a style element whose start tag holds `inside` applies to the screen: no media named, or one of them all or
289
+ * screen (a media query, or a value written with references, does not count: in doubt its rules hide nothing). */
290
+ function forScreen(inside) {
291
+ let media = null;
292
+ eachAttribute(inside, (name, value) => {
293
+ if (name.toLowerCase() !== "media")
294
+ return false;
295
+ media = value;
296
+ return true;
297
+ });
298
+ if (media === null)
299
+ return true;
300
+ return media.split(",").some((one) => SCREEN_MEDIA.has(one.trim().toLowerCase().replace(/\s+/g, " ")));
301
+ }
302
+ /**
303
+ * What `styles` (a page's own live style elements, InertDrop's styles) declare of each class's display and visibility:
304
+ * a rule at the top level whose selector is that one class (`.x` in `.x, .y { … }`) declares for it, a later rule's
305
+ * declaration after an earlier one's (after); a class a selector names otherwise (`.open .x`, `.x:hover`, `div.x`) is in
306
+ * doubt for each property that rule sets, since when it applies is not known here. A rule inside an at-rule (`@media`)
307
+ * is passed, and a style element not for the screen (forScreen) counts for nothing. Its style text is read once, at
308
+ * most STYLE_READ_CHARS of it, each character a bounded number of times.
309
+ */
310
+ export function pageHidingOf(styles) {
311
+ const hiding = new Map();
312
+ const declare = (name, declared) => {
313
+ const known = hiding.get(name) ?? {};
314
+ for (const property of ["display", "visibility"]) {
315
+ const next = declared[property];
316
+ if (next)
317
+ known[property] = after(known[property], next);
318
+ }
319
+ hiding.set(name, known);
320
+ };
321
+ let read = 0;
322
+ for (const { inside, css } of styles) {
323
+ if (read >= STYLE_READ_CHARS)
324
+ break;
325
+ const text = css.slice(0, STYLE_READ_CHARS - read);
326
+ read += text.length;
327
+ if (!forScreen(inside))
328
+ continue;
329
+ rulesOf(text, (selectors, block) => {
330
+ const declared = declarationsOf(block);
331
+ if (!declared.display && !declared.visibility)
332
+ return;
333
+ for (const selector of selectors.split(",")) {
334
+ const one = ONE_CLASS.exec(selector);
335
+ if (one) {
336
+ declare(one[1], declared);
337
+ continue;
338
+ }
339
+ const doubt = {};
340
+ for (const property of ["display", "visibility"])
341
+ if (declared[property])
342
+ doubt[property] = { outcome: "doubt", important: false };
343
+ for (const named of selector.matchAll(CLASSES_IN))
344
+ declare(named[1], doubt);
345
+ }
346
+ });
347
+ }
348
+ return hiding;
349
+ }
350
+ /** Each rule at the top level of `css` (its comments dropped): its selector text and its declaration block; an at-rule
351
+ * (a statement to its ";", or its block whole) is passed. One forward pass: each character is read once. */
352
+ function rulesOf(css, rule) {
353
+ const text = css.includes("/*") ? css.replace(/\/\*[\s\S]*?(?:\*\/|$)/g, " ") : css;
354
+ const { length } = text;
355
+ for (let at = 0; at < length;) {
356
+ const code = text.charCodeAt(at);
357
+ // Between rules: spaces, and a stray ";" or "}".
358
+ if (isSpace(code) || code === 59 || code === 125) {
359
+ at += 1;
360
+ continue;
361
+ }
362
+ const start = at;
363
+ let stop = at;
364
+ while (stop < length && text.charCodeAt(stop) !== 123 && text.charCodeAt(stop) !== 59)
365
+ stop += 1;
366
+ if (stop >= length)
367
+ break;
368
+ // A ";" before any "{": an at-rule statement ("@import …;"), or text that is no rule: passed.
369
+ if (text.charCodeAt(stop) === 59) {
370
+ at = stop + 1;
371
+ continue;
372
+ }
373
+ let depth = 1;
374
+ let end = stop + 1;
375
+ for (; end < length && depth > 0; end += 1) {
376
+ const c = text.charCodeAt(end);
377
+ if (c === 123)
378
+ depth += 1;
379
+ else if (c === 125)
380
+ depth -= 1;
381
+ }
382
+ if (code !== 64)
383
+ rule(text.slice(start, stop), text.slice(stop + 1, depth === 0 ? end - 1 : end));
384
+ at = end;
385
+ }
386
+ }
387
+ /** One property's outcome among an element's classes: theirs where every class that declares it agrees and none is in
388
+ * doubt; doubt where they disagree (which rule comes last across classes is not modelled), !important if any is. */
389
+ function amongClasses(declared) {
390
+ if (declared.length === 0)
391
+ return undefined;
392
+ const important = declared.some((one) => one.important);
393
+ const outcomes = new Set(declared.map((one) => one.outcome));
394
+ return { outcome: outcomes.size === 1 ? declared[0].outcome : "doubt", important };
395
+ }
396
+ /**
397
+ * Whether a start tag named `name`, holding `inside` (insideOf), hides its element from everyone. Each property is
398
+ * decided on its own: an inline declaration over its classes' (unless a class's !important one it contradicts, then
399
+ * doubt), the classes' over the browser's own (the hidden attribute and a dialog not open, display none). Its classes
400
+ * declare by the page's one-class rules (`page`) and HIDING_CLASSES. `why`: "display" when display comes to "hide",
401
+ * else "visibility" when visibility does, else null; doubt hides nothing. `shows`: the element sets visibility itself
402
+ * to anything but hidden (or in doubt), so a visibility-hidden ancestor's hiding may not reach it. A class or style
403
+ * value's character references are decoded first (`decode`).
404
+ */
405
+ function hides(name, inside, page, decode) {
406
+ let hiddenAttribute = false;
407
+ let open = false;
408
+ let style = "";
409
+ let classes = "";
410
+ eachAttribute(inside, (named, value) => {
411
+ const attribute = named.toLowerCase();
412
+ if (attribute === "hidden")
413
+ hiddenAttribute = true;
414
+ else if (attribute === "open")
415
+ open = true;
416
+ else if (attribute === "style" && !style)
417
+ style = value.includes("&") ? decode(value) : value;
418
+ else if (attribute === "class" && !classes)
419
+ classes = value.includes("&") ? decode(value) : value;
420
+ return false;
421
+ });
422
+ const inline = style ? declarationsOf(style) : {};
423
+ const byClass = { display: [], visibility: [] };
424
+ for (const one of classes.split(/\s+/)) {
425
+ if (!one)
426
+ continue;
427
+ if (HIDING_CLASSES.has(one))
428
+ byClass[one === "invisible" ? "visibility" : "display"].push({ outcome: "hide", important: false });
429
+ const declared = page.get(one);
430
+ if (declared?.display)
431
+ byClass.display.push(declared.display);
432
+ if (declared?.visibility)
433
+ byClass.visibility.push(declared.visibility);
434
+ }
435
+ const browser = hiddenAttribute || (name === "dialog" && !open);
436
+ const decided = (property) => {
437
+ const own = inline[property];
438
+ const fromClasses = amongClasses(byClass[property]);
439
+ if (own)
440
+ return fromClasses?.important && !own.important && fromClasses.outcome !== own.outcome ? "doubt" : own.outcome;
441
+ if (fromClasses)
442
+ return fromClasses.outcome;
443
+ return property === "display" && browser ? "hide" : undefined;
444
+ };
445
+ const visibility = decided("visibility");
446
+ return { why: decided("display") === "hide" ? "display" : visibility === "hide" ? "visibility" : null, shows: visibility === "show" || visibility === "doubt" };
447
+ }
448
+ /**
449
+ * A page without the elements it hides from everyone (hides), in steps so a caller on the event loop can yield between
450
+ * them (dropHidden, all at once). Run on a page its inert blocks are dropped from (InertDrop), with what its own live
451
+ * styles declare (pageHidingOf). A hidden element goes from its start tag to the end tag that closes it, a same-name
452
+ * element inside it counted; one that no end tag closes hides nothing, since where it ends is not known without a full
453
+ * tree. A hidden element that holds no content (VOID) goes alone. Visibility is inherited and a descendant can set it
454
+ * back: an element hidden by visibility alone goes only if nothing inside it shows itself again (hides' `shows`), and is
455
+ * kept whole otherwise. Only a descendant that is shown counts: one display-hidden itself, or inside a display-hidden
456
+ * element within the visibility-hidden one, shows nothing. display:none is not undone by a descendant, so its element
457
+ * always goes. Linear: one pass pairs each start tag with its end tag (an open-tag stack per name), marking the
458
+ * innermost open visibility-hidden element when a shown start tag inside it shows itself, and handing the mark to the
459
+ * element around it (its `outer`) as each closes; each tag costs a bounded amount of work, however the page nests or
460
+ * fails to (an element closed out of order is kept, and its stack entry left to be popped when it reaches the top).
461
+ * Then the hidden ranges are cut, the outermost of each nest.
462
+ */
463
+ export class HiddenDrop {
464
+ html;
465
+ page;
466
+ decode;
467
+ #next;
468
+ #open = new Map();
469
+ /** The visibility-hidden elements open now, outermost first; one closed out of order stays until it reaches the top. */
470
+ #invisible = [];
471
+ /** How many display-hidden elements are open now. */
472
+ #displayOpen = 0;
473
+ #ranges = [];
474
+ #at = 0;
475
+ #done = false;
476
+ constructor(html, page, decode) {
477
+ this.html = html;
478
+ this.page = page;
479
+ this.decode = decode;
480
+ this.#next = tagsOf(html);
481
+ }
482
+ /** Pairs the tags found in the next `chars` characters or so; true once the whole page is read. */
483
+ step(chars) {
484
+ const { html } = this;
485
+ const until = this.#at + chars;
486
+ while (!this.#done && this.#at < until) {
487
+ const tag = this.#next(this.#at);
488
+ if (!tag || tag.gt < 0) {
489
+ this.#done = true;
490
+ break;
491
+ }
492
+ this.#at = tag.gt + 1;
493
+ const { name } = tag;
494
+ if (!name)
495
+ continue;
496
+ if (tag.end) {
497
+ const open = this.#open.get(name)?.pop();
498
+ if (open) {
499
+ // An end tag closes what its element left open: a p or li opened after it ends here (HTML's optional end tags).
500
+ for (const implied of IMPLIED_END) {
501
+ const stack = this.#open.get(implied);
502
+ while (stack && stack.length > 0 && stack[stack.length - 1].lt > open.lt)
503
+ this.#close(stack.pop(), tag.lt);
504
+ }
505
+ this.#close(open, tag.gt + 1);
506
+ }
507
+ continue;
508
+ }
509
+ // A start tag that closes an open p or li (optional end tags): that element ends here, so its hiding stops here and
510
+ // never reaches text a browser shows.
511
+ if (P_CLOSERS.has(name))
512
+ this.#closeImplied("p", tag.lt);
513
+ if (name === "li") {
514
+ const last = this.#open.get("li")?.at(-1);
515
+ // Only an li not inside a list opened after it.
516
+ if (last && !LISTS.some((list) => (this.#open.get(list)?.at(-1)?.lt ?? -1) > last.lt))
517
+ this.#closeImplied("li", tag.lt);
518
+ }
519
+ const { why, shows } = hides(name, insideOf(html, name, tag.lt, tag.gt), this.page, this.decode);
520
+ // It shows itself inside a visibility-hidden element, and is shown (not display-hidden itself, and no display-hidden
521
+ // element open between them): that element is kept (and, as it closes, those around it).
522
+ const inner = this.#invisible.at(-1);
523
+ if (shows && why !== "display" && inner && this.#displayOpen === inner.displayOpen)
524
+ inner.shown = true;
525
+ const hidden = why !== null;
526
+ if (VOID.has(name)) {
527
+ if (hidden)
528
+ this.#ranges.push([tag.lt, tag.gt + 1]);
529
+ continue;
530
+ }
531
+ // Text, not markup, to its end tag (an iframe's fallback, an xmp): its "<" are not tags.
532
+ if (TEXT_ONLY.has(name)) {
533
+ const closer = CLOSER[name];
534
+ closer.lastIndex = tag.gt + 1;
535
+ const close = closer.exec(html);
536
+ if (hidden && close)
537
+ this.#ranges.push([tag.lt, close.index + close[0].length]);
538
+ if (!close) {
539
+ this.#done = true;
540
+ break;
541
+ }
542
+ this.#at = close.index + close[0].length;
543
+ continue;
544
+ }
545
+ let open = this.#open.get(name);
546
+ if (!open)
547
+ this.#open.set(name, (open = []));
548
+ const opened = { lt: tag.lt, why, shown: false, closed: false, outer: inner, displayOpen: this.#displayOpen };
549
+ open.push(opened);
550
+ if (why === "visibility")
551
+ this.#invisible.push(opened);
552
+ if (why === "display")
553
+ this.#displayOpen += 1;
554
+ }
555
+ return this.#done;
556
+ }
557
+ /** The innermost open `name` element, ended by a start tag at `at` that implies its end tag. */
558
+ #closeImplied(name, at) {
559
+ const open = this.#open.get(name)?.pop();
560
+ if (open)
561
+ this.#close(open, at);
562
+ }
563
+ /** `opened` closed by the end tag that ends at `end`: a display-hidden element goes; a visibility-hidden one goes
564
+ * unless something shown inside it showed itself again, and is then kept with its mark handed to the one around it
565
+ * (`outer`). One closed out of nesting order (by a stray end tag) is kept, since where its hiding reaches is in doubt,
566
+ * and its stack entry is left, marked closed, to be popped when it reaches the top: no search, no shift. */
567
+ #close(opened, end) {
568
+ if (opened.why === "display") {
569
+ this.#displayOpen = Math.max(0, this.#displayOpen - 1);
570
+ this.#ranges.push([opened.lt, end]);
571
+ return;
572
+ }
573
+ if (opened.why !== "visibility")
574
+ return;
575
+ const invisible = this.#invisible;
576
+ opened.closed = true;
577
+ if (invisible.at(-1) === opened) {
578
+ invisible.pop();
579
+ while (invisible.at(-1)?.closed)
580
+ invisible.pop();
581
+ }
582
+ else
583
+ opened.shown = true;
584
+ if (!opened.shown) {
585
+ this.#ranges.push([opened.lt, end]);
586
+ return;
587
+ }
588
+ // Kept, it shows in the one around it only if no display-hidden element was open between them.
589
+ if (opened.outer && opened.displayOpen === opened.outer.displayOpen)
590
+ opened.outer.shown = true;
591
+ }
592
+ /** The page without its hidden ranges (once step has answered true; the page as it is before that). */
593
+ result() {
594
+ if (!this.#done || this.#ranges.length === 0)
595
+ return this.html;
596
+ const ranges = this.#ranges.sort((a, b) => a[0] - b[0]);
597
+ const kept = [];
598
+ let from = 0;
599
+ for (const [start, end] of ranges) {
600
+ if (end <= from)
601
+ continue;
602
+ if (start > from)
603
+ kept.push(this.html.slice(from, start));
604
+ from = Math.max(from, end);
605
+ }
606
+ kept.push(this.html.slice(from));
607
+ return kept.join("");
608
+ }
609
+ }
610
+ /** Elements whose end tag HTML lets a page omit; an end tag of an element around one ends it too. */
611
+ const IMPLIED_END = ["p", "li"];
612
+ const LISTS = ["ul", "ol", "menu"];
613
+ /** Start tags that close an open p. */
614
+ const P_CLOSERS = new Set([
615
+ "address", "article", "aside", "blockquote", "details", "div", "dl", "fieldset", "figcaption", "figure", "footer", "form",
616
+ "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "main", "menu", "nav", "ol", "p", "pre", "section", "table", "ul",
617
+ ]);
618
+ /** `html` without the elements it hides from everyone, all at once (see HiddenDrop); `page` and `decode` as there. */
619
+ export function dropHidden(html, page, decode) {
620
+ const drop = new HiddenDrop(html, page, decode);
621
+ drop.step(Number.POSITIVE_INFINITY);
622
+ return drop.result();
623
+ }
176
624
  /** `html` cut to at most `max` characters, after the last tag that ends before the cut, so no tag is left half; at
177
625
  * `max` itself when that tag ends more than 64 KiB before it (a page of one long text), or one before it when `max`
178
626
  * falls between the two halves of a character written as a surrogate pair, so no half of one is left. */
@@ -275,6 +723,18 @@ const TITLE_END = /<\/title(?=[\s/>])/gi;
275
723
  * tagsOf ends a tag by: a quote right after the "=" and its spaces; one that no quote closes runs to the tag's end.
276
724
  */
277
725
  function attrOf(inside, name, decode) {
726
+ let found = null;
727
+ eachAttribute(inside, (named, value) => {
728
+ if (named.length !== name.length || named.toLowerCase() !== name)
729
+ return false;
730
+ found = value.includes("&") ? decode(value) : value;
731
+ return true;
732
+ });
733
+ return found;
734
+ }
735
+ /** Each attribute of a tag's `inside` (insideOf), in order, read as attrOf says: `visit` gets its name as written and its
736
+ * raw value ("" for one with no value), and stops the walk by answering true. */
737
+ function eachAttribute(inside, visit) {
278
738
  const { length } = inside;
279
739
  for (let at = 0; at < length;) {
280
740
  const code = inside.charCodeAt(at);
@@ -286,12 +746,12 @@ function attrOf(inside, name, decode) {
286
746
  do
287
747
  at += 1;
288
748
  while (at < length && !endsName(inside.charCodeAt(at)));
289
- const named = at - from === name.length && inside.slice(from, at).toLowerCase() === name;
749
+ const name = inside.slice(from, at);
290
750
  while (isSpace(inside.charCodeAt(at)))
291
751
  at += 1;
292
752
  if (inside.charCodeAt(at) !== EQUALS) {
293
- if (named)
294
- return "";
753
+ if (visit(name, ""))
754
+ return;
295
755
  continue;
296
756
  }
297
757
  at += 1;
@@ -311,12 +771,9 @@ function attrOf(inside, name, decode) {
311
771
  at += 1;
312
772
  end = at;
313
773
  }
314
- if (named) {
315
- const value = inside.slice(start, end);
316
- return value.includes("&") ? decode(value) : value;
317
- }
774
+ if (visit(name, inside.slice(start, end)))
775
+ return;
318
776
  }
319
- return null;
320
777
  }
321
778
  /** What a tag at `lt` named `name` holds between its name and its ">" at `gt`. */
322
779
  const insideOf = (html, name, lt, gt) => html.slice(lt + 1 + name.length, gt);
@@ -41,6 +41,9 @@ export function isPrivateAddress(ip) {
41
41
  return true;
42
42
  if (groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff)
43
43
  return isPrivateAddress([groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join("."));
44
+ // 6to4 (2002::/16, RFC 3056) carries an IPv4 address in its next 32 bits, which a relay delivers to: judged as that one.
45
+ if (groups[0] === 0x2002)
46
+ return isPrivateAddress([groups[1] >> 8, groups[1] & 0xff, groups[2] >> 8, groups[2] & 0xff].join("."));
44
47
  return V6_PRIVATE.some(([base, bits]) => inV6(groups, v6Groups(base), bits));
45
48
  }
46
49
  /** Answers `address` for any name: the address a request was checked against is the one it connects to. */
@@ -1,7 +1,8 @@
1
1
  // A fetched HTML page read as its title, its text and its links, without ever holding the event loop. The host's tools
2
2
  // run in the run's own process, so a read that holds the loop holds the run's heartbeat with it, and a run whose
3
3
  // heartbeat lapses is fenced: go.dev/dl, 3.15 MB of tables, held it past 240 s in one markdown conversion. So one linear
4
- // pass drops the blocks that hold no readable text (page-scan.ts); what is left is cut at READ_MAX_CHARS (the page is
4
+ // pass drops the blocks that hold no readable text, and another the elements the page hides from everyone (page-scan.ts:
5
+ // a help pop-up until its link is clicked is not the page's text); what is left is cut at READ_MAX_CHARS (the page is
5
6
  // then `truncated`, and `html_chars` is its whole length); and a worker thread converts that to markdown, in pieces
6
7
  // small enough that the converter stays linear (page-markdown.ts), within a deadline. A worker that has not answered by
7
8
  // then, or that fails, is stopped, and the text is the page's plain text, read on the loop in steps that yield it
@@ -10,7 +11,7 @@
10
11
  import { setImmediate as yieldToLoop } from "node:timers/promises";
11
12
  import { Worker } from "node:worker_threads";
12
13
  import { decodeEntities } from "./entities.js";
13
- import { cutHtml, InertDrop, PageScan } from "./page-scan.js";
14
+ import { cutHtml, HiddenDrop, InertDrop, pageHidingOf, PageScan } from "./page-scan.js";
14
15
  /** The most of a page's HTML that is read, counted once its script, style, svg, noscript and template blocks and its
15
16
  * comments are dropped. A million characters of go.dev/dl converts in about 0.8 s on a busy 48-core host and gives
16
17
  * 590,000 characters of markdown, ten times the 60,000 one web_fetch answer carries. */
@@ -28,7 +29,11 @@ const STEP_CHARS = 32_768;
28
29
  */
29
30
  export async function readPage(html, url, { deadlineMs = READ_DEADLINE_MS, signal } = {}) {
30
31
  signal?.throwIfAborted();
31
- const kept = await dropOnLoop(html, signal);
32
+ // What the page hides from everyone who reads it (a help pop-up until its link is clicked, a closed dialog) is not
33
+ // its text: what its own live style elements declare is read from those the inert drop took out.
34
+ const inert = new InertDrop(html);
35
+ await stepsOnLoop(inert, signal);
36
+ const kept = await dropHiddenOnLoop(inert.result(), pageHidingOf(inert.styles()), signal);
32
37
  const read = cutHtml(kept, READ_MAX_CHARS);
33
38
  const cut = read.length < kept.length ? { truncated: true, html_chars: html.length } : {};
34
39
  await yieldToLoop();
@@ -78,10 +83,21 @@ function inWorker(job, deadlineMs, signal) {
78
83
  /** `html` without its inert blocks (InertDrop), STEP_CHARS at a time with the loop yielded between steps. */
79
84
  export async function dropOnLoop(html, signal) {
80
85
  const drop = new InertDrop(html);
81
- while (!drop.step(STEP_CHARS)) {
86
+ await stepsOnLoop(drop, signal);
87
+ return drop.result();
88
+ }
89
+ /** Runs `pass` STEP_CHARS at a time, the loop yielded between steps, to its end. */
90
+ async function stepsOnLoop(pass, signal) {
91
+ while (!pass.step(STEP_CHARS)) {
82
92
  await yieldToLoop();
83
93
  signal?.throwIfAborted();
84
94
  }
95
+ }
96
+ /** `html` without the elements it hides from everyone (HiddenDrop), with `page` what its own live styles declare
97
+ * (pageHidingOf), STEP_CHARS at a time with the loop yielded between steps. */
98
+ async function dropHiddenOnLoop(html, page, signal) {
99
+ const drop = new HiddenDrop(html, page, decodeEntities);
100
+ await stepsOnLoop(drop, signal);
85
101
  return drop.result();
86
102
  }
87
103
  /** `html`'s plain text, title and links, scanned STEP_CHARS at a time with the loop yielded between steps. */
@@ -421,7 +421,10 @@ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT,
421
421
  const url = pageUrl(call, await s.driver.url());
422
422
  const receipt = await file(s, call, { tool: "screenshot", args: region ? { ...requested, clip: region.clip, ...region.of } : requested, status: "ok", facts: { final_url: url }, body: Buffer.from(shot.data, "base64"), ext: shot.size.type === "png" ? "png" : "jpg" });
423
423
  const first = shot.adjusted?.first;
424
- const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} was over the ${shot.adjusted.why === "long_edge" ? "2,000 px long edge" : "size"} budget; this is ${region ? "the clip" : "the viewport"} at ${shot.size.width}x${shot.size.height}`;
424
+ const over = shot.adjusted?.why === "long_edge" ? "was over the 2,000 px long edge budget" : shot.adjusted?.why === "base64_bytes" ? "was over the size budget" : "could not be measured";
425
+ const taken = shot.options.clip ? "the clip" : shot.options.fullPage ? "the full page" : "the viewport";
426
+ const scaled = shot.options.scale !== undefined && shot.options.scale < 1 ? ", scaled down" : "";
427
+ const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} ${over}; this is ${taken} at ${shot.size.width}x${shot.size.height}${scaled}`;
425
428
  const lines = [news, `url: ${url}`, `evidence: ${receipt.path ?? "(not filed)"}`, adjusted].filter(Boolean).join("\n");
426
429
  const media = { kind: "screenshot", path: receipt.bodyPath, mime: shot.mimeType, bytes: Buffer.from(shot.data, "base64").byteLength, width: shot.size.width, height: shot.size.height, ...(region ? { clip: region.clip, ...region.of } : {}), ...(ask ? { ask } : {}) };
427
430
  return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256, media } };
@@ -19,17 +19,7 @@ export type CapturedState = {
19
19
  /** Only cookies the allowed origins would be sent (see `cookieKept`). Partitioned cookies are not kept. */
20
20
  cookies: VaultCookie[];
21
21
  localStorage: Record<string, Record<string, string>>;
22
- }
23
- /** `origins`: the origins the context was captured for. A context holds a whole browser profile, so it is restored only for a request that covers EVERY one of them (`coversEveryOrigin`); one that records none is refused. */
24
- | {
25
- kind: "provider-context";
26
- capturedAt: string;
27
- provider: "browserbase" | "kernel";
28
- contextId: string;
29
- origins?: string[];
30
22
  };
31
- /** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
32
- export declare function coversEveryOrigin(requested: readonly string[], saved: readonly string[] | undefined): boolean;
33
23
  export interface SessionVaultPort {
34
24
  /** At release and at the end of a handoff, and, for a context a new session made, right after the create (so a restart still finds it). The host supplied the origins; nothing outside them was captured.
35
25
  * A capture has a deadline (20 s) after which nothing more is read or written, but a write already handed to this port cannot be recalled: ignore one whose `capturedAt` is older than the one you hold. */
@@ -11,19 +11,6 @@ export function clampSessionCookieAgeMs(requested) {
11
11
  throw new RangeError("the session-cookie age cap is a positive number of milliseconds");
12
12
  return Math.min(requested, MAX_SESSION_COOKIE_AGE_MS);
13
13
  }
14
- /** Whether `requested` includes every origin in `saved` (as scheme://host[:port], any order): a provider context is all-or-nothing, so a request for some of its origins would bring back the others' sign-ins. */
15
- export function coversEveryOrigin(requested, saved) {
16
- if (!saved || saved.length === 0)
17
- return false;
18
- const origin = (value) => { try {
19
- return new URL(value).origin;
20
- }
21
- catch {
22
- return null;
23
- } };
24
- const have = new Set(requested.map(origin));
25
- return saved.every((value) => { const o = origin(value); return o !== null && have.has(o); });
26
- }
27
14
  const SESSION_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/;
28
15
  /** The host's allowed origins, each `scheme://host[:port]` of http or https, normalised and de-duplicated; anything else is refused. */
29
16
  export function vaultOrigins(origins) {
@@ -224,9 +224,9 @@ export function cdpDriver() {
224
224
  const selector = place?.selector ?? null;
225
225
  return page.evaluate(`(${LOCATE_JS}).call(document, ${JSON.stringify(selector)})`);
226
226
  },
227
- screenshot: async ({ type = "png", quality, fullPage, clip }) => {
228
- const data = await page.screenshot({ type, ...(quality === undefined ? {} : { quality }), ...(fullPage === undefined ? {} : { fullPage }), ...(clip ? { clip } : {}) });
229
- return { data, mimeType: type === "jpeg" ? "image/jpeg" : "image/png" };
227
+ screenshot: async (options) => {
228
+ const data = await page.screenshot(options);
229
+ return { data, mimeType: options.type === "jpeg" ? "image/jpeg" : "image/png" };
230
230
  },
231
231
  // One evaluate reads all four, so they are one document's.
232
232
  page: () => page.evaluate(() => ({