@raibitsi/tiresias 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +88 -0
  2. package/README.md +72 -33
  3. package/data/README.md +33 -0
  4. package/data/guard-ordinary-urls.txt +102 -0
  5. package/data/guard-starter.jsonl +424 -0
  6. package/dist/aimed.js +23 -0
  7. package/dist/anchor.js +26 -13
  8. package/dist/browsecli.js +17 -13
  9. package/dist/cdp.js +22 -1
  10. package/dist/cleanup.js +3 -1
  11. package/dist/cli.js +216 -68
  12. package/dist/config.js +75 -1
  13. package/dist/confirm.js +20 -5
  14. package/dist/cooldown.js +57 -32
  15. package/dist/dailylog.js +11 -5
  16. package/dist/dom.js +13 -8
  17. package/dist/doors.js +1 -1
  18. package/dist/engine.js +9 -4
  19. package/dist/envelope.js +10 -1
  20. package/dist/fetcher.js +29 -2
  21. package/dist/filereply.js +7 -1
  22. package/dist/formflow.js +21 -3
  23. package/dist/forms.js +337 -42
  24. package/dist/getactions.js +119 -0
  25. package/dist/guard.js +1240 -0
  26. package/dist/guardclient.js +245 -0
  27. package/dist/guardnetwork.js +22 -0
  28. package/dist/history.js +11 -5
  29. package/dist/hook.js +2 -0
  30. package/dist/hosts.js +34 -27
  31. package/dist/keys.js +420 -18
  32. package/dist/local.js +537 -0
  33. package/dist/mcp.js +56 -25
  34. package/dist/needs.js +9 -5
  35. package/dist/parentwatch.js +6 -16
  36. package/dist/pdf.js +4 -1
  37. package/dist/pdfchild.js +0 -2
  38. package/dist/pdfproc.js +26 -11
  39. package/dist/pdfwatch.js +14 -0
  40. package/dist/pipeline.js +117 -26
  41. package/dist/preformatted.js +12 -0
  42. package/dist/profiles.js +3 -3
  43. package/dist/queue.js +24 -20
  44. package/dist/reader.js +441 -87
  45. package/dist/receipts.js +7 -2
  46. package/dist/render.js +156 -27
  47. package/dist/renderactions.js +69 -0
  48. package/dist/resume.js +8 -6
  49. package/dist/rules.js +24 -5
  50. package/dist/safebrowsing.js +3 -3
  51. package/dist/safety.js +15 -7
  52. package/dist/sandbox.js +1 -1
  53. package/dist/saved.js +24 -9
  54. package/dist/savedlock.js +99 -5
  55. package/dist/settings.js +10 -3
  56. package/dist/shelf.js +1 -1
  57. package/dist/shield.js +15 -0
  58. package/dist/slips.js +43 -1
  59. package/dist/store.js +119 -25
  60. package/dist/tabs.js +1 -1
  61. package/dist/textview.js +16 -0
  62. package/dist/translate.js +22 -2
  63. package/dist/types.js +9 -2
  64. package/dist/watchflow.js +1 -2
  65. package/dist/work.js +10 -2
  66. package/docs/LOCAL-MODELS.md +271 -0
  67. package/package.json +5 -2
package/dist/receipts.js CHANGED
@@ -23,15 +23,20 @@ export function bound(r) {
23
23
  // sent) or "file" (a file warning). Older lines have neither: a line with `sent: "form"` is a form, one with
24
24
  // `file: "warning"` a file (whose `kind` field there was the FILE's kind), anything else a quote.
25
25
  export const RECORD_VERSION = 1;
26
+ // Returns how many of the logged copies were cut (0: the log holds every receipt whole).
26
27
  export function record(receipts) {
27
28
  const now = Date.now();
28
29
  const recorded_at = new Date(now).toISOString();
30
+ let cut = 0;
29
31
  for (const r of receipts) {
30
32
  const row = bound({ v: RECORD_VERSION, kind: "quote", ...r, recorded_at });
33
+ if (row.cut)
34
+ cut++;
31
35
  appendLine(`receipts/${utcDay(now)}.jsonl`, row);
32
36
  }
37
+ return cut;
33
38
  }
34
- export async function query(f) {
39
+ export async function query(f, scan = {}) {
35
40
  const since = parseSince(f.since);
36
41
  // The URL as typed, or as a URL parser writes it ("https://example.com" is recorded as "https://example.com/").
37
42
  let alt;
@@ -54,7 +59,7 @@ export async function query(f) {
54
59
  bound(r); // filters match the stored values; only what is returned is cut
55
60
  r.record_fingerprint = record_fingerprint;
56
61
  return true;
57
- }, f.page ?? 1);
62
+ }, f.page ?? 1, 40_000, scan);
58
63
  }
59
64
  const MAX_SENT = 16_000;
60
65
  export function recordSent(r) {
package/dist/render.js CHANGED
@@ -24,14 +24,18 @@ import net from "node:net";
24
24
  import os from "node:os";
25
25
  import path from "node:path";
26
26
  import { Cdp, CdpError } from "./cdp.js";
27
+ import { getActionTarget, getActionTargetsMatch } from "./getactions.js";
27
28
  import { callContext } from "./context.js";
28
29
  import { Busy, Slots } from "./slots.js";
29
30
  import { ConfigError, TiresiasError } from "./errors.js";
31
+ import { runUntrusted } from "./sandbox.js";
32
+ import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_LIVE_ACTION, RenderActionCache } from "./renderactions.js";
30
33
  import { politeFetch } from "./fetcher.js";
31
34
  import { shutdownSignal } from "./lifetime.js";
32
35
  import { READ_SLACK, fetchCap } from "./safety.js";
33
36
  import { envMayLoosen, inForceConfig } from "./confirm.js";
34
37
  import { LIMITS as INSPECT, VIEWPORT, checkInspection, inspectScript, newMark, wallVerdict } from "./inspect.js";
38
+ import { STRUCTURED_HEADING } from "./reader.js";
35
39
  export const RENDER_MODES = ["off", "ask", "auto"];
36
40
  export const DEFAULT_RENDER = "off";
37
41
  export function parseRender(v) {
@@ -60,7 +64,7 @@ export function loadRenderMode() {
60
64
  return cfg;
61
65
  }
62
66
  export const BLOCK_REASONS = [
63
- "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
67
+ "form_action", "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
64
68
  "rules", "wall", "site_settings", "unsafe_address", "safe_browsing", "file", "too_large", "fetch_failed", "signing", "error",
65
69
  ];
66
70
  export const RENDER_TEXT = {
@@ -75,7 +79,7 @@ export const RENDERED_TEXT = "Rendered: Tiresias ran the page's JavaScript in a
75
79
  // ---- limits ----
76
80
  export const LIMITS = {
77
81
  ms: 20_000, // the whole render
78
- quietMs: 500, // the page is settled when no request has been pending this long, after its load event
82
+ quietMs: 500, // no paused request or fetch for this long after loading finishes
79
83
  requests: 100,
80
84
  bytes: 10 * 1024 * 1024,
81
85
  perRequestBytes: 4 * 1024 * 1024, // what one request reserves from the budget (and may read at most)
@@ -119,9 +123,9 @@ export function findChrome() {
119
123
  return (CHROME_PATHS[process.platform] ?? []).find(runnable) ?? null;
120
124
  }
121
125
  // ---- the empty JavaScript shell test (auto mode) ----
122
- // A page read as HTML whose text is almost nothing and that carries scripts.
123
- export function looksLikeShell(textChars, body) {
124
- return textChars < 200 && /<script\b/i.test(body);
126
+ // v1.2: short link menus can be shells; a substantial unlinked block or a useful list of links is content.
127
+ export function looksLikeShell(parsed, hasScript) {
128
+ return hasScript && parsed.text_chars < 200 && parsed.links.length < 20 && !parsed.sections.some(s => !(s.generated && s.heading === STRUCTURED_HEADING) && s.blocks.some(b => b.text.length >= 80 && b.marks.length === 0));
125
129
  }
126
130
  // ---- process lifetime ----
127
131
  const live = new Map(); // every running Chrome's killer, by process id
@@ -410,6 +414,9 @@ async function renderNow(pageUrl, body, fn) {
410
414
  const chrome = findChrome();
411
415
  if (!chrome)
412
416
  throw new RenderUnavailable();
417
+ const forms = await runUntrusted({ op: "form_actions", body, url: pageUrl }, body.length);
418
+ const actionCache = new RenderActionCache();
419
+ const actions = forms.incomplete ? null : actionCache.scan(forms.actions);
413
420
  hookExit();
414
421
  sweepProfiles();
415
422
  const started = Date.now();
@@ -420,6 +427,8 @@ async function renderNow(pageUrl, body, fn) {
420
427
  const signals = [ctl.signal, shutdownSignal, ...(outer ? [outer] : [])];
421
428
  const signal = AbortSignal.any(signals);
422
429
  const timer = setTimeout(() => ctl.abort(new Stop("timeout")), renderMs());
430
+ let navigationTimer;
431
+ let settleTimer;
423
432
  let dataDir;
424
433
  let proxy;
425
434
  let child;
@@ -462,40 +471,62 @@ async function renderNow(pageUrl, body, fn) {
462
471
  const mainFrame = { id: "" };
463
472
  let docServed = false;
464
473
  let pending = 0;
474
+ let outstanding = 0; // bounded metadata for paused requests, including local replies and refusals
465
475
  let lastActivity = Date.now();
466
476
  let loaded = false;
477
+ let domComplete = false;
478
+ let refusedNavigation = false;
467
479
  const budget = new ByteBudget(LIMITS.bytes, Math.min(fetchCap(), LIMITS.perRequestBytes));
468
480
  const fetchFor = async (e) => {
469
481
  const p = e.params;
470
482
  const id = p.requestId;
471
483
  // "Aborted", not "BlockedByClient": a refused navigation then leaves the page as it was instead of committing
472
484
  // Chrome's own error page in its place (which would be read as the page).
473
- const fail = (r) => (block(r), cdp.send("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
485
+ const fail = (r) => (block(r), cmd("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
474
486
  const url = String(p.request?.url ?? "");
475
487
  const type = String(p.resourceType ?? "");
476
488
  const method = String(p.request?.method ?? "");
477
489
  // The page itself: served from the body the plain read already fetched, exactly once, in the main frame.
478
490
  if (type === "Document" && e.sessionId === pageSession && p.frameId === mainFrame.id && !docServed && noFragment(url) === noFragment(pageUrl)) {
479
491
  docServed = true;
480
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
492
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
481
493
  return;
482
494
  }
483
495
  top.requests++;
484
496
  if (e.sessionId !== pageSession)
485
- return void fail("worker");
486
- if (type === "Document")
487
- return void fail(p.frameId === mainFrame.id ? "navigation" : "frame");
497
+ return await fail("worker");
498
+ if (type === "Document") {
499
+ if (p.frameId !== mainFrame.id)
500
+ return await fail("frame");
501
+ if (!loaded) {
502
+ // Chrome aborts the old document's load before this refusal, so its load event may never arrive.
503
+ refusedNavigation = true;
504
+ if (domComplete) {
505
+ loaded = true;
506
+ lastActivity = Date.now();
507
+ }
508
+ else if (!navigationTimer) {
509
+ // Give parsing a chance to finish, but return the plain read promptly if it was aborted too.
510
+ navigationTimer = setTimeout(() => {
511
+ navigationTimer = undefined;
512
+ if (!domComplete && !loaded)
513
+ ctl.abort(new Stop("navigated"));
514
+ }, LIMITS.quietMs);
515
+ }
516
+ }
517
+ return await fail("navigation");
518
+ }
488
519
  if (!/^https?:/i.test(url))
489
- return void fail("scheme");
520
+ return await fail("scheme");
490
521
  if (type === "Preflight") {
491
522
  // A CORS check before a GET with extra headers. The page's own headers are never sent (Tiresias sends
492
523
  // its own), so it is answered here and the real request still goes through the rules.
493
524
  const want = String(p.request?.headers?.["Access-Control-Request-Method"] ?? p.request?.headers?.["access-control-request-method"] ?? "");
494
525
  if (want.toUpperCase() !== "GET")
495
- return void fail("not_get");
526
+ return await fail("not_get");
496
527
  const origin = String(p.request?.headers?.Origin ?? p.request?.headers?.origin ?? "*");
497
528
  const asked = String(p.request?.headers?.["Access-Control-Request-Headers"] ?? p.request?.headers?.["access-control-request-headers"] ?? "");
498
- await cdp.send("Fetch.fulfillRequest", {
529
+ await cmd("Fetch.fulfillRequest", {
499
530
  requestId: id, responseCode: 204,
500
531
  responseHeaders: [{ name: "access-control-allow-origin", value: origin.slice(0, 2048) }, { name: "access-control-allow-methods", value: "GET" }, ...(asked ? [{ name: "access-control-allow-headers", value: asked.slice(0, 2048) }] : [])],
501
532
  }, e.sessionId).catch(() => { });
@@ -503,11 +534,86 @@ async function renderNow(pageUrl, body, fn) {
503
534
  return;
504
535
  }
505
536
  if (method !== "GET")
506
- return void fail("not_get");
537
+ return await fail("not_get");
538
+ const target = new URL(url);
539
+ const actionTarget = getActionTarget(target);
540
+ if (actions === null || actions.some((a) => getActionTargetsMatch(a, actionTarget)))
541
+ return await fail("form_action");
507
542
  if (!TYPES_FETCHED.has(type))
508
- return void fail("not_needed");
543
+ return await fail("not_needed");
544
+ // Check live forms too: scripts may have added a form or changed its action since the HTML arrived.
545
+ try {
546
+ const world = await cmd("Page.createIsolatedWorld", { frameId: mainFrame.id, worldName: "tiresias", grantUniveralAccess: false }, pageSession);
547
+ const r = await cmd("Runtime.evaluate", {
548
+ contextId: world.executionContextId, returnByValue: true, timeout: 1000,
549
+ expression: `(() => { /* get_form_actions */
550
+ const getter = (proto, name) => Object.getOwnPropertyDescriptor(proto, name).get;
551
+ const forms = getter(Document.prototype, 'forms').call(document);
552
+ const formCount = getter(HTMLCollection.prototype, 'length').call(forms);
553
+ const formAt = HTMLCollection.prototype.item;
554
+ // Native selectors and indexed access avoid named form properties and page wrappers.
555
+ // Include attributes on every element, even controls outside their form and non-controls.
556
+ const nodes = Document.prototype.querySelectorAll.call(document, '[formaction], [formmethod]');
557
+ const nodeCount = getter(NodeList.prototype, 'length').call(nodes);
558
+ if (formCount > ${MAX_FORM_ACTIONS} || nodeCount > ${LIMITS.elements}) return null;
559
+ const nodeAt = NodeList.prototype.item;
560
+ const attr = Element.prototype.getAttribute;
561
+ const localName = getter(Element.prototype, 'localName');
562
+ const namespace = getter(Element.prototype, 'namespaceURI');
563
+ const base = getter(Node.prototype, 'baseURI');
564
+ const page = getter(Document.prototype, 'URL').call(document);
565
+ const method = getter(HTMLFormElement.prototype, 'method');
566
+ const action = getter(HTMLFormElement.prototype, 'action');
567
+ const buttonAction = getter(HTMLButtonElement.prototype, 'formAction');
568
+ const buttonMethod = getter(HTMLButtonElement.prototype, 'formMethod');
569
+ const inputAction = getter(HTMLInputElement.prototype, 'formAction');
570
+ const inputMethod = getter(HTMLInputElement.prototype, 'formMethod');
571
+ const out = [];
572
+ let total = 0, count = formCount, getOverride = false;
573
+ const add = a => {
574
+ if (typeof a !== 'string' || a.length > ${MAX_LIVE_ACTION} || a.includes('\\0')) throw new Error('Unreadable action');
575
+ total += a.length;
576
+ if (total > ${MAX_FORM_ACTION_CHARS}) throw new Error('Action budget');
577
+ out.push(a);
578
+ };
579
+ for (let i = 0; i < nodeCount; i++) {
580
+ const n = nodeAt.call(nodes, i);
581
+ const name = namespace.call(n) === 'http://www.w3.org/1999/xhtml' ? localName.call(n) : '';
582
+ const actionGetter = name === 'button' ? buttonAction : name === 'input' ? inputAction : null;
583
+ const methodGetter = name === 'button' ? buttonMethod : name === 'input' ? inputMethod : null;
584
+ const fm = attr.call(n, 'formmethod');
585
+ if (fm !== null) {
586
+ const m = methodGetter ? methodGetter.call(n) : fm;
587
+ if (typeof m !== 'string') return null;
588
+ if (m.toLowerCase() !== 'post' && m.toLowerCase() !== 'dialog') getOverride = true;
589
+ }
590
+ const fa = attr.call(n, 'formaction');
591
+ if (fa !== null) {
592
+ if (++count > ${MAX_FORM_ACTIONS} || typeof fa !== 'string' || fa.length > ${MAX_LIVE_ACTION} || fa.includes('\\0')) return null;
593
+ // Non-controls have no formAction getter; apply the same URL rule ourselves.
594
+ const a = actionGetter ? actionGetter.call(n) : fa === '' ? page : new URL(fa, base.call(n)).href;
595
+ add(a);
596
+ }
597
+ }
598
+ for (let i = 0; i < formCount; i++) {
599
+ const f = formAt.call(forms, i);
600
+ const m = method.call(f);
601
+ if (typeof m !== 'string') return null;
602
+ if (!getOverride && (m.toLowerCase() === 'post' || m.toLowerCase() === 'dialog')) continue;
603
+ add(action.call(f));
604
+ }
605
+ return out;
606
+ })()`,
607
+ }, pageSession);
608
+ const live = actionCache.scan(r.result?.value);
609
+ if (live === null || live.some((a) => getActionTargetsMatch(a, actionTarget)))
610
+ return await fail("form_action");
611
+ }
612
+ catch {
613
+ return await fail("form_action");
614
+ }
509
615
  if (top.requests > LIMITS.requests)
510
- return void fail("request_cap");
616
+ return await fail("request_cap");
511
617
  pending++;
512
618
  // Codex round 1 (HIGH): the byte cap is a shared budget, RESERVED before a request starts (each request may read
513
619
  // at most what it reserved), so requests running at once can never together exceed it. What a request did not
@@ -520,7 +626,7 @@ async function renderNow(pageUrl, body, fn) {
520
626
  // The reservation bounds what the connection may receive (headers, 1xx, compressed bytes: safety.ts Wire), less
521
627
  // one socket read of slack, so the charge never passes it; the decoded body is held to it too.
522
628
  if (reserved <= READ_SLACK)
523
- return void fail("byte_cap");
629
+ return await fail("byte_cap");
524
630
  const res = await callContext.run({ ...callContext.getStore(), signal, renderBudget: budget }, () => politeFetch(url, { maxBytes: reserved, wireBytes: reserved - READ_SLACK, followRedirects: false }));
525
631
  // Chrome must see the redirect itself: module bases and cross-origin checks depend on every URL.
526
632
  // safeFetch validated the target (including https downgrades); the next paused request goes through
@@ -533,29 +639,29 @@ async function renderNow(pageUrl, body, fn) {
533
639
  if (KEEP_HEADERS.has(k))
534
640
  headers.push({ name: k, value: v.slice(0, 2048) });
535
641
  top.fetched++;
536
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
642
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
537
643
  return;
538
644
  }
539
645
  // A file (a program, an archive, any binary the file plan names) or a PDF is never handed to the page: without a
540
646
  // read kind, the fetcher found a file in its first bytes.
541
647
  if (res.file || !res.read || res.read === "pdf")
542
- return void fail("file");
648
+ return await fail("file");
543
649
  const buf = Buffer.from(res.body, "utf8");
544
650
  // Charged what was really received (after decompression, before charset decoding: Codex round 2, a UTF-16
545
651
  // body shrinks when decoded) or what is handed to the page, whichever is larger; never less than it fetched.
546
652
  charge = Math.max(res.size ?? reserved, buf.length);
547
653
  if (charge > reserved)
548
- return void fail("byte_cap"); // re-encoding made it larger than what was reserved
654
+ return await fail("byte_cap"); // re-encoding made it larger than what was reserved
549
655
  const headers = [];
550
656
  for (const [k, v] of res.headers)
551
657
  if (KEEP_HEADERS.has(k) && k !== "content-type")
552
658
  headers.push({ name: k, value: v.slice(0, 2048) });
553
659
  headers.push({ name: "content-type", value: utf8Type(res.headers.get("content-type")) });
554
660
  top.fetched++;
555
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
661
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
556
662
  }
557
663
  catch (err) {
558
- fail(signal.aborted ? "fetch_failed" : blockOf(err));
664
+ await fail(signal.aborted ? "fetch_failed" : blockOf(err));
559
665
  }
560
666
  finally {
561
667
  if (reserved > 0)
@@ -565,6 +671,8 @@ async function renderNow(pageUrl, body, fn) {
565
671
  }
566
672
  };
567
673
  cdp.on((e) => {
674
+ if (signal.aborted)
675
+ return;
568
676
  const p = e.params;
569
677
  if (e.method === "Target.attachedToTarget") {
570
678
  const t = p.targetInfo ?? {};
@@ -585,7 +693,15 @@ async function renderNow(pageUrl, body, fn) {
585
693
  }
586
694
  if (e.method === "Fetch.requestPaused") {
587
695
  lastActivity = Date.now();
588
- void fetchFor(e);
696
+ if (outstanding >= LIMITS.requests + 1)
697
+ return ctl.abort(new Stop("too_large"));
698
+ outstanding++;
699
+ void fetchFor(e).catch(() => {
700
+ ctl.abort(new Stop("browser_failed"));
701
+ }).finally(() => {
702
+ outstanding--;
703
+ lastActivity = Date.now();
704
+ });
589
705
  return;
590
706
  }
591
707
  if (e.method === "Page.javascriptDialogOpening" && e.sessionId === pageSession) {
@@ -593,9 +709,20 @@ async function renderNow(pageUrl, body, fn) {
593
709
  cdp.send("Page.handleJavaScriptDialog", { accept: false }, pageSession).catch(() => { });
594
710
  return;
595
711
  }
596
- if (e.method === "Page.loadEventFired" && e.sessionId === pageSession) {
712
+ if (e.method === "Page.domContentEventFired" && e.sessionId === pageSession && docServed) {
713
+ domComplete = true;
714
+ if (refusedNavigation) {
715
+ loaded = true;
716
+ lastActivity = Date.now();
717
+ clearTimeout(navigationTimer);
718
+ navigationTimer = undefined;
719
+ }
720
+ }
721
+ if (e.method === "Page.loadEventFired" && e.sessionId === pageSession && docServed) {
597
722
  loaded = true;
598
723
  lastActivity = Date.now();
724
+ clearTimeout(navigationTimer);
725
+ navigationTimer = undefined;
599
726
  }
600
727
  });
601
728
  // The browser, a throwaway context with downloads refused, and everything new attached paused.
@@ -631,14 +758,14 @@ async function renderNow(pageUrl, body, fn) {
631
758
  await cmd("Emulation.setDeviceMetricsOverride", { width: VIEWPORT.width, height: VIEWPORT.height, deviceScaleFactor: 1, mobile: false }, s).catch(() => { });
632
759
  await cmd("Runtime.runIfWaitingForDebugger", {}, s);
633
760
  await cmd("Page.navigate", { url: pageUrl }, s);
634
- // Settled: the load event came and no request has been pending for quietMs; or the time cap.
761
+ // Settled: loading finished and every paused handler and fetch has finished, then quietMs elapsed.
635
762
  await race(new Promise((resolve) => {
636
763
  const tick = () => {
637
764
  if (signal.aborted)
638
765
  return; // the race has already ended; stop polling
639
- if (loaded && pending === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
766
+ if (loaded && pending === 0 && outstanding === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
640
767
  return resolve();
641
- setTimeout(tick, 100);
768
+ settleTimer = setTimeout(tick, 100);
642
769
  };
643
770
  tick();
644
771
  }));
@@ -677,6 +804,8 @@ async function renderNow(pageUrl, body, fn) {
677
804
  }
678
805
  finally {
679
806
  clearTimeout(timer);
807
+ clearTimeout(navigationTimer);
808
+ clearTimeout(settleTimer);
680
809
  ctl.abort(new Stop("cancelled"));
681
810
  if (child) {
682
811
  killTree(child);
@@ -0,0 +1,69 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ import { getActionTarget } from "./getactions.js";
3
+ import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES } from "./types.js";
4
+ // The limits live in types.ts so the page reader (forms.ts, run in the worker) does not load this
5
+ // module and, through it, the whole request path.
6
+ export { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES };
7
+ // Collectors cap source actions. The static union can emit one target per source
8
+ // for the page URL and each web base; deduplication can only reduce this bound.
9
+ export const MAX_FORM_ACTION_TARGETS = MAX_FORM_ACTIONS * (MAX_FORM_BASES + 1);
10
+ export const MAX_LIVE_ACTION = 8192;
11
+ // null means a valid non-network action. Invalid, credentialed and oversized actions
12
+ // throw instead, including on non-web schemes, so callers fail closed.
13
+ export function networkFormAction(raw, base, max) {
14
+ if (raw.length > max)
15
+ throw new Error("Oversized form action");
16
+ const u = new URL(raw, base);
17
+ if (u.href.length > max || u.username || u.password)
18
+ throw new Error("Unsafe form action");
19
+ return u.protocol === "http:" || u.protocol === "https:" ? u : null;
20
+ }
21
+ // Owned by one render, shared by its static scan and all live scans. FIFO eviction
22
+ // bounds both keys and values by entry count and source characters. A complete scan
23
+ // of live actions fits in the cache. Larger static scans may evict entries, but
24
+ // retain every target in the scan's separate result set. Later scans reuse
25
+ // retained entries and safely recompute evicted ones.
26
+ export class RenderActionCache {
27
+ entries = new Map();
28
+ chars = 0;
29
+ scan(input) {
30
+ if (!Array.isArray(input) || input.length > MAX_FORM_ACTION_TARGETS)
31
+ return null;
32
+ let total = 0;
33
+ // Static input is already deduplicated, web-only and fragment-free: its
34
+ // resolved lengths cannot exceed the collector's max(raw, resolved) charges.
35
+ for (const a of input) {
36
+ if (typeof a !== "string" || a.length > MAX_LIVE_ACTION)
37
+ return null;
38
+ total += a.length;
39
+ if (total > MAX_FORM_ACTION_CHARS)
40
+ return null;
41
+ }
42
+ const targets = new Set();
43
+ try {
44
+ for (const a of new Set(input)) {
45
+ let target = this.entries.get(a);
46
+ if (target === undefined) {
47
+ // Both the static resolver and the native live getter return absolute
48
+ // URLs. A relative getter value signals failed resolution (for example
49
+ // under an opaque base); do not silently resolve it against the page.
50
+ const u = networkFormAction(a, undefined, MAX_LIVE_ACTION);
51
+ target = u ? getActionTarget(u) : null;
52
+ while (this.entries.size >= MAX_FORM_ACTIONS || this.chars + a.length > MAX_FORM_ACTION_CHARS) {
53
+ const first = this.entries.keys().next().value;
54
+ this.entries.delete(first);
55
+ this.chars -= first.length;
56
+ }
57
+ this.entries.set(a, target);
58
+ this.chars += a.length;
59
+ }
60
+ if (target !== null)
61
+ targets.add(target);
62
+ }
63
+ return [...targets];
64
+ }
65
+ catch {
66
+ return null;
67
+ }
68
+ }
69
+ }
package/dist/resume.js CHANGED
@@ -21,6 +21,7 @@ export const STILL_WALLED = "The page the person provided still shows a wall (it
21
21
  // response decides what it is (its first bytes, then its file name); a file that is not a page is a file
22
22
  // warning, fingerprinted like a download.
23
23
  export async function fromPerson(t, p, opts, meta, agent) {
24
+ const source = p.file !== undefined ? "provided_by_person" : "provided_by_agent";
24
25
  const cap = fetchCap();
25
26
  if (p.text !== undefined && p.file !== undefined)
26
27
  throw new ProvidedError("Give the page as text or as a file, not both.");
@@ -57,7 +58,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
57
58
  const f = hashPieces(v.file, v.flags, name, src);
58
59
  close();
59
60
  close = () => { };
60
- return await fileWarning(t.url, t.url, f, meta, agent, "provided_by_person", opts);
61
+ return await fileWarning(t.url, t.url, f, meta, agent, source, opts);
61
62
  }
62
63
  }
63
64
  finally {
@@ -71,7 +72,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
71
72
  const { parsed, wall } = await parsePage(k, body, k === "pdf" ? raw : undefined, t.url, v.flags, looksHtml(null, body));
72
73
  throwIfCancelled();
73
74
  if (wall)
74
- return env(t.url, "wall", { message: STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
75
+ return env(t.url, "wall", { message: source === "provided_by_agent" ? "The page the agent provided still shows a wall. The ticket stays open for a person." : STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
75
76
  let door = "text";
76
77
  let detail = detailOf(k);
77
78
  if (k === "html" && parsed.structured_count) {
@@ -83,11 +84,11 @@ export async function fromPerson(t, p, opts, meta, agent) {
83
84
  // The receipt URL is the ticket's URL; the entry exists only in memory and is never saved.
84
85
  const entry = {
85
86
  parser: PARSER_VERSION, url: t.url, source_url: t.url, final_url: t.url, door, door_detail: detail, hash: "", content_type: "",
86
- fetched_at: now, checked_at: now, parsed, source: "provided_by_person",
87
+ fetched_at: now, checked_at: now, parsed, source,
87
88
  };
88
89
  const out = await finishPage("read", t.url, entry, false, null, opts, null, now, () => { }, meta, { fresh: false });
89
90
  if (out.status === "ok")
90
- out.message = `${PROVIDED}${out.message ? " " + out.message : ""}`;
91
+ out.message = `${source === "provided_by_agent" ? "Read from text an agent provided. Tiresias sent no request to the site for it. It was not cached or compared for changes." : PROVIDED}${out.message ? " " + out.message : ""}`;
91
92
  return out;
92
93
  }
93
94
  export const keepDays = () => {
@@ -127,10 +128,11 @@ export async function resume(id, provided, ctx = DEFAULT) {
127
128
  try {
128
129
  const opts = { question: ticket.question, section: ticket.section, render: ticket.render, translate: ticket.translate };
129
130
  const person = !!provided && (provided.text !== undefined || provided.file !== undefined);
130
- const meta = person ? { source: "provided_by_person" } : {};
131
+ const source = provided?.file !== undefined ? "provided_by_person" : "provided_by_agent";
132
+ const meta = person ? { source } : {};
131
133
  out = await guarded("resume", ticket.url, ctx, ticket.question, (m) => (person ? fromPerson(ticket, provided, opts, m, ctx.agent) : run("read", ticket.url, { ...opts, fresh: true }, m, ctx.agent)), meta, { existing: ticket.id }, !person);
132
134
  if (person)
133
- out.source = "provided_by_person";
135
+ out.source = source;
134
136
  solved = out.status === "ok" || (!person && out.status === "unchanged");
135
137
  }
136
138
  finally {
package/dist/rules.js CHANGED
@@ -132,7 +132,7 @@ export function parseRobots(text, agent = "tiresias") {
132
132
  if (field === "crawl-delay") {
133
133
  const d = parseFloat(value);
134
134
  if (d > 0)
135
- cur.delay = d;
135
+ cur.delay = Math.max(cur.delay, d);
136
136
  }
137
137
  else if (value) {
138
138
  if (++total > MAX_RULES || value.length > MAX_PATTERN) {
@@ -152,7 +152,7 @@ export function parseRobots(text, agent = "tiresias") {
152
152
  chosen = groups.filter((g) => g.agents.includes("*"));
153
153
  return {
154
154
  rules: chosen.flatMap((g) => g.rules),
155
- crawlDelay: chosen.map((g) => g.delay).find((d) => d > 0) ?? 0,
155
+ crawlDelay: chosen.reduce((delay, g) => Math.max(delay, g.delay), 0),
156
156
  sitemaps,
157
157
  overflow,
158
158
  };
@@ -253,6 +253,10 @@ async function fetchRobots(origin) {
253
253
  let reserved = budget ? ROBOTS_COST : 0;
254
254
  let charged = reserved; // a failure is charged in full (bytes may have arrived)
255
255
  const turns = new Turns(); // robots.txt holds its site's turn until read AND parsed, its delay noted (rounds 74, 86)
256
+ const holder = callContext.getStore()?.robotsPasses;
257
+ // Per FETCH, not per call: redirects and a 304 retry invalidate this fetch's pass; another site's
258
+ // fresh robots fetch in the same call can still pair. Only this fetching async context owns it.
259
+ const visit = { requests: 0 };
256
260
  try {
257
261
  try {
258
262
  const fetchRobots = () => safeFetch(origin + "/robots.txt", {
@@ -265,6 +269,11 @@ async function fetchRobots(origin) {
265
269
  // and stop at a challenged response (any status, including a redirect).
266
270
  if (hop.status === 429 || hop.status === 503)
267
271
  setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
272
+ if (hop.redirect && hop.headers.has("retry-after")) {
273
+ const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
274
+ if (seconds > 0)
275
+ throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, seconds, hop.url.href);
276
+ }
268
277
  const w = detectWall({ status: hop.status, headers: hop.headers }, true);
269
278
  if (w) {
270
279
  keep = !priv && !/(^|[\s,])no-store\b/i.test(hop.headers.get("cache-control") ?? "");
@@ -272,10 +281,14 @@ async function fetchRobots(origin) {
272
281
  }
273
282
  },
274
283
  beforeRequest: async (u, signal) => {
284
+ if (u.origin !== origin && (u.pathname !== "/robots.txt" || u.search))
285
+ throw new Refused("robots.txt redirected to another site's path, so Tiresias assumes the site says no.");
275
286
  if (cooldownLeft(u.origin) > 0)
276
287
  throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, cooldownLeft(u.origin), u.href);
288
+ visit.requests++;
289
+ visit.pass = undefined;
277
290
  turns.leaveOthers(u);
278
- await turns.take(u, spacingMs() / 1000, signal); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
291
+ await turns.take(u, spacingMs() / 1000, signal, { robots: visit }); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
279
292
  },
280
293
  });
281
294
  let res = await fetchRobots();
@@ -313,6 +326,8 @@ async function fetchRobots(origin) {
313
326
  robots = closed("robots.txt returned an unexpected HTTP 304, so Tiresias assumes the site says no.", 60_000);
314
327
  else if (res.status === 206)
315
328
  robots = closed("robots.txt came back partial (HTTP 206), so Tiresias assumes the site says no.", 60_000);
329
+ else if (res.status >= 200 && res.status < 300 && res.truncated)
330
+ robots = closed("robots.txt is larger than 512 KB, too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR);
316
331
  else if (res.status >= 200 && res.status < 300 && (res.bytes !== undefined || res.file !== undefined))
317
332
  // v0.9 hardening (Codex round 2b, HIGH): a robots.txt the fetcher took for a PDF or a file comes back with no
318
333
  // text, which would parse as "no rules". What Tiresias cannot read as text, it takes as "the site says no".
@@ -323,7 +338,7 @@ async function fetchRobots(origin) {
323
338
  ? closed("robots.txt is too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR)
324
339
  : { mode: "parsed", rules: p.rules, crawlDelay: p.crawlDelay, sitemaps: p.sitemaps, expires: Date.now() + ONE_HOUR };
325
340
  if (!p.overflow)
326
- noteDelay(origin, Math.min(p.crawlDelay, 30));
341
+ noteDelay(origin, p.crawlDelay);
327
342
  }
328
343
  else if (res.status === 429 || res.status >= 500) {
329
344
  // RFC 9309: if robots.txt cannot be reached because of a server error,
@@ -331,7 +346,9 @@ async function fetchRobots(origin) {
331
346
  // The cooldown itself was recorded when the headers arrived (onHeaders above).
332
347
  const ra = res.headers.get("retry-after") ?? undefined;
333
348
  const secs = res.status === 429 || res.status === 503 ? cooldownSeconds(res.status, ra ?? null) : undefined;
334
- robots = closed(`robots.txt returned HTTP ${res.status}, so Tiresias assumes the site says no.`, 60_000, { retryAfter: ra, retrySeconds: secs });
349
+ // A 429 or 503 is the site asking for a wait, and the reply says so: "assumes the site says no" read as a
350
+ // refusal for good, and a caller gave up on a page it could have read a minute later (field list, 2026-11-14).
351
+ robots = closed(secs !== undefined ? `robots.txt returned HTTP ${res.status}: the site asked Tiresias to wait, so nothing on it is read yet. This is a wait, not a no. Ask again in ${secs} seconds.` : `robots.txt returned HTTP ${res.status}, so Tiresias assumes the site says no.`, 60_000, { retryAfter: ra, retrySeconds: secs });
335
352
  }
336
353
  else
337
354
  robots = { mode: "missing", rules: [], crawlDelay: 0, sitemaps: [], expires: Date.now() + ONE_HOUR };
@@ -356,6 +373,8 @@ async function fetchRobots(origin) {
356
373
  }
357
374
  finally {
358
375
  turns.giveBack();
376
+ if (visit.requests === 1 && visit.pass)
377
+ holder?.pending.set(visit.pass.origin, visit.pass);
359
378
  if (budget && reserved)
360
379
  budget.release(reserved, charged);
361
380
  }
@@ -24,7 +24,7 @@ import os from "node:os";
24
24
  import path from "node:path";
25
25
  import { ConfigError, loadSafetyConfig } from "./config.js";
26
26
  import { ENV_PHRASE, envMayLoosen } from "./confirm.js";
27
- import { FetchError, Flagged, SafeBrowsingError } from "./errors.js";
27
+ import { FetchError, Flagged, SafeBrowsingError, SafetyError } from "./errors.js";
28
28
  import { politeFetch } from "./fetcher.js";
29
29
  import { callContext } from "./context.js";
30
30
  import { indexList, lookup } from "./safelist.js";
@@ -480,8 +480,8 @@ async function webRisk(url, key, priv, signal) {
480
480
  parsed = JSON.parse(res.body.toString("utf8"));
481
481
  }
482
482
  catch (e) {
483
- if (e instanceof SafeBrowsingError)
484
- throw e; // a bad test endpoint setting: loud
483
+ if (e instanceof SafeBrowsingError || e instanceof SafetyError)
484
+ throw e; // policy refusals are never "unavailable"
485
485
  if (cancelled())
486
486
  throw new FetchError("The request was cancelled.", "cancelled");
487
487
  return { state: "unavailable" }; // never keep the error (its text could hold the request URL); a timeout lands here too