@raibitsi/tiresias 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +199 -0
  2. package/README.md +50 -27
  3. package/dist/background.js +6 -3
  4. package/dist/browsecli.js +4 -3
  5. package/dist/cdp.js +22 -1
  6. package/dist/cli.js +82 -29
  7. package/dist/config.js +32 -26
  8. package/dist/confirm.js +16 -7
  9. package/dist/cooldown.js +65 -34
  10. package/dist/dailylog.js +11 -5
  11. package/dist/engine.js +5 -4
  12. package/dist/envelope.js +20 -2
  13. package/dist/fetcher.js +5 -0
  14. package/dist/filereply.js +7 -1
  15. package/dist/files.js +4 -4
  16. package/dist/formflow.js +21 -3
  17. package/dist/forms.js +257 -34
  18. package/dist/getactions.js +99 -0
  19. package/dist/history.js +8 -5
  20. package/dist/hook.js +2 -0
  21. package/dist/hosts.js +15 -13
  22. package/dist/inside.js +2 -2
  23. package/dist/inspect.js +4 -1
  24. package/dist/keys.js +420 -18
  25. package/dist/mcp.js +10 -10
  26. package/dist/needs.js +9 -5
  27. package/dist/parentwatch.js +6 -16
  28. package/dist/pdf.js +4 -1
  29. package/dist/pdfchild.js +0 -2
  30. package/dist/pdfproc.js +26 -11
  31. package/dist/pdfwatch.js +14 -0
  32. package/dist/pipeline.js +25 -8
  33. package/dist/profiles.js +3 -3
  34. package/dist/queue.js +15 -7
  35. package/dist/reader.js +43 -18
  36. package/dist/receipts.js +2 -2
  37. package/dist/render.js +103 -24
  38. package/dist/resume.js +17 -8
  39. package/dist/rules.js +29 -7
  40. package/dist/safebrowsing.js +3 -3
  41. package/dist/safety.js +9 -3
  42. package/dist/sandbox.js +1 -1
  43. package/dist/saved.js +108 -60
  44. package/dist/savedlock.js +208 -0
  45. package/dist/settings.js +7 -3
  46. package/dist/shelf.js +1 -1
  47. package/dist/sites.js +9 -4
  48. package/dist/slips.js +43 -1
  49. package/dist/store.js +119 -25
  50. package/dist/tabs.js +1 -1
  51. package/dist/textview.js +10 -0
  52. package/dist/translate.js +22 -2
  53. package/dist/types.js +2 -2
  54. package/dist/watchflow.js +1 -2
  55. package/dist/work.js +4 -2
  56. package/package.json +2 -1
package/dist/render.js CHANGED
@@ -24,9 +24,11 @@ import net from "node:net";
24
24
  import os from "node:os";
25
25
  import path from "node:path";
26
26
  import { Cdp, CdpError } from "./cdp.js";
27
+ import { getActionTarget, getActionTargetsMatch } from "./getactions.js";
27
28
  import { callContext } from "./context.js";
28
29
  import { Busy, Slots } from "./slots.js";
29
30
  import { ConfigError, TiresiasError } from "./errors.js";
31
+ import { runUntrusted } from "./sandbox.js";
30
32
  import { politeFetch } from "./fetcher.js";
31
33
  import { shutdownSignal } from "./lifetime.js";
32
34
  import { READ_SLACK, fetchCap } from "./safety.js";
@@ -60,7 +62,7 @@ export function loadRenderMode() {
60
62
  return cfg;
61
63
  }
62
64
  export const BLOCK_REASONS = [
63
- "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
65
+ "form_action", "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
64
66
  "rules", "wall", "site_settings", "unsafe_address", "safe_browsing", "file", "too_large", "fetch_failed", "signing", "error",
65
67
  ];
66
68
  export const RENDER_TEXT = {
@@ -75,7 +77,7 @@ export const RENDERED_TEXT = "Rendered: Tiresias ran the page's JavaScript in a
75
77
  // ---- limits ----
76
78
  export const LIMITS = {
77
79
  ms: 20_000, // the whole render
78
- quietMs: 500, // the page is settled when no request has been pending this long, after its load event
80
+ quietMs: 500, // no paused request or fetch for this long after loading finishes
79
81
  requests: 100,
80
82
  bytes: 10 * 1024 * 1024,
81
83
  perRequestBytes: 4 * 1024 * 1024, // what one request reserves from the budget (and may read at most)
@@ -410,6 +412,8 @@ async function renderNow(pageUrl, body, fn) {
410
412
  const chrome = findChrome();
411
413
  if (!chrome)
412
414
  throw new RenderUnavailable();
415
+ const forms = await runUntrusted({ op: "forms", body, url: pageUrl }, body.length);
416
+ const actions = forms.forms.filter((f) => f.method === "GET" && f.action).map((f) => getActionTarget(new URL(f.action)));
413
417
  hookExit();
414
418
  sweepProfiles();
415
419
  const started = Date.now();
@@ -420,6 +424,8 @@ async function renderNow(pageUrl, body, fn) {
420
424
  const signals = [ctl.signal, shutdownSignal, ...(outer ? [outer] : [])];
421
425
  const signal = AbortSignal.any(signals);
422
426
  const timer = setTimeout(() => ctl.abort(new Stop("timeout")), renderMs());
427
+ let navigationTimer;
428
+ let settleTimer;
423
429
  let dataDir;
424
430
  let proxy;
425
431
  let child;
@@ -462,40 +468,62 @@ async function renderNow(pageUrl, body, fn) {
462
468
  const mainFrame = { id: "" };
463
469
  let docServed = false;
464
470
  let pending = 0;
471
+ let outstanding = 0; // bounded metadata for paused requests, including local replies and refusals
465
472
  let lastActivity = Date.now();
466
473
  let loaded = false;
474
+ let domComplete = false;
475
+ let refusedNavigation = false;
467
476
  const budget = new ByteBudget(LIMITS.bytes, Math.min(fetchCap(), LIMITS.perRequestBytes));
468
477
  const fetchFor = async (e) => {
469
478
  const p = e.params;
470
479
  const id = p.requestId;
471
480
  // "Aborted", not "BlockedByClient": a refused navigation then leaves the page as it was instead of committing
472
481
  // Chrome's own error page in its place (which would be read as the page).
473
- const fail = (r) => (block(r), cdp.send("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
482
+ const fail = (r) => (block(r), cmd("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
474
483
  const url = String(p.request?.url ?? "");
475
484
  const type = String(p.resourceType ?? "");
476
485
  const method = String(p.request?.method ?? "");
477
486
  // The page itself: served from the body the plain read already fetched, exactly once, in the main frame.
478
487
  if (type === "Document" && e.sessionId === pageSession && p.frameId === mainFrame.id && !docServed && noFragment(url) === noFragment(pageUrl)) {
479
488
  docServed = true;
480
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
489
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
481
490
  return;
482
491
  }
483
492
  top.requests++;
484
493
  if (e.sessionId !== pageSession)
485
- return void fail("worker");
486
- if (type === "Document")
487
- return void fail(p.frameId === mainFrame.id ? "navigation" : "frame");
494
+ return await fail("worker");
495
+ if (type === "Document") {
496
+ if (p.frameId !== mainFrame.id)
497
+ return await fail("frame");
498
+ if (!loaded) {
499
+ // Chrome aborts the old document's load before this refusal, so its load event may never arrive.
500
+ refusedNavigation = true;
501
+ if (domComplete) {
502
+ loaded = true;
503
+ lastActivity = Date.now();
504
+ }
505
+ else if (!navigationTimer) {
506
+ // Give parsing a chance to finish, but return the plain read promptly if it was aborted too.
507
+ navigationTimer = setTimeout(() => {
508
+ navigationTimer = undefined;
509
+ if (!domComplete && !loaded)
510
+ ctl.abort(new Stop("navigated"));
511
+ }, LIMITS.quietMs);
512
+ }
513
+ }
514
+ return await fail("navigation");
515
+ }
488
516
  if (!/^https?:/i.test(url))
489
- return void fail("scheme");
517
+ return await fail("scheme");
490
518
  if (type === "Preflight") {
491
519
  // A CORS check before a GET with extra headers. The page's own headers are never sent (Tiresias sends
492
520
  // its own), so it is answered here and the real request still goes through the rules.
493
521
  const want = String(p.request?.headers?.["Access-Control-Request-Method"] ?? p.request?.headers?.["access-control-request-method"] ?? "");
494
522
  if (want.toUpperCase() !== "GET")
495
- return void fail("not_get");
523
+ return await fail("not_get");
496
524
  const origin = String(p.request?.headers?.Origin ?? p.request?.headers?.origin ?? "*");
497
525
  const asked = String(p.request?.headers?.["Access-Control-Request-Headers"] ?? p.request?.headers?.["access-control-request-headers"] ?? "");
498
- await cdp.send("Fetch.fulfillRequest", {
526
+ await cmd("Fetch.fulfillRequest", {
499
527
  requestId: id, responseCode: 204,
500
528
  responseHeaders: [{ name: "access-control-allow-origin", value: origin.slice(0, 2048) }, { name: "access-control-allow-methods", value: "GET" }, ...(asked ? [{ name: "access-control-allow-headers", value: asked.slice(0, 2048) }] : [])],
501
529
  }, e.sessionId).catch(() => { });
@@ -503,11 +531,39 @@ async function renderNow(pageUrl, body, fn) {
503
531
  return;
504
532
  }
505
533
  if (method !== "GET")
506
- return void fail("not_get");
534
+ return await fail("not_get");
535
+ const target = new URL(url);
536
+ const actionTarget = getActionTarget(target);
537
+ if (forms.omitted || actions.some((a) => getActionTargetsMatch(a, actionTarget)))
538
+ return await fail("form_action");
507
539
  if (!TYPES_FETCHED.has(type))
508
- return void fail("not_needed");
540
+ return await fail("not_needed");
541
+ // Check live forms too: scripts may have added a form or changed its action since the HTML arrived.
542
+ try {
543
+ const world = await cmd("Page.createIsolatedWorld", { frameId: mainFrame.id, worldName: "tiresias", grantUniveralAccess: false }, pageSession);
544
+ const r = await cmd("Runtime.evaluate", {
545
+ contextId: world.executionContextId, returnByValue: true, timeout: 1000,
546
+ expression: `(() => { /* get_form_actions */
547
+ const forms = Object.getOwnPropertyDescriptor(Document.prototype, 'forms').get.call(document);
548
+ if (forms.length > 20) return null;
549
+ const method = Object.getOwnPropertyDescriptor(HTMLFormElement.prototype, 'method').get;
550
+ const action = Object.getOwnPropertyDescriptor(HTMLFormElement.prototype, 'action').get;
551
+ const out = [];
552
+ for (let i = 0; i < forms.length; i++) if (method.call(forms[i]).toLowerCase() === 'get') out.push(action.call(forms[i]));
553
+ return out;
554
+ })()`,
555
+ }, pageSession);
556
+ const live = r.result?.value;
557
+ if (!Array.isArray(live) || live.length > 20 || live.some((a) => typeof a !== "string" || a.length > 8192))
558
+ return await fail("form_action");
559
+ if (live.some((a) => { const u = new URL(a, pageUrl); return getActionTargetsMatch(getActionTarget(u), actionTarget); }))
560
+ return await fail("form_action");
561
+ }
562
+ catch {
563
+ return await fail("form_action");
564
+ }
509
565
  if (top.requests > LIMITS.requests)
510
- return void fail("request_cap");
566
+ return await fail("request_cap");
511
567
  pending++;
512
568
  // Codex round 1 (HIGH): the byte cap is a shared budget, RESERVED before a request starts (each request may read
513
569
  // at most what it reserved), so requests running at once can never together exceed it. What a request did not
@@ -520,7 +576,7 @@ async function renderNow(pageUrl, body, fn) {
520
576
  // The reservation bounds what the connection may receive (headers, 1xx, compressed bytes: safety.ts Wire), less
521
577
  // one socket read of slack, so the charge never passes it; the decoded body is held to it too.
522
578
  if (reserved <= READ_SLACK)
523
- return void fail("byte_cap");
579
+ return await fail("byte_cap");
524
580
  const res = await callContext.run({ ...callContext.getStore(), signal, renderBudget: budget }, () => politeFetch(url, { maxBytes: reserved, wireBytes: reserved - READ_SLACK, followRedirects: false }));
525
581
  // Chrome must see the redirect itself: module bases and cross-origin checks depend on every URL.
526
582
  // safeFetch validated the target (including https downgrades); the next paused request goes through
@@ -533,29 +589,29 @@ async function renderNow(pageUrl, body, fn) {
533
589
  if (KEEP_HEADERS.has(k))
534
590
  headers.push({ name: k, value: v.slice(0, 2048) });
535
591
  top.fetched++;
536
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
592
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
537
593
  return;
538
594
  }
539
595
  // A file (a program, an archive, any binary the file plan names) or a PDF is never handed to the page: without a
540
596
  // read kind, the fetcher found a file in its first bytes.
541
597
  if (res.file || !res.read || res.read === "pdf")
542
- return void fail("file");
598
+ return await fail("file");
543
599
  const buf = Buffer.from(res.body, "utf8");
544
600
  // Charged what was really received (after decompression, before charset decoding: Codex round 2, a UTF-16
545
601
  // body shrinks when decoded) or what is handed to the page, whichever is larger; never less than it fetched.
546
602
  charge = Math.max(res.size ?? reserved, buf.length);
547
603
  if (charge > reserved)
548
- return void fail("byte_cap"); // re-encoding made it larger than what was reserved
604
+ return await fail("byte_cap"); // re-encoding made it larger than what was reserved
549
605
  const headers = [];
550
606
  for (const [k, v] of res.headers)
551
607
  if (KEEP_HEADERS.has(k) && k !== "content-type")
552
608
  headers.push({ name: k, value: v.slice(0, 2048) });
553
609
  headers.push({ name: "content-type", value: utf8Type(res.headers.get("content-type")) });
554
610
  top.fetched++;
555
- await cdp.send("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
611
+ await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
556
612
  }
557
613
  catch (err) {
558
- fail(signal.aborted ? "fetch_failed" : blockOf(err));
614
+ await fail(signal.aborted ? "fetch_failed" : blockOf(err));
559
615
  }
560
616
  finally {
561
617
  if (reserved > 0)
@@ -565,6 +621,8 @@ async function renderNow(pageUrl, body, fn) {
565
621
  }
566
622
  };
567
623
  cdp.on((e) => {
624
+ if (signal.aborted)
625
+ return;
568
626
  const p = e.params;
569
627
  if (e.method === "Target.attachedToTarget") {
570
628
  const t = p.targetInfo ?? {};
@@ -585,7 +643,15 @@ async function renderNow(pageUrl, body, fn) {
585
643
  }
586
644
  if (e.method === "Fetch.requestPaused") {
587
645
  lastActivity = Date.now();
588
- void fetchFor(e);
646
+ if (outstanding >= LIMITS.requests + 1)
647
+ return ctl.abort(new Stop("too_large"));
648
+ outstanding++;
649
+ void fetchFor(e).catch(() => {
650
+ ctl.abort(new Stop("browser_failed"));
651
+ }).finally(() => {
652
+ outstanding--;
653
+ lastActivity = Date.now();
654
+ });
589
655
  return;
590
656
  }
591
657
  if (e.method === "Page.javascriptDialogOpening" && e.sessionId === pageSession) {
@@ -593,9 +659,20 @@ async function renderNow(pageUrl, body, fn) {
593
659
  cdp.send("Page.handleJavaScriptDialog", { accept: false }, pageSession).catch(() => { });
594
660
  return;
595
661
  }
596
- if (e.method === "Page.loadEventFired" && e.sessionId === pageSession) {
662
+ if (e.method === "Page.domContentEventFired" && e.sessionId === pageSession && docServed) {
663
+ domComplete = true;
664
+ if (refusedNavigation) {
665
+ loaded = true;
666
+ lastActivity = Date.now();
667
+ clearTimeout(navigationTimer);
668
+ navigationTimer = undefined;
669
+ }
670
+ }
671
+ if (e.method === "Page.loadEventFired" && e.sessionId === pageSession && docServed) {
597
672
  loaded = true;
598
673
  lastActivity = Date.now();
674
+ clearTimeout(navigationTimer);
675
+ navigationTimer = undefined;
599
676
  }
600
677
  });
601
678
  // The browser, a throwaway context with downloads refused, and everything new attached paused.
@@ -631,14 +708,14 @@ async function renderNow(pageUrl, body, fn) {
631
708
  await cmd("Emulation.setDeviceMetricsOverride", { width: VIEWPORT.width, height: VIEWPORT.height, deviceScaleFactor: 1, mobile: false }, s).catch(() => { });
632
709
  await cmd("Runtime.runIfWaitingForDebugger", {}, s);
633
710
  await cmd("Page.navigate", { url: pageUrl }, s);
634
- // Settled: the load event came and no request has been pending for quietMs; or the time cap.
711
+ // Settled: loading finished and every paused handler and fetch has finished, then quietMs elapsed.
635
712
  await race(new Promise((resolve) => {
636
713
  const tick = () => {
637
714
  if (signal.aborted)
638
715
  return; // the race has already ended; stop polling
639
- if (loaded && pending === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
716
+ if (loaded && pending === 0 && outstanding === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
640
717
  return resolve();
641
- setTimeout(tick, 100);
718
+ settleTimer = setTimeout(tick, 100);
642
719
  };
643
720
  tick();
644
721
  }));
@@ -677,6 +754,8 @@ async function renderNow(pageUrl, body, fn) {
677
754
  }
678
755
  finally {
679
756
  clearTimeout(timer);
757
+ clearTimeout(navigationTimer);
758
+ clearTimeout(settleTimer);
680
759
  ctl.abort(new Stop("cancelled"));
681
760
  if (child) {
682
761
  killTree(child);
package/dist/resume.js CHANGED
@@ -2,7 +2,7 @@
2
2
  // W-06: handoff tickets and resume, including a page a person provided.
3
3
  // (Split out of engine.ts in v0.5.1, behavior unchanged.)
4
4
  import path from "node:path";
5
- import { ConfigError, loadLimits } from "./config.js";
5
+ import { ConfigError, DEFAULT_LIMITS, readConfigObject } from "./config.js";
6
6
  import { classify } from "./files.js";
7
7
  import * as handoffs from "./handoffs.js";
8
8
  import { decode, fetchCap, throwIfCancelled } from "./safety.js";
@@ -21,6 +21,7 @@ export const STILL_WALLED = "The page the person provided still shows a wall (it
21
21
  // response decides what it is (its first bytes, then its file name); a file that is not a page is a file
22
22
  // warning, fingerprinted like a download.
23
23
  export async function fromPerson(t, p, opts, meta, agent) {
24
+ const source = p.file !== undefined ? "provided_by_person" : "provided_by_agent";
24
25
  const cap = fetchCap();
25
26
  if (p.text !== undefined && p.file !== undefined)
26
27
  throw new ProvidedError("Give the page as text or as a file, not both.");
@@ -57,7 +58,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
57
58
  const f = hashPieces(v.file, v.flags, name, src);
58
59
  close();
59
60
  close = () => { };
60
- return await fileWarning(t.url, t.url, f, meta, agent, "provided_by_person", opts);
61
+ return await fileWarning(t.url, t.url, f, meta, agent, source, opts);
61
62
  }
62
63
  }
63
64
  finally {
@@ -71,7 +72,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
71
72
  const { parsed, wall } = await parsePage(k, body, k === "pdf" ? raw : undefined, t.url, v.flags, looksHtml(null, body));
72
73
  throwIfCancelled();
73
74
  if (wall)
74
- return env(t.url, "wall", { message: STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
75
+ return env(t.url, "wall", { message: source === "provided_by_agent" ? "The page the agent provided still shows a wall. The ticket stays open for a person." : STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
75
76
  let door = "text";
76
77
  let detail = detailOf(k);
77
78
  if (k === "html" && parsed.structured_count) {
@@ -83,14 +84,21 @@ export async function fromPerson(t, p, opts, meta, agent) {
83
84
  // The receipt URL is the ticket's URL; the entry exists only in memory and is never saved.
84
85
  const entry = {
85
86
  parser: PARSER_VERSION, url: t.url, source_url: t.url, final_url: t.url, door, door_detail: detail, hash: "", content_type: "",
86
- fetched_at: now, checked_at: now, parsed, source: "provided_by_person",
87
+ fetched_at: now, checked_at: now, parsed, source,
87
88
  };
88
89
  const out = await finishPage("read", t.url, entry, false, null, opts, null, now, () => { }, meta, { fresh: false });
89
90
  if (out.status === "ok")
90
- out.message = `${PROVIDED}${out.message ? " " + out.message : ""}`;
91
+ out.message = `${source === "provided_by_agent" ? "Read from text an agent provided. Tiresias sent no request to the site for it. It was not cached or compared for changes." : PROVIDED}${out.message ? " " + out.message : ""}`;
91
92
  return out;
92
93
  }
93
- export const keepDays = () => loadLimits().keep_handoff_days;
94
+ export const keepDays = () => {
95
+ const value = readConfigObject()?.keep_handoff_days;
96
+ if (value === undefined)
97
+ return DEFAULT_LIMITS.keep_handoff_days;
98
+ if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1)
99
+ throw new ConfigError('config.json: "keep_handoff_days" must be a whole number of 1 or more.');
100
+ return value;
101
+ };
94
102
  // W-06: pick up a read that stopped at a wall. Without `provided`, the original read runs again
95
103
  // (same URL, question, section, render and translate options) with every rule in force: robots.txt, waits, signing and wall
96
104
  // detection; it always asks the site (a cached copy is never taken as proof the wall is gone).
@@ -120,10 +128,11 @@ export async function resume(id, provided, ctx = DEFAULT) {
120
128
  try {
121
129
  const opts = { question: ticket.question, section: ticket.section, render: ticket.render, translate: ticket.translate };
122
130
  const person = !!provided && (provided.text !== undefined || provided.file !== undefined);
123
- const meta = person ? { source: "provided_by_person" } : {};
131
+ const source = provided?.file !== undefined ? "provided_by_person" : "provided_by_agent";
132
+ const meta = person ? { source } : {};
124
133
  out = await guarded("resume", ticket.url, ctx, ticket.question, (m) => (person ? fromPerson(ticket, provided, opts, m, ctx.agent) : run("read", ticket.url, { ...opts, fresh: true }, m, ctx.agent)), meta, { existing: ticket.id }, !person);
125
134
  if (person)
126
- out.source = "provided_by_person";
135
+ out.source = source;
127
136
  solved = out.status === "ok" || (!person && out.status === "unchanged");
128
137
  }
129
138
  finally {
package/dist/rules.js CHANGED
@@ -132,7 +132,7 @@ export function parseRobots(text, agent = "tiresias") {
132
132
  if (field === "crawl-delay") {
133
133
  const d = parseFloat(value);
134
134
  if (d > 0)
135
- cur.delay = d;
135
+ cur.delay = Math.max(cur.delay, d);
136
136
  }
137
137
  else if (value) {
138
138
  if (++total > MAX_RULES || value.length > MAX_PATTERN) {
@@ -152,7 +152,7 @@ export function parseRobots(text, agent = "tiresias") {
152
152
  chosen = groups.filter((g) => g.agents.includes("*"));
153
153
  return {
154
154
  rules: chosen.flatMap((g) => g.rules),
155
- crawlDelay: chosen.map((g) => g.delay).find((d) => d > 0) ?? 0,
155
+ crawlDelay: chosen.reduce((delay, g) => Math.max(delay, g.delay), 0),
156
156
  sitemaps,
157
157
  overflow,
158
158
  };
@@ -250,11 +250,12 @@ async function fetchRobots(origin) {
250
250
  const ROBOTS_COST = ROBOTS_MAX + WIRE_EXTRA; // what its connections may receive (safety.ts Wire), plus one read of slack
251
251
  if (budget && !budget.take(ROBOTS_COST))
252
252
  throw new FetchError("The render's byte budget is spent.", "budget");
253
- let charged = budget ? ROBOTS_COST : 0; // a failure is charged in full (bytes may have arrived)
253
+ let reserved = budget ? ROBOTS_COST : 0;
254
+ let charged = reserved; // a failure is charged in full (bytes may have arrived)
254
255
  const turns = new Turns(); // robots.txt holds its site's turn until read AND parsed, its delay noted (rounds 74, 86)
255
256
  try {
256
257
  try {
257
- const res = await safeFetch(origin + "/robots.txt", {
258
+ const fetchRobots = () => safeFetch(origin + "/robots.txt", {
258
259
  maxBytes: ROBOTS_MAX,
259
260
  wireBytes: ROBOTS_COST - READ_SLACK,
260
261
  truncate: true,
@@ -264,6 +265,11 @@ async function fetchRobots(origin) {
264
265
  // and stop at a challenged response (any status, including a redirect).
265
266
  if (hop.status === 429 || hop.status === 503)
266
267
  setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
268
+ if (hop.redirect && hop.headers.has("retry-after")) {
269
+ const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
270
+ if (seconds > 0)
271
+ throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, seconds, hop.url.href);
272
+ }
267
273
  const w = detectWall({ status: hop.status, headers: hop.headers }, true);
268
274
  if (w) {
269
275
  keep = !priv && !/(^|[\s,])no-store\b/i.test(hop.headers.get("cache-control") ?? "");
@@ -271,12 +277,26 @@ async function fetchRobots(origin) {
271
277
  }
272
278
  },
273
279
  beforeRequest: async (u, signal) => {
280
+ if (u.origin !== origin && (u.pathname !== "/robots.txt" || u.search))
281
+ throw new Refused("robots.txt redirected to another site's path, so Tiresias assumes the site says no.");
274
282
  if (cooldownLeft(u.origin) > 0)
275
283
  throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, cooldownLeft(u.origin), u.href);
276
284
  turns.leaveOthers(u);
277
285
  await turns.take(u, spacingMs() / 1000, signal); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
278
286
  },
279
287
  });
288
+ let res = await fetchRobots();
289
+ if (res.status === 304) {
290
+ turns.giveBack();
291
+ if (budget) {
292
+ budget.release(ROBOTS_COST, res.size ?? ROBOTS_COST);
293
+ reserved = charged = 0;
294
+ if (!budget.take(ROBOTS_COST))
295
+ throw new FetchError("The render's byte budget is spent.", "budget");
296
+ reserved = charged = ROBOTS_COST;
297
+ }
298
+ res = await fetchRobots();
299
+ }
280
300
  if (budget)
281
301
  charged = res.size ?? ROBOTS_COST; // what its connections received
282
302
  keep = !priv && !/(^|[\s,])no-store\b/i.test(res.headers.get("cache-control") ?? "");
@@ -296,6 +316,8 @@ async function fetchRobots(origin) {
296
316
  robots = { mode: "wall", rules: [], crawlDelay: 0, sitemaps: [], wall: w, wallUrl: res.url, expires: Date.now() + 60_000 };
297
317
  // v0.9 hardening (Codex round 3b, HIGH): every success is a robots.txt to follow (RFC 9309 2.3.1.1), not
298
318
  // only a 200; a partial one (206) cannot be followed faithfully, so it means "the site says no".
319
+ else if (res.status === 304)
320
+ robots = closed("robots.txt returned an unexpected HTTP 304, so Tiresias assumes the site says no.", 60_000);
299
321
  else if (res.status === 206)
300
322
  robots = closed("robots.txt came back partial (HTTP 206), so Tiresias assumes the site says no.", 60_000);
301
323
  else if (res.status >= 200 && res.status < 300 && (res.bytes !== undefined || res.file !== undefined))
@@ -308,7 +330,7 @@ async function fetchRobots(origin) {
308
330
  ? closed("robots.txt is too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR)
309
331
  : { mode: "parsed", rules: p.rules, crawlDelay: p.crawlDelay, sitemaps: p.sitemaps, expires: Date.now() + ONE_HOUR };
310
332
  if (!p.overflow)
311
- noteDelay(origin, Math.min(p.crawlDelay, 30));
333
+ noteDelay(origin, p.crawlDelay);
312
334
  }
313
335
  else if (res.status === 429 || res.status >= 500) {
314
336
  // RFC 9309: if robots.txt cannot be reached because of a server error,
@@ -341,8 +363,8 @@ async function fetchRobots(origin) {
341
363
  }
342
364
  finally {
343
365
  turns.giveBack();
344
- if (budget)
345
- budget.release(ROBOTS_COST, charged);
366
+ if (budget && reserved)
367
+ budget.release(reserved, charged);
346
368
  }
347
369
  // Cache-Control: no-store: keep it for this one call only, never across calls.
348
370
  if (keep)
@@ -24,7 +24,7 @@ import os from "node:os";
24
24
  import path from "node:path";
25
25
  import { ConfigError, loadSafetyConfig } from "./config.js";
26
26
  import { ENV_PHRASE, envMayLoosen } from "./confirm.js";
27
- import { FetchError, Flagged, SafeBrowsingError } from "./errors.js";
27
+ import { FetchError, Flagged, SafeBrowsingError, SafetyError } from "./errors.js";
28
28
  import { politeFetch } from "./fetcher.js";
29
29
  import { callContext } from "./context.js";
30
30
  import { indexList, lookup } from "./safelist.js";
@@ -480,8 +480,8 @@ async function webRisk(url, key, priv, signal) {
480
480
  parsed = JSON.parse(res.body.toString("utf8"));
481
481
  }
482
482
  catch (e) {
483
- if (e instanceof SafeBrowsingError)
484
- throw e; // a bad test endpoint setting: loud
483
+ if (e instanceof SafeBrowsingError || e instanceof SafetyError)
484
+ throw e; // policy refusals are never "unavailable"
485
485
  if (cancelled())
486
486
  throw new FetchError("The request was cancelled.", "cancelled");
487
487
  return { state: "unavailable" }; // never keep the error (its text could hold the request URL); a timeout lands here too
package/dist/safety.js CHANGED
@@ -8,6 +8,7 @@
8
8
  // The connection's own `lookup` hook resolves the name, validates every
9
9
  // answer, and hands that same answer to the socket. TLS still verifies the
10
10
  // hostname because we connect by name, not by IP.
11
+ import { guardGetAction } from "./getactions.js";
11
12
  import { createHash } from "node:crypto";
12
13
  import dns from "node:dns/promises";
13
14
  import fs from "node:fs";
@@ -684,13 +685,15 @@ export class Wire {
684
685
  }
685
686
  }
686
687
  }
687
- // v0.7: one plain GET, no redirects, no lookup pinning, capped by its connection's bytes (Wire). For API calls to a
688
+ // v0.7: one plain GET, no redirects, guarded lookup, capped by its connection's bytes (Wire). For API calls to a
688
689
  // fixed endpoint (Google Web Risk), never for a site. The body is read up to `bodyMax`; past it, null.
689
690
  export function boundedGet(url, headers, bodyMax, wire, signal) {
690
691
  return new Promise((resolve, reject) => {
691
- const u = new URL(url);
692
+ const u = parseSafeUrl(url);
693
+ assertLiteral(u);
694
+ assertPort(u);
692
695
  const mod = u.protocol === "https:" ? https : http;
693
- const req = mod.request({ hostname: u.hostname.replace(/^\[|\]$/g, ""), port: u.port || undefined, path: u.pathname + u.search, method: "GET", headers, agent: false, signal }, (res) => {
696
+ const req = mod.request({ hostname: u.hostname.replace(/^\[|\]$/g, ""), port: u.port || undefined, path: u.pathname + u.search, method: "GET", headers, lookup: guardedLookup, agent: false, signal }, (res) => {
694
697
  if (res.statusCode !== 200) {
695
698
  res.destroy();
696
699
  return resolve({ status: res.statusCode ?? 0, body: null });
@@ -840,6 +843,9 @@ export async function safeFetch(rawUrl, opts = {}) {
840
843
  if (sent)
841
844
  (sent.requests++, signed && sent.signed++);
842
845
  };
846
+ if (!opts.submit && (hop > 0 || method === "GET"))
847
+ guardGetAction(u);
848
+ opts.beforeSend?.(u);
843
849
  let msg;
844
850
  try {
845
851
  msg = await abortable(requestOnce(u, headers, ctl.signal, onSent, hop === 0 ? method : "GET", hop === 0 ? body : undefined, wire));
package/dist/sandbox.js CHANGED
@@ -81,7 +81,7 @@ export async function runUntrusted(job, size) {
81
81
  // decoder buffers are not counted by a worker's heap limit.
82
82
  if (job.op === "pdf")
83
83
  return (await import("./pdfproc.js")).runPdfChild(job);
84
- if (size <= INLINE_MAX)
84
+ if (size <= INLINE_MAX && !(job.op === "parse" && job.kind === "plain" && job.mode === "calendar"))
85
85
  return unwrap(handle(job));
86
86
  const sig = currentSignal();
87
87
  let release;