@raibitsi/tiresias 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +72 -33
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +17 -13
- package/dist/cdp.js +22 -1
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +216 -68
- package/dist/config.js +75 -1
- package/dist/confirm.js +20 -5
- package/dist/cooldown.js +57 -32
- package/dist/dailylog.js +11 -5
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +9 -4
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +29 -2
- package/dist/filereply.js +7 -1
- package/dist/formflow.js +21 -3
- package/dist/forms.js +337 -42
- package/dist/getactions.js +119 -0
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +11 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +34 -27
- package/dist/keys.js +420 -18
- package/dist/local.js +537 -0
- package/dist/mcp.js +56 -25
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +117 -26
- package/dist/preformatted.js +12 -0
- package/dist/profiles.js +3 -3
- package/dist/queue.js +24 -20
- package/dist/reader.js +441 -87
- package/dist/receipts.js +7 -2
- package/dist/render.js +156 -27
- package/dist/renderactions.js +69 -0
- package/dist/resume.js +8 -6
- package/dist/rules.js +24 -5
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +15 -7
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +24 -9
- package/dist/savedlock.js +99 -5
- package/dist/settings.js +10 -3
- package/dist/shelf.js +1 -1
- package/dist/shield.js +15 -0
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +16 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +9 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +10 -2
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
package/dist/receipts.js
CHANGED
|
@@ -23,15 +23,20 @@ export function bound(r) {
|
|
|
23
23
|
// sent) or "file" (a file warning). Older lines have neither: a line with `sent: "form"` is a form, one with
|
|
24
24
|
// `file: "warning"` a file (whose `kind` field there was the FILE's kind), anything else a quote.
|
|
25
25
|
export const RECORD_VERSION = 1;
|
|
26
|
+
// Returns how many of the logged copies were cut (0: the log holds every receipt whole).
|
|
26
27
|
export function record(receipts) {
|
|
27
28
|
const now = Date.now();
|
|
28
29
|
const recorded_at = new Date(now).toISOString();
|
|
30
|
+
let cut = 0;
|
|
29
31
|
for (const r of receipts) {
|
|
30
32
|
const row = bound({ v: RECORD_VERSION, kind: "quote", ...r, recorded_at });
|
|
33
|
+
if (row.cut)
|
|
34
|
+
cut++;
|
|
31
35
|
appendLine(`receipts/${utcDay(now)}.jsonl`, row);
|
|
32
36
|
}
|
|
37
|
+
return cut;
|
|
33
38
|
}
|
|
34
|
-
export async function query(f) {
|
|
39
|
+
export async function query(f, scan = {}) {
|
|
35
40
|
const since = parseSince(f.since);
|
|
36
41
|
// The URL as typed, or as a URL parser writes it ("https://example.com" is recorded as "https://example.com/").
|
|
37
42
|
let alt;
|
|
@@ -54,7 +59,7 @@ export async function query(f) {
|
|
|
54
59
|
bound(r); // filters match the stored values; only what is returned is cut
|
|
55
60
|
r.record_fingerprint = record_fingerprint;
|
|
56
61
|
return true;
|
|
57
|
-
}, f.page ?? 1);
|
|
62
|
+
}, f.page ?? 1, 40_000, scan);
|
|
58
63
|
}
|
|
59
64
|
const MAX_SENT = 16_000;
|
|
60
65
|
export function recordSent(r) {
|
package/dist/render.js
CHANGED
|
@@ -24,14 +24,18 @@ import net from "node:net";
|
|
|
24
24
|
import os from "node:os";
|
|
25
25
|
import path from "node:path";
|
|
26
26
|
import { Cdp, CdpError } from "./cdp.js";
|
|
27
|
+
import { getActionTarget, getActionTargetsMatch } from "./getactions.js";
|
|
27
28
|
import { callContext } from "./context.js";
|
|
28
29
|
import { Busy, Slots } from "./slots.js";
|
|
29
30
|
import { ConfigError, TiresiasError } from "./errors.js";
|
|
31
|
+
import { runUntrusted } from "./sandbox.js";
|
|
32
|
+
import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_LIVE_ACTION, RenderActionCache } from "./renderactions.js";
|
|
30
33
|
import { politeFetch } from "./fetcher.js";
|
|
31
34
|
import { shutdownSignal } from "./lifetime.js";
|
|
32
35
|
import { READ_SLACK, fetchCap } from "./safety.js";
|
|
33
36
|
import { envMayLoosen, inForceConfig } from "./confirm.js";
|
|
34
37
|
import { LIMITS as INSPECT, VIEWPORT, checkInspection, inspectScript, newMark, wallVerdict } from "./inspect.js";
|
|
38
|
+
import { STRUCTURED_HEADING } from "./reader.js";
|
|
35
39
|
export const RENDER_MODES = ["off", "ask", "auto"];
|
|
36
40
|
export const DEFAULT_RENDER = "off";
|
|
37
41
|
export function parseRender(v) {
|
|
@@ -60,7 +64,7 @@ export function loadRenderMode() {
|
|
|
60
64
|
return cfg;
|
|
61
65
|
}
|
|
62
66
|
export const BLOCK_REASONS = [
|
|
63
|
-
"not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
|
|
67
|
+
"form_action", "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
|
|
64
68
|
"rules", "wall", "site_settings", "unsafe_address", "safe_browsing", "file", "too_large", "fetch_failed", "signing", "error",
|
|
65
69
|
];
|
|
66
70
|
export const RENDER_TEXT = {
|
|
@@ -75,7 +79,7 @@ export const RENDERED_TEXT = "Rendered: Tiresias ran the page's JavaScript in a
|
|
|
75
79
|
// ---- limits ----
|
|
76
80
|
export const LIMITS = {
|
|
77
81
|
ms: 20_000, // the whole render
|
|
78
|
-
quietMs: 500, //
|
|
82
|
+
quietMs: 500, // no paused request or fetch for this long after loading finishes
|
|
79
83
|
requests: 100,
|
|
80
84
|
bytes: 10 * 1024 * 1024,
|
|
81
85
|
perRequestBytes: 4 * 1024 * 1024, // what one request reserves from the budget (and may read at most)
|
|
@@ -119,9 +123,9 @@ export function findChrome() {
|
|
|
119
123
|
return (CHROME_PATHS[process.platform] ?? []).find(runnable) ?? null;
|
|
120
124
|
}
|
|
121
125
|
// ---- the empty JavaScript shell test (auto mode) ----
|
|
122
|
-
//
|
|
123
|
-
export function looksLikeShell(
|
|
124
|
-
return
|
|
126
|
+
// v1.2: short link menus can be shells; a substantial unlinked block or a useful list of links is content.
|
|
127
|
+
export function looksLikeShell(parsed, hasScript) {
|
|
128
|
+
return hasScript && parsed.text_chars < 200 && parsed.links.length < 20 && !parsed.sections.some(s => !(s.generated && s.heading === STRUCTURED_HEADING) && s.blocks.some(b => b.text.length >= 80 && b.marks.length === 0));
|
|
125
129
|
}
|
|
126
130
|
// ---- process lifetime ----
|
|
127
131
|
const live = new Map(); // every running Chrome's killer, by process id
|
|
@@ -410,6 +414,9 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
410
414
|
const chrome = findChrome();
|
|
411
415
|
if (!chrome)
|
|
412
416
|
throw new RenderUnavailable();
|
|
417
|
+
const forms = await runUntrusted({ op: "form_actions", body, url: pageUrl }, body.length);
|
|
418
|
+
const actionCache = new RenderActionCache();
|
|
419
|
+
const actions = forms.incomplete ? null : actionCache.scan(forms.actions);
|
|
413
420
|
hookExit();
|
|
414
421
|
sweepProfiles();
|
|
415
422
|
const started = Date.now();
|
|
@@ -420,6 +427,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
420
427
|
const signals = [ctl.signal, shutdownSignal, ...(outer ? [outer] : [])];
|
|
421
428
|
const signal = AbortSignal.any(signals);
|
|
422
429
|
const timer = setTimeout(() => ctl.abort(new Stop("timeout")), renderMs());
|
|
430
|
+
let navigationTimer;
|
|
431
|
+
let settleTimer;
|
|
423
432
|
let dataDir;
|
|
424
433
|
let proxy;
|
|
425
434
|
let child;
|
|
@@ -462,40 +471,62 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
462
471
|
const mainFrame = { id: "" };
|
|
463
472
|
let docServed = false;
|
|
464
473
|
let pending = 0;
|
|
474
|
+
let outstanding = 0; // bounded metadata for paused requests, including local replies and refusals
|
|
465
475
|
let lastActivity = Date.now();
|
|
466
476
|
let loaded = false;
|
|
477
|
+
let domComplete = false;
|
|
478
|
+
let refusedNavigation = false;
|
|
467
479
|
const budget = new ByteBudget(LIMITS.bytes, Math.min(fetchCap(), LIMITS.perRequestBytes));
|
|
468
480
|
const fetchFor = async (e) => {
|
|
469
481
|
const p = e.params;
|
|
470
482
|
const id = p.requestId;
|
|
471
483
|
// "Aborted", not "BlockedByClient": a refused navigation then leaves the page as it was instead of committing
|
|
472
484
|
// Chrome's own error page in its place (which would be read as the page).
|
|
473
|
-
const fail = (r) => (block(r),
|
|
485
|
+
const fail = (r) => (block(r), cmd("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
|
|
474
486
|
const url = String(p.request?.url ?? "");
|
|
475
487
|
const type = String(p.resourceType ?? "");
|
|
476
488
|
const method = String(p.request?.method ?? "");
|
|
477
489
|
// The page itself: served from the body the plain read already fetched, exactly once, in the main frame.
|
|
478
490
|
if (type === "Document" && e.sessionId === pageSession && p.frameId === mainFrame.id && !docServed && noFragment(url) === noFragment(pageUrl)) {
|
|
479
491
|
docServed = true;
|
|
480
|
-
await
|
|
492
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
|
|
481
493
|
return;
|
|
482
494
|
}
|
|
483
495
|
top.requests++;
|
|
484
496
|
if (e.sessionId !== pageSession)
|
|
485
|
-
return
|
|
486
|
-
if (type === "Document")
|
|
487
|
-
|
|
497
|
+
return await fail("worker");
|
|
498
|
+
if (type === "Document") {
|
|
499
|
+
if (p.frameId !== mainFrame.id)
|
|
500
|
+
return await fail("frame");
|
|
501
|
+
if (!loaded) {
|
|
502
|
+
// Chrome aborts the old document's load before this refusal, so its load event may never arrive.
|
|
503
|
+
refusedNavigation = true;
|
|
504
|
+
if (domComplete) {
|
|
505
|
+
loaded = true;
|
|
506
|
+
lastActivity = Date.now();
|
|
507
|
+
}
|
|
508
|
+
else if (!navigationTimer) {
|
|
509
|
+
// Give parsing a chance to finish, but return the plain read promptly if it was aborted too.
|
|
510
|
+
navigationTimer = setTimeout(() => {
|
|
511
|
+
navigationTimer = undefined;
|
|
512
|
+
if (!domComplete && !loaded)
|
|
513
|
+
ctl.abort(new Stop("navigated"));
|
|
514
|
+
}, LIMITS.quietMs);
|
|
515
|
+
}
|
|
516
|
+
}
|
|
517
|
+
return await fail("navigation");
|
|
518
|
+
}
|
|
488
519
|
if (!/^https?:/i.test(url))
|
|
489
|
-
return
|
|
520
|
+
return await fail("scheme");
|
|
490
521
|
if (type === "Preflight") {
|
|
491
522
|
// A CORS check before a GET with extra headers. The page's own headers are never sent (Tiresias sends
|
|
492
523
|
// its own), so it is answered here and the real request still goes through the rules.
|
|
493
524
|
const want = String(p.request?.headers?.["Access-Control-Request-Method"] ?? p.request?.headers?.["access-control-request-method"] ?? "");
|
|
494
525
|
if (want.toUpperCase() !== "GET")
|
|
495
|
-
return
|
|
526
|
+
return await fail("not_get");
|
|
496
527
|
const origin = String(p.request?.headers?.Origin ?? p.request?.headers?.origin ?? "*");
|
|
497
528
|
const asked = String(p.request?.headers?.["Access-Control-Request-Headers"] ?? p.request?.headers?.["access-control-request-headers"] ?? "");
|
|
498
|
-
await
|
|
529
|
+
await cmd("Fetch.fulfillRequest", {
|
|
499
530
|
requestId: id, responseCode: 204,
|
|
500
531
|
responseHeaders: [{ name: "access-control-allow-origin", value: origin.slice(0, 2048) }, { name: "access-control-allow-methods", value: "GET" }, ...(asked ? [{ name: "access-control-allow-headers", value: asked.slice(0, 2048) }] : [])],
|
|
501
532
|
}, e.sessionId).catch(() => { });
|
|
@@ -503,11 +534,86 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
503
534
|
return;
|
|
504
535
|
}
|
|
505
536
|
if (method !== "GET")
|
|
506
|
-
return
|
|
537
|
+
return await fail("not_get");
|
|
538
|
+
const target = new URL(url);
|
|
539
|
+
const actionTarget = getActionTarget(target);
|
|
540
|
+
if (actions === null || actions.some((a) => getActionTargetsMatch(a, actionTarget)))
|
|
541
|
+
return await fail("form_action");
|
|
507
542
|
if (!TYPES_FETCHED.has(type))
|
|
508
|
-
return
|
|
543
|
+
return await fail("not_needed");
|
|
544
|
+
// Check live forms too: scripts may have added a form or changed its action since the HTML arrived.
|
|
545
|
+
try {
|
|
546
|
+
const world = await cmd("Page.createIsolatedWorld", { frameId: mainFrame.id, worldName: "tiresias", grantUniveralAccess: false }, pageSession);
|
|
547
|
+
const r = await cmd("Runtime.evaluate", {
|
|
548
|
+
contextId: world.executionContextId, returnByValue: true, timeout: 1000,
|
|
549
|
+
expression: `(() => { /* get_form_actions */
|
|
550
|
+
const getter = (proto, name) => Object.getOwnPropertyDescriptor(proto, name).get;
|
|
551
|
+
const forms = getter(Document.prototype, 'forms').call(document);
|
|
552
|
+
const formCount = getter(HTMLCollection.prototype, 'length').call(forms);
|
|
553
|
+
const formAt = HTMLCollection.prototype.item;
|
|
554
|
+
// Native selectors and indexed access avoid named form properties and page wrappers.
|
|
555
|
+
// Include attributes on every element, even controls outside their form and non-controls.
|
|
556
|
+
const nodes = Document.prototype.querySelectorAll.call(document, '[formaction], [formmethod]');
|
|
557
|
+
const nodeCount = getter(NodeList.prototype, 'length').call(nodes);
|
|
558
|
+
if (formCount > ${MAX_FORM_ACTIONS} || nodeCount > ${LIMITS.elements}) return null;
|
|
559
|
+
const nodeAt = NodeList.prototype.item;
|
|
560
|
+
const attr = Element.prototype.getAttribute;
|
|
561
|
+
const localName = getter(Element.prototype, 'localName');
|
|
562
|
+
const namespace = getter(Element.prototype, 'namespaceURI');
|
|
563
|
+
const base = getter(Node.prototype, 'baseURI');
|
|
564
|
+
const page = getter(Document.prototype, 'URL').call(document);
|
|
565
|
+
const method = getter(HTMLFormElement.prototype, 'method');
|
|
566
|
+
const action = getter(HTMLFormElement.prototype, 'action');
|
|
567
|
+
const buttonAction = getter(HTMLButtonElement.prototype, 'formAction');
|
|
568
|
+
const buttonMethod = getter(HTMLButtonElement.prototype, 'formMethod');
|
|
569
|
+
const inputAction = getter(HTMLInputElement.prototype, 'formAction');
|
|
570
|
+
const inputMethod = getter(HTMLInputElement.prototype, 'formMethod');
|
|
571
|
+
const out = [];
|
|
572
|
+
let total = 0, count = formCount, getOverride = false;
|
|
573
|
+
const add = a => {
|
|
574
|
+
if (typeof a !== 'string' || a.length > ${MAX_LIVE_ACTION} || a.includes('\\0')) throw new Error('Unreadable action');
|
|
575
|
+
total += a.length;
|
|
576
|
+
if (total > ${MAX_FORM_ACTION_CHARS}) throw new Error('Action budget');
|
|
577
|
+
out.push(a);
|
|
578
|
+
};
|
|
579
|
+
for (let i = 0; i < nodeCount; i++) {
|
|
580
|
+
const n = nodeAt.call(nodes, i);
|
|
581
|
+
const name = namespace.call(n) === 'http://www.w3.org/1999/xhtml' ? localName.call(n) : '';
|
|
582
|
+
const actionGetter = name === 'button' ? buttonAction : name === 'input' ? inputAction : null;
|
|
583
|
+
const methodGetter = name === 'button' ? buttonMethod : name === 'input' ? inputMethod : null;
|
|
584
|
+
const fm = attr.call(n, 'formmethod');
|
|
585
|
+
if (fm !== null) {
|
|
586
|
+
const m = methodGetter ? methodGetter.call(n) : fm;
|
|
587
|
+
if (typeof m !== 'string') return null;
|
|
588
|
+
if (m.toLowerCase() !== 'post' && m.toLowerCase() !== 'dialog') getOverride = true;
|
|
589
|
+
}
|
|
590
|
+
const fa = attr.call(n, 'formaction');
|
|
591
|
+
if (fa !== null) {
|
|
592
|
+
if (++count > ${MAX_FORM_ACTIONS} || typeof fa !== 'string' || fa.length > ${MAX_LIVE_ACTION} || fa.includes('\\0')) return null;
|
|
593
|
+
// Non-controls have no formAction getter; apply the same URL rule ourselves.
|
|
594
|
+
const a = actionGetter ? actionGetter.call(n) : fa === '' ? page : new URL(fa, base.call(n)).href;
|
|
595
|
+
add(a);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
for (let i = 0; i < formCount; i++) {
|
|
599
|
+
const f = formAt.call(forms, i);
|
|
600
|
+
const m = method.call(f);
|
|
601
|
+
if (typeof m !== 'string') return null;
|
|
602
|
+
if (!getOverride && (m.toLowerCase() === 'post' || m.toLowerCase() === 'dialog')) continue;
|
|
603
|
+
add(action.call(f));
|
|
604
|
+
}
|
|
605
|
+
return out;
|
|
606
|
+
})()`,
|
|
607
|
+
}, pageSession);
|
|
608
|
+
const live = actionCache.scan(r.result?.value);
|
|
609
|
+
if (live === null || live.some((a) => getActionTargetsMatch(a, actionTarget)))
|
|
610
|
+
return await fail("form_action");
|
|
611
|
+
}
|
|
612
|
+
catch {
|
|
613
|
+
return await fail("form_action");
|
|
614
|
+
}
|
|
509
615
|
if (top.requests > LIMITS.requests)
|
|
510
|
-
return
|
|
616
|
+
return await fail("request_cap");
|
|
511
617
|
pending++;
|
|
512
618
|
// Codex round 1 (HIGH): the byte cap is a shared budget, RESERVED before a request starts (each request may read
|
|
513
619
|
// at most what it reserved), so requests running at once can never together exceed it. What a request did not
|
|
@@ -520,7 +626,7 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
520
626
|
// The reservation bounds what the connection may receive (headers, 1xx, compressed bytes: safety.ts Wire), less
|
|
521
627
|
// one socket read of slack, so the charge never passes it; the decoded body is held to it too.
|
|
522
628
|
if (reserved <= READ_SLACK)
|
|
523
|
-
return
|
|
629
|
+
return await fail("byte_cap");
|
|
524
630
|
const res = await callContext.run({ ...callContext.getStore(), signal, renderBudget: budget }, () => politeFetch(url, { maxBytes: reserved, wireBytes: reserved - READ_SLACK, followRedirects: false }));
|
|
525
631
|
// Chrome must see the redirect itself: module bases and cross-origin checks depend on every URL.
|
|
526
632
|
// safeFetch validated the target (including https downgrades); the next paused request goes through
|
|
@@ -533,29 +639,29 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
533
639
|
if (KEEP_HEADERS.has(k))
|
|
534
640
|
headers.push({ name: k, value: v.slice(0, 2048) });
|
|
535
641
|
top.fetched++;
|
|
536
|
-
await
|
|
642
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
|
|
537
643
|
return;
|
|
538
644
|
}
|
|
539
645
|
// A file (a program, an archive, any binary the file plan names) or a PDF is never handed to the page: without a
|
|
540
646
|
// read kind, the fetcher found a file in its first bytes.
|
|
541
647
|
if (res.file || !res.read || res.read === "pdf")
|
|
542
|
-
return
|
|
648
|
+
return await fail("file");
|
|
543
649
|
const buf = Buffer.from(res.body, "utf8");
|
|
544
650
|
// Charged what was really received (after decompression, before charset decoding: Codex round 2, a UTF-16
|
|
545
651
|
// body shrinks when decoded) or what is handed to the page, whichever is larger; never less than it fetched.
|
|
546
652
|
charge = Math.max(res.size ?? reserved, buf.length);
|
|
547
653
|
if (charge > reserved)
|
|
548
|
-
return
|
|
654
|
+
return await fail("byte_cap"); // re-encoding made it larger than what was reserved
|
|
549
655
|
const headers = [];
|
|
550
656
|
for (const [k, v] of res.headers)
|
|
551
657
|
if (KEEP_HEADERS.has(k) && k !== "content-type")
|
|
552
658
|
headers.push({ name: k, value: v.slice(0, 2048) });
|
|
553
659
|
headers.push({ name: "content-type", value: utf8Type(res.headers.get("content-type")) });
|
|
554
660
|
top.fetched++;
|
|
555
|
-
await
|
|
661
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
|
|
556
662
|
}
|
|
557
663
|
catch (err) {
|
|
558
|
-
fail(signal.aborted ? "fetch_failed" : blockOf(err));
|
|
664
|
+
await fail(signal.aborted ? "fetch_failed" : blockOf(err));
|
|
559
665
|
}
|
|
560
666
|
finally {
|
|
561
667
|
if (reserved > 0)
|
|
@@ -565,6 +671,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
565
671
|
}
|
|
566
672
|
};
|
|
567
673
|
cdp.on((e) => {
|
|
674
|
+
if (signal.aborted)
|
|
675
|
+
return;
|
|
568
676
|
const p = e.params;
|
|
569
677
|
if (e.method === "Target.attachedToTarget") {
|
|
570
678
|
const t = p.targetInfo ?? {};
|
|
@@ -585,7 +693,15 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
585
693
|
}
|
|
586
694
|
if (e.method === "Fetch.requestPaused") {
|
|
587
695
|
lastActivity = Date.now();
|
|
588
|
-
|
|
696
|
+
if (outstanding >= LIMITS.requests + 1)
|
|
697
|
+
return ctl.abort(new Stop("too_large"));
|
|
698
|
+
outstanding++;
|
|
699
|
+
void fetchFor(e).catch(() => {
|
|
700
|
+
ctl.abort(new Stop("browser_failed"));
|
|
701
|
+
}).finally(() => {
|
|
702
|
+
outstanding--;
|
|
703
|
+
lastActivity = Date.now();
|
|
704
|
+
});
|
|
589
705
|
return;
|
|
590
706
|
}
|
|
591
707
|
if (e.method === "Page.javascriptDialogOpening" && e.sessionId === pageSession) {
|
|
@@ -593,9 +709,20 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
593
709
|
cdp.send("Page.handleJavaScriptDialog", { accept: false }, pageSession).catch(() => { });
|
|
594
710
|
return;
|
|
595
711
|
}
|
|
596
|
-
if (e.method === "Page.
|
|
712
|
+
if (e.method === "Page.domContentEventFired" && e.sessionId === pageSession && docServed) {
|
|
713
|
+
domComplete = true;
|
|
714
|
+
if (refusedNavigation) {
|
|
715
|
+
loaded = true;
|
|
716
|
+
lastActivity = Date.now();
|
|
717
|
+
clearTimeout(navigationTimer);
|
|
718
|
+
navigationTimer = undefined;
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
if (e.method === "Page.loadEventFired" && e.sessionId === pageSession && docServed) {
|
|
597
722
|
loaded = true;
|
|
598
723
|
lastActivity = Date.now();
|
|
724
|
+
clearTimeout(navigationTimer);
|
|
725
|
+
navigationTimer = undefined;
|
|
599
726
|
}
|
|
600
727
|
});
|
|
601
728
|
// The browser, a throwaway context with downloads refused, and everything new attached paused.
|
|
@@ -631,14 +758,14 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
631
758
|
await cmd("Emulation.setDeviceMetricsOverride", { width: VIEWPORT.width, height: VIEWPORT.height, deviceScaleFactor: 1, mobile: false }, s).catch(() => { });
|
|
632
759
|
await cmd("Runtime.runIfWaitingForDebugger", {}, s);
|
|
633
760
|
await cmd("Page.navigate", { url: pageUrl }, s);
|
|
634
|
-
// Settled:
|
|
761
|
+
// Settled: loading finished and every paused handler and fetch has finished, then quietMs elapsed.
|
|
635
762
|
await race(new Promise((resolve) => {
|
|
636
763
|
const tick = () => {
|
|
637
764
|
if (signal.aborted)
|
|
638
765
|
return; // the race has already ended; stop polling
|
|
639
|
-
if (loaded && pending === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
|
|
766
|
+
if (loaded && pending === 0 && outstanding === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
|
|
640
767
|
return resolve();
|
|
641
|
-
setTimeout(tick, 100);
|
|
768
|
+
settleTimer = setTimeout(tick, 100);
|
|
642
769
|
};
|
|
643
770
|
tick();
|
|
644
771
|
}));
|
|
@@ -677,6 +804,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
677
804
|
}
|
|
678
805
|
finally {
|
|
679
806
|
clearTimeout(timer);
|
|
807
|
+
clearTimeout(navigationTimer);
|
|
808
|
+
clearTimeout(settleTimer);
|
|
680
809
|
ctl.abort(new Stop("cancelled"));
|
|
681
810
|
if (child) {
|
|
682
811
|
killTree(child);
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
import { getActionTarget } from "./getactions.js";
|
|
3
|
+
import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES } from "./types.js";
|
|
4
|
+
// The limits live in types.ts so the page reader (forms.ts, run in the worker) does not load this
|
|
5
|
+
// module and, through it, the whole request path.
|
|
6
|
+
export { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES };
|
|
7
|
+
// Collectors cap source actions. The static union can emit one target per source
|
|
8
|
+
// for the page URL and each web base; deduplication can only reduce this bound.
|
|
9
|
+
export const MAX_FORM_ACTION_TARGETS = MAX_FORM_ACTIONS * (MAX_FORM_BASES + 1);
|
|
10
|
+
export const MAX_LIVE_ACTION = 8192;
|
|
11
|
+
// null means a valid non-network action. Invalid, credentialed and oversized actions
|
|
12
|
+
// throw instead, including on non-web schemes, so callers fail closed.
|
|
13
|
+
export function networkFormAction(raw, base, max) {
|
|
14
|
+
if (raw.length > max)
|
|
15
|
+
throw new Error("Oversized form action");
|
|
16
|
+
const u = new URL(raw, base);
|
|
17
|
+
if (u.href.length > max || u.username || u.password)
|
|
18
|
+
throw new Error("Unsafe form action");
|
|
19
|
+
return u.protocol === "http:" || u.protocol === "https:" ? u : null;
|
|
20
|
+
}
|
|
21
|
+
// Owned by one render, shared by its static scan and all live scans. FIFO eviction
|
|
22
|
+
// bounds both keys and values by entry count and source characters. A complete scan
|
|
23
|
+
// of live actions fits in the cache. Larger static scans may evict entries, but
|
|
24
|
+
// retain every target in the scan's separate result set. Later scans reuse
|
|
25
|
+
// retained entries and safely recompute evicted ones.
|
|
26
|
+
export class RenderActionCache {
|
|
27
|
+
entries = new Map();
|
|
28
|
+
chars = 0;
|
|
29
|
+
scan(input) {
|
|
30
|
+
if (!Array.isArray(input) || input.length > MAX_FORM_ACTION_TARGETS)
|
|
31
|
+
return null;
|
|
32
|
+
let total = 0;
|
|
33
|
+
// Static input is already deduplicated, web-only and fragment-free: its
|
|
34
|
+
// resolved lengths cannot exceed the collector's max(raw, resolved) charges.
|
|
35
|
+
for (const a of input) {
|
|
36
|
+
if (typeof a !== "string" || a.length > MAX_LIVE_ACTION)
|
|
37
|
+
return null;
|
|
38
|
+
total += a.length;
|
|
39
|
+
if (total > MAX_FORM_ACTION_CHARS)
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
const targets = new Set();
|
|
43
|
+
try {
|
|
44
|
+
for (const a of new Set(input)) {
|
|
45
|
+
let target = this.entries.get(a);
|
|
46
|
+
if (target === undefined) {
|
|
47
|
+
// Both the static resolver and the native live getter return absolute
|
|
48
|
+
// URLs. A relative getter value signals failed resolution (for example
|
|
49
|
+
// under an opaque base); do not silently resolve it against the page.
|
|
50
|
+
const u = networkFormAction(a, undefined, MAX_LIVE_ACTION);
|
|
51
|
+
target = u ? getActionTarget(u) : null;
|
|
52
|
+
while (this.entries.size >= MAX_FORM_ACTIONS || this.chars + a.length > MAX_FORM_ACTION_CHARS) {
|
|
53
|
+
const first = this.entries.keys().next().value;
|
|
54
|
+
this.entries.delete(first);
|
|
55
|
+
this.chars -= first.length;
|
|
56
|
+
}
|
|
57
|
+
this.entries.set(a, target);
|
|
58
|
+
this.chars += a.length;
|
|
59
|
+
}
|
|
60
|
+
if (target !== null)
|
|
61
|
+
targets.add(target);
|
|
62
|
+
}
|
|
63
|
+
return [...targets];
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
}
|
package/dist/resume.js
CHANGED
|
@@ -21,6 +21,7 @@ export const STILL_WALLED = "The page the person provided still shows a wall (it
|
|
|
21
21
|
// response decides what it is (its first bytes, then its file name); a file that is not a page is a file
|
|
22
22
|
// warning, fingerprinted like a download.
|
|
23
23
|
export async function fromPerson(t, p, opts, meta, agent) {
|
|
24
|
+
const source = p.file !== undefined ? "provided_by_person" : "provided_by_agent";
|
|
24
25
|
const cap = fetchCap();
|
|
25
26
|
if (p.text !== undefined && p.file !== undefined)
|
|
26
27
|
throw new ProvidedError("Give the page as text or as a file, not both.");
|
|
@@ -57,7 +58,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
57
58
|
const f = hashPieces(v.file, v.flags, name, src);
|
|
58
59
|
close();
|
|
59
60
|
close = () => { };
|
|
60
|
-
return await fileWarning(t.url, t.url, f, meta, agent,
|
|
61
|
+
return await fileWarning(t.url, t.url, f, meta, agent, source, opts);
|
|
61
62
|
}
|
|
62
63
|
}
|
|
63
64
|
finally {
|
|
@@ -71,7 +72,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
71
72
|
const { parsed, wall } = await parsePage(k, body, k === "pdf" ? raw : undefined, t.url, v.flags, looksHtml(null, body));
|
|
72
73
|
throwIfCancelled();
|
|
73
74
|
if (wall)
|
|
74
|
-
return env(t.url, "wall", { message: STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
|
|
75
|
+
return env(t.url, "wall", { message: source === "provided_by_agent" ? "The page the agent provided still shows a wall. The ticket stays open for a person." : STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
|
|
75
76
|
let door = "text";
|
|
76
77
|
let detail = detailOf(k);
|
|
77
78
|
if (k === "html" && parsed.structured_count) {
|
|
@@ -83,11 +84,11 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
83
84
|
// The receipt URL is the ticket's URL; the entry exists only in memory and is never saved.
|
|
84
85
|
const entry = {
|
|
85
86
|
parser: PARSER_VERSION, url: t.url, source_url: t.url, final_url: t.url, door, door_detail: detail, hash: "", content_type: "",
|
|
86
|
-
fetched_at: now, checked_at: now, parsed, source
|
|
87
|
+
fetched_at: now, checked_at: now, parsed, source,
|
|
87
88
|
};
|
|
88
89
|
const out = await finishPage("read", t.url, entry, false, null, opts, null, now, () => { }, meta, { fresh: false });
|
|
89
90
|
if (out.status === "ok")
|
|
90
|
-
out.message = `${PROVIDED}${out.message ? " " + out.message : ""}`;
|
|
91
|
+
out.message = `${source === "provided_by_agent" ? "Read from text an agent provided. Tiresias sent no request to the site for it. It was not cached or compared for changes." : PROVIDED}${out.message ? " " + out.message : ""}`;
|
|
91
92
|
return out;
|
|
92
93
|
}
|
|
93
94
|
export const keepDays = () => {
|
|
@@ -127,10 +128,11 @@ export async function resume(id, provided, ctx = DEFAULT) {
|
|
|
127
128
|
try {
|
|
128
129
|
const opts = { question: ticket.question, section: ticket.section, render: ticket.render, translate: ticket.translate };
|
|
129
130
|
const person = !!provided && (provided.text !== undefined || provided.file !== undefined);
|
|
130
|
-
const
|
|
131
|
+
const source = provided?.file !== undefined ? "provided_by_person" : "provided_by_agent";
|
|
132
|
+
const meta = person ? { source } : {};
|
|
131
133
|
out = await guarded("resume", ticket.url, ctx, ticket.question, (m) => (person ? fromPerson(ticket, provided, opts, m, ctx.agent) : run("read", ticket.url, { ...opts, fresh: true }, m, ctx.agent)), meta, { existing: ticket.id }, !person);
|
|
132
134
|
if (person)
|
|
133
|
-
out.source =
|
|
135
|
+
out.source = source;
|
|
134
136
|
solved = out.status === "ok" || (!person && out.status === "unchanged");
|
|
135
137
|
}
|
|
136
138
|
finally {
|
package/dist/rules.js
CHANGED
|
@@ -132,7 +132,7 @@ export function parseRobots(text, agent = "tiresias") {
|
|
|
132
132
|
if (field === "crawl-delay") {
|
|
133
133
|
const d = parseFloat(value);
|
|
134
134
|
if (d > 0)
|
|
135
|
-
cur.delay = d;
|
|
135
|
+
cur.delay = Math.max(cur.delay, d);
|
|
136
136
|
}
|
|
137
137
|
else if (value) {
|
|
138
138
|
if (++total > MAX_RULES || value.length > MAX_PATTERN) {
|
|
@@ -152,7 +152,7 @@ export function parseRobots(text, agent = "tiresias") {
|
|
|
152
152
|
chosen = groups.filter((g) => g.agents.includes("*"));
|
|
153
153
|
return {
|
|
154
154
|
rules: chosen.flatMap((g) => g.rules),
|
|
155
|
-
crawlDelay: chosen.
|
|
155
|
+
crawlDelay: chosen.reduce((delay, g) => Math.max(delay, g.delay), 0),
|
|
156
156
|
sitemaps,
|
|
157
157
|
overflow,
|
|
158
158
|
};
|
|
@@ -253,6 +253,10 @@ async function fetchRobots(origin) {
|
|
|
253
253
|
let reserved = budget ? ROBOTS_COST : 0;
|
|
254
254
|
let charged = reserved; // a failure is charged in full (bytes may have arrived)
|
|
255
255
|
const turns = new Turns(); // robots.txt holds its site's turn until read AND parsed, its delay noted (rounds 74, 86)
|
|
256
|
+
const holder = callContext.getStore()?.robotsPasses;
|
|
257
|
+
// Per FETCH, not per call: redirects and a 304 retry invalidate this fetch's pass; another site's
|
|
258
|
+
// fresh robots fetch in the same call can still pair. Only this fetching async context owns it.
|
|
259
|
+
const visit = { requests: 0 };
|
|
256
260
|
try {
|
|
257
261
|
try {
|
|
258
262
|
const fetchRobots = () => safeFetch(origin + "/robots.txt", {
|
|
@@ -265,6 +269,11 @@ async function fetchRobots(origin) {
|
|
|
265
269
|
// and stop at a challenged response (any status, including a redirect).
|
|
266
270
|
if (hop.status === 429 || hop.status === 503)
|
|
267
271
|
setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
272
|
+
if (hop.redirect && hop.headers.has("retry-after")) {
|
|
273
|
+
const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
274
|
+
if (seconds > 0)
|
|
275
|
+
throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, seconds, hop.url.href);
|
|
276
|
+
}
|
|
268
277
|
const w = detectWall({ status: hop.status, headers: hop.headers }, true);
|
|
269
278
|
if (w) {
|
|
270
279
|
keep = !priv && !/(^|[\s,])no-store\b/i.test(hop.headers.get("cache-control") ?? "");
|
|
@@ -272,10 +281,14 @@ async function fetchRobots(origin) {
|
|
|
272
281
|
}
|
|
273
282
|
},
|
|
274
283
|
beforeRequest: async (u, signal) => {
|
|
284
|
+
if (u.origin !== origin && (u.pathname !== "/robots.txt" || u.search))
|
|
285
|
+
throw new Refused("robots.txt redirected to another site's path, so Tiresias assumes the site says no.");
|
|
275
286
|
if (cooldownLeft(u.origin) > 0)
|
|
276
287
|
throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, cooldownLeft(u.origin), u.href);
|
|
288
|
+
visit.requests++;
|
|
289
|
+
visit.pass = undefined;
|
|
277
290
|
turns.leaveOthers(u);
|
|
278
|
-
await turns.take(u, spacingMs() / 1000, signal); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
|
|
291
|
+
await turns.take(u, spacingMs() / 1000, signal, { robots: visit }); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
|
|
279
292
|
},
|
|
280
293
|
});
|
|
281
294
|
let res = await fetchRobots();
|
|
@@ -313,6 +326,8 @@ async function fetchRobots(origin) {
|
|
|
313
326
|
robots = closed("robots.txt returned an unexpected HTTP 304, so Tiresias assumes the site says no.", 60_000);
|
|
314
327
|
else if (res.status === 206)
|
|
315
328
|
robots = closed("robots.txt came back partial (HTTP 206), so Tiresias assumes the site says no.", 60_000);
|
|
329
|
+
else if (res.status >= 200 && res.status < 300 && res.truncated)
|
|
330
|
+
robots = closed("robots.txt is larger than 512 KB, too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR);
|
|
316
331
|
else if (res.status >= 200 && res.status < 300 && (res.bytes !== undefined || res.file !== undefined))
|
|
317
332
|
// v0.9 hardening (Codex round 2b, HIGH): a robots.txt the fetcher took for a PDF or a file comes back with no
|
|
318
333
|
// text, which would parse as "no rules". What Tiresias cannot read as text, it takes as "the site says no".
|
|
@@ -323,7 +338,7 @@ async function fetchRobots(origin) {
|
|
|
323
338
|
? closed("robots.txt is too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR)
|
|
324
339
|
: { mode: "parsed", rules: p.rules, crawlDelay: p.crawlDelay, sitemaps: p.sitemaps, expires: Date.now() + ONE_HOUR };
|
|
325
340
|
if (!p.overflow)
|
|
326
|
-
noteDelay(origin,
|
|
341
|
+
noteDelay(origin, p.crawlDelay);
|
|
327
342
|
}
|
|
328
343
|
else if (res.status === 429 || res.status >= 500) {
|
|
329
344
|
// RFC 9309: if robots.txt cannot be reached because of a server error,
|
|
@@ -331,7 +346,9 @@ async function fetchRobots(origin) {
|
|
|
331
346
|
// The cooldown itself was recorded when the headers arrived (onHeaders above).
|
|
332
347
|
const ra = res.headers.get("retry-after") ?? undefined;
|
|
333
348
|
const secs = res.status === 429 || res.status === 503 ? cooldownSeconds(res.status, ra ?? null) : undefined;
|
|
334
|
-
|
|
349
|
+
// A 429 or 503 is the site asking for a wait, and the reply says so: "assumes the site says no" read as a
|
|
350
|
+
// refusal for good, and a caller gave up on a page it could have read a minute later (field list, 2026-11-14).
|
|
351
|
+
robots = closed(secs !== undefined ? `robots.txt returned HTTP ${res.status}: the site asked Tiresias to wait, so nothing on it is read yet. This is a wait, not a no. Ask again in ${secs} seconds.` : `robots.txt returned HTTP ${res.status}, so Tiresias assumes the site says no.`, 60_000, { retryAfter: ra, retrySeconds: secs });
|
|
335
352
|
}
|
|
336
353
|
else
|
|
337
354
|
robots = { mode: "missing", rules: [], crawlDelay: 0, sitemaps: [], expires: Date.now() + ONE_HOUR };
|
|
@@ -356,6 +373,8 @@ async function fetchRobots(origin) {
|
|
|
356
373
|
}
|
|
357
374
|
finally {
|
|
358
375
|
turns.giveBack();
|
|
376
|
+
if (visit.requests === 1 && visit.pass)
|
|
377
|
+
holder?.pending.set(visit.pass.origin, visit.pass);
|
|
359
378
|
if (budget && reserved)
|
|
360
379
|
budget.release(reserved, charged);
|
|
361
380
|
}
|
package/dist/safebrowsing.js
CHANGED
|
@@ -24,7 +24,7 @@ import os from "node:os";
|
|
|
24
24
|
import path from "node:path";
|
|
25
25
|
import { ConfigError, loadSafetyConfig } from "./config.js";
|
|
26
26
|
import { ENV_PHRASE, envMayLoosen } from "./confirm.js";
|
|
27
|
-
import { FetchError, Flagged, SafeBrowsingError } from "./errors.js";
|
|
27
|
+
import { FetchError, Flagged, SafeBrowsingError, SafetyError } from "./errors.js";
|
|
28
28
|
import { politeFetch } from "./fetcher.js";
|
|
29
29
|
import { callContext } from "./context.js";
|
|
30
30
|
import { indexList, lookup } from "./safelist.js";
|
|
@@ -480,8 +480,8 @@ async function webRisk(url, key, priv, signal) {
|
|
|
480
480
|
parsed = JSON.parse(res.body.toString("utf8"));
|
|
481
481
|
}
|
|
482
482
|
catch (e) {
|
|
483
|
-
if (e instanceof SafeBrowsingError)
|
|
484
|
-
throw e; //
|
|
483
|
+
if (e instanceof SafeBrowsingError || e instanceof SafetyError)
|
|
484
|
+
throw e; // policy refusals are never "unavailable"
|
|
485
485
|
if (cancelled())
|
|
486
486
|
throw new FetchError("The request was cancelled.", "cancelled");
|
|
487
487
|
return { state: "unavailable" }; // never keep the error (its text could hold the request URL); a timeout lands here too
|