@raibitsi/tiresias 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +199 -0
- package/README.md +50 -27
- package/dist/background.js +6 -3
- package/dist/browsecli.js +4 -3
- package/dist/cdp.js +22 -1
- package/dist/cli.js +82 -29
- package/dist/config.js +32 -26
- package/dist/confirm.js +16 -7
- package/dist/cooldown.js +65 -34
- package/dist/dailylog.js +11 -5
- package/dist/engine.js +5 -4
- package/dist/envelope.js +20 -2
- package/dist/fetcher.js +5 -0
- package/dist/filereply.js +7 -1
- package/dist/files.js +4 -4
- package/dist/formflow.js +21 -3
- package/dist/forms.js +257 -34
- package/dist/getactions.js +99 -0
- package/dist/history.js +8 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +15 -13
- package/dist/inside.js +2 -2
- package/dist/inspect.js +4 -1
- package/dist/keys.js +420 -18
- package/dist/mcp.js +10 -10
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +25 -8
- package/dist/profiles.js +3 -3
- package/dist/queue.js +15 -7
- package/dist/reader.js +43 -18
- package/dist/receipts.js +2 -2
- package/dist/render.js +103 -24
- package/dist/resume.js +17 -8
- package/dist/rules.js +29 -7
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +9 -3
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +108 -60
- package/dist/savedlock.js +208 -0
- package/dist/settings.js +7 -3
- package/dist/shelf.js +1 -1
- package/dist/sites.js +9 -4
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +10 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +2 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +4 -2
- package/package.json +2 -1
package/dist/render.js
CHANGED
|
@@ -24,9 +24,11 @@ import net from "node:net";
|
|
|
24
24
|
import os from "node:os";
|
|
25
25
|
import path from "node:path";
|
|
26
26
|
import { Cdp, CdpError } from "./cdp.js";
|
|
27
|
+
import { getActionTarget, getActionTargetsMatch } from "./getactions.js";
|
|
27
28
|
import { callContext } from "./context.js";
|
|
28
29
|
import { Busy, Slots } from "./slots.js";
|
|
29
30
|
import { ConfigError, TiresiasError } from "./errors.js";
|
|
31
|
+
import { runUntrusted } from "./sandbox.js";
|
|
30
32
|
import { politeFetch } from "./fetcher.js";
|
|
31
33
|
import { shutdownSignal } from "./lifetime.js";
|
|
32
34
|
import { READ_SLACK, fetchCap } from "./safety.js";
|
|
@@ -60,7 +62,7 @@ export function loadRenderMode() {
|
|
|
60
62
|
return cfg;
|
|
61
63
|
}
|
|
62
64
|
export const BLOCK_REASONS = [
|
|
63
|
-
"not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
|
|
65
|
+
"form_action", "not_get", "not_needed", "frame", "navigation", "worker", "popup", "scheme", "request_cap", "byte_cap",
|
|
64
66
|
"rules", "wall", "site_settings", "unsafe_address", "safe_browsing", "file", "too_large", "fetch_failed", "signing", "error",
|
|
65
67
|
];
|
|
66
68
|
export const RENDER_TEXT = {
|
|
@@ -75,7 +77,7 @@ export const RENDERED_TEXT = "Rendered: Tiresias ran the page's JavaScript in a
|
|
|
75
77
|
// ---- limits ----
|
|
76
78
|
export const LIMITS = {
|
|
77
79
|
ms: 20_000, // the whole render
|
|
78
|
-
quietMs: 500, //
|
|
80
|
+
quietMs: 500, // no paused request or fetch for this long after loading finishes
|
|
79
81
|
requests: 100,
|
|
80
82
|
bytes: 10 * 1024 * 1024,
|
|
81
83
|
perRequestBytes: 4 * 1024 * 1024, // what one request reserves from the budget (and may read at most)
|
|
@@ -410,6 +412,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
410
412
|
const chrome = findChrome();
|
|
411
413
|
if (!chrome)
|
|
412
414
|
throw new RenderUnavailable();
|
|
415
|
+
const forms = await runUntrusted({ op: "forms", body, url: pageUrl }, body.length);
|
|
416
|
+
const actions = forms.forms.filter((f) => f.method === "GET" && f.action).map((f) => getActionTarget(new URL(f.action)));
|
|
413
417
|
hookExit();
|
|
414
418
|
sweepProfiles();
|
|
415
419
|
const started = Date.now();
|
|
@@ -420,6 +424,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
420
424
|
const signals = [ctl.signal, shutdownSignal, ...(outer ? [outer] : [])];
|
|
421
425
|
const signal = AbortSignal.any(signals);
|
|
422
426
|
const timer = setTimeout(() => ctl.abort(new Stop("timeout")), renderMs());
|
|
427
|
+
let navigationTimer;
|
|
428
|
+
let settleTimer;
|
|
423
429
|
let dataDir;
|
|
424
430
|
let proxy;
|
|
425
431
|
let child;
|
|
@@ -462,40 +468,62 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
462
468
|
const mainFrame = { id: "" };
|
|
463
469
|
let docServed = false;
|
|
464
470
|
let pending = 0;
|
|
471
|
+
let outstanding = 0; // bounded metadata for paused requests, including local replies and refusals
|
|
465
472
|
let lastActivity = Date.now();
|
|
466
473
|
let loaded = false;
|
|
474
|
+
let domComplete = false;
|
|
475
|
+
let refusedNavigation = false;
|
|
467
476
|
const budget = new ByteBudget(LIMITS.bytes, Math.min(fetchCap(), LIMITS.perRequestBytes));
|
|
468
477
|
const fetchFor = async (e) => {
|
|
469
478
|
const p = e.params;
|
|
470
479
|
const id = p.requestId;
|
|
471
480
|
// "Aborted", not "BlockedByClient": a refused navigation then leaves the page as it was instead of committing
|
|
472
481
|
// Chrome's own error page in its place (which would be read as the page).
|
|
473
|
-
const fail = (r) => (block(r),
|
|
482
|
+
const fail = (r) => (block(r), cmd("Fetch.failRequest", { requestId: id, errorReason: "Aborted" }, e.sessionId).catch(() => { }));
|
|
474
483
|
const url = String(p.request?.url ?? "");
|
|
475
484
|
const type = String(p.resourceType ?? "");
|
|
476
485
|
const method = String(p.request?.method ?? "");
|
|
477
486
|
// The page itself: served from the body the plain read already fetched, exactly once, in the main frame.
|
|
478
487
|
if (type === "Document" && e.sessionId === pageSession && p.frameId === mainFrame.id && !docServed && noFragment(url) === noFragment(pageUrl)) {
|
|
479
488
|
docServed = true;
|
|
480
|
-
await
|
|
489
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: 200, responseHeaders: [{ name: "content-type", value: "text/html; charset=utf-8" }], body: b64(body) }, e.sessionId).catch(() => { });
|
|
481
490
|
return;
|
|
482
491
|
}
|
|
483
492
|
top.requests++;
|
|
484
493
|
if (e.sessionId !== pageSession)
|
|
485
|
-
return
|
|
486
|
-
if (type === "Document")
|
|
487
|
-
|
|
494
|
+
return await fail("worker");
|
|
495
|
+
if (type === "Document") {
|
|
496
|
+
if (p.frameId !== mainFrame.id)
|
|
497
|
+
return await fail("frame");
|
|
498
|
+
if (!loaded) {
|
|
499
|
+
// Chrome aborts the old document's load before this refusal, so its load event may never arrive.
|
|
500
|
+
refusedNavigation = true;
|
|
501
|
+
if (domComplete) {
|
|
502
|
+
loaded = true;
|
|
503
|
+
lastActivity = Date.now();
|
|
504
|
+
}
|
|
505
|
+
else if (!navigationTimer) {
|
|
506
|
+
// Give parsing a chance to finish, but return the plain read promptly if it was aborted too.
|
|
507
|
+
navigationTimer = setTimeout(() => {
|
|
508
|
+
navigationTimer = undefined;
|
|
509
|
+
if (!domComplete && !loaded)
|
|
510
|
+
ctl.abort(new Stop("navigated"));
|
|
511
|
+
}, LIMITS.quietMs);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
return await fail("navigation");
|
|
515
|
+
}
|
|
488
516
|
if (!/^https?:/i.test(url))
|
|
489
|
-
return
|
|
517
|
+
return await fail("scheme");
|
|
490
518
|
if (type === "Preflight") {
|
|
491
519
|
// A CORS check before a GET with extra headers. The page's own headers are never sent (Tiresias sends
|
|
492
520
|
// its own), so it is answered here and the real request still goes through the rules.
|
|
493
521
|
const want = String(p.request?.headers?.["Access-Control-Request-Method"] ?? p.request?.headers?.["access-control-request-method"] ?? "");
|
|
494
522
|
if (want.toUpperCase() !== "GET")
|
|
495
|
-
return
|
|
523
|
+
return await fail("not_get");
|
|
496
524
|
const origin = String(p.request?.headers?.Origin ?? p.request?.headers?.origin ?? "*");
|
|
497
525
|
const asked = String(p.request?.headers?.["Access-Control-Request-Headers"] ?? p.request?.headers?.["access-control-request-headers"] ?? "");
|
|
498
|
-
await
|
|
526
|
+
await cmd("Fetch.fulfillRequest", {
|
|
499
527
|
requestId: id, responseCode: 204,
|
|
500
528
|
responseHeaders: [{ name: "access-control-allow-origin", value: origin.slice(0, 2048) }, { name: "access-control-allow-methods", value: "GET" }, ...(asked ? [{ name: "access-control-allow-headers", value: asked.slice(0, 2048) }] : [])],
|
|
501
529
|
}, e.sessionId).catch(() => { });
|
|
@@ -503,11 +531,39 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
503
531
|
return;
|
|
504
532
|
}
|
|
505
533
|
if (method !== "GET")
|
|
506
|
-
return
|
|
534
|
+
return await fail("not_get");
|
|
535
|
+
const target = new URL(url);
|
|
536
|
+
const actionTarget = getActionTarget(target);
|
|
537
|
+
if (forms.omitted || actions.some((a) => getActionTargetsMatch(a, actionTarget)))
|
|
538
|
+
return await fail("form_action");
|
|
507
539
|
if (!TYPES_FETCHED.has(type))
|
|
508
|
-
return
|
|
540
|
+
return await fail("not_needed");
|
|
541
|
+
// Check live forms too: scripts may have added a form or changed its action since the HTML arrived.
|
|
542
|
+
try {
|
|
543
|
+
const world = await cmd("Page.createIsolatedWorld", { frameId: mainFrame.id, worldName: "tiresias", grantUniveralAccess: false }, pageSession);
|
|
544
|
+
const r = await cmd("Runtime.evaluate", {
|
|
545
|
+
contextId: world.executionContextId, returnByValue: true, timeout: 1000,
|
|
546
|
+
expression: `(() => { /* get_form_actions */
|
|
547
|
+
const forms = Object.getOwnPropertyDescriptor(Document.prototype, 'forms').get.call(document);
|
|
548
|
+
if (forms.length > 20) return null;
|
|
549
|
+
const method = Object.getOwnPropertyDescriptor(HTMLFormElement.prototype, 'method').get;
|
|
550
|
+
const action = Object.getOwnPropertyDescriptor(HTMLFormElement.prototype, 'action').get;
|
|
551
|
+
const out = [];
|
|
552
|
+
for (let i = 0; i < forms.length; i++) if (method.call(forms[i]).toLowerCase() === 'get') out.push(action.call(forms[i]));
|
|
553
|
+
return out;
|
|
554
|
+
})()`,
|
|
555
|
+
}, pageSession);
|
|
556
|
+
const live = r.result?.value;
|
|
557
|
+
if (!Array.isArray(live) || live.length > 20 || live.some((a) => typeof a !== "string" || a.length > 8192))
|
|
558
|
+
return await fail("form_action");
|
|
559
|
+
if (live.some((a) => { const u = new URL(a, pageUrl); return getActionTargetsMatch(getActionTarget(u), actionTarget); }))
|
|
560
|
+
return await fail("form_action");
|
|
561
|
+
}
|
|
562
|
+
catch {
|
|
563
|
+
return await fail("form_action");
|
|
564
|
+
}
|
|
509
565
|
if (top.requests > LIMITS.requests)
|
|
510
|
-
return
|
|
566
|
+
return await fail("request_cap");
|
|
511
567
|
pending++;
|
|
512
568
|
// Codex round 1 (HIGH): the byte cap is a shared budget, RESERVED before a request starts (each request may read
|
|
513
569
|
// at most what it reserved), so requests running at once can never together exceed it. What a request did not
|
|
@@ -520,7 +576,7 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
520
576
|
// The reservation bounds what the connection may receive (headers, 1xx, compressed bytes: safety.ts Wire), less
|
|
521
577
|
// one socket read of slack, so the charge never passes it; the decoded body is held to it too.
|
|
522
578
|
if (reserved <= READ_SLACK)
|
|
523
|
-
return
|
|
579
|
+
return await fail("byte_cap");
|
|
524
580
|
const res = await callContext.run({ ...callContext.getStore(), signal, renderBudget: budget }, () => politeFetch(url, { maxBytes: reserved, wireBytes: reserved - READ_SLACK, followRedirects: false }));
|
|
525
581
|
// Chrome must see the redirect itself: module bases and cross-origin checks depend on every URL.
|
|
526
582
|
// safeFetch validated the target (including https downgrades); the next paused request goes through
|
|
@@ -533,29 +589,29 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
533
589
|
if (KEEP_HEADERS.has(k))
|
|
534
590
|
headers.push({ name: k, value: v.slice(0, 2048) });
|
|
535
591
|
top.fetched++;
|
|
536
|
-
await
|
|
592
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status, responseHeaders: headers, body: "" }, e.sessionId).catch(() => { });
|
|
537
593
|
return;
|
|
538
594
|
}
|
|
539
595
|
// A file (a program, an archive, any binary the file plan names) or a PDF is never handed to the page: without a
|
|
540
596
|
// read kind, the fetcher found a file in its first bytes.
|
|
541
597
|
if (res.file || !res.read || res.read === "pdf")
|
|
542
|
-
return
|
|
598
|
+
return await fail("file");
|
|
543
599
|
const buf = Buffer.from(res.body, "utf8");
|
|
544
600
|
// Charged what was really received (after decompression, before charset decoding: Codex round 2, a UTF-16
|
|
545
601
|
// body shrinks when decoded) or what is handed to the page, whichever is larger; never less than it fetched.
|
|
546
602
|
charge = Math.max(res.size ?? reserved, buf.length);
|
|
547
603
|
if (charge > reserved)
|
|
548
|
-
return
|
|
604
|
+
return await fail("byte_cap"); // re-encoding made it larger than what was reserved
|
|
549
605
|
const headers = [];
|
|
550
606
|
for (const [k, v] of res.headers)
|
|
551
607
|
if (KEEP_HEADERS.has(k) && k !== "content-type")
|
|
552
608
|
headers.push({ name: k, value: v.slice(0, 2048) });
|
|
553
609
|
headers.push({ name: "content-type", value: utf8Type(res.headers.get("content-type")) });
|
|
554
610
|
top.fetched++;
|
|
555
|
-
await
|
|
611
|
+
await cmd("Fetch.fulfillRequest", { requestId: id, responseCode: res.status || 200, responseHeaders: headers, body: buf.toString("base64") }, e.sessionId).catch(() => { });
|
|
556
612
|
}
|
|
557
613
|
catch (err) {
|
|
558
|
-
fail(signal.aborted ? "fetch_failed" : blockOf(err));
|
|
614
|
+
await fail(signal.aborted ? "fetch_failed" : blockOf(err));
|
|
559
615
|
}
|
|
560
616
|
finally {
|
|
561
617
|
if (reserved > 0)
|
|
@@ -565,6 +621,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
565
621
|
}
|
|
566
622
|
};
|
|
567
623
|
cdp.on((e) => {
|
|
624
|
+
if (signal.aborted)
|
|
625
|
+
return;
|
|
568
626
|
const p = e.params;
|
|
569
627
|
if (e.method === "Target.attachedToTarget") {
|
|
570
628
|
const t = p.targetInfo ?? {};
|
|
@@ -585,7 +643,15 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
585
643
|
}
|
|
586
644
|
if (e.method === "Fetch.requestPaused") {
|
|
587
645
|
lastActivity = Date.now();
|
|
588
|
-
|
|
646
|
+
if (outstanding >= LIMITS.requests + 1)
|
|
647
|
+
return ctl.abort(new Stop("too_large"));
|
|
648
|
+
outstanding++;
|
|
649
|
+
void fetchFor(e).catch(() => {
|
|
650
|
+
ctl.abort(new Stop("browser_failed"));
|
|
651
|
+
}).finally(() => {
|
|
652
|
+
outstanding--;
|
|
653
|
+
lastActivity = Date.now();
|
|
654
|
+
});
|
|
589
655
|
return;
|
|
590
656
|
}
|
|
591
657
|
if (e.method === "Page.javascriptDialogOpening" && e.sessionId === pageSession) {
|
|
@@ -593,9 +659,20 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
593
659
|
cdp.send("Page.handleJavaScriptDialog", { accept: false }, pageSession).catch(() => { });
|
|
594
660
|
return;
|
|
595
661
|
}
|
|
596
|
-
if (e.method === "Page.
|
|
662
|
+
if (e.method === "Page.domContentEventFired" && e.sessionId === pageSession && docServed) {
|
|
663
|
+
domComplete = true;
|
|
664
|
+
if (refusedNavigation) {
|
|
665
|
+
loaded = true;
|
|
666
|
+
lastActivity = Date.now();
|
|
667
|
+
clearTimeout(navigationTimer);
|
|
668
|
+
navigationTimer = undefined;
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
if (e.method === "Page.loadEventFired" && e.sessionId === pageSession && docServed) {
|
|
597
672
|
loaded = true;
|
|
598
673
|
lastActivity = Date.now();
|
|
674
|
+
clearTimeout(navigationTimer);
|
|
675
|
+
navigationTimer = undefined;
|
|
599
676
|
}
|
|
600
677
|
});
|
|
601
678
|
// The browser, a throwaway context with downloads refused, and everything new attached paused.
|
|
@@ -631,14 +708,14 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
631
708
|
await cmd("Emulation.setDeviceMetricsOverride", { width: VIEWPORT.width, height: VIEWPORT.height, deviceScaleFactor: 1, mobile: false }, s).catch(() => { });
|
|
632
709
|
await cmd("Runtime.runIfWaitingForDebugger", {}, s);
|
|
633
710
|
await cmd("Page.navigate", { url: pageUrl }, s);
|
|
634
|
-
// Settled:
|
|
711
|
+
// Settled: loading finished and every paused handler and fetch has finished, then quietMs elapsed.
|
|
635
712
|
await race(new Promise((resolve) => {
|
|
636
713
|
const tick = () => {
|
|
637
714
|
if (signal.aborted)
|
|
638
715
|
return; // the race has already ended; stop polling
|
|
639
|
-
if (loaded && pending === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
|
|
716
|
+
if (loaded && pending === 0 && outstanding === 0 && Date.now() - lastActivity >= LIMITS.quietMs)
|
|
640
717
|
return resolve();
|
|
641
|
-
setTimeout(tick, 100);
|
|
718
|
+
settleTimer = setTimeout(tick, 100);
|
|
642
719
|
};
|
|
643
720
|
tick();
|
|
644
721
|
}));
|
|
@@ -677,6 +754,8 @@ async function renderNow(pageUrl, body, fn) {
|
|
|
677
754
|
}
|
|
678
755
|
finally {
|
|
679
756
|
clearTimeout(timer);
|
|
757
|
+
clearTimeout(navigationTimer);
|
|
758
|
+
clearTimeout(settleTimer);
|
|
680
759
|
ctl.abort(new Stop("cancelled"));
|
|
681
760
|
if (child) {
|
|
682
761
|
killTree(child);
|
package/dist/resume.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// W-06: handoff tickets and resume, including a page a person provided.
|
|
3
3
|
// (Split out of engine.ts in v0.5.1, behavior unchanged.)
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { ConfigError,
|
|
5
|
+
import { ConfigError, DEFAULT_LIMITS, readConfigObject } from "./config.js";
|
|
6
6
|
import { classify } from "./files.js";
|
|
7
7
|
import * as handoffs from "./handoffs.js";
|
|
8
8
|
import { decode, fetchCap, throwIfCancelled } from "./safety.js";
|
|
@@ -21,6 +21,7 @@ export const STILL_WALLED = "The page the person provided still shows a wall (it
|
|
|
21
21
|
// response decides what it is (its first bytes, then its file name); a file that is not a page is a file
|
|
22
22
|
// warning, fingerprinted like a download.
|
|
23
23
|
export async function fromPerson(t, p, opts, meta, agent) {
|
|
24
|
+
const source = p.file !== undefined ? "provided_by_person" : "provided_by_agent";
|
|
24
25
|
const cap = fetchCap();
|
|
25
26
|
if (p.text !== undefined && p.file !== undefined)
|
|
26
27
|
throw new ProvidedError("Give the page as text or as a file, not both.");
|
|
@@ -57,7 +58,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
57
58
|
const f = hashPieces(v.file, v.flags, name, src);
|
|
58
59
|
close();
|
|
59
60
|
close = () => { };
|
|
60
|
-
return await fileWarning(t.url, t.url, f, meta, agent,
|
|
61
|
+
return await fileWarning(t.url, t.url, f, meta, agent, source, opts);
|
|
61
62
|
}
|
|
62
63
|
}
|
|
63
64
|
finally {
|
|
@@ -71,7 +72,7 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
71
72
|
const { parsed, wall } = await parsePage(k, body, k === "pdf" ? raw : undefined, t.url, v.flags, looksHtml(null, body));
|
|
72
73
|
throwIfCancelled();
|
|
73
74
|
if (wall)
|
|
74
|
-
return env(t.url, "wall", { message: STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
|
|
75
|
+
return env(t.url, "wall", { message: source === "provided_by_agent" ? "The page the agent provided still shows a wall. The ticket stays open for a person." : STILL_WALLED, handoff: { reason: wall.code, explanation: wall.explanation }, uc: { handoff: { open_this: t.open_this ?? t.url } } });
|
|
75
76
|
let door = "text";
|
|
76
77
|
let detail = detailOf(k);
|
|
77
78
|
if (k === "html" && parsed.structured_count) {
|
|
@@ -83,14 +84,21 @@ export async function fromPerson(t, p, opts, meta, agent) {
|
|
|
83
84
|
// The receipt URL is the ticket's URL; the entry exists only in memory and is never saved.
|
|
84
85
|
const entry = {
|
|
85
86
|
parser: PARSER_VERSION, url: t.url, source_url: t.url, final_url: t.url, door, door_detail: detail, hash: "", content_type: "",
|
|
86
|
-
fetched_at: now, checked_at: now, parsed, source
|
|
87
|
+
fetched_at: now, checked_at: now, parsed, source,
|
|
87
88
|
};
|
|
88
89
|
const out = await finishPage("read", t.url, entry, false, null, opts, null, now, () => { }, meta, { fresh: false });
|
|
89
90
|
if (out.status === "ok")
|
|
90
|
-
out.message = `${PROVIDED}${out.message ? " " + out.message : ""}`;
|
|
91
|
+
out.message = `${source === "provided_by_agent" ? "Read from text an agent provided. Tiresias sent no request to the site for it. It was not cached or compared for changes." : PROVIDED}${out.message ? " " + out.message : ""}`;
|
|
91
92
|
return out;
|
|
92
93
|
}
|
|
93
|
-
export const keepDays = () =>
|
|
94
|
+
export const keepDays = () => {
|
|
95
|
+
const value = readConfigObject()?.keep_handoff_days;
|
|
96
|
+
if (value === undefined)
|
|
97
|
+
return DEFAULT_LIMITS.keep_handoff_days;
|
|
98
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1)
|
|
99
|
+
throw new ConfigError('config.json: "keep_handoff_days" must be a whole number of 1 or more.');
|
|
100
|
+
return value;
|
|
101
|
+
};
|
|
94
102
|
// W-06: pick up a read that stopped at a wall. Without `provided`, the original read runs again
|
|
95
103
|
// (same URL, question, section, render and translate options) with every rule in force: robots.txt, waits, signing and wall
|
|
96
104
|
// detection; it always asks the site (a cached copy is never taken as proof the wall is gone).
|
|
@@ -120,10 +128,11 @@ export async function resume(id, provided, ctx = DEFAULT) {
|
|
|
120
128
|
try {
|
|
121
129
|
const opts = { question: ticket.question, section: ticket.section, render: ticket.render, translate: ticket.translate };
|
|
122
130
|
const person = !!provided && (provided.text !== undefined || provided.file !== undefined);
|
|
123
|
-
const
|
|
131
|
+
const source = provided?.file !== undefined ? "provided_by_person" : "provided_by_agent";
|
|
132
|
+
const meta = person ? { source } : {};
|
|
124
133
|
out = await guarded("resume", ticket.url, ctx, ticket.question, (m) => (person ? fromPerson(ticket, provided, opts, m, ctx.agent) : run("read", ticket.url, { ...opts, fresh: true }, m, ctx.agent)), meta, { existing: ticket.id }, !person);
|
|
125
134
|
if (person)
|
|
126
|
-
out.source =
|
|
135
|
+
out.source = source;
|
|
127
136
|
solved = out.status === "ok" || (!person && out.status === "unchanged");
|
|
128
137
|
}
|
|
129
138
|
finally {
|
package/dist/rules.js
CHANGED
|
@@ -132,7 +132,7 @@ export function parseRobots(text, agent = "tiresias") {
|
|
|
132
132
|
if (field === "crawl-delay") {
|
|
133
133
|
const d = parseFloat(value);
|
|
134
134
|
if (d > 0)
|
|
135
|
-
cur.delay = d;
|
|
135
|
+
cur.delay = Math.max(cur.delay, d);
|
|
136
136
|
}
|
|
137
137
|
else if (value) {
|
|
138
138
|
if (++total > MAX_RULES || value.length > MAX_PATTERN) {
|
|
@@ -152,7 +152,7 @@ export function parseRobots(text, agent = "tiresias") {
|
|
|
152
152
|
chosen = groups.filter((g) => g.agents.includes("*"));
|
|
153
153
|
return {
|
|
154
154
|
rules: chosen.flatMap((g) => g.rules),
|
|
155
|
-
crawlDelay: chosen.
|
|
155
|
+
crawlDelay: chosen.reduce((delay, g) => Math.max(delay, g.delay), 0),
|
|
156
156
|
sitemaps,
|
|
157
157
|
overflow,
|
|
158
158
|
};
|
|
@@ -250,11 +250,12 @@ async function fetchRobots(origin) {
|
|
|
250
250
|
const ROBOTS_COST = ROBOTS_MAX + WIRE_EXTRA; // what its connections may receive (safety.ts Wire), plus one read of slack
|
|
251
251
|
if (budget && !budget.take(ROBOTS_COST))
|
|
252
252
|
throw new FetchError("The render's byte budget is spent.", "budget");
|
|
253
|
-
let
|
|
253
|
+
let reserved = budget ? ROBOTS_COST : 0;
|
|
254
|
+
let charged = reserved; // a failure is charged in full (bytes may have arrived)
|
|
254
255
|
const turns = new Turns(); // robots.txt holds its site's turn until read AND parsed, its delay noted (rounds 74, 86)
|
|
255
256
|
try {
|
|
256
257
|
try {
|
|
257
|
-
const
|
|
258
|
+
const fetchRobots = () => safeFetch(origin + "/robots.txt", {
|
|
258
259
|
maxBytes: ROBOTS_MAX,
|
|
259
260
|
wireBytes: ROBOTS_COST - READ_SLACK,
|
|
260
261
|
truncate: true,
|
|
@@ -264,6 +265,11 @@ async function fetchRobots(origin) {
|
|
|
264
265
|
// and stop at a challenged response (any status, including a redirect).
|
|
265
266
|
if (hop.status === 429 || hop.status === 503)
|
|
266
267
|
setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
268
|
+
if (hop.redirect && hop.headers.has("retry-after")) {
|
|
269
|
+
const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
270
|
+
if (seconds > 0)
|
|
271
|
+
throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, seconds, hop.url.href);
|
|
272
|
+
}
|
|
267
273
|
const w = detectWall({ status: hop.status, headers: hop.headers }, true);
|
|
268
274
|
if (w) {
|
|
269
275
|
keep = !priv && !/(^|[\s,])no-store\b/i.test(hop.headers.get("cache-control") ?? "");
|
|
@@ -271,12 +277,26 @@ async function fetchRobots(origin) {
|
|
|
271
277
|
}
|
|
272
278
|
},
|
|
273
279
|
beforeRequest: async (u, signal) => {
|
|
280
|
+
if (u.origin !== origin && (u.pathname !== "/robots.txt" || u.search))
|
|
281
|
+
throw new Refused("robots.txt redirected to another site's path, so Tiresias assumes the site says no.");
|
|
274
282
|
if (cooldownLeft(u.origin) > 0)
|
|
275
283
|
throw new Refused("The site asked Tiresias to wait (Retry-After). Not calling it again yet.", undefined, cooldownLeft(u.origin), u.href);
|
|
276
284
|
turns.leaveOthers(u);
|
|
277
285
|
await turns.take(u, spacingMs() / 1000, signal); // v0.5.1: robots.txt waits its turn too (v0.6: and a site's own spacing, in the queue)
|
|
278
286
|
},
|
|
279
287
|
});
|
|
288
|
+
let res = await fetchRobots();
|
|
289
|
+
if (res.status === 304) {
|
|
290
|
+
turns.giveBack();
|
|
291
|
+
if (budget) {
|
|
292
|
+
budget.release(ROBOTS_COST, res.size ?? ROBOTS_COST);
|
|
293
|
+
reserved = charged = 0;
|
|
294
|
+
if (!budget.take(ROBOTS_COST))
|
|
295
|
+
throw new FetchError("The render's byte budget is spent.", "budget");
|
|
296
|
+
reserved = charged = ROBOTS_COST;
|
|
297
|
+
}
|
|
298
|
+
res = await fetchRobots();
|
|
299
|
+
}
|
|
280
300
|
if (budget)
|
|
281
301
|
charged = res.size ?? ROBOTS_COST; // what its connections received
|
|
282
302
|
keep = !priv && !/(^|[\s,])no-store\b/i.test(res.headers.get("cache-control") ?? "");
|
|
@@ -296,6 +316,8 @@ async function fetchRobots(origin) {
|
|
|
296
316
|
robots = { mode: "wall", rules: [], crawlDelay: 0, sitemaps: [], wall: w, wallUrl: res.url, expires: Date.now() + 60_000 };
|
|
297
317
|
// v0.9 hardening (Codex round 3b, HIGH): every success is a robots.txt to follow (RFC 9309 2.3.1.1), not
|
|
298
318
|
// only a 200; a partial one (206) cannot be followed faithfully, so it means "the site says no".
|
|
319
|
+
else if (res.status === 304)
|
|
320
|
+
robots = closed("robots.txt returned an unexpected HTTP 304, so Tiresias assumes the site says no.", 60_000);
|
|
299
321
|
else if (res.status === 206)
|
|
300
322
|
robots = closed("robots.txt came back partial (HTTP 206), so Tiresias assumes the site says no.", 60_000);
|
|
301
323
|
else if (res.status >= 200 && res.status < 300 && (res.bytes !== undefined || res.file !== undefined))
|
|
@@ -308,7 +330,7 @@ async function fetchRobots(origin) {
|
|
|
308
330
|
? closed("robots.txt is too large to apply faithfully, so Tiresias assumes the site says no.", ONE_HOUR)
|
|
309
331
|
: { mode: "parsed", rules: p.rules, crawlDelay: p.crawlDelay, sitemaps: p.sitemaps, expires: Date.now() + ONE_HOUR };
|
|
310
332
|
if (!p.overflow)
|
|
311
|
-
noteDelay(origin,
|
|
333
|
+
noteDelay(origin, p.crawlDelay);
|
|
312
334
|
}
|
|
313
335
|
else if (res.status === 429 || res.status >= 500) {
|
|
314
336
|
// RFC 9309: if robots.txt cannot be reached because of a server error,
|
|
@@ -341,8 +363,8 @@ async function fetchRobots(origin) {
|
|
|
341
363
|
}
|
|
342
364
|
finally {
|
|
343
365
|
turns.giveBack();
|
|
344
|
-
if (budget)
|
|
345
|
-
budget.release(
|
|
366
|
+
if (budget && reserved)
|
|
367
|
+
budget.release(reserved, charged);
|
|
346
368
|
}
|
|
347
369
|
// Cache-Control: no-store: keep it for this one call only, never across calls.
|
|
348
370
|
if (keep)
|
package/dist/safebrowsing.js
CHANGED
|
@@ -24,7 +24,7 @@ import os from "node:os";
|
|
|
24
24
|
import path from "node:path";
|
|
25
25
|
import { ConfigError, loadSafetyConfig } from "./config.js";
|
|
26
26
|
import { ENV_PHRASE, envMayLoosen } from "./confirm.js";
|
|
27
|
-
import { FetchError, Flagged, SafeBrowsingError } from "./errors.js";
|
|
27
|
+
import { FetchError, Flagged, SafeBrowsingError, SafetyError } from "./errors.js";
|
|
28
28
|
import { politeFetch } from "./fetcher.js";
|
|
29
29
|
import { callContext } from "./context.js";
|
|
30
30
|
import { indexList, lookup } from "./safelist.js";
|
|
@@ -480,8 +480,8 @@ async function webRisk(url, key, priv, signal) {
|
|
|
480
480
|
parsed = JSON.parse(res.body.toString("utf8"));
|
|
481
481
|
}
|
|
482
482
|
catch (e) {
|
|
483
|
-
if (e instanceof SafeBrowsingError)
|
|
484
|
-
throw e; //
|
|
483
|
+
if (e instanceof SafeBrowsingError || e instanceof SafetyError)
|
|
484
|
+
throw e; // policy refusals are never "unavailable"
|
|
485
485
|
if (cancelled())
|
|
486
486
|
throw new FetchError("The request was cancelled.", "cancelled");
|
|
487
487
|
return { state: "unavailable" }; // never keep the error (its text could hold the request URL); a timeout lands here too
|
package/dist/safety.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// The connection's own `lookup` hook resolves the name, validates every
|
|
9
9
|
// answer, and hands that same answer to the socket. TLS still verifies the
|
|
10
10
|
// hostname because we connect by name, not by IP.
|
|
11
|
+
import { guardGetAction } from "./getactions.js";
|
|
11
12
|
import { createHash } from "node:crypto";
|
|
12
13
|
import dns from "node:dns/promises";
|
|
13
14
|
import fs from "node:fs";
|
|
@@ -684,13 +685,15 @@ export class Wire {
|
|
|
684
685
|
}
|
|
685
686
|
}
|
|
686
687
|
}
|
|
687
|
-
// v0.7: one plain GET, no redirects,
|
|
688
|
+
// v0.7: one plain GET, no redirects, guarded lookup, capped by its connection's bytes (Wire). For API calls to a
|
|
688
689
|
// fixed endpoint (Google Web Risk), never for a site. The body is read up to `bodyMax`; past it, null.
|
|
689
690
|
export function boundedGet(url, headers, bodyMax, wire, signal) {
|
|
690
691
|
return new Promise((resolve, reject) => {
|
|
691
|
-
const u =
|
|
692
|
+
const u = parseSafeUrl(url);
|
|
693
|
+
assertLiteral(u);
|
|
694
|
+
assertPort(u);
|
|
692
695
|
const mod = u.protocol === "https:" ? https : http;
|
|
693
|
-
const req = mod.request({ hostname: u.hostname.replace(/^\[|\]$/g, ""), port: u.port || undefined, path: u.pathname + u.search, method: "GET", headers, agent: false, signal }, (res) => {
|
|
696
|
+
const req = mod.request({ hostname: u.hostname.replace(/^\[|\]$/g, ""), port: u.port || undefined, path: u.pathname + u.search, method: "GET", headers, lookup: guardedLookup, agent: false, signal }, (res) => {
|
|
694
697
|
if (res.statusCode !== 200) {
|
|
695
698
|
res.destroy();
|
|
696
699
|
return resolve({ status: res.statusCode ?? 0, body: null });
|
|
@@ -840,6 +843,9 @@ export async function safeFetch(rawUrl, opts = {}) {
|
|
|
840
843
|
if (sent)
|
|
841
844
|
(sent.requests++, signed && sent.signed++);
|
|
842
845
|
};
|
|
846
|
+
if (!opts.submit && (hop > 0 || method === "GET"))
|
|
847
|
+
guardGetAction(u);
|
|
848
|
+
opts.beforeSend?.(u);
|
|
843
849
|
let msg;
|
|
844
850
|
try {
|
|
845
851
|
msg = await abortable(requestOnce(u, headers, ctl.signal, onSent, hop === 0 ? method : "GET", hop === 0 ? body : undefined, wire));
|
package/dist/sandbox.js
CHANGED
|
@@ -81,7 +81,7 @@ export async function runUntrusted(job, size) {
|
|
|
81
81
|
// decoder buffers are not counted by a worker's heap limit.
|
|
82
82
|
if (job.op === "pdf")
|
|
83
83
|
return (await import("./pdfproc.js")).runPdfChild(job);
|
|
84
|
-
if (size <= INLINE_MAX)
|
|
84
|
+
if (size <= INLINE_MAX && !(job.op === "parse" && job.kind === "plain" && job.mode === "calendar"))
|
|
85
85
|
return unwrap(handle(job));
|
|
86
86
|
const sig = currentSignal();
|
|
87
87
|
let release;
|