@raibitsi/tiresias 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +199 -0
- package/README.md +50 -27
- package/dist/background.js +6 -3
- package/dist/browsecli.js +4 -3
- package/dist/cdp.js +22 -1
- package/dist/cli.js +82 -29
- package/dist/config.js +32 -26
- package/dist/confirm.js +16 -7
- package/dist/cooldown.js +65 -34
- package/dist/dailylog.js +11 -5
- package/dist/engine.js +5 -4
- package/dist/envelope.js +20 -2
- package/dist/fetcher.js +5 -0
- package/dist/filereply.js +7 -1
- package/dist/files.js +4 -4
- package/dist/formflow.js +21 -3
- package/dist/forms.js +257 -34
- package/dist/getactions.js +99 -0
- package/dist/history.js +8 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +15 -13
- package/dist/inside.js +2 -2
- package/dist/inspect.js +4 -1
- package/dist/keys.js +420 -18
- package/dist/mcp.js +10 -10
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +25 -8
- package/dist/profiles.js +3 -3
- package/dist/queue.js +15 -7
- package/dist/reader.js +43 -18
- package/dist/receipts.js +2 -2
- package/dist/render.js +103 -24
- package/dist/resume.js +17 -8
- package/dist/rules.js +29 -7
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +9 -3
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +108 -60
- package/dist/savedlock.js +208 -0
- package/dist/settings.js +7 -3
- package/dist/shelf.js +1 -1
- package/dist/sites.js +9 -4
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +10 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +2 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +4 -2
- package/package.json +2 -1
package/dist/mcp.js
CHANGED
|
@@ -144,9 +144,9 @@ export function createServer() {
|
|
|
144
144
|
since: z.string().optional().describe("A date like 2026-10-01, or 7d / 24h"),
|
|
145
145
|
page: PAGE,
|
|
146
146
|
},
|
|
147
|
-
}, async ({ url, since, page }) => {
|
|
147
|
+
}, async ({ url, since, page }, extra) => {
|
|
148
148
|
try {
|
|
149
|
-
const r = await engine.queryReceipts({ url, since, page });
|
|
149
|
+
const r = await engine.queryReceipts({ url, since, page }, { signal: extra.signal, deadline: Date.now() + 60_000 });
|
|
150
150
|
// Counts and page numbers are Tiresias's own; the records came from the web.
|
|
151
151
|
return text(receiptsEnvelope(url ?? "", r));
|
|
152
152
|
}
|
|
@@ -164,9 +164,9 @@ export function createServer() {
|
|
|
164
164
|
agent: z.string().optional().describe("Only this agent, e.g. claude-code"),
|
|
165
165
|
page: PAGE,
|
|
166
166
|
},
|
|
167
|
-
}, async ({ since, site, agent, page }) => {
|
|
167
|
+
}, async ({ since, site, agent, page }, extra) => {
|
|
168
168
|
try {
|
|
169
|
-
const r = await engine.queryHistory({ since, site, agent, page });
|
|
169
|
+
const r = await engine.queryHistory({ since, site, agent, page }, { signal: extra.signal, deadline: Date.now() + 60_000 });
|
|
170
170
|
// Records hold URLs, agent names and purposes from outside, so they sit inside untrusted_content.
|
|
171
171
|
return text(historyEnvelope(r));
|
|
172
172
|
}
|
|
@@ -182,9 +182,9 @@ export function createServer() {
|
|
|
182
182
|
day: z.string().optional().describe("The day, YYYY-MM-DD (local time). Default today."),
|
|
183
183
|
page: PAGE,
|
|
184
184
|
},
|
|
185
|
-
}, async ({ day, page }) => {
|
|
185
|
+
}, async ({ day, page }, extra) => {
|
|
186
186
|
try {
|
|
187
|
-
const r = await engine.dailyLog(day, page);
|
|
187
|
+
const r = await engine.dailyLog(day, page, { signal: extra.signal, deadline: Date.now() + 60_000 });
|
|
188
188
|
// URLs and agent text in the log came from outside, so the page sits inside untrusted_content.
|
|
189
189
|
return text(listEnv("", { day: r.day, entries: r.entries, log: r.text }, { page: r.page, pages: r.pages, total: r.entries }));
|
|
190
190
|
}
|
|
@@ -245,7 +245,7 @@ export function createServer() {
|
|
|
245
245
|
server.registerTool("resume", {
|
|
246
246
|
title: "Resume after a wall",
|
|
247
247
|
annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true },
|
|
248
|
-
description: "Pick up a read that stopped at a wall, by its handoff ticket, once a person is through. Without text: runs the same read again (a wall still there comes back with the same ticket). With text:
|
|
248
|
+
description: "Pick up a read that stopped at a wall, by its handoff ticket, once a person is through. Without text: runs the same read again (a wall still there comes back with the same ticket). With text: agent-supplied page text, read with NO request to the site (source: provided_by_agent); Tiresias cannot verify that a person supplied it.",
|
|
249
249
|
inputSchema: {
|
|
250
250
|
ticket: z.string().describe("The ticket id from handoff.ticket"),
|
|
251
251
|
text: z.string().optional().describe("The page as the person saved or copied it, after getting through the wall"),
|
|
@@ -296,7 +296,7 @@ export function createServer() {
|
|
|
296
296
|
server.registerTool("form_preview", {
|
|
297
297
|
title: "Preview a form",
|
|
298
298
|
annotations: { readOnlyHint: true, destructiveHint: false, openWorldHint: true },
|
|
299
|
-
description: "Preview a form without sending it: returns the EXACT request form_submit would send (hidden values shown as [set by the site]), a preview_hash, and whether a permission slip covers the target (form.slip.covered).
|
|
299
|
+
description: "Preview a form without sending it: returns the EXACT request form_submit would send (hidden values shown as [set by the site]), a preview_hash, and whether a permission slip covers the target (form.slip.covered). The previewed form is not sent. Minimal politeness state still applies; with the owner's get_form_actions=slip, fetching a known GET form action requires and spends a slip use. Sign-in, password, file upload, payment, purchase and CAPTCHA forms are not_allowed.",
|
|
300
300
|
inputSchema: { url: z.string().describe("The page the form is on"), form: FORM, values: VALUES, private: PRIVATE },
|
|
301
301
|
}, async ({ url, form, values, private: priv }, extra) => {
|
|
302
302
|
try {
|
|
@@ -332,9 +332,9 @@ export function createServer() {
|
|
|
332
332
|
since: z.string().optional().describe("Start of the window: a date like 2026-10-01, or 30d / 24h. Default 30d."),
|
|
333
333
|
page: PAGE,
|
|
334
334
|
},
|
|
335
|
-
}, async ({ since, page }) => {
|
|
335
|
+
}, async ({ since, page }, extra) => {
|
|
336
336
|
try {
|
|
337
|
-
return text(await engine.needsReport(since, page));
|
|
337
|
+
return text(await engine.needsReport(since, page, { signal: extra.signal, deadline: Date.now() + 60_000 }));
|
|
338
338
|
}
|
|
339
339
|
catch (e) {
|
|
340
340
|
return text(errorEnvelope("", e.message), true);
|
package/dist/needs.js
CHANGED
|
@@ -73,7 +73,7 @@ const hostOf = (url) => {
|
|
|
73
73
|
return null;
|
|
74
74
|
}
|
|
75
75
|
};
|
|
76
|
-
export async function buildNeeds(sinceText = DEFAULT_SINCE, page = 1, now = Date.now()) {
|
|
76
|
+
export async function buildNeeds(sinceText = DEFAULT_SINCE, page = 1, now = Date.now(), scan = {}) {
|
|
77
77
|
const since = parseSince(sinceText) ?? now - 30 * 86_400_000;
|
|
78
78
|
const firstDay = utcDay(since);
|
|
79
79
|
const walls = new Tally("walls");
|
|
@@ -122,8 +122,12 @@ export async function buildNeeds(sinceText = DEFAULT_SINCE, page = 1, now = Date
|
|
|
122
122
|
hand.tickets++;
|
|
123
123
|
else if (e.ticket === "refused_cap")
|
|
124
124
|
hand.refused_cap++;
|
|
125
|
-
if (e.tool === "resume" && (e.status === "ok" || (e.status === "unchanged" && !e.source)))
|
|
126
|
-
|
|
125
|
+
if (e.tool === "resume" && (e.status === "ok" || (e.status === "unchanged" && !e.source))) {
|
|
126
|
+
if (e.source === "provided_by_agent")
|
|
127
|
+
hand.resolved_agent = (hand.resolved_agent ?? 0) + 1;
|
|
128
|
+
else
|
|
129
|
+
hand[e.source ? "resolved_person" : "resolved_retry"]++;
|
|
130
|
+
}
|
|
127
131
|
if (e.status === "wall")
|
|
128
132
|
walls.add(`${host}\n${e.reason ?? ""}`, { host, reason: e.reason ?? "unknown" });
|
|
129
133
|
if (e.tool === "read" && e.matched === 0 && typeof e.purpose === "string")
|
|
@@ -154,7 +158,7 @@ export async function buildNeeds(sinceText = DEFAULT_SINCE, page = 1, now = Date
|
|
|
154
158
|
s.door = true;
|
|
155
159
|
}
|
|
156
160
|
return false; // collect only; nothing is kept in pages
|
|
157
|
-
});
|
|
161
|
+
}, 1, 40_000, scan);
|
|
158
162
|
const wallRows = walls.rows();
|
|
159
163
|
const emptyRows = empty.rows();
|
|
160
164
|
const betterRows = better.rows();
|
|
@@ -259,7 +263,7 @@ export function toMarkdown(r) {
|
|
|
259
263
|
L.push(`- Pages read more than once: ${c.rereads.groups} (${c.rereads.records} reads).`);
|
|
260
264
|
L.push(`- Sites visited ${OFTEN}+ times with no agent door: ${c.no_door.groups}.`);
|
|
261
265
|
const hc = c.handoffs;
|
|
262
|
-
L.push(`- Handoff tickets kept at walls: ${hc.tickets}${hc.refused_cap ? ` (${hc.refused_cap} more refused at the cap)` : ""}; resolved by a retry: ${hc.resolved_retry}; by a page a person provided: ${hc.resolved_person}. Expired tickets are not counted.`);
|
|
266
|
+
L.push(`- Handoff tickets kept at walls: ${hc.tickets}${hc.refused_cap ? ` (${hc.refused_cap} more refused at the cap)` : ""}; resolved by a retry: ${hc.resolved_retry}; by a page a person provided: ${hc.resolved_person}${hc.resolved_agent ? `; by text an agent provided: ${hc.resolved_agent}` : ""}. Expired tickets are not counted.`);
|
|
263
267
|
const fc = c.flagged;
|
|
264
268
|
if (fc)
|
|
265
269
|
L.push(`- Safe Browsing: ${fc.phishing_list} stopped by Phishing.Database, ${fc.web_risk} refused by Google Web Risk, ${fc.read_anyway} read anyway with allow_flagged.`);
|
package/dist/parentwatch.js
CHANGED
|
@@ -1,18 +1,8 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// W-17:
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
export function startParentWatchdog(intervalMs = 200) {
|
|
9
|
-
const code = `
|
|
10
|
-
const { workerData } = require("node:worker_threads");
|
|
11
|
-
setInterval(() => {
|
|
12
|
-
if (process.ppid !== workerData.ppid) process.kill(process.pid, "SIGKILL");
|
|
13
|
-
}, workerData.intervalMs);
|
|
14
|
-
`;
|
|
15
|
-
const w = new Worker(code, { eval: true, workerData: { ppid: process.ppid, intervalMs } });
|
|
16
|
-
w.unref(); // it never keeps the child alive on its own
|
|
17
|
-
return w;
|
|
2
|
+
// W-17: watch the parent's IPC channel outside the PDF child, whose parser may never yield.
|
|
3
|
+
import { fork } from "node:child_process";
|
|
4
|
+
export function startParentWatchdog(pid) {
|
|
5
|
+
return fork(new URL("./pdfwatch.js", import.meta.url), [String(pid)], {
|
|
6
|
+
execArgv: [], env: {}, stdio: ["ignore", "ignore", "ignore", "ipc"],
|
|
7
|
+
});
|
|
18
8
|
}
|
package/dist/pdf.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
// page is rendered, no font is installed (disableFontFace)
|
|
12
12
|
// - pdf.js runs its own "worker" as ordinary code inside this thread (it has no separate thread in
|
|
13
13
|
// Node), so nothing is spawned outside our sandbox
|
|
14
|
+
import { holdsSecret, SecretWithheld } from "./echo.js";
|
|
14
15
|
import path from "node:path";
|
|
15
16
|
import { fileURLToPath } from "node:url";
|
|
16
17
|
import { PdfError } from "./errors.js";
|
|
@@ -114,7 +115,7 @@ async function drawnText(page, OPS) {
|
|
|
114
115
|
return { white: "", runs: [] };
|
|
115
116
|
}
|
|
116
117
|
}
|
|
117
|
-
export async function parsePdf(bytes, baseUrl) {
|
|
118
|
+
export async function parsePdf(bytes, baseUrl, secrets) {
|
|
118
119
|
return quietly(async () => {
|
|
119
120
|
const pdfjs = await import(PDFJS);
|
|
120
121
|
const dir = packageDir();
|
|
@@ -163,6 +164,8 @@ export async function parsePdf(bytes, baseUrl) {
|
|
|
163
164
|
catch {
|
|
164
165
|
// no usable metadata
|
|
165
166
|
}
|
|
167
|
+
if (holdsSecret([title], secrets))
|
|
168
|
+
throw new SecretWithheld();
|
|
166
169
|
const pages = [];
|
|
167
170
|
const hidden = [];
|
|
168
171
|
let kept = 0;
|
package/dist/pdfchild.js
CHANGED
|
@@ -3,9 +3,7 @@
|
|
|
3
3
|
// PDF that eats memory in ways a worker thread's heap limit cannot see (decoder buffers) can only
|
|
4
4
|
// kill THIS process. It reads one job over the IPC channel, answers once, and waits to be killed. If the channel
|
|
5
5
|
// closes (the parent died) it exits at once. Its stdout is not connected to anything.
|
|
6
|
-
import { startParentWatchdog } from "./parentwatch.js";
|
|
7
6
|
import { capDone, handleAsync } from "./work.js";
|
|
8
|
-
startParentWatchdog(); // dies with the parent even if this thread is busy
|
|
9
7
|
process.on("disconnect", () => process.exit(0)); // never outlive the parent
|
|
10
8
|
// After answering, this process waits to be killed: the parent checks its memory once more first, and
|
|
11
9
|
// the answer carries this process's peak memory (maxRSS, in kB) so a spike between the parent's checks
|
package/dist/pdfproc.js
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
// page_too_complex and nothing is kept. FAIL CLOSED: if the memory of the child cannot be watched (Windows, no
|
|
10
10
|
// `ps`, no /proc, or a probe that fails mid-way) PDFs are not read, with a clear message. The child has an empty environment, no shell, no stdin or
|
|
11
11
|
// stdout, and is killed when this process exits.
|
|
12
|
+
import { SecretWithheld } from "./echo.js";
|
|
12
13
|
import { execFile, fork } from "node:child_process";
|
|
13
14
|
import fs from "node:fs";
|
|
14
15
|
import path from "node:path";
|
|
@@ -18,6 +19,7 @@ import { currentSignal } from "./safety.js";
|
|
|
18
19
|
import { PageTooComplex, budgetMs, workerHeapMb } from "./sandbox.js";
|
|
19
20
|
import { MAX_PDF_RESULT_CHARS } from "./work.js";
|
|
20
21
|
import { Slots } from "./slots.js";
|
|
22
|
+
import { startParentWatchdog } from "./parentwatch.js";
|
|
21
23
|
export const PDF_MEMORY_MB = 1024;
|
|
22
24
|
const POLL_MS = 50;
|
|
23
25
|
const MAX_CONCURRENT = 2;
|
|
@@ -34,7 +36,7 @@ const live = new Set();
|
|
|
34
36
|
// v0.5.1: the child runs under Node's permission model. It may READ only Tiresias's own folder (dist/) and
|
|
35
37
|
// the node_modules folder that holds pdfjs-dist (pdf.js, its fonts and cmaps, and the reader's own
|
|
36
38
|
// dependencies installed beside it); it may write nothing, start no process and load no native add-on.
|
|
37
|
-
//
|
|
39
|
+
// Its watchdog runs outside the child (parentwatch.ts). So a PDF that took over the
|
|
38
40
|
// parser still cannot read ~/.tiresias (the keys) or the owner's files. (Node 22's permission model does
|
|
39
41
|
// not cover the network; the child is given no address and no code that fetches.)
|
|
40
42
|
export const childExecArgv = (heap) => [`--max-old-space-size=${heap}`, ...childPermissions()];
|
|
@@ -52,7 +54,7 @@ export function childPermissions() {
|
|
|
52
54
|
}
|
|
53
55
|
};
|
|
54
56
|
const dirs = [...new Set([app, modules, real(app), real(modules)])];
|
|
55
|
-
return ["--permission", ...dirs.map((d) => `--allow-fs-read=${path.join(d, "*")}`)
|
|
57
|
+
return ["--permission", ...dirs.map((d) => `--allow-fs-read=${path.join(d, "*")}`)];
|
|
56
58
|
}
|
|
57
59
|
export const activePdfPids = () => [...live].map((c) => c.pid).filter(Boolean);
|
|
58
60
|
// No child may outlive this process. (The child also exits when its IPC channel closes.)
|
|
@@ -141,6 +143,7 @@ function start(job) {
|
|
|
141
143
|
serialization: "advanced",
|
|
142
144
|
});
|
|
143
145
|
live.add(child);
|
|
146
|
+
const watchdog = startParentWatchdog(child.pid);
|
|
144
147
|
child.stderr?.resume(); // drained and thrown away
|
|
145
148
|
let settled = false;
|
|
146
149
|
let polling = false;
|
|
@@ -154,6 +157,7 @@ function start(job) {
|
|
|
154
157
|
clearInterval(poll);
|
|
155
158
|
sig?.removeEventListener("abort", onAbort);
|
|
156
159
|
child.kill("SIGKILL"); // the child is done or being thrown away; it never stays
|
|
160
|
+
watchdog.kill("SIGKILL");
|
|
157
161
|
live.delete(child);
|
|
158
162
|
fn();
|
|
159
163
|
};
|
|
@@ -225,6 +229,8 @@ function start(job) {
|
|
|
225
229
|
return tooComplex();
|
|
226
230
|
return void accept(() => resolve(d.result), peakKb);
|
|
227
231
|
}
|
|
232
|
+
if (d.error === "secret_withheld")
|
|
233
|
+
return finish(() => reject(new SecretWithheld()));
|
|
228
234
|
if (d.error === "pdf") {
|
|
229
235
|
const msg = String(d.message ?? "This PDF could not be read.").slice(0, 300);
|
|
230
236
|
return void accept(() => reject(new PdfError(msg)), peakKb);
|
|
@@ -235,14 +241,23 @@ function start(job) {
|
|
|
235
241
|
tooComplex();
|
|
236
242
|
}
|
|
237
243
|
});
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
244
|
+
watchdog.once("error", tooComplex);
|
|
245
|
+
watchdog.once("exit", () => {
|
|
246
|
+
if (!settled && !answered)
|
|
247
|
+
tooComplex();
|
|
248
|
+
});
|
|
249
|
+
watchdog.once("message", () => {
|
|
250
|
+
if (settled)
|
|
251
|
+
return;
|
|
252
|
+
try {
|
|
253
|
+
child.send(job, (err) => {
|
|
254
|
+
if (err)
|
|
255
|
+
tooComplex();
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
tooComplex(); // a send that throws still cleans up the child, timers and listeners
|
|
260
|
+
}
|
|
261
|
+
});
|
|
247
262
|
});
|
|
248
263
|
}
|
package/dist/pdfwatch.js
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Only the parent starts this watchdog; no PDF bytes or page code reach it.
|
|
3
|
+
const pid = Number(process.argv[2]);
|
|
4
|
+
if (!Number.isSafeInteger(pid) || pid <= 0 || !process.send)
|
|
5
|
+
process.exit(1);
|
|
6
|
+
process.once("disconnect", () => {
|
|
7
|
+
try {
|
|
8
|
+
process.kill(pid, "SIGKILL");
|
|
9
|
+
}
|
|
10
|
+
catch { /* already gone */ }
|
|
11
|
+
process.exit(0);
|
|
12
|
+
});
|
|
13
|
+
process.send({ ready: true });
|
|
14
|
+
export {};
|
package/dist/pipeline.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// The read pipeline: choose a source, fetch it, decide walls and files, parse, cache, and build the reply
|
|
3
3
|
// (read, check and doors).
|
|
4
4
|
// (Split out of engine.ts in v0.5.1, behavior unchanged.)
|
|
5
|
+
import { noteGetActions } from "./getactions.js";
|
|
5
6
|
import { hostKey } from "./sitekey.js";
|
|
6
7
|
import { startDoorProbe } from "./background.js";
|
|
7
8
|
import { loadDoorsMode } from "./config.js";
|
|
@@ -12,7 +13,7 @@ import { TiresiasError } from "./errors.js";
|
|
|
12
13
|
import { politeFetch } from "./fetcher.js";
|
|
13
14
|
import { after304, fromResponse, isFresh } from "./freshness.js";
|
|
14
15
|
import { shown } from "./reader.js";
|
|
15
|
-
import { runUntrusted } from "./sandbox.js";
|
|
16
|
+
import { PageTooComplex, runUntrusted } from "./sandbox.js";
|
|
16
17
|
import { OVERLAY_TEXT, RENDERED_TEXT, RENDER_TEXT, loadRenderMode, looksLikeShell, renderHtml } from "./render.js";
|
|
17
18
|
import * as receipts from "./receipts.js";
|
|
18
19
|
import { checkAllowed } from "./rules.js";
|
|
@@ -71,8 +72,10 @@ export function parseJob(read, body, bytes, url, wall, lang, rendered) {
|
|
|
71
72
|
export const looksHtml = (contentType, body) => /html/.test(mimeOf(contentType)) || /<(!doctype html|html|body|form)\b/i.test(body.slice(0, 65536));
|
|
72
73
|
// `wallCheck`: the response is HTML whatever reader its name chose, so its wall markers are looked for
|
|
73
74
|
// (Codex round 63: a login page served as text/html at /login.csv is still a wall).
|
|
74
|
-
export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered) {
|
|
75
|
+
export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered, secrets) {
|
|
75
76
|
const job = parseJob(read, body, bytes, url, wallCheck, lang, rendered);
|
|
77
|
+
if (job.op === "pdf" && secrets)
|
|
78
|
+
job.secrets = secrets;
|
|
76
79
|
const r = await runUntrusted(job, job.op === "pdf" ? Infinity : body.length);
|
|
77
80
|
// What the response said against what it is goes first, so the shield's cap never pushes it out.
|
|
78
81
|
if (fileFlags.length)
|
|
@@ -99,9 +102,16 @@ export function pickSource(u, od) {
|
|
|
99
102
|
// built from that name too (Codex round 6).
|
|
100
103
|
const cu = new URL(u.href);
|
|
101
104
|
cu.hostname = hostKey(cu.hostname);
|
|
102
|
-
|
|
105
|
+
cu.hash = "";
|
|
106
|
+
const base = cu.pathname;
|
|
103
107
|
const stripped = base.replace(/\/$/, "");
|
|
104
|
-
const candidates = [base + ".md", stripped + ".md", base.endsWith("/") ? base + "index.html.md" : ""]
|
|
108
|
+
const candidates = [base + ".md", stripped + ".md", base.endsWith("/") ? base + "index.html.md" : ""].map((p) => {
|
|
109
|
+
if (!p)
|
|
110
|
+
return "";
|
|
111
|
+
const twin = new URL(cu.href);
|
|
112
|
+
twin.pathname = p;
|
|
113
|
+
return twin.href;
|
|
114
|
+
});
|
|
105
115
|
const hit = candidates.find((c) => c && od.llms_md.includes(c));
|
|
106
116
|
if (hit)
|
|
107
117
|
return { source: hit, detail: "markdown" };
|
|
@@ -156,6 +166,10 @@ export function paginate(items, size, page, budget) {
|
|
|
156
166
|
export const len = jsonBytes; // every page budget counts UTF-8 bytes of the JSON sent
|
|
157
167
|
export async function buildContent(entry, opts) {
|
|
158
168
|
const p = entry.parsed;
|
|
169
|
+
// Receipt measurement repeats the final URL for every unit, on the main thread (as in tabs.ts).
|
|
170
|
+
const receiptUnits = opts.section === undefined ? p.sections.length : (p.sections.find((s) => s.n === opts.section)?.blocks.length ?? 0);
|
|
171
|
+
if (receiptUnits * (entry.final_url.length + 600) > 64 * 1024 * 1024)
|
|
172
|
+
throw new PageTooComplex();
|
|
159
173
|
const page = opts.page ?? 1;
|
|
160
174
|
const fetchedAt = entry.fetched_at;
|
|
161
175
|
// Receipts cite the URL that was actually read, and quote exact extracted text.
|
|
@@ -278,7 +292,8 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
278
292
|
const renderMode = tool === "read" || tool === "check" ? loadRenderMode() : "off";
|
|
279
293
|
// Explicit renders and checks of rendered copies need a body and a new browser run, even after a 304.
|
|
280
294
|
// The one shortcut is an already rendered copy still fresh under the site's cache rules (below).
|
|
281
|
-
const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered));
|
|
295
|
+
const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered) || (renderMode === "auto" && !!cache?.shell && !cache.rendered));
|
|
296
|
+
const needsBody = rerender || (renderMode === "off" && !!cache?.rendered);
|
|
282
297
|
// W-28 (v0.8): a read that asks to save, when the owner allows it (a bad setting fails the read, by name). A save
|
|
283
298
|
// keeps what THIS read received, so it never uses a fresh copy, a 304 or a cached rendering: it always fetches.
|
|
284
299
|
const sv = tool === "read" && opts.save && savePolicy() === "ask" ? {} : undefined;
|
|
@@ -287,7 +302,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
287
302
|
// Only a copy whose fetch had no redirect at all (recorded by the fetch itself; a missing field
|
|
288
303
|
// means no) and that was read from the page itself, not llms.txt or a twin: the rules just
|
|
289
304
|
// checked are then the rules of the page being served (Codex rounds 38 and 39).
|
|
290
|
-
if (cache && !staleParse && (!
|
|
305
|
+
if (cache && !staleParse && (!needsBody || (renderMode !== "off" && cache.rendered)) && !sv && !opts.fresh && cache.redirected === false && cache.source_url === url && isFresh(cache)) {
|
|
291
306
|
meta.cache = "fresh";
|
|
292
307
|
const od = tool === "read" ? cachedDoors(u.origin) : null;
|
|
293
308
|
if (od)
|
|
@@ -295,7 +310,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
295
310
|
// W-30 (Codex round 1): a copy served from the cache with no door answers starts (or reports) the
|
|
296
311
|
// background checks too, so a cancelled or refused job is retried and a running one is reported.
|
|
297
312
|
const later = tool === "read" && !od && !priv && doorsMode === "background";
|
|
298
|
-
const out = await finishPage(tool, url, cache, true, null, opts, later ? pageOnly(u.origin) : od, new Date().toISOString(), () => { }, meta, { fresh: false }, false, { cache: "fresh", agent });
|
|
313
|
+
const out = await finishPage(tool, url, cache, true, null, opts, later ? pageOnly(u.origin) : od, new Date().toISOString(), () => { }, meta, { fresh: false }, false, { cache: "fresh", agent, ...(opts.render && renderMode === "off" ? { render: { status: "not_allowed" } } : {}) });
|
|
299
314
|
return later ? afterDeferred(out, u.origin) : out;
|
|
300
315
|
}
|
|
301
316
|
// check never probes; it just revisits where it read from last time.
|
|
@@ -313,7 +328,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
313
328
|
// While the checks are pending, the reply still lists the doors the page itself shows (feeds, an API
|
|
314
329
|
// description, structured data), from an empty origin answer (Codex round 1).
|
|
315
330
|
const shown = deferred ? pageOnly(u.origin) : od;
|
|
316
|
-
let { res, started, validatorsSent } = await fetchSource(source, cache, staleParse ||
|
|
331
|
+
let { res, started, validatorsSent } = await fetchSource(source, cache, staleParse || needsBody || !!sv, !!sv);
|
|
317
332
|
// An unsolicited 304 proves nothing about this parse or representation. Ask once for a full body.
|
|
318
333
|
if (res.status === 304 && (!validatorsSent || staleParse)) {
|
|
319
334
|
throwIfCancelled();
|
|
@@ -526,6 +541,8 @@ function afterDeferred(e, origin) {
|
|
|
526
541
|
return e;
|
|
527
542
|
}
|
|
528
543
|
export async function finishPage(tool, url, entry, unchanged, changedFrom, opts, od, now, persist, meta, diff, notKept = false, serve = {}) {
|
|
544
|
+
if (!entry.source)
|
|
545
|
+
noteGetActions(entry.parsed.forms, !notKept);
|
|
529
546
|
const wantsContent = opts.question !== undefined || opts.section !== undefined || opts.page !== undefined || (tool === "read" && (opts.translate !== undefined || !!opts.save));
|
|
530
547
|
// W-28 (v0.8): the snapshot is written before the receipts are recorded, so each recorded receipt names it.
|
|
531
548
|
const sv = tool === "read" ? serve.save : undefined;
|
package/dist/profiles.js
CHANGED
|
@@ -11,7 +11,7 @@ import { randomBytes } from "node:crypto";
|
|
|
11
11
|
import fs from "node:fs";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { parseSignatureAgent } from "./config.js";
|
|
14
|
-
import { KEY_FILE } from "./keys.js";
|
|
14
|
+
import { KEY_FILE, PREVIOUS_KEY_FILE } from "./keys.js";
|
|
15
15
|
import { DEFAULT_PROFILE, NOFOLLOW, NONBLOCK, PROFILE_RE, ProfileError, checkProfileName, checkSub, ensureRoot, profileDir, profilesRoot, readOwnedFile, readRoot, rootDir, } from "./store.js";
|
|
16
16
|
const META = "profile.json"; // { name, created_at }, written once at create
|
|
17
17
|
const me = () => (typeof process.getuid === "function" ? process.getuid() : -1);
|
|
@@ -138,8 +138,8 @@ function plan(dir) {
|
|
|
138
138
|
stack.push(f);
|
|
139
139
|
else if (s.isFile()) {
|
|
140
140
|
p.files.push({ file: f, size: s.size });
|
|
141
|
-
if (rel === path.join("keys", KEY_FILE))
|
|
142
|
-
p.hasKey = true;
|
|
141
|
+
if (rel === path.join("keys", KEY_FILE) || rel === path.join("keys", PREVIOUS_KEY_FILE))
|
|
142
|
+
p.hasKey = true; // v1.1: a rotation's previous key is a private key too
|
|
143
143
|
}
|
|
144
144
|
else
|
|
145
145
|
throw bad("something that is not a file or a folder");
|
package/dist/queue.js
CHANGED
|
@@ -9,8 +9,8 @@ import { canonOrigin } from "./sitekey.js";
|
|
|
9
9
|
import { noteApplied, siteSpacing } from "./sites.js";
|
|
10
10
|
// Bounded (v0.5.1): a long-running MCP server keeps at most MAX_HOSTS sites in each. A queue entry is
|
|
11
11
|
// removed when its last request has had its turn, and a site's waiting set when it is empty.
|
|
12
|
-
// When each site was last called. Kept while its spacing can still matter
|
|
13
|
-
//
|
|
12
|
+
// When each site was last called. Kept while its spacing can still matter,
|
|
13
|
+
// then pruned (Codex round 78: evicting by count could drop a spacing still owed).
|
|
14
14
|
const lastHit = new Map();
|
|
15
15
|
const HIT_KEEP_MS = 60_000;
|
|
16
16
|
// The earliest the NEXT request may start, as the last request's own spacing said (Codex round 79: a
|
|
@@ -142,17 +142,25 @@ export async function throttle(origin, delay, outer, url, hold = false) {
|
|
|
142
142
|
await Promise.race([prev, cancelled]);
|
|
143
143
|
// Holding the turn: first the cross-process lease (Codex round 76: waiting for it AFTER reserving a
|
|
144
144
|
// slot let the slot go stale), then the shared slot and its spacing.
|
|
145
|
-
|
|
146
|
-
activeRelease = await Promise.race([acquireActive(origin, mine.signal), cancelled]);
|
|
145
|
+
activeRelease = await Promise.race([acquireActive(origin, mine.signal), cancelled]);
|
|
147
146
|
// W-25 (v0.6): a site's own spacing (sites.ts) raises the spacing. Like every spacing since v0.5.1 it is per
|
|
148
147
|
// ORIGIN (scheme, host and port). Settings are the call's snapshot, read once when the call started
|
|
149
148
|
// (coordinator's rule after Codex round 11): a change saved meanwhile applies to calls that start after it.
|
|
150
149
|
const site = target ? siteSpacing(target) : 0;
|
|
151
150
|
const eff = Math.max(delay, site);
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
for (;;) {
|
|
152
|
+
if (mine.signal.aborted)
|
|
153
|
+
throw new Error("queue-cancelled");
|
|
154
|
+
if (cooldownLeft(origin) > 0)
|
|
155
|
+
throw cooldownRefusal(origin, url);
|
|
156
|
+
const slot = reserve(origin, eff * 1000, Math.max((lastHit.get(origin) ?? 0) + eff * 1000, nextOk.get(origin) ?? 0), true);
|
|
157
|
+
const wait = slot - Date.now();
|
|
158
|
+
if (wait <= 0)
|
|
159
|
+
break;
|
|
160
|
+
if (wait > 30_000)
|
|
161
|
+
throw new Refused("The site's spacing wait is longer than Tiresias will wait.", undefined, Math.ceil(wait / 1000), url);
|
|
155
162
|
await Promise.race([sleep(wait, mine.signal), cancelled]);
|
|
163
|
+
}
|
|
156
164
|
// The checks that matter: right before sending, not before queueing.
|
|
157
165
|
if (mine.signal.aborted)
|
|
158
166
|
throw new Error("queue-cancelled");
|
package/dist/reader.js
CHANGED
|
@@ -72,6 +72,11 @@ function collapse(parts) {
|
|
|
72
72
|
}
|
|
73
73
|
return { text, marks };
|
|
74
74
|
}
|
|
75
|
+
// Keep a UTF-16 boundary before an entire code point.
|
|
76
|
+
function splitEnd(text, end) {
|
|
77
|
+
const hi = text.charCodeAt(end - 1), lo = text.charCodeAt(end);
|
|
78
|
+
return hi >= 0xd800 && hi <= 0xdbff && lo >= 0xdc00 && lo <= 0xdfff ? end - 1 : end;
|
|
79
|
+
}
|
|
75
80
|
// Split an overlong paragraph at spaces. Every piece is an exact substring.
|
|
76
81
|
function splitBlock(b) {
|
|
77
82
|
if (b.text.length <= MAX_BLOCK)
|
|
@@ -85,6 +90,7 @@ function splitBlock(b) {
|
|
|
85
90
|
if (sp > start + MAX_BLOCK / 2)
|
|
86
91
|
end = sp;
|
|
87
92
|
}
|
|
93
|
+
end = splitEnd(b.text, end);
|
|
88
94
|
const lo = start;
|
|
89
95
|
const marks = b.marks.filter(([pos]) => (pos > lo || lo === 0) && pos <= end).map(([pos, n]) => [pos - lo, n]);
|
|
90
96
|
out.push({ text: b.text.slice(start, end), marks });
|
|
@@ -143,14 +149,14 @@ class Builder {
|
|
|
143
149
|
pre(raw, marks = [], lead = "", indent = "") {
|
|
144
150
|
if (!this.cur)
|
|
145
151
|
this.startTop();
|
|
146
|
-
for (let i = 0; i < raw.length;
|
|
147
|
-
const
|
|
152
|
+
for (let i = 0; i < raw.length;) {
|
|
153
|
+
const end = splitEnd(raw, Math.min(i + MAX_BLOCK, raw.length));
|
|
154
|
+
const piece = raw.slice(i, end);
|
|
148
155
|
// A fence longer than any run of backticks inside, so the text cannot close it (round 78).
|
|
149
156
|
let run = 0;
|
|
150
157
|
for (const m of piece.match(/`+/g) ?? [])
|
|
151
158
|
run = Math.max(run, m.length);
|
|
152
159
|
const fence = "`".repeat(Math.max(3, run + 1));
|
|
153
|
-
const end = i + piece.length;
|
|
154
160
|
const head = i === 0 && lead ? lead : indent;
|
|
155
161
|
const body = indent ? piece.split("\n").map((l) => indent + l).join("\n") : piece;
|
|
156
162
|
const m = marks
|
|
@@ -162,6 +168,7 @@ class Builder {
|
|
|
162
168
|
});
|
|
163
169
|
this.cur.blocks.push({ text: `${head}${fence}\n${body}\n${indent}${fence}`, marks: m });
|
|
164
170
|
this.shield.scan(piece, "visible text");
|
|
171
|
+
i = end;
|
|
165
172
|
}
|
|
166
173
|
}
|
|
167
174
|
// Text before the first heading goes under a label Tiresias makes, not text from the page: it gets no
|
|
@@ -646,6 +653,8 @@ function scanHead(root) {
|
|
|
646
653
|
found.title = n;
|
|
647
654
|
else if (n.name === "script" && /ld\+json/i.test(n.attribs?.type ?? ""))
|
|
648
655
|
found.ld.push(n);
|
|
656
|
+
else if (n.name === "base" && !found.base && n.attribs?.href !== undefined)
|
|
657
|
+
found.base = n;
|
|
649
658
|
else if (n.name === "link")
|
|
650
659
|
found.links.push(n);
|
|
651
660
|
else if (n.name === "meta" && found.metas.length < 200)
|
|
@@ -725,10 +734,11 @@ export function lineBlocks(text) {
|
|
|
725
734
|
let j = text.indexOf("\n", i);
|
|
726
735
|
if (j === -1)
|
|
727
736
|
j = text.length;
|
|
728
|
-
for (let k = i; k < j;
|
|
729
|
-
const e = Math.min(j, k + MAX_BLOCK);
|
|
737
|
+
for (let k = i; k < j;) {
|
|
738
|
+
const e = splitEnd(text, Math.min(j, k + MAX_BLOCK));
|
|
730
739
|
if (text.slice(k, e).trim())
|
|
731
740
|
lines.push([k, e]);
|
|
741
|
+
k = e;
|
|
732
742
|
}
|
|
733
743
|
i = j + 1;
|
|
734
744
|
}
|
|
@@ -762,8 +772,11 @@ export function lineBlocks(text) {
|
|
|
762
772
|
}
|
|
763
773
|
function rawBlocks(text) {
|
|
764
774
|
const out = [];
|
|
765
|
-
for (let i = 0; i < text.length;
|
|
766
|
-
|
|
775
|
+
for (let i = 0; i < text.length;) {
|
|
776
|
+
const end = splitEnd(text, Math.min(i + MAX_BLOCK, text.length));
|
|
777
|
+
out.push({ text: text.slice(i, end), marks: [] });
|
|
778
|
+
i = end;
|
|
779
|
+
}
|
|
767
780
|
return out;
|
|
768
781
|
}
|
|
769
782
|
// `keep`, when given, receives the forms WITH their hidden values (W-04 preview and submit only).
|
|
@@ -787,11 +800,20 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
787
800
|
shield.sheet.inherited.set(attrs, why);
|
|
788
801
|
}
|
|
789
802
|
}
|
|
790
|
-
const
|
|
803
|
+
const found = scanHead(doc);
|
|
804
|
+
let linkBase = baseUrl;
|
|
805
|
+
try {
|
|
806
|
+
const base = new URL(found.base?.attribs?.href ?? "", baseUrl);
|
|
807
|
+
if (base.protocol === "http:" || base.protocol === "https:")
|
|
808
|
+
linkBase = base.href;
|
|
809
|
+
}
|
|
810
|
+
catch {
|
|
811
|
+
// An unusable first base leaves links relative to the fetched page.
|
|
812
|
+
}
|
|
813
|
+
const b = new Builder(shield, linkBase);
|
|
791
814
|
chooseMain(doc, b);
|
|
792
815
|
walk(doc, b);
|
|
793
816
|
b.flush();
|
|
794
|
-
const found = scanHead(doc);
|
|
795
817
|
let title = found.title ? norm(rawText(found.title, MAX_TITLE * 2)).slice(0, MAX_TITLE) : "";
|
|
796
818
|
if (!title)
|
|
797
819
|
title = (b.sections.find((s) => s.level === 1)?.heading ?? "").slice(0, MAX_TITLE);
|
|
@@ -799,7 +821,7 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
799
821
|
// Features of the page itself.
|
|
800
822
|
const resolve = (h) => {
|
|
801
823
|
try {
|
|
802
|
-
return h ? new URL(h,
|
|
824
|
+
return h ? new URL(h, linkBase).href : null;
|
|
803
825
|
}
|
|
804
826
|
catch {
|
|
805
827
|
return null;
|
|
@@ -819,7 +841,7 @@ export function parseHtml(html, baseUrl, keep, rendered) {
|
|
|
819
841
|
service_desc.push(href);
|
|
820
842
|
}
|
|
821
843
|
const page = finish(b, { title, feeds, service_desc });
|
|
822
|
-
page.meta = pageMeta(found,
|
|
844
|
+
page.meta = pageMeta(found, linkBase);
|
|
823
845
|
for (const v of [page.meta?.byline, page.meta?.description, page.meta?.og?.title, page.meta?.og?.description])
|
|
824
846
|
if (v)
|
|
825
847
|
shield.scan(v, "page metadata");
|
|
@@ -945,8 +967,9 @@ export function parseSvg(body, baseUrl) {
|
|
|
945
967
|
}
|
|
946
968
|
const PLAIN_HEADING = { script: "(script)", csv: "(CSV)", calendar: "(calendar)", contact: "(contact)" };
|
|
947
969
|
export function parsePlain(text, baseUrl, mode) {
|
|
948
|
-
|
|
949
|
-
|
|
970
|
+
const omitted = text.length > MAX_EXPANDED ? text.length - splitEnd(text, MAX_EXPANDED) : 0;
|
|
971
|
+
if (omitted)
|
|
972
|
+
text = text.slice(0, text.length - omitted);
|
|
950
973
|
// Calendar and contact lines may be folded (a line break followed by a space or tab continues the line).
|
|
951
974
|
if (mode === "calendar" || mode === "contact")
|
|
952
975
|
text = text.replace(/\r?\n[ \t]/g, "");
|
|
@@ -959,7 +982,7 @@ export function parsePlain(text, baseUrl, mode) {
|
|
|
959
982
|
if (mode === "calendar" || mode === "contact")
|
|
960
983
|
for (const u of bareUrls(text))
|
|
961
984
|
b.link(u, u);
|
|
962
|
-
const page = finish(b, {});
|
|
985
|
+
const page = finish(b, { text_omitted: omitted || undefined });
|
|
963
986
|
if (mode === "calendar")
|
|
964
987
|
calendarEvents(text, page, shield); // v0.6: the invite's events, structured (ics.ts)
|
|
965
988
|
const own = [];
|
|
@@ -1312,8 +1335,9 @@ export function parseJsonText(text, baseUrl) {
|
|
|
1312
1335
|
}
|
|
1313
1336
|
if (pretty.length > MAX_EXPANDED) {
|
|
1314
1337
|
// raw text that is not JSON: keep the first MAX_EXPANDED characters and count the rest
|
|
1315
|
-
|
|
1316
|
-
|
|
1338
|
+
const end = splitEnd(pretty, MAX_EXPANDED);
|
|
1339
|
+
omitted = pretty.length - end;
|
|
1340
|
+
pretty = pretty.slice(0, end);
|
|
1317
1341
|
}
|
|
1318
1342
|
const shield = new Shield();
|
|
1319
1343
|
shield.scan(pretty, "JSON body");
|
|
@@ -1367,8 +1391,9 @@ function pdfBlocks(text) {
|
|
|
1367
1391
|
let l = line;
|
|
1368
1392
|
while (l.length > MAX_BLOCK) {
|
|
1369
1393
|
flush();
|
|
1370
|
-
|
|
1371
|
-
|
|
1394
|
+
const end = splitEnd(l, MAX_BLOCK);
|
|
1395
|
+
out.push({ text: l.slice(0, end), marks: [] });
|
|
1396
|
+
l = l.slice(end);
|
|
1372
1397
|
}
|
|
1373
1398
|
if (cur && cur.length + 1 + l.length > MAX_BLOCK)
|
|
1374
1399
|
flush();
|
package/dist/receipts.js
CHANGED
|
@@ -31,7 +31,7 @@ export function record(receipts) {
|
|
|
31
31
|
appendLine(`receipts/${utcDay(now)}.jsonl`, row);
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
|
-
export async function query(f) {
|
|
34
|
+
export async function query(f, scan = {}) {
|
|
35
35
|
const since = parseSince(f.since);
|
|
36
36
|
// The URL as typed, or as a URL parser writes it ("https://example.com" is recorded as "https://example.com/").
|
|
37
37
|
let alt;
|
|
@@ -54,7 +54,7 @@ export async function query(f) {
|
|
|
54
54
|
bound(r); // filters match the stored values; only what is returned is cut
|
|
55
55
|
r.record_fingerprint = record_fingerprint;
|
|
56
56
|
return true;
|
|
57
|
-
}, f.page ?? 1);
|
|
57
|
+
}, f.page ?? 1, 40_000, scan);
|
|
58
58
|
}
|
|
59
59
|
const MAX_SENT = 16_000;
|
|
60
60
|
export function recordSent(r) {
|