@raibitsi/tiresias 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +72 -33
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +17 -13
- package/dist/cdp.js +22 -1
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +216 -68
- package/dist/config.js +75 -1
- package/dist/confirm.js +20 -5
- package/dist/cooldown.js +57 -32
- package/dist/dailylog.js +11 -5
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +9 -4
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +29 -2
- package/dist/filereply.js +7 -1
- package/dist/formflow.js +21 -3
- package/dist/forms.js +337 -42
- package/dist/getactions.js +119 -0
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +11 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +34 -27
- package/dist/keys.js +420 -18
- package/dist/local.js +537 -0
- package/dist/mcp.js +56 -25
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +117 -26
- package/dist/preformatted.js +12 -0
- package/dist/profiles.js +3 -3
- package/dist/queue.js +24 -20
- package/dist/reader.js +441 -87
- package/dist/receipts.js +7 -2
- package/dist/render.js +156 -27
- package/dist/renderactions.js +69 -0
- package/dist/resume.js +8 -6
- package/dist/rules.js +24 -5
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +15 -7
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +24 -9
- package/dist/savedlock.js +99 -5
- package/dist/settings.js +10 -3
- package/dist/shelf.js +1 -1
- package/dist/shield.js +15 -0
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +16 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +9 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +10 -2
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
package/dist/pipeline.js
CHANGED
|
@@ -2,9 +2,10 @@
|
|
|
2
2
|
// The read pipeline: choose a source, fetch it, decide walls and files, parse, cache, and build the reply
|
|
3
3
|
// (read, check and doors).
|
|
4
4
|
// (Split out of engine.ts in v0.5.1, behavior unchanged.)
|
|
5
|
+
import { noteGetActions } from "./getactions.js";
|
|
5
6
|
import { hostKey } from "./sitekey.js";
|
|
6
7
|
import { startDoorProbe } from "./background.js";
|
|
7
|
-
import { loadDoorsMode } from "./config.js";
|
|
8
|
+
import { loadChoice, loadDoorsMode } from "./config.js";
|
|
8
9
|
import { cooldownSeconds } from "./cooldown.js";
|
|
9
10
|
import { FLAG_TEXT, mimeOf } from "./files.js";
|
|
10
11
|
import { cachedDoors, noStore, probeOrigin, summarize } from "./doors.js";
|
|
@@ -12,7 +13,8 @@ import { TiresiasError } from "./errors.js";
|
|
|
12
13
|
import { politeFetch } from "./fetcher.js";
|
|
13
14
|
import { after304, fromResponse, isFresh } from "./freshness.js";
|
|
14
15
|
import { shown } from "./reader.js";
|
|
15
|
-
import {
|
|
16
|
+
import { fencedText } from "./preformatted.js";
|
|
17
|
+
import { PageTooComplex, runUntrusted } from "./sandbox.js";
|
|
16
18
|
import { OVERLAY_TEXT, RENDERED_TEXT, RENDER_TEXT, loadRenderMode, looksLikeShell, renderHtml } from "./render.js";
|
|
17
19
|
import * as receipts from "./receipts.js";
|
|
18
20
|
import { checkAllowed } from "./rules.js";
|
|
@@ -24,7 +26,7 @@ import { stampLanguage } from "./lang.js";
|
|
|
24
26
|
import { fitTranslation, translateReply } from "./translate.js";
|
|
25
27
|
import { saveSnapshot, savePolicy, savedText } from "./saved.js";
|
|
26
28
|
export { stampLanguage } from "./lang.js";
|
|
27
|
-
import { BUDGET, DIFF_OMITTED, MAX_AGENT, NO_BASELINE, NO_STORE_DIFF, OLD_READER, OTHER_SOURCE, env, fitEnvelope, wallEnv } from "./envelope.js";
|
|
29
|
+
import { BUDGET, DIFF_OMITTED, LABEL_BYTES, MAX_AGENT, NO_BASELINE, NO_STORE_DIFF, OLD_READER, OTHER_SOURCE, env, fitEnvelope, wallEnv } from "./envelope.js";
|
|
28
30
|
import { fileOptions, fileWarning } from "./filereply.js";
|
|
29
31
|
// The private mark authenticates the inspection to the reader, but is not part of the page's identity.
|
|
30
32
|
export const renderedFingerprint = (html, mark) => sha256(mark ? html.replaceAll(mark, "data-t-inspected") : html);
|
|
@@ -71,8 +73,10 @@ export function parseJob(read, body, bytes, url, wall, lang, rendered) {
|
|
|
71
73
|
export const looksHtml = (contentType, body) => /html/.test(mimeOf(contentType)) || /<(!doctype html|html|body|form)\b/i.test(body.slice(0, 65536));
|
|
72
74
|
// `wallCheck`: the response is HTML whatever reader its name chose, so its wall markers are looked for
|
|
73
75
|
// (Codex round 63: a login page served as text/html at /login.csv is still a wall).
|
|
74
|
-
export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered) {
|
|
76
|
+
export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered, secrets) {
|
|
75
77
|
const job = parseJob(read, body, bytes, url, wallCheck, lang, rendered);
|
|
78
|
+
if (job.op === "pdf" && secrets)
|
|
79
|
+
job.secrets = secrets;
|
|
76
80
|
const r = await runUntrusted(job, job.op === "pdf" ? Infinity : body.length);
|
|
77
81
|
// What the response said against what it is goes first, so the shield's cap never pushes it out.
|
|
78
82
|
if (fileFlags.length)
|
|
@@ -161,8 +165,14 @@ export function paginate(items, size, page, budget) {
|
|
|
161
165
|
return { slice: pages[page - 1] ?? [], pages: Math.max(1, pages.length) };
|
|
162
166
|
}
|
|
163
167
|
export const len = jsonBytes; // every page budget counts UTF-8 bytes of the JSON sent
|
|
168
|
+
// `written_by` (W-39, local mode only): who wrote each part shown, on a page where visitor text was found; `label`:
|
|
169
|
+
// the owner's written_by setting is on, so the reply carries it.
|
|
164
170
|
export async function buildContent(entry, opts) {
|
|
165
171
|
const p = entry.parsed;
|
|
172
|
+
// Receipt measurement repeats the final URL for every unit, on the main thread (as in tabs.ts).
|
|
173
|
+
const receiptUnits = opts.section === undefined ? p.sections.length : (p.sections.find((s) => s.n === opts.section)?.blocks.length ?? 0);
|
|
174
|
+
if (receiptUnits * (entry.final_url.length + 600) > 64 * 1024 * 1024)
|
|
175
|
+
throw new PageTooComplex();
|
|
166
176
|
const page = opts.page ?? 1;
|
|
167
177
|
const fetchedAt = entry.fetched_at;
|
|
168
178
|
// Receipts cite the URL that was actually read, and quote exact extracted text.
|
|
@@ -171,16 +181,28 @@ export async function buildContent(entry, opts) {
|
|
|
171
181
|
const titleReceipt = p.title ? [rc(p.title, null)] : [];
|
|
172
182
|
// A heading Tiresias generated (a PDF's "Page N") is a label, not text from the source, so it gets no receipt.
|
|
173
183
|
const labelled = (n) => !!p.sections.find((x) => x.n === n)?.generated;
|
|
184
|
+
// W-34 (v1.2): local mode sets a smaller budget. A local reply carries no receipts, links or forms (local.ts), so
|
|
185
|
+
// its pages are measured by the text it does carry; the receipts are still made and recorded for what is shown.
|
|
186
|
+
const local = opts.budget !== undefined;
|
|
187
|
+
const textBudget = opts.budget ?? BUDGET.sectionText;
|
|
188
|
+
// W-39 (v1.2): local mode, on a page where the reader found visitor text. The setting is read only then, so a bad
|
|
189
|
+
// value fails only the reads it would change, by name. Each label counts against the budget.
|
|
190
|
+
const visitorPage = local && p.sections.some((s) => s.visitor || s.blocks.some((b) => b.visitor));
|
|
191
|
+
const label = visitorPage && loadChoice("written_by") === "on";
|
|
192
|
+
const labelCost = label ? LABEL_BYTES : 0;
|
|
174
193
|
if (opts.question) {
|
|
175
194
|
// Keep the best passages that fit the page budget, receipts included.
|
|
176
195
|
const passages = [];
|
|
177
196
|
let used = 0;
|
|
178
197
|
const found = await runUntrusted({ op: "search", parsed: p, question: opts.question, k: 5 }, p.searchable_chars + opts.question.length); // the question counts too (round 84)
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
198
|
+
const by = [];
|
|
199
|
+
for (const hit of found) {
|
|
200
|
+
const { visitor, ...x } = hit; // the reader's mark never goes out as is
|
|
201
|
+
const cost = local ? len(x) + labelCost : len(x) + len(rc(x.text, x.section)) + (labelled(x.section) ? 0 : len(rc(x.heading, x.section)));
|
|
202
|
+
if (passages.length && used + cost > textBudget)
|
|
182
203
|
break;
|
|
183
204
|
passages.push(x);
|
|
205
|
+
by.push(visitor ?? 0);
|
|
184
206
|
used += cost;
|
|
185
207
|
}
|
|
186
208
|
// Every returned fact is covered: each passage, and each distinct heading shown with one.
|
|
@@ -194,19 +216,50 @@ export async function buildContent(entry, opts) {
|
|
|
194
216
|
}
|
|
195
217
|
rs.push(rc(x.text, x.section));
|
|
196
218
|
}
|
|
197
|
-
return { uc: { ...base, question: opts.question.slice(0, 500), passages }, receipts: rs };
|
|
219
|
+
return { uc: { ...base, question: opts.question.slice(0, 500), passages }, receipts: rs, ...(visitorPage && by.length ? { written_by: by, label } : {}) };
|
|
198
220
|
}
|
|
199
221
|
if (opts.section !== undefined) {
|
|
200
222
|
const s = p.sections.find((x) => x.n === opts.section);
|
|
201
223
|
if (!s)
|
|
202
224
|
return { uc: {}, receipts: [], error: `No section ${opts.section}. This page has ${p.sections.length} section${p.sections.length === 1 ? "" : "s"}.` };
|
|
203
225
|
// The page size counts what is really sent: the text AND its receipt (which repeats the URL).
|
|
204
|
-
|
|
205
|
-
|
|
226
|
+
// v1.2: local boundaries include the actual reply's metadata, labels and guarded representation.
|
|
227
|
+
// Reserve cache metadata on every page so fresh and cached reads keep the same boundaries.
|
|
228
|
+
// v1.2: a preformatted block can hold many paragraphs. Page those independently so fitting one
|
|
229
|
+
// oversized block never discards its later paragraphs. Receipts keep the original source block.
|
|
230
|
+
const units = s.blocks.flatMap(source => {
|
|
231
|
+
const text = shown(source);
|
|
232
|
+
if (!local || !opts.localTextBytes)
|
|
233
|
+
return [{ text, source }];
|
|
234
|
+
// v1.2: every preformatted paragraph keeps its own balanced original fence. The guard and
|
|
235
|
+
// pagination measure those delimiters too; receipts still quote the complete source block.
|
|
236
|
+
const pre = fencedText(text);
|
|
237
|
+
const parts = pre ? pre.body.split(/\n(?:[ \t]*\n)+/).map(part => part.replace(/^\n+|\n+$/g, "")).filter(part => part.length > 0)
|
|
238
|
+
.map((part, i) => `${i ? pre.close : pre.open}\n${part}\n${pre.close}`) : text.split("\n\n");
|
|
239
|
+
return parts.map(text => ({ text, source }));
|
|
240
|
+
});
|
|
241
|
+
const overhead = local && opts.localSectionBytes ? opts.localSectionBytes({ ...base, cached_at: entry.fetched_at,
|
|
242
|
+
section: { n: s.n, heading: s.heading, text: "", page: units.length, pages: units.length } }, label) : labelCost;
|
|
243
|
+
const pg = paginate(units, (b) => local && opts.localTextBytes ? opts.localTextBytes(b.text) + 4
|
|
244
|
+
: len(b.text) + (local ? 0 : len(rc(b.source.text, s.n))) + 4, page, textBudget - overhead);
|
|
245
|
+
const rs = [...new Set(pg.slice.map(b => b.source))].map(b => rc(b.text, s.n));
|
|
206
246
|
rs.unshift(...(page === 1 ? titleReceipt : []), ...(s.generated ? [] : [rc(s.heading, s.n)])); // the heading is returned on every page (a generated label is not a quote)
|
|
247
|
+
const texts = pg.slice.map(b => b.text);
|
|
248
|
+
const spans = [];
|
|
249
|
+
if (visitorPage) {
|
|
250
|
+
let start = 0;
|
|
251
|
+
pg.slice.forEach((b, i) => {
|
|
252
|
+
if (b.source.visitor)
|
|
253
|
+
spans.push([start, start + texts[i].length, b.source.visitor]);
|
|
254
|
+
start += texts[i].length + 2; // the blank line between blocks
|
|
255
|
+
});
|
|
256
|
+
}
|
|
207
257
|
return {
|
|
208
|
-
uc: { ...(page === 1 ? base : { final_url: entry.final_url }), section: { n: s.n, heading: s.heading, text:
|
|
258
|
+
uc: { ...(page === 1 ? base : { final_url: entry.final_url }), section: { n: s.n, heading: s.heading, text: texts.join("\n\n"), page, pages: pg.pages } },
|
|
209
259
|
receipts: rs,
|
|
260
|
+
...(visitorPage ? { section_visitor: spans, heading_visitor: s.visitor ?? 0 } : {}),
|
|
261
|
+
// A page of a section is one part: the visitor's when any paragraph on it is (it errs toward the visitor).
|
|
262
|
+
...(visitorPage ? { written_by: [pg.slice.reduce((n, b) => (b.source.visitor && (!n || b.source.visitor < n) ? b.source.visitor : n), s.visitor ?? 0)], label } : {}),
|
|
210
263
|
};
|
|
211
264
|
}
|
|
212
265
|
// Outline: each entry returns a heading and a preview, so each gets a receipt.
|
|
@@ -215,7 +268,11 @@ export async function buildContent(entry, opts) {
|
|
|
215
268
|
const receipts = [...(s.generated ? [] : [rc(s.heading, s.n)]), ...(preview ? [rc(preview, s.n)] : [])];
|
|
216
269
|
return { item: { section: s.n, level: s.level, heading: s.heading, preview }, receipts };
|
|
217
270
|
});
|
|
218
|
-
|
|
271
|
+
// Local mode shows each entry as its number and heading only, against a budget in the same proportion as the
|
|
272
|
+
// default ones (outline to section text).
|
|
273
|
+
const out = local
|
|
274
|
+
? paginate(units, (u) => len({ section: u.item.section, heading: u.item.heading }), page, Math.round((textBudget * BUDGET.outline) / BUDGET.sectionText))
|
|
275
|
+
: paginate(units, len, page, BUDGET.outline);
|
|
219
276
|
const links = paginate(p.links, len, page, BUDGET.links);
|
|
220
277
|
// W-04: forms, as fields, paged like links. Like links they carry no receipts (structure, not quotes).
|
|
221
278
|
const forms = p.forms ? paginate(p.forms, len, page, BUDGET.forms) : null;
|
|
@@ -229,7 +286,7 @@ export async function buildContent(entry, opts) {
|
|
|
229
286
|
links: links.slice,
|
|
230
287
|
...(forms ? { forms: forms.slice, ...(page === 1 && p.forms_omitted ? { forms_omitted: p.forms_omitted } : {}), ...(forms.slice.length ? { forms_hint: FORMS_HINT } : {}) } : {}),
|
|
231
288
|
page,
|
|
232
|
-
pages: Math.max(out.pages, links.pages, forms?.pages ?? 1),
|
|
289
|
+
pages: local ? out.pages : Math.max(out.pages, links.pages, forms?.pages ?? 1), // local replies list no links or forms
|
|
233
290
|
section_count: p.sections.length,
|
|
234
291
|
links_omitted: p.links_omitted,
|
|
235
292
|
structured: p.structured_count || p.structured_skipped
|
|
@@ -240,6 +297,8 @@ export async function buildContent(entry, opts) {
|
|
|
240
297
|
receipts: rs,
|
|
241
298
|
};
|
|
242
299
|
}
|
|
300
|
+
// The fixed note on a read of an empty JavaScript shell that was not rendered (local mode looks for it: local.ts).
|
|
301
|
+
export const SHELL_NOTE = "This page looks like a JavaScript-only shell. Ask for rendering to read its content.";
|
|
243
302
|
export const FORMS_HINT = "Forms are listed as fields (hidden ones by name only). To see exactly what a form would send, call form_preview with this url, the form's index and your values. Sending needs a permission slip, which only the owner grants from the command line; a form with not_allowed is never sent.";
|
|
244
303
|
// One page of a kept list of changes. Added blocks are on the page now, so each gets a receipt.
|
|
245
304
|
// Removed blocks are not on the page any more: they carry no receipt, and `kind` says so.
|
|
@@ -273,19 +332,23 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
273
332
|
// reveals an earlier visit and never records this one.
|
|
274
333
|
const priv = isPrivate();
|
|
275
334
|
const cache = priv ? null : readJson(cacheFile);
|
|
335
|
+
// v1.2: old shell flags also obey the content test before a cached note or automatic render.
|
|
336
|
+
if (cache?.shell && !looksLikeShell(cache.parsed, true))
|
|
337
|
+
delete cache.shell;
|
|
276
338
|
// An entry parsed by an older reader keeps its provenance (where it was read
|
|
277
339
|
// from, its baseline) but its parsed form is rebuilt from the next response.
|
|
278
340
|
// Older rendered entries lack the inspection codes and predate the textarea inspection. Rebuild them.
|
|
279
|
-
const staleParse = !!cache && (cache.parser !== PARSER_VERSION || (!!cache.rendered && !cache.render_inspection));
|
|
341
|
+
const staleParse = !!cache && (cache.parser !== PARSER_VERSION || (!!cache.rendered && !cache.render_inspection) || (opts.budget !== undefined && (!cache.visitor_scopes || !cache.visitor_main_scopes || !cache.visitor_heading_text)));
|
|
280
342
|
// W-30: the owner's "doors" setting, read (and checked: a bad value fails the read, by name) on EVERY read,
|
|
281
343
|
// whichever path it takes (Codex round 2).
|
|
282
344
|
const doorsMode = tool === "read" ? loadDoorsMode() : "wait";
|
|
283
345
|
// W-10 (v0.7): the owner's render setting (a bad value fails the read, by name). A read that asks for rendering
|
|
284
346
|
// when it is allowed, and a check of a copy that was rendered, never reuse a plain copy as is.
|
|
285
347
|
const renderMode = tool === "read" || tool === "check" ? loadRenderMode() : "off";
|
|
348
|
+
const autoShell = renderMode === "auto" || (renderMode === "ask" && opts.renderIfShell === true);
|
|
286
349
|
// Explicit renders and checks of rendered copies need a body and a new browser run, even after a 304.
|
|
287
350
|
// The one shortcut is an already rendered copy still fresh under the site's cache rules (below).
|
|
288
|
-
const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered) || (
|
|
351
|
+
const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered) || (autoShell && !!cache?.shell && !cache.rendered));
|
|
289
352
|
const needsBody = rerender || (renderMode === "off" && !!cache?.rendered);
|
|
290
353
|
// W-28 (v0.8): a read that asks to save, when the owner allows it (a bad setting fails the read, by name). A save
|
|
291
354
|
// keeps what THIS read received, so it never uses a fresh copy, a 304 or a cached rendering: it always fetches.
|
|
@@ -387,6 +450,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
387
450
|
// All parsing of the page runs under a hard time budget (sandbox.ts).
|
|
388
451
|
let { parsed, wall: pageWall } = await parsePage(k, res.body, res.bytes, res.url, res.fileFlags, looksHtml(ct, res.body), res.headers.get("content-language"));
|
|
389
452
|
throwIfCancelled();
|
|
453
|
+
const hasScript = k === "html" && /<script\b/i.test(res.body);
|
|
390
454
|
// W-10 (v0.7): rendering. Asked for by the read, or (owner's "auto") an empty JavaScript shell, or a check of a
|
|
391
455
|
// copy that was rendered (so like is compared with like). The rendered HTML goes through the same reader and
|
|
392
456
|
// shield; a render that stops leaves the plain read in place, and the reply says why.
|
|
@@ -397,7 +461,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
397
461
|
// the same outline; when the page's own body is unchanged, the rendered copy is reused without starting a browser
|
|
398
462
|
// (an explicit render asks for a new rendering).
|
|
399
463
|
const plainHash = sha256(Buffer.concat([Buffer.from(`${k}|${(res.fileFlags ?? []).join(",")}|`), res.bytes ?? Buffer.from(res.body, "utf8")]));
|
|
400
|
-
const wantRender = !!opts.render || (renderMode !== "off" && !!cache?.rendered) || (
|
|
464
|
+
const wantRender = !!opts.render || (renderMode !== "off" && !!cache?.rendered) || (autoShell && looksLikeShell(parsed, hasScript));
|
|
401
465
|
const reuse = tool !== "check" && !opts.fresh && !opts.render && !sv && !staleParse && !!cache?.rendered && !!cache.render_hash && cache.plain_hash === plainHash && cache.source_url === source && cache.final_url === res.url;
|
|
402
466
|
// v0.7 track 2: a CAPTCHA or login wall found in the plain HTML's markers waits for the render's verdict when a render
|
|
403
467
|
// will decide this read (the browser sees whether the box is shown). A bot check (it is passed by running its script,
|
|
@@ -466,12 +530,12 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
|
|
|
466
530
|
const { fresh_until: _f, reuse_blocked: _b, ...rest } = cache; // a full 200 judges freshness from scratch
|
|
467
531
|
// Same bytes: the copy is kept, but how the site now describes it is taken fresh (round 83).
|
|
468
532
|
// A saving read uses the fresh parse; otherwise only the current language hint needs refreshing.
|
|
469
|
-
entry = { ...rest, ...fresh, parsed: sv ? parsed : { ...rest.parsed, language: parsed.language }, ...(renderedHash && render ? { render_inspection: inspectionCodes(render) } : {}), content_type: ct, shell:
|
|
533
|
+
entry = { ...rest, ...fresh, parsed: sv ? parsed : { ...rest.parsed, language: parsed.language }, ...(renderedHash && render ? { render_inspection: inspectionCodes(render) } : {}), content_type: ct, shell: looksLikeShell(parsed, hasScript) ? true : undefined, door_detail: doorDetail, redirected: res.redirected, etag: res.headers.get("etag") ?? cache.etag, last_modified: res.headers.get("last-modified") ?? cache.last_modified, checked_at: now };
|
|
470
534
|
}
|
|
471
535
|
else
|
|
472
536
|
entry = {
|
|
473
|
-
parser: PARSER_VERSION, url, source_url: source, final_url: res.url, door, door_detail: doorDetail, hash, content_type: ct, parsed,
|
|
474
|
-
...(
|
|
537
|
+
parser: PARSER_VERSION, visitor_scopes: true, visitor_main_scopes: true, visitor_heading_text: true, url, source_url: source, final_url: res.url, door, door_detail: doorDetail, hash, content_type: ct, parsed,
|
|
538
|
+
...(looksLikeShell(parsed, hasScript) ? { shell: true } : {}),
|
|
475
539
|
etag: res.headers.get("etag") ?? undefined,
|
|
476
540
|
last_modified: res.headers.get("last-modified") ?? undefined,
|
|
477
541
|
// an identical body keeps its original read time, and with it the agent that first fetched it
|
|
@@ -534,6 +598,8 @@ function afterDeferred(e, origin) {
|
|
|
534
598
|
return e;
|
|
535
599
|
}
|
|
536
600
|
export async function finishPage(tool, url, entry, unchanged, changedFrom, opts, od, now, persist, meta, diff, notKept = false, serve = {}) {
|
|
601
|
+
if (!entry.source)
|
|
602
|
+
noteGetActions(entry.parsed.forms, !notKept);
|
|
537
603
|
const wantsContent = opts.question !== undefined || opts.section !== undefined || opts.page !== undefined || (tool === "read" && (opts.translate !== undefined || !!opts.save));
|
|
538
604
|
// W-28 (v0.8): the snapshot is written before the receipts are recorded, so each recorded receipt names it.
|
|
539
605
|
const sv = tool === "read" ? serve.save : undefined;
|
|
@@ -559,10 +625,8 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
|
|
|
559
625
|
renderNote(e, serve.render ?? (entry.rendered ? cachedRender(entry) : undefined), meta);
|
|
560
626
|
if (serve.cache)
|
|
561
627
|
e.cache = serve.cache;
|
|
562
|
-
if (tool === "read" && entry.shell && !entry.rendered && !opts.render)
|
|
563
|
-
|
|
564
|
-
e.message = e.message ? `${e.message} ${note}` : note;
|
|
565
|
-
}
|
|
628
|
+
if (tool === "read" && entry.shell && !entry.rendered && !opts.render)
|
|
629
|
+
e.message = e.message ? `${e.message} ${SHELL_NOTE}` : SHELL_NOTE;
|
|
566
630
|
if (fromCache)
|
|
567
631
|
e.untrusted_content = { ...(e.untrusted_content ?? {}), ...cacheUc };
|
|
568
632
|
if (sv) {
|
|
@@ -639,6 +703,8 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
|
|
|
639
703
|
const c = await buildContent(entry, opts);
|
|
640
704
|
if (opts.question !== undefined && Array.isArray(c.uc.passages))
|
|
641
705
|
meta.matched = c.uc.passages.length;
|
|
706
|
+
if (c.written_by)
|
|
707
|
+
meta.written_by = c.written_by; // W-39: the levels go to the history, for the owner
|
|
642
708
|
if (c.error) {
|
|
643
709
|
persist(); // the page itself was read fine; only the section asked for is missing
|
|
644
710
|
if (sv)
|
|
@@ -657,6 +723,12 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
|
|
|
657
723
|
});
|
|
658
724
|
if (entry.parsed.forms && reply.counts)
|
|
659
725
|
reply.counts.forms = entry.parsed.forms.length + (entry.parsed.forms_omitted ?? 0); // W-04
|
|
726
|
+
if (c.written_by && c.label)
|
|
727
|
+
reply.written_by = c.written_by; // W-39: local.ts makes the labels from it
|
|
728
|
+
if (c.section_visitor)
|
|
729
|
+
reply.section_visitor = c.section_visitor;
|
|
730
|
+
if (c.heading_visitor !== undefined)
|
|
731
|
+
reply.heading_visitor = c.heading_visitor;
|
|
660
732
|
const out = withCache(reply);
|
|
661
733
|
// W-26 (v0.7): the owner's translation hook, only when asked and only on a read. The receipts above were
|
|
662
734
|
// built from the original text; the translation never becomes one.
|
|
@@ -671,10 +743,26 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
|
|
|
671
743
|
delete r.snapshot;
|
|
672
744
|
persist();
|
|
673
745
|
saveNow(kept);
|
|
674
|
-
|
|
675
|
-
|
|
746
|
+
const recorded = !priv && recordReceipts(kept);
|
|
747
|
+
if (tool === "read" && opts.receipts === "log" && recorded)
|
|
748
|
+
leaveReceiptsInLog(out);
|
|
676
749
|
return savedOn(out);
|
|
677
750
|
}
|
|
751
|
+
// v1.2 (field list, 2026-11-14): a reply repeats every heading, preview and paragraph it shows as a receipt, which
|
|
752
|
+
// about doubles what the reading model takes in. A caller may ask for the receipts to stay in the local log
|
|
753
|
+
// (read's `receipts: "log"`). The receipts are made and recorded exactly as before, and counts.receipts still says
|
|
754
|
+
// how many; only the list is left out of this reply, and the reply says so with a fixed top-level mark. It is left
|
|
755
|
+
// out only when every receipt was written whole: in private mode, when the log could not be written, or when the
|
|
756
|
+
// logged copy of any receipt was cut for length (Codex review, 2026-11-14), the log does not hold them, so the
|
|
757
|
+
// reply carries them as usual. A reply with no content (the one-line "unchanged" answer) has none to leave out,
|
|
758
|
+
// and a file reply (filereply.ts) carries its receipts as before.
|
|
759
|
+
export function leaveReceiptsInLog(e) {
|
|
760
|
+
const uc = e.untrusted_content;
|
|
761
|
+
if (!uc || !Array.isArray(uc.receipts))
|
|
762
|
+
return;
|
|
763
|
+
delete uc.receipts;
|
|
764
|
+
e.receipts_in_log = true;
|
|
765
|
+
}
|
|
678
766
|
// The top-level `saved` (fixed codes, Tiresias's own hash and counts) and its fixed sentence.
|
|
679
767
|
export function addSaved(e, top) {
|
|
680
768
|
const previous = e.saved;
|
|
@@ -695,12 +783,15 @@ export function renderNote(e, render, meta) {
|
|
|
695
783
|
e.message = e.message ? `${e.message} ${text}` : text;
|
|
696
784
|
return e;
|
|
697
785
|
}
|
|
786
|
+
// True when every receipt was written to the log WHOLE. The logged copy of a very long quote or URL is cut
|
|
787
|
+
// (receipts.ts), so a cut record, like a failed write, means the log does not hold what the reply quoted.
|
|
698
788
|
export function recordReceipts(rs) {
|
|
699
789
|
try {
|
|
700
|
-
receipts.record(rs);
|
|
790
|
+
return receipts.record(rs) === 0;
|
|
701
791
|
}
|
|
702
792
|
catch (e) {
|
|
703
793
|
console.error(`Could not write receipts: ${e.message}`); // a log failure never fails the browsing call
|
|
794
|
+
return false;
|
|
704
795
|
}
|
|
705
796
|
}
|
|
706
797
|
export const noteDoors = (meta, ad) => {
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// v1.2: recognize the reader's fenced presentation, including list indentation and a local warning.
|
|
3
|
+
export function fencedText(text) {
|
|
4
|
+
const head = /^((?:\[[^\n]*\]\n)?[ \t]*(?:(?:[-*+]|\d+[.)]) )?)(`{3,})\n/.exec(text);
|
|
5
|
+
if (!head)
|
|
6
|
+
return undefined;
|
|
7
|
+
const end = text.lastIndexOf("\n");
|
|
8
|
+
const close = text.slice(end + 1);
|
|
9
|
+
if (end < head[0].length || close.trimStart() !== head[2])
|
|
10
|
+
return undefined;
|
|
11
|
+
return { open: head[0].slice(0, -1), body: text.slice(head[0].length, end), close };
|
|
12
|
+
}
|
package/dist/profiles.js
CHANGED
|
@@ -11,7 +11,7 @@ import { randomBytes } from "node:crypto";
|
|
|
11
11
|
import fs from "node:fs";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { parseSignatureAgent } from "./config.js";
|
|
14
|
-
import { KEY_FILE } from "./keys.js";
|
|
14
|
+
import { KEY_FILE, PREVIOUS_KEY_FILE } from "./keys.js";
|
|
15
15
|
import { DEFAULT_PROFILE, NOFOLLOW, NONBLOCK, PROFILE_RE, ProfileError, checkProfileName, checkSub, ensureRoot, profileDir, profilesRoot, readOwnedFile, readRoot, rootDir, } from "./store.js";
|
|
16
16
|
const META = "profile.json"; // { name, created_at }, written once at create
|
|
17
17
|
const me = () => (typeof process.getuid === "function" ? process.getuid() : -1);
|
|
@@ -138,8 +138,8 @@ function plan(dir) {
|
|
|
138
138
|
stack.push(f);
|
|
139
139
|
else if (s.isFile()) {
|
|
140
140
|
p.files.push({ file: f, size: s.size });
|
|
141
|
-
if (rel === path.join("keys", KEY_FILE))
|
|
142
|
-
p.hasKey = true;
|
|
141
|
+
if (rel === path.join("keys", KEY_FILE) || rel === path.join("keys", PREVIOUS_KEY_FILE))
|
|
142
|
+
p.hasKey = true; // v1.1: a rotation's previous key is a private key too
|
|
143
143
|
}
|
|
144
144
|
else
|
|
145
145
|
throw bad("something that is not a file or a folder");
|
package/dist/queue.js
CHANGED
|
@@ -1,16 +1,13 @@
|
|
|
1
|
-
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// N-06: the per-site queue. Every request (pages, discovery probes, robots.txt) waits its turn here and
|
|
3
|
-
// is spaced from the one before it by the site's spacing (1 s by default, or a larger Crawl-delay).
|
|
4
1
|
import { cooldownLeft, onCooldown } from "./cooldown.js";
|
|
5
2
|
import { FetchError, Refused } from "./errors.js";
|
|
6
|
-
import { acquireActive, extendNext, noteCompletion, reserve } from "./hosts.js";
|
|
3
|
+
import { acquireActive, extendNext, noteCompletion, reserve, sharesSpacing, spacingMs } from "./hosts.js";
|
|
7
4
|
import { MAX_HOSTS } from "./lru.js";
|
|
8
5
|
import { canonOrigin } from "./sitekey.js";
|
|
9
6
|
import { noteApplied, siteSpacing } from "./sites.js";
|
|
10
7
|
// Bounded (v0.5.1): a long-running MCP server keeps at most MAX_HOSTS sites in each. A queue entry is
|
|
11
8
|
// removed when its last request has had its turn, and a site's waiting set when it is empty.
|
|
12
|
-
// When each site was last called. Kept while its spacing can still matter
|
|
13
|
-
//
|
|
9
|
+
// When each site was last called. Kept while its spacing can still matter,
|
|
10
|
+
// then pruned (Codex round 78: evicting by count could drop a spacing still owed).
|
|
14
11
|
const lastHit = new Map();
|
|
15
12
|
const HIT_KEEP_MS = 60_000;
|
|
16
13
|
// The earliest the NEXT request may start, as the last request's own spacing said (Codex round 79: a
|
|
@@ -79,15 +76,7 @@ export function sleep(ms, signal) {
|
|
|
79
76
|
signal.addEventListener("abort", stop, { once: true });
|
|
80
77
|
});
|
|
81
78
|
}
|
|
82
|
-
|
|
83
|
-
// Crawl-delay). The slot is reserved in the shared hosts/ file too (hosts.ts), so other processes space
|
|
84
|
-
// their requests to this site after it.
|
|
85
|
-
// A queued request that is cancelled (its call ended, or a cooldown began)
|
|
86
|
-
// leaves the queue at once and does not move the clock.
|
|
87
|
-
// With `hold`, the turn is kept until the returned function is called (Codex round 73: one request at a
|
|
88
|
-
// time means until its answer has been read, not only until it was sent); without it, the turn passes on
|
|
89
|
-
// as soon as the request may start.
|
|
90
|
-
export async function throttle(origin, delay, outer, url, hold = false) {
|
|
79
|
+
export async function throttle(origin, delay, outer, url, hold = false, options = {}) {
|
|
91
80
|
origin = canonOrigin(origin); // v0.6: "example.com." and "example.com" share one queue and one clock
|
|
92
81
|
let target = null;
|
|
93
82
|
try {
|
|
@@ -116,6 +105,8 @@ export async function throttle(origin, delay, outer, url, hold = false) {
|
|
|
116
105
|
const tail = prev.then(() => turn);
|
|
117
106
|
chains.set(origin, tail);
|
|
118
107
|
let activeRelease = () => { };
|
|
108
|
+
let activeHeld = false;
|
|
109
|
+
const visit = options.robots;
|
|
119
110
|
let admittedMs = 0; // the spacing this request was admitted with
|
|
120
111
|
const done = () => {
|
|
121
112
|
// v0.6 (Codex round 9): a held turn ends when the answer has been read. The site's clocks are moved on from
|
|
@@ -126,7 +117,9 @@ export async function throttle(origin, delay, outer, url, hold = false) {
|
|
|
126
117
|
const d = Math.max(admittedMs, delayOf.get(origin) ?? 0); // a Crawl-delay learned meanwhile counts too
|
|
127
118
|
lastHit.set(origin, t);
|
|
128
119
|
nextOk.set(origin, Math.max(nextOk.get(origin) ?? 0, t + d)); // this process's clock (private calls too)
|
|
129
|
-
noteCompletion(origin, t, t + d); // other processes
|
|
120
|
+
const shared = noteCompletion(origin, t, t + d); // other processes too
|
|
121
|
+
if (visit?.requests === 1 && (!sharesSpacing() || activeHeld))
|
|
122
|
+
visit.pass = { origin, last: t, next: nextOk.get(origin), shared };
|
|
130
123
|
}
|
|
131
124
|
activeRelease();
|
|
132
125
|
release();
|
|
@@ -142,7 +135,7 @@ export async function throttle(origin, delay, outer, url, hold = false) {
|
|
|
142
135
|
await Promise.race([prev, cancelled]);
|
|
143
136
|
// Holding the turn: first the cross-process lease (Codex round 76: waiting for it AFTER reserving a
|
|
144
137
|
// slot let the slot go stale), then the shared slot and its spacing.
|
|
145
|
-
activeRelease = await Promise.race([acquireActive(origin, mine.signal), cancelled]);
|
|
138
|
+
activeRelease = await Promise.race([acquireActive(origin, mine.signal, () => { activeHeld = true; }), cancelled]);
|
|
146
139
|
// W-25 (v0.6): a site's own spacing (sites.ts) raises the spacing. Like every spacing since v0.5.1 it is per
|
|
147
140
|
// ORIGIN (scheme, host and port). Settings are the call's snapshot, read once when the call started
|
|
148
141
|
// (coordinator's rule after Codex round 11): a change saved meanwhile applies to calls that start after it.
|
|
@@ -153,10 +146,21 @@ export async function throttle(origin, delay, outer, url, hold = false) {
|
|
|
153
146
|
throw new Error("queue-cancelled");
|
|
154
147
|
if (cooldownLeft(origin) > 0)
|
|
155
148
|
throw cooldownRefusal(origin, url);
|
|
156
|
-
const
|
|
149
|
+
const pass = options.pass;
|
|
150
|
+
const canPass = pass?.origin === origin && eff * 1000 <= spacingMs() &&
|
|
151
|
+
(delayOf.get(origin) ?? 0) <= spacingMs() && lastHit.get(origin) === pass.last &&
|
|
152
|
+
nextOk.get(origin) === pass.next && pass.next === pass.last + spacingMs();
|
|
153
|
+
// Private, unshared calls waive only their local clock. Shared calls also need a successfully
|
|
154
|
+
// held lease and the exact stamp this robots completion wrote, checked again under reserve's lock.
|
|
155
|
+
// A stale call-owned pass is harmless: any intervening request changes these clocks/stamps.
|
|
156
|
+
const stamp = canPass && (!sharesSpacing() || (activeHeld && pass.shared?.last === pass.last && pass.shared?.next === pass.next))
|
|
157
|
+
? (pass.shared ?? { last: pass.last, next: pass.next }) : undefined;
|
|
158
|
+
const slot = reserve(origin, eff * 1000, Math.max((lastHit.get(origin) ?? 0) + eff * 1000, nextOk.get(origin) ?? 0), true, stamp);
|
|
157
159
|
const wait = slot - Date.now();
|
|
158
160
|
if (wait <= 0)
|
|
159
161
|
break;
|
|
162
|
+
if (wait > 30_000)
|
|
163
|
+
throw new Refused("The site's spacing wait is longer than Tiresias will wait.", undefined, Math.ceil(wait / 1000), url);
|
|
160
164
|
await Promise.race([sleep(wait, mine.signal), cancelled]);
|
|
161
165
|
}
|
|
162
166
|
// The checks that matter: right before sending, not before queueing.
|
|
@@ -198,9 +202,9 @@ export class Turns {
|
|
|
198
202
|
leaveOthers(_u) {
|
|
199
203
|
this.giveBack();
|
|
200
204
|
}
|
|
201
|
-
async take(u, delay, signal) {
|
|
205
|
+
async take(u, delay, signal, options = {}) {
|
|
202
206
|
this.giveBack();
|
|
203
|
-
this.held = { origin: u.origin, release: await throttle(u.origin, delay, signal, u.href, true) };
|
|
207
|
+
this.held = { origin: u.origin, release: await throttle(u.origin, delay, signal, u.href, true, options) };
|
|
204
208
|
}
|
|
205
209
|
giveBack() {
|
|
206
210
|
this.held?.release();
|