@raibitsi/tiresias 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +88 -0
  2. package/README.md +72 -33
  3. package/data/README.md +33 -0
  4. package/data/guard-ordinary-urls.txt +102 -0
  5. package/data/guard-starter.jsonl +424 -0
  6. package/dist/aimed.js +23 -0
  7. package/dist/anchor.js +26 -13
  8. package/dist/browsecli.js +17 -13
  9. package/dist/cdp.js +22 -1
  10. package/dist/cleanup.js +3 -1
  11. package/dist/cli.js +216 -68
  12. package/dist/config.js +75 -1
  13. package/dist/confirm.js +20 -5
  14. package/dist/cooldown.js +57 -32
  15. package/dist/dailylog.js +11 -5
  16. package/dist/dom.js +13 -8
  17. package/dist/doors.js +1 -1
  18. package/dist/engine.js +9 -4
  19. package/dist/envelope.js +10 -1
  20. package/dist/fetcher.js +29 -2
  21. package/dist/filereply.js +7 -1
  22. package/dist/formflow.js +21 -3
  23. package/dist/forms.js +337 -42
  24. package/dist/getactions.js +119 -0
  25. package/dist/guard.js +1240 -0
  26. package/dist/guardclient.js +245 -0
  27. package/dist/guardnetwork.js +22 -0
  28. package/dist/history.js +11 -5
  29. package/dist/hook.js +2 -0
  30. package/dist/hosts.js +34 -27
  31. package/dist/keys.js +420 -18
  32. package/dist/local.js +537 -0
  33. package/dist/mcp.js +56 -25
  34. package/dist/needs.js +9 -5
  35. package/dist/parentwatch.js +6 -16
  36. package/dist/pdf.js +4 -1
  37. package/dist/pdfchild.js +0 -2
  38. package/dist/pdfproc.js +26 -11
  39. package/dist/pdfwatch.js +14 -0
  40. package/dist/pipeline.js +117 -26
  41. package/dist/preformatted.js +12 -0
  42. package/dist/profiles.js +3 -3
  43. package/dist/queue.js +24 -20
  44. package/dist/reader.js +441 -87
  45. package/dist/receipts.js +7 -2
  46. package/dist/render.js +156 -27
  47. package/dist/renderactions.js +69 -0
  48. package/dist/resume.js +8 -6
  49. package/dist/rules.js +24 -5
  50. package/dist/safebrowsing.js +3 -3
  51. package/dist/safety.js +15 -7
  52. package/dist/sandbox.js +1 -1
  53. package/dist/saved.js +24 -9
  54. package/dist/savedlock.js +99 -5
  55. package/dist/settings.js +10 -3
  56. package/dist/shelf.js +1 -1
  57. package/dist/shield.js +15 -0
  58. package/dist/slips.js +43 -1
  59. package/dist/store.js +119 -25
  60. package/dist/tabs.js +1 -1
  61. package/dist/textview.js +16 -0
  62. package/dist/translate.js +22 -2
  63. package/dist/types.js +9 -2
  64. package/dist/watchflow.js +1 -2
  65. package/dist/work.js +10 -2
  66. package/docs/LOCAL-MODELS.md +271 -0
  67. package/package.json +5 -2
package/dist/pipeline.js CHANGED
@@ -2,9 +2,10 @@
2
2
  // The read pipeline: choose a source, fetch it, decide walls and files, parse, cache, and build the reply
3
3
  // (read, check and doors).
4
4
  // (Split out of engine.ts in v0.5.1, behavior unchanged.)
5
+ import { noteGetActions } from "./getactions.js";
5
6
  import { hostKey } from "./sitekey.js";
6
7
  import { startDoorProbe } from "./background.js";
7
- import { loadDoorsMode } from "./config.js";
8
+ import { loadChoice, loadDoorsMode } from "./config.js";
8
9
  import { cooldownSeconds } from "./cooldown.js";
9
10
  import { FLAG_TEXT, mimeOf } from "./files.js";
10
11
  import { cachedDoors, noStore, probeOrigin, summarize } from "./doors.js";
@@ -12,7 +13,8 @@ import { TiresiasError } from "./errors.js";
12
13
  import { politeFetch } from "./fetcher.js";
13
14
  import { after304, fromResponse, isFresh } from "./freshness.js";
14
15
  import { shown } from "./reader.js";
15
- import { runUntrusted } from "./sandbox.js";
16
+ import { fencedText } from "./preformatted.js";
17
+ import { PageTooComplex, runUntrusted } from "./sandbox.js";
16
18
  import { OVERLAY_TEXT, RENDERED_TEXT, RENDER_TEXT, loadRenderMode, looksLikeShell, renderHtml } from "./render.js";
17
19
  import * as receipts from "./receipts.js";
18
20
  import { checkAllowed } from "./rules.js";
@@ -24,7 +26,7 @@ import { stampLanguage } from "./lang.js";
24
26
  import { fitTranslation, translateReply } from "./translate.js";
25
27
  import { saveSnapshot, savePolicy, savedText } from "./saved.js";
26
28
  export { stampLanguage } from "./lang.js";
27
- import { BUDGET, DIFF_OMITTED, MAX_AGENT, NO_BASELINE, NO_STORE_DIFF, OLD_READER, OTHER_SOURCE, env, fitEnvelope, wallEnv } from "./envelope.js";
29
+ import { BUDGET, DIFF_OMITTED, LABEL_BYTES, MAX_AGENT, NO_BASELINE, NO_STORE_DIFF, OLD_READER, OTHER_SOURCE, env, fitEnvelope, wallEnv } from "./envelope.js";
28
30
  import { fileOptions, fileWarning } from "./filereply.js";
29
31
  // The private mark authenticates the inspection to the reader, but is not part of the page's identity.
30
32
  export const renderedFingerprint = (html, mark) => sha256(mark ? html.replaceAll(mark, "data-t-inspected") : html);
@@ -71,8 +73,10 @@ export function parseJob(read, body, bytes, url, wall, lang, rendered) {
71
73
  export const looksHtml = (contentType, body) => /html/.test(mimeOf(contentType)) || /<(!doctype html|html|body|form)\b/i.test(body.slice(0, 65536));
72
74
  // `wallCheck`: the response is HTML whatever reader its name chose, so its wall markers are looked for
73
75
  // (Codex round 63: a login page served as text/html at /login.csv is still a wall).
74
- export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered) {
76
+ export async function parsePage(read, body, bytes, url, fileFlags = [], wallCheck = false, lang, rendered, secrets) {
75
77
  const job = parseJob(read, body, bytes, url, wallCheck, lang, rendered);
78
+ if (job.op === "pdf" && secrets)
79
+ job.secrets = secrets;
76
80
  const r = await runUntrusted(job, job.op === "pdf" ? Infinity : body.length);
77
81
  // What the response said against what it is goes first, so the shield's cap never pushes it out.
78
82
  if (fileFlags.length)
@@ -161,8 +165,14 @@ export function paginate(items, size, page, budget) {
161
165
  return { slice: pages[page - 1] ?? [], pages: Math.max(1, pages.length) };
162
166
  }
163
167
  export const len = jsonBytes; // every page budget counts UTF-8 bytes of the JSON sent
168
+ // `written_by` (W-39, local mode only): who wrote each part shown, on a page where visitor text was found; `label`:
169
+ // the owner's written_by setting is on, so the reply carries it.
164
170
  export async function buildContent(entry, opts) {
165
171
  const p = entry.parsed;
172
+ // Receipt measurement repeats the final URL for every unit, on the main thread (as in tabs.ts).
173
+ const receiptUnits = opts.section === undefined ? p.sections.length : (p.sections.find((s) => s.n === opts.section)?.blocks.length ?? 0);
174
+ if (receiptUnits * (entry.final_url.length + 600) > 64 * 1024 * 1024)
175
+ throw new PageTooComplex();
166
176
  const page = opts.page ?? 1;
167
177
  const fetchedAt = entry.fetched_at;
168
178
  // Receipts cite the URL that was actually read, and quote exact extracted text.
@@ -171,16 +181,28 @@ export async function buildContent(entry, opts) {
171
181
  const titleReceipt = p.title ? [rc(p.title, null)] : [];
172
182
  // A heading Tiresias generated (a PDF's "Page N") is a label, not text from the source, so it gets no receipt.
173
183
  const labelled = (n) => !!p.sections.find((x) => x.n === n)?.generated;
184
+ // W-34 (v1.2): local mode sets a smaller budget. A local reply carries no receipts, links or forms (local.ts), so
185
+ // its pages are measured by the text it does carry; the receipts are still made and recorded for what is shown.
186
+ const local = opts.budget !== undefined;
187
+ const textBudget = opts.budget ?? BUDGET.sectionText;
188
+ // W-39 (v1.2): local mode, on a page where the reader found visitor text. The setting is read only then, so a bad
189
+ // value fails only the reads it would change, by name. Each label counts against the budget.
190
+ const visitorPage = local && p.sections.some((s) => s.visitor || s.blocks.some((b) => b.visitor));
191
+ const label = visitorPage && loadChoice("written_by") === "on";
192
+ const labelCost = label ? LABEL_BYTES : 0;
174
193
  if (opts.question) {
175
194
  // Keep the best passages that fit the page budget, receipts included.
176
195
  const passages = [];
177
196
  let used = 0;
178
197
  const found = await runUntrusted({ op: "search", parsed: p, question: opts.question, k: 5 }, p.searchable_chars + opts.question.length); // the question counts too (round 84)
179
- for (const x of found) {
180
- const cost = len(x) + len(rc(x.text, x.section)) + (labelled(x.section) ? 0 : len(rc(x.heading, x.section)));
181
- if (passages.length && used + cost > BUDGET.sectionText)
198
+ const by = [];
199
+ for (const hit of found) {
200
+ const { visitor, ...x } = hit; // the reader's mark never goes out as is
201
+ const cost = local ? len(x) + labelCost : len(x) + len(rc(x.text, x.section)) + (labelled(x.section) ? 0 : len(rc(x.heading, x.section)));
202
+ if (passages.length && used + cost > textBudget)
182
203
  break;
183
204
  passages.push(x);
205
+ by.push(visitor ?? 0);
184
206
  used += cost;
185
207
  }
186
208
  // Every returned fact is covered: each passage, and each distinct heading shown with one.
@@ -194,19 +216,50 @@ export async function buildContent(entry, opts) {
194
216
  }
195
217
  rs.push(rc(x.text, x.section));
196
218
  }
197
- return { uc: { ...base, question: opts.question.slice(0, 500), passages }, receipts: rs };
219
+ return { uc: { ...base, question: opts.question.slice(0, 500), passages }, receipts: rs, ...(visitorPage && by.length ? { written_by: by, label } : {}) };
198
220
  }
199
221
  if (opts.section !== undefined) {
200
222
  const s = p.sections.find((x) => x.n === opts.section);
201
223
  if (!s)
202
224
  return { uc: {}, receipts: [], error: `No section ${opts.section}. This page has ${p.sections.length} section${p.sections.length === 1 ? "" : "s"}.` };
203
225
  // The page size counts what is really sent: the text AND its receipt (which repeats the URL).
204
- const pg = paginate(s.blocks, (b) => len(shown(b)) + len(rc(b.text, s.n)) + 4, page, BUDGET.sectionText);
205
- const rs = pg.slice.map((b) => rc(b.text, s.n));
226
+ // v1.2: local boundaries include the actual reply's metadata, labels and guarded representation.
227
+ // Reserve cache metadata on every page so fresh and cached reads keep the same boundaries.
228
+ // v1.2: a preformatted block can hold many paragraphs. Page those independently so fitting one
229
+ // oversized block never discards its later paragraphs. Receipts keep the original source block.
230
+ const units = s.blocks.flatMap(source => {
231
+ const text = shown(source);
232
+ if (!local || !opts.localTextBytes)
233
+ return [{ text, source }];
234
+ // v1.2: every preformatted paragraph keeps its own balanced original fence. The guard and
235
+ // pagination measure those delimiters too; receipts still quote the complete source block.
236
+ const pre = fencedText(text);
237
+ const parts = pre ? pre.body.split(/\n(?:[ \t]*\n)+/).map(part => part.replace(/^\n+|\n+$/g, "")).filter(part => part.length > 0)
238
+ .map((part, i) => `${i ? pre.close : pre.open}\n${part}\n${pre.close}`) : text.split("\n\n");
239
+ return parts.map(text => ({ text, source }));
240
+ });
241
+ const overhead = local && opts.localSectionBytes ? opts.localSectionBytes({ ...base, cached_at: entry.fetched_at,
242
+ section: { n: s.n, heading: s.heading, text: "", page: units.length, pages: units.length } }, label) : labelCost;
243
+ const pg = paginate(units, (b) => local && opts.localTextBytes ? opts.localTextBytes(b.text) + 4
244
+ : len(b.text) + (local ? 0 : len(rc(b.source.text, s.n))) + 4, page, textBudget - overhead);
245
+ const rs = [...new Set(pg.slice.map(b => b.source))].map(b => rc(b.text, s.n));
206
246
  rs.unshift(...(page === 1 ? titleReceipt : []), ...(s.generated ? [] : [rc(s.heading, s.n)])); // the heading is returned on every page (a generated label is not a quote)
247
+ const texts = pg.slice.map(b => b.text);
248
+ const spans = [];
249
+ if (visitorPage) {
250
+ let start = 0;
251
+ pg.slice.forEach((b, i) => {
252
+ if (b.source.visitor)
253
+ spans.push([start, start + texts[i].length, b.source.visitor]);
254
+ start += texts[i].length + 2; // the blank line between blocks
255
+ });
256
+ }
207
257
  return {
208
- uc: { ...(page === 1 ? base : { final_url: entry.final_url }), section: { n: s.n, heading: s.heading, text: pg.slice.map(shown).join("\n\n"), page, pages: pg.pages } },
258
+ uc: { ...(page === 1 ? base : { final_url: entry.final_url }), section: { n: s.n, heading: s.heading, text: texts.join("\n\n"), page, pages: pg.pages } },
209
259
  receipts: rs,
260
+ ...(visitorPage ? { section_visitor: spans, heading_visitor: s.visitor ?? 0 } : {}),
261
+ // A page of a section is one part: the visitor's when any paragraph on it is (it errs toward the visitor).
262
+ ...(visitorPage ? { written_by: [pg.slice.reduce((n, b) => (b.source.visitor && (!n || b.source.visitor < n) ? b.source.visitor : n), s.visitor ?? 0)], label } : {}),
210
263
  };
211
264
  }
212
265
  // Outline: each entry returns a heading and a preview, so each gets a receipt.
@@ -215,7 +268,11 @@ export async function buildContent(entry, opts) {
215
268
  const receipts = [...(s.generated ? [] : [rc(s.heading, s.n)]), ...(preview ? [rc(preview, s.n)] : [])];
216
269
  return { item: { section: s.n, level: s.level, heading: s.heading, preview }, receipts };
217
270
  });
218
- const out = paginate(units, len, page, BUDGET.outline);
271
+ // Local mode shows each entry as its number and heading only, against a budget in the same proportion as the
272
+ // default ones (outline to section text).
273
+ const out = local
274
+ ? paginate(units, (u) => len({ section: u.item.section, heading: u.item.heading }), page, Math.round((textBudget * BUDGET.outline) / BUDGET.sectionText))
275
+ : paginate(units, len, page, BUDGET.outline);
219
276
  const links = paginate(p.links, len, page, BUDGET.links);
220
277
  // W-04: forms, as fields, paged like links. Like links they carry no receipts (structure, not quotes).
221
278
  const forms = p.forms ? paginate(p.forms, len, page, BUDGET.forms) : null;
@@ -229,7 +286,7 @@ export async function buildContent(entry, opts) {
229
286
  links: links.slice,
230
287
  ...(forms ? { forms: forms.slice, ...(page === 1 && p.forms_omitted ? { forms_omitted: p.forms_omitted } : {}), ...(forms.slice.length ? { forms_hint: FORMS_HINT } : {}) } : {}),
231
288
  page,
232
- pages: Math.max(out.pages, links.pages, forms?.pages ?? 1),
289
+ pages: local ? out.pages : Math.max(out.pages, links.pages, forms?.pages ?? 1), // local replies list no links or forms
233
290
  section_count: p.sections.length,
234
291
  links_omitted: p.links_omitted,
235
292
  structured: p.structured_count || p.structured_skipped
@@ -240,6 +297,8 @@ export async function buildContent(entry, opts) {
240
297
  receipts: rs,
241
298
  };
242
299
  }
300
+ // The fixed note on a read of an empty JavaScript shell that was not rendered (local mode looks for it: local.ts).
301
+ export const SHELL_NOTE = "This page looks like a JavaScript-only shell. Ask for rendering to read its content.";
243
302
  export const FORMS_HINT = "Forms are listed as fields (hidden ones by name only). To see exactly what a form would send, call form_preview with this url, the form's index and your values. Sending needs a permission slip, which only the owner grants from the command line; a form with not_allowed is never sent.";
244
303
  // One page of a kept list of changes. Added blocks are on the page now, so each gets a receipt.
245
304
  // Removed blocks are not on the page any more: they carry no receipt, and `kind` says so.
@@ -273,19 +332,23 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
273
332
  // reveals an earlier visit and never records this one.
274
333
  const priv = isPrivate();
275
334
  const cache = priv ? null : readJson(cacheFile);
335
+ // v1.2: old shell flags also obey the content test before a cached note or automatic render.
336
+ if (cache?.shell && !looksLikeShell(cache.parsed, true))
337
+ delete cache.shell;
276
338
  // An entry parsed by an older reader keeps its provenance (where it was read
277
339
  // from, its baseline) but its parsed form is rebuilt from the next response.
278
340
  // Older rendered entries lack the inspection codes and predate the textarea inspection. Rebuild them.
279
- const staleParse = !!cache && (cache.parser !== PARSER_VERSION || (!!cache.rendered && !cache.render_inspection));
341
+ const staleParse = !!cache && (cache.parser !== PARSER_VERSION || (!!cache.rendered && !cache.render_inspection) || (opts.budget !== undefined && (!cache.visitor_scopes || !cache.visitor_main_scopes || !cache.visitor_heading_text)));
280
342
  // W-30: the owner's "doors" setting, read (and checked: a bad value fails the read, by name) on EVERY read,
281
343
  // whichever path it takes (Codex round 2).
282
344
  const doorsMode = tool === "read" ? loadDoorsMode() : "wait";
283
345
  // W-10 (v0.7): the owner's render setting (a bad value fails the read, by name). A read that asks for rendering
284
346
  // when it is allowed, and a check of a copy that was rendered, never reuse a plain copy as is.
285
347
  const renderMode = tool === "read" || tool === "check" ? loadRenderMode() : "off";
348
+ const autoShell = renderMode === "auto" || (renderMode === "ask" && opts.renderIfShell === true);
286
349
  // Explicit renders and checks of rendered copies need a body and a new browser run, even after a 304.
287
350
  // The one shortcut is an already rendered copy still fresh under the site's cache rules (below).
288
- const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered) || (renderMode === "auto" && !!cache?.shell && !cache.rendered));
351
+ const rerender = renderMode !== "off" && (!!opts.render || (tool === "check" && !!cache?.rendered) || (autoShell && !!cache?.shell && !cache.rendered));
289
352
  const needsBody = rerender || (renderMode === "off" && !!cache?.rendered);
290
353
  // W-28 (v0.8): a read that asks to save, when the owner allows it (a bad setting fails the read, by name). A save
291
354
  // keeps what THIS read received, so it never uses a fresh copy, a 304 or a cached rendering: it always fetches.
@@ -387,6 +450,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
387
450
  // All parsing of the page runs under a hard time budget (sandbox.ts).
388
451
  let { parsed, wall: pageWall } = await parsePage(k, res.body, res.bytes, res.url, res.fileFlags, looksHtml(ct, res.body), res.headers.get("content-language"));
389
452
  throwIfCancelled();
453
+ const hasScript = k === "html" && /<script\b/i.test(res.body);
390
454
  // W-10 (v0.7): rendering. Asked for by the read, or (owner's "auto") an empty JavaScript shell, or a check of a
391
455
  // copy that was rendered (so like is compared with like). The rendered HTML goes through the same reader and
392
456
  // shield; a render that stops leaves the plain read in place, and the reply says why.
@@ -397,7 +461,7 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
397
461
  // the same outline; when the page's own body is unchanged, the rendered copy is reused without starting a browser
398
462
  // (an explicit render asks for a new rendering).
399
463
  const plainHash = sha256(Buffer.concat([Buffer.from(`${k}|${(res.fileFlags ?? []).join(",")}|`), res.bytes ?? Buffer.from(res.body, "utf8")]));
400
- const wantRender = !!opts.render || (renderMode !== "off" && !!cache?.rendered) || (renderMode === "auto" && k === "html" && looksLikeShell(parsed.text_chars, res.body));
464
+ const wantRender = !!opts.render || (renderMode !== "off" && !!cache?.rendered) || (autoShell && looksLikeShell(parsed, hasScript));
401
465
  const reuse = tool !== "check" && !opts.fresh && !opts.render && !sv && !staleParse && !!cache?.rendered && !!cache.render_hash && cache.plain_hash === plainHash && cache.source_url === source && cache.final_url === res.url;
402
466
  // v0.7 track 2: a CAPTCHA or login wall found in the plain HTML's markers waits for the render's verdict when a render
403
467
  // will decide this read (the browser sees whether the box is shown). A bot check (it is passed by running its script,
@@ -466,12 +530,12 @@ export async function run(tool, rawUrl, opts, meta = {}, agent) {
466
530
  const { fresh_until: _f, reuse_blocked: _b, ...rest } = cache; // a full 200 judges freshness from scratch
467
531
  // Same bytes: the copy is kept, but how the site now describes it is taken fresh (round 83).
468
532
  // A saving read uses the fresh parse; otherwise only the current language hint needs refreshing.
469
- entry = { ...rest, ...fresh, parsed: sv ? parsed : { ...rest.parsed, language: parsed.language }, ...(renderedHash && render ? { render_inspection: inspectionCodes(render) } : {}), content_type: ct, shell: k === "html" && looksLikeShell(parsed.text_chars, res.body) ? true : undefined, door_detail: doorDetail, redirected: res.redirected, etag: res.headers.get("etag") ?? cache.etag, last_modified: res.headers.get("last-modified") ?? cache.last_modified, checked_at: now };
533
+ entry = { ...rest, ...fresh, parsed: sv ? parsed : { ...rest.parsed, language: parsed.language }, ...(renderedHash && render ? { render_inspection: inspectionCodes(render) } : {}), content_type: ct, shell: looksLikeShell(parsed, hasScript) ? true : undefined, door_detail: doorDetail, redirected: res.redirected, etag: res.headers.get("etag") ?? cache.etag, last_modified: res.headers.get("last-modified") ?? cache.last_modified, checked_at: now };
470
534
  }
471
535
  else
472
536
  entry = {
473
- parser: PARSER_VERSION, url, source_url: source, final_url: res.url, door, door_detail: doorDetail, hash, content_type: ct, parsed,
474
- ...(k === "html" && looksLikeShell(parsed.text_chars, res.body) ? { shell: true } : {}),
537
+ parser: PARSER_VERSION, visitor_scopes: true, visitor_main_scopes: true, visitor_heading_text: true, url, source_url: source, final_url: res.url, door, door_detail: doorDetail, hash, content_type: ct, parsed,
538
+ ...(looksLikeShell(parsed, hasScript) ? { shell: true } : {}),
475
539
  etag: res.headers.get("etag") ?? undefined,
476
540
  last_modified: res.headers.get("last-modified") ?? undefined,
477
541
  // an identical body keeps its original read time, and with it the agent that first fetched it
@@ -534,6 +598,8 @@ function afterDeferred(e, origin) {
534
598
  return e;
535
599
  }
536
600
  export async function finishPage(tool, url, entry, unchanged, changedFrom, opts, od, now, persist, meta, diff, notKept = false, serve = {}) {
601
+ if (!entry.source)
602
+ noteGetActions(entry.parsed.forms, !notKept);
537
603
  const wantsContent = opts.question !== undefined || opts.section !== undefined || opts.page !== undefined || (tool === "read" && (opts.translate !== undefined || !!opts.save));
538
604
  // W-28 (v0.8): the snapshot is written before the receipts are recorded, so each recorded receipt names it.
539
605
  const sv = tool === "read" ? serve.save : undefined;
@@ -559,10 +625,8 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
559
625
  renderNote(e, serve.render ?? (entry.rendered ? cachedRender(entry) : undefined), meta);
560
626
  if (serve.cache)
561
627
  e.cache = serve.cache;
562
- if (tool === "read" && entry.shell && !entry.rendered && !opts.render) {
563
- const note = "This page looks like a JavaScript-only shell. Ask for rendering to read its content.";
564
- e.message = e.message ? `${e.message} ${note}` : note;
565
- }
628
+ if (tool === "read" && entry.shell && !entry.rendered && !opts.render)
629
+ e.message = e.message ? `${e.message} ${SHELL_NOTE}` : SHELL_NOTE;
566
630
  if (fromCache)
567
631
  e.untrusted_content = { ...(e.untrusted_content ?? {}), ...cacheUc };
568
632
  if (sv) {
@@ -639,6 +703,8 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
639
703
  const c = await buildContent(entry, opts);
640
704
  if (opts.question !== undefined && Array.isArray(c.uc.passages))
641
705
  meta.matched = c.uc.passages.length;
706
+ if (c.written_by)
707
+ meta.written_by = c.written_by; // W-39: the levels go to the history, for the owner
642
708
  if (c.error) {
643
709
  persist(); // the page itself was read fine; only the section asked for is missing
644
710
  if (sv)
@@ -657,6 +723,12 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
657
723
  });
658
724
  if (entry.parsed.forms && reply.counts)
659
725
  reply.counts.forms = entry.parsed.forms.length + (entry.parsed.forms_omitted ?? 0); // W-04
726
+ if (c.written_by && c.label)
727
+ reply.written_by = c.written_by; // W-39: local.ts makes the labels from it
728
+ if (c.section_visitor)
729
+ reply.section_visitor = c.section_visitor;
730
+ if (c.heading_visitor !== undefined)
731
+ reply.heading_visitor = c.heading_visitor;
660
732
  const out = withCache(reply);
661
733
  // W-26 (v0.7): the owner's translation hook, only when asked and only on a read. The receipts above were
662
734
  // built from the original text; the translation never becomes one.
@@ -671,10 +743,26 @@ export async function finishPage(tool, url, entry, unchanged, changedFrom, opts,
671
743
  delete r.snapshot;
672
744
  persist();
673
745
  saveNow(kept);
674
- if (!priv)
675
- recordReceipts(kept);
746
+ const recorded = !priv && recordReceipts(kept);
747
+ if (tool === "read" && opts.receipts === "log" && recorded)
748
+ leaveReceiptsInLog(out);
676
749
  return savedOn(out);
677
750
  }
751
+ // v1.2 (field list, 2026-11-14): a reply repeats every heading, preview and paragraph it shows as a receipt, which
752
+ // about doubles what the reading model takes in. A caller may ask for the receipts to stay in the local log
753
+ // (read's `receipts: "log"`). The receipts are made and recorded exactly as before, and counts.receipts still says
754
+ // how many; only the list is left out of this reply, and the reply says so with a fixed top-level mark. It is left
755
+ // out only when every receipt was written whole: in private mode, when the log could not be written, or when the
756
+ // logged copy of any receipt was cut for length (Codex review, 2026-11-14), the log does not hold them, so the
757
+ // reply carries them as usual. A reply with no content (the one-line "unchanged" answer) has none to leave out,
758
+ // and a file reply (filereply.ts) carries its receipts as before.
759
+ export function leaveReceiptsInLog(e) {
760
+ const uc = e.untrusted_content;
761
+ if (!uc || !Array.isArray(uc.receipts))
762
+ return;
763
+ delete uc.receipts;
764
+ e.receipts_in_log = true;
765
+ }
678
766
  // The top-level `saved` (fixed codes, Tiresias's own hash and counts) and its fixed sentence.
679
767
  export function addSaved(e, top) {
680
768
  const previous = e.saved;
@@ -695,12 +783,15 @@ export function renderNote(e, render, meta) {
695
783
  e.message = e.message ? `${e.message} ${text}` : text;
696
784
  return e;
697
785
  }
786
+ // True when every receipt was written to the log WHOLE. The logged copy of a very long quote or URL is cut
787
+ // (receipts.ts), so a cut record, like a failed write, means the log does not hold what the reply quoted.
698
788
  export function recordReceipts(rs) {
699
789
  try {
700
- receipts.record(rs);
790
+ return receipts.record(rs) === 0;
701
791
  }
702
792
  catch (e) {
703
793
  console.error(`Could not write receipts: ${e.message}`); // a log failure never fails the browsing call
794
+ return false;
704
795
  }
705
796
  }
706
797
  export const noteDoors = (meta, ad) => {
@@ -0,0 +1,12 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // v1.2: recognize the reader's fenced presentation, including list indentation and a local warning.
3
+ export function fencedText(text) {
4
+ const head = /^((?:\[[^\n]*\]\n)?[ \t]*(?:(?:[-*+]|\d+[.)]) )?)(`{3,})\n/.exec(text);
5
+ if (!head)
6
+ return undefined;
7
+ const end = text.lastIndexOf("\n");
8
+ const close = text.slice(end + 1);
9
+ if (end < head[0].length || close.trimStart() !== head[2])
10
+ return undefined;
11
+ return { open: head[0].slice(0, -1), body: text.slice(head[0].length, end), close };
12
+ }
package/dist/profiles.js CHANGED
@@ -11,7 +11,7 @@ import { randomBytes } from "node:crypto";
11
11
  import fs from "node:fs";
12
12
  import path from "node:path";
13
13
  import { parseSignatureAgent } from "./config.js";
14
- import { KEY_FILE } from "./keys.js";
14
+ import { KEY_FILE, PREVIOUS_KEY_FILE } from "./keys.js";
15
15
  import { DEFAULT_PROFILE, NOFOLLOW, NONBLOCK, PROFILE_RE, ProfileError, checkProfileName, checkSub, ensureRoot, profileDir, profilesRoot, readOwnedFile, readRoot, rootDir, } from "./store.js";
16
16
  const META = "profile.json"; // { name, created_at }, written once at create
17
17
  const me = () => (typeof process.getuid === "function" ? process.getuid() : -1);
@@ -138,8 +138,8 @@ function plan(dir) {
138
138
  stack.push(f);
139
139
  else if (s.isFile()) {
140
140
  p.files.push({ file: f, size: s.size });
141
- if (rel === path.join("keys", KEY_FILE))
142
- p.hasKey = true;
141
+ if (rel === path.join("keys", KEY_FILE) || rel === path.join("keys", PREVIOUS_KEY_FILE))
142
+ p.hasKey = true; // v1.1: a rotation's previous key is a private key too
143
143
  }
144
144
  else
145
145
  throw bad("something that is not a file or a folder");
package/dist/queue.js CHANGED
@@ -1,16 +1,13 @@
1
- // SPDX-License-Identifier: Apache-2.0
2
- // N-06: the per-site queue. Every request (pages, discovery probes, robots.txt) waits its turn here and
3
- // is spaced from the one before it by the site's spacing (1 s by default, or a larger Crawl-delay).
4
1
  import { cooldownLeft, onCooldown } from "./cooldown.js";
5
2
  import { FetchError, Refused } from "./errors.js";
6
- import { acquireActive, extendNext, noteCompletion, reserve } from "./hosts.js";
3
+ import { acquireActive, extendNext, noteCompletion, reserve, sharesSpacing, spacingMs } from "./hosts.js";
7
4
  import { MAX_HOSTS } from "./lru.js";
8
5
  import { canonOrigin } from "./sitekey.js";
9
6
  import { noteApplied, siteSpacing } from "./sites.js";
10
7
  // Bounded (v0.5.1): a long-running MCP server keeps at most MAX_HOSTS sites in each. A queue entry is
11
8
  // removed when its last request has had its turn, and a site's waiting set when it is empty.
12
- // When each site was last called. Kept while its spacing can still matter (the longest wait is a 30-second
13
- // Crawl-delay), then pruned (Codex round 78: evicting by count could drop a spacing still owed).
9
+ // When each site was last called. Kept while its spacing can still matter,
10
+ // then pruned (Codex round 78: evicting by count could drop a spacing still owed).
14
11
  const lastHit = new Map();
15
12
  const HIT_KEEP_MS = 60_000;
16
13
  // The earliest the NEXT request may start, as the last request's own spacing said (Codex round 79: a
@@ -79,15 +76,7 @@ export function sleep(ms, signal) {
79
76
  signal.addEventListener("abort", stop, { once: true });
80
77
  });
81
78
  }
82
- // Serialise requests to one host and space them by `delay` seconds (the default spacing, or a larger
83
- // Crawl-delay). The slot is reserved in the shared hosts/ file too (hosts.ts), so other processes space
84
- // their requests to this site after it.
85
- // A queued request that is cancelled (its call ended, or a cooldown began)
86
- // leaves the queue at once and does not move the clock.
87
- // With `hold`, the turn is kept until the returned function is called (Codex round 73: one request at a
88
- // time means until its answer has been read, not only until it was sent); without it, the turn passes on
89
- // as soon as the request may start.
90
- export async function throttle(origin, delay, outer, url, hold = false) {
79
+ export async function throttle(origin, delay, outer, url, hold = false, options = {}) {
91
80
  origin = canonOrigin(origin); // v0.6: "example.com." and "example.com" share one queue and one clock
92
81
  let target = null;
93
82
  try {
@@ -116,6 +105,8 @@ export async function throttle(origin, delay, outer, url, hold = false) {
116
105
  const tail = prev.then(() => turn);
117
106
  chains.set(origin, tail);
118
107
  let activeRelease = () => { };
108
+ let activeHeld = false;
109
+ const visit = options.robots;
119
110
  let admittedMs = 0; // the spacing this request was admitted with
120
111
  const done = () => {
121
112
  // v0.6 (Codex round 9): a held turn ends when the answer has been read. The site's clocks are moved on from
@@ -126,7 +117,9 @@ export async function throttle(origin, delay, outer, url, hold = false) {
126
117
  const d = Math.max(admittedMs, delayOf.get(origin) ?? 0); // a Crawl-delay learned meanwhile counts too
127
118
  lastHit.set(origin, t);
128
119
  nextOk.set(origin, Math.max(nextOk.get(origin) ?? 0, t + d)); // this process's clock (private calls too)
129
- noteCompletion(origin, t, t + d); // other processes: last = completion, next = max(next, completion + delay)
120
+ const shared = noteCompletion(origin, t, t + d); // other processes too
121
+ if (visit?.requests === 1 && (!sharesSpacing() || activeHeld))
122
+ visit.pass = { origin, last: t, next: nextOk.get(origin), shared };
130
123
  }
131
124
  activeRelease();
132
125
  release();
@@ -142,7 +135,7 @@ export async function throttle(origin, delay, outer, url, hold = false) {
142
135
  await Promise.race([prev, cancelled]);
143
136
  // Holding the turn: first the cross-process lease (Codex round 76: waiting for it AFTER reserving a
144
137
  // slot let the slot go stale), then the shared slot and its spacing.
145
- activeRelease = await Promise.race([acquireActive(origin, mine.signal), cancelled]);
138
+ activeRelease = await Promise.race([acquireActive(origin, mine.signal, () => { activeHeld = true; }), cancelled]);
146
139
  // W-25 (v0.6): a site's own spacing (sites.ts) raises the spacing. Like every spacing since v0.5.1 it is per
147
140
  // ORIGIN (scheme, host and port). Settings are the call's snapshot, read once when the call started
148
141
  // (coordinator's rule after Codex round 11): a change saved meanwhile applies to calls that start after it.
@@ -153,10 +146,21 @@ export async function throttle(origin, delay, outer, url, hold = false) {
153
146
  throw new Error("queue-cancelled");
154
147
  if (cooldownLeft(origin) > 0)
155
148
  throw cooldownRefusal(origin, url);
156
- const slot = reserve(origin, eff * 1000, Math.max((lastHit.get(origin) ?? 0) + eff * 1000, nextOk.get(origin) ?? 0), true);
149
+ const pass = options.pass;
150
+ const canPass = pass?.origin === origin && eff * 1000 <= spacingMs() &&
151
+ (delayOf.get(origin) ?? 0) <= spacingMs() && lastHit.get(origin) === pass.last &&
152
+ nextOk.get(origin) === pass.next && pass.next === pass.last + spacingMs();
153
+ // Private, unshared calls waive only their local clock. Shared calls also need a successfully
154
+ // held lease and the exact stamp this robots completion wrote, checked again under reserve's lock.
155
+ // A stale call-owned pass is harmless: any intervening request changes these clocks/stamps.
156
+ const stamp = canPass && (!sharesSpacing() || (activeHeld && pass.shared?.last === pass.last && pass.shared?.next === pass.next))
157
+ ? (pass.shared ?? { last: pass.last, next: pass.next }) : undefined;
158
+ const slot = reserve(origin, eff * 1000, Math.max((lastHit.get(origin) ?? 0) + eff * 1000, nextOk.get(origin) ?? 0), true, stamp);
157
159
  const wait = slot - Date.now();
158
160
  if (wait <= 0)
159
161
  break;
162
+ if (wait > 30_000)
163
+ throw new Refused("The site's spacing wait is longer than Tiresias will wait.", undefined, Math.ceil(wait / 1000), url);
160
164
  await Promise.race([sleep(wait, mine.signal), cancelled]);
161
165
  }
162
166
  // The checks that matter: right before sending, not before queueing.
@@ -198,9 +202,9 @@ export class Turns {
198
202
  leaveOthers(_u) {
199
203
  this.giveBack();
200
204
  }
201
- async take(u, delay, signal) {
205
+ async take(u, delay, signal, options = {}) {
202
206
  this.giveBack();
203
- this.held = { origin: u.origin, release: await throttle(u.origin, delay, signal, u.href, true) };
207
+ this.held = { origin: u.origin, release: await throttle(u.origin, delay, signal, u.href, true, options) };
204
208
  }
205
209
  giveBack() {
206
210
  this.held?.release();