@raibitsi/tiresias 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +88 -0
  2. package/README.md +72 -33
  3. package/data/README.md +33 -0
  4. package/data/guard-ordinary-urls.txt +102 -0
  5. package/data/guard-starter.jsonl +424 -0
  6. package/dist/aimed.js +23 -0
  7. package/dist/anchor.js +26 -13
  8. package/dist/browsecli.js +17 -13
  9. package/dist/cdp.js +22 -1
  10. package/dist/cleanup.js +3 -1
  11. package/dist/cli.js +216 -68
  12. package/dist/config.js +75 -1
  13. package/dist/confirm.js +20 -5
  14. package/dist/cooldown.js +57 -32
  15. package/dist/dailylog.js +11 -5
  16. package/dist/dom.js +13 -8
  17. package/dist/doors.js +1 -1
  18. package/dist/engine.js +9 -4
  19. package/dist/envelope.js +10 -1
  20. package/dist/fetcher.js +29 -2
  21. package/dist/filereply.js +7 -1
  22. package/dist/formflow.js +21 -3
  23. package/dist/forms.js +337 -42
  24. package/dist/getactions.js +119 -0
  25. package/dist/guard.js +1240 -0
  26. package/dist/guardclient.js +245 -0
  27. package/dist/guardnetwork.js +22 -0
  28. package/dist/history.js +11 -5
  29. package/dist/hook.js +2 -0
  30. package/dist/hosts.js +34 -27
  31. package/dist/keys.js +420 -18
  32. package/dist/local.js +537 -0
  33. package/dist/mcp.js +56 -25
  34. package/dist/needs.js +9 -5
  35. package/dist/parentwatch.js +6 -16
  36. package/dist/pdf.js +4 -1
  37. package/dist/pdfchild.js +0 -2
  38. package/dist/pdfproc.js +26 -11
  39. package/dist/pdfwatch.js +14 -0
  40. package/dist/pipeline.js +117 -26
  41. package/dist/preformatted.js +12 -0
  42. package/dist/profiles.js +3 -3
  43. package/dist/queue.js +24 -20
  44. package/dist/reader.js +441 -87
  45. package/dist/receipts.js +7 -2
  46. package/dist/render.js +156 -27
  47. package/dist/renderactions.js +69 -0
  48. package/dist/resume.js +8 -6
  49. package/dist/rules.js +24 -5
  50. package/dist/safebrowsing.js +3 -3
  51. package/dist/safety.js +15 -7
  52. package/dist/sandbox.js +1 -1
  53. package/dist/saved.js +24 -9
  54. package/dist/savedlock.js +99 -5
  55. package/dist/settings.js +10 -3
  56. package/dist/shelf.js +1 -1
  57. package/dist/shield.js +15 -0
  58. package/dist/slips.js +43 -1
  59. package/dist/store.js +119 -25
  60. package/dist/tabs.js +1 -1
  61. package/dist/textview.js +16 -0
  62. package/dist/translate.js +22 -2
  63. package/dist/types.js +9 -2
  64. package/dist/watchflow.js +1 -2
  65. package/dist/work.js +10 -2
  66. package/docs/LOCAL-MODELS.md +271 -0
  67. package/package.json +5 -2
package/dist/config.js CHANGED
@@ -3,6 +3,7 @@
3
3
  // Absent means these defaults. A bad value is a clear error, never a guess.
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
+ import { isOwnerAddress } from "./guardnetwork.js";
6
7
  import { releaseLock, stillOwned, takeLock } from "./ownedlock.js";
7
8
  import { NOFOLLOW, NONBLOCK, home, ownFile, randomTmp, readOwnedText } from "./store.js";
8
9
  // W-32 (v0.9): readers of a risky setting use the value in force (an unconfirmed loosening is held back).
@@ -64,6 +65,63 @@ export function loadTranslator() {
64
65
  return null;
65
66
  return parseTranslator(o.translator);
66
67
  }
68
+ export const GUARD_TIMEOUT = 4;
69
+ const GUARD_KEYS = new Set(["url", "model", "prefix", "timeout_seconds", "allow_public"]);
70
+ const BAD_GUARD = 'config.json: "guard" must be {"url": "http://localhost:1234/v1", "model": "<embedding model id>", "prefix": "...", "timeout_seconds": 1 to 60}. Set it with: tiresias guard setup --url <url> --model <id>';
71
+ const CONTROL = /[\u0000-\u001f\u007f]/;
72
+ // Addresses written as an IP that are public. A host NAME is judged when the call is made (guardclient.ts), from
73
+ // the addresses it resolves to.
74
+ const PUBLIC_LITERAL = (host, isPrivate) => {
75
+ const h = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
76
+ return /^[0-9.]+$/.test(h) || h.includes(":") ? !isPrivate(h) : false;
77
+ };
78
+ export function parseGuardUrl(v, isPrivate = isOwnerAddress) {
79
+ if (typeof v !== "string" || v.length === 0 || v.length > 2048 || CONTROL.test(v))
80
+ throw new ConfigError(BAD_GUARD);
81
+ let u;
82
+ try {
83
+ u = new URL(v);
84
+ }
85
+ catch {
86
+ throw new ConfigError('The guard "url" is not a valid address. Give the base of an OpenAI-compatible API, like http://localhost:1234/v1');
87
+ }
88
+ if (u.protocol !== "http:" && u.protocol !== "https:")
89
+ throw new ConfigError('The guard "url" must start with http:// or https://.');
90
+ if (u.username || u.password)
91
+ throw new ConfigError('The guard "url" must not contain a user name or password. A key goes in the environment variable TIRESIAS_GUARD_KEY.');
92
+ if (u.search || u.hash || v.includes("?") || v.includes("#"))
93
+ throw new ConfigError('The guard "url" must not have a query or a # part.');
94
+ if (u.protocol === "http:" && PUBLIC_LITERAL(u.hostname, isPrivate))
95
+ throw new ConfigError("Plain http to a public address is refused: page text would cross the internet unencrypted. Use https, or an address on your own network.");
96
+ return (u.origin + u.pathname).replace(/\/+$/, "");
97
+ }
98
+ export function parseGuard(v, isPrivate) {
99
+ if (typeof v !== "object" || v === null || Array.isArray(v))
100
+ throw new ConfigError(BAD_GUARD);
101
+ const o = v;
102
+ for (const k of Object.keys(o))
103
+ if (!GUARD_KEYS.has(k))
104
+ throw new ConfigError(BAD_GUARD);
105
+ const url = parseGuardUrl(o.url, isPrivate);
106
+ if (typeof o.model !== "string" || o.model.length === 0 || o.model.length > 200 || CONTROL.test(o.model))
107
+ throw new ConfigError(BAD_GUARD);
108
+ const prefix = Object.hasOwn(o, "prefix") ? o.prefix : "";
109
+ if (typeof prefix !== "string" || prefix.length > 200 || CONTROL.test(prefix))
110
+ throw new ConfigError(BAD_GUARD);
111
+ const t = Object.hasOwn(o, "timeout_seconds") ? o.timeout_seconds : GUARD_TIMEOUT;
112
+ if (typeof t !== "number" || !Number.isSafeInteger(t) || t < 1 || t > 60)
113
+ throw new ConfigError(BAD_GUARD);
114
+ if (Object.hasOwn(o, "allow_public") && o.allow_public !== true)
115
+ throw new ConfigError(BAD_GUARD);
116
+ return { url, model: o.model, prefix, timeout_seconds: t, ...(o.allow_public === true ? { allow_public: true } : {}) };
117
+ }
118
+ // The meaning check of this profile, or null when none is set. A bad setting is a ConfigError.
119
+ export function loadGuard(isPrivate) {
120
+ const o = inForceConfig();
121
+ if (!o || !Object.hasOwn(o, "guard"))
122
+ return null;
123
+ return parseGuard(o.guard, isPrivate);
124
+ }
67
125
  // The whole config.json as an object (null when there is none). A bad file is a ConfigError.
68
126
  function readConfigFile() {
69
127
  const raw = readOwnedText("config.json");
@@ -196,7 +254,23 @@ export function loadLimits() {
196
254
  // ---- v0.6 (track 3: settings) ----
197
255
  // Keys checked only where they are used (like "scanner"): a bad one fails the calls that use it, by name,
198
256
  // and never stops clear or the daily trim.
199
- const OWN_CHECK_KEYS = ["doors", "sites", "translator", "rendering", "saving", "anchoring", "anchor_calendars"]; // translator: v0.7 W-26; rendering: v0.7 W-10; saving: v0.8 W-28; anchoring, anchor_calendars: v0.8 W-08
257
+ export const SECURITY_CHOICES = {
258
+ get_form_actions: ["read", "slip"],
259
+ private_spacing: ["shared", "none"],
260
+ hook_secrets: ["hidden", "inherit"],
261
+ text_views: ["marked", "wrapped"],
262
+ written_by: ["on", "off"], // W-39 (v1.2): local mode's written_by label
263
+ };
264
+ export function parseChoice(name, value) {
265
+ if (typeof value === "string" && SECURITY_CHOICES[name].includes(value))
266
+ return value;
267
+ throw new ConfigError(`config.json: "${name}" must be ${SECURITY_CHOICES[name].map((s) => JSON.stringify(s)).join(" or ")}.`);
268
+ }
269
+ export function loadChoice(name) {
270
+ const o = inForceConfig();
271
+ return o && Object.hasOwn(o, name) ? parseChoice(name, o[name]) : SECURITY_CHOICES[name][0];
272
+ }
273
+ const OWN_CHECK_KEYS = [...Object.keys(SECURITY_CHOICES), "doors", "sites", "translator", "rendering", "saving", "anchoring", "anchor_calendars", "guard"]; // translator: v0.7 W-26; rendering: v0.7 W-10; saving: v0.8 W-28; anchoring, anchor_calendars: v0.8 W-08; guard: v1.2 W-38
200
274
  export const KNOWN_KEYS = [...Object.keys(DEFAULT_LIMITS), ...SIGNING_KEYS, ...SAFETY_KEYS, "scanner", ...OWN_CHECK_KEYS];
201
275
  export const DEFAULT_DOORS = "background";
202
276
  export function parseDoors(v) {
package/dist/confirm.js CHANGED
@@ -13,7 +13,7 @@
13
13
  import fs from "node:fs";
14
14
  import path from "node:path";
15
15
  import { callContext } from "./context.js";
16
- import { DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
16
+ import { SECURITY_CHOICES, parseChoice, DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseGuard, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
17
17
  import { parseRender } from "./render.js";
18
18
  import { ConfigError } from "./errors.js";
19
19
  import * as history from "./history.js";
@@ -46,7 +46,11 @@ const calendarSet = (v) => {
46
46
  }
47
47
  };
48
48
  const renderRank = rank(["off", "ask", "auto"]);
49
+ const publicGuard = (v) => !!v && typeof v === "object" && v.allow_public === true;
50
+ const guardUrl = (v) => (v && typeof v === "object" ? String(v.url) : "");
49
51
  export const TOP = [
52
+ { key: "private_spacing", looser: (c, n) => c !== "none" && n === "none", phrase: () => "YES, TURN OFF PRIVATE SPACING", what: () => "private spacing across processes turned off" },
53
+ { key: "hook_secrets", looser: (c, n) => c !== "inherit" && n === "inherit", phrase: () => "YES, SHARE TIRESIAS SECRETS WITH HOOKS", what: () => "Tiresias secrets shared with hooks" },
50
54
  { key: "saving", looser: (c, n) => rank(["off", "ask"])(n) > rank(["off", "ask"])(c), phrase: () => "YES, TURN ON SAVING", what: () => "saving turned on" },
51
55
  {
52
56
  key: "rendering",
@@ -68,7 +72,11 @@ export const TOP = [
68
72
  what: () => "anchor calendars were added",
69
73
  },
70
74
  { key: "safe_browsing", looser: (c, n) => c !== false && n === false, phrase: () => "YES, TURN OFF SAFE BROWSING", what: () => "Safe Browsing turned off" },
75
+ // W-38 (v1.2): the meaning check sends page text to the owner's embedding endpoint. Sending it to an address
76
+ // outside the owner's network (allow_public) is the loosening; a new address with it set needs the phrase again.
77
+ { key: "guard", looser: (c, n) => publicGuard(n) && !(publicGuard(c) && guardUrl(c) === guardUrl(n)), phrase: () => GUARD_PHRASE, what: () => "the meaning check was allowed to send page text outside your network" },
71
78
  ];
79
+ export const GUARD_PHRASE = "YES, SEND PAGE TEXT OUTSIDE MY NETWORK";
72
80
  const TOP_BY = new Map(TOP.map((t) => [t.key, t]));
73
81
  export const RISKY_KEYS = TOP.map((t) => t.key);
74
82
  export const isRisky = (name) => TOP_BY.has(name) || SITE_RISKY.includes(name);
@@ -408,7 +416,9 @@ function ratchet(raw, rec) {
408
416
  }
409
417
  function validTop(key, v) {
410
418
  try {
411
- if (key === "saving")
419
+ if (Object.hasOwn(SECURITY_CHOICES, key))
420
+ parseChoice(key, v);
421
+ else if (key === "saving")
412
422
  parseSaving(v);
413
423
  else if (key === "rendering")
414
424
  parseRender(v);
@@ -422,6 +432,8 @@ function validTop(key, v) {
422
432
  parseCalendars(v);
423
433
  else if (key === "safe_browsing")
424
434
  parseSafeBrowsing(v);
435
+ else if (key === "guard")
436
+ parseGuard(v);
425
437
  return true;
426
438
  }
427
439
  catch {
@@ -589,7 +601,7 @@ export class ConfirmError extends ConfigError {
589
601
  }
590
602
  // ---- the notice in replies ----
591
603
  // Fixed text only (setting names, validated site names, times, fixed words). Undefined when there is nothing to say.
592
- export function notice(now = Date.now()) {
604
+ export function notice(now = Date.now(), includeGuard = true) {
593
605
  try {
594
606
  const lines = [];
595
607
  const { rec, state } = loadRecord();
@@ -602,12 +614,15 @@ export function notice(now = Date.now()) {
602
614
  catch {
603
615
  raw = null; // a bad config.json is reported by the calls that use it
604
616
  }
605
- for (const w of clamp(raw, rec).waiting)
617
+ const effective = clamp(raw, rec);
618
+ // Guard notices belong only to local replies while the check is set, so off restores their former output.
619
+ const showGuard = includeGuard && !!effective.obj?.guard;
620
+ for (const w of effective.waiting.filter((w) => w.setting !== "guard" || showGuard))
606
621
  lines.push(`config.json loosens ${w.setting}${w.site ? ` for ${w.site}` : ""}, but that was not confirmed, so it is not in force. To confirm: ${w.command}`);
607
622
  for (const e of envNotices(rec, raw))
608
623
  lines.push(e);
609
624
  const since = now - NOTICE_HOURS * 3_600_000;
610
- const recent = rec.recent.filter((c) => c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
625
+ const recent = rec.recent.filter((c) => (c.setting !== "guard" || showGuard) && c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
611
626
  for (const c of recent)
612
627
  lines.push(`${c.at}: ${describe(c)}. See tiresias history.`);
613
628
  if (!lines.length)
package/dist/cooldown.js CHANGED
@@ -1,6 +1,6 @@
1
1
  // SPDX-License-Identifier: Apache-2.0
2
- // N-06: Retry-After is enforced, not just reported. When a site says 429 or
3
- // 503, we remember "do not call this origin until T", across processes, and
2
+ // N-06: Retry-After is enforced, not just reported. When a site says 429,
3
+ // 503 or asks for a wait on a redirect, we remember "do not call this origin until T", across processes, and
4
4
  // every later request (page, probe, robots refresh) checks it first.
5
5
  //
6
6
  // Storage is an append-only log with no lock. A lock cannot be made safe
@@ -13,7 +13,8 @@ import crypto from "node:crypto";
13
13
  import fs from "node:fs";
14
14
  import path from "node:path";
15
15
  import { canonOrigin } from "./sitekey.js";
16
- import { appendLine, cachePath, readJson, readLogSync, rootDir } from "./store.js";
16
+ import { Refused } from "./errors.js";
17
+ import { appendLine, cachePath, readJson, scanLogSync, rootDir } from "./store.js";
17
18
  // A wait over 10 years is not a real Retry-After. It is treated as invalid and
18
19
  // replaced by a fixed 24 hour cooldown. Anything below that limit is kept in
19
20
  // full, never shortened.
@@ -65,8 +66,7 @@ const LEGACY_OLD = "cooldowns.old.jsonl"; // written by v0.1.4, read only, never
65
66
  const MAX_ORIGIN = 400; // keeps every line well under 512 bytes (one atomic append)
66
67
  const mem = new Map(); // cooldowns set by this process (at most MEM_MAX)
67
68
  // v0.9 hardening (Codex round 6c, HIGH): `mem` no longer keeps every wait for the server's whole life: past MEM_MAX
68
- // entries, the expired ones are dropped. A wait that has not ended is never dropped (Codex round 7c: the shared file
69
- // is read only from its last 8 MiB, so it cannot stand in for one).
69
+ // entries, the expired ones are dropped. A wait that has not ended is never dropped.
70
70
  // v0.9 client trust (Travis approved, 2026-10-31; closes the class stopped in client-trust round 1): when `mem` is
71
71
  // full of waits that have not ended, a new one goes into a FIXED-SIZE store instead of growing memory: SLOTS
72
72
  // deadlines (8 MB), each origin written to K slots chosen by a keyed hash (a random key per process, so a site cannot
@@ -126,9 +126,6 @@ function remember(origin, until) {
126
126
  }
127
127
  export const cooldownStoreBytes = () => sketch?.byteLength ?? 0; // for tests
128
128
  export const cooldownMemSize = () => mem.size; // for tests
129
- // v0.5.1: the log is read again only when one of its files changed (size, mtime or inode), not on every
130
- // check (a request checks about eleven times). The stamp names the store root, so a new TIRESIAS_HOME
131
- // is read afresh.
132
129
  let memo = null;
133
130
  let reads = 0;
134
131
  export const loadCount = () => reads; // for tests
@@ -137,9 +134,11 @@ function stamp() {
137
134
  for (const f of [path.join(rootDir(), CUR), path.join(rootDir(), LEGACY_OLD), safePath(() => cachePath(LEGACY, "shared"))]) {
138
135
  try {
139
136
  const st = f ? fs.lstatSync(f) : null;
140
- parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ino}` : "-");
137
+ parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ctimeMs}:${st.dev}:${st.ino}:${st.mode}` : "-");
141
138
  }
142
- catch {
139
+ catch (e) {
140
+ if (e.code !== "ENOENT")
141
+ throw e;
143
142
  parts.push("-");
144
143
  }
145
144
  }
@@ -154,26 +153,54 @@ const safePath = (f) => {
154
153
  }
155
154
  };
156
155
  function load() {
157
- const st = stamp();
158
- if (memo && memo.stamp === st)
159
- return memo.map;
160
- const map = loadNow();
161
- memo = { stamp: st, map };
162
- return map;
156
+ try {
157
+ const st = stamp();
158
+ if (memo && memo.stamp === st)
159
+ return memo.map;
160
+ const root = rootDir();
161
+ const map = loadNow(memo?.root === root ? memo.map : undefined);
162
+ memo = { root, stamp: st, map };
163
+ return map;
164
+ }
165
+ catch {
166
+ memo = null;
167
+ throw new Refused("The site's Retry-After waits could not be read. No request was sent.");
168
+ }
163
169
  }
164
- // Latest deadline per origin. Expired and malformed entries are ignored; there
165
- // is no cap on the number of origins. Only the last 8 MB of the log is read.
166
- function loadNow() {
167
- reads++;
168
- const now = Date.now();
170
+ // Replay every live deadline. Overflow uses the same bounded, conservative slots as the in-process list.
171
+ function newIndex() {
169
172
  const best = new Map();
173
+ let overflow = null;
170
174
  const take = (origin, until) => {
171
- if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= now)
175
+ if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= Date.now())
172
176
  return;
173
177
  const key = canonOrigin(origin); // v0.6: one name per site (sitekey.ts)
174
- if (until > (best.get(key) ?? 0))
175
- best.set(key, until);
178
+ if (best.has(key) || (best.size < MEM_MAX && key.length <= MAX_ORIGIN)) {
179
+ if (until > (best.get(key) ?? 0))
180
+ best.set(key, until);
181
+ }
182
+ else {
183
+ overflow ??= new Float64Array(SLOTS);
184
+ for (const i of slotsOf(key))
185
+ if (until > overflow[i])
186
+ overflow[i] = until;
187
+ }
188
+ };
189
+ return {
190
+ take,
191
+ logs: new Map(),
192
+ exact: (origin) => best.get(origin) ?? 0,
193
+ get: (origin) => {
194
+ let least = Infinity;
195
+ if (overflow)
196
+ for (const i of slotsOf(origin))
197
+ least = Math.min(least, overflow[i]);
198
+ return Math.max(best.get(origin) ?? 0, least === Infinity ? 0 : least);
199
+ },
176
200
  };
201
+ }
202
+ function loadNow(index = newIndex()) {
203
+ reads++;
177
204
  let legacy = null;
178
205
  try {
179
206
  legacy = readJson(cachePath(LEGACY, "shared")); // W-18: shared, at the root
@@ -182,13 +209,11 @@ function loadNow() {
182
209
  // the root's cache/ is a link or someone else's: the legacy store is not read (Codex round 46)
183
210
  }
184
211
  if (legacy && typeof legacy === "object")
185
- for (const [o, u] of Object.entries(legacy))
186
- take(o, u);
187
- for (const e of readLogSync(LEGACY_OLD))
188
- take(e?.origin, e?.until); // v0.1.4 rotated file, read only
189
- for (const e of readLogSync(CUR))
190
- take(e?.origin, e?.until);
191
- return best;
212
+ for (const [origin, until] of Object.entries(legacy))
213
+ index.take(origin, until);
214
+ for (const name of [LEGACY_OLD, CUR])
215
+ index.logs.set(name, scanLogSync(name, (e) => index.take(e?.origin, e?.until), index.logs.get(name)));
216
+ return index;
192
217
  }
193
218
  // Fetchers register here so queued requests can be cancelled the moment a
194
219
  // cooldown starts for their site.
@@ -199,7 +224,7 @@ export function setCooldown(origin, status, header) {
199
224
  const until = Date.now() + seconds * 1000;
200
225
  // Exact sources only: the fixed-size store may answer later than the truth (a collision), and taking that as
201
226
  // "already written" would keep the line from other processes.
202
- const known = Math.max(mem.get(origin) ?? 0, load().get(origin) ?? 0);
227
+ const known = Math.max(mem.get(origin) ?? 0, load().exact(origin));
203
228
  remember(origin, until); // this process honors it at once
204
229
  // Only a later deadline is worth a line. Never shortens anything.
205
230
  if (until > known && origin.length <= MAX_ORIGIN)
package/dist/dailylog.js CHANGED
@@ -2,6 +2,7 @@
2
2
  // W-13: the daily log. A readable page for one day, built from history.jsonl
3
3
  // (and receipts.jsonl) on request. It is never stored as a second copy; the
4
4
  // only file it can create is an export the owner asks for with --to.
5
+ import { textView } from "./textview.js";
5
6
  import fs from "node:fs";
6
7
  import path from "node:path";
7
8
  import { pathToFileURL } from "node:url";
@@ -53,6 +54,9 @@ function outcome(e) {
53
54
  }
54
55
  // One settings change in plain words, from the record's fixed codes (history.ts keeps only known codes).
55
56
  function settingWords(e) {
57
+ const by0 = e.by === "terminal" ? "from a terminal (likely a person)" : e.by === "not_terminal" ? "NOT from a terminal (an agent, a script or cron)" : "by an unknown caller";
58
+ if (e.tool === "keys")
59
+ return `${e.event === "slip_key_rotated" ? `slip key rotated${e.revoked !== undefined ? `, ${e.revoked} slip${e.revoked === 1 ? "" : "s"} revoked` : ""}` : e.event === "signing_key_rotation_finished" ? "signing key rotation finished (previous key dropped)" : "signing key rotated"}, ${by0}`;
56
60
  const what = e.change === "never_ask" ? "confirmations for risky settings turned OFF (never ask again)"
57
61
  : e.change === "ask_again" ? "confirmations for risky settings turned back on"
58
62
  : `${e.setting ?? "a setting"}${e.site_key ? ` for ${code(e.site_key)}` : ""} ${e.change === "looser" ? "turned on or loosened" : "turned off or made stricter"}`;
@@ -64,7 +68,8 @@ const hhmm = (iso) => {
64
68
  const d = new Date(iso);
65
69
  return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`;
66
70
  };
67
- export async function buildLog(dayText) {
71
+ export async function buildLog(dayText, scan = {}) {
72
+ scan = { ...scan, deadline: scan.deadline ?? Date.now() + 60_000 };
68
73
  const { day, start, end } = parseDay(dayText);
69
74
  const entries = [];
70
75
  const changes = []; // W-32 (v0.9): risky settings changed this day get their own section, not counted as lookups
@@ -96,7 +101,8 @@ export async function buildLog(dayText) {
96
101
  }
97
102
  return false;
98
103
  }
99
- if (e.tool === "settings") {
104
+ if (e.tool === "settings" || e.tool === "keys") {
105
+ // v1.1: a key rotation is listed with the settings changes (it is just as risky)
100
106
  changeCount++;
101
107
  if (changes.length < MAX_LOG_LOOKUPS)
102
108
  changes.push(e);
@@ -112,7 +118,7 @@ export async function buildLog(dayText) {
112
118
  otherAgents++;
113
119
  results.set(outcome(e), (results.get(outcome(e)) ?? 0) + 1); // a fixed set of outcomes
114
120
  return false;
115
- });
121
+ }, 1, PAGE_BUDGET, scan);
116
122
  entries.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
117
123
  changes.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
118
124
  const perUrl = new Map();
@@ -136,7 +142,7 @@ export async function buildLog(dayText) {
136
142
  }
137
143
  }
138
144
  return false;
139
- });
145
+ }, 1, PAGE_BUDGET, scan);
140
146
  const link = (name) => `[${name}](${pathToFileURL(path.join(homeDir(), name)).href})`;
141
147
  const days = [...utcDays].sort().join(", ");
142
148
  const lines = [];
@@ -234,7 +240,7 @@ export async function exportLog(folder, dayText) {
234
240
  throw new Error("Private mode: nothing is written, so the log is not exported. Run it without --to to print it.");
235
241
  // 1. Build the whole text first. Nothing on disk is touched until it exists.
236
242
  const log = await buildLog(dayText);
237
- const body = fullText(log);
243
+ const body = textView(fullText(log));
238
244
  // 2. Resolve the folder ONCE and use only that real path from here on.
239
245
  let real;
240
246
  try {
package/dist/dom.js CHANGED
@@ -4,17 +4,19 @@
4
4
  import { DomHandler, Parser } from "htmlparser2";
5
5
  export const MAX_DOM_DEPTH = 256;
6
6
  export function parseDocument(body) {
7
- const handler = new DomHandler();
7
+ const handler = new DomHandler(undefined, { withStartIndices: true });
8
+ const source = new WeakMap();
8
9
  const opens = [];
9
10
  // The bounded parser can forget an ancestor before its source end tag. Keep source scopes with
10
11
  // push/pop (not htmlparser2's shifting stacks), so flattening cannot turn hidden descendants into
11
12
  // shown siblings. Each attribute set refers to its original parent; the reader resolves these in
12
13
  // source order, once the page's style sheet is known, without copying attributes or walking chains.
13
- const inherited = [];
14
+ const inherited = []; // third entry: the source parent is inside code/pre
14
15
  const scopes = [];
15
16
  const byName = new Map();
16
17
  const popScope = () => {
17
18
  const s = scopes.pop();
19
+ source.get(s.attrs).end = parser.startIndex;
18
20
  if (s.previous === undefined)
19
21
  byName.delete(s.name);
20
22
  else
@@ -28,12 +30,14 @@ export function parseDocument(body) {
28
30
  const close = handler.onclosetag.bind(handler);
29
31
  const text = handler.ontext.bind(handler);
30
32
  handler.onopentag = (name, attrs) => {
31
- if (scopes.length)
32
- inherited.push([attrs, scopes[scopes.length - 1].attrs]);
33
+ const parent = scopes[scopes.length - 1];
34
+ source.set(attrs, { parent: parent?.attrs });
35
+ if (parent)
36
+ inherited.push([attrs, parent.attrs, parent.code]);
33
37
  const key = name.toLowerCase();
34
38
  const previous = byName.get(key);
35
39
  byName.set(key, scopes.length);
36
- scopes.push({ name: key, attrs, previous });
40
+ scopes.push({ name: key, attrs, code: key === "code" || key === "pre" || parent?.code === true, previous });
37
41
  opens.push({ ...bounds, attrs });
38
42
  open(name, attrs);
39
43
  };
@@ -51,7 +55,7 @@ export function parseDocument(body) {
51
55
  return text(data);
52
56
  // Text after a flattened child's close (including raw text) has no element to inherit on.
53
57
  const attrs = {};
54
- inherited.push([attrs, scope.attrs]);
58
+ inherited.push([attrs, scope.attrs, scope.code]);
55
59
  open("thidden", attrs);
56
60
  text(data);
57
61
  close();
@@ -87,6 +91,7 @@ export function parseDocument(body) {
87
91
  popScope();
88
92
  }
89
93
  }
90
- new BoundedParser(handler).end(body);
91
- return { doc: handler.root, flattened, inherited };
94
+ const parser = new BoundedParser(handler);
95
+ parser.end(body);
96
+ return { doc: handler.root, flattened, inherited, source };
92
97
  }
package/dist/doors.js CHANGED
@@ -40,7 +40,7 @@ export const noStore = (h) => /(^|[\s,])no-store\b/i.test(h.get("cache-control")
40
40
  // is not "no door": it throws, so the sequential checks stop and nothing is cached (the next read asks again).
41
41
  async function probe(url, maxBytes, ctl, strict = false) {
42
42
  try {
43
- const res = await politeFetch(url, { maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
43
+ const res = await politeFetch(url, { robotsPass: false, maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
44
44
  if (res.status === 402)
45
45
  throw new PaymentRequired(res.headers, res.body, res.url);
46
46
  if (strict && (res.status >= 500 || res.status === 429))
package/dist/engine.js CHANGED
@@ -17,7 +17,12 @@ export * from "./resume.js";
17
17
  export * from "./formflow.js";
18
18
  export const read = async (url, opts = {}, ctx = DEFAULT) => {
19
19
  const saveFallback = opts.save ? saveRefusal() : undefined;
20
- const out = await guarded("read", url, ctx, opts.question, (m) => run("read", url, opts, m, ctx.agent), {}, { create: { question: opts.question, section: opts.section, render: opts.render, translate: opts.translate } });
20
+ const out = await guarded("read", url, ctx, opts.question, async (m) => {
21
+ const e = await run("read", url, opts, m, ctx.agent);
22
+ if (opts.budget !== undefined)
23
+ await opts.beforeLocalRecord?.(e, m);
24
+ return e;
25
+ }, {}, { create: { question: opts.question, section: opts.section, render: opts.render, translate: opts.translate } });
21
26
  // W-28 (v0.8): a read that asked to save always says what happened to the save (the pipeline sets it when it got
22
27
  // as far as a page or a file).
23
28
  if (opts.save && !out.saved)
@@ -56,13 +61,13 @@ export function slipList(page = 1) {
56
61
  export const claimSlip = slips.claimSlip;
57
62
  export const checkSlip = slips.checkSlip;
58
63
  export const queryHistory = history.query;
59
- export const dailyLog = async (day, page = 1) => {
60
- const log = await dailylog.buildLog(day);
64
+ export const dailyLog = async (day, page = 1, scan = {}) => {
65
+ const log = await dailylog.buildLog(day, scan);
61
66
  return { day: log.day, entries: log.entries, ...dailylog.pageOf(log, page) };
62
67
  };
63
68
  export const exportDailyLog = dailylog.exportLog;
64
69
  // W-12: generated from the history when asked, never stored.
65
- export const needsReport = (since, page = 1) => needs.buildNeeds(since, page);
70
+ export const needsReport = (since, page = 1, scan = {}) => needs.buildNeeds(since, page, Date.now(), scan);
66
71
  // W-19: tiresias safety status | update.
67
72
  export const safetyStatus = safebrowsing.status;
68
73
  export const safetyUpdate = safebrowsing.update;
package/dist/envelope.js CHANGED
@@ -80,6 +80,11 @@ export const receiptsEnvelope = (url, r) => {
80
80
  };
81
81
  export const historyEnvelope = (r) => listEnvelope("", { history: r.items }, r);
82
82
  export const STOPPED = "Stopped. Tiresias does not retry or work around walls.";
83
+ // W-39 (v1.2): the two values of a local passage's written_by, the wording measured in the lab.
84
+ export const BY_SITE = "the site";
85
+ export const BY_VISITOR = "a visitor to the site, not the site itself";
86
+ // What one label adds to a passage at most (the longer value, its key and a comma), for the reply budget.
87
+ export const LABEL_BYTES = jsonBytes({ written_by: BY_VISITOR }) - 2 + 1;
83
88
  // W-02 fixed texts.
84
89
  export const NO_BASELINE = "There was nothing to compare against: no earlier copy of this page is kept (first visit, a cleared or expired cache, or private mode).";
85
90
  export const NO_STORE_DIFF = "There was nothing to compare against: the site asks that this page not be stored (no-store), so no copy is kept.";
@@ -216,7 +221,10 @@ export async function guarded(...args) {
216
221
  catch (e) {
217
222
  configError = e;
218
223
  }
219
- return callContext.run({ ...callContext.getStore(), config, configError }, () => guardedCall(...args));
224
+ // One holder per tool/CLI call, before withSignal/runPrivate make shallow copies. Background door
225
+ // jobs create their own context without a holder, and an in-flight robots waiter gets no pass.
226
+ return callContext.run({ ...callContext.getStore(), config, configError, getActions: undefined,
227
+ robotsPasses: { pending: new Map(), attempted: new Set() } }, () => guardedCall(...args));
220
228
  }
221
229
  async function guardedCall(tool, url, ctx, purpose, fn, meta = {}, hand, safetyCheck = true, // W-19: false only for a page a person provided (nothing is requested)
222
230
  record = true) {
@@ -289,6 +297,7 @@ record = true) {
289
297
  lang: out.language?.code, // W-26 (v0.7)
290
298
  rendered: meta.rendered, // W-10 (v0.7)
291
299
  saved: meta.saved, // W-28 (v0.8)
300
+ written_by: meta.written_by, // W-39 (v1.2)
292
301
  });
293
302
  }
294
303
  catch (e) {
package/dist/fetcher.js CHANGED
@@ -1,12 +1,14 @@
1
1
  // SPDX-License-Identifier: Apache-2.0
2
2
  // The polite fetcher: safety (safety.ts) plus the site's rules (rules.ts).
3
3
  // Every request except robots.txt itself goes through politeFetch.
4
+ import { callContext } from "./context.js";
4
5
  import { cooldownLeft, setCooldown } from "./cooldown.js";
5
6
  import { Refused, Walled } from "./errors.js";
6
7
  import { spacingMs } from "./hosts.js";
7
8
  import { Turns, cooldownRefusal } from "./queue.js";
8
9
  import { checkAllowed } from "./rules.js";
9
10
  import { assertSiteAllowed } from "./sites.js";
11
+ import { canonOrigin } from "./sitekey.js";
10
12
  import { safeFetch } from "./safety.js";
11
13
  import { detectWall } from "./walls.js";
12
14
  export { Refused, Walled } from "./errors.js";
@@ -37,12 +39,20 @@ export async function politeFetch(url, opts = {}) {
37
39
  }
38
40
  }
39
41
  async function politeFetchHeld(url, opts, turn) {
42
+ const holder = callContext.getStore()?.robotsPasses;
43
+ const canClaim = opts.robotsPass !== false && !opts.submit && opts.method !== "POST";
44
+ let first = true;
40
45
  const res = await safeFetch(url, {
41
46
  ...opts,
42
47
  onHeaders: (hop) => {
43
48
  // Record the wait the moment 429/503 headers arrive, even if the body never finishes.
44
49
  if (hop.status === 429 || hop.status === 503)
45
50
  setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
51
+ if (hop.redirect && hop.headers.has("retry-after")) {
52
+ const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
53
+ if (seconds > 0)
54
+ throw new Refused(COOLDOWN_TEXT, undefined, seconds, hop.url.href);
55
+ }
46
56
  // A challenge (cf-mitigated) on ANY hop is a wall at that hop: a redirect is not
47
57
  // followed, and a 304 is not accepted as a revalidation. Plain 401/402/403 final
48
58
  // responses are handled with their body by the engine.
@@ -53,7 +63,24 @@ async function politeFetchHeld(url, opts, turn) {
53
63
  },
54
64
  beforeRequest: async (u, signal) => {
55
65
  turn.leaveOthers(u);
56
- const p = await policyCheck(u);
66
+ const origin = canonOrigin(u.origin);
67
+ const eligible = first && canClaim && holder && !holder.attempted.has(origin);
68
+ first = false;
69
+ // Claim before awaiting policy, even with no pending pass: only the first eligible fetch to
70
+ // this origin may use one. The policy check itself may fetch robots on direct fetcher paths.
71
+ if (eligible)
72
+ holder.attempted.add(origin);
73
+ let p;
74
+ let pass;
75
+ try {
76
+ p = await policyCheck(u);
77
+ }
78
+ finally {
79
+ if (eligible) {
80
+ pass = holder.pending.get(origin);
81
+ holder.pending.delete(origin); // consumed on admission or refusal, used or not
82
+ }
83
+ }
57
84
  if (!p.ok && "wall" in p)
58
85
  throw new Walled(p.wall, p.wallUrl);
59
86
  if (!p.ok)
@@ -61,7 +88,7 @@ async function politeFetchHeld(url, opts, turn) {
61
88
  const d = p;
62
89
  // v0.5.1: every request is queued and spaced, by at least the default spacing (1 s).
63
90
  // W-25 (v0.6): a site's own spacing raises it further, never lowers it (applied in the queue, queue.ts).
64
- await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal);
91
+ await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal, { pass: p.crawlDelay === 0 ? pass : undefined });
65
92
  },
66
93
  });
67
94
  return res;
package/dist/filereply.js CHANGED
@@ -15,7 +15,8 @@ import { NONBLOCK, isPrivate } from "./store.js";
15
15
  import { MAX_BLOCK } from "./reader.js";
16
16
  import { MAX_SAMPLE, detect, stampLanguage } from "./lang.js";
17
17
  import { translateReply } from "./translate.js";
18
- import { SecretWithheld, valueHoldsSecret } from "./echo.js";
18
+ import { SecretWithheld, noSecrets, valueHoldsSecret } from "./echo.js";
19
+ import { runUntrusted } from "./sandbox.js";
19
20
  import { BUDGET, env } from "./envelope.js";
20
21
  export class ProvidedError extends TiresiasError {
21
22
  kind = "provided";
@@ -81,6 +82,11 @@ export async function fileWarning(url, finalUrl, f, meta, agent, source, opts =
81
82
  }
82
83
  // v0.6: inside a zip or an Office file, in the worker, without unpacking it (inside.ts).
83
84
  const ins = await inspectFile(f);
85
+ if (ins?.office_text && !noSecrets(secrets)) {
86
+ const texts = [JSON.stringify(ins.office_text), ins.office_text.map((s) => s.text).join("\n")];
87
+ if (await runUntrusted({ op: "echo", texts, secrets: secrets }, Infinity))
88
+ throw new SecretWithheld();
89
+ }
84
90
  if (macroByName(f) || ins?.macros)
85
91
  fw.flags = [...new Set([...fw.flags, "macros"])];
86
92
  if (ins) {