@raibitsi/tiresias 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +88 -0
  2. package/README.md +72 -33
  3. package/data/README.md +33 -0
  4. package/data/guard-ordinary-urls.txt +102 -0
  5. package/data/guard-starter.jsonl +424 -0
  6. package/dist/aimed.js +23 -0
  7. package/dist/anchor.js +26 -13
  8. package/dist/browsecli.js +17 -13
  9. package/dist/cdp.js +22 -1
  10. package/dist/cleanup.js +3 -1
  11. package/dist/cli.js +216 -68
  12. package/dist/config.js +75 -1
  13. package/dist/confirm.js +20 -5
  14. package/dist/cooldown.js +57 -32
  15. package/dist/dailylog.js +11 -5
  16. package/dist/dom.js +13 -8
  17. package/dist/doors.js +1 -1
  18. package/dist/engine.js +9 -4
  19. package/dist/envelope.js +10 -1
  20. package/dist/fetcher.js +29 -2
  21. package/dist/filereply.js +7 -1
  22. package/dist/formflow.js +21 -3
  23. package/dist/forms.js +337 -42
  24. package/dist/getactions.js +119 -0
  25. package/dist/guard.js +1240 -0
  26. package/dist/guardclient.js +245 -0
  27. package/dist/guardnetwork.js +22 -0
  28. package/dist/history.js +11 -5
  29. package/dist/hook.js +2 -0
  30. package/dist/hosts.js +34 -27
  31. package/dist/keys.js +420 -18
  32. package/dist/local.js +537 -0
  33. package/dist/mcp.js +56 -25
  34. package/dist/needs.js +9 -5
  35. package/dist/parentwatch.js +6 -16
  36. package/dist/pdf.js +4 -1
  37. package/dist/pdfchild.js +0 -2
  38. package/dist/pdfproc.js +26 -11
  39. package/dist/pdfwatch.js +14 -0
  40. package/dist/pipeline.js +117 -26
  41. package/dist/preformatted.js +12 -0
  42. package/dist/profiles.js +3 -3
  43. package/dist/queue.js +24 -20
  44. package/dist/reader.js +441 -87
  45. package/dist/receipts.js +7 -2
  46. package/dist/render.js +156 -27
  47. package/dist/renderactions.js +69 -0
  48. package/dist/resume.js +8 -6
  49. package/dist/rules.js +24 -5
  50. package/dist/safebrowsing.js +3 -3
  51. package/dist/safety.js +15 -7
  52. package/dist/sandbox.js +1 -1
  53. package/dist/saved.js +24 -9
  54. package/dist/savedlock.js +99 -5
  55. package/dist/settings.js +10 -3
  56. package/dist/shelf.js +1 -1
  57. package/dist/shield.js +15 -0
  58. package/dist/slips.js +43 -1
  59. package/dist/store.js +119 -25
  60. package/dist/tabs.js +1 -1
  61. package/dist/textview.js +16 -0
  62. package/dist/translate.js +22 -2
  63. package/dist/types.js +9 -2
  64. package/dist/watchflow.js +1 -2
  65. package/dist/work.js +10 -2
  66. package/docs/LOCAL-MODELS.md +271 -0
  67. package/package.json +5 -2
@@ -0,0 +1,245 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // W-38 (v1.2): the meaning check's one network client. It talks to the owner's OWN embedding endpoint (LM Studio,
3
+ // Ollama, or any OpenAI-compatible /v1/embeddings), never to a site, so it is deliberately not the page fetcher:
4
+ // no robots.txt, no signing, no spacing, no cache, no Tiresias user agent promise. What it keeps from the fetcher's
5
+ // hardening is what protects the owner: one request, no redirect ever followed, a hard timeout, a cap on the bytes
6
+ // read, and a reply that is checked in full before a single number from it is used.
7
+ //
8
+ // Where it may send: the setting's address is resolved here, at call time, and the connection goes to the address
9
+ // that was checked (no second lookup). An address on this machine or the owner's network (loopback, private, link
10
+ // local, CGNAT 100.64.0.0/10 that Tailscale uses) is fine. Anything else means page text would leave the owner's
11
+ // network, so it needs `allow_public`, which only `tiresias guard setup` writes and only with the typed
12
+ // confirmation; plain http to such an address is refused whatever the setting says.
13
+ //
14
+ // The key: TIRESIAS_GUARD_KEY, read from the environment at call time and sent as a Bearer token. It is never
15
+ // stored, logged or put in a reply or an error.
16
+ import dns from "node:dns";
17
+ import http from "node:http";
18
+ import https from "node:https";
19
+ import net from "node:net";
20
+ import { isOwnerAddress } from "./guardnetwork.js";
21
+ const WORDS = {
22
+ refused_public: "the endpoint's address is not on this machine or your network, and sending page text there was not confirmed",
23
+ refused_http: "the endpoint's address is not on this machine or your network, and plain http to it is refused",
24
+ dns: "the endpoint's name could not be looked up",
25
+ network: "the endpoint could not be reached",
26
+ timeout: "the endpoint did not answer in time",
27
+ cancelled: "the call was cancelled",
28
+ http_status: "the endpoint answered with an error status",
29
+ redirect: "the endpoint answered with a redirect, which is never followed",
30
+ too_large: "the endpoint's answer was larger than allowed",
31
+ bad_json: "the endpoint's answer was not JSON",
32
+ bad_shape: "the endpoint's answer did not have a data list of embeddings",
33
+ wrong_count: "the endpoint returned a different number of embeddings than texts sent",
34
+ wrong_dimension: "the endpoint returned embeddings of an unexpected length",
35
+ not_finite: "the endpoint returned a number that is not finite",
36
+ zero_vector: "the endpoint returned an embedding of all zeros",
37
+ };
38
+ export class EmbedError extends Error {
39
+ code;
40
+ status;
41
+ constructor(code, status) {
42
+ super(`The embedding call failed: ${WORDS[code]}${status !== undefined ? ` (HTTP ${status})` : ""}.`);
43
+ this.code = code;
44
+ this.status = status;
45
+ }
46
+ }
47
+ // Bounds. A batch is at most MAX_BATCH texts; a reply at most MAX_REPLY bytes (64 texts of 4,096 numbers written
48
+ // out in full is about 6 MB); a dimension from 8 to 8,192.
49
+ export const MAX_BATCH = 64;
50
+ export const MAX_REPLY = 16 * 1024 * 1024;
51
+ export const MIN_DIM = 8;
52
+ export const MAX_DIM = 8192;
53
+ // v1.2: the whole input, including the configured prefix, must fit. Never silently embed a truncated text.
54
+ export const MAX_INPUT_CHARS = 8_000;
55
+ const isLoopbackName = (h) => h === "localhost" || h.endsWith(".localhost");
56
+ const bare = (h) => (h.startsWith("[") && h.endsWith("]") ? h.slice(1, -1) : h);
57
+ // The addresses a host name stands for, every one checked. A literal address is used as it is.
58
+ async function resolve(host, signal) {
59
+ if (signal.aborted)
60
+ throw new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled");
61
+ const h = bare(host);
62
+ if (net.isIP(h))
63
+ return [{ address: h, family: net.isIP(h) }];
64
+ const lookup = dns.promises.lookup(isLoopbackName(h) ? "localhost" : h, { all: true, verbatim: true });
65
+ const abort = new Promise((_, reject) => signal.addEventListener("abort", () => reject(new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled")), { once: true }));
66
+ let found;
67
+ try {
68
+ found = await Promise.race([lookup, abort]);
69
+ }
70
+ catch (e) {
71
+ if (e instanceof EmbedError)
72
+ throw e;
73
+ throw new EmbedError("dns");
74
+ }
75
+ if (!found.length)
76
+ throw new EmbedError("dns");
77
+ return found;
78
+ }
79
+ // "local" when every address is loopback, private, link local or CGNAT: page text stays on the owner's network.
80
+ export async function classify(url, signal = new AbortController().signal) {
81
+ const addrs = await resolve(new URL(url).hostname, signal);
82
+ return addrs.every((a) => isOwnerAddress(a.address)) ? "local" : "public";
83
+ }
84
+ // The whole check for one address: may page text be sent there under this setting?
85
+ export async function destination(cfg, signal) {
86
+ const u = new URL(cfg.url);
87
+ const addrs = await resolve(u.hostname, signal);
88
+ const local = addrs.every((a) => isOwnerAddress(a.address));
89
+ if (!local) {
90
+ if (u.protocol !== "https:")
91
+ throw new EmbedError("refused_http");
92
+ if (cfg.allow_public !== true)
93
+ throw new EmbedError("refused_public");
94
+ }
95
+ return addrs[0];
96
+ }
97
+ // One POST to <url>/embeddings with { model, input }, and the vectors from data[i].embedding, each made unit length.
98
+ // Throws an EmbedError with a fixed code on anything unexpected.
99
+ export async function embed(cfg, inputs, opts) {
100
+ if (inputs.length === 0)
101
+ return [];
102
+ if (inputs.length > MAX_BATCH)
103
+ throw new RangeError("too many texts in one batch");
104
+ if (inputs.some(t => t.length > MAX_INPUT_CHARS))
105
+ throw new RangeError("text too long for one embedding input");
106
+ const ctl = new AbortController();
107
+ const timer = setTimeout(() => ctl.abort("timeout"), opts.timeoutMs);
108
+ const outer = opts.signal;
109
+ const onOuter = () => ctl.abort("cancelled");
110
+ if (outer?.aborted)
111
+ ctl.abort("cancelled");
112
+ else
113
+ outer?.addEventListener("abort", onOuter, { once: true });
114
+ try {
115
+ const to = await destination(cfg, ctl.signal);
116
+ const body = Buffer.from(JSON.stringify({ model: cfg.model, input: inputs }));
117
+ const raw = await post(new URL(cfg.url.replace(/\/+$/, "") + "/embeddings"), body, to, ctl.signal);
118
+ return parseVectors(raw, inputs.length, opts.dim);
119
+ }
120
+ finally {
121
+ clearTimeout(timer);
122
+ outer?.removeEventListener("abort", onOuter);
123
+ }
124
+ }
125
+ function post(u, body, to, signal) {
126
+ return new Promise((resolveP, reject) => {
127
+ const fail = (code, status) => reject(new EmbedError(code, status));
128
+ if (signal.aborted)
129
+ return fail(signal.reason === "timeout" ? "timeout" : "cancelled");
130
+ const headers = { "content-type": "application/json", accept: "application/json", "content-length": String(body.length) };
131
+ const key = process.env.TIRESIAS_GUARD_KEY;
132
+ if (key)
133
+ headers.authorization = `Bearer ${key}`;
134
+ const mod = u.protocol === "https:" ? https : http;
135
+ // The connection goes to the address checked above: a lookup that answers with it, never a second DNS query.
136
+ const lookup = (_h, o, cb) => o?.all ? cb(null, [{ address: to.address, family: to.family }]) : cb(null, to.address, to.family);
137
+ const req = mod.request(u, { method: "POST", headers, lookup: lookup, agent: false, signal }, (res) => {
138
+ const status = res.statusCode ?? 0;
139
+ if (status >= 300 && status < 400) {
140
+ res.destroy();
141
+ return fail("redirect", status);
142
+ }
143
+ if (status !== 200) {
144
+ res.destroy();
145
+ return fail("http_status", status);
146
+ }
147
+ const declared = Number(res.headers["content-length"]);
148
+ if (Number.isFinite(declared) && declared > MAX_REPLY) {
149
+ res.destroy();
150
+ return fail("too_large");
151
+ }
152
+ const parts = [];
153
+ let total = 0;
154
+ res.on("data", (c) => {
155
+ total += c.length;
156
+ if (total > MAX_REPLY) {
157
+ res.destroy();
158
+ return fail("too_large");
159
+ }
160
+ parts.push(c);
161
+ });
162
+ res.on("end", () => resolveP(Buffer.concat(parts, total)));
163
+ res.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
164
+ });
165
+ req.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
166
+ req.end(body);
167
+ });
168
+ }
169
+ // The reply, checked whole: a data list of exactly `count` embeddings (if any item has an index, all must have a
170
+ // distinct index in range), each a list of finite numbers of one length (the stored dimension when there is one).
171
+ export function parseVectors(raw, count, dim) {
172
+ let j;
173
+ try {
174
+ j = JSON.parse(raw.toString("utf8"));
175
+ }
176
+ catch {
177
+ throw new EmbedError("bad_json");
178
+ }
179
+ const data = j && typeof j === "object" && !Array.isArray(j) ? j.data : undefined;
180
+ if (!Array.isArray(data))
181
+ throw new EmbedError("bad_shape");
182
+ if (data.length !== count)
183
+ throw new EmbedError("wrong_count");
184
+ let order = data;
185
+ const idx = data.map((d) => (d && typeof d === "object" ? d.index : undefined));
186
+ if (idx.some((i) => i !== undefined)) {
187
+ if (!idx.every((i) => Number.isInteger(i) && i >= 0 && i < count) || new Set(idx).size !== count)
188
+ throw new EmbedError("bad_shape");
189
+ order = new Array(count);
190
+ data.forEach((d, k) => (order[idx[k]] = d));
191
+ }
192
+ const out = [];
193
+ let len = dim;
194
+ for (const d of order) {
195
+ const e = d && typeof d === "object" ? d.embedding : undefined;
196
+ if (!Array.isArray(e))
197
+ throw new EmbedError("bad_shape");
198
+ if (len === undefined) {
199
+ if (e.length < MIN_DIM || e.length > MAX_DIM)
200
+ throw new EmbedError("wrong_dimension");
201
+ len = e.length;
202
+ }
203
+ if (e.length !== len)
204
+ throw new EmbedError("wrong_dimension");
205
+ const v = new Float32Array(len);
206
+ let n = 0;
207
+ let storedNorm = 0;
208
+ for (let i = 0; i < len; i++) {
209
+ const x = e[i];
210
+ if (typeof x !== "number" || !Number.isFinite(x))
211
+ throw new EmbedError("not_finite");
212
+ // Reject values outside the stored format before they can become infinities or an all-zero vector.
213
+ const stored = Math.fround(x);
214
+ if (!Number.isFinite(stored))
215
+ throw new EmbedError("not_finite");
216
+ storedNorm += stored * stored;
217
+ n += x * x;
218
+ }
219
+ n = Math.sqrt(n);
220
+ if (!(n > 0) || !Number.isFinite(n) || !(storedNorm > 0))
221
+ throw new EmbedError("zero_vector");
222
+ let finalNorm = 0;
223
+ for (let i = 0; i < len; i++) {
224
+ v[i] = e[i] / n;
225
+ if (!Number.isFinite(v[i]))
226
+ throw new EmbedError("not_finite");
227
+ finalNorm += v[i] * v[i];
228
+ }
229
+ if (!(finalNorm > 0) || !Number.isFinite(finalNorm))
230
+ throw new EmbedError("zero_vector");
231
+ out.push(v);
232
+ }
233
+ return out;
234
+ }
235
+ // Several batches in order, within one deadline.
236
+ export async function embedAll(cfg, inputs, opts) {
237
+ const out = [];
238
+ const size = Math.min(opts.batch ?? 32, MAX_BATCH);
239
+ for (let i = 0; i < inputs.length; i += size) {
240
+ const v = await embed(cfg, inputs.slice(i, i + size), { ...opts, dim: opts.dim ?? (out[0]?.length) });
241
+ out.push(...v);
242
+ opts.progress?.(out.length);
243
+ }
244
+ return out;
245
+ }
@@ -0,0 +1,22 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // The embedding endpoint may receive page text without confirmation only on the owner's own network.
3
+ import net from "node:net";
4
+ export function isOwnerAddress(ip) {
5
+ if (net.isIPv4(ip)) {
6
+ const [a, b] = ip.split(".").map(Number);
7
+ return a === 127 || a === 10 || (a === 172 && b >= 16 && b <= 31) ||
8
+ (a === 192 && b === 168) || (a === 169 && b === 254) || (a === 100 && b >= 64 && b <= 127);
9
+ }
10
+ const bare = ip.split("%")[0];
11
+ if (!net.isIPv6(bare))
12
+ return false;
13
+ // URL canonicalization gives mapped addresses one spelling, including when their tail was dotted IPv4.
14
+ const host = new URL(`http://[${bare}]/`).hostname.slice(1, -1);
15
+ const mapped = /^::ffff:([a-f0-9]+):([a-f0-9]+)$/.exec(host);
16
+ if (mapped) {
17
+ const hi = parseInt(mapped[1], 16), lo = parseInt(mapped[2], 16);
18
+ return isOwnerAddress(`${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`);
19
+ }
20
+ const first = parseInt(host.split(":")[0], 16);
21
+ return host === "::1" || (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80;
22
+ }
package/dist/history.js CHANGED
@@ -4,6 +4,7 @@
4
4
  // Tiresias is missing (W-12, needs.ts).
5
5
  import { appendLine, listLogs, pageLog, parseSince, utcDay } from "./store.js";
6
6
  import { LANG_SET } from "./lang.js";
7
+ const KEY_EVENTS = new Set(["signing_key_rotated", "signing_key_rotation_finished", "slip_key_rotated"]);
7
8
  // Bounds on what one record may hold, so a single line can never outgrow a history page.
8
9
  const MAX_AGENT = 200;
9
10
  const MAX_PURPOSE = 1000;
@@ -12,13 +13,14 @@ const MAX_URL_LOG = 4096;
12
13
  const REASONS = new Set(["bot_challenge", "captcha", "login_required", "forbidden", "owner_hand_off"]); // owner_hand_off: W-25 (v0.6)
13
14
  const DOORS = new Set(["mcp", "api", "feed", "structured", "text", "none"]);
14
15
  const MAX_MATCHED = 1000;
16
+ const MAX_WRITTEN_BY = 20; // W-39: a local reply shows at most 5 passages, or one page of a section
15
17
  const TICKETS = new Set(["created", "refused_cap"]);
16
18
  const CACHE = new Set(["fresh", "revalidated", "refetched"]);
17
19
  const FLAGS = new Set(["phishing_list", "web_risk"]);
18
20
  const SLIP_ID = /^[A-Za-z0-9_-]{24}$/;
19
21
  const FILE_KINDS = new Set(["program", "script", "shortcut", "addon", "archive", "office", "image", "media", "unknown"]);
20
22
  const SCANS = new Set(["bad", "clean", "unknown", "failed"]);
21
- const SETTINGS = new Set(["saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
23
+ const SETTINGS = new Set(["guard", "private_spacing", "hook_secrets", "saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
22
24
  const CHANGES = new Set(["looser", "stricter", "never_ask", "ask_again"]);
23
25
  const BYS = new Set(["terminal", "not_terminal"]);
24
26
  const HOWS = new Set(["phrase", "never_ask", "not_needed"]);
@@ -51,7 +53,7 @@ export function bound(e) {
51
53
  drop("llms_txt");
52
54
  if (e.ticket !== undefined && !(typeof e.ticket === "string" && TICKETS.has(e.ticket)))
53
55
  drop("ticket");
54
- if (e.source !== undefined && e.source !== "provided_by_person")
56
+ if (e.source !== undefined && e.source !== "provided_by_person" && e.source !== "provided_by_agent")
55
57
  drop("source");
56
58
  if (e.cache !== undefined && !(typeof e.cache === "string" && CACHE.has(e.cache)))
57
59
  drop("cache");
@@ -79,8 +81,12 @@ export function bound(e) {
79
81
  drop("by");
80
82
  if (e.confirmed !== undefined && !(typeof e.confirmed === "string" && HOWS.has(e.confirmed)))
81
83
  drop("confirmed");
82
- if (e.event !== undefined && e.event !== "memory_ceiling")
84
+ if (e.event !== undefined && !(e.event === "memory_ceiling" || (typeof e.event === "string" && KEY_EVENTS.has(e.event))))
83
85
  drop("event");
86
+ if (e.revoked !== undefined && !(Number.isSafeInteger(e.revoked) && e.revoked >= 0 && e.revoked <= 100_000))
87
+ drop("revoked");
88
+ if (e.written_by !== undefined && !(Array.isArray(e.written_by) && e.written_by.length >= 1 && e.written_by.length <= MAX_WRITTEN_BY && e.written_by.every((n) => n === 0 || n === 1 || n === 2 || n === 3)))
89
+ drop("written_by");
84
90
  if (e.memory_mb !== undefined && !(Number.isSafeInteger(e.memory_mb) && e.memory_mb >= 1 && e.memory_mb <= 65536))
85
91
  drop("memory_mb");
86
92
  if (over)
@@ -92,7 +98,7 @@ export function record(e) {
92
98
  const t = Date.parse(out.time);
93
99
  appendLine(`history/${utcDay(Number.isNaN(t) ? Date.now() : t)}.jsonl`, out);
94
100
  }
95
- export async function query(f) {
101
+ export async function query(f, scan = {}) {
96
102
  const since = parseSince(f.since);
97
103
  const site = f.site?.toLowerCase().replace(/^www\./, "");
98
104
  return pageLog(listLogs("history").map((f) => f.rel), (e) => {
@@ -115,5 +121,5 @@ export async function query(f) {
115
121
  }
116
122
  bound(e); // filters match the stored values; only what is returned is cut
117
123
  return true;
118
- }, f.page ?? 1);
124
+ }, f.page ?? 1, 40_000, scan);
119
125
  }
package/dist/hook.js CHANGED
@@ -7,6 +7,7 @@
7
7
  // answered, or within 200 ms of Tiresias being gone. Tiresias also kills the group at the cap, on cancellation
8
8
  // and once it has the answer, so a hook that starts helpers leaves nothing running. Its output is capped; its
9
9
  // standard error is ignored. Hooks are never run on Windows (no process groups to kill).
10
+ import { loadChoice } from "./config.js";
10
11
  import { spawn } from "node:child_process";
11
12
  import { fileURLToPath } from "node:url";
12
13
  import { currentSignal } from "./safety.js";
@@ -48,6 +49,7 @@ export function runHook(command, extra, timeoutMs, maxOut, input) {
48
49
  const cap = maxOut === 4096 ? [] : [`max=${maxOut}`]; // the runner's default is the scanner's 4 KB
49
50
  child = spawn(process.execPath, [runner, String(process.pid), String(timeoutMs), ...cap, command[0], ...command.slice(1), ...extra], {
50
51
  // Always its own group: the runner kills its WHOLE group, which must never be ours.
52
+ env: loadChoice("hook_secrets") === "inherit" ? { ...process.env } : Object.fromEntries(Object.entries(process.env).filter(([k]) => !/^TIRESIAS_.*(?:KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)/i.test(k))),
51
53
  shell: false, stdio: ["pipe", "pipe", "ignore"], detached: true, windowsHide: true,
52
54
  });
53
55
  }
package/dist/hosts.js CHANGED
@@ -3,18 +3,18 @@
3
3
  // in the shared hosts/ folder at the store root, holding the earliest time the next request to it may
4
4
  // start. A request reserves its slot under a short exclusive lock (an O_EXCL lock file, broken when
5
5
  // older than LOCK_STALE_MS), so the CLI and the MCP server running side by side space their requests
6
- // to one site as one queue does. Private mode and form_preview read the spacing but never write it (a
7
- // private visit leaves no trace of which site it called; a preview writes nothing); their spacing then
8
- // holds within the process only.
6
+ // to one site as one queue does. Private calls and previews share this minimal politeness state too.
9
7
  import { randomBytes } from "node:crypto";
10
8
  import fs from "node:fs";
11
9
  import path from "node:path";
10
+ import { loadChoice } from "./config.js";
12
11
  import { Refused } from "./errors.js";
13
12
  import { holdLease } from "./lifetime.js";
14
13
  import { lock } from "./lockfile.js";
15
14
  import { releaseLock, takeLock } from "./ownedlock.js";
16
15
  import { currentReadOnly } from "./safety.js";
17
16
  import { NOFOLLOW, NONBLOCK, ensureRoot, isPrivate, ownFile, readRoot, sha256, checkSub, mkdirOne } from "./store.js";
17
+ export const sharesSpacing = () => (!isPrivate() && !currentReadOnly()) || loadChoice("private_spacing") === "shared";
18
18
  export const HOSTS = "hosts";
19
19
  export const DEFAULT_SPACING_MS = 1_000;
20
20
  // The least time between two requests to one site. One second by default; TIRESIAS_HOST_SPACING_MS
@@ -26,49 +26,47 @@ export function spacingMs() {
26
26
  return DEFAULT_SPACING_MS;
27
27
  }
28
28
  const fileOf = (dir, origin) => path.join(dir, sha256(origin).slice(0, 32));
29
- // The earliest start the shared file allows for a request wanting `spacing` ms: after the next slot the
30
- // last reservation left, AND `spacing` after that reservation's own start (Codex round 69: a 1-second
31
- // reservation must not let a 30-second Crawl-delay request start a second later).
32
- // The spacing file, read safely (no links, no blocking, a regular file of ours, at most 256 bytes).
33
- function readSpacing(file) {
29
+ function readSpacing(file, strict = false) {
34
30
  let fd;
35
31
  try {
36
32
  fd = fs.openSync(file, fs.constants.O_RDONLY | NOFOLLOW | NONBLOCK);
37
33
  }
38
34
  catch {
39
- return { last: 0, next: 0 };
35
+ return undefined;
40
36
  }
41
37
  try {
42
38
  ownFile(fd, file);
43
39
  if (fs.fstatSync(fd).size > 256)
44
- return { last: 0, next: 0 };
40
+ return undefined;
45
41
  const o = JSON.parse(fs.readFileSync(fd, "utf8"));
46
42
  const next = Number(o?.next);
47
43
  const last = Number(o?.last);
44
+ if (strict && (typeof o?.last !== "number" || typeof o?.next !== "number" || !Number.isFinite(last) || !Number.isFinite(next)))
45
+ return undefined;
48
46
  return { last: Number.isFinite(last) ? last : 0, next: Number.isFinite(next) ? next : 0 };
49
47
  }
50
48
  catch {
51
- return { last: 0, next: 0 };
49
+ return undefined;
52
50
  }
53
51
  finally {
54
52
  fs.closeSync(fd);
55
53
  }
56
54
  }
57
55
  function readNext(file, spacing) {
58
- const { last, next } = readSpacing(file);
56
+ const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
59
57
  return Math.max(next, last ? last + spacing : 0);
60
58
  }
61
59
  // Reserve the next slot for `origin` with `spacing` ms between starts and return when it begins (epoch
62
60
  // ms). `earliest` is this process's own earliest time. Store trouble never fails a request: the slot is
63
61
  // then this process's own. `whenReady` leaves a future slot unclaimed so a cancelled wait moves nothing.
64
- export function reserve(origin, spacing, earliest, whenReady = false) {
62
+ export function reserve(origin, spacing, earliest, whenReady = false, pass) {
65
63
  const now = Date.now();
66
64
  let dir;
67
65
  try {
68
- if (isPrivate() || currentReadOnly()) {
66
+ if (!sharesSpacing()) {
69
67
  const r = readRoot();
70
68
  dir = r && checkSub(path.join(r, HOSTS)) ? path.join(r, HOSTS) : null;
71
- return Math.max(now, earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
69
+ return Math.max(now, pass ? 0 : earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
72
70
  }
73
71
  const root = ensureRoot();
74
72
  dir = path.join(root, HOSTS);
@@ -88,7 +86,10 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
88
86
  if (!held)
89
87
  return Math.max(now, earliest, readNext(file, spacing)) + spacing;
90
88
  try {
91
- const slot = Math.max(now, earliest, readNext(file, spacing));
89
+ // The active lease is still held. Only an exact, safely read completion stamp can be waived.
90
+ const current = pass ? readSpacing(file, true) : undefined;
91
+ const waive = pass && current?.last === pass.last && current?.next === pass.next;
92
+ const slot = waive ? now : Math.max(now, earliest, readNext(file, spacing));
92
93
  if (whenReady && slot > now)
93
94
  return slot;
94
95
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
@@ -104,6 +105,8 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
104
105
  }
105
106
  catch {
106
107
  fs.rmSync(tmp, { force: true });
108
+ if (waive)
109
+ return Math.max(now, earliest, readNext(file, spacing));
107
110
  }
108
111
  return slot;
109
112
  }
@@ -114,7 +117,7 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
114
117
  // One request at a time across processes too (Codex round 75): while a request to a site is being
115
118
  // answered, a small "active" file (the owner's pid and the time) sits beside its spacing file; others wait
116
119
  // until it is gone. A file whose process is gone, or that is older than a call can live (75 s), is broken.
117
- // Private calls and previews wait for it but never write it. Store trouble never blocks a request.
120
+ // Private calls and previews share it by default; unshared calls wait without writing. Store trouble never blocks a request.
118
121
  // v0.6 (coordinator's decision after Codex round 8): a lease is NEVER taken from a live process, however old it
119
122
  // is; a lease whose process is gone is stale at once. Only a lease whose owner cannot be told (no process id) is
120
123
  // judged by age, after this long (longer than any holder can live: a background job lasts up to 240 seconds).
@@ -204,9 +207,9 @@ function activeStale(f) {
204
207
  }
205
208
  }
206
209
  // A delay learned from robots.txt reaches other processes too (Codex round 84): the shared file's next
207
- // slot is moved to `until` if later. Never in private mode or a preview; no write without the lock.
210
+ // slot is moved to `until` if later. Shared by default for private calls and previews; no write without the lock.
208
211
  export function extendNext(origin, until) {
209
- if (isPrivate() || currentReadOnly())
212
+ if (!sharesSpacing())
210
213
  return;
211
214
  try {
212
215
  const r = readRoot();
@@ -217,7 +220,7 @@ export function extendNext(origin, until) {
217
220
  if (!lock(file))
218
221
  return;
219
222
  try {
220
- const { last, next } = readSpacing(file);
223
+ const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
221
224
  if (until <= next)
222
225
  return;
223
226
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
@@ -240,9 +243,9 @@ export function extendNext(origin, until) {
240
243
  }
241
244
  // v0.6 (Codex round 11, coordinator's decision): a held turn's completion is recorded WHOLE in the shared file:
242
245
  // last = the completion time, next = the later of the old next and completion + delay. Every reader, whatever
243
- // spacing it needs, then measures from the last completion. Never in private mode or a preview; under the lock.
246
+ // spacing it needs, then measures from the last completion. Shared by default for private calls and previews; under the lock.
244
247
  export function noteCompletion(origin, t, until) {
245
- if (isPrivate() || currentReadOnly())
248
+ if (!sharesSpacing())
246
249
  return;
247
250
  try {
248
251
  const r = readRoot();
@@ -253,16 +256,19 @@ export function noteCompletion(origin, t, until) {
253
256
  if (!lock(file))
254
257
  return;
255
258
  try {
256
- const { last, next } = readSpacing(file);
259
+ const previous = readSpacing(file, true);
260
+ const { last, next } = previous ?? readSpacing(file) ?? { last: 0, next: 0 };
261
+ const stamp = { last: Math.max(last, t), next: Math.max(next, until) };
257
262
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
258
263
  const fd = fs.openSync(tmp, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NOFOLLOW, 0o600);
259
264
  try {
260
- fs.writeSync(fd, JSON.stringify({ v: 1, last: Math.max(last, t), next: Math.max(next, until) }));
265
+ fs.writeSync(fd, JSON.stringify({ v: 1, ...stamp }));
261
266
  }
262
267
  finally {
263
268
  fs.closeSync(fd);
264
269
  }
265
270
  fs.renameSync(tmp, file);
271
+ return previous ? stamp : undefined;
266
272
  }
267
273
  finally {
268
274
  fs.rmSync(file + ".lock", { force: true });
@@ -279,10 +285,10 @@ const pause = (ms, signal) => new Promise((resolve, reject) => {
279
285
  const stop = () => (clearTimeout(t), reject(signal.reason));
280
286
  signal.addEventListener("abort", stop, { once: true });
281
287
  });
282
- export async function acquireActive(origin, signal) {
288
+ export async function acquireActive(origin, signal, onHeld) {
283
289
  let dir = null;
284
290
  try {
285
- if (!isPrivate() && !currentReadOnly()) {
291
+ if (sharesSpacing()) {
286
292
  // A writer makes hosts/ first, so the very first request to a site holds its lease too (round 77).
287
293
  const root = ensureRoot();
288
294
  const d = path.join(root, HOSTS);
@@ -301,7 +307,7 @@ export async function acquireActive(origin, signal) {
301
307
  if (!dir)
302
308
  return () => { };
303
309
  const f = fileOf(dir, origin) + ".active";
304
- const writer = !isPrivate() && !currentReadOnly();
310
+ const writer = sharesSpacing();
305
311
  const giveUp = Date.now() + leaseWaitMs();
306
312
  for (;;) {
307
313
  if (writer) {
@@ -321,6 +327,7 @@ export async function acquireActive(origin, signal) {
321
327
  fs.rmSync(f, { force: true });
322
328
  };
323
329
  const forget = holdLease(f, remove);
330
+ onHeld?.();
324
331
  return () => {
325
332
  remove();
326
333
  forget();