@raibitsi/tiresias 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +72 -33
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +17 -13
- package/dist/cdp.js +22 -1
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +216 -68
- package/dist/config.js +75 -1
- package/dist/confirm.js +20 -5
- package/dist/cooldown.js +57 -32
- package/dist/dailylog.js +11 -5
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +9 -4
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +29 -2
- package/dist/filereply.js +7 -1
- package/dist/formflow.js +21 -3
- package/dist/forms.js +337 -42
- package/dist/getactions.js +119 -0
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +11 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +34 -27
- package/dist/keys.js +420 -18
- package/dist/local.js +537 -0
- package/dist/mcp.js +56 -25
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +117 -26
- package/dist/preformatted.js +12 -0
- package/dist/profiles.js +3 -3
- package/dist/queue.js +24 -20
- package/dist/reader.js +441 -87
- package/dist/receipts.js +7 -2
- package/dist/render.js +156 -27
- package/dist/renderactions.js +69 -0
- package/dist/resume.js +8 -6
- package/dist/rules.js +24 -5
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +15 -7
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +24 -9
- package/dist/savedlock.js +99 -5
- package/dist/settings.js +10 -3
- package/dist/shelf.js +1 -1
- package/dist/shield.js +15 -0
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +16 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +9 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +10 -2
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
package/dist/config.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// Absent means these defaults. A bad value is a clear error, never a guess.
|
|
4
4
|
import fs from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
|
+
import { isOwnerAddress } from "./guardnetwork.js";
|
|
6
7
|
import { releaseLock, stillOwned, takeLock } from "./ownedlock.js";
|
|
7
8
|
import { NOFOLLOW, NONBLOCK, home, ownFile, randomTmp, readOwnedText } from "./store.js";
|
|
8
9
|
// W-32 (v0.9): readers of a risky setting use the value in force (an unconfirmed loosening is held back).
|
|
@@ -64,6 +65,63 @@ export function loadTranslator() {
|
|
|
64
65
|
return null;
|
|
65
66
|
return parseTranslator(o.translator);
|
|
66
67
|
}
|
|
68
|
+
export const GUARD_TIMEOUT = 4;
|
|
69
|
+
const GUARD_KEYS = new Set(["url", "model", "prefix", "timeout_seconds", "allow_public"]);
|
|
70
|
+
const BAD_GUARD = 'config.json: "guard" must be {"url": "http://localhost:1234/v1", "model": "<embedding model id>", "prefix": "...", "timeout_seconds": 1 to 60}. Set it with: tiresias guard setup --url <url> --model <id>';
|
|
71
|
+
const CONTROL = /[\u0000-\u001f\u007f]/;
|
|
72
|
+
// Addresses written as an IP that are public. A host NAME is judged when the call is made (guardclient.ts), from
|
|
73
|
+
// the addresses it resolves to.
|
|
74
|
+
const PUBLIC_LITERAL = (host, isPrivate) => {
|
|
75
|
+
const h = host.startsWith("[") && host.endsWith("]") ? host.slice(1, -1) : host;
|
|
76
|
+
return /^[0-9.]+$/.test(h) || h.includes(":") ? !isPrivate(h) : false;
|
|
77
|
+
};
|
|
78
|
+
export function parseGuardUrl(v, isPrivate = isOwnerAddress) {
|
|
79
|
+
if (typeof v !== "string" || v.length === 0 || v.length > 2048 || CONTROL.test(v))
|
|
80
|
+
throw new ConfigError(BAD_GUARD);
|
|
81
|
+
let u;
|
|
82
|
+
try {
|
|
83
|
+
u = new URL(v);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
throw new ConfigError('The guard "url" is not a valid address. Give the base of an OpenAI-compatible API, like http://localhost:1234/v1');
|
|
87
|
+
}
|
|
88
|
+
if (u.protocol !== "http:" && u.protocol !== "https:")
|
|
89
|
+
throw new ConfigError('The guard "url" must start with http:// or https://.');
|
|
90
|
+
if (u.username || u.password)
|
|
91
|
+
throw new ConfigError('The guard "url" must not contain a user name or password. A key goes in the environment variable TIRESIAS_GUARD_KEY.');
|
|
92
|
+
if (u.search || u.hash || v.includes("?") || v.includes("#"))
|
|
93
|
+
throw new ConfigError('The guard "url" must not have a query or a # part.');
|
|
94
|
+
if (u.protocol === "http:" && PUBLIC_LITERAL(u.hostname, isPrivate))
|
|
95
|
+
throw new ConfigError("Plain http to a public address is refused: page text would cross the internet unencrypted. Use https, or an address on your own network.");
|
|
96
|
+
return (u.origin + u.pathname).replace(/\/+$/, "");
|
|
97
|
+
}
|
|
98
|
+
export function parseGuard(v, isPrivate) {
|
|
99
|
+
if (typeof v !== "object" || v === null || Array.isArray(v))
|
|
100
|
+
throw new ConfigError(BAD_GUARD);
|
|
101
|
+
const o = v;
|
|
102
|
+
for (const k of Object.keys(o))
|
|
103
|
+
if (!GUARD_KEYS.has(k))
|
|
104
|
+
throw new ConfigError(BAD_GUARD);
|
|
105
|
+
const url = parseGuardUrl(o.url, isPrivate);
|
|
106
|
+
if (typeof o.model !== "string" || o.model.length === 0 || o.model.length > 200 || CONTROL.test(o.model))
|
|
107
|
+
throw new ConfigError(BAD_GUARD);
|
|
108
|
+
const prefix = Object.hasOwn(o, "prefix") ? o.prefix : "";
|
|
109
|
+
if (typeof prefix !== "string" || prefix.length > 200 || CONTROL.test(prefix))
|
|
110
|
+
throw new ConfigError(BAD_GUARD);
|
|
111
|
+
const t = Object.hasOwn(o, "timeout_seconds") ? o.timeout_seconds : GUARD_TIMEOUT;
|
|
112
|
+
if (typeof t !== "number" || !Number.isSafeInteger(t) || t < 1 || t > 60)
|
|
113
|
+
throw new ConfigError(BAD_GUARD);
|
|
114
|
+
if (Object.hasOwn(o, "allow_public") && o.allow_public !== true)
|
|
115
|
+
throw new ConfigError(BAD_GUARD);
|
|
116
|
+
return { url, model: o.model, prefix, timeout_seconds: t, ...(o.allow_public === true ? { allow_public: true } : {}) };
|
|
117
|
+
}
|
|
118
|
+
// The meaning check of this profile, or null when none is set. A bad setting is a ConfigError.
|
|
119
|
+
export function loadGuard(isPrivate) {
|
|
120
|
+
const o = inForceConfig();
|
|
121
|
+
if (!o || !Object.hasOwn(o, "guard"))
|
|
122
|
+
return null;
|
|
123
|
+
return parseGuard(o.guard, isPrivate);
|
|
124
|
+
}
|
|
67
125
|
// The whole config.json as an object (null when there is none). A bad file is a ConfigError.
|
|
68
126
|
function readConfigFile() {
|
|
69
127
|
const raw = readOwnedText("config.json");
|
|
@@ -196,7 +254,23 @@ export function loadLimits() {
|
|
|
196
254
|
// ---- v0.6 (track 3: settings) ----
|
|
197
255
|
// Keys checked only where they are used (like "scanner"): a bad one fails the calls that use it, by name,
|
|
198
256
|
// and never stops clear or the daily trim.
|
|
199
|
-
const
|
|
257
|
+
export const SECURITY_CHOICES = {
|
|
258
|
+
get_form_actions: ["read", "slip"],
|
|
259
|
+
private_spacing: ["shared", "none"],
|
|
260
|
+
hook_secrets: ["hidden", "inherit"],
|
|
261
|
+
text_views: ["marked", "wrapped"],
|
|
262
|
+
written_by: ["on", "off"], // W-39 (v1.2): local mode's written_by label
|
|
263
|
+
};
|
|
264
|
+
export function parseChoice(name, value) {
|
|
265
|
+
if (typeof value === "string" && SECURITY_CHOICES[name].includes(value))
|
|
266
|
+
return value;
|
|
267
|
+
throw new ConfigError(`config.json: "${name}" must be ${SECURITY_CHOICES[name].map((s) => JSON.stringify(s)).join(" or ")}.`);
|
|
268
|
+
}
|
|
269
|
+
export function loadChoice(name) {
|
|
270
|
+
const o = inForceConfig();
|
|
271
|
+
return o && Object.hasOwn(o, name) ? parseChoice(name, o[name]) : SECURITY_CHOICES[name][0];
|
|
272
|
+
}
|
|
273
|
+
const OWN_CHECK_KEYS = [...Object.keys(SECURITY_CHOICES), "doors", "sites", "translator", "rendering", "saving", "anchoring", "anchor_calendars", "guard"]; // translator: v0.7 W-26; rendering: v0.7 W-10; saving: v0.8 W-28; anchoring, anchor_calendars: v0.8 W-08; guard: v1.2 W-38
|
|
200
274
|
export const KNOWN_KEYS = [...Object.keys(DEFAULT_LIMITS), ...SIGNING_KEYS, ...SAFETY_KEYS, "scanner", ...OWN_CHECK_KEYS];
|
|
201
275
|
export const DEFAULT_DOORS = "background";
|
|
202
276
|
export function parseDoors(v) {
|
package/dist/confirm.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { callContext } from "./context.js";
|
|
16
|
-
import { DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
|
|
16
|
+
import { SECURITY_CHOICES, parseChoice, DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseGuard, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
|
|
17
17
|
import { parseRender } from "./render.js";
|
|
18
18
|
import { ConfigError } from "./errors.js";
|
|
19
19
|
import * as history from "./history.js";
|
|
@@ -46,7 +46,11 @@ const calendarSet = (v) => {
|
|
|
46
46
|
}
|
|
47
47
|
};
|
|
48
48
|
const renderRank = rank(["off", "ask", "auto"]);
|
|
49
|
+
const publicGuard = (v) => !!v && typeof v === "object" && v.allow_public === true;
|
|
50
|
+
const guardUrl = (v) => (v && typeof v === "object" ? String(v.url) : "");
|
|
49
51
|
export const TOP = [
|
|
52
|
+
{ key: "private_spacing", looser: (c, n) => c !== "none" && n === "none", phrase: () => "YES, TURN OFF PRIVATE SPACING", what: () => "private spacing across processes turned off" },
|
|
53
|
+
{ key: "hook_secrets", looser: (c, n) => c !== "inherit" && n === "inherit", phrase: () => "YES, SHARE TIRESIAS SECRETS WITH HOOKS", what: () => "Tiresias secrets shared with hooks" },
|
|
50
54
|
{ key: "saving", looser: (c, n) => rank(["off", "ask"])(n) > rank(["off", "ask"])(c), phrase: () => "YES, TURN ON SAVING", what: () => "saving turned on" },
|
|
51
55
|
{
|
|
52
56
|
key: "rendering",
|
|
@@ -68,7 +72,11 @@ export const TOP = [
|
|
|
68
72
|
what: () => "anchor calendars were added",
|
|
69
73
|
},
|
|
70
74
|
{ key: "safe_browsing", looser: (c, n) => c !== false && n === false, phrase: () => "YES, TURN OFF SAFE BROWSING", what: () => "Safe Browsing turned off" },
|
|
75
|
+
// W-38 (v1.2): the meaning check sends page text to the owner's embedding endpoint. Sending it to an address
|
|
76
|
+
// outside the owner's network (allow_public) is the loosening; a new address with it set needs the phrase again.
|
|
77
|
+
{ key: "guard", looser: (c, n) => publicGuard(n) && !(publicGuard(c) && guardUrl(c) === guardUrl(n)), phrase: () => GUARD_PHRASE, what: () => "the meaning check was allowed to send page text outside your network" },
|
|
71
78
|
];
|
|
79
|
+
export const GUARD_PHRASE = "YES, SEND PAGE TEXT OUTSIDE MY NETWORK";
|
|
72
80
|
const TOP_BY = new Map(TOP.map((t) => [t.key, t]));
|
|
73
81
|
export const RISKY_KEYS = TOP.map((t) => t.key);
|
|
74
82
|
export const isRisky = (name) => TOP_BY.has(name) || SITE_RISKY.includes(name);
|
|
@@ -408,7 +416,9 @@ function ratchet(raw, rec) {
|
|
|
408
416
|
}
|
|
409
417
|
function validTop(key, v) {
|
|
410
418
|
try {
|
|
411
|
-
if (key
|
|
419
|
+
if (Object.hasOwn(SECURITY_CHOICES, key))
|
|
420
|
+
parseChoice(key, v);
|
|
421
|
+
else if (key === "saving")
|
|
412
422
|
parseSaving(v);
|
|
413
423
|
else if (key === "rendering")
|
|
414
424
|
parseRender(v);
|
|
@@ -422,6 +432,8 @@ function validTop(key, v) {
|
|
|
422
432
|
parseCalendars(v);
|
|
423
433
|
else if (key === "safe_browsing")
|
|
424
434
|
parseSafeBrowsing(v);
|
|
435
|
+
else if (key === "guard")
|
|
436
|
+
parseGuard(v);
|
|
425
437
|
return true;
|
|
426
438
|
}
|
|
427
439
|
catch {
|
|
@@ -589,7 +601,7 @@ export class ConfirmError extends ConfigError {
|
|
|
589
601
|
}
|
|
590
602
|
// ---- the notice in replies ----
|
|
591
603
|
// Fixed text only (setting names, validated site names, times, fixed words). Undefined when there is nothing to say.
|
|
592
|
-
export function notice(now = Date.now()) {
|
|
604
|
+
export function notice(now = Date.now(), includeGuard = true) {
|
|
593
605
|
try {
|
|
594
606
|
const lines = [];
|
|
595
607
|
const { rec, state } = loadRecord();
|
|
@@ -602,12 +614,15 @@ export function notice(now = Date.now()) {
|
|
|
602
614
|
catch {
|
|
603
615
|
raw = null; // a bad config.json is reported by the calls that use it
|
|
604
616
|
}
|
|
605
|
-
|
|
617
|
+
const effective = clamp(raw, rec);
|
|
618
|
+
// Guard notices belong only to local replies while the check is set, so off restores their former output.
|
|
619
|
+
const showGuard = includeGuard && !!effective.obj?.guard;
|
|
620
|
+
for (const w of effective.waiting.filter((w) => w.setting !== "guard" || showGuard))
|
|
606
621
|
lines.push(`config.json loosens ${w.setting}${w.site ? ` for ${w.site}` : ""}, but that was not confirmed, so it is not in force. To confirm: ${w.command}`);
|
|
607
622
|
for (const e of envNotices(rec, raw))
|
|
608
623
|
lines.push(e);
|
|
609
624
|
const since = now - NOTICE_HOURS * 3_600_000;
|
|
610
|
-
const recent = rec.recent.filter((c) => c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
|
|
625
|
+
const recent = rec.recent.filter((c) => (c.setting !== "guard" || showGuard) && c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
|
|
611
626
|
for (const c of recent)
|
|
612
627
|
lines.push(`${c.at}: ${describe(c)}. See tiresias history.`);
|
|
613
628
|
if (!lines.length)
|
package/dist/cooldown.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// N-06: Retry-After is enforced, not just reported. When a site says 429
|
|
3
|
-
// 503, we remember "do not call this origin until T", across processes, and
|
|
2
|
+
// N-06: Retry-After is enforced, not just reported. When a site says 429,
|
|
3
|
+
// 503 or asks for a wait on a redirect, we remember "do not call this origin until T", across processes, and
|
|
4
4
|
// every later request (page, probe, robots refresh) checks it first.
|
|
5
5
|
//
|
|
6
6
|
// Storage is an append-only log with no lock. A lock cannot be made safe
|
|
@@ -13,7 +13,8 @@ import crypto from "node:crypto";
|
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { canonOrigin } from "./sitekey.js";
|
|
16
|
-
import {
|
|
16
|
+
import { Refused } from "./errors.js";
|
|
17
|
+
import { appendLine, cachePath, readJson, scanLogSync, rootDir } from "./store.js";
|
|
17
18
|
// A wait over 10 years is not a real Retry-After. It is treated as invalid and
|
|
18
19
|
// replaced by a fixed 24 hour cooldown. Anything below that limit is kept in
|
|
19
20
|
// full, never shortened.
|
|
@@ -65,8 +66,7 @@ const LEGACY_OLD = "cooldowns.old.jsonl"; // written by v0.1.4, read only, never
|
|
|
65
66
|
const MAX_ORIGIN = 400; // keeps every line well under 512 bytes (one atomic append)
|
|
66
67
|
const mem = new Map(); // cooldowns set by this process (at most MEM_MAX)
|
|
67
68
|
// v0.9 hardening (Codex round 6c, HIGH): `mem` no longer keeps every wait for the server's whole life: past MEM_MAX
|
|
68
|
-
// entries, the expired ones are dropped. A wait that has not ended is never dropped
|
|
69
|
-
// is read only from its last 8 MiB, so it cannot stand in for one).
|
|
69
|
+
// entries, the expired ones are dropped. A wait that has not ended is never dropped.
|
|
70
70
|
// v0.9 client trust (Travis approved, 2026-10-31; closes the class stopped in client-trust round 1): when `mem` is
|
|
71
71
|
// full of waits that have not ended, a new one goes into a FIXED-SIZE store instead of growing memory: SLOTS
|
|
72
72
|
// deadlines (8 MB), each origin written to K slots chosen by a keyed hash (a random key per process, so a site cannot
|
|
@@ -126,9 +126,6 @@ function remember(origin, until) {
|
|
|
126
126
|
}
|
|
127
127
|
export const cooldownStoreBytes = () => sketch?.byteLength ?? 0; // for tests
|
|
128
128
|
export const cooldownMemSize = () => mem.size; // for tests
|
|
129
|
-
// v0.5.1: the log is read again only when one of its files changed (size, mtime or inode), not on every
|
|
130
|
-
// check (a request checks about eleven times). The stamp names the store root, so a new TIRESIAS_HOME
|
|
131
|
-
// is read afresh.
|
|
132
129
|
let memo = null;
|
|
133
130
|
let reads = 0;
|
|
134
131
|
export const loadCount = () => reads; // for tests
|
|
@@ -137,9 +134,11 @@ function stamp() {
|
|
|
137
134
|
for (const f of [path.join(rootDir(), CUR), path.join(rootDir(), LEGACY_OLD), safePath(() => cachePath(LEGACY, "shared"))]) {
|
|
138
135
|
try {
|
|
139
136
|
const st = f ? fs.lstatSync(f) : null;
|
|
140
|
-
parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ino}` : "-");
|
|
137
|
+
parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ctimeMs}:${st.dev}:${st.ino}:${st.mode}` : "-");
|
|
141
138
|
}
|
|
142
|
-
catch {
|
|
139
|
+
catch (e) {
|
|
140
|
+
if (e.code !== "ENOENT")
|
|
141
|
+
throw e;
|
|
143
142
|
parts.push("-");
|
|
144
143
|
}
|
|
145
144
|
}
|
|
@@ -154,26 +153,54 @@ const safePath = (f) => {
|
|
|
154
153
|
}
|
|
155
154
|
};
|
|
156
155
|
function load() {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
156
|
+
try {
|
|
157
|
+
const st = stamp();
|
|
158
|
+
if (memo && memo.stamp === st)
|
|
159
|
+
return memo.map;
|
|
160
|
+
const root = rootDir();
|
|
161
|
+
const map = loadNow(memo?.root === root ? memo.map : undefined);
|
|
162
|
+
memo = { root, stamp: st, map };
|
|
163
|
+
return map;
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
memo = null;
|
|
167
|
+
throw new Refused("The site's Retry-After waits could not be read. No request was sent.");
|
|
168
|
+
}
|
|
163
169
|
}
|
|
164
|
-
//
|
|
165
|
-
|
|
166
|
-
function loadNow() {
|
|
167
|
-
reads++;
|
|
168
|
-
const now = Date.now();
|
|
170
|
+
// Replay every live deadline. Overflow uses the same bounded, conservative slots as the in-process list.
|
|
171
|
+
function newIndex() {
|
|
169
172
|
const best = new Map();
|
|
173
|
+
let overflow = null;
|
|
170
174
|
const take = (origin, until) => {
|
|
171
|
-
if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= now)
|
|
175
|
+
if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= Date.now())
|
|
172
176
|
return;
|
|
173
177
|
const key = canonOrigin(origin); // v0.6: one name per site (sitekey.ts)
|
|
174
|
-
if (
|
|
175
|
-
best.
|
|
178
|
+
if (best.has(key) || (best.size < MEM_MAX && key.length <= MAX_ORIGIN)) {
|
|
179
|
+
if (until > (best.get(key) ?? 0))
|
|
180
|
+
best.set(key, until);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
overflow ??= new Float64Array(SLOTS);
|
|
184
|
+
for (const i of slotsOf(key))
|
|
185
|
+
if (until > overflow[i])
|
|
186
|
+
overflow[i] = until;
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
return {
|
|
190
|
+
take,
|
|
191
|
+
logs: new Map(),
|
|
192
|
+
exact: (origin) => best.get(origin) ?? 0,
|
|
193
|
+
get: (origin) => {
|
|
194
|
+
let least = Infinity;
|
|
195
|
+
if (overflow)
|
|
196
|
+
for (const i of slotsOf(origin))
|
|
197
|
+
least = Math.min(least, overflow[i]);
|
|
198
|
+
return Math.max(best.get(origin) ?? 0, least === Infinity ? 0 : least);
|
|
199
|
+
},
|
|
176
200
|
};
|
|
201
|
+
}
|
|
202
|
+
function loadNow(index = newIndex()) {
|
|
203
|
+
reads++;
|
|
177
204
|
let legacy = null;
|
|
178
205
|
try {
|
|
179
206
|
legacy = readJson(cachePath(LEGACY, "shared")); // W-18: shared, at the root
|
|
@@ -182,13 +209,11 @@ function loadNow() {
|
|
|
182
209
|
// the root's cache/ is a link or someone else's: the legacy store is not read (Codex round 46)
|
|
183
210
|
}
|
|
184
211
|
if (legacy && typeof legacy === "object")
|
|
185
|
-
for (const [
|
|
186
|
-
take(
|
|
187
|
-
for (const
|
|
188
|
-
take(e?.origin, e?.until)
|
|
189
|
-
|
|
190
|
-
take(e?.origin, e?.until);
|
|
191
|
-
return best;
|
|
212
|
+
for (const [origin, until] of Object.entries(legacy))
|
|
213
|
+
index.take(origin, until);
|
|
214
|
+
for (const name of [LEGACY_OLD, CUR])
|
|
215
|
+
index.logs.set(name, scanLogSync(name, (e) => index.take(e?.origin, e?.until), index.logs.get(name)));
|
|
216
|
+
return index;
|
|
192
217
|
}
|
|
193
218
|
// Fetchers register here so queued requests can be cancelled the moment a
|
|
194
219
|
// cooldown starts for their site.
|
|
@@ -199,7 +224,7 @@ export function setCooldown(origin, status, header) {
|
|
|
199
224
|
const until = Date.now() + seconds * 1000;
|
|
200
225
|
// Exact sources only: the fixed-size store may answer later than the truth (a collision), and taking that as
|
|
201
226
|
// "already written" would keep the line from other processes.
|
|
202
|
-
const known = Math.max(mem.get(origin) ?? 0, load().
|
|
227
|
+
const known = Math.max(mem.get(origin) ?? 0, load().exact(origin));
|
|
203
228
|
remember(origin, until); // this process honors it at once
|
|
204
229
|
// Only a later deadline is worth a line. Never shortens anything.
|
|
205
230
|
if (until > known && origin.length <= MAX_ORIGIN)
|
package/dist/dailylog.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// W-13: the daily log. A readable page for one day, built from history.jsonl
|
|
3
3
|
// (and receipts.jsonl) on request. It is never stored as a second copy; the
|
|
4
4
|
// only file it can create is an export the owner asks for with --to.
|
|
5
|
+
import { textView } from "./textview.js";
|
|
5
6
|
import fs from "node:fs";
|
|
6
7
|
import path from "node:path";
|
|
7
8
|
import { pathToFileURL } from "node:url";
|
|
@@ -53,6 +54,9 @@ function outcome(e) {
|
|
|
53
54
|
}
|
|
54
55
|
// One settings change in plain words, from the record's fixed codes (history.ts keeps only known codes).
|
|
55
56
|
function settingWords(e) {
|
|
57
|
+
const by0 = e.by === "terminal" ? "from a terminal (likely a person)" : e.by === "not_terminal" ? "NOT from a terminal (an agent, a script or cron)" : "by an unknown caller";
|
|
58
|
+
if (e.tool === "keys")
|
|
59
|
+
return `${e.event === "slip_key_rotated" ? `slip key rotated${e.revoked !== undefined ? `, ${e.revoked} slip${e.revoked === 1 ? "" : "s"} revoked` : ""}` : e.event === "signing_key_rotation_finished" ? "signing key rotation finished (previous key dropped)" : "signing key rotated"}, ${by0}`;
|
|
56
60
|
const what = e.change === "never_ask" ? "confirmations for risky settings turned OFF (never ask again)"
|
|
57
61
|
: e.change === "ask_again" ? "confirmations for risky settings turned back on"
|
|
58
62
|
: `${e.setting ?? "a setting"}${e.site_key ? ` for ${code(e.site_key)}` : ""} ${e.change === "looser" ? "turned on or loosened" : "turned off or made stricter"}`;
|
|
@@ -64,7 +68,8 @@ const hhmm = (iso) => {
|
|
|
64
68
|
const d = new Date(iso);
|
|
65
69
|
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`;
|
|
66
70
|
};
|
|
67
|
-
export async function buildLog(dayText) {
|
|
71
|
+
export async function buildLog(dayText, scan = {}) {
|
|
72
|
+
scan = { ...scan, deadline: scan.deadline ?? Date.now() + 60_000 };
|
|
68
73
|
const { day, start, end } = parseDay(dayText);
|
|
69
74
|
const entries = [];
|
|
70
75
|
const changes = []; // W-32 (v0.9): risky settings changed this day get their own section, not counted as lookups
|
|
@@ -96,7 +101,8 @@ export async function buildLog(dayText) {
|
|
|
96
101
|
}
|
|
97
102
|
return false;
|
|
98
103
|
}
|
|
99
|
-
if (e.tool === "settings") {
|
|
104
|
+
if (e.tool === "settings" || e.tool === "keys") {
|
|
105
|
+
// v1.1: a key rotation is listed with the settings changes (it is just as risky)
|
|
100
106
|
changeCount++;
|
|
101
107
|
if (changes.length < MAX_LOG_LOOKUPS)
|
|
102
108
|
changes.push(e);
|
|
@@ -112,7 +118,7 @@ export async function buildLog(dayText) {
|
|
|
112
118
|
otherAgents++;
|
|
113
119
|
results.set(outcome(e), (results.get(outcome(e)) ?? 0) + 1); // a fixed set of outcomes
|
|
114
120
|
return false;
|
|
115
|
-
});
|
|
121
|
+
}, 1, PAGE_BUDGET, scan);
|
|
116
122
|
entries.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
|
|
117
123
|
changes.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
|
|
118
124
|
const perUrl = new Map();
|
|
@@ -136,7 +142,7 @@ export async function buildLog(dayText) {
|
|
|
136
142
|
}
|
|
137
143
|
}
|
|
138
144
|
return false;
|
|
139
|
-
});
|
|
145
|
+
}, 1, PAGE_BUDGET, scan);
|
|
140
146
|
const link = (name) => `[${name}](${pathToFileURL(path.join(homeDir(), name)).href})`;
|
|
141
147
|
const days = [...utcDays].sort().join(", ");
|
|
142
148
|
const lines = [];
|
|
@@ -234,7 +240,7 @@ export async function exportLog(folder, dayText) {
|
|
|
234
240
|
throw new Error("Private mode: nothing is written, so the log is not exported. Run it without --to to print it.");
|
|
235
241
|
// 1. Build the whole text first. Nothing on disk is touched until it exists.
|
|
236
242
|
const log = await buildLog(dayText);
|
|
237
|
-
const body = fullText(log);
|
|
243
|
+
const body = textView(fullText(log));
|
|
238
244
|
// 2. Resolve the folder ONCE and use only that real path from here on.
|
|
239
245
|
let real;
|
|
240
246
|
try {
|
package/dist/dom.js
CHANGED
|
@@ -4,17 +4,19 @@
|
|
|
4
4
|
import { DomHandler, Parser } from "htmlparser2";
|
|
5
5
|
export const MAX_DOM_DEPTH = 256;
|
|
6
6
|
export function parseDocument(body) {
|
|
7
|
-
const handler = new DomHandler();
|
|
7
|
+
const handler = new DomHandler(undefined, { withStartIndices: true });
|
|
8
|
+
const source = new WeakMap();
|
|
8
9
|
const opens = [];
|
|
9
10
|
// The bounded parser can forget an ancestor before its source end tag. Keep source scopes with
|
|
10
11
|
// push/pop (not htmlparser2's shifting stacks), so flattening cannot turn hidden descendants into
|
|
11
12
|
// shown siblings. Each attribute set refers to its original parent; the reader resolves these in
|
|
12
13
|
// source order, once the page's style sheet is known, without copying attributes or walking chains.
|
|
13
|
-
const inherited = [];
|
|
14
|
+
const inherited = []; // third entry: the source parent is inside code/pre
|
|
14
15
|
const scopes = [];
|
|
15
16
|
const byName = new Map();
|
|
16
17
|
const popScope = () => {
|
|
17
18
|
const s = scopes.pop();
|
|
19
|
+
source.get(s.attrs).end = parser.startIndex;
|
|
18
20
|
if (s.previous === undefined)
|
|
19
21
|
byName.delete(s.name);
|
|
20
22
|
else
|
|
@@ -28,12 +30,14 @@ export function parseDocument(body) {
|
|
|
28
30
|
const close = handler.onclosetag.bind(handler);
|
|
29
31
|
const text = handler.ontext.bind(handler);
|
|
30
32
|
handler.onopentag = (name, attrs) => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
const parent = scopes[scopes.length - 1];
|
|
34
|
+
source.set(attrs, { parent: parent?.attrs });
|
|
35
|
+
if (parent)
|
|
36
|
+
inherited.push([attrs, parent.attrs, parent.code]);
|
|
33
37
|
const key = name.toLowerCase();
|
|
34
38
|
const previous = byName.get(key);
|
|
35
39
|
byName.set(key, scopes.length);
|
|
36
|
-
scopes.push({ name: key, attrs, previous });
|
|
40
|
+
scopes.push({ name: key, attrs, code: key === "code" || key === "pre" || parent?.code === true, previous });
|
|
37
41
|
opens.push({ ...bounds, attrs });
|
|
38
42
|
open(name, attrs);
|
|
39
43
|
};
|
|
@@ -51,7 +55,7 @@ export function parseDocument(body) {
|
|
|
51
55
|
return text(data);
|
|
52
56
|
// Text after a flattened child's close (including raw text) has no element to inherit on.
|
|
53
57
|
const attrs = {};
|
|
54
|
-
inherited.push([attrs, scope.attrs]);
|
|
58
|
+
inherited.push([attrs, scope.attrs, scope.code]);
|
|
55
59
|
open("thidden", attrs);
|
|
56
60
|
text(data);
|
|
57
61
|
close();
|
|
@@ -87,6 +91,7 @@ export function parseDocument(body) {
|
|
|
87
91
|
popScope();
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
|
-
new BoundedParser(handler)
|
|
91
|
-
|
|
94
|
+
const parser = new BoundedParser(handler);
|
|
95
|
+
parser.end(body);
|
|
96
|
+
return { doc: handler.root, flattened, inherited, source };
|
|
92
97
|
}
|
package/dist/doors.js
CHANGED
|
@@ -40,7 +40,7 @@ export const noStore = (h) => /(^|[\s,])no-store\b/i.test(h.get("cache-control")
|
|
|
40
40
|
// is not "no door": it throws, so the sequential checks stop and nothing is cached (the next read asks again).
|
|
41
41
|
async function probe(url, maxBytes, ctl, strict = false) {
|
|
42
42
|
try {
|
|
43
|
-
const res = await politeFetch(url, { maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
|
|
43
|
+
const res = await politeFetch(url, { robotsPass: false, maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
|
|
44
44
|
if (res.status === 402)
|
|
45
45
|
throw new PaymentRequired(res.headers, res.body, res.url);
|
|
46
46
|
if (strict && (res.status >= 500 || res.status === 429))
|
package/dist/engine.js
CHANGED
|
@@ -17,7 +17,12 @@ export * from "./resume.js";
|
|
|
17
17
|
export * from "./formflow.js";
|
|
18
18
|
export const read = async (url, opts = {}, ctx = DEFAULT) => {
|
|
19
19
|
const saveFallback = opts.save ? saveRefusal() : undefined;
|
|
20
|
-
const out = await guarded("read", url, ctx, opts.question, (m) =>
|
|
20
|
+
const out = await guarded("read", url, ctx, opts.question, async (m) => {
|
|
21
|
+
const e = await run("read", url, opts, m, ctx.agent);
|
|
22
|
+
if (opts.budget !== undefined)
|
|
23
|
+
await opts.beforeLocalRecord?.(e, m);
|
|
24
|
+
return e;
|
|
25
|
+
}, {}, { create: { question: opts.question, section: opts.section, render: opts.render, translate: opts.translate } });
|
|
21
26
|
// W-28 (v0.8): a read that asked to save always says what happened to the save (the pipeline sets it when it got
|
|
22
27
|
// as far as a page or a file).
|
|
23
28
|
if (opts.save && !out.saved)
|
|
@@ -56,13 +61,13 @@ export function slipList(page = 1) {
|
|
|
56
61
|
export const claimSlip = slips.claimSlip;
|
|
57
62
|
export const checkSlip = slips.checkSlip;
|
|
58
63
|
export const queryHistory = history.query;
|
|
59
|
-
export const dailyLog = async (day, page = 1) => {
|
|
60
|
-
const log = await dailylog.buildLog(day);
|
|
64
|
+
export const dailyLog = async (day, page = 1, scan = {}) => {
|
|
65
|
+
const log = await dailylog.buildLog(day, scan);
|
|
61
66
|
return { day: log.day, entries: log.entries, ...dailylog.pageOf(log, page) };
|
|
62
67
|
};
|
|
63
68
|
export const exportDailyLog = dailylog.exportLog;
|
|
64
69
|
// W-12: generated from the history when asked, never stored.
|
|
65
|
-
export const needsReport = (since, page = 1) => needs.buildNeeds(since, page);
|
|
70
|
+
export const needsReport = (since, page = 1, scan = {}) => needs.buildNeeds(since, page, Date.now(), scan);
|
|
66
71
|
// W-19: tiresias safety status | update.
|
|
67
72
|
export const safetyStatus = safebrowsing.status;
|
|
68
73
|
export const safetyUpdate = safebrowsing.update;
|
package/dist/envelope.js
CHANGED
|
@@ -80,6 +80,11 @@ export const receiptsEnvelope = (url, r) => {
|
|
|
80
80
|
};
|
|
81
81
|
export const historyEnvelope = (r) => listEnvelope("", { history: r.items }, r);
|
|
82
82
|
export const STOPPED = "Stopped. Tiresias does not retry or work around walls.";
|
|
83
|
+
// W-39 (v1.2): the two values of a local passage's written_by, the wording measured in the lab.
|
|
84
|
+
export const BY_SITE = "the site";
|
|
85
|
+
export const BY_VISITOR = "a visitor to the site, not the site itself";
|
|
86
|
+
// What one label adds to a passage at most (the longer value, its key and a comma), for the reply budget.
|
|
87
|
+
export const LABEL_BYTES = jsonBytes({ written_by: BY_VISITOR }) - 2 + 1;
|
|
83
88
|
// W-02 fixed texts.
|
|
84
89
|
export const NO_BASELINE = "There was nothing to compare against: no earlier copy of this page is kept (first visit, a cleared or expired cache, or private mode).";
|
|
85
90
|
export const NO_STORE_DIFF = "There was nothing to compare against: the site asks that this page not be stored (no-store), so no copy is kept.";
|
|
@@ -216,7 +221,10 @@ export async function guarded(...args) {
|
|
|
216
221
|
catch (e) {
|
|
217
222
|
configError = e;
|
|
218
223
|
}
|
|
219
|
-
|
|
224
|
+
// One holder per tool/CLI call, before withSignal/runPrivate make shallow copies. Background door
|
|
225
|
+
// jobs create their own context without a holder, and an in-flight robots waiter gets no pass.
|
|
226
|
+
return callContext.run({ ...callContext.getStore(), config, configError, getActions: undefined,
|
|
227
|
+
robotsPasses: { pending: new Map(), attempted: new Set() } }, () => guardedCall(...args));
|
|
220
228
|
}
|
|
221
229
|
async function guardedCall(tool, url, ctx, purpose, fn, meta = {}, hand, safetyCheck = true, // W-19: false only for a page a person provided (nothing is requested)
|
|
222
230
|
record = true) {
|
|
@@ -289,6 +297,7 @@ record = true) {
|
|
|
289
297
|
lang: out.language?.code, // W-26 (v0.7)
|
|
290
298
|
rendered: meta.rendered, // W-10 (v0.7)
|
|
291
299
|
saved: meta.saved, // W-28 (v0.8)
|
|
300
|
+
written_by: meta.written_by, // W-39 (v1.2)
|
|
292
301
|
});
|
|
293
302
|
}
|
|
294
303
|
catch (e) {
|
package/dist/fetcher.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// The polite fetcher: safety (safety.ts) plus the site's rules (rules.ts).
|
|
3
3
|
// Every request except robots.txt itself goes through politeFetch.
|
|
4
|
+
import { callContext } from "./context.js";
|
|
4
5
|
import { cooldownLeft, setCooldown } from "./cooldown.js";
|
|
5
6
|
import { Refused, Walled } from "./errors.js";
|
|
6
7
|
import { spacingMs } from "./hosts.js";
|
|
7
8
|
import { Turns, cooldownRefusal } from "./queue.js";
|
|
8
9
|
import { checkAllowed } from "./rules.js";
|
|
9
10
|
import { assertSiteAllowed } from "./sites.js";
|
|
11
|
+
import { canonOrigin } from "./sitekey.js";
|
|
10
12
|
import { safeFetch } from "./safety.js";
|
|
11
13
|
import { detectWall } from "./walls.js";
|
|
12
14
|
export { Refused, Walled } from "./errors.js";
|
|
@@ -37,12 +39,20 @@ export async function politeFetch(url, opts = {}) {
|
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
async function politeFetchHeld(url, opts, turn) {
|
|
42
|
+
const holder = callContext.getStore()?.robotsPasses;
|
|
43
|
+
const canClaim = opts.robotsPass !== false && !opts.submit && opts.method !== "POST";
|
|
44
|
+
let first = true;
|
|
40
45
|
const res = await safeFetch(url, {
|
|
41
46
|
...opts,
|
|
42
47
|
onHeaders: (hop) => {
|
|
43
48
|
// Record the wait the moment 429/503 headers arrive, even if the body never finishes.
|
|
44
49
|
if (hop.status === 429 || hop.status === 503)
|
|
45
50
|
setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
51
|
+
if (hop.redirect && hop.headers.has("retry-after")) {
|
|
52
|
+
const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
53
|
+
if (seconds > 0)
|
|
54
|
+
throw new Refused(COOLDOWN_TEXT, undefined, seconds, hop.url.href);
|
|
55
|
+
}
|
|
46
56
|
// A challenge (cf-mitigated) on ANY hop is a wall at that hop: a redirect is not
|
|
47
57
|
// followed, and a 304 is not accepted as a revalidation. Plain 401/402/403 final
|
|
48
58
|
// responses are handled with their body by the engine.
|
|
@@ -53,7 +63,24 @@ async function politeFetchHeld(url, opts, turn) {
|
|
|
53
63
|
},
|
|
54
64
|
beforeRequest: async (u, signal) => {
|
|
55
65
|
turn.leaveOthers(u);
|
|
56
|
-
const
|
|
66
|
+
const origin = canonOrigin(u.origin);
|
|
67
|
+
const eligible = first && canClaim && holder && !holder.attempted.has(origin);
|
|
68
|
+
first = false;
|
|
69
|
+
// Claim before awaiting policy, even with no pending pass: only the first eligible fetch to
|
|
70
|
+
// this origin may use one. The policy check itself may fetch robots on direct fetcher paths.
|
|
71
|
+
if (eligible)
|
|
72
|
+
holder.attempted.add(origin);
|
|
73
|
+
let p;
|
|
74
|
+
let pass;
|
|
75
|
+
try {
|
|
76
|
+
p = await policyCheck(u);
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
if (eligible) {
|
|
80
|
+
pass = holder.pending.get(origin);
|
|
81
|
+
holder.pending.delete(origin); // consumed on admission or refusal, used or not
|
|
82
|
+
}
|
|
83
|
+
}
|
|
57
84
|
if (!p.ok && "wall" in p)
|
|
58
85
|
throw new Walled(p.wall, p.wallUrl);
|
|
59
86
|
if (!p.ok)
|
|
@@ -61,7 +88,7 @@ async function politeFetchHeld(url, opts, turn) {
|
|
|
61
88
|
const d = p;
|
|
62
89
|
// v0.5.1: every request is queued and spaced, by at least the default spacing (1 s).
|
|
63
90
|
// W-25 (v0.6): a site's own spacing raises it further, never lowers it (applied in the queue, queue.ts).
|
|
64
|
-
await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal);
|
|
91
|
+
await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal, { pass: p.crawlDelay === 0 ? pass : undefined });
|
|
65
92
|
},
|
|
66
93
|
});
|
|
67
94
|
return res;
|
package/dist/filereply.js
CHANGED
|
@@ -15,7 +15,8 @@ import { NONBLOCK, isPrivate } from "./store.js";
|
|
|
15
15
|
import { MAX_BLOCK } from "./reader.js";
|
|
16
16
|
import { MAX_SAMPLE, detect, stampLanguage } from "./lang.js";
|
|
17
17
|
import { translateReply } from "./translate.js";
|
|
18
|
-
import { SecretWithheld, valueHoldsSecret } from "./echo.js";
|
|
18
|
+
import { SecretWithheld, noSecrets, valueHoldsSecret } from "./echo.js";
|
|
19
|
+
import { runUntrusted } from "./sandbox.js";
|
|
19
20
|
import { BUDGET, env } from "./envelope.js";
|
|
20
21
|
export class ProvidedError extends TiresiasError {
|
|
21
22
|
kind = "provided";
|
|
@@ -81,6 +82,11 @@ export async function fileWarning(url, finalUrl, f, meta, agent, source, opts =
|
|
|
81
82
|
}
|
|
82
83
|
// v0.6: inside a zip or an Office file, in the worker, without unpacking it (inside.ts).
|
|
83
84
|
const ins = await inspectFile(f);
|
|
85
|
+
if (ins?.office_text && !noSecrets(secrets)) {
|
|
86
|
+
const texts = [JSON.stringify(ins.office_text), ins.office_text.map((s) => s.text).join("\n")];
|
|
87
|
+
if (await runUntrusted({ op: "echo", texts, secrets: secrets }, Infinity))
|
|
88
|
+
throw new SecretWithheld();
|
|
89
|
+
}
|
|
84
90
|
if (macroByName(f) || ins?.macros)
|
|
85
91
|
fw.flags = [...new Set([...fw.flags, "macros"])];
|
|
86
92
|
if (ins) {
|