@raibitsi/tiresias 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -0
- package/README.md +72 -33
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +17 -13
- package/dist/cdp.js +22 -1
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +216 -68
- package/dist/config.js +75 -1
- package/dist/confirm.js +20 -5
- package/dist/cooldown.js +57 -32
- package/dist/dailylog.js +11 -5
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +9 -4
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +29 -2
- package/dist/filereply.js +7 -1
- package/dist/formflow.js +21 -3
- package/dist/forms.js +337 -42
- package/dist/getactions.js +119 -0
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +11 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +34 -27
- package/dist/keys.js +420 -18
- package/dist/local.js +537 -0
- package/dist/mcp.js +56 -25
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +117 -26
- package/dist/preformatted.js +12 -0
- package/dist/profiles.js +3 -3
- package/dist/queue.js +24 -20
- package/dist/reader.js +441 -87
- package/dist/receipts.js +7 -2
- package/dist/render.js +156 -27
- package/dist/renderactions.js +69 -0
- package/dist/resume.js +8 -6
- package/dist/rules.js +24 -5
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +15 -7
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +24 -9
- package/dist/savedlock.js +99 -5
- package/dist/settings.js +10 -3
- package/dist/shelf.js +1 -1
- package/dist/shield.js +15 -0
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +16 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +9 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +10 -2
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// W-38 (v1.2): the meaning check's one network client. It talks to the owner's OWN embedding endpoint (LM Studio,
|
|
3
|
+
// Ollama, or any OpenAI-compatible /v1/embeddings), never to a site, so it is deliberately not the page fetcher:
|
|
4
|
+
// no robots.txt, no signing, no spacing, no cache, no Tiresias user agent promise. What it keeps from the fetcher's
|
|
5
|
+
// hardening is what protects the owner: one request, no redirect ever followed, a hard timeout, a cap on the bytes
|
|
6
|
+
// read, and a reply that is checked in full before a single number from it is used.
|
|
7
|
+
//
|
|
8
|
+
// Where it may send: the setting's address is resolved here, at call time, and the connection goes to the address
|
|
9
|
+
// that was checked (no second lookup). An address on this machine or the owner's network (loopback, private, link
|
|
10
|
+
// local, CGNAT 100.64.0.0/10 that Tailscale uses) is fine. Anything else means page text would leave the owner's
|
|
11
|
+
// network, so it needs `allow_public`, which only `tiresias guard setup` writes and only with the typed
|
|
12
|
+
// confirmation; plain http to such an address is refused whatever the setting says.
|
|
13
|
+
//
|
|
14
|
+
// The key: TIRESIAS_GUARD_KEY, read from the environment at call time and sent as a Bearer token. It is never
|
|
15
|
+
// stored, logged or put in a reply or an error.
|
|
16
|
+
import dns from "node:dns";
|
|
17
|
+
import http from "node:http";
|
|
18
|
+
import https from "node:https";
|
|
19
|
+
import net from "node:net";
|
|
20
|
+
import { isOwnerAddress } from "./guardnetwork.js";
|
|
21
|
+
const WORDS = {
|
|
22
|
+
refused_public: "the endpoint's address is not on this machine or your network, and sending page text there was not confirmed",
|
|
23
|
+
refused_http: "the endpoint's address is not on this machine or your network, and plain http to it is refused",
|
|
24
|
+
dns: "the endpoint's name could not be looked up",
|
|
25
|
+
network: "the endpoint could not be reached",
|
|
26
|
+
timeout: "the endpoint did not answer in time",
|
|
27
|
+
cancelled: "the call was cancelled",
|
|
28
|
+
http_status: "the endpoint answered with an error status",
|
|
29
|
+
redirect: "the endpoint answered with a redirect, which is never followed",
|
|
30
|
+
too_large: "the endpoint's answer was larger than allowed",
|
|
31
|
+
bad_json: "the endpoint's answer was not JSON",
|
|
32
|
+
bad_shape: "the endpoint's answer did not have a data list of embeddings",
|
|
33
|
+
wrong_count: "the endpoint returned a different number of embeddings than texts sent",
|
|
34
|
+
wrong_dimension: "the endpoint returned embeddings of an unexpected length",
|
|
35
|
+
not_finite: "the endpoint returned a number that is not finite",
|
|
36
|
+
zero_vector: "the endpoint returned an embedding of all zeros",
|
|
37
|
+
};
|
|
38
|
+
export class EmbedError extends Error {
|
|
39
|
+
code;
|
|
40
|
+
status;
|
|
41
|
+
constructor(code, status) {
|
|
42
|
+
super(`The embedding call failed: ${WORDS[code]}${status !== undefined ? ` (HTTP ${status})` : ""}.`);
|
|
43
|
+
this.code = code;
|
|
44
|
+
this.status = status;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Bounds. A batch is at most MAX_BATCH texts; a reply at most MAX_REPLY bytes (64 texts of 4,096 numbers written
|
|
48
|
+
// out in full is about 6 MB); a dimension from 8 to 8,192.
|
|
49
|
+
export const MAX_BATCH = 64;
|
|
50
|
+
export const MAX_REPLY = 16 * 1024 * 1024;
|
|
51
|
+
export const MIN_DIM = 8;
|
|
52
|
+
export const MAX_DIM = 8192;
|
|
53
|
+
// v1.2: the whole input, including the configured prefix, must fit. Never silently embed a truncated text.
|
|
54
|
+
export const MAX_INPUT_CHARS = 8_000;
|
|
55
|
+
const isLoopbackName = (h) => h === "localhost" || h.endsWith(".localhost");
|
|
56
|
+
const bare = (h) => (h.startsWith("[") && h.endsWith("]") ? h.slice(1, -1) : h);
|
|
57
|
+
// The addresses a host name stands for, every one checked. A literal address is used as it is.
|
|
58
|
+
async function resolve(host, signal) {
|
|
59
|
+
if (signal.aborted)
|
|
60
|
+
throw new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled");
|
|
61
|
+
const h = bare(host);
|
|
62
|
+
if (net.isIP(h))
|
|
63
|
+
return [{ address: h, family: net.isIP(h) }];
|
|
64
|
+
const lookup = dns.promises.lookup(isLoopbackName(h) ? "localhost" : h, { all: true, verbatim: true });
|
|
65
|
+
const abort = new Promise((_, reject) => signal.addEventListener("abort", () => reject(new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled")), { once: true }));
|
|
66
|
+
let found;
|
|
67
|
+
try {
|
|
68
|
+
found = await Promise.race([lookup, abort]);
|
|
69
|
+
}
|
|
70
|
+
catch (e) {
|
|
71
|
+
if (e instanceof EmbedError)
|
|
72
|
+
throw e;
|
|
73
|
+
throw new EmbedError("dns");
|
|
74
|
+
}
|
|
75
|
+
if (!found.length)
|
|
76
|
+
throw new EmbedError("dns");
|
|
77
|
+
return found;
|
|
78
|
+
}
|
|
79
|
+
// "local" when every address is loopback, private, link local or CGNAT: page text stays on the owner's network.
|
|
80
|
+
export async function classify(url, signal = new AbortController().signal) {
|
|
81
|
+
const addrs = await resolve(new URL(url).hostname, signal);
|
|
82
|
+
return addrs.every((a) => isOwnerAddress(a.address)) ? "local" : "public";
|
|
83
|
+
}
|
|
84
|
+
// The whole check for one address: may page text be sent there under this setting?
|
|
85
|
+
export async function destination(cfg, signal) {
|
|
86
|
+
const u = new URL(cfg.url);
|
|
87
|
+
const addrs = await resolve(u.hostname, signal);
|
|
88
|
+
const local = addrs.every((a) => isOwnerAddress(a.address));
|
|
89
|
+
if (!local) {
|
|
90
|
+
if (u.protocol !== "https:")
|
|
91
|
+
throw new EmbedError("refused_http");
|
|
92
|
+
if (cfg.allow_public !== true)
|
|
93
|
+
throw new EmbedError("refused_public");
|
|
94
|
+
}
|
|
95
|
+
return addrs[0];
|
|
96
|
+
}
|
|
97
|
+
// One POST to <url>/embeddings with { model, input }, and the vectors from data[i].embedding, each made unit length.
|
|
98
|
+
// Throws an EmbedError with a fixed code on anything unexpected.
|
|
99
|
+
export async function embed(cfg, inputs, opts) {
|
|
100
|
+
if (inputs.length === 0)
|
|
101
|
+
return [];
|
|
102
|
+
if (inputs.length > MAX_BATCH)
|
|
103
|
+
throw new RangeError("too many texts in one batch");
|
|
104
|
+
if (inputs.some(t => t.length > MAX_INPUT_CHARS))
|
|
105
|
+
throw new RangeError("text too long for one embedding input");
|
|
106
|
+
const ctl = new AbortController();
|
|
107
|
+
const timer = setTimeout(() => ctl.abort("timeout"), opts.timeoutMs);
|
|
108
|
+
const outer = opts.signal;
|
|
109
|
+
const onOuter = () => ctl.abort("cancelled");
|
|
110
|
+
if (outer?.aborted)
|
|
111
|
+
ctl.abort("cancelled");
|
|
112
|
+
else
|
|
113
|
+
outer?.addEventListener("abort", onOuter, { once: true });
|
|
114
|
+
try {
|
|
115
|
+
const to = await destination(cfg, ctl.signal);
|
|
116
|
+
const body = Buffer.from(JSON.stringify({ model: cfg.model, input: inputs }));
|
|
117
|
+
const raw = await post(new URL(cfg.url.replace(/\/+$/, "") + "/embeddings"), body, to, ctl.signal);
|
|
118
|
+
return parseVectors(raw, inputs.length, opts.dim);
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
outer?.removeEventListener("abort", onOuter);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function post(u, body, to, signal) {
|
|
126
|
+
return new Promise((resolveP, reject) => {
|
|
127
|
+
const fail = (code, status) => reject(new EmbedError(code, status));
|
|
128
|
+
if (signal.aborted)
|
|
129
|
+
return fail(signal.reason === "timeout" ? "timeout" : "cancelled");
|
|
130
|
+
const headers = { "content-type": "application/json", accept: "application/json", "content-length": String(body.length) };
|
|
131
|
+
const key = process.env.TIRESIAS_GUARD_KEY;
|
|
132
|
+
if (key)
|
|
133
|
+
headers.authorization = `Bearer ${key}`;
|
|
134
|
+
const mod = u.protocol === "https:" ? https : http;
|
|
135
|
+
// The connection goes to the address checked above: a lookup that answers with it, never a second DNS query.
|
|
136
|
+
const lookup = (_h, o, cb) => o?.all ? cb(null, [{ address: to.address, family: to.family }]) : cb(null, to.address, to.family);
|
|
137
|
+
const req = mod.request(u, { method: "POST", headers, lookup: lookup, agent: false, signal }, (res) => {
|
|
138
|
+
const status = res.statusCode ?? 0;
|
|
139
|
+
if (status >= 300 && status < 400) {
|
|
140
|
+
res.destroy();
|
|
141
|
+
return fail("redirect", status);
|
|
142
|
+
}
|
|
143
|
+
if (status !== 200) {
|
|
144
|
+
res.destroy();
|
|
145
|
+
return fail("http_status", status);
|
|
146
|
+
}
|
|
147
|
+
const declared = Number(res.headers["content-length"]);
|
|
148
|
+
if (Number.isFinite(declared) && declared > MAX_REPLY) {
|
|
149
|
+
res.destroy();
|
|
150
|
+
return fail("too_large");
|
|
151
|
+
}
|
|
152
|
+
const parts = [];
|
|
153
|
+
let total = 0;
|
|
154
|
+
res.on("data", (c) => {
|
|
155
|
+
total += c.length;
|
|
156
|
+
if (total > MAX_REPLY) {
|
|
157
|
+
res.destroy();
|
|
158
|
+
return fail("too_large");
|
|
159
|
+
}
|
|
160
|
+
parts.push(c);
|
|
161
|
+
});
|
|
162
|
+
res.on("end", () => resolveP(Buffer.concat(parts, total)));
|
|
163
|
+
res.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
|
|
164
|
+
});
|
|
165
|
+
req.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
|
|
166
|
+
req.end(body);
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
// The reply, checked whole: a data list of exactly `count` embeddings (if any item has an index, all must have a
|
|
170
|
+
// distinct index in range), each a list of finite numbers of one length (the stored dimension when there is one).
|
|
171
|
+
export function parseVectors(raw, count, dim) {
|
|
172
|
+
let j;
|
|
173
|
+
try {
|
|
174
|
+
j = JSON.parse(raw.toString("utf8"));
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
throw new EmbedError("bad_json");
|
|
178
|
+
}
|
|
179
|
+
const data = j && typeof j === "object" && !Array.isArray(j) ? j.data : undefined;
|
|
180
|
+
if (!Array.isArray(data))
|
|
181
|
+
throw new EmbedError("bad_shape");
|
|
182
|
+
if (data.length !== count)
|
|
183
|
+
throw new EmbedError("wrong_count");
|
|
184
|
+
let order = data;
|
|
185
|
+
const idx = data.map((d) => (d && typeof d === "object" ? d.index : undefined));
|
|
186
|
+
if (idx.some((i) => i !== undefined)) {
|
|
187
|
+
if (!idx.every((i) => Number.isInteger(i) && i >= 0 && i < count) || new Set(idx).size !== count)
|
|
188
|
+
throw new EmbedError("bad_shape");
|
|
189
|
+
order = new Array(count);
|
|
190
|
+
data.forEach((d, k) => (order[idx[k]] = d));
|
|
191
|
+
}
|
|
192
|
+
const out = [];
|
|
193
|
+
let len = dim;
|
|
194
|
+
for (const d of order) {
|
|
195
|
+
const e = d && typeof d === "object" ? d.embedding : undefined;
|
|
196
|
+
if (!Array.isArray(e))
|
|
197
|
+
throw new EmbedError("bad_shape");
|
|
198
|
+
if (len === undefined) {
|
|
199
|
+
if (e.length < MIN_DIM || e.length > MAX_DIM)
|
|
200
|
+
throw new EmbedError("wrong_dimension");
|
|
201
|
+
len = e.length;
|
|
202
|
+
}
|
|
203
|
+
if (e.length !== len)
|
|
204
|
+
throw new EmbedError("wrong_dimension");
|
|
205
|
+
const v = new Float32Array(len);
|
|
206
|
+
let n = 0;
|
|
207
|
+
let storedNorm = 0;
|
|
208
|
+
for (let i = 0; i < len; i++) {
|
|
209
|
+
const x = e[i];
|
|
210
|
+
if (typeof x !== "number" || !Number.isFinite(x))
|
|
211
|
+
throw new EmbedError("not_finite");
|
|
212
|
+
// Reject values outside the stored format before they can become infinities or an all-zero vector.
|
|
213
|
+
const stored = Math.fround(x);
|
|
214
|
+
if (!Number.isFinite(stored))
|
|
215
|
+
throw new EmbedError("not_finite");
|
|
216
|
+
storedNorm += stored * stored;
|
|
217
|
+
n += x * x;
|
|
218
|
+
}
|
|
219
|
+
n = Math.sqrt(n);
|
|
220
|
+
if (!(n > 0) || !Number.isFinite(n) || !(storedNorm > 0))
|
|
221
|
+
throw new EmbedError("zero_vector");
|
|
222
|
+
let finalNorm = 0;
|
|
223
|
+
for (let i = 0; i < len; i++) {
|
|
224
|
+
v[i] = e[i] / n;
|
|
225
|
+
if (!Number.isFinite(v[i]))
|
|
226
|
+
throw new EmbedError("not_finite");
|
|
227
|
+
finalNorm += v[i] * v[i];
|
|
228
|
+
}
|
|
229
|
+
if (!(finalNorm > 0) || !Number.isFinite(finalNorm))
|
|
230
|
+
throw new EmbedError("zero_vector");
|
|
231
|
+
out.push(v);
|
|
232
|
+
}
|
|
233
|
+
return out;
|
|
234
|
+
}
|
|
235
|
+
// Several batches in order, within one deadline.
|
|
236
|
+
export async function embedAll(cfg, inputs, opts) {
|
|
237
|
+
const out = [];
|
|
238
|
+
const size = Math.min(opts.batch ?? 32, MAX_BATCH);
|
|
239
|
+
for (let i = 0; i < inputs.length; i += size) {
|
|
240
|
+
const v = await embed(cfg, inputs.slice(i, i + size), { ...opts, dim: opts.dim ?? (out[0]?.length) });
|
|
241
|
+
out.push(...v);
|
|
242
|
+
opts.progress?.(out.length);
|
|
243
|
+
}
|
|
244
|
+
return out;
|
|
245
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// The embedding endpoint may receive page text without confirmation only on the owner's own network.
|
|
3
|
+
import net from "node:net";
|
|
4
|
+
export function isOwnerAddress(ip) {
|
|
5
|
+
if (net.isIPv4(ip)) {
|
|
6
|
+
const [a, b] = ip.split(".").map(Number);
|
|
7
|
+
return a === 127 || a === 10 || (a === 172 && b >= 16 && b <= 31) ||
|
|
8
|
+
(a === 192 && b === 168) || (a === 169 && b === 254) || (a === 100 && b >= 64 && b <= 127);
|
|
9
|
+
}
|
|
10
|
+
const bare = ip.split("%")[0];
|
|
11
|
+
if (!net.isIPv6(bare))
|
|
12
|
+
return false;
|
|
13
|
+
// URL canonicalization gives mapped addresses one spelling, including when their tail was dotted IPv4.
|
|
14
|
+
const host = new URL(`http://[${bare}]/`).hostname.slice(1, -1);
|
|
15
|
+
const mapped = /^::ffff:([a-f0-9]+):([a-f0-9]+)$/.exec(host);
|
|
16
|
+
if (mapped) {
|
|
17
|
+
const hi = parseInt(mapped[1], 16), lo = parseInt(mapped[2], 16);
|
|
18
|
+
return isOwnerAddress(`${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`);
|
|
19
|
+
}
|
|
20
|
+
const first = parseInt(host.split(":")[0], 16);
|
|
21
|
+
return host === "::1" || (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80;
|
|
22
|
+
}
|
package/dist/history.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// Tiresias is missing (W-12, needs.ts).
|
|
5
5
|
import { appendLine, listLogs, pageLog, parseSince, utcDay } from "./store.js";
|
|
6
6
|
import { LANG_SET } from "./lang.js";
|
|
7
|
+
const KEY_EVENTS = new Set(["signing_key_rotated", "signing_key_rotation_finished", "slip_key_rotated"]);
|
|
7
8
|
// Bounds on what one record may hold, so a single line can never outgrow a history page.
|
|
8
9
|
const MAX_AGENT = 200;
|
|
9
10
|
const MAX_PURPOSE = 1000;
|
|
@@ -12,13 +13,14 @@ const MAX_URL_LOG = 4096;
|
|
|
12
13
|
const REASONS = new Set(["bot_challenge", "captcha", "login_required", "forbidden", "owner_hand_off"]); // owner_hand_off: W-25 (v0.6)
|
|
13
14
|
const DOORS = new Set(["mcp", "api", "feed", "structured", "text", "none"]);
|
|
14
15
|
const MAX_MATCHED = 1000;
|
|
16
|
+
const MAX_WRITTEN_BY = 20; // W-39: a local reply shows at most 5 passages, or one page of a section
|
|
15
17
|
const TICKETS = new Set(["created", "refused_cap"]);
|
|
16
18
|
const CACHE = new Set(["fresh", "revalidated", "refetched"]);
|
|
17
19
|
const FLAGS = new Set(["phishing_list", "web_risk"]);
|
|
18
20
|
const SLIP_ID = /^[A-Za-z0-9_-]{24}$/;
|
|
19
21
|
const FILE_KINDS = new Set(["program", "script", "shortcut", "addon", "archive", "office", "image", "media", "unknown"]);
|
|
20
22
|
const SCANS = new Set(["bad", "clean", "unknown", "failed"]);
|
|
21
|
-
const SETTINGS = new Set(["saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
|
|
23
|
+
const SETTINGS = new Set(["guard", "private_spacing", "hook_secrets", "saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
|
|
22
24
|
const CHANGES = new Set(["looser", "stricter", "never_ask", "ask_again"]);
|
|
23
25
|
const BYS = new Set(["terminal", "not_terminal"]);
|
|
24
26
|
const HOWS = new Set(["phrase", "never_ask", "not_needed"]);
|
|
@@ -51,7 +53,7 @@ export function bound(e) {
|
|
|
51
53
|
drop("llms_txt");
|
|
52
54
|
if (e.ticket !== undefined && !(typeof e.ticket === "string" && TICKETS.has(e.ticket)))
|
|
53
55
|
drop("ticket");
|
|
54
|
-
if (e.source !== undefined && e.source !== "provided_by_person")
|
|
56
|
+
if (e.source !== undefined && e.source !== "provided_by_person" && e.source !== "provided_by_agent")
|
|
55
57
|
drop("source");
|
|
56
58
|
if (e.cache !== undefined && !(typeof e.cache === "string" && CACHE.has(e.cache)))
|
|
57
59
|
drop("cache");
|
|
@@ -79,8 +81,12 @@ export function bound(e) {
|
|
|
79
81
|
drop("by");
|
|
80
82
|
if (e.confirmed !== undefined && !(typeof e.confirmed === "string" && HOWS.has(e.confirmed)))
|
|
81
83
|
drop("confirmed");
|
|
82
|
-
if (e.event !== undefined && e.event
|
|
84
|
+
if (e.event !== undefined && !(e.event === "memory_ceiling" || (typeof e.event === "string" && KEY_EVENTS.has(e.event))))
|
|
83
85
|
drop("event");
|
|
86
|
+
if (e.revoked !== undefined && !(Number.isSafeInteger(e.revoked) && e.revoked >= 0 && e.revoked <= 100_000))
|
|
87
|
+
drop("revoked");
|
|
88
|
+
if (e.written_by !== undefined && !(Array.isArray(e.written_by) && e.written_by.length >= 1 && e.written_by.length <= MAX_WRITTEN_BY && e.written_by.every((n) => n === 0 || n === 1 || n === 2 || n === 3)))
|
|
89
|
+
drop("written_by");
|
|
84
90
|
if (e.memory_mb !== undefined && !(Number.isSafeInteger(e.memory_mb) && e.memory_mb >= 1 && e.memory_mb <= 65536))
|
|
85
91
|
drop("memory_mb");
|
|
86
92
|
if (over)
|
|
@@ -92,7 +98,7 @@ export function record(e) {
|
|
|
92
98
|
const t = Date.parse(out.time);
|
|
93
99
|
appendLine(`history/${utcDay(Number.isNaN(t) ? Date.now() : t)}.jsonl`, out);
|
|
94
100
|
}
|
|
95
|
-
export async function query(f) {
|
|
101
|
+
export async function query(f, scan = {}) {
|
|
96
102
|
const since = parseSince(f.since);
|
|
97
103
|
const site = f.site?.toLowerCase().replace(/^www\./, "");
|
|
98
104
|
return pageLog(listLogs("history").map((f) => f.rel), (e) => {
|
|
@@ -115,5 +121,5 @@ export async function query(f) {
|
|
|
115
121
|
}
|
|
116
122
|
bound(e); // filters match the stored values; only what is returned is cut
|
|
117
123
|
return true;
|
|
118
|
-
}, f.page ?? 1);
|
|
124
|
+
}, f.page ?? 1, 40_000, scan);
|
|
119
125
|
}
|
package/dist/hook.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// answered, or within 200 ms of Tiresias being gone. Tiresias also kills the group at the cap, on cancellation
|
|
8
8
|
// and once it has the answer, so a hook that starts helpers leaves nothing running. Its output is capped; its
|
|
9
9
|
// standard error is ignored. Hooks are never run on Windows (no process groups to kill).
|
|
10
|
+
import { loadChoice } from "./config.js";
|
|
10
11
|
import { spawn } from "node:child_process";
|
|
11
12
|
import { fileURLToPath } from "node:url";
|
|
12
13
|
import { currentSignal } from "./safety.js";
|
|
@@ -48,6 +49,7 @@ export function runHook(command, extra, timeoutMs, maxOut, input) {
|
|
|
48
49
|
const cap = maxOut === 4096 ? [] : [`max=${maxOut}`]; // the runner's default is the scanner's 4 KB
|
|
49
50
|
child = spawn(process.execPath, [runner, String(process.pid), String(timeoutMs), ...cap, command[0], ...command.slice(1), ...extra], {
|
|
50
51
|
// Always its own group: the runner kills its WHOLE group, which must never be ours.
|
|
52
|
+
env: loadChoice("hook_secrets") === "inherit" ? { ...process.env } : Object.fromEntries(Object.entries(process.env).filter(([k]) => !/^TIRESIAS_.*(?:KEY|SECRET|TOKEN|PASSWORD|CREDENTIAL)/i.test(k))),
|
|
51
53
|
shell: false, stdio: ["pipe", "pipe", "ignore"], detached: true, windowsHide: true,
|
|
52
54
|
});
|
|
53
55
|
}
|
package/dist/hosts.js
CHANGED
|
@@ -3,18 +3,18 @@
|
|
|
3
3
|
// in the shared hosts/ folder at the store root, holding the earliest time the next request to it may
|
|
4
4
|
// start. A request reserves its slot under a short exclusive lock (an O_EXCL lock file, broken when
|
|
5
5
|
// older than LOCK_STALE_MS), so the CLI and the MCP server running side by side space their requests
|
|
6
|
-
// to one site as one queue does. Private
|
|
7
|
-
// private visit leaves no trace of which site it called; a preview writes nothing); their spacing then
|
|
8
|
-
// holds within the process only.
|
|
6
|
+
// to one site as one queue does. Private calls and previews share this minimal politeness state too.
|
|
9
7
|
import { randomBytes } from "node:crypto";
|
|
10
8
|
import fs from "node:fs";
|
|
11
9
|
import path from "node:path";
|
|
10
|
+
import { loadChoice } from "./config.js";
|
|
12
11
|
import { Refused } from "./errors.js";
|
|
13
12
|
import { holdLease } from "./lifetime.js";
|
|
14
13
|
import { lock } from "./lockfile.js";
|
|
15
14
|
import { releaseLock, takeLock } from "./ownedlock.js";
|
|
16
15
|
import { currentReadOnly } from "./safety.js";
|
|
17
16
|
import { NOFOLLOW, NONBLOCK, ensureRoot, isPrivate, ownFile, readRoot, sha256, checkSub, mkdirOne } from "./store.js";
|
|
17
|
+
export const sharesSpacing = () => (!isPrivate() && !currentReadOnly()) || loadChoice("private_spacing") === "shared";
|
|
18
18
|
export const HOSTS = "hosts";
|
|
19
19
|
export const DEFAULT_SPACING_MS = 1_000;
|
|
20
20
|
// The least time between two requests to one site. One second by default; TIRESIAS_HOST_SPACING_MS
|
|
@@ -26,49 +26,47 @@ export function spacingMs() {
|
|
|
26
26
|
return DEFAULT_SPACING_MS;
|
|
27
27
|
}
|
|
28
28
|
const fileOf = (dir, origin) => path.join(dir, sha256(origin).slice(0, 32));
|
|
29
|
-
|
|
30
|
-
// last reservation left, AND `spacing` after that reservation's own start (Codex round 69: a 1-second
|
|
31
|
-
// reservation must not let a 30-second Crawl-delay request start a second later).
|
|
32
|
-
// The spacing file, read safely (no links, no blocking, a regular file of ours, at most 256 bytes).
|
|
33
|
-
function readSpacing(file) {
|
|
29
|
+
function readSpacing(file, strict = false) {
|
|
34
30
|
let fd;
|
|
35
31
|
try {
|
|
36
32
|
fd = fs.openSync(file, fs.constants.O_RDONLY | NOFOLLOW | NONBLOCK);
|
|
37
33
|
}
|
|
38
34
|
catch {
|
|
39
|
-
return
|
|
35
|
+
return undefined;
|
|
40
36
|
}
|
|
41
37
|
try {
|
|
42
38
|
ownFile(fd, file);
|
|
43
39
|
if (fs.fstatSync(fd).size > 256)
|
|
44
|
-
return
|
|
40
|
+
return undefined;
|
|
45
41
|
const o = JSON.parse(fs.readFileSync(fd, "utf8"));
|
|
46
42
|
const next = Number(o?.next);
|
|
47
43
|
const last = Number(o?.last);
|
|
44
|
+
if (strict && (typeof o?.last !== "number" || typeof o?.next !== "number" || !Number.isFinite(last) || !Number.isFinite(next)))
|
|
45
|
+
return undefined;
|
|
48
46
|
return { last: Number.isFinite(last) ? last : 0, next: Number.isFinite(next) ? next : 0 };
|
|
49
47
|
}
|
|
50
48
|
catch {
|
|
51
|
-
return
|
|
49
|
+
return undefined;
|
|
52
50
|
}
|
|
53
51
|
finally {
|
|
54
52
|
fs.closeSync(fd);
|
|
55
53
|
}
|
|
56
54
|
}
|
|
57
55
|
function readNext(file, spacing) {
|
|
58
|
-
const { last, next } = readSpacing(file);
|
|
56
|
+
const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
|
|
59
57
|
return Math.max(next, last ? last + spacing : 0);
|
|
60
58
|
}
|
|
61
59
|
// Reserve the next slot for `origin` with `spacing` ms between starts and return when it begins (epoch
|
|
62
60
|
// ms). `earliest` is this process's own earliest time. Store trouble never fails a request: the slot is
|
|
63
61
|
// then this process's own. `whenReady` leaves a future slot unclaimed so a cancelled wait moves nothing.
|
|
64
|
-
export function reserve(origin, spacing, earliest, whenReady = false) {
|
|
62
|
+
export function reserve(origin, spacing, earliest, whenReady = false, pass) {
|
|
65
63
|
const now = Date.now();
|
|
66
64
|
let dir;
|
|
67
65
|
try {
|
|
68
|
-
if (
|
|
66
|
+
if (!sharesSpacing()) {
|
|
69
67
|
const r = readRoot();
|
|
70
68
|
dir = r && checkSub(path.join(r, HOSTS)) ? path.join(r, HOSTS) : null;
|
|
71
|
-
return Math.max(now, earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
|
|
69
|
+
return Math.max(now, pass ? 0 : earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
|
|
72
70
|
}
|
|
73
71
|
const root = ensureRoot();
|
|
74
72
|
dir = path.join(root, HOSTS);
|
|
@@ -88,7 +86,10 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
|
|
|
88
86
|
if (!held)
|
|
89
87
|
return Math.max(now, earliest, readNext(file, spacing)) + spacing;
|
|
90
88
|
try {
|
|
91
|
-
|
|
89
|
+
// The active lease is still held. Only an exact, safely read completion stamp can be waived.
|
|
90
|
+
const current = pass ? readSpacing(file, true) : undefined;
|
|
91
|
+
const waive = pass && current?.last === pass.last && current?.next === pass.next;
|
|
92
|
+
const slot = waive ? now : Math.max(now, earliest, readNext(file, spacing));
|
|
92
93
|
if (whenReady && slot > now)
|
|
93
94
|
return slot;
|
|
94
95
|
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
@@ -104,6 +105,8 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
|
|
|
104
105
|
}
|
|
105
106
|
catch {
|
|
106
107
|
fs.rmSync(tmp, { force: true });
|
|
108
|
+
if (waive)
|
|
109
|
+
return Math.max(now, earliest, readNext(file, spacing));
|
|
107
110
|
}
|
|
108
111
|
return slot;
|
|
109
112
|
}
|
|
@@ -114,7 +117,7 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
|
|
|
114
117
|
// One request at a time across processes too (Codex round 75): while a request to a site is being
|
|
115
118
|
// answered, a small "active" file (the owner's pid and the time) sits beside its spacing file; others wait
|
|
116
119
|
// until it is gone. A file whose process is gone, or that is older than a call can live (75 s), is broken.
|
|
117
|
-
// Private calls and previews
|
|
120
|
+
// Private calls and previews share it by default; unshared calls wait without writing. Store trouble never blocks a request.
|
|
118
121
|
// v0.6 (coordinator's decision after Codex round 8): a lease is NEVER taken from a live process, however old it
|
|
119
122
|
// is; a lease whose process is gone is stale at once. Only a lease whose owner cannot be told (no process id) is
|
|
120
123
|
// judged by age, after this long (longer than any holder can live: a background job lasts up to 240 seconds).
|
|
@@ -204,9 +207,9 @@ function activeStale(f) {
|
|
|
204
207
|
}
|
|
205
208
|
}
|
|
206
209
|
// A delay learned from robots.txt reaches other processes too (Codex round 84): the shared file's next
|
|
207
|
-
// slot is moved to `until` if later.
|
|
210
|
+
// slot is moved to `until` if later. Shared by default for private calls and previews; no write without the lock.
|
|
208
211
|
export function extendNext(origin, until) {
|
|
209
|
-
if (
|
|
212
|
+
if (!sharesSpacing())
|
|
210
213
|
return;
|
|
211
214
|
try {
|
|
212
215
|
const r = readRoot();
|
|
@@ -217,7 +220,7 @@ export function extendNext(origin, until) {
|
|
|
217
220
|
if (!lock(file))
|
|
218
221
|
return;
|
|
219
222
|
try {
|
|
220
|
-
const { last, next } = readSpacing(file);
|
|
223
|
+
const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
|
|
221
224
|
if (until <= next)
|
|
222
225
|
return;
|
|
223
226
|
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
@@ -240,9 +243,9 @@ export function extendNext(origin, until) {
|
|
|
240
243
|
}
|
|
241
244
|
// v0.6 (Codex round 11, coordinator's decision): a held turn's completion is recorded WHOLE in the shared file:
|
|
242
245
|
// last = the completion time, next = the later of the old next and completion + delay. Every reader, whatever
|
|
243
|
-
// spacing it needs, then measures from the last completion.
|
|
246
|
+
// spacing it needs, then measures from the last completion. Shared by default for private calls and previews; under the lock.
|
|
244
247
|
export function noteCompletion(origin, t, until) {
|
|
245
|
-
if (
|
|
248
|
+
if (!sharesSpacing())
|
|
246
249
|
return;
|
|
247
250
|
try {
|
|
248
251
|
const r = readRoot();
|
|
@@ -253,16 +256,19 @@ export function noteCompletion(origin, t, until) {
|
|
|
253
256
|
if (!lock(file))
|
|
254
257
|
return;
|
|
255
258
|
try {
|
|
256
|
-
const
|
|
259
|
+
const previous = readSpacing(file, true);
|
|
260
|
+
const { last, next } = previous ?? readSpacing(file) ?? { last: 0, next: 0 };
|
|
261
|
+
const stamp = { last: Math.max(last, t), next: Math.max(next, until) };
|
|
257
262
|
const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
258
263
|
const fd = fs.openSync(tmp, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NOFOLLOW, 0o600);
|
|
259
264
|
try {
|
|
260
|
-
fs.writeSync(fd, JSON.stringify({ v: 1,
|
|
265
|
+
fs.writeSync(fd, JSON.stringify({ v: 1, ...stamp }));
|
|
261
266
|
}
|
|
262
267
|
finally {
|
|
263
268
|
fs.closeSync(fd);
|
|
264
269
|
}
|
|
265
270
|
fs.renameSync(tmp, file);
|
|
271
|
+
return previous ? stamp : undefined;
|
|
266
272
|
}
|
|
267
273
|
finally {
|
|
268
274
|
fs.rmSync(file + ".lock", { force: true });
|
|
@@ -279,10 +285,10 @@ const pause = (ms, signal) => new Promise((resolve, reject) => {
|
|
|
279
285
|
const stop = () => (clearTimeout(t), reject(signal.reason));
|
|
280
286
|
signal.addEventListener("abort", stop, { once: true });
|
|
281
287
|
});
|
|
282
|
-
export async function acquireActive(origin, signal) {
|
|
288
|
+
export async function acquireActive(origin, signal, onHeld) {
|
|
283
289
|
let dir = null;
|
|
284
290
|
try {
|
|
285
|
-
if (
|
|
291
|
+
if (sharesSpacing()) {
|
|
286
292
|
// A writer makes hosts/ first, so the very first request to a site holds its lease too (round 77).
|
|
287
293
|
const root = ensureRoot();
|
|
288
294
|
const d = path.join(root, HOSTS);
|
|
@@ -301,7 +307,7 @@ export async function acquireActive(origin, signal) {
|
|
|
301
307
|
if (!dir)
|
|
302
308
|
return () => { };
|
|
303
309
|
const f = fileOf(dir, origin) + ".active";
|
|
304
|
-
const writer =
|
|
310
|
+
const writer = sharesSpacing();
|
|
305
311
|
const giveUp = Date.now() + leaseWaitMs();
|
|
306
312
|
for (;;) {
|
|
307
313
|
if (writer) {
|
|
@@ -321,6 +327,7 @@ export async function acquireActive(origin, signal) {
|
|
|
321
327
|
fs.rmSync(f, { force: true });
|
|
322
328
|
};
|
|
323
329
|
const forget = holdLease(f, remove);
|
|
330
|
+
onHeld?.();
|
|
324
331
|
return () => {
|
|
325
332
|
remove();
|
|
326
333
|
forget();
|