@raibitsi/tiresias 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,245 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // W-38 (v1.2): the meaning check's one network client. It talks to the owner's OWN embedding endpoint (LM Studio,
3
+ // Ollama, or any OpenAI-compatible /v1/embeddings), never to a site, so it is deliberately not the page fetcher:
4
+ // no robots.txt, no signing, no spacing, no cache, no Tiresias user agent promise. What it keeps from the fetcher's
5
+ // hardening is what protects the owner: one request, no redirect ever followed, a hard timeout, a cap on the bytes
6
+ // read, and a reply that is checked in full before a single number from it is used.
7
+ //
8
+ // Where it may send: the setting's address is resolved here, at call time, and the connection goes to the address
9
+ // that was checked (no second lookup). An address on this machine or the owner's network (loopback, private, link
10
+ // local, CGNAT 100.64.0.0/10 that Tailscale uses) is fine. Anything else means page text would leave the owner's
11
+ // network, so it needs `allow_public`, which only `tiresias guard setup` writes and only with the typed
12
+ // confirmation; plain http to such an address is refused whatever the setting says.
13
+ //
14
+ // The key: TIRESIAS_GUARD_KEY, read from the environment at call time and sent as a Bearer token. It is never
15
+ // stored, logged or put in a reply or an error.
16
+ import dns from "node:dns";
17
+ import http from "node:http";
18
+ import https from "node:https";
19
+ import net from "node:net";
20
+ import { isOwnerAddress } from "./guardnetwork.js";
21
+ const WORDS = {
22
+ refused_public: "the endpoint's address is not on this machine or your network, and sending page text there was not confirmed",
23
+ refused_http: "the endpoint's address is not on this machine or your network, and plain http to it is refused",
24
+ dns: "the endpoint's name could not be looked up",
25
+ network: "the endpoint could not be reached",
26
+ timeout: "the endpoint did not answer in time",
27
+ cancelled: "the call was cancelled",
28
+ http_status: "the endpoint answered with an error status",
29
+ redirect: "the endpoint answered with a redirect, which is never followed",
30
+ too_large: "the endpoint's answer was larger than allowed",
31
+ bad_json: "the endpoint's answer was not JSON",
32
+ bad_shape: "the endpoint's answer did not have a data list of embeddings",
33
+ wrong_count: "the endpoint returned a different number of embeddings than texts sent",
34
+ wrong_dimension: "the endpoint returned embeddings of an unexpected length",
35
+ not_finite: "the endpoint returned a number that is not finite",
36
+ zero_vector: "the endpoint returned an embedding of all zeros",
37
+ };
38
+ export class EmbedError extends Error {
39
+ code;
40
+ status;
41
+ constructor(code, status) {
42
+ super(`The embedding call failed: ${WORDS[code]}${status !== undefined ? ` (HTTP ${status})` : ""}.`);
43
+ this.code = code;
44
+ this.status = status;
45
+ }
46
+ }
47
+ // Bounds. A batch is at most MAX_BATCH texts; a reply at most MAX_REPLY bytes (64 texts of 4,096 numbers written
48
+ // out in full is about 6 MB); a dimension from 8 to 8,192.
49
+ export const MAX_BATCH = 64;
50
+ export const MAX_REPLY = 16 * 1024 * 1024;
51
+ export const MIN_DIM = 8;
52
+ export const MAX_DIM = 8192;
53
+ // v1.2: the whole input, including the configured prefix, must fit. Never silently embed a truncated text.
54
+ export const MAX_INPUT_CHARS = 8_000;
55
+ const isLoopbackName = (h) => h === "localhost" || h.endsWith(".localhost");
56
+ const bare = (h) => (h.startsWith("[") && h.endsWith("]") ? h.slice(1, -1) : h);
57
+ // The addresses a host name stands for, every one checked. A literal address is used as it is.
58
+ async function resolve(host, signal) {
59
+ if (signal.aborted)
60
+ throw new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled");
61
+ const h = bare(host);
62
+ if (net.isIP(h))
63
+ return [{ address: h, family: net.isIP(h) }];
64
+ const lookup = dns.promises.lookup(isLoopbackName(h) ? "localhost" : h, { all: true, verbatim: true });
65
+ const abort = new Promise((_, reject) => signal.addEventListener("abort", () => reject(new EmbedError(signal.reason === "timeout" ? "timeout" : "cancelled")), { once: true }));
66
+ let found;
67
+ try {
68
+ found = await Promise.race([lookup, abort]);
69
+ }
70
+ catch (e) {
71
+ if (e instanceof EmbedError)
72
+ throw e;
73
+ throw new EmbedError("dns");
74
+ }
75
+ if (!found.length)
76
+ throw new EmbedError("dns");
77
+ return found;
78
+ }
79
+ // "local" when every address is loopback, private, link local or CGNAT: page text stays on the owner's network.
80
+ export async function classify(url, signal = new AbortController().signal) {
81
+ const addrs = await resolve(new URL(url).hostname, signal);
82
+ return addrs.every((a) => isOwnerAddress(a.address)) ? "local" : "public";
83
+ }
84
+ // The whole check for one address: may page text be sent there under this setting?
85
+ export async function destination(cfg, signal) {
86
+ const u = new URL(cfg.url);
87
+ const addrs = await resolve(u.hostname, signal);
88
+ const local = addrs.every((a) => isOwnerAddress(a.address));
89
+ if (!local) {
90
+ if (u.protocol !== "https:")
91
+ throw new EmbedError("refused_http");
92
+ if (cfg.allow_public !== true)
93
+ throw new EmbedError("refused_public");
94
+ }
95
+ return addrs[0];
96
+ }
97
+ // One POST to <url>/embeddings with { model, input }, and the vectors from data[i].embedding, each made unit length.
98
+ // Throws an EmbedError with a fixed code on anything unexpected.
99
+ export async function embed(cfg, inputs, opts) {
100
+ if (inputs.length === 0)
101
+ return [];
102
+ if (inputs.length > MAX_BATCH)
103
+ throw new RangeError("too many texts in one batch");
104
+ if (inputs.some(t => t.length > MAX_INPUT_CHARS))
105
+ throw new RangeError("text too long for one embedding input");
106
+ const ctl = new AbortController();
107
+ const timer = setTimeout(() => ctl.abort("timeout"), opts.timeoutMs);
108
+ const outer = opts.signal;
109
+ const onOuter = () => ctl.abort("cancelled");
110
+ if (outer?.aborted)
111
+ ctl.abort("cancelled");
112
+ else
113
+ outer?.addEventListener("abort", onOuter, { once: true });
114
+ try {
115
+ const to = await destination(cfg, ctl.signal);
116
+ const body = Buffer.from(JSON.stringify({ model: cfg.model, input: inputs }));
117
+ const raw = await post(new URL(cfg.url.replace(/\/+$/, "") + "/embeddings"), body, to, ctl.signal);
118
+ return parseVectors(raw, inputs.length, opts.dim);
119
+ }
120
+ finally {
121
+ clearTimeout(timer);
122
+ outer?.removeEventListener("abort", onOuter);
123
+ }
124
+ }
125
+ function post(u, body, to, signal) {
126
+ return new Promise((resolveP, reject) => {
127
+ const fail = (code, status) => reject(new EmbedError(code, status));
128
+ if (signal.aborted)
129
+ return fail(signal.reason === "timeout" ? "timeout" : "cancelled");
130
+ const headers = { "content-type": "application/json", accept: "application/json", "content-length": String(body.length) };
131
+ const key = process.env.TIRESIAS_GUARD_KEY;
132
+ if (key)
133
+ headers.authorization = `Bearer ${key}`;
134
+ const mod = u.protocol === "https:" ? https : http;
135
+ // The connection goes to the address checked above: a lookup that answers with it, never a second DNS query.
136
+ const lookup = (_h, o, cb) => o?.all ? cb(null, [{ address: to.address, family: to.family }]) : cb(null, to.address, to.family);
137
+ const req = mod.request(u, { method: "POST", headers, lookup: lookup, agent: false, signal }, (res) => {
138
+ const status = res.statusCode ?? 0;
139
+ if (status >= 300 && status < 400) {
140
+ res.destroy();
141
+ return fail("redirect", status);
142
+ }
143
+ if (status !== 200) {
144
+ res.destroy();
145
+ return fail("http_status", status);
146
+ }
147
+ const declared = Number(res.headers["content-length"]);
148
+ if (Number.isFinite(declared) && declared > MAX_REPLY) {
149
+ res.destroy();
150
+ return fail("too_large");
151
+ }
152
+ const parts = [];
153
+ let total = 0;
154
+ res.on("data", (c) => {
155
+ total += c.length;
156
+ if (total > MAX_REPLY) {
157
+ res.destroy();
158
+ return fail("too_large");
159
+ }
160
+ parts.push(c);
161
+ });
162
+ res.on("end", () => resolveP(Buffer.concat(parts, total)));
163
+ res.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
164
+ });
165
+ req.on("error", () => fail(signal.aborted ? (signal.reason === "timeout" ? "timeout" : "cancelled") : "network"));
166
+ req.end(body);
167
+ });
168
+ }
169
+ // The reply, checked whole: a data list of exactly `count` embeddings (if any item has an index, all must have a
170
+ // distinct index in range), each a list of finite numbers of one length (the stored dimension when there is one).
171
+ export function parseVectors(raw, count, dim) {
172
+ let j;
173
+ try {
174
+ j = JSON.parse(raw.toString("utf8"));
175
+ }
176
+ catch {
177
+ throw new EmbedError("bad_json");
178
+ }
179
+ const data = j && typeof j === "object" && !Array.isArray(j) ? j.data : undefined;
180
+ if (!Array.isArray(data))
181
+ throw new EmbedError("bad_shape");
182
+ if (data.length !== count)
183
+ throw new EmbedError("wrong_count");
184
+ let order = data;
185
+ const idx = data.map((d) => (d && typeof d === "object" ? d.index : undefined));
186
+ if (idx.some((i) => i !== undefined)) {
187
+ if (!idx.every((i) => Number.isInteger(i) && i >= 0 && i < count) || new Set(idx).size !== count)
188
+ throw new EmbedError("bad_shape");
189
+ order = new Array(count);
190
+ data.forEach((d, k) => (order[idx[k]] = d));
191
+ }
192
+ const out = [];
193
+ let len = dim;
194
+ for (const d of order) {
195
+ const e = d && typeof d === "object" ? d.embedding : undefined;
196
+ if (!Array.isArray(e))
197
+ throw new EmbedError("bad_shape");
198
+ if (len === undefined) {
199
+ if (e.length < MIN_DIM || e.length > MAX_DIM)
200
+ throw new EmbedError("wrong_dimension");
201
+ len = e.length;
202
+ }
203
+ if (e.length !== len)
204
+ throw new EmbedError("wrong_dimension");
205
+ const v = new Float32Array(len);
206
+ let n = 0;
207
+ let storedNorm = 0;
208
+ for (let i = 0; i < len; i++) {
209
+ const x = e[i];
210
+ if (typeof x !== "number" || !Number.isFinite(x))
211
+ throw new EmbedError("not_finite");
212
+ // Reject values outside the stored format before they can become infinities or an all-zero vector.
213
+ const stored = Math.fround(x);
214
+ if (!Number.isFinite(stored))
215
+ throw new EmbedError("not_finite");
216
+ storedNorm += stored * stored;
217
+ n += x * x;
218
+ }
219
+ n = Math.sqrt(n);
220
+ if (!(n > 0) || !Number.isFinite(n) || !(storedNorm > 0))
221
+ throw new EmbedError("zero_vector");
222
+ let finalNorm = 0;
223
+ for (let i = 0; i < len; i++) {
224
+ v[i] = e[i] / n;
225
+ if (!Number.isFinite(v[i]))
226
+ throw new EmbedError("not_finite");
227
+ finalNorm += v[i] * v[i];
228
+ }
229
+ if (!(finalNorm > 0) || !Number.isFinite(finalNorm))
230
+ throw new EmbedError("zero_vector");
231
+ out.push(v);
232
+ }
233
+ return out;
234
+ }
235
+ // Several batches in order, within one deadline.
236
+ export async function embedAll(cfg, inputs, opts) {
237
+ const out = [];
238
+ const size = Math.min(opts.batch ?? 32, MAX_BATCH);
239
+ for (let i = 0; i < inputs.length; i += size) {
240
+ const v = await embed(cfg, inputs.slice(i, i + size), { ...opts, dim: opts.dim ?? (out[0]?.length) });
241
+ out.push(...v);
242
+ opts.progress?.(out.length);
243
+ }
244
+ return out;
245
+ }
@@ -0,0 +1,22 @@
1
+ // SPDX-License-Identifier: Apache-2.0
2
+ // The embedding endpoint may receive page text without confirmation only on the owner's own network.
3
+ import net from "node:net";
4
+ export function isOwnerAddress(ip) {
5
+ if (net.isIPv4(ip)) {
6
+ const [a, b] = ip.split(".").map(Number);
7
+ return a === 127 || a === 10 || (a === 172 && b >= 16 && b <= 31) ||
8
+ (a === 192 && b === 168) || (a === 169 && b === 254) || (a === 100 && b >= 64 && b <= 127);
9
+ }
10
+ const bare = ip.split("%")[0];
11
+ if (!net.isIPv6(bare))
12
+ return false;
13
+ // URL canonicalization gives mapped addresses one spelling, including when their tail was dotted IPv4.
14
+ const host = new URL(`http://[${bare}]/`).hostname.slice(1, -1);
15
+ const mapped = /^::ffff:([a-f0-9]+):([a-f0-9]+)$/.exec(host);
16
+ if (mapped) {
17
+ const hi = parseInt(mapped[1], 16), lo = parseInt(mapped[2], 16);
18
+ return isOwnerAddress(`${hi >> 8}.${hi & 255}.${lo >> 8}.${lo & 255}`);
19
+ }
20
+ const first = parseInt(host.split(":")[0], 16);
21
+ return host === "::1" || (first & 0xfe00) === 0xfc00 || (first & 0xffc0) === 0xfe80;
22
+ }
package/dist/history.js CHANGED
@@ -13,13 +13,14 @@ const MAX_URL_LOG = 4096;
13
13
  const REASONS = new Set(["bot_challenge", "captcha", "login_required", "forbidden", "owner_hand_off"]); // owner_hand_off: W-25 (v0.6)
14
14
  const DOORS = new Set(["mcp", "api", "feed", "structured", "text", "none"]);
15
15
  const MAX_MATCHED = 1000;
16
+ const MAX_WRITTEN_BY = 20; // W-39: a local reply shows at most 5 passages, or one page of a section
16
17
  const TICKETS = new Set(["created", "refused_cap"]);
17
18
  const CACHE = new Set(["fresh", "revalidated", "refetched"]);
18
19
  const FLAGS = new Set(["phishing_list", "web_risk"]);
19
20
  const SLIP_ID = /^[A-Za-z0-9_-]{24}$/;
20
21
  const FILE_KINDS = new Set(["program", "script", "shortcut", "addon", "archive", "office", "image", "media", "unknown"]);
21
22
  const SCANS = new Set(["bad", "clean", "unknown", "failed"]);
22
- const SETTINGS = new Set(["private_spacing", "hook_secrets", "saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
23
+ const SETTINGS = new Set(["guard", "private_spacing", "hook_secrets", "saving", "rendering", "translator", "scanner", "anchoring", "anchor_calendars", "safe_browsing", "ports", "spacing_seconds", "never_visit", "hand_off", "confirmations"]);
23
24
  const CHANGES = new Set(["looser", "stricter", "never_ask", "ask_again"]);
24
25
  const BYS = new Set(["terminal", "not_terminal"]);
25
26
  const HOWS = new Set(["phrase", "never_ask", "not_needed"]);
@@ -84,6 +85,8 @@ export function bound(e) {
84
85
  drop("event");
85
86
  if (e.revoked !== undefined && !(Number.isSafeInteger(e.revoked) && e.revoked >= 0 && e.revoked <= 100_000))
86
87
  drop("revoked");
88
+ if (e.written_by !== undefined && !(Array.isArray(e.written_by) && e.written_by.length >= 1 && e.written_by.length <= MAX_WRITTEN_BY && e.written_by.every((n) => n === 0 || n === 1 || n === 2 || n === 3)))
89
+ drop("written_by");
87
90
  if (e.memory_mb !== undefined && !(Number.isSafeInteger(e.memory_mb) && e.memory_mb >= 1 && e.memory_mb <= 65536))
88
91
  drop("memory_mb");
89
92
  if (over)
package/dist/hosts.js CHANGED
@@ -14,7 +14,7 @@ import { lock } from "./lockfile.js";
14
14
  import { releaseLock, takeLock } from "./ownedlock.js";
15
15
  import { currentReadOnly } from "./safety.js";
16
16
  import { NOFOLLOW, NONBLOCK, ensureRoot, isPrivate, ownFile, readRoot, sha256, checkSub, mkdirOne } from "./store.js";
17
- const sharesSpacing = () => (!isPrivate() && !currentReadOnly()) || loadChoice("private_spacing") === "shared";
17
+ export const sharesSpacing = () => (!isPrivate() && !currentReadOnly()) || loadChoice("private_spacing") === "shared";
18
18
  export const HOSTS = "hosts";
19
19
  export const DEFAULT_SPACING_MS = 1_000;
20
20
  // The least time between two requests to one site. One second by default; TIRESIAS_HOST_SPACING_MS
@@ -26,49 +26,47 @@ export function spacingMs() {
26
26
  return DEFAULT_SPACING_MS;
27
27
  }
28
28
  const fileOf = (dir, origin) => path.join(dir, sha256(origin).slice(0, 32));
29
- // The earliest start the shared file allows for a request wanting `spacing` ms: after the next slot the
30
- // last reservation left, AND `spacing` after that reservation's own start (Codex round 69: a 1-second
31
- // reservation must not let a 30-second Crawl-delay request start a second later).
32
- // The spacing file, read safely (no links, no blocking, a regular file of ours, at most 256 bytes).
33
- function readSpacing(file) {
29
+ function readSpacing(file, strict = false) {
34
30
  let fd;
35
31
  try {
36
32
  fd = fs.openSync(file, fs.constants.O_RDONLY | NOFOLLOW | NONBLOCK);
37
33
  }
38
34
  catch {
39
- return { last: 0, next: 0 };
35
+ return undefined;
40
36
  }
41
37
  try {
42
38
  ownFile(fd, file);
43
39
  if (fs.fstatSync(fd).size > 256)
44
- return { last: 0, next: 0 };
40
+ return undefined;
45
41
  const o = JSON.parse(fs.readFileSync(fd, "utf8"));
46
42
  const next = Number(o?.next);
47
43
  const last = Number(o?.last);
44
+ if (strict && (typeof o?.last !== "number" || typeof o?.next !== "number" || !Number.isFinite(last) || !Number.isFinite(next)))
45
+ return undefined;
48
46
  return { last: Number.isFinite(last) ? last : 0, next: Number.isFinite(next) ? next : 0 };
49
47
  }
50
48
  catch {
51
- return { last: 0, next: 0 };
49
+ return undefined;
52
50
  }
53
51
  finally {
54
52
  fs.closeSync(fd);
55
53
  }
56
54
  }
57
55
  function readNext(file, spacing) {
58
- const { last, next } = readSpacing(file);
56
+ const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
59
57
  return Math.max(next, last ? last + spacing : 0);
60
58
  }
61
59
  // Reserve the next slot for `origin` with `spacing` ms between starts and return when it begins (epoch
62
60
  // ms). `earliest` is this process's own earliest time. Store trouble never fails a request: the slot is
63
61
  // then this process's own. `whenReady` leaves a future slot unclaimed so a cancelled wait moves nothing.
64
- export function reserve(origin, spacing, earliest, whenReady = false) {
62
+ export function reserve(origin, spacing, earliest, whenReady = false, pass) {
65
63
  const now = Date.now();
66
64
  let dir;
67
65
  try {
68
66
  if (!sharesSpacing()) {
69
67
  const r = readRoot();
70
68
  dir = r && checkSub(path.join(r, HOSTS)) ? path.join(r, HOSTS) : null;
71
- return Math.max(now, earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
69
+ return Math.max(now, pass ? 0 : earliest, dir ? readNext(fileOf(dir, origin), spacing) : 0);
72
70
  }
73
71
  const root = ensureRoot();
74
72
  dir = path.join(root, HOSTS);
@@ -88,7 +86,10 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
88
86
  if (!held)
89
87
  return Math.max(now, earliest, readNext(file, spacing)) + spacing;
90
88
  try {
91
- const slot = Math.max(now, earliest, readNext(file, spacing));
89
+ // The active lease is still held. Only an exact, safely read completion stamp can be waived.
90
+ const current = pass ? readSpacing(file, true) : undefined;
91
+ const waive = pass && current?.last === pass.last && current?.next === pass.next;
92
+ const slot = waive ? now : Math.max(now, earliest, readNext(file, spacing));
92
93
  if (whenReady && slot > now)
93
94
  return slot;
94
95
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
@@ -104,6 +105,8 @@ export function reserve(origin, spacing, earliest, whenReady = false) {
104
105
  }
105
106
  catch {
106
107
  fs.rmSync(tmp, { force: true });
108
+ if (waive)
109
+ return Math.max(now, earliest, readNext(file, spacing));
107
110
  }
108
111
  return slot;
109
112
  }
@@ -217,7 +220,7 @@ export function extendNext(origin, until) {
217
220
  if (!lock(file))
218
221
  return;
219
222
  try {
220
- const { last, next } = readSpacing(file);
223
+ const { last, next } = readSpacing(file) ?? { last: 0, next: 0 };
221
224
  if (until <= next)
222
225
  return;
223
226
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
@@ -253,16 +256,19 @@ export function noteCompletion(origin, t, until) {
253
256
  if (!lock(file))
254
257
  return;
255
258
  try {
256
- const { last, next } = readSpacing(file);
259
+ const previous = readSpacing(file, true);
260
+ const { last, next } = previous ?? readSpacing(file) ?? { last: 0, next: 0 };
261
+ const stamp = { last: Math.max(last, t), next: Math.max(next, until) };
257
262
  const tmp = `${file}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
258
263
  const fd = fs.openSync(tmp, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NOFOLLOW, 0o600);
259
264
  try {
260
- fs.writeSync(fd, JSON.stringify({ v: 1, last: Math.max(last, t), next: Math.max(next, until) }));
265
+ fs.writeSync(fd, JSON.stringify({ v: 1, ...stamp }));
261
266
  }
262
267
  finally {
263
268
  fs.closeSync(fd);
264
269
  }
265
270
  fs.renameSync(tmp, file);
271
+ return previous ? stamp : undefined;
266
272
  }
267
273
  finally {
268
274
  fs.rmSync(file + ".lock", { force: true });
@@ -279,7 +285,7 @@ const pause = (ms, signal) => new Promise((resolve, reject) => {
279
285
  const stop = () => (clearTimeout(t), reject(signal.reason));
280
286
  signal.addEventListener("abort", stop, { once: true });
281
287
  });
282
- export async function acquireActive(origin, signal) {
288
+ export async function acquireActive(origin, signal, onHeld) {
283
289
  let dir = null;
284
290
  try {
285
291
  if (sharesSpacing()) {
@@ -321,6 +327,7 @@ export async function acquireActive(origin, signal) {
321
327
  fs.rmSync(f, { force: true });
322
328
  };
323
329
  const forget = holdLease(f, remove);
330
+ onHeld?.();
324
331
  return () => {
325
332
  remove();
326
333
  forget();