@raibitsi/tiresias 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +199 -0
- package/README.md +50 -27
- package/dist/background.js +6 -3
- package/dist/browsecli.js +4 -3
- package/dist/cdp.js +22 -1
- package/dist/cli.js +82 -29
- package/dist/config.js +32 -26
- package/dist/confirm.js +16 -7
- package/dist/cooldown.js +65 -34
- package/dist/dailylog.js +11 -5
- package/dist/engine.js +5 -4
- package/dist/envelope.js +20 -2
- package/dist/fetcher.js +5 -0
- package/dist/filereply.js +7 -1
- package/dist/files.js +4 -4
- package/dist/formflow.js +21 -3
- package/dist/forms.js +257 -34
- package/dist/getactions.js +99 -0
- package/dist/history.js +8 -5
- package/dist/hook.js +2 -0
- package/dist/hosts.js +15 -13
- package/dist/inside.js +2 -2
- package/dist/inspect.js +4 -1
- package/dist/keys.js +420 -18
- package/dist/mcp.js +10 -10
- package/dist/needs.js +9 -5
- package/dist/parentwatch.js +6 -16
- package/dist/pdf.js +4 -1
- package/dist/pdfchild.js +0 -2
- package/dist/pdfproc.js +26 -11
- package/dist/pdfwatch.js +14 -0
- package/dist/pipeline.js +25 -8
- package/dist/profiles.js +3 -3
- package/dist/queue.js +15 -7
- package/dist/reader.js +43 -18
- package/dist/receipts.js +2 -2
- package/dist/render.js +103 -24
- package/dist/resume.js +17 -8
- package/dist/rules.js +29 -7
- package/dist/safebrowsing.js +3 -3
- package/dist/safety.js +9 -3
- package/dist/sandbox.js +1 -1
- package/dist/saved.js +108 -60
- package/dist/savedlock.js +208 -0
- package/dist/settings.js +7 -3
- package/dist/shelf.js +1 -1
- package/dist/sites.js +9 -4
- package/dist/slips.js +43 -1
- package/dist/store.js +119 -25
- package/dist/tabs.js +1 -1
- package/dist/textview.js +10 -0
- package/dist/translate.js +22 -2
- package/dist/types.js +2 -2
- package/dist/watchflow.js +1 -2
- package/dist/work.js +4 -2
- package/package.json +2 -1
package/dist/cooldown.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
-
// N-06: Retry-After is enforced, not just reported. When a site says 429
|
|
3
|
-
// 503, we remember "do not call this origin until T", across processes, and
|
|
2
|
+
// N-06: Retry-After is enforced, not just reported. When a site says 429,
|
|
3
|
+
// 503 or asks for a wait on a redirect, we remember "do not call this origin until T", across processes, and
|
|
4
4
|
// every later request (page, probe, robots refresh) checks it first.
|
|
5
5
|
//
|
|
6
6
|
// Storage is an append-only log with no lock. A lock cannot be made safe
|
|
@@ -13,12 +13,19 @@ import crypto from "node:crypto";
|
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { canonOrigin } from "./sitekey.js";
|
|
16
|
-
import {
|
|
16
|
+
import { Refused } from "./errors.js";
|
|
17
|
+
import { appendLine, cachePath, readJson, scanLogSync, rootDir } from "./store.js";
|
|
17
18
|
// A wait over 10 years is not a real Retry-After. It is treated as invalid and
|
|
18
19
|
// replaced by a fixed 24 hour cooldown. Anything below that limit is kept in
|
|
19
20
|
// full, never shortened.
|
|
20
21
|
const ABSURD_SECONDS = 10 * 365 * 86_400;
|
|
21
22
|
export const ABSURD_FALLBACK_SECONDS = 86_400;
|
|
23
|
+
// The three HTTP-date spellings, excluding Date.parse's permissive numeric and ISO inputs.
|
|
24
|
+
const WEEKDAY = "(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)";
|
|
25
|
+
const MONTH = "(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)";
|
|
26
|
+
const CLOCK = "[0-2][0-9]:[0-5][0-9]:[0-5][0-9]";
|
|
27
|
+
const HTTP_DATE = new RegExp(`^(?:${WEEKDAY}, [0-3][0-9] ${MONTH} [0-9]{4} ${CLOCK} GMT|(?:Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday), [0-3][0-9]-${MONTH}-[0-9]{2} ${CLOCK} GMT|${WEEKDAY} ${MONTH} (?: [1-9]|[0-3][0-9]) ${CLOCK} [0-9]{4})$`);
|
|
28
|
+
const httpDate = (s) => HTTP_DATE.test(s) ? Date.parse(s) : NaN;
|
|
22
29
|
// Retry-After is either whole seconds or an HTTP date. Returns null when it
|
|
23
30
|
// is missing, unreadable or absurd.
|
|
24
31
|
export function parseRetryAfter(v) {
|
|
@@ -29,7 +36,7 @@ export function parseRetryAfter(v) {
|
|
|
29
36
|
if (/^\d+$/.test(t))
|
|
30
37
|
secs = Number(t);
|
|
31
38
|
else {
|
|
32
|
-
const when =
|
|
39
|
+
const when = httpDate(t);
|
|
33
40
|
if (Number.isNaN(when))
|
|
34
41
|
return null;
|
|
35
42
|
secs = Math.max(0, Math.ceil((when - Date.now()) / 1000));
|
|
@@ -43,7 +50,7 @@ export function cooldownSeconds(status, header) {
|
|
|
43
50
|
return parsed;
|
|
44
51
|
if (header && header.trim()) {
|
|
45
52
|
const t = header.trim();
|
|
46
|
-
const absurd = /^\d+$/.test(t) || !Number.isNaN(
|
|
53
|
+
const absurd = /^\d+$/.test(t) || !Number.isNaN(httpDate(t));
|
|
47
54
|
if (absurd)
|
|
48
55
|
return ABSURD_FALLBACK_SECONDS;
|
|
49
56
|
}
|
|
@@ -59,8 +66,7 @@ const LEGACY_OLD = "cooldowns.old.jsonl"; // written by v0.1.4, read only, never
|
|
|
59
66
|
const MAX_ORIGIN = 400; // keeps every line well under 512 bytes (one atomic append)
|
|
60
67
|
const mem = new Map(); // cooldowns set by this process (at most MEM_MAX)
|
|
61
68
|
// v0.9 hardening (Codex round 6c, HIGH): `mem` no longer keeps every wait for the server's whole life: past MEM_MAX
|
|
62
|
-
// entries, the expired ones are dropped. A wait that has not ended is never dropped
|
|
63
|
-
// is read only from its last 8 MiB, so it cannot stand in for one).
|
|
69
|
+
// entries, the expired ones are dropped. A wait that has not ended is never dropped.
|
|
64
70
|
// v0.9 client trust (Travis approved, 2026-10-31; closes the class stopped in client-trust round 1): when `mem` is
|
|
65
71
|
// full of waits that have not ended, a new one goes into a FIXED-SIZE store instead of growing memory: SLOTS
|
|
66
72
|
// deadlines (8 MB), each origin written to K slots chosen by a keyed hash (a random key per process, so a site cannot
|
|
@@ -120,9 +126,6 @@ function remember(origin, until) {
|
|
|
120
126
|
}
|
|
121
127
|
export const cooldownStoreBytes = () => sketch?.byteLength ?? 0; // for tests
|
|
122
128
|
export const cooldownMemSize = () => mem.size; // for tests
|
|
123
|
-
// v0.5.1: the log is read again only when one of its files changed (size, mtime or inode), not on every
|
|
124
|
-
// check (a request checks about eleven times). The stamp names the store root, so a new TIRESIAS_HOME
|
|
125
|
-
// is read afresh.
|
|
126
129
|
let memo = null;
|
|
127
130
|
let reads = 0;
|
|
128
131
|
export const loadCount = () => reads; // for tests
|
|
@@ -131,9 +134,11 @@ function stamp() {
|
|
|
131
134
|
for (const f of [path.join(rootDir(), CUR), path.join(rootDir(), LEGACY_OLD), safePath(() => cachePath(LEGACY, "shared"))]) {
|
|
132
135
|
try {
|
|
133
136
|
const st = f ? fs.lstatSync(f) : null;
|
|
134
|
-
parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ino}` : "-");
|
|
137
|
+
parts.push(st ? `${st.size}:${st.mtimeMs}:${st.ctimeMs}:${st.dev}:${st.ino}:${st.mode}` : "-");
|
|
135
138
|
}
|
|
136
|
-
catch {
|
|
139
|
+
catch (e) {
|
|
140
|
+
if (e.code !== "ENOENT")
|
|
141
|
+
throw e;
|
|
137
142
|
parts.push("-");
|
|
138
143
|
}
|
|
139
144
|
}
|
|
@@ -148,26 +153,54 @@ const safePath = (f) => {
|
|
|
148
153
|
}
|
|
149
154
|
};
|
|
150
155
|
function load() {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
try {
|
|
157
|
+
const st = stamp();
|
|
158
|
+
if (memo && memo.stamp === st)
|
|
159
|
+
return memo.map;
|
|
160
|
+
const root = rootDir();
|
|
161
|
+
const map = loadNow(memo?.root === root ? memo.map : undefined);
|
|
162
|
+
memo = { root, stamp: st, map };
|
|
163
|
+
return map;
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
memo = null;
|
|
167
|
+
throw new Refused("The site's Retry-After waits could not be read. No request was sent.");
|
|
168
|
+
}
|
|
157
169
|
}
|
|
158
|
-
//
|
|
159
|
-
|
|
160
|
-
function loadNow() {
|
|
161
|
-
reads++;
|
|
162
|
-
const now = Date.now();
|
|
170
|
+
// Replay every live deadline. Overflow uses the same bounded, conservative slots as the in-process list.
|
|
171
|
+
function newIndex() {
|
|
163
172
|
const best = new Map();
|
|
173
|
+
let overflow = null;
|
|
164
174
|
const take = (origin, until) => {
|
|
165
|
-
if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= now)
|
|
175
|
+
if (typeof origin !== "string" || typeof until !== "number" || !Number.isFinite(until) || until <= Date.now())
|
|
166
176
|
return;
|
|
167
177
|
const key = canonOrigin(origin); // v0.6: one name per site (sitekey.ts)
|
|
168
|
-
if (
|
|
169
|
-
best.
|
|
178
|
+
if (best.has(key) || (best.size < MEM_MAX && key.length <= MAX_ORIGIN)) {
|
|
179
|
+
if (until > (best.get(key) ?? 0))
|
|
180
|
+
best.set(key, until);
|
|
181
|
+
}
|
|
182
|
+
else {
|
|
183
|
+
overflow ??= new Float64Array(SLOTS);
|
|
184
|
+
for (const i of slotsOf(key))
|
|
185
|
+
if (until > overflow[i])
|
|
186
|
+
overflow[i] = until;
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
return {
|
|
190
|
+
take,
|
|
191
|
+
logs: new Map(),
|
|
192
|
+
exact: (origin) => best.get(origin) ?? 0,
|
|
193
|
+
get: (origin) => {
|
|
194
|
+
let least = Infinity;
|
|
195
|
+
if (overflow)
|
|
196
|
+
for (const i of slotsOf(origin))
|
|
197
|
+
least = Math.min(least, overflow[i]);
|
|
198
|
+
return Math.max(best.get(origin) ?? 0, least === Infinity ? 0 : least);
|
|
199
|
+
},
|
|
170
200
|
};
|
|
201
|
+
}
|
|
202
|
+
function loadNow(index = newIndex()) {
|
|
203
|
+
reads++;
|
|
171
204
|
let legacy = null;
|
|
172
205
|
try {
|
|
173
206
|
legacy = readJson(cachePath(LEGACY, "shared")); // W-18: shared, at the root
|
|
@@ -176,13 +209,11 @@ function loadNow() {
|
|
|
176
209
|
// the root's cache/ is a link or someone else's: the legacy store is not read (Codex round 46)
|
|
177
210
|
}
|
|
178
211
|
if (legacy && typeof legacy === "object")
|
|
179
|
-
for (const [
|
|
180
|
-
take(
|
|
181
|
-
for (const
|
|
182
|
-
take(e?.origin, e?.until)
|
|
183
|
-
|
|
184
|
-
take(e?.origin, e?.until);
|
|
185
|
-
return best;
|
|
212
|
+
for (const [origin, until] of Object.entries(legacy))
|
|
213
|
+
index.take(origin, until);
|
|
214
|
+
for (const name of [LEGACY_OLD, CUR])
|
|
215
|
+
index.logs.set(name, scanLogSync(name, (e) => index.take(e?.origin, e?.until), index.logs.get(name)));
|
|
216
|
+
return index;
|
|
186
217
|
}
|
|
187
218
|
// Fetchers register here so queued requests can be cancelled the moment a
|
|
188
219
|
// cooldown starts for their site.
|
|
@@ -193,7 +224,7 @@ export function setCooldown(origin, status, header) {
|
|
|
193
224
|
const until = Date.now() + seconds * 1000;
|
|
194
225
|
// Exact sources only: the fixed-size store may answer later than the truth (a collision), and taking that as
|
|
195
226
|
// "already written" would keep the line from other processes.
|
|
196
|
-
const known = Math.max(mem.get(origin) ?? 0, load().
|
|
227
|
+
const known = Math.max(mem.get(origin) ?? 0, load().exact(origin));
|
|
197
228
|
remember(origin, until); // this process honors it at once
|
|
198
229
|
// Only a later deadline is worth a line. Never shortens anything.
|
|
199
230
|
if (until > known && origin.length <= MAX_ORIGIN)
|
package/dist/dailylog.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// W-13: the daily log. A readable page for one day, built from history.jsonl
|
|
3
3
|
// (and receipts.jsonl) on request. It is never stored as a second copy; the
|
|
4
4
|
// only file it can create is an export the owner asks for with --to.
|
|
5
|
+
import { textView } from "./textview.js";
|
|
5
6
|
import fs from "node:fs";
|
|
6
7
|
import path from "node:path";
|
|
7
8
|
import { pathToFileURL } from "node:url";
|
|
@@ -53,6 +54,9 @@ function outcome(e) {
|
|
|
53
54
|
}
|
|
54
55
|
// One settings change in plain words, from the record's fixed codes (history.ts keeps only known codes).
|
|
55
56
|
function settingWords(e) {
|
|
57
|
+
const by0 = e.by === "terminal" ? "from a terminal (likely a person)" : e.by === "not_terminal" ? "NOT from a terminal (an agent, a script or cron)" : "by an unknown caller";
|
|
58
|
+
if (e.tool === "keys")
|
|
59
|
+
return `${e.event === "slip_key_rotated" ? `slip key rotated${e.revoked !== undefined ? `, ${e.revoked} slip${e.revoked === 1 ? "" : "s"} revoked` : ""}` : e.event === "signing_key_rotation_finished" ? "signing key rotation finished (previous key dropped)" : "signing key rotated"}, ${by0}`;
|
|
56
60
|
const what = e.change === "never_ask" ? "confirmations for risky settings turned OFF (never ask again)"
|
|
57
61
|
: e.change === "ask_again" ? "confirmations for risky settings turned back on"
|
|
58
62
|
: `${e.setting ?? "a setting"}${e.site_key ? ` for ${code(e.site_key)}` : ""} ${e.change === "looser" ? "turned on or loosened" : "turned off or made stricter"}`;
|
|
@@ -64,7 +68,8 @@ const hhmm = (iso) => {
|
|
|
64
68
|
const d = new Date(iso);
|
|
65
69
|
return `${String(d.getHours()).padStart(2, "0")}:${String(d.getMinutes()).padStart(2, "0")}`;
|
|
66
70
|
};
|
|
67
|
-
export async function buildLog(dayText) {
|
|
71
|
+
export async function buildLog(dayText, scan = {}) {
|
|
72
|
+
scan = { ...scan, deadline: scan.deadline ?? Date.now() + 60_000 };
|
|
68
73
|
const { day, start, end } = parseDay(dayText);
|
|
69
74
|
const entries = [];
|
|
70
75
|
const changes = []; // W-32 (v0.9): risky settings changed this day get their own section, not counted as lookups
|
|
@@ -96,7 +101,8 @@ export async function buildLog(dayText) {
|
|
|
96
101
|
}
|
|
97
102
|
return false;
|
|
98
103
|
}
|
|
99
|
-
if (e.tool === "settings") {
|
|
104
|
+
if (e.tool === "settings" || e.tool === "keys") {
|
|
105
|
+
// v1.1: a key rotation is listed with the settings changes (it is just as risky)
|
|
100
106
|
changeCount++;
|
|
101
107
|
if (changes.length < MAX_LOG_LOOKUPS)
|
|
102
108
|
changes.push(e);
|
|
@@ -112,7 +118,7 @@ export async function buildLog(dayText) {
|
|
|
112
118
|
otherAgents++;
|
|
113
119
|
results.set(outcome(e), (results.get(outcome(e)) ?? 0) + 1); // a fixed set of outcomes
|
|
114
120
|
return false;
|
|
115
|
-
});
|
|
121
|
+
}, 1, PAGE_BUDGET, scan);
|
|
116
122
|
entries.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
|
|
117
123
|
changes.sort((a, b) => Date.parse(a.time) - Date.parse(b.time));
|
|
118
124
|
const perUrl = new Map();
|
|
@@ -136,7 +142,7 @@ export async function buildLog(dayText) {
|
|
|
136
142
|
}
|
|
137
143
|
}
|
|
138
144
|
return false;
|
|
139
|
-
});
|
|
145
|
+
}, 1, PAGE_BUDGET, scan);
|
|
140
146
|
const link = (name) => `[${name}](${pathToFileURL(path.join(homeDir(), name)).href})`;
|
|
141
147
|
const days = [...utcDays].sort().join(", ");
|
|
142
148
|
const lines = [];
|
|
@@ -234,7 +240,7 @@ export async function exportLog(folder, dayText) {
|
|
|
234
240
|
throw new Error("Private mode: nothing is written, so the log is not exported. Run it without --to to print it.");
|
|
235
241
|
// 1. Build the whole text first. Nothing on disk is touched until it exists.
|
|
236
242
|
const log = await buildLog(dayText);
|
|
237
|
-
const body = fullText(log);
|
|
243
|
+
const body = textView(fullText(log));
|
|
238
244
|
// 2. Resolve the folder ONCE and use only that real path from here on.
|
|
239
245
|
let real;
|
|
240
246
|
try {
|
package/dist/engine.js
CHANGED
|
@@ -16,11 +16,12 @@ export * from "./watchflow.js";
|
|
|
16
16
|
export * from "./resume.js";
|
|
17
17
|
export * from "./formflow.js";
|
|
18
18
|
export const read = async (url, opts = {}, ctx = DEFAULT) => {
|
|
19
|
+
const saveFallback = opts.save ? saveRefusal() : undefined;
|
|
19
20
|
const out = await guarded("read", url, ctx, opts.question, (m) => run("read", url, opts, m, ctx.agent), {}, { create: { question: opts.question, section: opts.section, render: opts.render, translate: opts.translate } });
|
|
20
21
|
// W-28 (v0.8): a read that asked to save always says what happened to the save (the pipeline sets it when it got
|
|
21
22
|
// as far as a page or a file).
|
|
22
23
|
if (opts.save && !out.saved)
|
|
23
|
-
addSaved(out, out.private ? { status: "private" } :
|
|
24
|
+
addSaved(out, out.private ? { status: "private" } : saveFallback);
|
|
24
25
|
return out;
|
|
25
26
|
};
|
|
26
27
|
function saveRefusal() {
|
|
@@ -55,13 +56,13 @@ export function slipList(page = 1) {
|
|
|
55
56
|
export const claimSlip = slips.claimSlip;
|
|
56
57
|
export const checkSlip = slips.checkSlip;
|
|
57
58
|
export const queryHistory = history.query;
|
|
58
|
-
export const dailyLog = async (day, page = 1) => {
|
|
59
|
-
const log = await dailylog.buildLog(day);
|
|
59
|
+
export const dailyLog = async (day, page = 1, scan = {}) => {
|
|
60
|
+
const log = await dailylog.buildLog(day, scan);
|
|
60
61
|
return { day: log.day, entries: log.entries, ...dailylog.pageOf(log, page) };
|
|
61
62
|
};
|
|
62
63
|
export const exportDailyLog = dailylog.exportLog;
|
|
63
64
|
// W-12: generated from the history when asked, never stored.
|
|
64
|
-
export const needsReport = (since, page = 1) => needs.buildNeeds(since, page);
|
|
65
|
+
export const needsReport = (since, page = 1, scan = {}) => needs.buildNeeds(since, page, Date.now(), scan);
|
|
65
66
|
// W-19: tiresias safety status | update.
|
|
66
67
|
export const safetyStatus = safebrowsing.status;
|
|
67
68
|
export const safetyUpdate = safebrowsing.update;
|
package/dist/envelope.js
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
// Reply building: the envelope, its budget, the error map, the handoff ticket for a wall, and guarded(),
|
|
3
|
+
// the wrapper every tool call runs in (deadline, Safe Browsing, private mode, history).
|
|
4
|
+
// (Split out of engine.ts in v0.5.1, behavior unchanged.)
|
|
5
|
+
import { snapshotConfig } from "./config.js";
|
|
1
6
|
import { TiresiasError } from "./errors.js";
|
|
2
7
|
import * as handoffs from "./handoffs.js";
|
|
3
8
|
import * as history from "./history.js";
|
|
@@ -202,7 +207,18 @@ export function errorEnv(url, e) {
|
|
|
202
207
|
return env(url, "error", { message: e.message });
|
|
203
208
|
}
|
|
204
209
|
}
|
|
205
|
-
export async function guarded(
|
|
210
|
+
export async function guarded(...args) {
|
|
211
|
+
let config;
|
|
212
|
+
let configError;
|
|
213
|
+
try {
|
|
214
|
+
config = snapshotConfig();
|
|
215
|
+
}
|
|
216
|
+
catch (e) {
|
|
217
|
+
configError = e;
|
|
218
|
+
}
|
|
219
|
+
return callContext.run({ ...callContext.getStore(), config, configError, getActions: undefined }, () => guardedCall(...args));
|
|
220
|
+
}
|
|
221
|
+
async function guardedCall(tool, url, ctx, purpose, fn, meta = {}, hand, safetyCheck = true, // W-19: false only for a page a person provided (nothing is requested)
|
|
206
222
|
record = true) {
|
|
207
223
|
const priv = !!ctx.private || sessionPrivate();
|
|
208
224
|
// W-15: the once-a-day trim. Not in private mode (a private call leaves the store as it found it).
|
|
@@ -218,6 +234,8 @@ record = true) {
|
|
|
218
234
|
try {
|
|
219
235
|
// W-19: the asked URL is checked first, before robots.txt or anything else, and before a copy is
|
|
220
236
|
// served from the cache. Every later hop on a new host is checked by the fetcher (safety.ts).
|
|
237
|
+
if (callContext.getStore()?.configError)
|
|
238
|
+
throw callContext.getStore().configError;
|
|
221
239
|
assertProfile(); // W-18: a profile that does not exist fails here, by name, before anything else
|
|
222
240
|
gate = safetyCheck ? safebrowsing.makeGate({ allowFlagged: !!ctx.allow_flagged, priv, readOnly: !record }) : undefined;
|
|
223
241
|
const g = gate;
|
|
@@ -265,7 +283,7 @@ record = true) {
|
|
|
265
283
|
return out;
|
|
266
284
|
try {
|
|
267
285
|
history.record({
|
|
268
|
-
time: new Date().toISOString(), tool, url, agent: ctx.agent, purpose
|
|
286
|
+
time: new Date().toISOString(), tool, url, agent: ctx.agent, purpose, status: out.status, door: out.door,
|
|
269
287
|
reason: out.handoff?.reason, matched: meta.matched, best_door: meta.best_door, llms_txt: meta.llms_txt,
|
|
270
288
|
ticket: meta.ticket, source: meta.source, cache: meta.cache, flag: meta.flag, slip: meta.slip, file_kind: meta.file_kind, scan: meta.scan,
|
|
271
289
|
lang: out.language?.code, // W-26 (v0.7)
|
package/dist/fetcher.js
CHANGED
|
@@ -43,6 +43,11 @@ async function politeFetchHeld(url, opts, turn) {
|
|
|
43
43
|
// Record the wait the moment 429/503 headers arrive, even if the body never finishes.
|
|
44
44
|
if (hop.status === 429 || hop.status === 503)
|
|
45
45
|
setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
46
|
+
if (hop.redirect && hop.headers.has("retry-after")) {
|
|
47
|
+
const seconds = setCooldown(hop.url.origin, hop.status, hop.headers.get("retry-after"));
|
|
48
|
+
if (seconds > 0)
|
|
49
|
+
throw new Refused(COOLDOWN_TEXT, undefined, seconds, hop.url.href);
|
|
50
|
+
}
|
|
46
51
|
// A challenge (cf-mitigated) on ANY hop is a wall at that hop: a redirect is not
|
|
47
52
|
// followed, and a 304 is not accepted as a revalidation. Plain 401/402/403 final
|
|
48
53
|
// responses are handled with their body by the engine.
|
package/dist/filereply.js
CHANGED
|
@@ -15,7 +15,8 @@ import { NONBLOCK, isPrivate } from "./store.js";
|
|
|
15
15
|
import { MAX_BLOCK } from "./reader.js";
|
|
16
16
|
import { MAX_SAMPLE, detect, stampLanguage } from "./lang.js";
|
|
17
17
|
import { translateReply } from "./translate.js";
|
|
18
|
-
import { SecretWithheld, valueHoldsSecret } from "./echo.js";
|
|
18
|
+
import { SecretWithheld, noSecrets, valueHoldsSecret } from "./echo.js";
|
|
19
|
+
import { runUntrusted } from "./sandbox.js";
|
|
19
20
|
import { BUDGET, env } from "./envelope.js";
|
|
20
21
|
export class ProvidedError extends TiresiasError {
|
|
21
22
|
kind = "provided";
|
|
@@ -81,6 +82,11 @@ export async function fileWarning(url, finalUrl, f, meta, agent, source, opts =
|
|
|
81
82
|
}
|
|
82
83
|
// v0.6: inside a zip or an Office file, in the worker, without unpacking it (inside.ts).
|
|
83
84
|
const ins = await inspectFile(f);
|
|
85
|
+
if (ins?.office_text && !noSecrets(secrets)) {
|
|
86
|
+
const texts = [JSON.stringify(ins.office_text), ins.office_text.map((s) => s.text).join("\n")];
|
|
87
|
+
if (await runUntrusted({ op: "echo", texts, secrets: secrets }, Infinity))
|
|
88
|
+
throw new SecretWithheld();
|
|
89
|
+
}
|
|
84
90
|
if (macroByName(f) || ins?.macros)
|
|
85
91
|
fw.flags = [...new Set([...fw.flags, "macros"])];
|
|
86
92
|
if (ins) {
|
package/dist/files.js
CHANGED
|
@@ -75,7 +75,7 @@ export function signature(b, tail) {
|
|
|
75
75
|
return { family: "archive", kind: "addon" }; // Chrome extension
|
|
76
76
|
// Archives and disk images
|
|
77
77
|
if (at(b, 0, "PK\x03\x04") || at(b, 0, "PK\x05\x06") || at(b, 0, "PK\x07\x08"))
|
|
78
|
-
return { family: "archive", kind: "archive" }; // zip and its family
|
|
78
|
+
return { family: "archive", kind: "archive", zip: true }; // zip and its family
|
|
79
79
|
if (at(b, 0, "Rar!\x1a\x07"))
|
|
80
80
|
return { family: "archive", kind: "archive" };
|
|
81
81
|
if (at(b, 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]))
|
|
@@ -350,11 +350,11 @@ function doubleExtension(name) {
|
|
|
350
350
|
}
|
|
351
351
|
// A zip, OLE or Chrome add-on is one family with several kinds. The kind inside the family is taken from
|
|
352
352
|
// what the response says it is, since nothing inside the file is read.
|
|
353
|
-
function refine(kind, sigFamily, ct, names) {
|
|
353
|
+
function refine(kind, sigFamily, ct, names, zip = false) {
|
|
354
354
|
const exts = [names.disposition, ...(names.others ?? []), names.url].map(lastExt).filter((x) => !!x);
|
|
355
355
|
if (sigFamily === "ole")
|
|
356
356
|
return exts.includes("msi") || ct === "application/x-msi" ? "program" : "office";
|
|
357
|
-
if (kind !== "archive" ||
|
|
357
|
+
if (kind !== "archive" || !zip)
|
|
358
358
|
return kind;
|
|
359
359
|
if (exts.some((e) => ADDON_EXT.has(e)) || ct === "application/x-xpinstall" || ct === "application/x-chrome-extension")
|
|
360
360
|
return "addon";
|
|
@@ -438,7 +438,7 @@ function decide(head, m, names, sig) {
|
|
|
438
438
|
return { file: "unknown", flags: [] };
|
|
439
439
|
}
|
|
440
440
|
if (sig && sig.kind !== "script")
|
|
441
|
-
return { file: refine(sig.kind, sig.family, m, names), flags: [] };
|
|
441
|
+
return { file: refine(sig.kind, sig.family, m, names, sig.zip), flags: [] };
|
|
442
442
|
if (!sig && m === "application/pdf")
|
|
443
443
|
return { read: "pdf", flags: [] }; // W-17: the declared type alone (a broken PDF fails to read)
|
|
444
444
|
if (looksBinary(head))
|
package/dist/formflow.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// W-04: forms as permission-gated actions (form_preview, form_submit).
|
|
3
3
|
// (Split out of engine.ts in v0.5.1, behavior unchanged.)
|
|
4
|
+
import { noteGetActions } from "./getactions.js";
|
|
4
5
|
import { assertSiteAllowed } from "./sites.js";
|
|
5
6
|
import { cooldownSeconds } from "./cooldown.js";
|
|
6
|
-
import { COOLDOWN_TEXT, CRAWL_DELAY_TEXT, policyCheck, politeFetch } from "./fetcher.js";
|
|
7
|
+
import { COOLDOWN_TEXT, CRAWL_DELAY_TEXT, policyCheck, politeFetch, Refused } from "./fetcher.js";
|
|
7
8
|
import { runUntrusted } from "./sandbox.js";
|
|
8
9
|
import * as receipts from "./receipts.js";
|
|
9
10
|
import { assertPublicUrl, currentGate, currentSent, parseSafeUrl, throwIfCancelled } from "./safety.js";
|
|
@@ -21,7 +22,7 @@ import { buildContent, csvCount, detailOf, earlyOutcome, looksHtml, parsePage, r
|
|
|
21
22
|
// form_preview: fetch the page fresh (robots, waits, Safe Browsing and signing, like read with fresh),
|
|
22
23
|
// read its forms, check the agent's values, and show the EXACT request that would be sent, hidden
|
|
23
24
|
// values replaced by "[set by the site]", with a preview hash and whether a slip covers the target.
|
|
24
|
-
//
|
|
25
|
+
// Keeps no page record; shared politeness state still applies. A gated GET read can spend a slip use.
|
|
25
26
|
//
|
|
26
27
|
// form_submit: in ONE call, fetch the page fresh again, re-read the form, rebuild the request and its
|
|
27
28
|
// hash; a different hash is form_changed and nothing is sent. Then the TARGET is checked (Safe
|
|
@@ -67,6 +68,7 @@ export async function loadForm(url, index, jar) {
|
|
|
67
68
|
return formEnv(url, "error", { index, reason: "not_html" }, NOT_HTML, { final_url: res.url });
|
|
68
69
|
const r = await runUntrusted({ op: "forms", body: res.body, url: res.url }, res.body.length);
|
|
69
70
|
throwIfCancelled();
|
|
71
|
+
noteGetActions(r.forms);
|
|
70
72
|
if (r.wall)
|
|
71
73
|
return wallEnv(url, r.wall, res.url, { final_url: res.url });
|
|
72
74
|
const form = r.forms.find((f) => f.index === index);
|
|
@@ -186,13 +188,23 @@ export async function afterSubmit(url, top, res, secrets, meta, agent) {
|
|
|
186
188
|
if (!k)
|
|
187
189
|
return formEnv(url, "ok", top, `${SENT} The site accepted it (HTTP ${res.status}). Its answer was empty.`, uc);
|
|
188
190
|
// The answer is read like a page (outline with receipts), in the worker, and never cached.
|
|
189
|
-
|
|
191
|
+
let page;
|
|
192
|
+
try {
|
|
193
|
+
page = await parsePage(k, res.body, res.bytes, res.url, res.fileFlags, looksHtml(res.headers.get("content-type"), res.body), res.headers.get("content-language"), undefined, secrets);
|
|
194
|
+
}
|
|
195
|
+
catch (err) {
|
|
196
|
+
if (err instanceof SecretWithheld)
|
|
197
|
+
return withheldEnv(url, top, res.status, answerSize(res));
|
|
198
|
+
throw err;
|
|
199
|
+
}
|
|
200
|
+
const { parsed, wall } = page;
|
|
190
201
|
// Check the full extracted text, before previews can cut off a long secret or receipts can be recorded.
|
|
191
202
|
// Joining blocks also covers a secret split by the reader's paragraph-size cap.
|
|
192
203
|
if (!noSecrets(secrets) && await answerHoldsSecret([
|
|
193
204
|
JSON.stringify(parsed), parsed.sections.flatMap((s) => s.blocks.map((b) => b.text)).join("\n"),
|
|
194
205
|
], secrets))
|
|
195
206
|
return withheldEnv(url, top, res.status, answerSize(res));
|
|
207
|
+
noteGetActions(parsed.forms);
|
|
196
208
|
if (wall) {
|
|
197
209
|
const e = wallEnv(url, wall, res.url, uc);
|
|
198
210
|
e.message = `${SENT} The page that came back shows a wall: a person has to take it from here.`;
|
|
@@ -272,6 +284,12 @@ export async function formSubmit(url, index, values, previewHash, ctx = DEFAULT)
|
|
|
272
284
|
method: b.method,
|
|
273
285
|
body: b.body,
|
|
274
286
|
submit: true,
|
|
287
|
+
beforeSend: (u) => {
|
|
288
|
+
if (!slips.claimedStillValid(claim, "submit_form", u.href)) {
|
|
289
|
+
top.reason = "no_slip";
|
|
290
|
+
throw new Refused("The claimed permission slip expired or was revoked before sending. Nothing was sent; the reserved use stays spent.");
|
|
291
|
+
}
|
|
292
|
+
},
|
|
275
293
|
cookies: jar,
|
|
276
294
|
files: fileOptions(), // W-20: an answer that is a file is fingerprinted and flagged
|
|
277
295
|
// Honest about where the form came from: the page's origin and address, as a browser sends them.
|