@raibitsi/tiresias 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/README.md +27 -11
- package/data/README.md +33 -0
- package/data/guard-ordinary-urls.txt +102 -0
- package/data/guard-starter.jsonl +424 -0
- package/dist/aimed.js +23 -0
- package/dist/anchor.js +26 -13
- package/dist/browsecli.js +16 -13
- package/dist/cleanup.js +3 -1
- package/dist/cli.js +144 -49
- package/dist/config.js +60 -1
- package/dist/confirm.js +15 -4
- package/dist/dom.js +13 -8
- package/dist/doors.js +1 -1
- package/dist/engine.js +6 -1
- package/dist/envelope.js +10 -1
- package/dist/fetcher.js +24 -2
- package/dist/forms.js +80 -8
- package/dist/getactions.js +28 -8
- package/dist/guard.js +1240 -0
- package/dist/guardclient.js +245 -0
- package/dist/guardnetwork.js +22 -0
- package/dist/history.js +4 -1
- package/dist/hosts.js +24 -17
- package/dist/local.js +537 -0
- package/dist/mcp.js +46 -15
- package/dist/pipeline.js +106 -24
- package/dist/preformatted.js +12 -0
- package/dist/queue.js +20 -18
- package/dist/reader.js +441 -87
- package/dist/receipts.js +5 -0
- package/dist/render.js +65 -15
- package/dist/renderactions.js +69 -0
- package/dist/rules.js +14 -2
- package/dist/safety.js +6 -4
- package/dist/settings.js +5 -2
- package/dist/shield.js +15 -0
- package/dist/textview.js +8 -2
- package/dist/types.js +9 -2
- package/dist/work.js +6 -0
- package/docs/LOCAL-MODELS.md +271 -0
- package/package.json +5 -2
package/dist/confirm.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import fs from "node:fs";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { callContext } from "./context.js";
|
|
16
|
-
import { SECURITY_CHOICES, parseChoice, DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
|
|
16
|
+
import { SECURITY_CHOICES, parseChoice, DEFAULT_CALENDARS, parseAnchoring, parseCalendars, parseGuard, parseSafeBrowsing, parseSaving, parseScanner, parseTranslator, readConfigObject, writeConfig } from "./config.js";
|
|
17
17
|
import { parseRender } from "./render.js";
|
|
18
18
|
import { ConfigError } from "./errors.js";
|
|
19
19
|
import * as history from "./history.js";
|
|
@@ -46,6 +46,8 @@ const calendarSet = (v) => {
|
|
|
46
46
|
}
|
|
47
47
|
};
|
|
48
48
|
const renderRank = rank(["off", "ask", "auto"]);
|
|
49
|
+
const publicGuard = (v) => !!v && typeof v === "object" && v.allow_public === true;
|
|
50
|
+
const guardUrl = (v) => (v && typeof v === "object" ? String(v.url) : "");
|
|
49
51
|
export const TOP = [
|
|
50
52
|
{ key: "private_spacing", looser: (c, n) => c !== "none" && n === "none", phrase: () => "YES, TURN OFF PRIVATE SPACING", what: () => "private spacing across processes turned off" },
|
|
51
53
|
{ key: "hook_secrets", looser: (c, n) => c !== "inherit" && n === "inherit", phrase: () => "YES, SHARE TIRESIAS SECRETS WITH HOOKS", what: () => "Tiresias secrets shared with hooks" },
|
|
@@ -70,7 +72,11 @@ export const TOP = [
|
|
|
70
72
|
what: () => "anchor calendars were added",
|
|
71
73
|
},
|
|
72
74
|
{ key: "safe_browsing", looser: (c, n) => c !== false && n === false, phrase: () => "YES, TURN OFF SAFE BROWSING", what: () => "Safe Browsing turned off" },
|
|
75
|
+
// W-38 (v1.2): the meaning check sends page text to the owner's embedding endpoint. Sending it to an address
|
|
76
|
+
// outside the owner's network (allow_public) is the loosening; a new address with it set needs the phrase again.
|
|
77
|
+
{ key: "guard", looser: (c, n) => publicGuard(n) && !(publicGuard(c) && guardUrl(c) === guardUrl(n)), phrase: () => GUARD_PHRASE, what: () => "the meaning check was allowed to send page text outside your network" },
|
|
73
78
|
];
|
|
79
|
+
export const GUARD_PHRASE = "YES, SEND PAGE TEXT OUTSIDE MY NETWORK";
|
|
74
80
|
const TOP_BY = new Map(TOP.map((t) => [t.key, t]));
|
|
75
81
|
export const RISKY_KEYS = TOP.map((t) => t.key);
|
|
76
82
|
export const isRisky = (name) => TOP_BY.has(name) || SITE_RISKY.includes(name);
|
|
@@ -426,6 +432,8 @@ function validTop(key, v) {
|
|
|
426
432
|
parseCalendars(v);
|
|
427
433
|
else if (key === "safe_browsing")
|
|
428
434
|
parseSafeBrowsing(v);
|
|
435
|
+
else if (key === "guard")
|
|
436
|
+
parseGuard(v);
|
|
429
437
|
return true;
|
|
430
438
|
}
|
|
431
439
|
catch {
|
|
@@ -593,7 +601,7 @@ export class ConfirmError extends ConfigError {
|
|
|
593
601
|
}
|
|
594
602
|
// ---- the notice in replies ----
|
|
595
603
|
// Fixed text only (setting names, validated site names, times, fixed words). Undefined when there is nothing to say.
|
|
596
|
-
export function notice(now = Date.now()) {
|
|
604
|
+
export function notice(now = Date.now(), includeGuard = true) {
|
|
597
605
|
try {
|
|
598
606
|
const lines = [];
|
|
599
607
|
const { rec, state } = loadRecord();
|
|
@@ -606,12 +614,15 @@ export function notice(now = Date.now()) {
|
|
|
606
614
|
catch {
|
|
607
615
|
raw = null; // a bad config.json is reported by the calls that use it
|
|
608
616
|
}
|
|
609
|
-
|
|
617
|
+
const effective = clamp(raw, rec);
|
|
618
|
+
// Guard notices belong only to local replies while the check is set, so off restores their former output.
|
|
619
|
+
const showGuard = includeGuard && !!effective.obj?.guard;
|
|
620
|
+
for (const w of effective.waiting.filter((w) => w.setting !== "guard" || showGuard))
|
|
610
621
|
lines.push(`config.json loosens ${w.setting}${w.site ? ` for ${w.site}` : ""}, but that was not confirmed, so it is not in force. To confirm: ${w.command}`);
|
|
611
622
|
for (const e of envNotices(rec, raw))
|
|
612
623
|
lines.push(e);
|
|
613
624
|
const since = now - NOTICE_HOURS * 3_600_000;
|
|
614
|
-
const recent = rec.recent.filter((c) => c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
|
|
625
|
+
const recent = rec.recent.filter((c) => (c.setting !== "guard" || showGuard) && c.change !== "stricter" && Date.parse(c.at) >= since).reverse();
|
|
615
626
|
for (const c of recent)
|
|
616
627
|
lines.push(`${c.at}: ${describe(c)}. See tiresias history.`);
|
|
617
628
|
if (!lines.length)
|
package/dist/dom.js
CHANGED
|
@@ -4,17 +4,19 @@
|
|
|
4
4
|
import { DomHandler, Parser } from "htmlparser2";
|
|
5
5
|
export const MAX_DOM_DEPTH = 256;
|
|
6
6
|
export function parseDocument(body) {
|
|
7
|
-
const handler = new DomHandler();
|
|
7
|
+
const handler = new DomHandler(undefined, { withStartIndices: true });
|
|
8
|
+
const source = new WeakMap();
|
|
8
9
|
const opens = [];
|
|
9
10
|
// The bounded parser can forget an ancestor before its source end tag. Keep source scopes with
|
|
10
11
|
// push/pop (not htmlparser2's shifting stacks), so flattening cannot turn hidden descendants into
|
|
11
12
|
// shown siblings. Each attribute set refers to its original parent; the reader resolves these in
|
|
12
13
|
// source order, once the page's style sheet is known, without copying attributes or walking chains.
|
|
13
|
-
const inherited = [];
|
|
14
|
+
const inherited = []; // third entry: the source parent is inside code/pre
|
|
14
15
|
const scopes = [];
|
|
15
16
|
const byName = new Map();
|
|
16
17
|
const popScope = () => {
|
|
17
18
|
const s = scopes.pop();
|
|
19
|
+
source.get(s.attrs).end = parser.startIndex;
|
|
18
20
|
if (s.previous === undefined)
|
|
19
21
|
byName.delete(s.name);
|
|
20
22
|
else
|
|
@@ -28,12 +30,14 @@ export function parseDocument(body) {
|
|
|
28
30
|
const close = handler.onclosetag.bind(handler);
|
|
29
31
|
const text = handler.ontext.bind(handler);
|
|
30
32
|
handler.onopentag = (name, attrs) => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
+
const parent = scopes[scopes.length - 1];
|
|
34
|
+
source.set(attrs, { parent: parent?.attrs });
|
|
35
|
+
if (parent)
|
|
36
|
+
inherited.push([attrs, parent.attrs, parent.code]);
|
|
33
37
|
const key = name.toLowerCase();
|
|
34
38
|
const previous = byName.get(key);
|
|
35
39
|
byName.set(key, scopes.length);
|
|
36
|
-
scopes.push({ name: key, attrs, previous });
|
|
40
|
+
scopes.push({ name: key, attrs, code: key === "code" || key === "pre" || parent?.code === true, previous });
|
|
37
41
|
opens.push({ ...bounds, attrs });
|
|
38
42
|
open(name, attrs);
|
|
39
43
|
};
|
|
@@ -51,7 +55,7 @@ export function parseDocument(body) {
|
|
|
51
55
|
return text(data);
|
|
52
56
|
// Text after a flattened child's close (including raw text) has no element to inherit on.
|
|
53
57
|
const attrs = {};
|
|
54
|
-
inherited.push([attrs, scope.attrs]);
|
|
58
|
+
inherited.push([attrs, scope.attrs, scope.code]);
|
|
55
59
|
open("thidden", attrs);
|
|
56
60
|
text(data);
|
|
57
61
|
close();
|
|
@@ -87,6 +91,7 @@ export function parseDocument(body) {
|
|
|
87
91
|
popScope();
|
|
88
92
|
}
|
|
89
93
|
}
|
|
90
|
-
new BoundedParser(handler)
|
|
91
|
-
|
|
94
|
+
const parser = new BoundedParser(handler);
|
|
95
|
+
parser.end(body);
|
|
96
|
+
return { doc: handler.root, flattened, inherited, source };
|
|
92
97
|
}
|
package/dist/doors.js
CHANGED
|
@@ -40,7 +40,7 @@ export const noStore = (h) => /(^|[\s,])no-store\b/i.test(h.get("cache-control")
|
|
|
40
40
|
// is not "no door": it throws, so the sequential checks stop and nothing is cached (the next read asks again).
|
|
41
41
|
async function probe(url, maxBytes, ctl, strict = false) {
|
|
42
42
|
try {
|
|
43
|
-
const res = await politeFetch(url, { maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
|
|
43
|
+
const res = await politeFetch(url, { robotsPass: false, maxBytes, truncate: true, timeoutMs: 8000, signal: ctl.signal });
|
|
44
44
|
if (res.status === 402)
|
|
45
45
|
throw new PaymentRequired(res.headers, res.body, res.url);
|
|
46
46
|
if (strict && (res.status >= 500 || res.status === 429))
|
package/dist/engine.js
CHANGED
|
@@ -17,7 +17,12 @@ export * from "./resume.js";
|
|
|
17
17
|
export * from "./formflow.js";
|
|
18
18
|
export const read = async (url, opts = {}, ctx = DEFAULT) => {
|
|
19
19
|
const saveFallback = opts.save ? saveRefusal() : undefined;
|
|
20
|
-
const out = await guarded("read", url, ctx, opts.question, (m) =>
|
|
20
|
+
const out = await guarded("read", url, ctx, opts.question, async (m) => {
|
|
21
|
+
const e = await run("read", url, opts, m, ctx.agent);
|
|
22
|
+
if (opts.budget !== undefined)
|
|
23
|
+
await opts.beforeLocalRecord?.(e, m);
|
|
24
|
+
return e;
|
|
25
|
+
}, {}, { create: { question: opts.question, section: opts.section, render: opts.render, translate: opts.translate } });
|
|
21
26
|
// W-28 (v0.8): a read that asked to save always says what happened to the save (the pipeline sets it when it got
|
|
22
27
|
// as far as a page or a file).
|
|
23
28
|
if (opts.save && !out.saved)
|
package/dist/envelope.js
CHANGED
|
@@ -80,6 +80,11 @@ export const receiptsEnvelope = (url, r) => {
|
|
|
80
80
|
};
|
|
81
81
|
export const historyEnvelope = (r) => listEnvelope("", { history: r.items }, r);
|
|
82
82
|
export const STOPPED = "Stopped. Tiresias does not retry or work around walls.";
|
|
83
|
+
// W-39 (v1.2): the two values of a local passage's written_by, the wording measured in the lab.
|
|
84
|
+
export const BY_SITE = "the site";
|
|
85
|
+
export const BY_VISITOR = "a visitor to the site, not the site itself";
|
|
86
|
+
// What one label adds to a passage at most (the longer value, its key and a comma), for the reply budget.
|
|
87
|
+
export const LABEL_BYTES = jsonBytes({ written_by: BY_VISITOR }) - 2 + 1;
|
|
83
88
|
// W-02 fixed texts.
|
|
84
89
|
export const NO_BASELINE = "There was nothing to compare against: no earlier copy of this page is kept (first visit, a cleared or expired cache, or private mode).";
|
|
85
90
|
export const NO_STORE_DIFF = "There was nothing to compare against: the site asks that this page not be stored (no-store), so no copy is kept.";
|
|
@@ -216,7 +221,10 @@ export async function guarded(...args) {
|
|
|
216
221
|
catch (e) {
|
|
217
222
|
configError = e;
|
|
218
223
|
}
|
|
219
|
-
|
|
224
|
+
// One holder per tool/CLI call, before withSignal/runPrivate make shallow copies. Background door
|
|
225
|
+
// jobs create their own context without a holder, and an in-flight robots waiter gets no pass.
|
|
226
|
+
return callContext.run({ ...callContext.getStore(), config, configError, getActions: undefined,
|
|
227
|
+
robotsPasses: { pending: new Map(), attempted: new Set() } }, () => guardedCall(...args));
|
|
220
228
|
}
|
|
221
229
|
async function guardedCall(tool, url, ctx, purpose, fn, meta = {}, hand, safetyCheck = true, // W-19: false only for a page a person provided (nothing is requested)
|
|
222
230
|
record = true) {
|
|
@@ -289,6 +297,7 @@ record = true) {
|
|
|
289
297
|
lang: out.language?.code, // W-26 (v0.7)
|
|
290
298
|
rendered: meta.rendered, // W-10 (v0.7)
|
|
291
299
|
saved: meta.saved, // W-28 (v0.8)
|
|
300
|
+
written_by: meta.written_by, // W-39 (v1.2)
|
|
292
301
|
});
|
|
293
302
|
}
|
|
294
303
|
catch (e) {
|
package/dist/fetcher.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
// SPDX-License-Identifier: Apache-2.0
|
|
2
2
|
// The polite fetcher: safety (safety.ts) plus the site's rules (rules.ts).
|
|
3
3
|
// Every request except robots.txt itself goes through politeFetch.
|
|
4
|
+
import { callContext } from "./context.js";
|
|
4
5
|
import { cooldownLeft, setCooldown } from "./cooldown.js";
|
|
5
6
|
import { Refused, Walled } from "./errors.js";
|
|
6
7
|
import { spacingMs } from "./hosts.js";
|
|
7
8
|
import { Turns, cooldownRefusal } from "./queue.js";
|
|
8
9
|
import { checkAllowed } from "./rules.js";
|
|
9
10
|
import { assertSiteAllowed } from "./sites.js";
|
|
11
|
+
import { canonOrigin } from "./sitekey.js";
|
|
10
12
|
import { safeFetch } from "./safety.js";
|
|
11
13
|
import { detectWall } from "./walls.js";
|
|
12
14
|
export { Refused, Walled } from "./errors.js";
|
|
@@ -37,6 +39,9 @@ export async function politeFetch(url, opts = {}) {
|
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
async function politeFetchHeld(url, opts, turn) {
|
|
42
|
+
const holder = callContext.getStore()?.robotsPasses;
|
|
43
|
+
const canClaim = opts.robotsPass !== false && !opts.submit && opts.method !== "POST";
|
|
44
|
+
let first = true;
|
|
40
45
|
const res = await safeFetch(url, {
|
|
41
46
|
...opts,
|
|
42
47
|
onHeaders: (hop) => {
|
|
@@ -58,7 +63,24 @@ async function politeFetchHeld(url, opts, turn) {
|
|
|
58
63
|
},
|
|
59
64
|
beforeRequest: async (u, signal) => {
|
|
60
65
|
turn.leaveOthers(u);
|
|
61
|
-
const
|
|
66
|
+
const origin = canonOrigin(u.origin);
|
|
67
|
+
const eligible = first && canClaim && holder && !holder.attempted.has(origin);
|
|
68
|
+
first = false;
|
|
69
|
+
// Claim before awaiting policy, even with no pending pass: only the first eligible fetch to
|
|
70
|
+
// this origin may use one. The policy check itself may fetch robots on direct fetcher paths.
|
|
71
|
+
if (eligible)
|
|
72
|
+
holder.attempted.add(origin);
|
|
73
|
+
let p;
|
|
74
|
+
let pass;
|
|
75
|
+
try {
|
|
76
|
+
p = await policyCheck(u);
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
if (eligible) {
|
|
80
|
+
pass = holder.pending.get(origin);
|
|
81
|
+
holder.pending.delete(origin); // consumed on admission or refusal, used or not
|
|
82
|
+
}
|
|
83
|
+
}
|
|
62
84
|
if (!p.ok && "wall" in p)
|
|
63
85
|
throw new Walled(p.wall, p.wallUrl);
|
|
64
86
|
if (!p.ok)
|
|
@@ -66,7 +88,7 @@ async function politeFetchHeld(url, opts, turn) {
|
|
|
66
88
|
const d = p;
|
|
67
89
|
// v0.5.1: every request is queued and spaced, by at least the default spacing (1 s).
|
|
68
90
|
// W-25 (v0.6): a site's own spacing raises it further, never lowers it (applied in the queue, queue.ts).
|
|
69
|
-
await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal);
|
|
91
|
+
await turn.take(u, Math.max(d.crawlDelay, spacingMs() / 1000), signal, { pass: p.crawlDelay === 0 ? pass : undefined });
|
|
70
92
|
},
|
|
71
93
|
});
|
|
72
94
|
return res;
|
package/dist/forms.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// (duplicate_names), a listing or a default value too large to show exactly (too_large).
|
|
18
18
|
import { createHash } from "node:crypto";
|
|
19
19
|
import { listsFor, stripAccents } from "./shieldphrases.js";
|
|
20
|
-
import { MAX_URL } from "./types.js";
|
|
20
|
+
import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES, MAX_URL } from "./types.js";
|
|
21
21
|
export const MAX_FORMS = 20; // per page; the rest are counted (forms_omitted)
|
|
22
22
|
export const MAX_FIELDS = 100; // per form; a form with more is listed but never sent (too_many_fields)
|
|
23
23
|
export const MAX_OPTIONS = 50; // per select, radio or checkbox field; the rest are counted and cannot be chosen
|
|
@@ -687,8 +687,8 @@ export function hasCardNumber(v) {
|
|
|
687
687
|
}
|
|
688
688
|
function resolveAction(raw, pageUrl, base) {
|
|
689
689
|
try {
|
|
690
|
-
//
|
|
691
|
-
const u = raw === undefined || raw
|
|
690
|
+
// Only a missing or exactly empty action sends to the page itself. Whitespace is parsed against <base href>.
|
|
691
|
+
const u = raw === undefined || raw === "" ? new URL(pageUrl) : new URL(raw, base);
|
|
692
692
|
if ((u.protocol !== "http:" && u.protocol !== "https:") || u.username || u.password)
|
|
693
693
|
return null;
|
|
694
694
|
u.hash = "";
|
|
@@ -742,12 +742,14 @@ class FormBuilder {
|
|
|
742
742
|
// Every <form> on the page, in order, with its fields. A form nested in another (invalid HTML) is
|
|
743
743
|
// read as part of the outer one, as browsers do. Fields linked from outside a form with form="id"
|
|
744
744
|
// are not included.
|
|
745
|
-
|
|
746
|
-
const budget = { left: TEXT_BUDGET };
|
|
745
|
+
function scanForms(doc, pageUrl, actionsOnly = false) {
|
|
747
746
|
let base = pageUrl;
|
|
748
747
|
let baseSeen = false;
|
|
748
|
+
const bases = new Set(); // render policy only; the listing keeps its first-base resolver
|
|
749
749
|
const labelFor = new Map();
|
|
750
750
|
const formNodes = [];
|
|
751
|
+
const submitterNodes = []; // render policy only, including attributes on non-controls
|
|
752
|
+
let getOverride = false;
|
|
751
753
|
const ids = new Map(); // every element by id, for the audit
|
|
752
754
|
const labelsFor = new Map(); // every label for= of the page, for the audit
|
|
753
755
|
const formUp = new Map();
|
|
@@ -766,13 +768,38 @@ export function extractForms(doc, pageUrl, flattened = false) {
|
|
|
766
768
|
}
|
|
767
769
|
const { n, inForm, up } = item;
|
|
768
770
|
if (++count > NODE_BUDGET)
|
|
769
|
-
break; //
|
|
771
|
+
break; // an incomplete action scan fails closed
|
|
770
772
|
span.set(n, { a: clock++, z: Number.MAX_SAFE_INTEGER });
|
|
771
773
|
stack.push({ close: n });
|
|
772
774
|
let inside = inForm;
|
|
775
|
+
if (actionsOnly && (n.type === "tag" || n.type === "script" || n.type === "style")) {
|
|
776
|
+
const a = n.attribs ?? {};
|
|
777
|
+
// htmlparser2 preserves literal NULs; Chrome replaces them with U+FFFD. Numeric
|
|
778
|
+
// character references to zero already decode to U+FFFD in both parsers.
|
|
779
|
+
if ([a.action, a.formaction, n.name === "base" ? a.href : undefined].some(v => v?.includes("\0")))
|
|
780
|
+
return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
|
|
781
|
+
if (a.formmethod !== undefined && !["post", "dialog"].includes(a.formmethod.toLowerCase()))
|
|
782
|
+
getOverride = true;
|
|
783
|
+
if (a.formaction !== undefined)
|
|
784
|
+
submitterNodes.push(n);
|
|
785
|
+
if (formNodes.length + submitterNodes.length > MAX_FORM_ACTIONS)
|
|
786
|
+
return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
|
|
787
|
+
}
|
|
773
788
|
if (n.type === "tag") {
|
|
774
789
|
const a = n.attribs ?? {};
|
|
775
|
-
if (n.name === "base" &&
|
|
790
|
+
if (actionsOnly && n.name === "base" && a.href !== undefined) {
|
|
791
|
+
try {
|
|
792
|
+
const b = new URL(a.href, pageUrl);
|
|
793
|
+
if (b.protocol === "http:" || b.protocol === "https:")
|
|
794
|
+
bases.add(b.href);
|
|
795
|
+
}
|
|
796
|
+
catch {
|
|
797
|
+
// Invalid and non-web bases contribute no candidates; the page URL is always tried.
|
|
798
|
+
}
|
|
799
|
+
if (bases.size > MAX_FORM_BASES)
|
|
800
|
+
return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
|
|
801
|
+
}
|
|
802
|
+
if (!actionsOnly && n.name === "base" && !baseSeen && a.href !== undefined) {
|
|
776
803
|
baseSeen = true;
|
|
777
804
|
try {
|
|
778
805
|
const b = new URL(a.href.trim(), pageUrl);
|
|
@@ -787,7 +814,9 @@ export function extractForms(doc, pageUrl, flattened = false) {
|
|
|
787
814
|
labelFor.set(a.for, n);
|
|
788
815
|
if (n.name === "label" && a.for)
|
|
789
816
|
labelsFor.set(a.for, [...(labelsFor.get(a.for) ?? []), n].slice(0, MAX_REFS + 1));
|
|
790
|
-
if (n.name === "form" && !inForm) {
|
|
817
|
+
if (n.name === "form" && (!inForm || actionsOnly)) {
|
|
818
|
+
if (actionsOnly && formNodes.length + submitterNodes.length >= MAX_FORM_ACTIONS)
|
|
819
|
+
return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
|
|
791
820
|
formNodes.push(n);
|
|
792
821
|
formUp.set(n, up);
|
|
793
822
|
inside = true;
|
|
@@ -801,6 +830,49 @@ export function extractForms(doc, pageUrl, flattened = false) {
|
|
|
801
830
|
for (let i = n.children.length - 1; i >= 0; i--)
|
|
802
831
|
stack.push({ n: n.children[i], inForm: inside, up: next });
|
|
803
832
|
}
|
|
833
|
+
return { incomplete: count > NODE_BUDGET, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
|
|
834
|
+
}
|
|
835
|
+
export function extractFormActions(doc, pageUrl) {
|
|
836
|
+
const { formNodes, submitterNodes, getOverride, bases, incomplete } = scanForms(doc, pageUrl, true);
|
|
837
|
+
if (incomplete)
|
|
838
|
+
return { actions: [], incomplete: true };
|
|
839
|
+
const candidates = new Set([pageUrl, ...bases]);
|
|
840
|
+
const actions = new Set();
|
|
841
|
+
let total = 0;
|
|
842
|
+
try {
|
|
843
|
+
const rawActions = formNodes.filter(f => {
|
|
844
|
+
// Invalid method values submit as GET in browsers. The listing still refuses them.
|
|
845
|
+
const method = (f.attribs?.method ?? "").toLowerCase();
|
|
846
|
+
return getOverride || (method !== "post" && method !== "dialog");
|
|
847
|
+
}).map(f => f.attribs?.action ?? "");
|
|
848
|
+
rawActions.push(...submitterNodes.map(n => n.attribs.formaction));
|
|
849
|
+
for (const raw of rawActions) {
|
|
850
|
+
if (raw.length > MAX_URL)
|
|
851
|
+
return { actions: [], incomplete: true };
|
|
852
|
+
// Take the union instead of guessing the browser's effective base or special-casing
|
|
853
|
+
// empty/whitespace actions. Include inert and later bases, and always the page URL.
|
|
854
|
+
for (const candidate of candidates) {
|
|
855
|
+
const action = new URL(raw, candidate);
|
|
856
|
+
if (action.href.length > MAX_URL || action.username || action.password)
|
|
857
|
+
return { actions: [], incomplete: true };
|
|
858
|
+
total += Math.max(raw.length, action.href.length);
|
|
859
|
+
if (total > MAX_FORM_ACTION_CHARS)
|
|
860
|
+
return { actions: [], incomplete: true };
|
|
861
|
+
if (action.protocol === "http:" || action.protocol === "https:") {
|
|
862
|
+
action.hash = "";
|
|
863
|
+
actions.add(action.href);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
return { actions: [...actions], incomplete: false };
|
|
868
|
+
}
|
|
869
|
+
catch {
|
|
870
|
+
return { actions: [], incomplete: true };
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
export function extractForms(doc, pageUrl, flattened = false) {
|
|
874
|
+
const { formNodes, base, labelFor, ids, labelsFor, formUp, span } = scanForms(doc, pageUrl);
|
|
875
|
+
const budget = { left: TEXT_BUDGET };
|
|
804
876
|
const labelCache = new Map();
|
|
805
877
|
const labelText = (n) => {
|
|
806
878
|
let t = labelCache.get(n);
|
package/dist/getactions.js
CHANGED
|
@@ -2,12 +2,24 @@
|
|
|
2
2
|
// Known GET form targets are policy, not a guess about what an arbitrary GET does.
|
|
3
3
|
import { loadChoice } from "./config.js";
|
|
4
4
|
import { Refused } from "./errors.js";
|
|
5
|
-
import { cachePath, homeDir, isPrivate, readJson, writeJson } from "./store.js";
|
|
5
|
+
import { cachePath, homeDir, isPrivate, readJson, sha256, writeJson } from "./store.js";
|
|
6
6
|
import { normalizePath } from "./rules.js";
|
|
7
7
|
import { callContext } from "./context.js";
|
|
8
8
|
import { claimSlip, claimedStillValid } from "./slips.js";
|
|
9
|
-
const shared = { seen: new Set()
|
|
9
|
+
const shared = { seen: new Set() };
|
|
10
10
|
const MAX_SEEN = 4096;
|
|
11
|
+
// Every shared overflow target enters this fixed 8 KiB Bloom filter, saved or not.
|
|
12
|
+
// With 65,536 bits and 4 probes, estimated false positive rates (uniform hashes) are
|
|
13
|
+
// 0.239% at 4,096, 16.0% at 16,384 and 92.9% at 65,536 overflow targets. No false
|
|
14
|
+
// negatives during this process; collisions only require extra slips. Private observations never enter it.
|
|
15
|
+
const sharedOverflow = new Uint8Array(8192);
|
|
16
|
+
function overflowBits(key) {
|
|
17
|
+
const hash = sha256(key);
|
|
18
|
+
return [0, 4, 8, 12].map((i) => parseInt(hash.slice(i, i + 4), 16));
|
|
19
|
+
}
|
|
20
|
+
function inSharedOverflow(key) {
|
|
21
|
+
return overflowBits(key).every((bit) => (sharedOverflow[bit >>> 3] & (1 << (bit & 7))) !== 0);
|
|
22
|
+
}
|
|
11
23
|
// A denial comparison, never a URL to send or a permission-slip scope. Reuse the robots path
|
|
12
24
|
// normalizer for unreserved octets and escape case, then fold ambiguous server routing spellings
|
|
13
25
|
// conservatively: repeated decoding, separators, path parameters, Unicode, case and dot segments.
|
|
@@ -74,11 +86,19 @@ export function noteGetActions(forms = [], keep = false) {
|
|
|
74
86
|
if (f.method !== "GET" || !f.action)
|
|
75
87
|
continue;
|
|
76
88
|
const key = actionKey(new URL(f.action));
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
state.
|
|
81
|
-
|
|
89
|
+
const mk = memoryKey(key);
|
|
90
|
+
const saved = keep && !isPrivate() && !callContext.getStore()?.readOnly;
|
|
91
|
+
if (state && !state.seen.has(mk)) {
|
|
92
|
+
if (state.seen.size < MAX_SEEN)
|
|
93
|
+
state.seen.add(mk);
|
|
94
|
+
else if (state === shared)
|
|
95
|
+
for (const bit of overflowBits(mk))
|
|
96
|
+
sharedOverflow[bit >>> 3] |= 1 << (bit & 7);
|
|
97
|
+
else if ("full" in state)
|
|
98
|
+
state.full = true; // private overflow fails closed only for this call
|
|
99
|
+
}
|
|
100
|
+
// Record policy in memory before writing, so a failed write also fails closed.
|
|
101
|
+
if (saved)
|
|
82
102
|
writeJson(cachePath(key), { get_form_action: true, fetched_at: new Date().toISOString() });
|
|
83
103
|
}
|
|
84
104
|
}
|
|
@@ -89,7 +109,7 @@ export function guardGetAction(u) {
|
|
|
89
109
|
const key = `get-form:${target}`;
|
|
90
110
|
const state = callContext.getStore()?.getActions;
|
|
91
111
|
const wildcard = `get-form:${new URL(target).origin}/*`;
|
|
92
|
-
const known = key === wildcard || [
|
|
112
|
+
const known = key === wildcard || [key, wildcard].some((k) => inSharedOverflow(memoryKey(k))) || state?.full || [state, shared].some((s) => s?.seen.has(memoryKey(key)) || s?.seen.has(memoryKey(wildcard))) ||
|
|
93
113
|
[key, wildcard].some((k) => readJson(cachePath(k))?.get_form_action === true);
|
|
94
114
|
if (!known)
|
|
95
115
|
return;
|