@raibitsi/tiresias 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +88 -0
  2. package/README.md +72 -33
  3. package/data/README.md +33 -0
  4. package/data/guard-ordinary-urls.txt +102 -0
  5. package/data/guard-starter.jsonl +424 -0
  6. package/dist/aimed.js +23 -0
  7. package/dist/anchor.js +26 -13
  8. package/dist/browsecli.js +17 -13
  9. package/dist/cdp.js +22 -1
  10. package/dist/cleanup.js +3 -1
  11. package/dist/cli.js +216 -68
  12. package/dist/config.js +75 -1
  13. package/dist/confirm.js +20 -5
  14. package/dist/cooldown.js +57 -32
  15. package/dist/dailylog.js +11 -5
  16. package/dist/dom.js +13 -8
  17. package/dist/doors.js +1 -1
  18. package/dist/engine.js +9 -4
  19. package/dist/envelope.js +10 -1
  20. package/dist/fetcher.js +29 -2
  21. package/dist/filereply.js +7 -1
  22. package/dist/formflow.js +21 -3
  23. package/dist/forms.js +337 -42
  24. package/dist/getactions.js +119 -0
  25. package/dist/guard.js +1240 -0
  26. package/dist/guardclient.js +245 -0
  27. package/dist/guardnetwork.js +22 -0
  28. package/dist/history.js +11 -5
  29. package/dist/hook.js +2 -0
  30. package/dist/hosts.js +34 -27
  31. package/dist/keys.js +420 -18
  32. package/dist/local.js +537 -0
  33. package/dist/mcp.js +56 -25
  34. package/dist/needs.js +9 -5
  35. package/dist/parentwatch.js +6 -16
  36. package/dist/pdf.js +4 -1
  37. package/dist/pdfchild.js +0 -2
  38. package/dist/pdfproc.js +26 -11
  39. package/dist/pdfwatch.js +14 -0
  40. package/dist/pipeline.js +117 -26
  41. package/dist/preformatted.js +12 -0
  42. package/dist/profiles.js +3 -3
  43. package/dist/queue.js +24 -20
  44. package/dist/reader.js +441 -87
  45. package/dist/receipts.js +7 -2
  46. package/dist/render.js +156 -27
  47. package/dist/renderactions.js +69 -0
  48. package/dist/resume.js +8 -6
  49. package/dist/rules.js +24 -5
  50. package/dist/safebrowsing.js +3 -3
  51. package/dist/safety.js +15 -7
  52. package/dist/sandbox.js +1 -1
  53. package/dist/saved.js +24 -9
  54. package/dist/savedlock.js +99 -5
  55. package/dist/settings.js +10 -3
  56. package/dist/shelf.js +1 -1
  57. package/dist/shield.js +15 -0
  58. package/dist/slips.js +43 -1
  59. package/dist/store.js +119 -25
  60. package/dist/tabs.js +1 -1
  61. package/dist/textview.js +16 -0
  62. package/dist/translate.js +22 -2
  63. package/dist/types.js +9 -2
  64. package/dist/watchflow.js +1 -2
  65. package/dist/work.js +10 -2
  66. package/docs/LOCAL-MODELS.md +271 -0
  67. package/package.json +5 -2
package/dist/formflow.js CHANGED
@@ -1,9 +1,10 @@
1
1
  // SPDX-License-Identifier: Apache-2.0
2
2
  // W-04: forms as permission-gated actions (form_preview, form_submit).
3
3
  // (Split out of engine.ts in v0.5.1, behavior unchanged.)
4
+ import { noteGetActions } from "./getactions.js";
4
5
  import { assertSiteAllowed } from "./sites.js";
5
6
  import { cooldownSeconds } from "./cooldown.js";
6
- import { COOLDOWN_TEXT, CRAWL_DELAY_TEXT, policyCheck, politeFetch } from "./fetcher.js";
7
+ import { COOLDOWN_TEXT, CRAWL_DELAY_TEXT, policyCheck, politeFetch, Refused } from "./fetcher.js";
7
8
  import { runUntrusted } from "./sandbox.js";
8
9
  import * as receipts from "./receipts.js";
9
10
  import { assertPublicUrl, currentGate, currentSent, parseSafeUrl, throwIfCancelled } from "./safety.js";
@@ -21,7 +22,7 @@ import { buildContent, csvCount, detailOf, earlyOutcome, looksHtml, parsePage, r
21
22
  // form_preview: fetch the page fresh (robots, waits, Safe Browsing and signing, like read with fresh),
22
23
  // read its forms, check the agent's values, and show the EXACT request that would be sent, hidden
23
24
  // values replaced by "[set by the site]", with a preview hash and whether a slip covers the target.
24
- // Writes nothing: no cache, no history line, no receipts, no slip use.
25
+ // Keeps no page record; shared politeness state still applies. A gated GET read can spend a slip use.
25
26
  //
26
27
  // form_submit: in ONE call, fetch the page fresh again, re-read the form, rebuild the request and its
27
28
  // hash; a different hash is form_changed and nothing is sent. Then the TARGET is checked (Safe
@@ -67,6 +68,7 @@ export async function loadForm(url, index, jar) {
67
68
  return formEnv(url, "error", { index, reason: "not_html" }, NOT_HTML, { final_url: res.url });
68
69
  const r = await runUntrusted({ op: "forms", body: res.body, url: res.url }, res.body.length);
69
70
  throwIfCancelled();
71
+ noteGetActions(r.forms);
70
72
  if (r.wall)
71
73
  return wallEnv(url, r.wall, res.url, { final_url: res.url });
72
74
  const form = r.forms.find((f) => f.index === index);
@@ -186,13 +188,23 @@ export async function afterSubmit(url, top, res, secrets, meta, agent) {
186
188
  if (!k)
187
189
  return formEnv(url, "ok", top, `${SENT} The site accepted it (HTTP ${res.status}). Its answer was empty.`, uc);
188
190
  // The answer is read like a page (outline with receipts), in the worker, and never cached.
189
- const { parsed, wall } = await parsePage(k, res.body, res.bytes, res.url, res.fileFlags, looksHtml(res.headers.get("content-type"), res.body), res.headers.get("content-language"));
191
+ let page;
192
+ try {
193
+ page = await parsePage(k, res.body, res.bytes, res.url, res.fileFlags, looksHtml(res.headers.get("content-type"), res.body), res.headers.get("content-language"), undefined, secrets);
194
+ }
195
+ catch (err) {
196
+ if (err instanceof SecretWithheld)
197
+ return withheldEnv(url, top, res.status, answerSize(res));
198
+ throw err;
199
+ }
200
+ const { parsed, wall } = page;
190
201
  // Check the full extracted text, before previews can cut off a long secret or receipts can be recorded.
191
202
  // Joining blocks also covers a secret split by the reader's paragraph-size cap.
192
203
  if (!noSecrets(secrets) && await answerHoldsSecret([
193
204
  JSON.stringify(parsed), parsed.sections.flatMap((s) => s.blocks.map((b) => b.text)).join("\n"),
194
205
  ], secrets))
195
206
  return withheldEnv(url, top, res.status, answerSize(res));
207
+ noteGetActions(parsed.forms);
196
208
  if (wall) {
197
209
  const e = wallEnv(url, wall, res.url, uc);
198
210
  e.message = `${SENT} The page that came back shows a wall: a person has to take it from here.`;
@@ -272,6 +284,12 @@ export async function formSubmit(url, index, values, previewHash, ctx = DEFAULT)
272
284
  method: b.method,
273
285
  body: b.body,
274
286
  submit: true,
287
+ beforeSend: (u) => {
288
+ if (!slips.claimedStillValid(claim, "submit_form", u.href)) {
289
+ top.reason = "no_slip";
290
+ throw new Refused("The claimed permission slip expired or was revoked before sending. Nothing was sent; the reserved use stays spent.");
291
+ }
292
+ },
275
293
  cookies: jar,
276
294
  files: fileOptions(), // W-20: an answer that is a file is fingerprinted and flagged
277
295
  // Honest about where the form came from: the page's origin and address, as a browser sends them.
package/dist/forms.js CHANGED
@@ -16,7 +16,8 @@
16
16
  // whole: more than 100 fields (too_many_fields), one visible name used by two different fields
17
17
  // (duplicate_names), a listing or a default value too large to show exactly (too_large).
18
18
  import { createHash } from "node:crypto";
19
- import { MAX_URL } from "./types.js";
19
+ import { listsFor, stripAccents } from "./shieldphrases.js";
20
+ import { MAX_FORM_ACTIONS, MAX_FORM_ACTION_CHARS, MAX_FORM_BASES, MAX_URL } from "./types.js";
20
21
  export const MAX_FORMS = 20; // per page; the rest are counted (forms_omitted)
21
22
  export const MAX_FIELDS = 100; // per form; a form with more is listed but never sent (too_many_fields)
22
23
  export const MAX_OPTIONS = 50; // per select, radio or checkbox field; the rest are counted and cannot be chosen
@@ -41,9 +42,9 @@ const ORDER = ["login_or_password", "captcha", "payment", "file_upload", "nested
41
42
  export const NOT_ALLOWED_TEXT = {
42
43
  unknown_control: "This form has a field of a kind Tiresias does not know (an input type outside the standard ones), so it does not send the form.",
43
44
  nested_control: "This form has a field inside another control (a button, a list of options, a template), which Tiresias cannot list or check as a field, so it does not send the form.",
44
- login_or_password: "This form asks for a password or a login code. Tiresias never handles logins or passwords: a person has to sign in themselves.",
45
+ login_or_password: "This form asks for sign-in, a password or a login code. Tiresias never handles logins or passwords: a person has to sign in themselves.",
45
46
  file_upload: "This form uploads files (a file field or multipart encoding). Tiresias does not send files.",
46
- payment: "This form looks like a payment form (card number, security code, expiry, IBAN or routing number). This version of Tiresias never pays.",
47
+ payment: "This form looks like a payment or purchase form (a payment action, card number, security code, expiry, IBAN or routing number). This version of Tiresias never pays.",
47
48
  captcha: "This form has a CAPTCHA or bot check. A person has to fill it in; Tiresias never solves one.",
48
49
  bad_action: "This form sends to something that is not an http or https address, so Tiresias does not use it.",
49
50
  bad_method: "This form uses a method other than GET or POST, so Tiresias does not use it.",
@@ -100,6 +101,9 @@ const cut = (s, n) => {
100
101
  const t = norm(s).slice(0, n);
101
102
  return t || undefined;
102
103
  };
104
+ // Characters that render as nothing (zero-width, soft hyphen, bidi marks, variation selectors and the
105
+ // like) are removed before any text is judged (round 90).
106
+ const invisible = (t) => t.replace(/[\p{Cf}\p{Default_Ignorable_Code_Point}]/gu, "");
103
107
  // THE text normalization of the form guard (round 93, one path for every text it judges): invisible and
104
108
  // format characters removed, NFKC (fullwidth letters, ligatures, compatibility digits), case-folded
105
109
  // (upper then lower, so "ß" is "ss" and the long s is "s"), invisible characters removed again.
@@ -120,9 +124,11 @@ function tokens(text) {
120
124
  const base = invisible(text).normalize("NFKC");
121
125
  const folded = fold(base);
122
126
  const camel = fold(base.replace(/(\p{Ll})(\p{Lu})/gu, "$1 $2"));
127
+ const runs = [folded, camel].map((s) => s.split(/[^a-z0-9]+/).filter(Boolean));
123
128
  return {
124
- words: [...folded.split(/[^a-z0-9]+/), ...camel.split(/[^a-z0-9]+/)].filter(Boolean),
129
+ words: runs.flat(),
125
130
  chunks: folded.split(/\s+/).map((c) => c.replace(/[^a-z0-9]/g, "")).filter(Boolean),
131
+ runs,
126
132
  };
127
133
  }
128
134
  // "data" is an <object>'s address (Codex round 79: a CAPTCHA embedded that way was not refused).
@@ -136,7 +142,7 @@ const OPAQUE = new Set(["button", "select", "textarea", "template", "script", "s
136
142
  // The input types Tiresias knows; any other refuses the form (round 87: it was read as text).
137
143
  const KNOWN_INPUTS = new Set([...TEXT_TYPES, "hidden", "checkbox", "radio", "submit", "image", "reset", "button", "password", "file"]);
138
144
  const MAX_AUDIT_TEXT = 1024 * 1024; // all texts the audit reads in one form
139
- // THE one list of markers, single words and phrases alike. A phrase also matches with its words run
145
+ // The existing safety markers, single words and phrases alike. A phrase also matches with its words run
140
146
  // together in any way ("sign in code" is also "signin code", "sign incode" and "signincode").
141
147
  const MARKER_PHRASES = [
142
148
  ["payment", ["card number", "card num", "card no", "credit card", "debit card", "cc num", "cc number", "cc exp", "cc expiry", "cc csc", "cc cvv", "cc cvc",
@@ -144,6 +150,164 @@ const MARKER_PHRASES = [
144
150
  ["login_or_password", ["password", "pass code", "passphrase", "login code", "sign in code", "one time", "otp", "verification code", "2fa", "two factor", "authenticator", "pin code"]],
145
151
  ["captcha", ["captcha", "turnstile", "not a robot", "not robot", "are you a human", "are you human"]],
146
152
  ];
153
+ // Actions use whole words only, never chunks or skeletons: "display", "buyer", "designing" and
154
+ // "blog in" are ordinary text. Inflections are explicit so unrelated longer words stay harmless.
155
+ const ACTION_WORDS = [
156
+ ["payment", ["pay", "pays", "paying", "paid", "payment", "payments", "purchase", "purchases", "purchased", "purchasing", "buy", "buys", "buying", "bought", "checkout", "checkouts", "placeorder"]],
157
+ ["login_or_password", ["login", "logins", "signin", "signins"]],
158
+ ];
159
+ // Preserve the original ASCII-token phrase checks on every text that feeds the word bag,
160
+ // including the whole subtree with text nodes spaced apart.
161
+ const LEGACY_ACTION_PHRASES = [
162
+ ...["check", "checks", "checked", "checking"].map((w) => ["payment", w, "out"]),
163
+ ...["place", "places", "placed", "placing"].flatMap((w) => ["order", "orders"].map((o) => ["payment", w, o])),
164
+ ...["log", "logs", "logged", "logging", "sign", "signs", "signed", "signing"].map((w) => ["login_or_password", w, "in"]),
165
+ ];
166
+ // New phrases require ordered Unicode words in one attribute or block, in either inline
167
+ // tokenisation. Separate attributes or blocks cannot form a new phrase.
168
+ const ACTION_PHRASES = [
169
+ ...LEGACY_ACTION_PHRASES,
170
+ ...["confirm", "confirms", "confirmed", "confirming", "complete", "completes", "completed", "completing",
171
+ "submit", "submits", "submitted", "submitting", "finish", "finishes", "finished", "finishing",
172
+ "finalize", "finalizes", "finalized", "finalizing", "finalise", "finalises", "finalised", "finalising"]
173
+ .flatMap((w) => ["order", "orders", "purchase", "purchases", "payment", "payments"].map((o) => ["payment", w, o])),
174
+ ["payment", "order", "now"],
175
+ ];
176
+ // Closed, language-specific grammar; at most two of these may separate phrase parts.
177
+ const FILLERS = {
178
+ en: ["your", "my", "the", "this", "our", "their", "me"],
179
+ es: ["mi", "tu", "su", "el", "la", "los", "las", "mis", "tus", "sus", "este", "esta"],
180
+ fr: ["ma", "mon", "mes", "votre", "vos", "notre", "nos", "la", "le", "les", "ce", "cette"],
181
+ de: ["meine", "meinen", "mein", "ihre", "ihren", "ihr", "die", "der", "das", "deine", "deinen", "dein"],
182
+ pt: ["meu", "minha", "seu", "sua", "o", "a", "os", "as", "este", "esta"],
183
+ it: ["mio", "mia", "tuo", "tua", "suo", "sua", "il", "lo", "la", "un", "una"],
184
+ nl: ["mijn", "je", "jouw", "uw", "de", "het", "dit", "deze"],
185
+ ru: ["мой", "моя", "мои", "мою", "ваш", "ваша", "ваши", "вашу", "свой", "свою", "этот"],
186
+ zh: ["我的", "您的", "你的"], ja: ["私の", "あなたの"], ko: ["내", "나의", "귀하의"],
187
+ ar: ["هذا", "هذه"], hi: ["मेरा", "मेरे", "अपना", "अपने", "आपका", "आपके", "की"],
188
+ };
189
+ // Only complete control labels accept these complements, never arbitrary trailing text.
190
+ const COMPLEMENTS = {
191
+ en: [["with email", "to account"], []], es: [["con email", "con correo", "a cuenta"], []],
192
+ fr: [["avec email", "par email", "a compte"], ["par carte"]],
193
+ de: [["mit email", "mit e mail", "in konto"], []], pt: [["com email", "com e mail", "na conta"], []],
194
+ it: [["con email", "con e mail", "al account"], []], nl: [["met email", "met e mail", "op account"], []],
195
+ ru: [["по почте", "в аккаунт"], []], ar: [["إلى حسابك", "الى حسابك", "بالبريد"], []],
196
+ };
197
+ const OTHER_ACTIONS = {
198
+ en: { anywhere: [[], []], submit: [["log in", "sign in"], ["order"]], now: ["now"] },
199
+ es: { anywhere: [["iniciar sesion", "inicia sesion", "inicie sesion"],
200
+ ["pagar", "pague", "comprar", "finalizar compra", "finalizar pedido", "realizar pedido", "hacer pedido", "confirmar pedido", "tramitar pedido"]],
201
+ submit: [["acceder", "accede", "acceda"], ["paga", "compra", "compre"]], now: ["ahora"] },
202
+ fr: { anywhere: [["se connecter", "me connecter", "connectez vous", "ouvrir une session"],
203
+ ["payez", "acheter", "achetez", "passer a la caisse", "passer commande", "valider la commande", "confirmer la commande"]],
204
+ submit: [["connexion"], ["payer", "achat", "commander"]], now: ["maintenant"] },
205
+ de: { anywhere: [["einloggen", "einloggen sie sich"],
206
+ ["bezahlen", "kaufen", "kaufe", "zur kasse", "bestellung aufgeben", "jetzt bestellen", "bestellung bestatigen"]],
207
+ submit: [["anmelden", "anmeldung"], ["zahlen", "kauf", "bestellen", "zahlungspflichtig bestellen"]], now: ["jetzt"] },
208
+ pt: { anywhere: [["iniciar sessao", "iniciar sessao de utilizador", "fazer login", "entrar na conta"],
209
+ ["pagar", "pague", "comprar", "finalizar compra", "finalizar pedido", "fazer pedido", "efetuar pedido", "confirmar pedido", "concluir pedido"]],
210
+ submit: [["entrar"], ["compre", "compra"]], now: ["agora"] },
211
+ it: { anywhere: [["effettua l accesso", "inizia sessione"],
212
+ ["pagare", "acquista", "acquistare", "acquisto", "comprare", "procedi al pagamento", "vai alla cassa", "effettua ordine", "effettua l ordine", "conferma ordine"]],
213
+ submit: [["accedi", "accedere"], ["paga", "compra", "ordina"]], now: ["ora", "adesso"] },
214
+ nl: { anywhere: [["inloggen", "log in", "meld je aan"],
215
+ ["betalen", "betaal", "kopen", "aankoop", "afrekenen", "bestelling plaatsen", "plaats bestelling", "bestelling bevestigen"]],
216
+ submit: [["aanmelden", "aanmelding"], ["koop", "bestel"]], now: ["nu"] },
217
+ ru: { anywhere: [["вход в систему", "авторизоваться"],
218
+ ["оплатить", "оплата", "платить", "купить", "покупка", "оформить покупку", "оформить заказ", "разместить заказ", "подтвердить заказ", "заказать"]],
219
+ submit: [["войти", "вход"], ["приобрести"]], now: ["сейчас"] },
220
+ zh: { anywhere: [["登录", "登錄", "登入"],
221
+ ["支付", "付款", "购买", "購買", "结账", "結帳", "結賬", "结算", "結算", "下单", "下單", "提交订单", "提交訂單", "确认订单", "確認訂單"]],
222
+ submit: [[], ["买", "買", "订购", "訂購"]], now: ["现在", "現在", "立即"] },
223
+ ja: { anywhere: [["ログイン", "サインイン"],
224
+ ["支払う", "支払い", "お支払", "購入", "買う", "チェックアウト", "レジに進む", "注文する", "注文を確定", "注文を送信", "注文確定", "注文を完了する"]],
225
+ submit: [[], []], now: ["今すぐ"] },
226
+ ko: { anywhere: [["로그인", "로그인하기"],
227
+ ["결제", "지불", "구매", "구입", "체크아웃", "주문하기", "주문 제출", "주문 확정", "주문 완료"]],
228
+ submit: [[], ["사기"]], now: ["지금"] },
229
+ ar: { anywhere: [["تسجيل الدخول", "تسجيل دخول"],
230
+ ["ادفع", "إدفع", "شراء", "اشتر", "اشتري", "اشترى", "إتمام الشراء", "اتمام الشراء", "تقديم الطلب", "إرسال الطلب", "ارسال الطلب", "تأكيد الطلب"]],
231
+ submit: [["الدخول"], ["دفع", "الدفع", "اطلب"]], now: ["الآن", "الان"] },
232
+ hi: { anywhere: [["साइन इन", "लॉग इन", "लाग इन", "लॉगिन", "प्रवेश करें"],
233
+ ["भुगतान", "खरीदें", "खरीदना", "खरीदारी", "चेकआउट", "चेक आउट", "ऑर्डर दें", "आर्डर दें", "ऑर्डर करें", "ऑर्डर पक्का करें", "ऑर्डर पुष्टि करें"]],
234
+ submit: [[], []], now: ["अभी"] },
235
+ };
236
+ const NO_ACTION_EDGES = new Set(["zh", "ja", "ko"]);
237
+ const LATIN_ACTIONS = new Set(["es", "fr", "de", "pt", "it", "nl"]);
238
+ const WORD_SEP = "[^\\p{L}\\p{M}\\p{N}]+";
239
+ const actionBody = (phrase, language) => {
240
+ const fillers = language ? FILLERS[language] : undefined;
241
+ const sep = fillers?.length ? `${WORD_SEP}(?:(?:${fillers.join("|")})${WORD_SEP}){0,2}` : WORD_SEP;
242
+ const words = fold(phrase).split(" ");
243
+ // Existing phrases include determiners ("confirmer la commande"). Treat those
244
+ // as fillers too, so the same zero-to-two bound applies to every spelling.
245
+ return words.filter((w, i) => i === 0 || i === words.length - 1 || !fillers?.includes(w)).join(sep);
246
+ };
247
+ const actionPattern = (language, phrase) => {
248
+ const body = actionBody(phrase, language);
249
+ return new RegExp(NO_ACTION_EDGES.has(language) ? body : `(?<![\\p{L}\\p{M}\\p{N}])${body}(?![\\p{L}\\p{M}\\p{N}])`, "u");
250
+ };
251
+ const OTHER_ACTION_PATTERNS = new Map(Object.entries(OTHER_ACTIONS).map(([language, { anywhere, submit, now }]) => {
252
+ const sep = NO_ACTION_EDGES.has(language) ? "\\s*" : WORD_SEP;
253
+ const when = now.map(w => actionBody(w)).join("|");
254
+ return [language, {
255
+ anywhere: anywhere.map((phrases) => phrases.map((p) => actionPattern(language, p))),
256
+ submit: submit.map((words, group) => words.map((w) => {
257
+ const complement = (COMPLEMENTS[language]?.[group] ?? []).map(p => actionBody(p, language)).join("|");
258
+ const action = `${actionBody(w)}${complement ? `(?:${sep}(?:${complement}))?` : ""}`;
259
+ return new RegExp(`^(?:(?:${when})${sep})?${action}(?:${sep}(?:${when}))?$`, "u");
260
+ })),
261
+ }];
262
+ }));
263
+ // One first-word lookup per token; each candidate has a constant, bounded gap.
264
+ function phraseIndex(phrases) {
265
+ const index = new Map();
266
+ for (const [code, first, second] of phrases) {
267
+ if (!index.has(first))
268
+ index.set(first, new Map());
269
+ index.get(first).set(second, code);
270
+ }
271
+ return index;
272
+ }
273
+ const LEGACY_INDEX = phraseIndex(LEGACY_ACTION_PHRASES);
274
+ const ACTION_INDEX = phraseIndex(ACTION_PHRASES);
275
+ const EN_FILLERS = new Set(FILLERS.en);
276
+ function phraseRefusals(run, index, b, gaps = false) {
277
+ for (let i = 0; i + 1 < run.length; i++) {
278
+ const ends = index.get(run[i]);
279
+ if (!ends)
280
+ continue;
281
+ for (let j = i + 1; j < run.length && j <= i + (gaps ? 3 : 1); j++) {
282
+ const code = ends.get(run[j]);
283
+ if (code)
284
+ b.np(code);
285
+ if (!EN_FILLERS.has(run[j]))
286
+ break;
287
+ }
288
+ }
289
+ }
290
+ function otherActionRefusals(text, b, submit = false) {
291
+ const folded = norm(fold(text));
292
+ for (const language of ["en", ...listsFor(folded)]) {
293
+ const value = LATIN_ACTIONS.has(language) ? stripAccents(folded) : folded;
294
+ const patterns = OTHER_ACTION_PATTERNS.get(language);
295
+ const groups = submit ? patterns.submit : patterns.anywhere;
296
+ if (groups[0].some((re) => re.test(value)))
297
+ b.np("login_or_password");
298
+ if (groups[1].some((re) => re.test(value)))
299
+ b.np("payment");
300
+ }
301
+ }
302
+ function actionRefusals(text, b) {
303
+ const base = invisible(text).normalize("NFKC");
304
+ const runs = [fold(base), fold(base.replace(/(\p{Ll})(\p{Lu})/gu, "$1 $2"))]
305
+ .map((s) => s.split(/[^\p{L}\p{M}\p{N}]+/u).filter(Boolean));
306
+ for (const run of runs) {
307
+ phraseRefusals(run, ACTION_INDEX, b, true);
308
+ }
309
+ otherActionRefusals(text, b);
310
+ }
147
311
  // Every marker's run-together form, searched inside every chunk (round 94, coordinator's decision): a
148
312
  // marker anywhere inside a word counts ("recaptcha", "mypassword", "footpath" for "otp").
149
313
  // Markers whose run-together form has SHORT_MARKER characters or more are found inside skeletons (across
@@ -165,9 +329,6 @@ const joinings = (ws) => {
165
329
  // phrase also matches with some of its words run together ("signin" + "code").
166
330
  const MARKERS = MARKER_PHRASES.flatMap(([code, ps]) => ps.flatMap((p) => joinings(p.split(" ")).map((w) => [code, w])));
167
331
  const ID_LISTS = ["aria-labelledby", "aria-describedby", "aria-details", "aria-errormessage", "aria-owns"];
168
- // Characters that render as nothing (zero-width, soft hyphen, bidi marks, variation selectors and the
169
- // like) are removed before any text is judged (round 90).
170
- const invisible = (t) => t.replace(/[\p{Cf}\p{Default_Ignorable_Code_Point}]/gu, "");
171
332
  const MAX_REFS = 50; // ids in one reference attribute, and labels for one id; more refuses (round 89)
172
333
  // Elements that break a line of text. Anything else (custom elements included) is inline, as in a browser.
173
334
  const BLOCK = new Set([
@@ -175,6 +336,24 @@ const BLOCK = new Set([
175
336
  "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hgroup", "hr", "input", "legend", "li", "main", "menu", "nav", "ol",
176
337
  "optgroup", "option", "p", "pre", "section", "select", "summary", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "tr", "ul",
177
338
  ]);
339
+ function judgeText(text, cache) {
340
+ const old = cache.get(text);
341
+ if (old)
342
+ return old;
343
+ const { words, chunks, runs } = tokens(text);
344
+ const basic = new Set(), actions = new Set(), submit = new Set();
345
+ for (const run of runs)
346
+ phraseRefusals(run, LEGACY_INDEX, { np: c => basic.add(c) });
347
+ for (const c of new Set(chunks))
348
+ for (const [code, inside] of MARKER_SHORT)
349
+ if (c.includes(inside))
350
+ basic.add(code);
351
+ actionRefusals(text, { np: c => actions.add(c) });
352
+ otherActionRefusals(text, { np: c => submit.add(c) }, true);
353
+ const value = { words: [...new Set(words)], skeleton: skeleton(text), card: hasCardNumber(text), basic, actions, submit };
354
+ cache.set(text, value);
355
+ return value;
356
+ }
178
357
  function auditForm(form, b, ctx) {
179
358
  let used = 0;
180
359
  let nodes = 0;
@@ -184,30 +363,37 @@ function auditForm(form, b, ctx) {
184
363
  b.np("too_large");
185
364
  };
186
365
  const bag = new Set(); // every word the form and everything tied to it says
187
- const chunkSeen = new Set(); // chunks already searched
188
- const skeletons = []; // every skeleton read, for the long words of the many-word markers
189
- // judge: one text's words and chunks (short markers inside chunks); with `skel`, also its skeleton (kept
190
- // for the MARKERS check) and its card digits
191
- const judge = (text, skel) => {
366
+ const skeletons = new Set();
367
+ // Charge every read even on a cache hit: sharing classification never grants extra budget.
368
+ const charged = (text) => {
192
369
  used += text.length;
193
- if (used > MAX_AUDIT_TEXT)
194
- return tooLarge();
195
- const { words, chunks } = tokens(text);
196
- for (const w of words)
197
- bag.add(w);
198
- for (const c of chunks) {
199
- if (chunkSeen.has(c))
200
- continue;
201
- chunkSeen.add(c);
202
- for (const [code, inside] of MARKER_SHORT)
203
- if (c.includes(inside))
204
- b.np(code);
370
+ if (used > MAX_AUDIT_TEXT) {
371
+ tooLarge();
372
+ return undefined;
205
373
  }
374
+ return judgeText(text, ctx.texts);
375
+ };
376
+ const judge = (text, skel) => {
377
+ const value = charged(text);
378
+ if (!value)
379
+ return;
380
+ for (const w of value.words)
381
+ bag.add(w);
382
+ for (const c of value.basic)
383
+ b.np(c);
206
384
  if (!skel)
207
385
  return;
208
- if (hasCardNumber(text))
386
+ for (const c of value.actions)
387
+ b.np(c);
388
+ if (value.card)
209
389
  b.np("payment");
210
- skeletons.push(skeleton(text));
390
+ skeletons.add(value.skeleton);
391
+ };
392
+ const submitText = (text) => {
393
+ const value = charged(text);
394
+ if (value)
395
+ for (const c of value.submit)
396
+ b.np(c);
211
397
  };
212
398
  // each attribute value is its own text (round 93: never joined); each element's attributes are judged
213
399
  // exactly once, whether reached as an ancestor or in a walk (round 96: twice spent the budget twice)
@@ -216,8 +402,14 @@ function auditForm(form, b, ctx) {
216
402
  if (attrSeen.has(n))
217
403
  return;
218
404
  attrSeen.add(n);
219
- for (const v of Object.values(n.attribs ?? {}))
405
+ const c = n.attribs ?? {};
406
+ for (const v of Object.values(c))
220
407
  judge(v, true);
408
+ if (n.name === "input" && ["submit", "button", "image"].includes((c.type ?? "").trim().toLowerCase())) {
409
+ for (const v of [c.value, c.alt])
410
+ if (v !== undefined)
411
+ submitText(v);
412
+ }
221
413
  };
222
414
  const units = [];
223
415
  const queued = new Set();
@@ -311,25 +503,39 @@ function auditForm(form, b, ctx) {
311
503
  // in the page's own text, which split a word or a card number)
312
504
  const blocks = [];
313
505
  let cur = "";
506
+ let curSpaced = "";
507
+ let buttonText;
314
508
  let size = 0;
315
509
  const endBlock = () => {
316
510
  if (cur)
317
- blocks.push(cur);
511
+ blocks.push({ joined: cur, spaced: curSpaced });
318
512
  cur = "";
513
+ curSpaced = "";
319
514
  };
320
515
  const work = [{ n: root, inside: false }];
321
516
  while (work.length && !stop) {
322
517
  const it = work.pop();
518
+ if ("buttonEnd" in it) {
519
+ submitText(buttonText.joined);
520
+ submitText(buttonText.spaced);
521
+ buttonText = undefined;
522
+ continue;
523
+ }
323
524
  if ("sep" in it) {
324
525
  endBlock();
325
526
  continue;
326
527
  }
327
528
  const { n: x, inside } = it; // the kept subtrees are the union found above, so no new budget is needed
328
529
  if (x.type === "text") {
329
- if (hasCardNumber(x.data))
530
+ if (charged(x.data ?? "")?.card)
330
531
  b.np("payment");
331
532
  spaced += (x.data ?? "") + " ";
332
533
  cur += x.data ?? "";
534
+ curSpaced += (x.data ?? "") + " ";
535
+ if (buttonText) {
536
+ buttonText.joined += x.data ?? "";
537
+ buttonText.spaced += (x.data ?? "") + " ";
538
+ }
333
539
  size += (x.data ?? "").length;
334
540
  if (spaced.length + size > MAX_AUDIT_TEXT)
335
541
  return tooLarge();
@@ -349,6 +555,12 @@ function auditForm(form, b, ctx) {
349
555
  b.np("unknown_control");
350
556
  attrs(x); // every attribute value: titles, labels, placeholders, values...
351
557
  block = BLOCK.has(nm);
558
+ // Capture a button's entire text in the same bounded walk. Nested controls already
559
+ // fail closed; do not copy their text into multiple nested accumulators.
560
+ if (nm === "button" && !buttonText) {
561
+ buttonText = { joined: "", spaced: "" };
562
+ work.push({ buttonEnd: true });
563
+ }
352
564
  }
353
565
  if (block) {
354
566
  endBlock();
@@ -365,19 +577,29 @@ function auditForm(form, b, ctx) {
365
577
  // each block of inline-joined text is one text: its skeleton and its card digits (round 95: a card
366
578
  // number split across inline elements). Any white space in a block, line breaks, tabs and CR included,
367
579
  // is ordinary spacing (round 96).
368
- for (const piece of blocks)
369
- judge(piece.replace(/\s+/g, " "), true);
580
+ for (const piece of blocks) {
581
+ judge(piece.joined.replace(/\s+/g, " "), true);
582
+ if (piece.spaced !== piece.joined) {
583
+ const value = charged(piece.spaced);
584
+ if (value)
585
+ for (const c of value.actions)
586
+ b.np(c);
587
+ }
588
+ }
370
589
  }
371
590
  if (stop)
372
591
  return;
373
592
  // EVERY marker of SHORT_MARKER characters or more, and every many-word marker with its words apart: a word
374
593
  // (or run-together piece) of SHORT_MARKER letters or more counts when it is inside any skeleton
375
594
  // ("SeCuRiTy Code" as "securitycode"; "mySeCuRiTy" and "code"), a shorter one only as a whole word
376
- const all = skeletons.join("|");
595
+ const all = [...skeletons].join("|");
377
596
  const has = (w) => bag.has(w) || (w.length >= SHORT_MARKER && all.includes(w));
378
597
  for (const [code, need] of MARKERS)
379
598
  if (need.every(has))
380
599
  b.np(code);
600
+ for (const [code, words] of ACTION_WORDS)
601
+ if (words.some((w) => bag.has(w)))
602
+ b.np(code);
381
603
  }
382
604
  // ONE set of per-element refusals (Codex round 85: the whole-form scan had its own, shorter copy). Used by
383
605
  // the whole-form scan, which visits every element, and by the field walk. Label-based payment checks stay
@@ -386,7 +608,7 @@ function elementRefusals(name, c, b) {
386
608
  if (captchaMarker(c) || EMBEDDED.has(name))
387
609
  b.np("captcha");
388
610
  const auto = (c.autocomplete ?? "").toLowerCase().split(/\s+/);
389
- if (auto.some((x) => x === "current-password" || x === "new-password" || x === "one-time-code"))
611
+ if (auto.some((x) => x === "username" || x === "current-password" || x === "new-password" || x === "one-time-code"))
390
612
  b.np("login_or_password");
391
613
  if (auto.some((x) => x.startsWith("cc-")))
392
614
  b.np("payment");
@@ -465,8 +687,8 @@ export function hasCardNumber(v) {
465
687
  }
466
688
  function resolveAction(raw, pageUrl, base) {
467
689
  try {
468
- // A missing or empty action sends to the page itself; otherwise it is resolved against <base href>.
469
- const u = raw === undefined || raw.trim() === "" ? new URL(pageUrl) : new URL(raw.trim(), base);
690
+ // Only a missing or exactly empty action sends to the page itself. Whitespace is parsed against <base href>.
691
+ const u = raw === undefined || raw === "" ? new URL(pageUrl) : new URL(raw, base);
470
692
  if ((u.protocol !== "http:" && u.protocol !== "https:") || u.username || u.password)
471
693
  return null;
472
694
  u.hash = "";
@@ -520,12 +742,14 @@ class FormBuilder {
520
742
  // Every <form> on the page, in order, with its fields. A form nested in another (invalid HTML) is
521
743
  // read as part of the outer one, as browsers do. Fields linked from outside a form with form="id"
522
744
  // are not included.
523
- export function extractForms(doc, pageUrl, flattened = false) {
524
- const budget = { left: TEXT_BUDGET };
745
+ function scanForms(doc, pageUrl, actionsOnly = false) {
525
746
  let base = pageUrl;
526
747
  let baseSeen = false;
748
+ const bases = new Set(); // render policy only; the listing keeps its first-base resolver
527
749
  const labelFor = new Map();
528
750
  const formNodes = [];
751
+ const submitterNodes = []; // render policy only, including attributes on non-controls
752
+ let getOverride = false;
529
753
  const ids = new Map(); // every element by id, for the audit
530
754
  const labelsFor = new Map(); // every label for= of the page, for the audit
531
755
  const formUp = new Map();
@@ -544,13 +768,38 @@ export function extractForms(doc, pageUrl, flattened = false) {
544
768
  }
545
769
  const { n, inForm, up } = item;
546
770
  if (++count > NODE_BUDGET)
547
- break; // forms past the budget are never found, so never sent
771
+ break; // an incomplete action scan fails closed
548
772
  span.set(n, { a: clock++, z: Number.MAX_SAFE_INTEGER });
549
773
  stack.push({ close: n });
550
774
  let inside = inForm;
775
+ if (actionsOnly && (n.type === "tag" || n.type === "script" || n.type === "style")) {
776
+ const a = n.attribs ?? {};
777
+ // htmlparser2 preserves literal NULs; Chrome replaces them with U+FFFD. Numeric
778
+ // character references to zero already decode to U+FFFD in both parsers.
779
+ if ([a.action, a.formaction, n.name === "base" ? a.href : undefined].some(v => v?.includes("\0")))
780
+ return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
781
+ if (a.formmethod !== undefined && !["post", "dialog"].includes(a.formmethod.toLowerCase()))
782
+ getOverride = true;
783
+ if (a.formaction !== undefined)
784
+ submitterNodes.push(n);
785
+ if (formNodes.length + submitterNodes.length > MAX_FORM_ACTIONS)
786
+ return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
787
+ }
551
788
  if (n.type === "tag") {
552
789
  const a = n.attribs ?? {};
553
- if (n.name === "base" && !baseSeen && a.href !== undefined) {
790
+ if (actionsOnly && n.name === "base" && a.href !== undefined) {
791
+ try {
792
+ const b = new URL(a.href, pageUrl);
793
+ if (b.protocol === "http:" || b.protocol === "https:")
794
+ bases.add(b.href);
795
+ }
796
+ catch {
797
+ // Invalid and non-web bases contribute no candidates; the page URL is always tried.
798
+ }
799
+ if (bases.size > MAX_FORM_BASES)
800
+ return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
801
+ }
802
+ if (!actionsOnly && n.name === "base" && !baseSeen && a.href !== undefined) {
554
803
  baseSeen = true;
555
804
  try {
556
805
  const b = new URL(a.href.trim(), pageUrl);
@@ -565,7 +814,9 @@ export function extractForms(doc, pageUrl, flattened = false) {
565
814
  labelFor.set(a.for, n);
566
815
  if (n.name === "label" && a.for)
567
816
  labelsFor.set(a.for, [...(labelsFor.get(a.for) ?? []), n].slice(0, MAX_REFS + 1));
568
- if (n.name === "form" && !inForm) {
817
+ if (n.name === "form" && (!inForm || actionsOnly)) {
818
+ if (actionsOnly && formNodes.length + submitterNodes.length >= MAX_FORM_ACTIONS)
819
+ return { incomplete: true, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
569
820
  formNodes.push(n);
570
821
  formUp.set(n, up);
571
822
  inside = true;
@@ -579,6 +830,49 @@ export function extractForms(doc, pageUrl, flattened = false) {
579
830
  for (let i = n.children.length - 1; i >= 0; i--)
580
831
  stack.push({ n: n.children[i], inForm: inside, up: next });
581
832
  }
833
+ return { incomplete: count > NODE_BUDGET, base, bases, formNodes, submitterNodes, getOverride, labelFor, ids, labelsFor, formUp, span };
834
+ }
835
+ export function extractFormActions(doc, pageUrl) {
836
+ const { formNodes, submitterNodes, getOverride, bases, incomplete } = scanForms(doc, pageUrl, true);
837
+ if (incomplete)
838
+ return { actions: [], incomplete: true };
839
+ const candidates = new Set([pageUrl, ...bases]);
840
+ const actions = new Set();
841
+ let total = 0;
842
+ try {
843
+ const rawActions = formNodes.filter(f => {
844
+ // Invalid method values submit as GET in browsers. The listing still refuses them.
845
+ const method = (f.attribs?.method ?? "").toLowerCase();
846
+ return getOverride || (method !== "post" && method !== "dialog");
847
+ }).map(f => f.attribs?.action ?? "");
848
+ rawActions.push(...submitterNodes.map(n => n.attribs.formaction));
849
+ for (const raw of rawActions) {
850
+ if (raw.length > MAX_URL)
851
+ return { actions: [], incomplete: true };
852
+ // Take the union instead of guessing the browser's effective base or special-casing
853
+ // empty/whitespace actions. Include inert and later bases, and always the page URL.
854
+ for (const candidate of candidates) {
855
+ const action = new URL(raw, candidate);
856
+ if (action.href.length > MAX_URL || action.username || action.password)
857
+ return { actions: [], incomplete: true };
858
+ total += Math.max(raw.length, action.href.length);
859
+ if (total > MAX_FORM_ACTION_CHARS)
860
+ return { actions: [], incomplete: true };
861
+ if (action.protocol === "http:" || action.protocol === "https:") {
862
+ action.hash = "";
863
+ actions.add(action.href);
864
+ }
865
+ }
866
+ }
867
+ return { actions: [...actions], incomplete: false };
868
+ }
869
+ catch {
870
+ return { actions: [], incomplete: true };
871
+ }
872
+ }
873
+ export function extractForms(doc, pageUrl, flattened = false) {
874
+ const { formNodes, base, labelFor, ids, labelsFor, formUp, span } = scanForms(doc, pageUrl);
875
+ const budget = { left: TEXT_BUDGET };
582
876
  const labelCache = new Map();
583
877
  const labelText = (n) => {
584
878
  let t = labelCache.get(n);
@@ -586,11 +880,12 @@ export function extractForms(doc, pageUrl, flattened = false) {
586
880
  labelCache.set(n, (t = textOf(n, budget)));
587
881
  return t;
588
882
  };
883
+ const texts = new Map();
589
884
  const forms = formNodes.slice(0, MAX_FORMS).map((f, i) => {
590
885
  const ancestors = [];
591
886
  for (let x = formUp.get(f) ?? null; x; x = x.up)
592
887
  ancestors.push(x.n);
593
- const form = readForm(f, i + 1, pageUrl, base, labelFor, labelText, budget, { ids, labelsFor, ancestors, span });
888
+ const form = readForm(f, i + 1, pageUrl, base, labelFor, labelText, budget, { ids, labelsFor, ancestors, span, texts });
594
889
  // Flattening may have changed a control's owner or its ancestors: never submit a partial form.
595
890
  if (flattened && !form.not_allowed)
596
891
  form.not_allowed = "too_large";