chaffjs 0.28.0 → 0.29.0

package/src/facts/fact-scope.ts

@@ -139,3 +139,9 @@ export const scopedFacts = (facts: readonly Fact[], tree: StructureNode, source:
       return { ...fact, ...scopeOf(fact, around.at(-1), at), part: partOf(around, tree, fact.value.start, summaryWords) };
     });
 };
+
+/** 文書のその位置が、冒頭、要約の節、本文のどれか。 */
+export const partsAt = (offsets: readonly number[], tree: StructureNode, summaryWords: readonly string[]): FactPart[] => {
+  const nodes = inDocumentOrder(tree).filter((node) => CONTAINERS.has(node.kind));
+  return offsets.map((offset) => partOf(containersAt(nodes, offset), tree, offset, summaryWords));
+};

package/src/facts/labelled-facts.ts

@@ -19,6 +19,8 @@ export type FactWords = {
   readonly determiners: readonly string[];
   /** 名前にならない語（それ、it）。 */
   readonly vague: readonly string[];
+  /** 名前の前に置かれる決定の前置き（協議の結果、, it was decided that）。読点を含んでも条件ではないので、外してから名前を読む。 */
+  readonly leads: readonly string[];
   readonly attributes: readonly AttributePhrase[];
 };
 
@@ -38,6 +40,7 @@ const LETTER = /\p{L}/u;
 /** 名前に書かない記号。コードや属性（{style="…"}、"key"=）の中の値は、項目の値ではない。 */
 const CODE_MARK = /[{}<>=[\]"`]/u;
 const LATIN_WORD = /^[a-z ]+$/iu;
+const LATIN_LETTER = /^[a-z]$/iu;
 
 const lineStartOf = (source: string, offset: number): number => source.lastIndexOf("\n", offset - 1) + 1;
 
@@ -54,6 +57,15 @@ const beforeSeparator = (head: string, separator: string): string | undefined =>
   return rest;
 };
 
+/** 英字の前置きが語の途中で切れているか（「we agreed that」は「we agreed thatching」の頭ではない）。 */
+const endsInsideWord = (text: string, length: number): boolean => LATIN_LETTER.test(text.charAt(length - 1)) && LATIN_LETTER.test(text.charAt(length));
+
+/** 頭の決定の前置きを外した節。 */
+const withoutLead = (clause: string, words: FactWords): string => {
+  const lead = words.leads.find((phrase) => clause.toLowerCase().startsWith(phrase.toLowerCase()) && !endsInsideWord(clause, phrase.length));
+  return lead === undefined ? clause : clause.slice(lead.length).trimStart();
+};
+
 /** 最後の文の切れ目より後ろ。 */
 const afterLastBreak = (text: string): string => {
   const breaks = [...text.matchAll(SENTENCE_BREAK)];
@@ -97,7 +109,7 @@ const labelledFact = (source: string, value: FactValue, words: FactWords): Fact
   for (const separator of words.separators) {
     const before = beforeSeparator(head, separator);
     if (before === undefined) continue;
-    const clause = afterLastBreak(before).replace(BLOCK_MARK, "").replace(ITEM_MARK, "");
+    const clause = withoutLead(afterLastBreak(before).replace(BLOCK_MARK, "").replace(ITEM_MARK, "").trimStart(), words);
     if (CLAUSE_COMMA.test(clause)) return undefined;
     const label = withoutEdgeMarks(clause);
     const key = keyOf(label, words);

package/src/facts/table-facts.ts

@@ -11,12 +11,12 @@ import { withoutEdgeMarks } from "./trim-marks.ts";
  * 升に値だけを書いたときだけ読む。「3,000円（税込）」のように条件を書いた升は読まない。
  * 行の見出しが値を含むか字を含まない表（日付の列から始まる日程表）は、行に名前が無いので読まない。
  */
-type Cell = Span & { readonly text: string };
+export type Cell = Span & { readonly text: string };
 
 const LETTER = /\p{L}/u;
 const TWO_COLUMNS = 2;
 
-const cellsOf = (line: Line): Cell[] => {
+export const cellsOf = (line: Line): Cell[] => {
   const bounds = [-1, ...[...line.text.matchAll(CELL_SEPARATOR)].map((match) => match.index), line.text.length];
   const cells = bounds.slice(1).map((end, index) => {
     const from = (bounds[index] ?? 0) + 1;
@@ -29,7 +29,7 @@ const cellsOf = (line: Line): Cell[] => {
 const plain = withoutEdgeMarks;
 
 /** 区切り行（|---|）の前の行が見出し、後ろに続く | の行が本体。 */
-const tablesOf = (lines: readonly Line[]): { header: Line; rows: Line[] }[] =>
+export const tablesOf = (lines: readonly Line[]): { header: Line; rows: Line[] }[] =>
   lines.flatMap((line, index) => {
     const header = lines[index - 1];
     if (!TABLE_RULE.test(line.text) || header === undefined || !header.text.includes("|")) return [];

package/src/grade/ai-score.ts

@@ -8,7 +8,7 @@ export type GradeAiScore = {
   readonly notScored: NotScored["reason"] | null;
   readonly signs: number;
   readonly compared: number;
-  /** The signs that counted: rule ids, and structure measures as structure:<id>. */
+  /** The signs that counted: rule ids, structure measures as structure:<id>, and ai-generated-composite:strict. */
   readonly shown: readonly string[];
 };
 

package/src/grade/text.ts

@@ -103,7 +103,7 @@ const aside = (open: string, inner: string, close: string): string => (inner ===
 export const GRADE_TEXT: Texts<GradeText> = {
   ja: {
     usage:
-      "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
+      "使い方: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <前の results.jsonl>] [--allow-stamp-mismatch] [--variant-key <欄の名前>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <ジャンル>]",
     noReference: "reference が無い（事実は reference と照らす）",
     noCitations: "citations が無い（chaff は出力から引用を推測しない）",
     noContexts: "contexts が無い（回答の事実は、検索で取った一節と照らす）",
@@ -135,7 +135,7 @@ export const GRADE_TEXT: Texts<GradeText> = {
   },
   en: {
     usage:
-      "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
+      "usage: chaff grade <items.jsonl> [--out <results.jsonl>] [--baseline <earlier results.jsonl>] [--allow-stamp-mismatch] [--variant-key <field>] [--format text|json|markdown] [--json] [--compact] [--experimental] [--genre <genre>]",
     noReference: "no reference given (facts are checked against a reference)",
     noCitations: "no citations given (chaff does not guess quotations from the output)",
     noContexts: "no contexts given (the output's facts are checked against retrieved passages)",

package/src/heading-numbering.ts

@@ -11,8 +11,8 @@ const ARABIC = "[0-9０-９]+";
 const KANJI = "[一二三四五六七八九十百]+";
 const ROMAN = "[IVXⅠ-Ⅻ]+";
 
-/** 番号の後ろに続く見出しの言葉の頭。字で始まる。「8.2.3 (2026-08-06)」の版や「3 reasons」の数量は番号と読まない。 */
-const WORDS_FOLLOW = "(?=\\s*[\\p{L}])";
+/** 番号の後ろに続く見出しの言葉の頭。字か、コードや引用の開き（`check()`、「概要」）で始まる。「8.2.3 (2026-08-06)」の版や「3 reasons」の数量は番号と読まない。 */
+const WORDS_FOLLOW = '(?=\\s*[\\p{L}`「『“"])';
 
 /**
  * 字だけで決まる書き方。上から順に試す。ラテン文字一字とローマ数字（A. と II.）は同じ書き方とする。どちらも番号の後ろに言葉が続くときだけ。

package/src/html/fetch-page.ts

@@ -0,0 +1,64 @@
+// Fetches a page as text, in the encoding it declares, with a timeout. A non-2xx answer throws an error whose cause
+// carries the HTTP status, so a caller can tell a dead page (404) from a transient one (503). Every error names the URL.
+import { decodeFetched } from "./fetched-text.ts";
+
+export class HttpStatusError extends Error {
+  readonly status: number;
+
+  constructor(status: number) {
+    super(`HTTP ${String(status)}`);
+    this.status = status;
+  }
+}
+
+/** The page's text and the Content-Type it was sent with (null when it had none). */
+export type FetchedPage = { readonly text: string; readonly contentType: string | null };
+
+/** What fetches: the global fetch, or a stand-in in tests. */
+export type Fetcher = (url: string, init: { readonly signal: AbortSignal }) => Promise<Response>;
+
+export type FetchLimits = {
+  readonly timeout_ms: number;
+  /** A body longer than this is refused while it is read, before it is decoded. Undefined: no limit. */
+  readonly max_bytes?: number | undefined;
+};
+
+const tooLarge = (max_bytes: number): Error => new Error(`the page is larger than ${String(max_bytes)} bytes`);
+
+/** A read that gave bytes; a finished read (done) is not one. */
+const isChunk = (read: unknown): read is { readonly value: Uint8Array } =>
+  typeof read === "object" && read !== null && Reflect.get(read, "done") !== true && Reflect.get(read, "value") instanceof Uint8Array;
+
+/** The body's bytes, stopping as soon as they pass max_bytes. */
+const bodyBytes = async (response: Response, max_bytes: number | undefined): Promise<Uint8Array> => {
+  if (max_bytes === undefined || response.body === null) return new Uint8Array(await response.arrayBuffer());
+  const declared = Number(response.headers.get("content-length") ?? Number.NaN);
+  if (declared > max_bytes) throw tooLarge(max_bytes);
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  for (let read: unknown = await reader.read(); isChunk(read); read = await reader.read()) {
+    size += read.value.byteLength;
+    if (size > max_bytes) {
+      await reader.cancel();
+      throw tooLarge(max_bytes);
+    }
+    chunks.push(read.value);
+  }
+  return Buffer.concat(chunks);
+};
+
+export const fetchPage = async (url: string, limits: FetchLimits, fetcher: Fetcher = fetch): Promise<FetchedPage> => {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), limits.timeout_ms);
+  try {
+    const response = await fetcher(url, { signal: controller.signal });
+    if (!response.ok) throw new HttpStatusError(response.status);
+    const contentType = response.headers.get("content-type");
+    return { text: decodeFetched(await bodyBytes(response, limits.max_bytes), contentType), contentType };
+  } catch (err) {
+    throw new Error(`${url}: ${err instanceof Error ? err.message : String(err)}`, { cause: err });
+  } finally {
+    clearTimeout(timer);
+  }
+};

package/src/html/fetched-text.ts

@@ -0,0 +1,32 @@
+// The text of a fetched body in the encoding it declares: the Content-Type header first, then a <meta charset> or an
+// XML declaration near the top of an HTML or XML body, otherwise UTF-8. Many Japanese government pages are still
+// Shift_JIS. Pure.
+
+/** How far into the body a declaration is looked for; HTML requires <meta charset> within the first 1024 bytes. */
+const DECLARATION_BYTES = 1024;
+
+const CHARSET = /charset\s*=\s*["']?([\w.:-]+)/iu;
+
+const declaredInHead = (head: string): string | undefined =>
+  /<meta\b[^>]*\bcharset\s*=\s*["']?([\w.:-]+)/iu.exec(head)?.[1] ?? /^\s*<\?xml\b[^>]*\bencoding\s*=\s*["']([\w.:-]+)["']/iu.exec(head)?.[1];
+
+/** Only a markup body can declare its own encoding; Markdown or JSON that quotes a <meta> tag declares nothing. */
+const isMarkup = (contentType: string | null): boolean => contentType === null || /html|xml/iu.test(contentType);
+
+/** The label the body declares, lower-cased, or "utf-8" when it declares none. */
+export const charsetOf = (contentType: string | null, head: string): string =>
+  (CHARSET.exec(contentType ?? "")?.[1] ?? (isMarkup(contentType) ? declaredInHead(head) : undefined) ?? "utf-8").toLowerCase();
+
+const decoderFor = (label: string): TextDecoder => {
+  try {
+    return new TextDecoder(label);
+  } catch {
+    return new TextDecoder("utf-8");
+  }
+};
+
+/** An unknown label is read as UTF-8, as before a declaration was looked for. */
+export const decodeFetched = (bytes: Uint8Array, contentType: string | null): string => {
+  const head = new TextDecoder("latin1").decode(bytes.subarray(0, DECLARATION_BYTES));
+  return decoderFor(charsetOf(contentType, head)).decode(bytes);
+};

package/src/html/html-content-block.ts

@@ -0,0 +1,107 @@
+// The content of a page that marks none (no <main>, no role="main", no sole <article>): the innermost block that
+// holds every heading (the <h1> title among them) and nearly all the text written in sentences. A site's menus, side columns and footer are
+// labels and links, not sentences, so they fall outside; when the sentences are spread over blocks, no block
+// qualifies and the whole page is read. Pure; a regular-expression reading.
+import { CLOSED_SENTENCE, elementRanges, plainText, type ElementRange } from "./html-elements.ts";
+import { decodeEntities } from "./markup-text.ts";
+
+const TAG = /<\/?[a-z!][^>]*>/giu;
+
+// Tags that end a line of text; inline tags (a link, a span, a <br> inside a paragraph) run on inside a sentence.
+const LINE_BREAK = /^<\/?(?:p|div|li|dd|dt|td|th|h[1-6]|section|article|header|blockquote|ul|ol|dl|table|tr)\b/iu;
+
+const SENTENCE_END = new RegExp(CLOSED_SENTENCE.source, "gu");
+
+type Sentence = { readonly at: number; readonly text: string };
+
+type Scan = { readonly start: number; readonly line: string; readonly found: Sentence[] };
+
+/** A text between tags: each sentence closed in it is found with the text before it on its line; the rest runs on. */
+const scanText = (scan: Scan, text: string, at: number): Scan => {
+  const start = scan.line === "" ? at : scan.start;
+  const ends = [...text.matchAll(SENTENCE_END)].map((end) => end.index + end[0].length);
+  if (ends.length === 0) return { start, line: scan.line + text, found: scan.found };
+  ends.forEach((end, index) => scan.found.push({ at: start, text: (index === 0 ? scan.line : "") + text.slice(ends[index - 1] ?? 0, end) }));
+  return { start: at, line: text.slice(ends.at(-1) ?? text.length), found: scan.found };
+};
+
+type TagScan = Scan & { readonly from: number };
+
+/** The text before a tag, then the tag: a line break starts a new line. */
+const scanTag = (html: string, scan: TagScan, tag: RegExpExecArray): TagScan => {
+  const before = html.slice(scan.from, tag.index);
+  const scanned = before === "" ? scan : scanText(scan, decodeEntities(before), scan.from);
+  const next = LINE_BREAK.test(tag[0]) ? { start: tag.index, line: "", found: scanned.found } : scanned;
+  return { ...next, from: tag.index + tag[0].length };
+};
+
+/** Each sentence on the page with the words before it on its line, and where that line starts; in document order. */
+const sentences = (html: string): Sentence[] => {
+  const scan = [...html.matchAll(TAG)].reduce<TagScan>((state, tag) => scanTag(html, state, tag), { start: 0, line: "", found: [], from: 0 });
+  const rest = html.slice(scan.from);
+  return (rest === "" ? scan : scanText(scan, decodeEntities(rest), scan.from)).found;
+};
+
+const TITLE = /<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/giu;
+
+type Weighed = { readonly at: number; readonly weight: number };
+
+/**
+ * How much text each sentence carries: its length, or nothing when it only repeats the page's title (a question set as
+ * the title and again in a breadcrumb), so that a short page's title does not outweigh its body.
+ */
+const weighed = (html: string): Weighed[] => {
+  const titles = new Set([...html.matchAll(TITLE)].map((title) => plainText(title[1] ?? "")));
+  return sentences(html).map((sentence) => {
+    const text = sentence.text.replace(/\s+/gu, " ").trim();
+    return { at: sentence.at, weight: titles.has(text) ? 0 : text.length };
+  });
+};
+
+/** The total weight of the sentences whose line starts in [start, end), from running totals (totals[i] = before i). */
+const weightWithin = (all: readonly Weighed[], totals: readonly number[], start: number, end: number): number => {
+  const firstAtOrAfter = (at: number, low = 0, high = all.length): number => {
+    if (low >= high) return low;
+    const middle = Math.floor((low + high) / 2);
+    return (all[middle]?.at ?? 0) < at ? firstAtOrAfter(at, middle + 1, high) : firstAtOrAfter(at, low, middle);
+  };
+  return (totals[firstAtOrAfter(end)] ?? 0) - (totals[firstAtOrAfter(start)] ?? 0);
+};
+
+const runningTotals = (all: readonly Weighed[]): number[] =>
+  all.reduce<number[]>(
+    (sums, sentence) => {
+      sums.push((sums.at(-1) ?? 0) + sentence.weight);
+      return sums;
+    },
+    [0],
+  );
+
+// Share of the page's sentence text the block must hold: a line in the footer may fall outside it, a section of the
+// document may not.
+const CONTENT_SHARE = 0.9;
+
+const CONTENT_BLOCKS = ["div", "section"];
+
+/**
+ * The inside of the content block, or undefined when the page has no sentence, no block to single out, or no <h1>:
+ * without a marked title, a title set in a plain block (and its byline) could fall outside.
+ */
+export const contentBlock = (html: string): string | undefined => {
+  if (!/<h1\b/iu.test(html)) return undefined;
+  const headings = [...html.matchAll(/<h[1-6]\b/giu)].map((heading) => heading.index);
+  const all = weighed(html);
+  const totals = runningTotals(all);
+  const whole = totals.at(-1) ?? 0;
+  if (whole === 0) return undefined;
+  // Every heading is the document's (a list under "Members" has no sentence), and nothing written in sentences may
+  // stand before the block ("Updated." above the title): only a footer's line after it.
+  const holdsContent = (range: ElementRange): boolean =>
+    headings.every((heading) => heading > range.start && heading < range.end) &&
+    weightWithin(all, totals, 0, range.start) === 0 &&
+    weightWithin(all, totals, range.start, range.end) >= whole * CONTENT_SHARE;
+  const smallest = CONTENT_BLOCKS.flatMap((tag) => elementRanges(html, tag))
+    .filter(holdsContent)
+    .toSorted((left, right) => left.end - left.start - (right.end - right.start))[0];
+  return smallest?.inner;
+};

package/src/html/html-elements.ts

@@ -0,0 +1,127 @@
+// The pieces of an HTML page the converters look at: tags, links, attributes and elements matched to their own
+// closing tag. Pure; a regular-expression reading, not a parser.
+import { decodeEntities } from "./markup-text.ts";
+
+// In a tag a quote opens a value only after "=", and the value ends only at its matching quote, whatever it holds.
+const TAG_BODY = String.raw`(?:[^>=]|=\s*"[^"]*"|=\s*'[^']*'|=(?!\s*["']))*`;
+
+// A comment, a processing instruction, a script and a style are matched whole, so that no tag is read inside them
+// (the converter drops them anyway), and a "<!--" inside a value is escaped before comments are looked for.
+const TAG = new RegExp(String.raw`<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<(script|style)\b${TAG_BODY}>[\s\S]*?<\/\1\s*>|<[a-z][a-z0-9-]*${TAG_BODY}>`, "giu");
+
+const QUOTED_VALUE = /=(\s*)("[^"]*"|'[^']*')/gu;
+
+const withValuesEscaped = (tag: string): string =>
+  tag.replace(QUOTED_VALUE, (_whole: string, space: string, value: string) => `=${space}${value.replaceAll("<", "&lt;").replaceAll(">", "&gt;")}`);
+
+/**
+ * Each "<" and ">" inside a quoted attribute value (title="published under the <span>…</span>") as a character
+ * reference, which means the same in a value, so that a scanner reading a tag up to its first ">" reads all of it.
+ */
+export const withAttributeMarkupEscaped = (html: string): string => html.replace(TAG, (tag: string) => withValuesEscaped(tag));
+
+/** Apply step until the text stops changing: nested elements are removed from the inside out. */
+export const untilStable = (text: string, step: (text: string) => string): string => {
+  const next = step(text);
+  return next === text ? text : untilStable(next, step);
+};
+
+export const stripTags = (html: string): string => html.replace(/<\/?[a-z!][^>]*>/giu, "");
+
+export const plainText = (html: string): string => decodeEntities(stripTags(html)).replace(/\s+/gu, " ").trim();
+
+/** Text put back into markup, so that decoding it again gives the same text. */
+export const asMarkup = (text: string): string => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;");
+
+/** A full stop, question or exclamation mark closing a sentence, also before a closing quote or bracket; not the point in "3.5". */
+export const CLOSED_SENTENCE = /[。．！？]|[.!?][)\]"'”’]*(?=\s|$)/u;
+
+/** Whether the markup closes a sentence, each tag read as a space so that "Done.</li><li>Next" ends one. */
+export const hasSentence = (html: string): boolean => CLOSED_SENTENCE.test(plainText(html.replace(/<\/?[a-z!][^>]*>/giu, " ")));
+
+export const hasNoWords = (text: string): boolean => !/[\p{L}\p{N}]/u.test(text);
+
+export const ANY_LINK = /<a\b[^>]*>[\s\S]*?<\/a\s*>/giu;
+
+const HREF = /\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/iu;
+
+const PAGE_EXTENSION = /^(?:[sx]?html?|php|aspx?|jsp|cgi)$/iu;
+
+// The host is not part of the path: "https://example.com" names a site, not a file ending in .com.
+const SCHEME_AND_HOST = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^/?#]*/iu;
+
+// A link that opens no page (tel:, mailto:, sms:) carries its own content, a number or an address.
+const OTHER_SCHEME = /^\s*(?!https?:)[a-z][a-z0-9+.-]*:(?!\/\/)/iu;
+
+const hrefOf = (link: string): string => {
+  const href = HREF.exec(link);
+  return href?.[1] ?? href?.[2] ?? href?.[3] ?? "";
+};
+
+/** The extension of the file a link opens ("pdf", "mp3"), or undefined for a page or a site. */
+export const fileExtension = (link: string): string | undefined => {
+  const path = hrefOf(link).replace(SCHEME_AND_HOST, "").split(/[?#]/u)[0] ?? "";
+  const extension = /\.([a-z0-9]{1,5})$/iu.exec(path.split("/").at(-1) ?? "")?.[1];
+  return extension !== undefined && !PAGE_EXTENSION.test(extension) ? extension : undefined;
+};
+
+/** A link to a file (an appendix as PDF, a table as .xlsx), or one that opens no page, is part of the document. */
+export const isFileLink = (link: string): boolean => OTHER_SCHEME.test(hrefOf(link)) || fileExtension(link) !== undefined;
+
+/** Attributes before a given one, each skipped whole so that a quoted value (title="x role=main") is not read as one. */
+export const ATTRIBUTES = String.raw`(?:\s+[^\s"'>=/]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'>]+))?)*?`;
+
+export type ElementRange = { readonly start: number; readonly end: number; readonly openTag: string; readonly inner: string };
+
+type RangeScan = { readonly open: readonly RegExpExecArray[]; readonly ranges: readonly ElementRange[] };
+
+const closeLast = (html: string, scan: RangeScan, close: RegExpExecArray): RangeScan => {
+  const opener = scan.open.at(-1);
+  if (opener === undefined) return scan;
+  const range = {
+    start: opener.index,
+    end: close.index + close[0].length,
+    openTag: opener[0],
+    inner: html.slice(opener.index + opener[0].length, close.index),
+  };
+  return { open: scan.open.slice(0, -1), ranges: [...scan.ranges, range] };
+};
+
+/** Every <tag>…</tag>, nested ones matched to their own closing tag, in document order (outer before inner). */
+export const elementRanges = (html: string, tag: string): ElementRange[] =>
+  [...html.matchAll(new RegExp(`<(/?)${tag}\\b[^>]*>`, "giu"))]
+    .reduce<RangeScan>((scan, match) => (match[1] === "/" ? closeLast(html, scan, match) : { open: [...scan.open, match], ranges: scan.ranges }), {
+      open: [],
+      ranges: [],
+    })
+    .ranges.toSorted((left, right) => left.start - right.start);
+
+export const isInside = (outer: ElementRange, inner: ElementRange): boolean => outer.start < inner.start && inner.end <= outer.end;
+
+/** The ranges (in document order) each replaced by a space; one inside another cut one goes with it. */
+export const withoutRanges = (html: string, ranges: readonly ElementRange[]): string => {
+  const chosen = ranges.reduce<ElementRange[]>((kept, range) => {
+    const insideKept = (kept.at(-1)?.end ?? 0) > range.start;
+    if (!insideKept) kept.push(range);
+    return kept;
+  }, []);
+  const cut = chosen.reduce<{ readonly parts: readonly string[]; readonly from: number }>(
+    (acc, range) => ({ parts: [...acc.parts, html.slice(acc.from, range.start), " "], from: range.end }),
+    { parts: [], from: 0 },
+  );
+  return [...cut.parts, html.slice(cut.from)].join("");
+};
+
+/** The <tag> elements that are chrome, each replaced by a space; one inside another dropped one goes with it. */
+export const withoutElementsWhere = (html: string, tag: string, isChrome: (range: ElementRange) => boolean): string =>
+  withoutRanges(html, elementRanges(html, tag).filter(isChrome));
+
+/** Every <tag> whose opening matches, for each tag that opens that way, replaced by a space. */
+export const withoutElementsOpening = (html: string, opening: RegExp, isChrome: (range: ElementRange) => boolean): string => {
+  const tags = new Set([...html.matchAll(opening)].map((match) => (match[1] ?? "").toLowerCase()));
+  return [...tags].reduce((text, tag) => withoutElementsWhere(text, tag, isChrome), html);
+};
+
+export const HEADING_TAGS = ["h1", "h2", "h3", "h4", "h5", "h6"];
+
+export const headingRanges = (html: string): ElementRange[] => HEADING_TAGS.flatMap((tag) => elementRanges(html, tag));

package/src/html/html-footnote-marks.ts

@@ -0,0 +1,20 @@
+// A footnote's reference mark: a superscript that is a link (<sup><a …>1</a></sup>), or a link to a place on the same
+// page holding nothing but a superscript (<a href="#fn1"><sup>1</sup></a>). Read as text the number sticks to the
+// sentence it follows ("recessions.1 Over"), so the mark is dropped; the note itself stays where the page has it. A
+// superscript that is a link is a mark wherever the link goes (the notes may be on another page); a link around a
+// superscript is one only when it stays on this page, since "Brand<a href=…><sup>TM</sup></a>" is a sign. A superscript
+// with no link ("25<sup>th</sup>") is text. Pure.
+
+const LINK_IN_SUPERSCRIPT = /<sup\b[^>]*>\s*<a\b[^>]*>[^<]*<\/a\s*>\s*<\/sup\s*>/giu;
+
+const SUPERSCRIPT_LINK = /<a\b([^>]*)>\s*<sup\b[^>]*>([^<]*)<\/sup\s*>\s*<\/a\s*>/giu;
+
+// The fragment must name something: href="#" alone goes nowhere and is a control.
+const IN_PAGE_HREF = /\shref\s*=\s*(?:"#[^"\s]+"|'#[^'\s]+'|#[^\s"'>]+)/iu;
+
+const isFootnoteLink = (attributes: string, mark: string): boolean => IN_PAGE_HREF.test(attributes) && mark.trim() !== "";
+
+export const withoutFootnoteMarks = (html: string): string =>
+  html
+    .replace(LINK_IN_SUPERSCRIPT, "")
+    .replace(SUPERSCRIPT_LINK, (link: string, attributes: string, mark: string) => (isFootnoteLink(attributes, mark) ? "" : link));

package/src/html/html-heading-links.ts

@@ -0,0 +1,93 @@
+// A heading's link to its own section (a permalink beside or after the title), taken out of an HTML page. Pure.
+import { ANY_LINK, ATTRIBUTES, elementRanges, hasNoWords, plainText, type ElementRange } from "./html-elements.ts";
+import { decodeEntities } from "./markup-text.ts";
+
+const ID = new RegExp(String.raw`<([a-z][a-z0-9-]*)${ATTRIBUTES}\s+id\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))`, "giu");
+
+type IdTarget = { readonly start: number; readonly tag: string };
+
+/** Where each id's element opens, and which element it is; the first one when an id is repeated. */
+const idTargets = (html: string): Map<string, IdTarget> =>
+  [...html.matchAll(ID)].reduce((targets, match) => {
+    const id = decodeEntities(match[2] ?? match[3] ?? match[4] ?? "");
+    return targets.has(id) ? targets : targets.set(id, { start: match.index, tag: (match[1] ?? "").toLowerCase() });
+  }, new Map<string, IdTarget>());
+
+const FRAGMENT = /^<a\b[^>]*\bhref\s*=\s*["']?[^"'#\s>]*#([^"'\s>]+)/iu;
+
+/** The id a link's fragment names, its character references and percent escapes undone. */
+const fragmentId = (link: string): string | undefined => {
+  const fragment = FRAGMENT.exec(link)?.[1];
+  if (fragment === undefined) return undefined;
+  const decoded = decodeEntities(fragment);
+  try {
+    return decodeURIComponent(decoded);
+  } catch {
+    return decoded;
+  }
+};
+
+const HEADING_OPENING = /<h[1-6]\b/iu;
+
+type PageIds = {
+  readonly ids: ReadonlyMap<string, IdTarget>;
+  readonly html: string;
+  readonly rangesOf: (tag: string) => readonly ElementRange[];
+};
+
+type HeadingPlace = PageIds & { readonly start: number; readonly end: number };
+
+/** The target opens before the heading and closes after it, with no other heading opening in between. */
+const isSectionOf = (target: IdTarget, place: HeadingPlace): boolean =>
+  place.rangesOf(target.tag).some((range) => range.start === target.start && range.end >= place.end) &&
+  !HEADING_OPENING.test(place.html.slice(target.start, place.start));
+
+/** A link to the heading's own section: the heading, something inside it, or the block it opens (a permalink, "Copy link to Context"). */
+const isSelfLink = (link: string, place: HeadingPlace): boolean => {
+  const id = fragmentId(link);
+  const target = id === undefined ? undefined : place.ids.get(id);
+  if (target === undefined || target.start >= place.end) return false;
+  return target.start >= place.start || isSectionOf(target, place);
+};
+
+/** A permalink mark inside the heading (¶, #, an icon); a self-link with words is the title, or part of it. */
+const isPermalinkMark = (link: string, place: HeadingPlace): boolean => hasNoWords(plainText(link)) && isSelfLink(link, place);
+
+const HEADING_THEN_LINK = /(<h([1-6])\b[^>]*>)([\s\S]*?)(<\/h\2\s*>)(\s*<a\b[^>]*>[\s\S]*?<\/a\s*>)?/giu;
+
+/** The heading as matched, without its permalink marks or the self-link standing after it. */
+const headingWithoutSelfLinks = (match: RegExpExecArray, page: PageIds): string => {
+  const [, open = "", , inside = "", close = "", after] = match;
+  const place = { ...page, start: match.index, end: match.index + open.length + inside.length + close.length };
+  const kept = inside.replace(ANY_LINK, (link: string) => (isPermalinkMark(link, place) ? " " : link));
+  const next = after === undefined || isSelfLink(after.trim(), place) ? "" : after;
+  return `${open}${kept}${close}${next}`;
+};
+
+/** elementRanges for each tag, computed once. */
+const rangesByTag = (html: string): ((tag: string) => readonly ElementRange[]) => {
+  const cache = new Map<string, readonly ElementRange[]>();
+  return (tag: string) => {
+    const known = cache.get(tag);
+    if (known !== undefined) return known;
+    const ranges = elementRanges(html, tag);
+    cache.set(tag, ranges);
+    return ranges;
+  };
+};
+
+/**
+ * A heading's link to its own section goes when it stands just after the heading ("Copy link to Context"), or inside
+ * it as a mark without words ("Scope ¶"). A link with words inside the heading is its title and stays.
+ */
+export const withoutHeadingSelfLinks = (html: string): string => {
+  const page = { ids: idTargets(html), html, rangesOf: rangesByTag(html) };
+  const cut = [...html.matchAll(HEADING_THEN_LINK)].reduce<{ readonly parts: readonly string[]; readonly from: number }>(
+    (acc, match) => ({
+      parts: [...acc.parts, html.slice(acc.from, match.index), headingWithoutSelfLinks(match, page)],
+      from: match.index + match[0].length,
+    }),
+    { parts: [], from: 0 },
+  );
+  return [...cut.parts, html.slice(cut.from)].join("");
+};

package/src/html/html-lost-tags.ts

@@ -0,0 +1,14 @@
+// A paragraph's start tag that lost its "<" upstream and so reached the page as text: the eCFR serves 16 CFR 310.4(b)
+// with "… 45 CFR 160.103. P&gt;(2) It is an abusive …", where "P>" is what is left of the <P> that opened paragraph (2).
+// Read as text it glues the paragraph to the one before and leaves "P>" in the sentence. Only a P and a ">" standing
+// after a sentence's end and right before a paragraph's designation ("(2)", "(iv)", "(A)") and a capital are taken for the lost
+// tag; "P>0.05", "P > (2)" and a formula opening a sentence ("P>(2) follows") stay text. Pure.
+
+const LOST_PARAGRAPH_TAG = /(?<=[.;:][)"'”’]?[ \t])[Pp](?:&gt;|>)(?=\([0-9A-Za-z]{1,4}\)[ \t]\p{Lu})/gu;
+
+/** Whether at is inside a tag. An attribute's ">" is escaped by then (withAttributeMarkupEscaped), so a tag has none before its end. */
+const isInsideTag = (html: string, at: number): boolean => html.lastIndexOf("<", at) > html.lastIndexOf(">", at - 1);
+
+/** The HTML with each lost paragraph start tag in its text put back as <p>. */
+export const withLostParagraphTagsRestored = (html: string): string =>
+  html.replace(LOST_PARAGRAPH_TAG, (lost: string, at: number) => (isInsideTag(html, at) ? lost : "<p>"));