chaffjs 0.27.1 → 0.29.0

package/src/html/html-content-block.ts

@@ -0,0 +1,107 @@
+// The content of a page that marks none (no <main>, no role="main", no sole <article>): the innermost block that
+// holds every heading (the <h1> title among them) and nearly all the text written in sentences. A site's menus, side columns and footer are
+// labels and links, not sentences, so they fall outside; when the sentences are spread over blocks, no block
+// qualifies and the whole page is read. Pure; a regular-expression reading.
+import { CLOSED_SENTENCE, elementRanges, plainText, type ElementRange } from "./html-elements.ts";
+import { decodeEntities } from "./markup-text.ts";
+
+const TAG = /<\/?[a-z!][^>]*>/giu;
+
+// Tags that end a line of text; inline tags (a link, a span, a <br> inside a paragraph) run on inside a sentence.
+const LINE_BREAK = /^<\/?(?:p|div|li|dd|dt|td|th|h[1-6]|section|article|header|blockquote|ul|ol|dl|table|tr)\b/iu;
+
+const SENTENCE_END = new RegExp(CLOSED_SENTENCE.source, "gu");
+
+type Sentence = { readonly at: number; readonly text: string };
+
+type Scan = { readonly start: number; readonly line: string; readonly found: Sentence[] };
+
+/** A text between tags: each sentence closed in it is found with the text before it on its line; the rest runs on. */
+const scanText = (scan: Scan, text: string, at: number): Scan => {
+  const start = scan.line === "" ? at : scan.start;
+  const ends = [...text.matchAll(SENTENCE_END)].map((end) => end.index + end[0].length);
+  if (ends.length === 0) return { start, line: scan.line + text, found: scan.found };
+  ends.forEach((end, index) => scan.found.push({ at: start, text: (index === 0 ? scan.line : "") + text.slice(ends[index - 1] ?? 0, end) }));
+  return { start: at, line: text.slice(ends.at(-1) ?? text.length), found: scan.found };
+};
+
+type TagScan = Scan & { readonly from: number };
+
+/** The text before a tag, then the tag: a line break starts a new line. */
+const scanTag = (html: string, scan: TagScan, tag: RegExpExecArray): TagScan => {
+  const before = html.slice(scan.from, tag.index);
+  const scanned = before === "" ? scan : scanText(scan, decodeEntities(before), scan.from);
+  const next = LINE_BREAK.test(tag[0]) ? { start: tag.index, line: "", found: scanned.found } : scanned;
+  return { ...next, from: tag.index + tag[0].length };
+};
+
+/** Each sentence on the page with the words before it on its line, and where that line starts; in document order. */
+const sentences = (html: string): Sentence[] => {
+  const scan = [...html.matchAll(TAG)].reduce<TagScan>((state, tag) => scanTag(html, state, tag), { start: 0, line: "", found: [], from: 0 });
+  const rest = html.slice(scan.from);
+  return (rest === "" ? scan : scanText(scan, decodeEntities(rest), scan.from)).found;
+};
+
+const TITLE = /<h1\b[^>]*>([\s\S]*?)<\/h1\s*>/giu;
+
+type Weighed = { readonly at: number; readonly weight: number };
+
+/**
+ * How much text each sentence carries: its length, or nothing when it only repeats the page's title (a question set as
+ * the title and again in a breadcrumb), so that a short page's title does not outweigh its body.
+ */
+const weighed = (html: string): Weighed[] => {
+  const titles = new Set([...html.matchAll(TITLE)].map((title) => plainText(title[1] ?? "")));
+  return sentences(html).map((sentence) => {
+    const text = sentence.text.replace(/\s+/gu, " ").trim();
+    return { at: sentence.at, weight: titles.has(text) ? 0 : text.length };
+  });
+};
+
+/** The total weight of the sentences whose line starts in [start, end), from running totals (totals[i] = before i). */
+const weightWithin = (all: readonly Weighed[], totals: readonly number[], start: number, end: number): number => {
+  const firstAtOrAfter = (at: number, low = 0, high = all.length): number => {
+    if (low >= high) return low;
+    const middle = Math.floor((low + high) / 2);
+    return (all[middle]?.at ?? 0) < at ? firstAtOrAfter(at, middle + 1, high) : firstAtOrAfter(at, low, middle);
+  };
+  return (totals[firstAtOrAfter(end)] ?? 0) - (totals[firstAtOrAfter(start)] ?? 0);
+};
+
+const runningTotals = (all: readonly Weighed[]): number[] =>
+  all.reduce<number[]>(
+    (sums, sentence) => {
+      sums.push((sums.at(-1) ?? 0) + sentence.weight);
+      return sums;
+    },
+    [0],
+  );
+
+// Share of the page's sentence text the block must hold: a line in the footer may fall outside it, a section of the
+// document may not.
+const CONTENT_SHARE = 0.9;
+
+const CONTENT_BLOCKS = ["div", "section"];
+
+/**
+ * The inside of the content block, or undefined when the page has no sentence, no block to single out, or no <h1>:
+ * without a marked title, a title set in a plain block (and its byline) could fall outside.
+ */
+export const contentBlock = (html: string): string | undefined => {
+  if (!/<h1\b/iu.test(html)) return undefined;
+  const headings = [...html.matchAll(/<h[1-6]\b/giu)].map((heading) => heading.index);
+  const all = weighed(html);
+  const totals = runningTotals(all);
+  const whole = totals.at(-1) ?? 0;
+  if (whole === 0) return undefined;
+  // Every heading is the document's (a list under "Members" has no sentence), and nothing written in sentences may
+  // stand before the block ("Updated." above the title): only a footer's line after it.
+  const holdsContent = (range: ElementRange): boolean =>
+    headings.every((heading) => heading > range.start && heading < range.end) &&
+    weightWithin(all, totals, 0, range.start) === 0 &&
+    weightWithin(all, totals, range.start, range.end) >= whole * CONTENT_SHARE;
+  const smallest = CONTENT_BLOCKS.flatMap((tag) => elementRanges(html, tag))
+    .filter(holdsContent)
+    .toSorted((left, right) => left.end - left.start - (right.end - right.start))[0];
+  return smallest?.inner;
+};

package/src/html/html-elements.ts

@@ -0,0 +1,127 @@
+// The pieces of an HTML page the converters look at: tags, links, attributes and elements matched to their own
+// closing tag. Pure; a regular-expression reading, not a parser.
+import { decodeEntities } from "./markup-text.ts";
+
+// In a tag a quote opens a value only after "=", and the value ends only at its matching quote, whatever it holds.
+const TAG_BODY = String.raw`(?:[^>=]|=\s*"[^"]*"|=\s*'[^']*'|=(?!\s*["']))*`;
+
+// A comment, a processing instruction, a script and a style are matched whole, so that no tag is read inside them
+// (the converter drops them anyway), and a "<!--" inside a value is escaped before comments are looked for.
+const TAG = new RegExp(String.raw`<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<(script|style)\b${TAG_BODY}>[\s\S]*?<\/\1\s*>|<[a-z][a-z0-9-]*${TAG_BODY}>`, "giu");
+
+const QUOTED_VALUE = /=(\s*)("[^"]*"|'[^']*')/gu;
+
+const withValuesEscaped = (tag: string): string =>
+  tag.replace(QUOTED_VALUE, (_whole: string, space: string, value: string) => `=${space}${value.replaceAll("<", "&lt;").replaceAll(">", "&gt;")}`);
+
+/**
+ * Each "<" and ">" inside a quoted attribute value (title="published under the <span>…</span>") as a character
+ * reference, which means the same in a value, so that a scanner reading a tag up to its first ">" reads all of it.
+ */
+export const withAttributeMarkupEscaped = (html: string): string => html.replace(TAG, (tag: string) => withValuesEscaped(tag));
+
+/** Apply step until the text stops changing: nested elements are removed from the inside out. */
+export const untilStable = (text: string, step: (text: string) => string): string => {
+  const next = step(text);
+  return next === text ? text : untilStable(next, step);
+};
+
+export const stripTags = (html: string): string => html.replace(/<\/?[a-z!][^>]*>/giu, "");
+
+export const plainText = (html: string): string => decodeEntities(stripTags(html)).replace(/\s+/gu, " ").trim();
+
+/** Text put back into markup, so that decoding it again gives the same text. */
+export const asMarkup = (text: string): string => text.replaceAll("&", "&amp;").replaceAll("<", "&lt;");
+
+/** A full stop, question or exclamation mark closing a sentence, also before a closing quote or bracket; not the point in "3.5". */
+export const CLOSED_SENTENCE = /[。．！？]|[.!?][)\]"'”’]*(?=\s|$)/u;
+
+/** Whether the markup closes a sentence, each tag read as a space so that "Done.</li><li>Next" ends one. */
+export const hasSentence = (html: string): boolean => CLOSED_SENTENCE.test(plainText(html.replace(/<\/?[a-z!][^>]*>/giu, " ")));
+
+export const hasNoWords = (text: string): boolean => !/[\p{L}\p{N}]/u.test(text);
+
+export const ANY_LINK = /<a\b[^>]*>[\s\S]*?<\/a\s*>/giu;
+
+const HREF = /\shref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/iu;
+
+const PAGE_EXTENSION = /^(?:[sx]?html?|php|aspx?|jsp|cgi)$/iu;
+
+// The host is not part of the path: "https://example.com" names a site, not a file ending in .com.
+const SCHEME_AND_HOST = /^(?:[a-z][a-z0-9+.-]*:)?\/\/[^/?#]*/iu;
+
+// A link that opens no page (tel:, mailto:, sms:) carries its own content, a number or an address.
+const OTHER_SCHEME = /^\s*(?!https?:)[a-z][a-z0-9+.-]*:(?!\/\/)/iu;
+
+const hrefOf = (link: string): string => {
+  const href = HREF.exec(link);
+  return href?.[1] ?? href?.[2] ?? href?.[3] ?? "";
+};
+
+/** The extension of the file a link opens ("pdf", "mp3"), or undefined for a page or a site. */
+export const fileExtension = (link: string): string | undefined => {
+  const path = hrefOf(link).replace(SCHEME_AND_HOST, "").split(/[?#]/u)[0] ?? "";
+  const extension = /\.([a-z0-9]{1,5})$/iu.exec(path.split("/").at(-1) ?? "")?.[1];
+  return extension !== undefined && !PAGE_EXTENSION.test(extension) ? extension : undefined;
+};
+
+/** A link to a file (an appendix as PDF, a table as .xlsx), or one that opens no page, is part of the document. */
+export const isFileLink = (link: string): boolean => OTHER_SCHEME.test(hrefOf(link)) || fileExtension(link) !== undefined;
+
+/** Attributes before a given one, each skipped whole so that a quoted value (title="x role=main") is not read as one. */
+export const ATTRIBUTES = String.raw`(?:\s+[^\s"'>=/]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s"'>]+))?)*?`;
+
+export type ElementRange = { readonly start: number; readonly end: number; readonly openTag: string; readonly inner: string };
+
+type RangeScan = { readonly open: readonly RegExpExecArray[]; readonly ranges: readonly ElementRange[] };
+
+const closeLast = (html: string, scan: RangeScan, close: RegExpExecArray): RangeScan => {
+  const opener = scan.open.at(-1);
+  if (opener === undefined) return scan;
+  const range = {
+    start: opener.index,
+    end: close.index + close[0].length,
+    openTag: opener[0],
+    inner: html.slice(opener.index + opener[0].length, close.index),
+  };
+  return { open: scan.open.slice(0, -1), ranges: [...scan.ranges, range] };
+};
+
+/** Every <tag>…</tag>, nested ones matched to their own closing tag, in document order (outer before inner). */
+export const elementRanges = (html: string, tag: string): ElementRange[] =>
+  [...html.matchAll(new RegExp(`<(/?)${tag}\\b[^>]*>`, "giu"))]
+    .reduce<RangeScan>((scan, match) => (match[1] === "/" ? closeLast(html, scan, match) : { open: [...scan.open, match], ranges: scan.ranges }), {
+      open: [],
+      ranges: [],
+    })
+    .ranges.toSorted((left, right) => left.start - right.start);
+
+export const isInside = (outer: ElementRange, inner: ElementRange): boolean => outer.start < inner.start && inner.end <= outer.end;
+
+/** The ranges (in document order) each replaced by a space; one inside another cut one goes with it. */
+export const withoutRanges = (html: string, ranges: readonly ElementRange[]): string => {
+  const chosen = ranges.reduce<ElementRange[]>((kept, range) => {
+    const insideKept = (kept.at(-1)?.end ?? 0) > range.start;
+    if (!insideKept) kept.push(range);
+    return kept;
+  }, []);
+  const cut = chosen.reduce<{ readonly parts: readonly string[]; readonly from: number }>(
+    (acc, range) => ({ parts: [...acc.parts, html.slice(acc.from, range.start), " "], from: range.end }),
+    { parts: [], from: 0 },
+  );
+  return [...cut.parts, html.slice(cut.from)].join("");
+};
+
+/** The <tag> elements that are chrome, each replaced by a space; one inside another dropped one goes with it. */
+export const withoutElementsWhere = (html: string, tag: string, isChrome: (range: ElementRange) => boolean): string =>
+  withoutRanges(html, elementRanges(html, tag).filter(isChrome));
+
+/** Every <tag> whose opening matches, for each tag that opens that way, replaced by a space. */
+export const withoutElementsOpening = (html: string, opening: RegExp, isChrome: (range: ElementRange) => boolean): string => {
+  const tags = new Set([...html.matchAll(opening)].map((match) => (match[1] ?? "").toLowerCase()));
+  return [...tags].reduce((text, tag) => withoutElementsWhere(text, tag, isChrome), html);
+};
+
+export const HEADING_TAGS = ["h1", "h2", "h3", "h4", "h5", "h6"];
+
+export const headingRanges = (html: string): ElementRange[] => HEADING_TAGS.flatMap((tag) => elementRanges(html, tag));

package/src/html/html-footnote-marks.ts

@@ -0,0 +1,20 @@
+// A footnote's reference mark: a superscript that is a link (<sup><a …>1</a></sup>), or a link to a place on the same
+// page holding nothing but a superscript (<a href="#fn1"><sup>1</sup></a>). Read as text the number sticks to the
+// sentence it follows ("recessions.1 Over"), so the mark is dropped; the note itself stays where the page has it. A
+// superscript that is a link is a mark wherever the link goes (the notes may be on another page); a link around a
+// superscript is one only when it stays on this page, since "Brand<a href=…><sup>TM</sup></a>" is a sign. A superscript
+// with no link ("25<sup>th</sup>") is text. Pure.
+
+const LINK_IN_SUPERSCRIPT = /<sup\b[^>]*>\s*<a\b[^>]*>[^<]*<\/a\s*>\s*<\/sup\s*>/giu;
+
+const SUPERSCRIPT_LINK = /<a\b([^>]*)>\s*<sup\b[^>]*>([^<]*)<\/sup\s*>\s*<\/a\s*>/giu;
+
+// The fragment must name something: href="#" alone goes nowhere and is a control.
+const IN_PAGE_HREF = /\shref\s*=\s*(?:"#[^"\s]+"|'#[^'\s]+'|#[^\s"'>]+)/iu;
+
+const isFootnoteLink = (attributes: string, mark: string): boolean => IN_PAGE_HREF.test(attributes) && mark.trim() !== "";
+
+export const withoutFootnoteMarks = (html: string): string =>
+  html
+    .replace(LINK_IN_SUPERSCRIPT, "")
+    .replace(SUPERSCRIPT_LINK, (link: string, attributes: string, mark: string) => (isFootnoteLink(attributes, mark) ? "" : link));

package/src/html/html-heading-links.ts

@@ -0,0 +1,93 @@
+// A heading's link to its own section (a permalink beside or after the title), taken out of an HTML page. Pure.
+import { ANY_LINK, ATTRIBUTES, elementRanges, hasNoWords, plainText, type ElementRange } from "./html-elements.ts";
+import { decodeEntities } from "./markup-text.ts";
+
+const ID = new RegExp(String.raw`<([a-z][a-z0-9-]*)${ATTRIBUTES}\s+id\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))`, "giu");
+
+type IdTarget = { readonly start: number; readonly tag: string };
+
+/** Where each id's element opens, and which element it is; the first one when an id is repeated. */
+const idTargets = (html: string): Map<string, IdTarget> =>
+  [...html.matchAll(ID)].reduce((targets, match) => {
+    const id = decodeEntities(match[2] ?? match[3] ?? match[4] ?? "");
+    return targets.has(id) ? targets : targets.set(id, { start: match.index, tag: (match[1] ?? "").toLowerCase() });
+  }, new Map<string, IdTarget>());
+
+const FRAGMENT = /^<a\b[^>]*\bhref\s*=\s*["']?[^"'#\s>]*#([^"'\s>]+)/iu;
+
+/** The id a link's fragment names, its character references and percent escapes undone. */
+const fragmentId = (link: string): string | undefined => {
+  const fragment = FRAGMENT.exec(link)?.[1];
+  if (fragment === undefined) return undefined;
+  const decoded = decodeEntities(fragment);
+  try {
+    return decodeURIComponent(decoded);
+  } catch {
+    return decoded;
+  }
+};
+
+const HEADING_OPENING = /<h[1-6]\b/iu;
+
+type PageIds = {
+  readonly ids: ReadonlyMap<string, IdTarget>;
+  readonly html: string;
+  readonly rangesOf: (tag: string) => readonly ElementRange[];
+};
+
+type HeadingPlace = PageIds & { readonly start: number; readonly end: number };
+
+/** The target opens before the heading and closes after it, with no other heading opening in between. */
+const isSectionOf = (target: IdTarget, place: HeadingPlace): boolean =>
+  place.rangesOf(target.tag).some((range) => range.start === target.start && range.end >= place.end) &&
+  !HEADING_OPENING.test(place.html.slice(target.start, place.start));
+
+/** A link to the heading's own section: the heading, something inside it, or the block it opens (a permalink, "Copy link to Context"). */
+const isSelfLink = (link: string, place: HeadingPlace): boolean => {
+  const id = fragmentId(link);
+  const target = id === undefined ? undefined : place.ids.get(id);
+  if (target === undefined || target.start >= place.end) return false;
+  return target.start >= place.start || isSectionOf(target, place);
+};
+
+/** A permalink mark inside the heading (¶, #, an icon); a self-link with words is the title, or part of it. */
+const isPermalinkMark = (link: string, place: HeadingPlace): boolean => hasNoWords(plainText(link)) && isSelfLink(link, place);
+
+const HEADING_THEN_LINK = /(<h([1-6])\b[^>]*>)([\s\S]*?)(<\/h\2\s*>)(\s*<a\b[^>]*>[\s\S]*?<\/a\s*>)?/giu;
+
+/** The heading as matched, without its permalink marks or the self-link standing after it. */
+const headingWithoutSelfLinks = (match: RegExpExecArray, page: PageIds): string => {
+  const [, open = "", , inside = "", close = "", after] = match;
+  const place = { ...page, start: match.index, end: match.index + open.length + inside.length + close.length };
+  const kept = inside.replace(ANY_LINK, (link: string) => (isPermalinkMark(link, place) ? " " : link));
+  const next = after === undefined || isSelfLink(after.trim(), place) ? "" : after;
+  return `${open}${kept}${close}${next}`;
+};
+
+/** elementRanges for each tag, computed once. */
+const rangesByTag = (html: string): ((tag: string) => readonly ElementRange[]) => {
+  const cache = new Map<string, readonly ElementRange[]>();
+  return (tag: string) => {
+    const known = cache.get(tag);
+    if (known !== undefined) return known;
+    const ranges = elementRanges(html, tag);
+    cache.set(tag, ranges);
+    return ranges;
+  };
+};
+
+/**
+ * A heading's link to its own section goes when it stands just after the heading ("Copy link to Context"), or inside
+ * it as a mark without words ("Scope ¶"). A link with words inside the heading is its title and stays.
+ */
+export const withoutHeadingSelfLinks = (html: string): string => {
+  const page = { ids: idTargets(html), html, rangesOf: rangesByTag(html) };
+  const cut = [...html.matchAll(HEADING_THEN_LINK)].reduce<{ readonly parts: readonly string[]; readonly from: number }>(
+    (acc, match) => ({
+      parts: [...acc.parts, html.slice(acc.from, match.index), headingWithoutSelfLinks(match, page)],
+      from: match.index + match[0].length,
+    }),
+    { parts: [], from: 0 },
+  );
+  return [...cut.parts, html.slice(cut.from)].join("");
+};

package/src/html/html-lost-tags.ts

@@ -0,0 +1,14 @@
+// A paragraph's start tag that lost its "<" upstream and so reached the page as text: the eCFR serves 16 CFR 310.4(b)
+// with "… 45 CFR 160.103. P&gt;(2) It is an abusive …", where "P>" is what is left of the <P> that opened paragraph (2).
+// Read as text it glues the paragraph to the one before and leaves "P>" in the sentence. Only a P and a ">" standing
+// after a sentence's end and right before a paragraph's designation ("(2)", "(iv)", "(A)") and a capital are taken for the lost
+// tag; "P>0.05", "P > (2)" and a formula opening a sentence ("P>(2) follows") stay text. Pure.
+
+const LOST_PARAGRAPH_TAG = /(?<=[.;:][)"'”’]?[ \t])[Pp](?:&gt;|>)(?=\([0-9A-Za-z]{1,4}\)[ \t]\p{Lu})/gu;
+
+/** Whether at is inside a tag. An attribute's ">" is escaped by then (withAttributeMarkupEscaped), so a tag has none before its end. */
+const isInsideTag = (html: string, at: number): boolean => html.lastIndexOf("<", at) > html.lastIndexOf(">", at - 1);
+
+/** The HTML with each lost paragraph start tag in its text put back as <p>. */
+export const withLostParagraphTagsRestored = (html: string): string =>
+  html.replace(LOST_PARAGRAPH_TAG, (lost: string, at: number) => (isInsideTag(html, at) ? lost : "<p>"));