@v1nvn/readability 0.43.0 → 0.45.0

package/src/pipeline/context.ts

@@ -0,0 +1,88 @@
+import type { ReadabilityParseResult } from './readability.ts';
+
+interface GatingSignal {
+  readonly likely: boolean;
+  readonly reason: string;
+}
+
+interface PaginationSignal {
+  readonly nextUrl?: string;
+  readonly selector?: string;
+  readonly type: 'infinite' | 'paginated';
+}
+
+interface PresetSignal {
+  readonly applied: boolean;
+  readonly reason?: 'detectors-missed' | 'overridden';
+  readonly site: string;
+}
+
+export interface StructuredData {
+  readonly '@type'?: string;
+  readonly [key: string]: unknown;
+}
+
+export interface Metadata {
+  readonly baseUrl?: string;
+  readonly byline?: string;
+  readonly canonical?: string;
+  readonly estimator?: string;
+  readonly excerpt?: string;
+  readonly lang?: string;
+  readonly publishedTime?: string;
+  readonly readingTimeMin?: number;
+  readonly siteName?: string;
+  readonly structured?: StructuredData;
+  readonly title?: string;
+  readonly tokenEstimate?: number;
+  readonly wordCount?: number;
+}
+
+export interface SanitizationDiagnostics {
+  readonly iframes: number;
+  readonly scripts: number;
+}
+
+export interface TraceStage {
+  readonly ms: number;
+  readonly stage: string;
+}
+
+export interface CacheSignal {
+  readonly hit: boolean;
+  readonly normalizedHash: string;
+  readonly originalHash: string;
+}
+
+export interface Diagnostics {
+  readonly boilerplateRemoved?: number;
+  readonly cache?: CacheSignal;
+  readonly chromeRemoved?: number;
+  readonly extractedNode?: string;
+  readonly fallbackUsed: boolean;
+  readonly gated?: GatingSignal;
+  readonly imagesResolved?: number;
+  readonly pagination?: PaginationSignal;
+  readonly preset?: PresetSignal;
+  readonly readerable?: boolean;
+  readonly removedNodes?: number;
+  readonly sanitization?: SanitizationDiagnostics;
+  readonly trace?: readonly TraceStage[];
+  readonly truncated: boolean;
+}
+
+export interface ExtractionContext {
+  readonly article?: null | Readonly<ReadabilityParseResult>;
+  readonly baseUrl?: string;
+  readonly diagnostics?: Diagnostics;
+  readonly document?: Document;
+  readonly documentElementCount?: number;
+  readonly html: string;
+  readonly markdown?: string;
+  readonly metadata?: Metadata;
+  readonly readerable?: boolean;
+  readonly rootSelector?: string;
+  readonly sanitizedHtml?: string;
+  // Must be the same window the document came from — DOMPurify is bound to it.
+  readonly window?: Window;
+}

package/src/pipeline/dom.ts

@@ -0,0 +1,23 @@
+import { JSDOM } from 'jsdom';
+
+export interface BuiltDocument {
+  readonly document: Document;
+  readonly window: Window;
+}
+
+// Built only from caller-supplied `html`; `baseUrl` is origin context for
+// absolutization, never fetched. Parse and the downstream Readability walk run
+// synchronously on the event loop — no worker isolation or wall-clock timeout
+// guards them. A worker pool was deferred rather than rejected: the synchronous
+// parse can't be interrupted in-process, and `worker_threads` can't load the dev
+// hot-reload loop's in-memory Vite modules. `maxNodes` is the only input-size
+// bound until a holistic worker strategy lands; a standalone `timeout` option
+// would mislead (it can't interrupt the parse), so it ships with the worker.
+export function buildDocument(html: string, baseUrl?: string): BuiltDocument {
+  const dom = new JSDOM(html, { url: baseUrl });
+  return { document: dom.window.document, window: dom.window };
+}
+
+export function isElement(node: Node): node is Element {
+  return node.nodeType === 1;
+}

package/src/pipeline/footnotes.ts

@@ -0,0 +1,181 @@
+import { buildDocument } from './dom.ts';
+
+export interface FootnoteResult {
+  readonly footnoteDefs: readonly string[];
+  readonly html: string;
+}
+
+// jsdom-parsing every turndown call would dominate hot paths; this pre-check
+// gates on a string scan so footnote-less articles pay nothing.
+const FOOTNOTE_SIGNAL_RE =
+  /cite_note|cite_ref|class="footnotes"|class="references"|data-footnote|role="doc-endnote|<sup\b[^>]*>\s*<a\s[^>]*href="#/i;
+
+// First match wins per group; an `<ol class="footnotes">` is the common case.
+const DEFINITION_CONTAINER_SELECTORS = [
+  'ol.footnotes',
+  'ol[class*="footnotes"]',
+  'ol.references',
+  'ol[class*="references"]',
+  'section[class*="footnote"]',
+  'div[class*="footnote"]',
+  '[role="doc-endnotes"]',
+  '[role="doc-bibliography"]',
+] as const;
+
+const STANDALONE_DEF_ID_RE = /^(?:fn|cite_note|footnote|note)[:_-]/i;
+
+interface RefHit {
+  readonly defId: string;
+  readonly n: number;
+  readonly sup: Element;
+}
+
+// Some sites prepend a back-to-text caret or "Jump to" label to each definition.
+// Stripping only a leading occurrence keeps accidental mid-text carets intact.
+const BACKREF_LEADING_RE = /^(?:↑\s?|↩\s?|\^\s|Jump to\s*)/;
+
+function cleanDefText(text: string): string {
+  return text.replace(/\s+/g, ' ').trim().replace(BACKREF_LEADING_RE, '');
+}
+
+function collectDefinitions(document: Document): {
+  readonly containers: ReadonlySet<Element>;
+  readonly defs: ReadonlyMap<string, string>;
+  readonly standaloneIds: ReadonlySet<string>;
+} {
+  const containers = new Set<Element>();
+  const defs = new Map<string, string>();
+  const standaloneIds = new Set<string>();
+
+  for (const selector of DEFINITION_CONTAINER_SELECTORS) {
+    let matched: NodeListOf<Element>;
+    try {
+      matched = document.querySelectorAll(selector);
+    } catch {
+      continue;
+    }
+    for (const container of Array.from(matched)) {
+      if (!containers.has(container)) {
+        containers.add(container);
+      }
+      for (const item of Array.from(
+        container.querySelectorAll('li, [role="doc-endnote"]'),
+      )) {
+        const id = item.id;
+        if (!id || defs.has(id)) {
+          continue;
+        }
+        const text = cleanDefText(item.textContent);
+        if (text) {
+          defs.set(id, text);
+        }
+      }
+    }
+  }
+
+  // Some sites scatter `<li id="fn1">` outside any container; collect those too.
+  for (const li of Array.from(document.querySelectorAll('li[id]'))) {
+    const id = li.id;
+    if (!id || defs.has(id) || !STANDALONE_DEF_ID_RE.test(id)) {
+      continue;
+    }
+    const text = cleanDefText(li.textContent);
+    if (text) {
+      defs.set(id, text);
+      standaloneIds.add(id);
+    }
+  }
+
+  return { containers, defs, standaloneIds };
+}
+
+export function processFootnotes(html: string): FootnoteResult | null {
+  if (!html || !FOOTNOTE_SIGNAL_RE.test(html)) {
+    return null;
+  }
+
+  let document: Document;
+  try {
+    document = buildDocument(html).document;
+  } catch {
+    return null;
+  }
+
+  const { containers, defs, standaloneIds } = collectDefinitions(document);
+  if (defs.size === 0) {
+    return null;
+  }
+
+  const refHits: RefHit[] = [];
+  const defIdToNumber = new Map<string, number>();
+  for (const sup of Array.from(document.querySelectorAll('sup'))) {
+    if (!sup.isConnected) {
+      continue;
+    }
+    try {
+      const anchor = sup.querySelector('a[href^="#"]');
+      if (!anchor) {
+        continue;
+      }
+      const frag = (anchor.getAttribute('href') ?? '').slice(1);
+      if (!frag || !defs.has(frag)) {
+        continue;
+      }
+      let n = defIdToNumber.get(frag);
+      if (n === undefined) {
+        n = defIdToNumber.size + 1;
+        defIdToNumber.set(frag, n);
+      }
+      refHits.push({ defId: frag, n, sup });
+    } catch {
+      // Defensive: malformed <sup> must not abort the whole pass.
+    }
+  }
+
+  if (refHits.length === 0) {
+    return null;
+  }
+
+  for (const { n, sup } of refHits) {
+    sup.replaceWith(document.createTextNode(`[^${n}]`));
+  }
+
+  // Drop the rendered references list so it doesn't duplicate the appended
+  // `[^N]:` block. Removing containers also takes care of the standalone <li>
+  // elements that lived inside them; only out-of-container standalones need
+  // individual removal here.
+  for (const container of containers) {
+    if (container.isConnected) {
+      container.remove();
+    }
+  }
+  for (const id of standaloneIds) {
+    const el = document.getElementById(id);
+    if (el?.isConnected) {
+      el.remove();
+    }
+  }
+
+  const numberToDefId = new Map<number, string>();
+  for (const [id, n] of defIdToNumber) {
+    numberToDefId.set(n, id);
+  }
+  const footnoteDefs: string[] = [];
+  for (let n = 1; n <= numberToDefId.size; n++) {
+    const id = numberToDefId.get(n);
+    if (id === undefined) {
+      break;
+    }
+    const text = defs.get(id);
+    if (text !== undefined) {
+      footnoteDefs.push(text);
+    }
+  }
+
+  // buildDocument wraps both full documents and bare fragments in a <body>;
+  // serializing its innerHTML returns the equivalent of the input region.
+  return {
+    footnoteDefs,
+    html: document.body.innerHTML,
+  };
+}

package/src/pipeline/math.ts

@@ -0,0 +1,130 @@
+// Math renderers leave the original LaTeX in a small, engine-agnostic pocket:
+// the `<annotation encoding="application/x-tex">` child of a `<math>` element.
+// KaTeX wraps it inside `.katex`, arXiv/LaTeXML (ar5iv) and MDN ship it in a
+// bare `<math display="…">`, and MathJax v2/v3 do not use `<annotation>` at
+// all — their source lives in `<script type="math/tex">` that the
+// script-removal pass strips. Move each into one controlled marker span BEFORE
+// normalize's `<script>` removal so every engine shares a single turndown rule
+// that emits the LaTeX verbatim.
+
+const MARKER_CLASS = 'rdrm-math';
+const DISPLAY_ATTR = 'data-display';
+const KATEX_CLASS = 'katex';
+const KATEX_DISPLAY_CLASS = 'katex-display';
+const TEX_ANNOTATION_SELECTOR = 'annotation[encoding="application/x-tex"]';
+const MATHJAX_INLINE_TYPE = 'math/tex';
+const MATHJAX_DISPLAY_TYPE = 'math/tex; mode=display';
+// arXiv/LaTeXML (`ltx_*`) and common docs-site display-math wrappers. A bare
+// `<math display="block">` also flags display, but pages often omit the
+// attribute and signal block context only through the wrapper class.
+const MATHML_DISPLAY_SELECTOR =
+  '.ltx_equation, .ltx_displaymath, .equation-display, .math-display';
+// Placeholder for a math container whose annotation and alttext are both empty
+// or absent; rendered markup has no recoverable LaTeX, so emit a token rather
+// than crash.
+const BROKEN_PLACEHOLDER = '[?]';
+
+function createMarker(
+  document: Document,
+  tex: string,
+  display: boolean,
+): HTMLElement {
+  const marker = document.createElement('span');
+  marker.className = MARKER_CLASS;
+  marker.setAttribute(DISPLAY_ATTR, display ? 'true' : 'false');
+  marker.textContent = tex;
+  return marker;
+}
+
+// Drives off the annotation, not the wrapper, so one pass recovers LaTeX from
+// every engine that emits `<annotation encoding="application/x-tex">` — KaTeX
+// inside `.katex`, arXiv/MDN inside a bare `<math>`. The container replaced is
+// engine-specific: the `.katex` span (which also drops the `.katex-html`
+// rendered sibling), or the bare `<math>` element. Replacing a container
+// detaches other annotations that lived inside it, so disconnected nodes
+// reached later in document order are skipped. arXiv carries the LaTeX in
+// `<math alttext="…">` as well, which is the fallback when the annotation text
+// is empty.
+function convertAnnotations(document: Document): void {
+  const annotations = document.querySelectorAll(TEX_ANNOTATION_SELECTOR);
+  for (const annotation of Array.from(annotations)) {
+    if (!annotation.isConnected) {
+      continue;
+    }
+    try {
+      const katex = annotation.closest(`.${KATEX_CLASS}`);
+      const math = annotation.closest('math');
+      let container: Element = annotation;
+      let display = false;
+      if (katex) {
+        container = katex;
+        display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+      } else if (math) {
+        container = math;
+        display =
+          math.getAttribute('display') === 'block' ||
+          math.closest(MATHML_DISPLAY_SELECTOR) !== null;
+      }
+      const tex =
+        annotation.textContent.trim() ||
+        (math?.getAttribute('alttext') ?? '').trim();
+      container.replaceWith(
+        createMarker(document, tex || BROKEN_PLACEHOLDER, display),
+      );
+    } catch {
+      annotation.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+    }
+  }
+}
+
+// A `.katex` that survived the annotation pass had no recoverable annotation
+// (stripped or never present). Its `.katex-html` rendered tree carries no
+// recoverable LaTeX, so swap the whole span for a placeholder marker rather
+// than let rendered spans leak into turndown.
+function convertOrphanedKatex(document: Document): void {
+  for (const katex of Array.from(
+    document.getElementsByClassName(KATEX_CLASS),
+  )) {
+    if (!katex.isConnected) {
+      continue;
+    }
+    const display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+    katex.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, display));
+  }
+}
+
+// MathJax v2/v3 inline + display source scripts. These `<script>` nodes are
+// removed by normalize's generic script-removal AND by DOMPurify, so they must
+// be converted here, before that loop runs. MathJax scripts do not use
+// `<annotation>`, so they are handled separately from `convertAnnotations`.
+function convertMathJax(document: Document): void {
+  const scripts = document.querySelectorAll(
+    `script[type="${MATHJAX_INLINE_TYPE}"], script[type="${MATHJAX_DISPLAY_TYPE}"]`,
+  );
+  for (const script of Array.from(scripts)) {
+    if (!script.isConnected) {
+      continue;
+    }
+    try {
+      const rawType = script.getAttribute('type') ?? '';
+      const display = rawType.includes('mode=display');
+      const tex = script.textContent.trim();
+      script.replaceWith(
+        createMarker(document, tex || BROKEN_PLACEHOLDER, display),
+      );
+    } catch {
+      script.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+    }
+  }
+}
+
+// Walks the pre-normalize document and rewrites every math container —
+// annotation-bearing (KaTeX, bare MathML) or MathJax script — into a
+// `<span class="rdrm-math" data-display="…">LATEX</span>` marker. Idempotent:
+// markers carry no `.katex`/`<math>`/annotation, so a second pass is a no-op.
+// Defensive per node so malformed markup never aborts the pass.
+export function extractMath(document: Document): void {
+  convertAnnotations(document);
+  convertOrphanedKatex(document);
+  convertMathJax(document);
+}