@v1nvn/readability 0.44.0 → 0.45.1

package/src/pipeline/footnotes.js

@@ -0,0 +1,148 @@
+import { buildDocument } from "./dom.js";
+// jsdom-parsing every turndown call would dominate hot paths; this pre-check
+// gates on a string scan so footnote-less articles pay nothing.
+const FOOTNOTE_SIGNAL_RE = /cite_note|cite_ref|class="footnotes"|class="references"|data-footnote|role="doc-endnote|<sup\b[^>]*>\s*<a\s[^>]*href="#/i;
+// First match wins per group; an `<ol class="footnotes">` is the common case.
+const DEFINITION_CONTAINER_SELECTORS = [
+    'ol.footnotes',
+    'ol[class*="footnotes"]',
+    'ol.references',
+    'ol[class*="references"]',
+    'section[class*="footnote"]',
+    'div[class*="footnote"]',
+    '[role="doc-endnotes"]',
+    '[role="doc-bibliography"]',
+];
+const STANDALONE_DEF_ID_RE = /^(?:fn|cite_note|footnote|note)[:_-]/i;
+// Some sites prepend a back-to-text caret or "Jump to" label to each definition.
+// Stripping only a leading occurrence keeps accidental mid-text carets intact.
+const BACKREF_LEADING_RE = /^(?:↑\s?|↩\s?|\^\s|Jump to\s*)/;
+function cleanDefText(text) {
+    return text.replace(/\s+/g, ' ').trim().replace(BACKREF_LEADING_RE, '');
+}
+function collectDefinitions(document) {
+    const containers = new Set();
+    const defs = new Map();
+    const standaloneIds = new Set();
+    for (const selector of DEFINITION_CONTAINER_SELECTORS) {
+        let matched;
+        try {
+            matched = document.querySelectorAll(selector);
+        }
+        catch {
+            continue;
+        }
+        for (const container of Array.from(matched)) {
+            if (!containers.has(container)) {
+                containers.add(container);
+            }
+            for (const item of Array.from(container.querySelectorAll('li, [role="doc-endnote"]'))) {
+                const id = item.id;
+                if (!id || defs.has(id)) {
+                    continue;
+                }
+                const text = cleanDefText(item.textContent);
+                if (text) {
+                    defs.set(id, text);
+                }
+            }
+        }
+    }
+    // Some sites scatter `<li id="fn1">` outside any container; collect those too.
+    for (const li of Array.from(document.querySelectorAll('li[id]'))) {
+        const id = li.id;
+        if (!id || defs.has(id) || !STANDALONE_DEF_ID_RE.test(id)) {
+            continue;
+        }
+        const text = cleanDefText(li.textContent);
+        if (text) {
+            defs.set(id, text);
+            standaloneIds.add(id);
+        }
+    }
+    return { containers, defs, standaloneIds };
+}
+export function processFootnotes(html) {
+    if (!html || !FOOTNOTE_SIGNAL_RE.test(html)) {
+        return null;
+    }
+    let document;
+    try {
+        document = buildDocument(html).document;
+    }
+    catch {
+        return null;
+    }
+    const { containers, defs, standaloneIds } = collectDefinitions(document);
+    if (defs.size === 0) {
+        return null;
+    }
+    const refHits = [];
+    const defIdToNumber = new Map();
+    for (const sup of Array.from(document.querySelectorAll('sup'))) {
+        if (!sup.isConnected) {
+            continue;
+        }
+        try {
+            const anchor = sup.querySelector('a[href^="#"]');
+            if (!anchor) {
+                continue;
+            }
+            const frag = (anchor.getAttribute('href') ?? '').slice(1);
+            if (!frag || !defs.has(frag)) {
+                continue;
+            }
+            let n = defIdToNumber.get(frag);
+            if (n === undefined) {
+                n = defIdToNumber.size + 1;
+                defIdToNumber.set(frag, n);
+            }
+            refHits.push({ defId: frag, n, sup });
+        }
+        catch {
+            // Defensive: malformed <sup> must not abort the whole pass.
+        }
+    }
+    if (refHits.length === 0) {
+        return null;
+    }
+    for (const { n, sup } of refHits) {
+        sup.replaceWith(document.createTextNode(`[^${n}]`));
+    }
+    // Drop the rendered references list so it doesn't duplicate the appended
+    // `[^N]:` block. Removing containers also takes care of the standalone <li>
+    // elements that lived inside them; only out-of-container standalones need
+    // individual removal here.
+    for (const container of containers) {
+        if (container.isConnected) {
+            container.remove();
+        }
+    }
+    for (const id of standaloneIds) {
+        const el = document.getElementById(id);
+        if (el?.isConnected) {
+            el.remove();
+        }
+    }
+    const numberToDefId = new Map();
+    for (const [id, n] of defIdToNumber) {
+        numberToDefId.set(n, id);
+    }
+    const footnoteDefs = [];
+    for (let n = 1; n <= numberToDefId.size; n++) {
+        const id = numberToDefId.get(n);
+        if (id === undefined) {
+            break;
+        }
+        const text = defs.get(id);
+        if (text !== undefined) {
+            footnoteDefs.push(text);
+        }
+    }
+    // buildDocument wraps both full documents and bare fragments in a <body>;
+    // serializing its innerHTML returns the equivalent of the input region.
+    return {
+        footnoteDefs,
+        html: document.body.innerHTML,
+    };
+}

package/src/pipeline/footnotes.ts

@@ -0,0 +1,181 @@
+import { buildDocument } from './dom.ts';
+
+export interface FootnoteResult {
+  readonly footnoteDefs: readonly string[];
+  readonly html: string;
+}
+
+// jsdom-parsing every turndown call would dominate hot paths; this pre-check
+// gates on a string scan so footnote-less articles pay nothing.
+const FOOTNOTE_SIGNAL_RE =
+  /cite_note|cite_ref|class="footnotes"|class="references"|data-footnote|role="doc-endnote|<sup\b[^>]*>\s*<a\s[^>]*href="#/i;
+
+// First match wins per group; an `<ol class="footnotes">` is the common case.
+const DEFINITION_CONTAINER_SELECTORS = [
+  'ol.footnotes',
+  'ol[class*="footnotes"]',
+  'ol.references',
+  'ol[class*="references"]',
+  'section[class*="footnote"]',
+  'div[class*="footnote"]',
+  '[role="doc-endnotes"]',
+  '[role="doc-bibliography"]',
+] as const;
+
+const STANDALONE_DEF_ID_RE = /^(?:fn|cite_note|footnote|note)[:_-]/i;
+
+interface RefHit {
+  readonly defId: string;
+  readonly n: number;
+  readonly sup: Element;
+}
+
+// Some sites prepend a back-to-text caret or "Jump to" label to each definition.
+// Stripping only a leading occurrence keeps accidental mid-text carets intact.
+const BACKREF_LEADING_RE = /^(?:↑\s?|↩\s?|\^\s|Jump to\s*)/;
+
+function cleanDefText(text: string): string {
+  return text.replace(/\s+/g, ' ').trim().replace(BACKREF_LEADING_RE, '');
+}
+
+function collectDefinitions(document: Document): {
+  readonly containers: ReadonlySet<Element>;
+  readonly defs: ReadonlyMap<string, string>;
+  readonly standaloneIds: ReadonlySet<string>;
+} {
+  const containers = new Set<Element>();
+  const defs = new Map<string, string>();
+  const standaloneIds = new Set<string>();
+
+  for (const selector of DEFINITION_CONTAINER_SELECTORS) {
+    let matched: NodeListOf<Element>;
+    try {
+      matched = document.querySelectorAll(selector);
+    } catch {
+      continue;
+    }
+    for (const container of Array.from(matched)) {
+      if (!containers.has(container)) {
+        containers.add(container);
+      }
+      for (const item of Array.from(
+        container.querySelectorAll('li, [role="doc-endnote"]'),
+      )) {
+        const id = item.id;
+        if (!id || defs.has(id)) {
+          continue;
+        }
+        const text = cleanDefText(item.textContent);
+        if (text) {
+          defs.set(id, text);
+        }
+      }
+    }
+  }
+
+  // Some sites scatter `<li id="fn1">` outside any container; collect those too.
+  for (const li of Array.from(document.querySelectorAll('li[id]'))) {
+    const id = li.id;
+    if (!id || defs.has(id) || !STANDALONE_DEF_ID_RE.test(id)) {
+      continue;
+    }
+    const text = cleanDefText(li.textContent);
+    if (text) {
+      defs.set(id, text);
+      standaloneIds.add(id);
+    }
+  }
+
+  return { containers, defs, standaloneIds };
+}
+
+export function processFootnotes(html: string): FootnoteResult | null {
+  if (!html || !FOOTNOTE_SIGNAL_RE.test(html)) {
+    return null;
+  }
+
+  let document: Document;
+  try {
+    document = buildDocument(html).document;
+  } catch {
+    return null;
+  }
+
+  const { containers, defs, standaloneIds } = collectDefinitions(document);
+  if (defs.size === 0) {
+    return null;
+  }
+
+  const refHits: RefHit[] = [];
+  const defIdToNumber = new Map<string, number>();
+  for (const sup of Array.from(document.querySelectorAll('sup'))) {
+    if (!sup.isConnected) {
+      continue;
+    }
+    try {
+      const anchor = sup.querySelector('a[href^="#"]');
+      if (!anchor) {
+        continue;
+      }
+      const frag = (anchor.getAttribute('href') ?? '').slice(1);
+      if (!frag || !defs.has(frag)) {
+        continue;
+      }
+      let n = defIdToNumber.get(frag);
+      if (n === undefined) {
+        n = defIdToNumber.size + 1;
+        defIdToNumber.set(frag, n);
+      }
+      refHits.push({ defId: frag, n, sup });
+    } catch {
+      // Defensive: malformed <sup> must not abort the whole pass.
+    }
+  }
+
+  if (refHits.length === 0) {
+    return null;
+  }
+
+  for (const { n, sup } of refHits) {
+    sup.replaceWith(document.createTextNode(`[^${n}]`));
+  }
+
+  // Drop the rendered references list so it doesn't duplicate the appended
+  // `[^N]:` block. Removing containers also takes care of the standalone <li>
+  // elements that lived inside them; only out-of-container standalones need
+  // individual removal here.
+  for (const container of containers) {
+    if (container.isConnected) {
+      container.remove();
+    }
+  }
+  for (const id of standaloneIds) {
+    const el = document.getElementById(id);
+    if (el?.isConnected) {
+      el.remove();
+    }
+  }
+
+  const numberToDefId = new Map<number, string>();
+  for (const [id, n] of defIdToNumber) {
+    numberToDefId.set(n, id);
+  }
+  const footnoteDefs: string[] = [];
+  for (let n = 1; n <= numberToDefId.size; n++) {
+    const id = numberToDefId.get(n);
+    if (id === undefined) {
+      break;
+    }
+    const text = defs.get(id);
+    if (text !== undefined) {
+      footnoteDefs.push(text);
+    }
+  }
+
+  // buildDocument wraps both full documents and bare fragments in a <body>;
+  // serializing its innerHTML returns the equivalent of the input region.
+  return {
+    footnoteDefs,
+    html: document.body.innerHTML,
+  };
+}

package/src/pipeline/math.js

@@ -0,0 +1,113 @@
+// Math renderers leave the original LaTeX in a small, engine-agnostic pocket:
+// the `<annotation encoding="application/x-tex">` child of a `<math>` element.
+// KaTeX wraps it inside `.katex`, arXiv/LaTeXML (ar5iv) and MDN ship it in a
+// bare `<math display="…">`, and MathJax v2/v3 do not use `<annotation>` at
+// all — their source lives in `<script type="math/tex">` that the
+// script-removal pass strips. Move each into one controlled marker span BEFORE
+// normalize's `<script>` removal so every engine shares a single turndown rule
+// that emits the LaTeX verbatim.
+const MARKER_CLASS = 'rdrm-math';
+const DISPLAY_ATTR = 'data-display';
+const KATEX_CLASS = 'katex';
+const KATEX_DISPLAY_CLASS = 'katex-display';
+const TEX_ANNOTATION_SELECTOR = 'annotation[encoding="application/x-tex"]';
+const MATHJAX_INLINE_TYPE = 'math/tex';
+const MATHJAX_DISPLAY_TYPE = 'math/tex; mode=display';
+// arXiv/LaTeXML (`ltx_*`) and common docs-site display-math wrappers. A bare
+// `<math display="block">` also flags display, but pages often omit the
+// attribute and signal block context only through the wrapper class.
+const MATHML_DISPLAY_SELECTOR = '.ltx_equation, .ltx_displaymath, .equation-display, .math-display';
+// Placeholder for a math container whose annotation and alttext are both empty
+// or absent; rendered markup has no recoverable LaTeX, so emit a token rather
+// than crash.
+const BROKEN_PLACEHOLDER = '[?]';
+function createMarker(document, tex, display) {
+    const marker = document.createElement('span');
+    marker.className = MARKER_CLASS;
+    marker.setAttribute(DISPLAY_ATTR, display ? 'true' : 'false');
+    marker.textContent = tex;
+    return marker;
+}
+// Drives off the annotation, not the wrapper, so one pass recovers LaTeX from
+// every engine that emits `<annotation encoding="application/x-tex">` — KaTeX
+// inside `.katex`, arXiv/MDN inside a bare `<math>`. The container replaced is
+// engine-specific: the `.katex` span (which also drops the `.katex-html`
+// rendered sibling), or the bare `<math>` element. Replacing a container
+// detaches other annotations that lived inside it, so disconnected nodes
+// reached later in document order are skipped. arXiv carries the LaTeX in
+// `<math alttext="…">` as well, which is the fallback when the annotation text
+// is empty.
+function convertAnnotations(document) {
+    const annotations = document.querySelectorAll(TEX_ANNOTATION_SELECTOR);
+    for (const annotation of Array.from(annotations)) {
+        if (!annotation.isConnected) {
+            continue;
+        }
+        try {
+            const katex = annotation.closest(`.${KATEX_CLASS}`);
+            const math = annotation.closest('math');
+            let container = annotation;
+            let display = false;
+            if (katex) {
+                container = katex;
+                display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+            }
+            else if (math) {
+                container = math;
+                display =
+                    math.getAttribute('display') === 'block' ||
+                        math.closest(MATHML_DISPLAY_SELECTOR) !== null;
+            }
+            const tex = annotation.textContent.trim() ||
+                (math?.getAttribute('alttext') ?? '').trim();
+            container.replaceWith(createMarker(document, tex || BROKEN_PLACEHOLDER, display));
+        }
+        catch {
+            annotation.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+        }
+    }
+}
+// A `.katex` that survived the annotation pass had no recoverable annotation
+// (stripped or never present). Its `.katex-html` rendered tree carries no
+// recoverable LaTeX, so swap the whole span for a placeholder marker rather
+// than let rendered spans leak into turndown.
+function convertOrphanedKatex(document) {
+    for (const katex of Array.from(document.getElementsByClassName(KATEX_CLASS))) {
+        if (!katex.isConnected) {
+            continue;
+        }
+        const display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+        katex.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, display));
+    }
+}
+// MathJax v2/v3 inline + display source scripts. These `<script>` nodes are
+// removed by normalize's generic script-removal AND by DOMPurify, so they must
+// be converted here, before that loop runs. MathJax scripts do not use
+// `<annotation>`, so they are handled separately from `convertAnnotations`.
+function convertMathJax(document) {
+    const scripts = document.querySelectorAll(`script[type="${MATHJAX_INLINE_TYPE}"], script[type="${MATHJAX_DISPLAY_TYPE}"]`);
+    for (const script of Array.from(scripts)) {
+        if (!script.isConnected) {
+            continue;
+        }
+        try {
+            const rawType = script.getAttribute('type') ?? '';
+            const display = rawType.includes('mode=display');
+            const tex = script.textContent.trim();
+            script.replaceWith(createMarker(document, tex || BROKEN_PLACEHOLDER, display));
+        }
+        catch {
+            script.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+        }
+    }
+}
+// Walks the pre-normalize document and rewrites every math container —
+// annotation-bearing (KaTeX, bare MathML) or MathJax script — into a
+// `<span class="rdrm-math" data-display="…">LATEX</span>` marker. Idempotent:
+// markers carry no `.katex`/`<math>`/annotation, so a second pass is a no-op.
+// Defensive per node so malformed markup never aborts the pass.
+export function extractMath(document) {
+    convertAnnotations(document);
+    convertOrphanedKatex(document);
+    convertMathJax(document);
+}

package/src/pipeline/math.ts

@@ -0,0 +1,130 @@
+// Math renderers leave the original LaTeX in a small, engine-agnostic pocket:
+// the `<annotation encoding="application/x-tex">` child of a `<math>` element.
+// KaTeX wraps it inside `.katex`, arXiv/LaTeXML (ar5iv) and MDN ship it in a
+// bare `<math display="…">`, and MathJax v2/v3 do not use `<annotation>` at
+// all — their source lives in `<script type="math/tex">` that the
+// script-removal pass strips. Move each into one controlled marker span BEFORE
+// normalize's `<script>` removal so every engine shares a single turndown rule
+// that emits the LaTeX verbatim.
+
+const MARKER_CLASS = 'rdrm-math';
+const DISPLAY_ATTR = 'data-display';
+const KATEX_CLASS = 'katex';
+const KATEX_DISPLAY_CLASS = 'katex-display';
+const TEX_ANNOTATION_SELECTOR = 'annotation[encoding="application/x-tex"]';
+const MATHJAX_INLINE_TYPE = 'math/tex';
+const MATHJAX_DISPLAY_TYPE = 'math/tex; mode=display';
+// arXiv/LaTeXML (`ltx_*`) and common docs-site display-math wrappers. A bare
+// `<math display="block">` also flags display, but pages often omit the
+// attribute and signal block context only through the wrapper class.
+const MATHML_DISPLAY_SELECTOR =
+  '.ltx_equation, .ltx_displaymath, .equation-display, .math-display';
+// Placeholder for a math container whose annotation and alttext are both empty
+// or absent; rendered markup has no recoverable LaTeX, so emit a token rather
+// than crash.
+const BROKEN_PLACEHOLDER = '[?]';
+
+function createMarker(
+  document: Document,
+  tex: string,
+  display: boolean,
+): HTMLElement {
+  const marker = document.createElement('span');
+  marker.className = MARKER_CLASS;
+  marker.setAttribute(DISPLAY_ATTR, display ? 'true' : 'false');
+  marker.textContent = tex;
+  return marker;
+}
+
+// Drives off the annotation, not the wrapper, so one pass recovers LaTeX from
+// every engine that emits `<annotation encoding="application/x-tex">` — KaTeX
+// inside `.katex`, arXiv/MDN inside a bare `<math>`. The container replaced is
+// engine-specific: the `.katex` span (which also drops the `.katex-html`
+// rendered sibling), or the bare `<math>` element. Replacing a container
+// detaches other annotations that lived inside it, so disconnected nodes
+// reached later in document order are skipped. arXiv carries the LaTeX in
+// `<math alttext="…">` as well, which is the fallback when the annotation text
+// is empty.
+function convertAnnotations(document: Document): void {
+  const annotations = document.querySelectorAll(TEX_ANNOTATION_SELECTOR);
+  for (const annotation of Array.from(annotations)) {
+    if (!annotation.isConnected) {
+      continue;
+    }
+    try {
+      const katex = annotation.closest(`.${KATEX_CLASS}`);
+      const math = annotation.closest('math');
+      let container: Element = annotation;
+      let display = false;
+      if (katex) {
+        container = katex;
+        display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+      } else if (math) {
+        container = math;
+        display =
+          math.getAttribute('display') === 'block' ||
+          math.closest(MATHML_DISPLAY_SELECTOR) !== null;
+      }
+      const tex =
+        annotation.textContent.trim() ||
+        (math?.getAttribute('alttext') ?? '').trim();
+      container.replaceWith(
+        createMarker(document, tex || BROKEN_PLACEHOLDER, display),
+      );
+    } catch {
+      annotation.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+    }
+  }
+}
+
+// A `.katex` that survived the annotation pass had no recoverable annotation
+// (stripped or never present). Its `.katex-html` rendered tree carries no
+// recoverable LaTeX, so swap the whole span for a placeholder marker rather
+// than let rendered spans leak into turndown.
+function convertOrphanedKatex(document: Document): void {
+  for (const katex of Array.from(
+    document.getElementsByClassName(KATEX_CLASS),
+  )) {
+    if (!katex.isConnected) {
+      continue;
+    }
+    const display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+    katex.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, display));
+  }
+}
+
+// MathJax v2/v3 inline + display source scripts. These `<script>` nodes are
+// removed by normalize's generic script-removal AND by DOMPurify, so they must
+// be converted here, before that loop runs. MathJax scripts do not use
+// `<annotation>`, so they are handled separately from `convertAnnotations`.
+function convertMathJax(document: Document): void {
+  const scripts = document.querySelectorAll(
+    `script[type="${MATHJAX_INLINE_TYPE}"], script[type="${MATHJAX_DISPLAY_TYPE}"]`,
+  );
+  for (const script of Array.from(scripts)) {
+    if (!script.isConnected) {
+      continue;
+    }
+    try {
+      const rawType = script.getAttribute('type') ?? '';
+      const display = rawType.includes('mode=display');
+      const tex = script.textContent.trim();
+      script.replaceWith(
+        createMarker(document, tex || BROKEN_PLACEHOLDER, display),
+      );
+    } catch {
+      script.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+    }
+  }
+}
+
+// Walks the pre-normalize document and rewrites every math container —
+// annotation-bearing (KaTeX, bare MathML) or MathJax script — into a
+// `<span class="rdrm-math" data-display="…">LATEX</span>` marker. Idempotent:
+// markers carry no `.katex`/`<math>`/annotation, so a second pass is a no-op.
+// Defensive per node so malformed markup never aborts the pass.
+export function extractMath(document: Document): void {
+  convertAnnotations(document);
+  convertOrphanedKatex(document);
+  convertMathJax(document);
+}