@v1nvn/readability 0.45.0 → 0.45.1

package/src/pipeline/math.js

@@ -0,0 +1,113 @@
+// Math renderers leave the original LaTeX in a small, engine-agnostic pocket:
+// the `<annotation encoding="application/x-tex">` child of a `<math>` element.
+// KaTeX wraps it inside `.katex`, arXiv/LaTeXML (ar5iv) and MDN ship it in a
+// bare `<math display="…">`, and MathJax v2/v3 do not use `<annotation>` at
+// all — their source lives in `<script type="math/tex">` that the
+// script-removal pass strips. Move each into one controlled marker span BEFORE
+// normalize's `<script>` removal so every engine shares a single turndown rule
+// that emits the LaTeX verbatim.
+const MARKER_CLASS = 'rdrm-math';
+const DISPLAY_ATTR = 'data-display';
+const KATEX_CLASS = 'katex';
+const KATEX_DISPLAY_CLASS = 'katex-display';
+const TEX_ANNOTATION_SELECTOR = 'annotation[encoding="application/x-tex"]';
+const MATHJAX_INLINE_TYPE = 'math/tex';
+const MATHJAX_DISPLAY_TYPE = 'math/tex; mode=display';
+// arXiv/LaTeXML (`ltx_*`) and common docs-site display-math wrappers. A bare
+// `<math display="block">` also flags display, but pages often omit the
+// attribute and signal block context only through the wrapper class.
+const MATHML_DISPLAY_SELECTOR = '.ltx_equation, .ltx_displaymath, .equation-display, .math-display';
+// Placeholder for a math container whose annotation and alttext are both empty
+// or absent; rendered markup has no recoverable LaTeX, so emit a token rather
+// than crash.
+const BROKEN_PLACEHOLDER = '[?]';
+function createMarker(document, tex, display) {
+    const marker = document.createElement('span');
+    marker.className = MARKER_CLASS;
+    marker.setAttribute(DISPLAY_ATTR, display ? 'true' : 'false');
+    marker.textContent = tex;
+    return marker;
+}
+// Drives off the annotation, not the wrapper, so one pass recovers LaTeX from
+// every engine that emits `<annotation encoding="application/x-tex">` — KaTeX
+// inside `.katex`, arXiv/MDN inside a bare `<math>`. The container replaced is
+// engine-specific: the `.katex` span (which also drops the `.katex-html`
+// rendered sibling), or the bare `<math>` element. Replacing a container
+// detaches other annotations that lived inside it, so disconnected nodes
+// reached later in document order are skipped. arXiv carries the LaTeX in
+// `<math alttext="…">` as well, which is the fallback when the annotation text
+// is empty.
+function convertAnnotations(document) {
+    const annotations = document.querySelectorAll(TEX_ANNOTATION_SELECTOR);
+    for (const annotation of Array.from(annotations)) {
+        if (!annotation.isConnected) {
+            continue;
+        }
+        try {
+            const katex = annotation.closest(`.${KATEX_CLASS}`);
+            const math = annotation.closest('math');
+            let container = annotation;
+            let display = false;
+            if (katex) {
+                container = katex;
+                display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+            }
+            else if (math) {
+                container = math;
+                display =
+                    math.getAttribute('display') === 'block' ||
+                        math.closest(MATHML_DISPLAY_SELECTOR) !== null;
+            }
+            const tex = annotation.textContent.trim() ||
+                (math?.getAttribute('alttext') ?? '').trim();
+            container.replaceWith(createMarker(document, tex || BROKEN_PLACEHOLDER, display));
+        }
+        catch {
+            annotation.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+        }
+    }
+}
+// A `.katex` that survived the annotation pass had no recoverable annotation
+// (stripped or never present). Its `.katex-html` rendered tree carries no
+// recoverable LaTeX, so swap the whole span for a placeholder marker rather
+// than let rendered spans leak into turndown.
+function convertOrphanedKatex(document) {
+    for (const katex of Array.from(document.getElementsByClassName(KATEX_CLASS))) {
+        if (!katex.isConnected) {
+            continue;
+        }
+        const display = katex.closest(`.${KATEX_DISPLAY_CLASS}`) !== null;
+        katex.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, display));
+    }
+}
+// MathJax v2/v3 inline + display source scripts. These `<script>` nodes are
+// removed by normalize's generic script-removal AND by DOMPurify, so they must
+// be converted here, before that loop runs. MathJax scripts do not use
+// `<annotation>`, so they are handled separately from `convertAnnotations`.
+function convertMathJax(document) {
+    const scripts = document.querySelectorAll(`script[type="${MATHJAX_INLINE_TYPE}"], script[type="${MATHJAX_DISPLAY_TYPE}"]`);
+    for (const script of Array.from(scripts)) {
+        if (!script.isConnected) {
+            continue;
+        }
+        try {
+            const rawType = script.getAttribute('type') ?? '';
+            const display = rawType.includes('mode=display');
+            const tex = script.textContent.trim();
+            script.replaceWith(createMarker(document, tex || BROKEN_PLACEHOLDER, display));
+        }
+        catch {
+            script.replaceWith(createMarker(document, BROKEN_PLACEHOLDER, false));
+        }
+    }
+}
+// Walks the pre-normalize document and rewrites every math container —
+// annotation-bearing (KaTeX, bare MathML) or MathJax script — into a
+// `<span class="rdrm-math" data-display="…">LATEX</span>` marker. Idempotent:
+// markers carry no `.katex`/`<math>`/annotation, so a second pass is a no-op.
+// Defensive per node so malformed markup never aborts the pass.
+export function extractMath(document) {
+    convertAnnotations(document);
+    convertOrphanedKatex(document);
+    convertMathJax(document);
+}

package/src/pipeline/normalize.js

@@ -0,0 +1,507 @@
+import { extractMath } from "./math.js";
+import { KNOWN_LANGUAGE_TOKENS } from "./resolver.js";
+const NONCE_ATTR = 'nonce';
+export function normalizeDocument(document, options) {
+    // MathJax source lives in `<script type="math/tex">`, which the script-removal
+    // loop below strips — extract math into markers first so both engines share
+    // one turndown rule downstream.
+    extractMath(document);
+    const baseEls = document.querySelectorAll('base');
+    // Preserve <script type="application/ld+json">: structured metadata consumed by the cascade.
+    const scriptEls = document.querySelectorAll('script:not([type="application/ld+json"])');
+    baseEls.forEach(el => {
+        el.remove();
+    });
+    scriptEls.forEach(el => {
+        el.remove();
+    });
+    const withNonce = document.querySelectorAll(`[${NONCE_ATTR}]`);
+    withNonce.forEach(el => {
+        el.removeAttribute(NONCE_ATTR);
+    });
+    const chromeRemoved = options?.cleanChrome === false ? 0 : stripChrome(document);
+    const boilerplateRemoved = stripBoilerplate(document);
+    return {
+        boilerplateRemoved,
+        chromeRemoved,
+        iframes: 0,
+        scripts: scriptEls.length,
+    };
+}
+// Conservative, vendor-specific consent SDK selectors. Curated, not greedy —
+// intentionally NOT `[class*="consent"]` (that eats real article widgets).
+const CONSENT_SELECTORS = [
+    '[role="dialog"]',
+    '[aria-modal="true"]',
+    '#onetrust-banner-sdk',
+    '#onetrust-consent-sdk',
+    '#onetrust-pc-sdk',
+    '.cc-window',
+    '.cc-banner',
+    '.cc-revoke',
+    '.osano-cm-window',
+    '.osano-cm-dialog',
+    '.qc-cmp2-container',
+    '.qc-cmp-ui-container',
+    '#sp_message_container',
+    '[id^="sp_message_container_"]',
+    '#didomi-host',
+    '.didomi-popup-container',
+    '#truste-consent-track',
+    '#consent_blackbar',
+    '#cookie-banner',
+    '.cookie-banner',
+    '#consent-banner',
+    '.consent-banner',
+    '.cookie-bar',
+    '.gdpr-banner',
+    '.privacy-banner',
+];
+// jsdom has no layout, so "covers the viewport" must be inferred from inline
+// style. Requiring BOTH axes full is what protects a fixed nav bar: a nav is
+// full-width but its height is a fixed px (not 100%/100vh), and inset:0 is not
+// used. Dropping either axis from the conjunction nukes legit chrome.
+function isFullViewportOverlay(style) {
+    if (!/position\s*:\s*(?:fixed|sticky)/i.test(style)) {
+        return false;
+    }
+    const zIndexMatch = /z-index\s*:\s*(\d+)/i.exec(style);
+    if (!zIndexMatch || Number.parseInt(zIndexMatch[1], 10) < 1000) {
+        return false;
+    }
+    // `inset:0` is shorthand for top/right/bottom/left = 0, so it satisfies BOTH
+    // axes at once and is checked up front.
+    if (/inset\s*:\s*0/i.test(style)) {
+        return true;
+    }
+    const widthFull = /width\s*:\s*100(?:%|vw)/i.test(style) ||
+        (/left\s*:\s*0/i.test(style) && /right\s*:\s*0/i.test(style));
+    const heightFull = /height\s*:\s*100(?:%|vh)/i.test(style) ||
+        (/top\s*:\s*0/i.test(style) && /bottom\s*:\s*0/i.test(style));
+    return widthFull && heightFull;
+}
+// Removes before Readability scores them — these poison density math and leak
+// into the article. Inline-style matches require the full isFullViewportOverlay
+// conjunction; `position:fixed` alone never qualifies.
+export function stripChrome(document) {
+    let removed = 0;
+    const consentEls = document.querySelectorAll(CONSENT_SELECTORS.join(','));
+    for (const el of consentEls) {
+        if (!el.isConnected) {
+            continue;
+        }
+        el.remove();
+        removed++;
+    }
+    const styledEls = document.querySelectorAll('[style]');
+    for (const el of styledEls) {
+        if (!el.isConnected) {
+            continue;
+        }
+        if (isFullViewportOverlay(el.getAttribute('style') ?? '')) {
+            el.remove();
+            removed++;
+        }
+    }
+    return removed;
+}
+// Unambiguous boilerplate signatures. Bare `related`/`subscribe` are
+// intentionally excluded — they match legit content (related-products section
+// on a store, a "subscribe to RSS" line inside an article) far too often.
+// Substring matching means `newsletter` also covers `newsletter-signup`, etc.;
+// the longer spellings are kept as explicit documentation of intended targets.
+const BOILERPLATE_TOKENS = [
+    'newsletter',
+    'newsletter-signup',
+    'mailing-list',
+    'email-signup',
+    'subscribe-form',
+    'signup-form',
+    'related-posts',
+    'related-post',
+    'read-next',
+    'more-from',
+    'you-might-also',
+    'recommended-posts',
+    'recommended',
+];
+function signatureMatches(el) {
+    const signature = `${el.getAttribute('class') ?? ''} ${el.getAttribute('id') ?? ''}`.toLowerCase();
+    return BOILERPLATE_TOKENS.some(token => signature.includes(token));
+}
+// A boilerplate block is always a structural CONTAINER. A heading or paragraph
+// can carry a token in its `id` (an anchor target like `id="related-posts"`) or
+// `class`, and individually such a leaf passes the footprint guard — so without
+// a container restriction the rule deletes real article content (e.g. an in-body
+// section heading whose id happens to match). Reject non-containers early.
+const BOILERPLATE_CONTAINER_TAGS = new Set([
+    'ASIDE',
+    'DIV',
+    'FORM',
+    'NAV',
+    'OL',
+    'SECTION',
+    'UL',
+]);
+// Strips "related posts" / newsletter signup / "read next" blocks Readability
+// sometimes retains. Subtractive and high-risk, so a token hit alone is never
+// enough: the footprint guard requires the candidate to be a small fraction of
+// the body. The body length is snapshotted once so every candidate is judged
+// against the same baseline — recomputing after each removal would make the
+// threshold depend on iteration order. A boilerplate block is always a small
+// slice of a real article; capping at 25% guarantees the worst case is leaving
+// a small block in, never deleting a section that is itself a substantial part
+// of the content. Idempotent and order-independent.
+//
+// Only outermost boilerplate roots are candidates. A single signature CONTAINER
+// (e.g. <aside class="newsletter-subscribe">) puts the token on every descendant
+// ("newsletter-title", "newsletter-submit", …); stripping those leaves
+// independently mangles the container when the footprint guard preserves the
+// root — a newsletter form shorn of its title and submit button. The ancestor
+// walk rejects any element whose class+id already matched on an ancestor.
+// querySelectorAll yields outer-before-inner order, and removing a root detaches
+// its descendants (caught by !isConnected above), so this walk uniformly rejects
+// leaves of both removed roots and preserved roots.
+export function stripBoilerplate(document) {
+    const bodyLength = document.body.textContent.length;
+    const limit = 0.25 * bodyLength;
+    let removed = 0;
+    for (const el of document.querySelectorAll('[class],[id]')) {
+        if (!el.isConnected) {
+            continue;
+        }
+        if (!BOILERPLATE_CONTAINER_TAGS.has(el.tagName)) {
+            continue;
+        }
+        if (!signatureMatches(el)) {
+            continue;
+        }
+        let ancestor = el.parentElement;
+        while (ancestor && !signatureMatches(ancestor)) {
+            ancestor = ancestor.parentElement;
+        }
+        if (ancestor) {
+            continue;
+        }
+        if (el.textContent.length >= limit) {
+            continue;
+        }
+        el.remove();
+        removed++;
+    }
+    return removed;
+}
+// Scope the document by CSS selector: drop every `exclude` match, then (if
+// `include` matches) replace the body with the first matching subtree.
+export function applySelectors(document, selectors) {
+    if (!selectors) {
+        return;
+    }
+    if (selectors.exclude) {
+        for (const selector of selectors.exclude) {
+            document.querySelectorAll(selector).forEach(el => {
+                el.remove();
+            });
+        }
+    }
+    if (selectors.include) {
+        const body = document.body;
+        const root = body.querySelector(selectors.include);
+        if (root && root !== body) {
+            body.innerHTML = '';
+            body.appendChild(root);
+        }
+    }
+}
+// Matched anywhere in a src (case-insensitive) to flag a lazy-load placeholder.
+const PLACEHOLDER_TOKENS = [
+    'placeholder',
+    'blank',
+    'spacer',
+    'lazy',
+    'loading',
+    '1x1',
+    'transparent',
+    'pixel',
+    'dummy',
+];
+export function isPlaceholderSrc(src) {
+    if (!src) {
+        return true;
+    }
+    if (src.startsWith('data:')) {
+        return true;
+    }
+    const lowered = src.toLowerCase();
+    return PLACEHOLDER_TOKENS.some(token => lowered.includes(token));
+}
+function usableAttr(el, names) {
+    for (const name of names) {
+        const value = el.getAttribute(name);
+        if (value) {
+            return value;
+        }
+    }
+    return undefined;
+}
+// Largest-descriptor candidate wins (`Nw`/`Nx`, never mixed in valid srcset);
+// first candidate wins when none carry a descriptor. Malformed entries are skipped.
+function pickLargestSrcset(srcset) {
+    let bestUrl;
+    let bestValue = -1;
+    let firstUrl;
+    let sawDescriptor = false;
+    for (const raw of srcset.split(',')) {
+        const entry = raw.trim();
+        if (!entry) {
+            continue;
+        }
+        const splitAt = entry.search(/\s/);
+        const rawUrl = splitAt === -1 ? entry : entry.slice(0, splitAt);
+        const descriptor = splitAt === -1 ? '' : entry.slice(splitAt).trim();
+        const url = rawUrl.length >= 2 &&
+            ((rawUrl.startsWith('"') && rawUrl.endsWith('"')) ||
+                (rawUrl.startsWith("'") && rawUrl.endsWith("'")))
+            ? rawUrl.slice(1, -1)
+            : rawUrl;
+        if (!url) {
+            continue;
+        }
+        if (firstUrl === undefined) {
+            firstUrl = url;
+        }
+        if (!descriptor) {
+            continue;
+        }
+        const value = /^(\d+(?:\.\d+)?)[wx]$/.exec(descriptor);
+        if (!value) {
+            continue;
+        }
+        sawDescriptor = true;
+        const numeric = Number.parseFloat(value[1]);
+        if (numeric > bestValue) {
+            bestValue = numeric;
+            bestUrl = url;
+        }
+    }
+    return sawDescriptor ? bestUrl : firstUrl;
+}
+// Skip media-constrained <source>: art-direction crops for specific viewports,
+// not the default image the page shows.
+function resolveFromPicture(img) {
+    const picture = img.closest('picture');
+    if (!picture) {
+        return undefined;
+    }
+    for (const source of picture.querySelectorAll('source')) {
+        if (source.hasAttribute('media')) {
+            continue;
+        }
+        const srcset = source.getAttribute('srcset');
+        if (srcset) {
+            const url = pickLargestSrcset(srcset);
+            if (url) {
+                return url;
+            }
+        }
+    }
+    return undefined;
+}
+// Precedence (first usable wins): data-src → <picture><source srcset> → own
+// srcset (largest) → data-original → data-lazy-src.
+function resolveRealSource(img) {
+    const dataSrc = usableAttr(img, ['data-src']);
+    if (dataSrc) {
+        return dataSrc;
+    }
+    const fromPicture = resolveFromPicture(img);
+    if (fromPicture) {
+        return fromPicture;
+    }
+    const ownSrcset = img.getAttribute('srcset');
+    if (ownSrcset) {
+        const url = pickLargestSrcset(ownSrcset);
+        if (url) {
+            return url;
+        }
+    }
+    return usableAttr(img, ['data-original', 'data-lazy-src']);
+}
+// Swap placeholder src values for the real source before Readability clones.
+// Only a placeholder-like current src is rewritten, so a real src is left alone
+// even if a data-src also exists. Idempotent.
+export function resolveLazyImages(document) {
+    let resolved = 0;
+    for (const img of document.querySelectorAll('img')) {
+        const currentSrc = img.getAttribute('src') ?? '';
+        if (!isPlaceholderSrc(currentSrc)) {
+            continue;
+        }
+        const real = resolveRealSource(img);
+        if (real && real !== currentSrc) {
+            img.setAttribute('src', real);
+            resolved++;
+        }
+    }
+    return resolved;
+}
+// Readability keeps a class only by literal `classesToPreserve.includes(cls)`
+// equality, so non-canonical code-block conventions — GitHub's
+// `highlight-source-js`, sandpack's `sp-javascript`, SyntaxHighlighter's
+// `brush: js` — are stripped before turndown runs and the fence loses its
+// language tag. Rewrite each convention to the canonical
+// `<code class="language-X">` form before Readability clones the document.
+const HIGHLIGHT_SOURCE_PREFIX = 'highlight-source-';
+const LANGUAGE_PREFIX = 'language-';
+const LANG_PREFIX = 'lang-';
+const SP_PREFIX = 'sp-';
+const BRUSH_RE = /brush:\s*([A-Za-z][\w-]*)/;
+const BOGUS_TOKENS = new Set(['', 'highlight', 'source', 'sp']);
+function isValidToken(token) {
+    return !BOGUS_TOKENS.has(token);
+}
+function hasLanguageClass(el) {
+    for (const cls of el.classList) {
+        if (cls !== LANGUAGE_PREFIX && cls.startsWith(LANGUAGE_PREFIX)) {
+            return true;
+        }
+    }
+    return false;
+}
+// Pool the <pre>'s own classes, then ancestors up to the highlight wrapper
+// (capped at two levels), then any descendant <code>'s classes — in that order.
+function collectClassSources(pre, code) {
+    const sources = [
+        { classes: [...pre.classList], raw: pre.getAttribute('class') ?? '' },
+    ];
+    let depth = 0;
+    let el = pre.parentElement;
+    while (el && depth < 2) {
+        sources.push({
+            classes: [...el.classList],
+            raw: el.getAttribute('class') ?? '',
+        });
+        depth++;
+        if (el.tagName === 'DIV' &&
+            [...el.classList].some(c => c.startsWith('highlight'))) {
+            break;
+        }
+        el = el.parentElement;
+    }
+    if (code) {
+        sources.push({
+            classes: [...code.classList],
+            raw: code.getAttribute('class') ?? '',
+        });
+    }
+    return sources;
+}
+function resolveCodeToken(pre, code) {
+    const sources = collectClassSources(pre, code);
+    function findByPrefix(prefix) {
+        for (const src of sources) {
+            for (const cls of src.classes) {
+                if (cls.startsWith(prefix)) {
+                    const token = cls.slice(prefix.length).toLowerCase();
+                    if (isValidToken(token)) {
+                        return token;
+                    }
+                }
+            }
+        }
+        return null;
+    }
+    // Convention priority: highlight-source > language > lang > sp > brush.
+    // `brush:` is a class-attribute value, not a single class token, so it cannot
+    // be matched by prefix and is parsed from the raw attribute string instead.
+    const highlightToken = findByPrefix(HIGHLIGHT_SOURCE_PREFIX);
+    if (highlightToken) {
+        return { token: highlightToken, fromHighlightSource: true };
+    }
+    const languageToken = findByPrefix(LANGUAGE_PREFIX);
+    if (languageToken) {
+        return { token: languageToken, fromHighlightSource: false };
+    }
+    const langToken = findByPrefix(LANG_PREFIX);
+    if (langToken) {
+        return { token: langToken, fromHighlightSource: false };
+    }
+    // Sandpack emits infra classes alongside the language class (sp-cm,
+    // sp-pristine, sp-pre-placeholder); unlike the single-token highlight-source
+    // / language / lang conventions, SP must pick the class whose token is a
+    // recognized language or it grabs an infra string the preserve list strips.
+    for (const src of sources) {
+        for (const cls of src.classes) {
+            if (!cls.startsWith(SP_PREFIX)) {
+                continue;
+            }
+            const token = cls.slice(SP_PREFIX.length).toLowerCase();
+            if (KNOWN_LANGUAGE_TOKENS.has(token)) {
+                return { token, fromHighlightSource: false };
+            }
+        }
+    }
+    for (const src of sources) {
+        const match = BRUSH_RE.exec(src.raw);
+        if (match) {
+            const token = match[1].toLowerCase();
+            if (isValidToken(token)) {
+                return { token, fromHighlightSource: false };
+            }
+        }
+    }
+    return null;
+}
+function canonicalizePre(pre) {
+    const code = pre.querySelector('code');
+    // turndown reads the language from the <code> class; if one is present the
+    // block is already canonical and rewriting it can only drop information.
+    if (code && hasLanguageClass(code)) {
+        return false;
+    }
+    const match = resolveCodeToken(pre, code);
+    if (!match) {
+        return false;
+    }
+    let target = code;
+    if (!target) {
+        target = pre.ownerDocument.createElement('code');
+        while (pre.firstChild) {
+            target.appendChild(pre.firstChild);
+        }
+        pre.appendChild(target);
+    }
+    const classes = [`language-${match.token}`];
+    if (target.classList.contains('hljs')) {
+        classes.push('hljs');
+    }
+    target.setAttribute('class', classes.join(' '));
+    // GitHub carries the language on a <div class="highlight"> wrapper rather than
+    // the <pre>; once that hint is moved onto <code>, hoist the <pre> out so the
+    // block stands alone instead of inside a div Readability scores separately.
+    if (match.fromHighlightSource) {
+        const parent = pre.parentElement;
+        if (parent?.tagName === 'DIV' &&
+            [...parent.classList].some(c => c.startsWith('highlight'))) {
+            parent.replaceWith(pre);
+        }
+    }
+    return true;
+}
+export function canonicalizeCodeBlocks(document) {
+    let count = 0;
+    for (const pre of document.querySelectorAll('pre')) {
+        if (!pre.isConnected) {
+            continue;
+        }
+        try {
+            if (canonicalizePre(pre)) {
+                count++;
+            }
+        }
+        catch {
+            // Defensive: malformed markup must not break the extract pipeline.
+        }
+    }
+    return count;
+}

package/src/pipeline/readability.js

@@ -0,0 +1,10 @@
+import { isProbablyReaderable, Readability } from '@mozilla/readability';
+export function isReaderable(document) {
+    return isProbablyReaderable(document);
+}
+export function parseArticle(document, options) {
+    // Readability mutates its input document; clone so the pipeline's doc is untouched.
+    const clone = document.cloneNode(true);
+    const reader = new Readability(clone, options);
+    return reader.parse();
+}