@v1nvn/readability 0.45.0 → 0.45.1

package/src/policy/chunk.js

@@ -0,0 +1,365 @@
+// Token-bounded chunking of extracted markdown for RAG/embedding. Chunk
+// `tokenCount` uses the shared chars/4 estimator (policy/text.ts), keeping it
+// directly comparable to `metadata.tokenEstimate`. This is the server-side
+// chunking path for large documents; transport-level streaming was rejected —
+// MCP tool-result progress isn't universally host-supported, and this option
+// already keeps a multi-MB page from returning as one payload.
+import { hardSplitLines, headingText, parseBlocks } from "./markdown.js";
+import { estimateTokens } from "./text.js";
+const HEADING_FIRST_LINE = /^#{1,6}\s/;
+function splitBlocks(markdown) {
+    const parts = markdown.split(/\n{2,}/);
+    const blocks = [];
+    let heading = '';
+    for (const part of parts) {
+        const text = part.trim();
+        if (!text) {
+            continue;
+        }
+        if (HEADING_FIRST_LINE.test(text)) {
+            heading = headingText(text);
+        }
+        blocks.push({ headingContext: heading, text });
+    }
+    return blocks;
+}
+// Char strategy may break a code block — the semantic strategy avoids that.
+function splitOversizedBlock(block, maxChars) {
+    return hardSplitLines(block.text, maxChars).map(text => ({
+        headingContext: block.headingContext,
+        text,
+    }));
+}
+function toUnits(blocks, maxChars) {
+    const units = [];
+    for (const block of blocks) {
+        if (block.text.length <= maxChars) {
+            units.push(block);
+        }
+        else {
+            units.push(...splitOversizedBlock(block, maxChars));
+        }
+    }
+    return units;
+}
+function chunkMarkdownChar(markdown, options) {
+    if (!markdown) {
+        return [];
+    }
+    const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+    const maxChars = maxTokens * 4;
+    const overlapTokens = Math.max(0, Math.floor(options.overlap));
+    // Overlap must be strictly less than maxChars — otherwise a chunk's tail
+    // could consume the entire budget and we'd loop forever re-emitting it.
+    const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+    const blocks = splitBlocks(markdown);
+    if (blocks.length === 0) {
+        return [];
+    }
+    const units = toUnits(blocks, maxChars);
+    if (units.length === 0) {
+        return [];
+    }
+    const chunks = [];
+    let i = 0;
+    let overlapText = '';
+    let overlapHeading = '';
+    while (i < units.length) {
+        const spans = [];
+        let chunkText = '';
+        if (overlapText) {
+            chunkText = overlapText;
+            spans.push({
+                end: overlapText.length,
+                headingContext: overlapHeading,
+                start: 0,
+            });
+        }
+        // Force at least one new unit per chunk so the loop terminates. If carrying
+        // the overlap would push the first new unit past maxChars, drop the overlap
+        // for this chunk (the unit was pre-split to <= maxChars so it always fits alone).
+        const firstUnit = units[i];
+        const sepLen = chunkText ? 2 : 0;
+        if (chunkText &&
+            chunkText.length + sepLen + firstUnit.text.length > maxChars) {
+            chunkText = '';
+            spans.length = 0;
+        }
+        {
+            const sep = chunkText ? '\n\n' : '';
+            const start = chunkText.length + sep.length;
+            chunkText = chunkText + sep + firstUnit.text;
+            spans.push({
+                end: chunkText.length,
+                headingContext: firstUnit.headingContext,
+                start,
+            });
+            i += 1;
+        }
+        while (i < units.length) {
+            const unit = units[i];
+            const candidate = `${chunkText}\n\n${unit.text}`;
+            if (candidate.length > maxChars) {
+                break;
+            }
+            const start = chunkText.length + 2;
+            chunkText = candidate;
+            spans.push({
+                end: chunkText.length,
+                headingContext: unit.headingContext,
+                start,
+            });
+            i += 1;
+        }
+        const text = chunkText.trim();
+        if (text) {
+            chunks.push({
+                headingContext: spans[0]?.headingContext ?? '',
+                index: chunks.length,
+                text,
+                tokenCount: estimateTokens(text).tokenEstimate,
+            });
+        }
+        if (overlapChars > 0 && i < units.length) {
+            const overlapStart = Math.max(0, chunkText.length - overlapChars);
+            const carrier = spans.find(span => overlapStart >= span.start && overlapStart < span.end) ?? spans.find(span => span.start >= overlapStart);
+            overlapHeading = carrier?.headingContext ?? '';
+            overlapText = chunkText.slice(overlapStart).replace(/^\n+/, '');
+        }
+        else {
+            overlapText = '';
+            overlapHeading = '';
+        }
+    }
+    return chunks;
+}
+function parseSemanticUnits(markdown) {
+    const units = [];
+    const stack = [];
+    function context() {
+        return stack.map(h => h.text).join(' > ');
+    }
+    for (const block of parseBlocks(markdown)) {
+        const text = markdown.slice(block.start, block.end);
+        if (block.kind === 'heading') {
+            while (stack.length > 0 && stack[stack.length - 1].level >= block.depth) {
+                stack.pop();
+            }
+            stack.push({ level: block.depth, text: headingText(text) });
+            units.push({ headingContext: context(), kind: 'heading', text });
+            continue;
+        }
+        units.push({
+            headingContext: context(),
+            kind: block.kind === 'code' ? 'code' : 'text',
+            text,
+        });
+    }
+    return units;
+}
+// A section = a heading unit plus its following non-heading units until the
+// next heading. Pre-first-heading content forms its own section with an empty
+// hierarchy path.
+function groupSections(units) {
+    const sections = [];
+    let current = [];
+    let currentContext = '';
+    function flush() {
+        if (current.length > 0) {
+            sections.push({ headingContext: currentContext, units: current });
+            current = [];
+        }
+    }
+    for (const unit of units) {
+        if (unit.kind === 'heading') {
+            flush();
+            current = [unit];
+            currentContext = unit.headingContext;
+        }
+        else {
+            if (current.length === 0) {
+                currentContext = unit.headingContext;
+            }
+            current.push(unit);
+        }
+    }
+    flush();
+    return sections;
+}
+function joinedLength(units) {
+    if (units.length === 0) {
+        return 0;
+    }
+    let total = units[0].text.length;
+    for (let i = 1; i < units.length; i++) {
+        total += 2 + units[i].text.length;
+    }
+    return total;
+}
+// Hard-split a text unit that alone exceeds the budget — line-aware first, then
+// hard-split any oversized line. Only ever applied to paragraphs; code blocks
+// are never passed through here.
+function splitOversizedTextUnit(unit, maxChars) {
+    return hardSplitLines(unit.text, maxChars).map(text => ({
+        headingContext: unit.headingContext,
+        kind: 'text',
+        text,
+    }));
+}
+// Pack sections into chunk groups <= maxChars, preferring heading boundaries.
+// A section that fits is never split; an oversized section is split into
+// sub-chunks by body unit (paragraphs may split; code blocks never do).
+function buildBaseGroups(sections, maxChars) {
+    const groups = [];
+    let current = [];
+    let currentLen = 0;
+    function emit() {
+        if (current.length > 0) {
+            groups.push(current);
+            current = [];
+            currentLen = 0;
+        }
+    }
+    function append(unit) {
+        const sep = current.length > 0 ? 2 : 0;
+        current.push(unit);
+        currentLen += sep + unit.text.length;
+    }
+    for (const section of sections) {
+        const sectionLen = joinedLength(section.units);
+        const sep = current.length > 0 ? 2 : 0;
+        if (currentLen + sep + sectionLen <= maxChars) {
+            for (const unit of section.units) {
+                append(unit);
+            }
+            continue;
+        }
+        // Doesn't fit merged — close the current group and retry the section alone
+        // so it has first dibs on the next group (it may still merge with a later
+        // section that fits behind it).
+        emit();
+        if (sectionLen <= maxChars) {
+            for (const unit of section.units) {
+                append(unit);
+            }
+            continue;
+        }
+        // Oversized section: pack body units under the heading; split oversized
+        // paragraphs, emit oversized code blocks whole.
+        const [headingUnit, ...body] = section.units;
+        current.push(headingUnit);
+        currentLen = headingUnit.text.length;
+        for (const unit of body) {
+            if (unit.kind === 'code' && unit.text.length > maxChars) {
+                // Semantic never splits a code fence: a fence cut mid-way is
+                // unrecoverable, so the block is emitted as its own chunk even though
+                // it exceeds the token budget. Deliberate tradeoff vs. the char cap.
+                emit();
+                groups.push([unit]);
+                continue;
+            }
+            const sepNow = current.length > 0 ? 2 : 0;
+            if (unit.text.length <= maxChars &&
+                currentLen + sepNow + unit.text.length <= maxChars) {
+                append(unit);
+                continue;
+            }
+            const pieces = unit.text.length > maxChars
+                ? splitOversizedTextUnit(unit, maxChars)
+                : [unit];
+            for (const piece of pieces) {
+                const pieceSep = current.length > 0 ? 2 : 0;
+                if (current.length > 0 &&
+                    currentLen + pieceSep + piece.text.length > maxChars) {
+                    emit();
+                }
+                append(piece);
+            }
+        }
+        // Leave the trailing sub-chunk in `current` so a following section can
+        // merge into it if it fits.
+    }
+    emit();
+    return groups;
+}
+// Carry the trailing text (non-code) units from the previous group as the
+// prefix of the next, up to overlapChars. Never carry into or out of a code
+// block: a trailing code run is skipped, then contiguous text units before it
+// are carried (stop at the first heading or code block encountered).
+function applySemanticOverlap(groups, overlapChars) {
+    const result = [];
+    for (let i = 0; i < groups.length; i++) {
+        const group = groups[i];
+        const carrier = [];
+        if (i > 0 && overlapChars > 0) {
+            const prev = groups[i - 1];
+            let end = prev.length;
+            while (end > 0 && prev[end - 1].kind === 'code') {
+                end -= 1;
+            }
+            let carrierLen = 0;
+            for (let j = end - 1; j >= 0; j--) {
+                const unit = prev[j];
+                if (unit.kind !== 'text') {
+                    break;
+                }
+                const sep = carrier.length > 0 ? 2 : 0;
+                if (carrierLen + sep + unit.text.length > overlapChars) {
+                    break;
+                }
+                carrier.unshift(unit);
+                carrierLen += sep + unit.text.length;
+            }
+        }
+        const units = carrier.length === 0 ? group : [...carrier, ...group];
+        result.push({
+            // The chunk's heading context follows its first non-carrier unit, so a
+            // heading-leading chunk keeps its heading's path even with overlap.
+            headingContext: group[0]?.headingContext ?? '',
+            units,
+        });
+    }
+    return result;
+}
+function chunkMarkdownSemantic(markdown, options) {
+    if (!markdown) {
+        return [];
+    }
+    const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+    const maxChars = maxTokens * 4;
+    const overlapTokens = Math.max(0, Math.floor(options.overlap));
+    const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+    const units = parseSemanticUnits(markdown);
+    if (units.length === 0) {
+        return [];
+    }
+    const sections = groupSections(units);
+    const baseGroups = buildBaseGroups(sections, maxChars);
+    const groups = applySemanticOverlap(baseGroups, overlapChars);
+    const chunks = [];
+    for (const group of groups) {
+        if (group.units.length === 0) {
+            continue;
+        }
+        const text = group.units
+            .map(unit => unit.text)
+            .join('\n\n')
+            .trim();
+        if (!text) {
+            continue;
+        }
+        chunks.push({
+            headingContext: group.headingContext,
+            index: chunks.length,
+            text,
+            tokenCount: estimateTokens(text).tokenEstimate,
+        });
+    }
+    return chunks;
+}
+export function chunkMarkdown(markdown, options) {
+    if (options.strategy === 'semantic') {
+        return chunkMarkdownSemantic(markdown, options);
+    }
+    return chunkMarkdownChar(markdown, options);
+}

package/src/policy/diagnostics.js

@@ -0,0 +1,53 @@
+function countElements(html, window) {
+    if (!window || !html) {
+        return 0;
+    }
+    const template = window.document.createElement('div');
+    template.innerHTML = html;
+    return template.querySelectorAll('*').length;
+}
+export function assembleDiagnostics(input) {
+    const articleElementCount = countElements(input.articleHtml ?? '', input.window);
+    const removedNodes = Math.max(0, (input.documentElementCount ?? 0) - articleElementCount);
+    return {
+        readerable: input.readerable,
+        extractedNode: input.extractedNode,
+        fallbackUsed: input.fallbackUsed ?? false,
+        gated: input.gated,
+        imagesResolved: input.imagesResolved,
+        pagination: input.pagination,
+        removedNodes,
+        boilerplateRemoved: input.boilerplateRemoved,
+        chromeRemoved: input.chromeRemoved,
+        sanitization: input.sanitization,
+        trace: input.trace,
+        truncated: input.truncated ?? false,
+        ...(input.cache ? { cache: input.cache } : {}),
+        ...(input.preset ? { preset: input.preset } : {}),
+    };
+}
+// Emits trace entries only when enabled, so the debug-off hot path allocates
+// nothing (the fn runs bare). Used by the orchestrator tools to time pipeline
+// stages without threading timers into lower layers.
+export class TraceCollector {
+    enabled;
+    entries = [];
+    constructor(enabled) {
+        this.enabled = enabled;
+    }
+    collect() {
+        return this.enabled ? this.entries : undefined;
+    }
+    run(stage, fn) {
+        if (!this.enabled) {
+            return fn();
+        }
+        const start = performance.now();
+        try {
+            return fn();
+        }
+        finally {
+            this.entries.push({ ms: performance.now() - start, stage });
+        }
+    }
+}

package/src/policy/explain.js

@@ -0,0 +1,104 @@
+import { Readability } from '@mozilla/readability';
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument, resolveLazyImages, } from "../pipeline/normalize.js";
+import { isReaderable } from "../pipeline/readability.js";
+import { resolveReadabilityOptions } from "../pipeline/resolver.js";
+import { assembleDiagnostics } from "./diagnostics.js";
+import { detectGating } from "./gating.js";
+import { detectPagination } from "./pagination.js";
+import { describeSelector } from "./sibling-scan.js";
+const DEFAULT_SNAPSHOT_MAX = 4000;
+const DEFAULT_TOP_N = 5;
+function readScored(el) {
+    const stamp = el
+        .readability;
+    const score = stamp?.contentScore;
+    if (typeof score !== 'number') {
+        return null;
+    }
+    const className = typeof el.className === 'string' ? el.className : '';
+    const candidate = {
+        className,
+        id: el.id,
+        score,
+        tag: el.tagName,
+        textLength: el.textContent.trim().length,
+    };
+    // Two classes cap the hint's width — Readability's expando is invisible to
+    // CSS and an nth-child chain would be brittle against the host's live DOM,
+    // so tag + id + a couple of classes is all a human needs to pick the node
+    // out of a small candidate list.
+    return { ...candidate, selector: describeSelector(el, 2) };
+}
+function truncateSnapshot(html, max) {
+    if (html.length <= max) {
+        return { html, truncated: false };
+    }
+    return { html: html.slice(0, max), truncated: true };
+}
+export function buildExplainReport(options) {
+    const { html, selectors, snapshotMaxChars = DEFAULT_SNAPSHOT_MAX, topN = DEFAULT_TOP_N, baseUrl, } = options;
+    const { document, window } = buildDocument(html, baseUrl);
+    // Mirror the extract pipeline's ordering: gating detection must precede
+    // normalization (chrome stripping would remove the overlay), and pagination
+    // detection runs before applySelectors (a caller's include could hide the
+    // sentinel but not the "more content exists" signal).
+    const gating = detectGating(document);
+    const documentElementCount = document.querySelectorAll('*').length;
+    const normalizeCounts = normalizeDocument(document);
+    resolveLazyImages(document);
+    const pagination = detectPagination(document, baseUrl);
+    applySelectors(document, selectors);
+    const snapshot = truncateSnapshot(document.body.innerHTML, snapshotMaxChars);
+    const readerable = isReaderable(document);
+    const readabilityOptions = resolveReadabilityOptions({});
+    // Clone so the normalized doc (and the snapshot above) is preserved untouched
+    // — Readability restructures its input during parse.
+    const clone = document.cloneNode(true);
+    // Grab node references before parse: Readability detaches scored candidates
+    // from the live tree while restructuring, so a post-parse querySelectorAll
+    // finds almost nothing. The object refs held here keep their `readability`
+    // stamps after being detached — that retention is what surfaces the real
+    // per-candidate scores without forking the library or parsing its debug log.
+    const heldNodes = Array.from(clone.querySelectorAll('*'));
+    const reader = new Readability(clone, readabilityOptions);
+    const article = reader.parse();
+    const parseSucceeded = !!article?.content;
+    const scored = [];
+    for (const el of heldNodes) {
+        const entry = readScored(el);
+        if (entry) {
+            scored.push(entry);
+        }
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const candidates = scored.slice(0, Math.max(0, topN));
+    const chosenRoot = scored[0] ?? null;
+    const diagnostics = assembleDiagnostics({
+        articleHtml: article?.content ?? '',
+        boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+        chromeRemoved: normalizeCounts.chromeRemoved,
+        documentElementCount,
+        extractedNode: 'readability',
+        fallbackUsed: false,
+        gated: gating,
+        pagination,
+        readerable,
+        window,
+    });
+    return {
+        candidates,
+        chosenRoot,
+        fallbackUsed: diagnostics.fallbackUsed,
+        gating: diagnostics.gated,
+        pagination: diagnostics.pagination,
+        parseSucceeded,
+        readerable: diagnostics.readerable ?? false,
+        removedNodes: {
+            boilerplate: diagnostics.boilerplateRemoved ?? 0,
+            chrome: diagnostics.chromeRemoved ?? 0,
+            total: diagnostics.removedNodes ?? 0,
+        },
+        snapshot,
+    };
+}

package/src/policy/fallback.js

@@ -0,0 +1,68 @@
+import { sanitizeHtml } from "../pipeline/sanitize.js";
+import { toMarkdown } from "../pipeline/turndown.js";
+// Below this a block is treated as not text-dense (nav/boilerplate-ish).
+const MIN_DENSE_BLOCK_CHARS = 200;
+function convert(element, options) {
+    const rawHtml = element.outerHTML;
+    let sanitizedHtml = rawHtml;
+    let sanitization = { iframes: 0, scripts: 0 };
+    if (options.sanitize) {
+        const res = sanitizeHtml(rawHtml, options.window);
+        sanitizedHtml = res.html;
+        sanitization = { iframes: res.iframesRemoved, scripts: res.scriptsRemoved };
+    }
+    const markdown = toMarkdown(sanitizedHtml, {
+        codeBlockStyle: options.codeBlockStyle,
+        gfm: options.gfm,
+        headingStyle: options.headingStyle,
+        images: options.images,
+        tables: options.tables,
+        baseUrl: options.baseUrl,
+    });
+    const textContent = element.textContent;
+    if (markdown.trim().length === 0) {
+        return undefined;
+    }
+    return { markdown, sanitization, sanitizedHtml, textContent };
+}
+function largestTextDenseBlock(document) {
+    let best;
+    let bestLen = 0;
+    const candidates = document.querySelectorAll('div, section');
+    candidates.forEach(el => {
+        const len = el.textContent.trim().length;
+        if (len > bestLen && len >= MIN_DENSE_BLOCK_CHARS) {
+            best = el;
+            bestLen = len;
+        }
+    });
+    return best;
+}
+export function extractViaFallback(document, options) {
+    const orderedSelectors = [
+        'article',
+        'main',
+        '[role=main]',
+    ];
+    for (const selector of orderedSelectors) {
+        const root = document.querySelector(selector);
+        if (root) {
+            const converted = convert(root, options);
+            if (converted) {
+                return { ...converted, rootSelector: selector };
+            }
+        }
+    }
+    const dense = largestTextDenseBlock(document);
+    if (dense) {
+        const converted = convert(dense, options);
+        if (converted) {
+            return { ...converted, rootSelector: 'largest-block' };
+        }
+    }
+    const converted = convert(document.body, options);
+    if (converted) {
+        return { ...converted, rootSelector: 'body' };
+    }
+    return null;
+}

package/src/policy/gating.js

@@ -0,0 +1,90 @@
+// Curated vendor/structural paywall selectors — intentionally NOT a greedy
+// `[class*="subscribe"]` (that catches newsletter CTAs on clean articles and
+// false-positives). Each entry names a known paywall surface.
+const PAYWALL_SELECTORS = [
+    '.piano',
+    '#piano',
+    '.tp-modal',
+    '.tp-active',
+    '[class*="subscribe-wall"]',
+    '[id*="subscribe-wall"]',
+    '[class*="metered-wall"]',
+    '[id*="metered-wall"]',
+    '.leaky-paywall',
+];
+// Class/id substring hits are verdicts, not signals. Measured on real captures:
+// Daily Mail free articles carry `<html class="… paywall-ineligible">` plus ~240
+// `is-paywalled` / `is-paywall-processed` feed badges — state about other
+// articles or processing markers, all matching `[class*="paywall"]`; Corriere's
+// free articles carry an `offer-header-piano` subscription header matching
+// `[class*="piano"]`. Real surfaces name the gate as the head noun (WIRED's
+// `paywall-modal`, the camel-cased `PaywallModalWrapper`).
+const PAYWALL_ATTR_CANDIDATES = '[class*="paywall"], [id*="paywall"], [class*="piano"], [id*="piano"]';
+const NEGATION_SEGMENTS = new Set([
+    'bypass',
+    'disabled',
+    'exempt',
+    'free',
+    'ineligible',
+    'no',
+    'non',
+    'not',
+    'off',
+    'optout',
+    'without',
+]);
+function namesGateSurface(classAndId, noun) {
+    for (const token of classAndId.split(/\s+/)) {
+        const segments = token
+            .replace(/([a-z])([A-Z])/g, '$1 $2')
+            .toLowerCase()
+            .split(/[^a-z]+/)
+            .filter(Boolean);
+        if (segments[0] !== noun) {
+            continue;
+        }
+        if (segments.some(segment => NEGATION_SEGMENTS.has(segment))) {
+            continue;
+        }
+        return true;
+    }
+    return false;
+}
+// Phrases that essentially never appear on a fully-unlocked article. Bare
+// "Subscribe" nav links / newsletter CTAs are intentionally excluded — they
+// are ubiquitous and would mislead the host into discarding complete content.
+const METERED_TEXT_RE = /(\d+)\s*(?:free\s*)?(?:articles?|stories?)\s*(?:left|remaining)|you\s+have\s+reached\s+(?:your\s+)?(?:free\s+)?(?:article\s+|story\s+)?limit|subscribe\s+to\s+(?:continue\s+)?reading|read\s+the\s+full\s+(?:article|story)|unlock\s+(?:this|full|all)\s+(?:article|story|content)|keep\s+reading\s+with/i;
+function findPaywallOverlay(document) {
+    for (const el of document.querySelectorAll(PAYWALL_ATTR_CANDIDATES)) {
+        const classAndId = `${el.getAttribute('class') ?? ''} ${el.getAttribute('id') ?? ''}`;
+        if (!el.isConnected) {
+            continue;
+        }
+        if (namesGateSurface(classAndId, 'paywall') ||
+            namesGateSurface(classAndId, 'piano')) {
+            return { likely: true, reason: 'paywall overlay' };
+        }
+    }
+    for (const selector of PAYWALL_SELECTORS) {
+        const el = document.querySelector(selector);
+        if (el?.isConnected) {
+            return { likely: true, reason: 'paywall overlay' };
+        }
+    }
+    return undefined;
+}
+// One textContent read is cheaper than a per-element scan of a large doc and
+// avoids ordering a query per selector. jsdom computes body.textContent once.
+function findMeteredMessage(document) {
+    const text = document.body.textContent;
+    if (METERED_TEXT_RE.test(text)) {
+        return { likely: true, reason: 'metered paywall message' };
+    }
+    return undefined;
+}
+// Reads the DOM only — never fetches, authenticates, or mutates the document.
+// Conservative by design: a false positive misleads the host into treating
+// complete content as truncated, which is worse than a miss.
+export function detectGating(document) {
+    return findPaywallOverlay(document) ?? findMeteredMessage(document);
+}