@v1nvn/readability 0.45.0 → 0.45.1

package/src/pipeline/resolver.js

@@ -0,0 +1,113 @@
+// Readability.js confirmed defaults.
+const DEFAULT_CHAR_THRESHOLD = 500;
+const DEFAULT_N_TOP_CANDIDATES = 5;
+// Readability keeps a class only by strict `classesToPreserve.includes(cls)`
+// equality (not regex), stripping code-block language tokens by default; list
+// the common highlight.js/prism `language-*` tokens so fences keep their tag.
+// `rdrm-math` is the math marker emitted by `pipeline/math.ts` — Readability
+// would otherwise strip it before turndown's math rule runs. Applies only when
+// `keepClasses` is false.
+const CLASSES_TO_PRESERVE = [
+    'hljs',
+    'language-asm',
+    'language-assembly',
+    'language-bash',
+    'language-c',
+    'language-clojure',
+    'language-cpp',
+    'language-cs',
+    'language-csharp',
+    'language-css',
+    'language-dart',
+    'language-diff',
+    'language-dockerfile',
+    'language-elixir',
+    'language-erlang',
+    'language-go',
+    'language-graphql',
+    'language-haskell',
+    'language-html',
+    'language-ini',
+    'language-java',
+    'language-javascript',
+    'language-js',
+    'language-jsx',
+    'language-json',
+    'language-kotlin',
+    'language-lisp',
+    'language-lua',
+    'language-md',
+    'language-markdown',
+    'language-objc',
+    'language-objectivec',
+    'language-perl',
+    'language-php',
+    'language-plaintext',
+    'language-powershell',
+    'language-py',
+    'language-python',
+    'language-r',
+    'language-rb',
+    'language-rs',
+    'language-ruby',
+    'language-rust',
+    'language-scala',
+    'language-sh',
+    'language-shell',
+    'language-sql',
+    'language-swift',
+    'language-text',
+    'language-toml',
+    'language-ts',
+    'language-tsx',
+    'language-typescript',
+    'language-vim',
+    'language-wasm',
+    'language-xml',
+    'language-yaml',
+    'language-yml',
+    'rdrm-math',
+];
+// Bare tokens (the `X` in `language-X`) shared by convention resolvers that
+// must validate a candidate against the preserve set, e.g. sandpack's `sp-*`
+// classes, which carry infra strings alongside the language token. Filter to
+// `language-` entries so non-language preserve additions (`hljs`, `rdrm-math`)
+// do not pollute the set with the empty token.
+export const KNOWN_LANGUAGE_TOKENS = new Set(CLASSES_TO_PRESERVE.filter(c => c.startsWith('language-')).map(c => c.slice('language-'.length)));
+function knobsForMode(mode) {
+    switch (mode) {
+        case 'aggressive':
+            return {
+                charThreshold: Math.round(DEFAULT_CHAR_THRESHOLD / 2),
+                nbTopCandidates: DEFAULT_N_TOP_CANDIDATES * 2,
+            };
+        case 'balanced':
+            // null ⇒ emit nothing; Readability uses its own defaults.
+            return null;
+        case 'conservative':
+            return {
+                charThreshold: DEFAULT_CHAR_THRESHOLD * 2,
+                nbTopCandidates: Math.max(1, Math.round(DEFAULT_N_TOP_CANDIDATES / 2)),
+            };
+    }
+}
+export function resolveReadabilityOptions(input) {
+    const mode = input.extraction ?? 'balanced';
+    const modeKnobs = knobsForMode(mode);
+    const keepClasses = input.keepClasses ?? false;
+    const charThreshold = input.minArticleLength !== undefined
+        ? input.minArticleLength
+        : modeKnobs?.charThreshold;
+    const nbTopCandidates = modeKnobs?.nbTopCandidates;
+    // Escape hatch spreads last so it wins verbatim over every derived knob.
+    return {
+        classesToPreserve: keepClasses ? [] : [...CLASSES_TO_PRESERVE],
+        keepClasses,
+        ...(charThreshold !== undefined ? { charThreshold } : {}),
+        ...(nbTopCandidates !== undefined ? { nbTopCandidates } : {}),
+        ...(input.maxNodes !== undefined
+            ? { maxElemsToParse: input.maxNodes }
+            : {}),
+        ...(input.readabilityOverrides ?? {}),
+    };
+}

package/src/pipeline/sanitize.js

@@ -0,0 +1,26 @@
+import DOMPurify from 'dompurify';
+function countRemoved(removed, tagName) {
+    let count = 0;
+    for (const entry of removed) {
+        // RemovedAttribute entries have no `element`; count element removals only.
+        if ('element' in entry &&
+            entry.element.nodeName.toUpperCase() === tagName) {
+            count += 1;
+        }
+    }
+    return count;
+}
+// DOMPurify scrubs Readability's output (scripts, inline event handlers,
+// iframes) before Turndown converts it to Markdown — it is a pipeline stage,
+// not a product. A sanitized-HTML output mode was rejected: the server emits
+// Markdown, and callers wanting sanitized HTML should run DOMPurify directly.
+export function sanitizeHtml(dirty, window) {
+    // DOMPurify's WindowLike is structurally compatible at runtime but not under TS.
+    const purify = DOMPurify(window);
+    const html = purify.sanitize(dirty);
+    return {
+        html,
+        iframesRemoved: countRemoved(purify.removed, 'IFRAME'),
+        scriptsRemoved: countRemoved(purify.removed, 'SCRIPT'),
+    };
+}

package/src/pipeline/tables.js

@@ -0,0 +1,239 @@
+import { resolveCellText } from "./cell-text.js";
+// `<thead>`/`<tbody>`/`<tfoot>` group `<tr>` rows; a `<table>` may also hold `<tr>`
+// directly. Walking these levels (and only these) keeps nested `<table>`s out of
+// the matrix — those are emitted as their own replacement by turndown.
+const SECTION_TAGS = new Set(['TBODY', 'TFOOT', 'THEAD']);
+const CELL_TAGS = new Set(['TD', 'TH']);
+function spanOf(cell, attr) {
+    const raw = cell.getAttribute(attr);
+    if (raw === null) {
+        return 1;
+    }
+    const parsed = Number.parseInt(raw, 10);
+    // HTML treats 0 (and any non-positive / unparseable value) as 1.
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        return 1;
+    }
+    return parsed;
+}
+function collectRows(table) {
+    const rows = [];
+    for (const child of Array.from(table.children)) {
+        if (child.tagName === 'TR') {
+            rows.push(child);
+        }
+        else if (SECTION_TAGS.has(child.tagName)) {
+            for (const tr of Array.from(child.children)) {
+                if (tr.tagName === 'TR') {
+                    rows.push(tr);
+                }
+            }
+        }
+    }
+    return rows;
+}
+function cellsOf(tr) {
+    return Array.from(tr.children).filter(child => CELL_TAGS.has(child.tagName));
+}
+// Text projection of buildCellGrid: origin cells → resolveCellText, span/pad slots → ''.
+export function parseTableMatrix(table) {
+    return buildCellGrid(table).map(row => row.map(cell => (cell === null ? '' : resolveCellText(cell))));
+}
+function escapeGfmCell(text) {
+    return text.replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
+}
+export function renderTableGfm(matrix) {
+    if (matrix.length === 0) {
+        return '';
+    }
+    const cols = matrix[0].length;
+    const lines = [];
+    lines.push(`| ${matrix[0].map(escapeGfmCell).join(' | ')} |`);
+    lines.push(`| ${Array.from({ length: cols }, () => '---').join(' | ')} |`);
+    for (let r = 1; r < matrix.length; r++) {
+        lines.push(`| ${matrix[r].map(escapeGfmCell).join(' | ')} |`);
+    }
+    return lines.join('\n');
+}
+function escapeCsvField(text) {
+    if (/[",\r\n]/.test(text)) {
+        return `"${text.replace(/"/g, '""')}"`;
+    }
+    return text;
+}
+export function renderTableCsv(matrix) {
+    if (matrix.length === 0) {
+        return '';
+    }
+    return matrix.map(row => row.map(escapeCsvField).join(',')).join('\n');
+}
+function headerKeys(header) {
+    return header.map((cell, i) => (cell === '' ? `column_${i}` : cell));
+}
+// Tracks the originating cell element at each grid position after resolving
+// rowspan/colspan, so resolveHeaderKeys can read td/th attributes the text
+// matrix discards. parseTableMatrix is the text projection of this grid; `null`
+// marks positions filled by a span (no origin cell).
+function buildCellGrid(table) {
+    const rows = collectRows(table);
+    if (rows.length === 0) {
+        return [];
+    }
+    const grid = [];
+    const occupied = [];
+    let maxCols = 0;
+    for (let r = 0; r < rows.length; r++) {
+        while (grid.length <= r) {
+            grid.push([]);
+            occupied.push([]);
+        }
+        const rowCells = grid[r];
+        const rowOccupied = occupied[r];
+        let col = 0;
+        for (const cell of cellsOf(rows[r])) {
+            while (rowOccupied[col]) {
+                col++;
+            }
+            const rowspan = spanOf(cell, 'rowspan');
+            const colspan = spanOf(cell, 'colspan');
+            rowCells[col] = cell;
+            rowOccupied[col] = true;
+            for (let dr = 0; dr < rowspan; dr++) {
+                for (let dc = 0; dc < colspan; dc++) {
+                    if (dr === 0 && dc === 0) {
+                        continue;
+                    }
+                    const rr = r + dr;
+                    while (grid.length <= rr) {
+                        grid.push([]);
+                        occupied.push([]);
+                    }
+                    const occ = occupied[rr];
+                    while (occ.length <= col + dc) {
+                        occ.push(false);
+                        grid[rr].push(null);
+                    }
+                    occ[col + dc] = true;
+                }
+            }
+            col += colspan;
+            if (col > maxCols) {
+                maxCols = col;
+            }
+        }
+    }
+    const dense = [];
+    for (const row of grid) {
+        dense.push(Array.from({ length: maxCols }, (_, i) => row[i] ?? null));
+    }
+    return dense;
+}
+function isHeaderRow(tr) {
+    const cells = cellsOf(tr);
+    return cells.length > 0 && cells.every(c => c.tagName === 'TH');
+}
+function slugify(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, '_')
+        .replace(/^_+|_+$/g, '');
+}
+function readCellLabel(cell) {
+    return (cell.getAttribute('aria-label') ??
+        cell.getAttribute('title') ??
+        cell.getAttribute('data-label'));
+}
+export function resolveHeaderKeys(table, matrix) {
+    if (matrix.length === 0) {
+        return [];
+    }
+    const width = matrix[0].length;
+    const keys = headerKeys(matrix[0]);
+    const rows = collectRows(table);
+    if (rows.length === 0) {
+        return keys;
+    }
+    const cellGrid = buildCellGrid(table);
+    // Headerless columns lose their name without a th; recover from a colspan
+    // parent in row 0, falling back to the position under that parent.
+    const parents = Array.from({ length: width }, () => '');
+    const parentChildren = new Map();
+    {
+        let col = 0;
+        for (const cell of cellsOf(rows[0])) {
+            const colspan = spanOf(cell, 'colspan');
+            const text = resolveCellText(cell);
+            if (colspan > 1 && text) {
+                const slug = slugify(text);
+                const kids = [];
+                for (let dc = 0; dc < colspan && col + dc < width; dc++) {
+                    parents[col + dc] = slug;
+                    kids.push(col + dc);
+                }
+                parentChildren.set(slug, kids);
+            }
+            col += colspan;
+        }
+    }
+    const hasSubRow = rows.length > 1 && isHeaderRow(rows[1]);
+    for (let c = 0; c < width; c++) {
+        const dataCell = cellGrid[1]?.[c];
+        if (dataCell) {
+            const label = readCellLabel(dataCell);
+            if (label) {
+                keys[c] = label;
+                continue;
+            }
+        }
+        const parent = parents[c];
+        if (!parent) {
+            continue;
+        }
+        if (hasSubRow) {
+            const sub = matrix[1][c] ? slugify(matrix[1][c]) : '';
+            if (sub) {
+                keys[c] = `${parent}_${sub}`;
+                continue;
+            }
+        }
+        if (matrix[0][c] === '') {
+            const kids = parentChildren.get(parent);
+            const idx = kids ? kids.indexOf(c) + 1 : c;
+            keys[c] = `${parent}_${idx}`;
+        }
+    }
+    return keys;
+}
+function resolveJsonKeys(matrix, keys) {
+    const width = matrix[0].length;
+    if (!keys) {
+        return headerKeys(matrix[0]);
+    }
+    return Array.from({ length: width }, (_, i) => keys[i] ?? `column_${i}`);
+}
+export function renderTableJson(matrix, keys) {
+    if (matrix.length < 2) {
+        return '[]';
+    }
+    const resolved = resolveJsonKeys(matrix, keys);
+    const records = [];
+    for (let r = 1; r < matrix.length; r++) {
+        const row = matrix[r];
+        const record = {};
+        for (let c = 0; c < resolved.length; c++) {
+            record[resolved[c]] = row[c] ?? '';
+        }
+        records.push(record);
+    }
+    return JSON.stringify(records, null, 2);
+}
+export function renderTable(matrix, format, keys) {
+    switch (format) {
+        case 'csv':
+            return renderTableCsv(matrix);
+        case 'gfm':
+            return renderTableGfm(matrix);
+        case 'json':
+            return renderTableJson(matrix, keys);
+    }
+}

package/src/pipeline/turndown.js

@@ -0,0 +1,153 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { processFootnotes } from "./footnotes.js";
+import { parseTableMatrix, renderTable } from "./tables.js";
+import { absolutize } from "./urls.js";
+export function toMarkdown(html, options) {
+    const service = new TurndownService({
+        bulletListMarker: '-',
+        // Code renders inline in the Markdown as fenced blocks; normalize rewrites
+        // source conventions to `<code class="language-X">` first, so the fence
+        // carries a language tag. A code-only extraction tool was rejected as
+        // redundant surface for a niche the main path already covers.
+        codeBlockStyle: options?.codeBlockStyle ?? 'fenced',
+        emDelimiter: '_',
+        fence: '```',
+        headingStyle: options?.headingStyle ?? 'atx',
+        strongDelimiter: '**',
+    });
+    if (options?.gfm !== false) {
+        service.use(gfm);
+    }
+    const tableFormat = options?.tables;
+    if (tableFormat !== undefined) {
+        // addRule prepends, so this shadows the gfm plugin's table rule AND its
+        // headerless-table keep filter (which lives in a separate _keep list, only
+        // consulted when no rule in the array matches). Matching every TABLE here
+        // means the keep list is never reached for tables.
+        service.addRule('tableMatrix', {
+            filter: node => node.nodeName === 'TABLE',
+            replacement: (_content, node) => {
+                const matrix = parseTableMatrix(node);
+                if (matrix.length === 0) {
+                    return '';
+                }
+                const body = renderTable(matrix, tableFormat);
+                // GFM rows are already markdown table syntax; csv/json need a fenced block.
+                if (tableFormat === 'gfm') {
+                    return `\n\n${body}\n\n`;
+                }
+                return `\n\n\`\`\`${tableFormat}\n${body}\n\`\`\`\n\n`;
+            },
+        });
+    }
+    const imageMode = options?.images;
+    const baseUrl = options?.baseUrl;
+    // `reference` mode collects definitions as images render; append them after the pass.
+    const references = [];
+    applyImagePolicy(service, imageMode, baseUrl, references);
+    // Readability pre-absolutizes anchors on the main path, but html_to_markdown
+    // and the extract fallback bypass it; turndown is shared, so absolutize here
+    // for every path. Empty/absent href falls through to bare content to match
+    // turndown's default link rule (which never matches when href is falsy).
+    service.addRule('anchorAbsolutize', {
+        filter: 'a',
+        replacement: (content, node) => {
+            const rawHref = node.getAttribute('href');
+            if (!rawHref) {
+                return content;
+            }
+            const href = absolutize(rawHref, baseUrl);
+            const title = node.getAttribute('title');
+            const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : '';
+            return `[${content}](${href}${titlePart})`;
+        },
+    });
+    // Math markers are emitted by `pipeline/math.ts` and survive Readability via the
+    // `rdrm-math` preserve entry. A rule's replacement is inserted verbatim, so
+    // the raw LaTeX backslashes survive turndown untouched.
+    service.addRule('mathMarker', {
+        filter: node => node.nodeName === 'SPAN' && node.classList.contains('rdrm-math'),
+        replacement: (_content, node) => {
+            const tex = node.textContent.trim();
+            if (!tex) {
+                return '';
+            }
+            const display = node.getAttribute('data-display') === 'true';
+            return display ? `$$${tex}$$` : `$${tex}$`;
+        },
+    });
+    const fnResult = processFootnotes(html);
+    const sourceHtml = fnResult?.html ?? html;
+    let body = service.turndown(sourceHtml);
+    if (fnResult) {
+        // Turndown sees `[^N]` as a shortcut-reference link and backslash-escapes
+        // the brackets; reverse that for the markers we emitted so they render as
+        // footnote refs. Code blocks pass through verbatim and never carry the
+        // escape, so they are unaffected.
+        for (let n = 1; n <= fnResult.footnoteDefs.length; n++) {
+            body = body.replaceAll(`\\[^${n}\\]`, `[^${n}]`);
+        }
+    }
+    const trailingBlocks = [];
+    if (imageMode === 'reference' && references.length > 0) {
+        trailingBlocks.push(references.map((ref, i) => `[img-${i + 1}]: ${ref}`).join('\n'));
+    }
+    if (fnResult && fnResult.footnoteDefs.length > 0) {
+        trailingBlocks.push(fnResult.footnoteDefs.map((def, i) => `[^${i + 1}]: ${def}`).join('\n'));
+    }
+    if (trailingBlocks.length === 0) {
+        return body;
+    }
+    return `${body.replace(/\n+$/, '')}\n\n${trailingBlocks.join('\n')}`;
+}
+function applyImagePolicy(service, mode, baseUrl, references) {
+    switch (mode) {
+        case 'drop': {
+            // `.remove('img')` doesn't reliably drop void elements in turndown 7.x.
+            service.addRule('dropImage', {
+                filter: 'img',
+                replacement: () => '',
+            });
+            break;
+        }
+        case 'keep':
+        case undefined: {
+            service.addRule('imageKeep', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    const alt = node.getAttribute('alt') ?? '';
+                    return src ? `![${alt}](${src})` : '';
+                },
+            });
+            break;
+        }
+        case 'reference': {
+            service.addRule('imageReference', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    if (!src) {
+                        return '';
+                    }
+                    references.push(src);
+                    const id = references.length;
+                    const alt = node.getAttribute('alt') ?? '';
+                    return `![${alt}][img-${id}]`;
+                },
+            });
+            break;
+        }
+        case 'src-only': {
+            service.addRule('imageSrcOnly', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    return src ? `\n\n${src}\n\n` : '';
+                },
+            });
+            break;
+        }
+    }
+}

package/src/pipeline/urls.js

@@ -0,0 +1,14 @@
+// Shared by the turndown rules (image/anchor absolutization) and the
+// extract_links tool. Returns the input unchanged when either side is missing
+// or the URL constructor rejects the pair, so callers never throw on bad input.
+export function absolutize(src, baseUrl) {
+    if (!src || !baseUrl) {
+        return src;
+    }
+    try {
+        return new URL(src, baseUrl).href;
+    }
+    catch {
+        return src;
+    }
+}