@v1nvn/readability 0.44.0 → 0.45.1

package/src/pipeline/tables.ts

@@ -0,0 +1,286 @@
+import { resolveCellText } from './cell-text.ts';
+
+export type TableFormat = 'csv' | 'gfm' | 'json';
+
+// `<thead>`/`<tbody>`/`<tfoot>` group `<tr>` rows; a `<table>` may also hold `<tr>`
+// directly. Walking these levels (and only these) keeps nested `<table>`s out of
+// the matrix — those are emitted as their own replacement by turndown.
+const SECTION_TAGS = new Set(['TBODY', 'TFOOT', 'THEAD']);
+const CELL_TAGS = new Set(['TD', 'TH']);
+
+function spanOf(cell: Element, attr: 'colspan' | 'rowspan'): number {
+  const raw = cell.getAttribute(attr);
+  if (raw === null) {
+    return 1;
+  }
+  const parsed = Number.parseInt(raw, 10);
+  // HTML treats 0 (and any non-positive / unparseable value) as 1.
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    return 1;
+  }
+  return parsed;
+}
+
+function collectRows(table: Element): readonly Element[] {
+  const rows: Element[] = [];
+  for (const child of Array.from(table.children)) {
+    if (child.tagName === 'TR') {
+      rows.push(child);
+    } else if (SECTION_TAGS.has(child.tagName)) {
+      for (const tr of Array.from(child.children)) {
+        if (tr.tagName === 'TR') {
+          rows.push(tr);
+        }
+      }
+    }
+  }
+  return rows;
+}
+
+function cellsOf(tr: Element): readonly Element[] {
+  return Array.from(tr.children).filter(child => CELL_TAGS.has(child.tagName));
+}
+
+// Text projection of buildCellGrid: origin cells → resolveCellText, span/pad slots → ''.
+export function parseTableMatrix(table: Element): string[][] {
+  return buildCellGrid(table).map(row =>
+    row.map(cell => (cell === null ? '' : resolveCellText(cell))),
+  );
+}
+
+function escapeGfmCell(text: string): string {
+  return text.replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
+}
+
+export function renderTableGfm(matrix: string[][]): string {
+  if (matrix.length === 0) {
+    return '';
+  }
+  const cols = matrix[0].length;
+  const lines: string[] = [];
+  lines.push(`| ${matrix[0].map(escapeGfmCell).join(' | ')} |`);
+  lines.push(`| ${Array.from({ length: cols }, () => '---').join(' | ')} |`);
+  for (let r = 1; r < matrix.length; r++) {
+    lines.push(`| ${matrix[r].map(escapeGfmCell).join(' | ')} |`);
+  }
+  return lines.join('\n');
+}
+
+function escapeCsvField(text: string): string {
+  if (/[",\r\n]/.test(text)) {
+    return `"${text.replace(/"/g, '""')}"`;
+  }
+  return text;
+}
+
+export function renderTableCsv(matrix: string[][]): string {
+  if (matrix.length === 0) {
+    return '';
+  }
+  return matrix.map(row => row.map(escapeCsvField).join(',')).join('\n');
+}
+
+function headerKeys(header: readonly string[]): string[] {
+  return header.map((cell, i) => (cell === '' ? `column_${i}` : cell));
+}
+
+// Tracks the originating cell element at each grid position after resolving
+// rowspan/colspan, so resolveHeaderKeys can read td/th attributes the text
+// matrix discards. parseTableMatrix is the text projection of this grid; `null`
+// marks positions filled by a span (no origin cell).
+function buildCellGrid(table: Element): (Element | null)[][] {
+  const rows = collectRows(table);
+  if (rows.length === 0) {
+    return [];
+  }
+  const grid: (Element | null)[][] = [];
+  const occupied: boolean[][] = [];
+  let maxCols = 0;
+
+  for (let r = 0; r < rows.length; r++) {
+    while (grid.length <= r) {
+      grid.push([]);
+      occupied.push([]);
+    }
+    const rowCells = grid[r];
+    const rowOccupied = occupied[r];
+
+    let col = 0;
+    for (const cell of cellsOf(rows[r])) {
+      while (rowOccupied[col]) {
+        col++;
+      }
+      const rowspan = spanOf(cell, 'rowspan');
+      const colspan = spanOf(cell, 'colspan');
+      rowCells[col] = cell;
+      rowOccupied[col] = true;
+      for (let dr = 0; dr < rowspan; dr++) {
+        for (let dc = 0; dc < colspan; dc++) {
+          if (dr === 0 && dc === 0) {
+            continue;
+          }
+          const rr = r + dr;
+          while (grid.length <= rr) {
+            grid.push([]);
+            occupied.push([]);
+          }
+          const occ = occupied[rr];
+          while (occ.length <= col + dc) {
+            occ.push(false);
+            grid[rr].push(null);
+          }
+          occ[col + dc] = true;
+        }
+      }
+      col += colspan;
+      if (col > maxCols) {
+        maxCols = col;
+      }
+    }
+  }
+
+  const dense: (Element | null)[][] = [];
+  for (const row of grid) {
+    dense.push(Array.from({ length: maxCols }, (_, i) => row[i] ?? null));
+  }
+  return dense;
+}
+
+function isHeaderRow(tr: Element): boolean {
+  const cells = cellsOf(tr);
+  return cells.length > 0 && cells.every(c => c.tagName === 'TH');
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '_')
+    .replace(/^_+|_+$/g, '');
+}
+
+function readCellLabel(cell: Element): null | string {
+  return (
+    cell.getAttribute('aria-label') ??
+    cell.getAttribute('title') ??
+    cell.getAttribute('data-label')
+  );
+}
+
+export function resolveHeaderKeys(
+  table: Element,
+  matrix: string[][],
+): string[] {
+  if (matrix.length === 0) {
+    return [];
+  }
+  const width = matrix[0].length;
+  const keys = headerKeys(matrix[0]);
+
+  const rows = collectRows(table);
+  if (rows.length === 0) {
+    return keys;
+  }
+
+  const cellGrid = buildCellGrid(table);
+
+  // Headerless columns lose their name without a th; recover from a colspan
+  // parent in row 0, falling back to the position under that parent.
+  const parents: string[] = Array.from({ length: width }, () => '');
+  const parentChildren = new Map<string, number[]>();
+  {
+    let col = 0;
+    for (const cell of cellsOf(rows[0])) {
+      const colspan = spanOf(cell, 'colspan');
+      const text = resolveCellText(cell);
+      if (colspan > 1 && text) {
+        const slug = slugify(text);
+        const kids: number[] = [];
+        for (let dc = 0; dc < colspan && col + dc < width; dc++) {
+          parents[col + dc] = slug;
+          kids.push(col + dc);
+        }
+        parentChildren.set(slug, kids);
+      }
+      col += colspan;
+    }
+  }
+
+  const hasSubRow = rows.length > 1 && isHeaderRow(rows[1]);
+
+  for (let c = 0; c < width; c++) {
+    const dataCell = cellGrid[1]?.[c];
+    if (dataCell) {
+      const label = readCellLabel(dataCell);
+      if (label) {
+        keys[c] = label;
+        continue;
+      }
+    }
+
+    const parent = parents[c];
+    if (!parent) {
+      continue;
+    }
+
+    if (hasSubRow) {
+      const sub = matrix[1][c] ? slugify(matrix[1][c]) : '';
+      if (sub) {
+        keys[c] = `${parent}_${sub}`;
+        continue;
+      }
+    }
+
+    if (matrix[0][c] === '') {
+      const kids = parentChildren.get(parent);
+      const idx = kids ? kids.indexOf(c) + 1 : c;
+      keys[c] = `${parent}_${idx}`;
+    }
+  }
+  return keys;
+}
+
+function resolveJsonKeys(
+  matrix: string[][],
+  keys: readonly string[] | undefined,
+): string[] {
+  const width = matrix[0].length;
+  if (!keys) {
+    return headerKeys(matrix[0]);
+  }
+  return Array.from({ length: width }, (_, i) => keys[i] ?? `column_${i}`);
+}
+
+export function renderTableJson(
+  matrix: string[][],
+  keys?: readonly string[],
+): string {
+  if (matrix.length < 2) {
+    return '[]';
+  }
+  const resolved = resolveJsonKeys(matrix, keys);
+  const records: Record<string, string>[] = [];
+  for (let r = 1; r < matrix.length; r++) {
+    const row = matrix[r];
+    const record: Record<string, string> = {};
+    for (let c = 0; c < resolved.length; c++) {
+      record[resolved[c]] = row[c] ?? '';
+    }
+    records.push(record);
+  }
+  return JSON.stringify(records, null, 2);
+}
+
+export function renderTable(
+  matrix: string[][],
+  format: TableFormat,
+  keys?: readonly string[],
+): string {
+  switch (format) {
+    case 'csv':
+      return renderTableCsv(matrix);
+    case 'gfm':
+      return renderTableGfm(matrix);
+    case 'json':
+      return renderTableJson(matrix, keys);
+  }
+}

package/src/pipeline/turndown.js

@@ -0,0 +1,153 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+import { processFootnotes } from "./footnotes.js";
+import { parseTableMatrix, renderTable } from "./tables.js";
+import { absolutize } from "./urls.js";
+export function toMarkdown(html, options) {
+    const service = new TurndownService({
+        bulletListMarker: '-',
+        // Code renders inline in the Markdown as fenced blocks; normalize rewrites
+        // source conventions to `<code class="language-X">` first, so the fence
+        // carries a language tag. A code-only extraction tool was rejected as
+        // redundant surface for a niche the main path already covers.
+        codeBlockStyle: options?.codeBlockStyle ?? 'fenced',
+        emDelimiter: '_',
+        fence: '```',
+        headingStyle: options?.headingStyle ?? 'atx',
+        strongDelimiter: '**',
+    });
+    if (options?.gfm !== false) {
+        service.use(gfm);
+    }
+    const tableFormat = options?.tables;
+    if (tableFormat !== undefined) {
+        // addRule prepends, so this shadows the gfm plugin's table rule AND its
+        // headerless-table keep filter (which lives in a separate _keep list, only
+        // consulted when no rule in the array matches). Matching every TABLE here
+        // means the keep list is never reached for tables.
+        service.addRule('tableMatrix', {
+            filter: node => node.nodeName === 'TABLE',
+            replacement: (_content, node) => {
+                const matrix = parseTableMatrix(node);
+                if (matrix.length === 0) {
+                    return '';
+                }
+                const body = renderTable(matrix, tableFormat);
+                // GFM rows are already markdown table syntax; csv/json need a fenced block.
+                if (tableFormat === 'gfm') {
+                    return `\n\n${body}\n\n`;
+                }
+                return `\n\n\`\`\`${tableFormat}\n${body}\n\`\`\`\n\n`;
+            },
+        });
+    }
+    const imageMode = options?.images;
+    const baseUrl = options?.baseUrl;
+    // `reference` mode collects definitions as images render; append them after the pass.
+    const references = [];
+    applyImagePolicy(service, imageMode, baseUrl, references);
+    // Readability pre-absolutizes anchors on the main path, but html_to_markdown
+    // and the extract fallback bypass it; turndown is shared, so absolutize here
+    // for every path. Empty/absent href falls through to bare content to match
+    // turndown's default link rule (which never matches when href is falsy).
+    service.addRule('anchorAbsolutize', {
+        filter: 'a',
+        replacement: (content, node) => {
+            const rawHref = node.getAttribute('href');
+            if (!rawHref) {
+                return content;
+            }
+            const href = absolutize(rawHref, baseUrl);
+            const title = node.getAttribute('title');
+            const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : '';
+            return `[${content}](${href}${titlePart})`;
+        },
+    });
+    // Math markers are emitted by `pipeline/math.ts` and survive Readability via the
+    // `rdrm-math` preserve entry. A rule's replacement is inserted verbatim, so
+    // the raw LaTeX backslashes survive turndown untouched.
+    service.addRule('mathMarker', {
+        filter: node => node.nodeName === 'SPAN' && node.classList.contains('rdrm-math'),
+        replacement: (_content, node) => {
+            const tex = node.textContent.trim();
+            if (!tex) {
+                return '';
+            }
+            const display = node.getAttribute('data-display') === 'true';
+            return display ? `$$${tex}$$` : `$${tex}$`;
+        },
+    });
+    const fnResult = processFootnotes(html);
+    const sourceHtml = fnResult?.html ?? html;
+    let body = service.turndown(sourceHtml);
+    if (fnResult) {
+        // Turndown sees `[^N]` as a shortcut-reference link and backslash-escapes
+        // the brackets; reverse that for the markers we emitted so they render as
+        // footnote refs. Code blocks pass through verbatim and never carry the
+        // escape, so they are unaffected.
+        for (let n = 1; n <= fnResult.footnoteDefs.length; n++) {
+            body = body.replaceAll(`\\[^${n}\\]`, `[^${n}]`);
+        }
+    }
+    const trailingBlocks = [];
+    if (imageMode === 'reference' && references.length > 0) {
+        trailingBlocks.push(references.map((ref, i) => `[img-${i + 1}]: ${ref}`).join('\n'));
+    }
+    if (fnResult && fnResult.footnoteDefs.length > 0) {
+        trailingBlocks.push(fnResult.footnoteDefs.map((def, i) => `[^${i + 1}]: ${def}`).join('\n'));
+    }
+    if (trailingBlocks.length === 0) {
+        return body;
+    }
+    return `${body.replace(/\n+$/, '')}\n\n${trailingBlocks.join('\n')}`;
+}
+function applyImagePolicy(service, mode, baseUrl, references) {
+    switch (mode) {
+        case 'drop': {
+            // `.remove('img')` doesn't reliably drop void elements in turndown 7.x.
+            service.addRule('dropImage', {
+                filter: 'img',
+                replacement: () => '',
+            });
+            break;
+        }
+        case 'keep':
+        case undefined: {
+            service.addRule('imageKeep', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    const alt = node.getAttribute('alt') ?? '';
+                    return src ? `![${alt}](${src})` : '';
+                },
+            });
+            break;
+        }
+        case 'reference': {
+            service.addRule('imageReference', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    if (!src) {
+                        return '';
+                    }
+                    references.push(src);
+                    const id = references.length;
+                    const alt = node.getAttribute('alt') ?? '';
+                    return `![${alt}][img-${id}]`;
+                },
+            });
+            break;
+        }
+        case 'src-only': {
+            service.addRule('imageSrcOnly', {
+                filter: 'img',
+                replacement: (_content, node) => {
+                    const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+                    return src ? `\n\n${src}\n\n` : '';
+                },
+            });
+            break;
+        }
+    }
+}

package/src/pipeline/turndown.ts

@@ -0,0 +1,191 @@
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+
+import type { TableFormat } from './tables.ts';
+
+import { processFootnotes } from './footnotes.ts';
+import { parseTableMatrix, renderTable } from './tables.ts';
+import { absolutize } from './urls.ts';
+
+export type CodeBlockStyle = 'fenced' | 'indented';
+export type HeadingStyle = 'atx' | 'setext';
+export type ImageMode = 'drop' | 'keep' | 'reference' | 'src-only';
+export type { TableFormat };
+
+export interface TurndownOptions {
+  readonly baseUrl?: string;
+  readonly codeBlockStyle?: CodeBlockStyle;
+  readonly gfm?: boolean;
+  readonly headingStyle?: HeadingStyle;
+  readonly images?: ImageMode;
+  readonly tables?: TableFormat;
+}
+
+export function toMarkdown(
+  html: string,
+  options?: Readonly<TurndownOptions>,
+): string {
+  const service = new TurndownService({
+    bulletListMarker: '-',
+    // Code renders inline in the Markdown as fenced blocks; normalize rewrites
+    // source conventions to `<code class="language-X">` first, so the fence
+    // carries a language tag. A code-only extraction tool was rejected as
+    // redundant surface for a niche the main path already covers.
+    codeBlockStyle: options?.codeBlockStyle ?? 'fenced',
+    emDelimiter: '_',
+    fence: '```',
+    headingStyle: options?.headingStyle ?? 'atx',
+    strongDelimiter: '**',
+  });
+
+  if (options?.gfm !== false) {
+    service.use(gfm);
+  }
+
+  const tableFormat = options?.tables;
+  if (tableFormat !== undefined) {
+    // addRule prepends, so this shadows the gfm plugin's table rule AND its
+    // headerless-table keep filter (which lives in a separate _keep list, only
+    // consulted when no rule in the array matches). Matching every TABLE here
+    // means the keep list is never reached for tables.
+    service.addRule('tableMatrix', {
+      filter: node => node.nodeName === 'TABLE',
+      replacement: (_content, node) => {
+        const matrix = parseTableMatrix(node);
+        if (matrix.length === 0) {
+          return '';
+        }
+        const body = renderTable(matrix, tableFormat);
+        // GFM rows are already markdown table syntax; csv/json need a fenced block.
+        if (tableFormat === 'gfm') {
+          return `\n\n${body}\n\n`;
+        }
+        return `\n\n\`\`\`${tableFormat}\n${body}\n\`\`\`\n\n`;
+      },
+    });
+  }
+
+  const imageMode = options?.images;
+  const baseUrl = options?.baseUrl;
+  // `reference` mode collects definitions as images render; append them after the pass.
+  const references: string[] = [];
+  applyImagePolicy(service, imageMode, baseUrl, references);
+  // Readability pre-absolutizes anchors on the main path, but html_to_markdown
+  // and the extract fallback bypass it; turndown is shared, so absolutize here
+  // for every path. Empty/absent href falls through to bare content to match
+  // turndown's default link rule (which never matches when href is falsy).
+  service.addRule('anchorAbsolutize', {
+    filter: 'a',
+    replacement: (content, node) => {
+      const rawHref = node.getAttribute('href');
+      if (!rawHref) {
+        return content;
+      }
+      const href = absolutize(rawHref, baseUrl);
+      const title = node.getAttribute('title');
+      const titlePart = title ? ` "${title.replace(/"/g, '\\"')}"` : '';
+      return `[${content}](${href}${titlePart})`;
+    },
+  });
+  // Math markers are emitted by `pipeline/math.ts` and survive Readability via the
+  // `rdrm-math` preserve entry. A rule's replacement is inserted verbatim, so
+  // the raw LaTeX backslashes survive turndown untouched.
+  service.addRule('mathMarker', {
+    filter: node =>
+      node.nodeName === 'SPAN' && node.classList.contains('rdrm-math'),
+    replacement: (_content, node) => {
+      const tex = node.textContent.trim();
+      if (!tex) {
+        return '';
+      }
+      const display = node.getAttribute('data-display') === 'true';
+      return display ? `$$${tex}$$` : `$${tex}$`;
+    },
+  });
+
+  const fnResult = processFootnotes(html);
+  const sourceHtml = fnResult?.html ?? html;
+  let body = service.turndown(sourceHtml);
+
+  if (fnResult) {
+    // Turndown sees `[^N]` as a shortcut-reference link and backslash-escapes
+    // the brackets; reverse that for the markers we emitted so they render as
+    // footnote refs. Code blocks pass through verbatim and never carry the
+    // escape, so they are unaffected.
+    for (let n = 1; n <= fnResult.footnoteDefs.length; n++) {
+      body = body.replaceAll(`\\[^${n}\\]`, `[^${n}]`);
+    }
+  }
+
+  const trailingBlocks: string[] = [];
+  if (imageMode === 'reference' && references.length > 0) {
+    trailingBlocks.push(
+      references.map((ref, i) => `[img-${i + 1}]: ${ref}`).join('\n'),
+    );
+  }
+  if (fnResult && fnResult.footnoteDefs.length > 0) {
+    trailingBlocks.push(
+      fnResult.footnoteDefs.map((def, i) => `[^${i + 1}]: ${def}`).join('\n'),
+    );
+  }
+  if (trailingBlocks.length === 0) {
+    return body;
+  }
+  return `${body.replace(/\n+$/, '')}\n\n${trailingBlocks.join('\n')}`;
+}
+
+function applyImagePolicy(
+  service: TurndownService,
+  mode: ImageMode | undefined,
+  baseUrl: string | undefined,
+  references: string[],
+): void {
+  switch (mode) {
+    case 'drop': {
+      // `.remove('img')` doesn't reliably drop void elements in turndown 7.x.
+      service.addRule('dropImage', {
+        filter: 'img',
+        replacement: () => '',
+      });
+      break;
+    }
+    case 'keep':
+    case undefined: {
+      service.addRule('imageKeep', {
+        filter: 'img',
+        replacement: (_content, node) => {
+          const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+          const alt = node.getAttribute('alt') ?? '';
+          return src ? `![${alt}](${src})` : '';
+        },
+      });
+      break;
+    }
+    case 'reference': {
+      service.addRule('imageReference', {
+        filter: 'img',
+        replacement: (_content, node) => {
+          const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+          if (!src) {
+            return '';
+          }
+          references.push(src);
+          const id = references.length;
+          const alt = node.getAttribute('alt') ?? '';
+          return `![${alt}][img-${id}]`;
+        },
+      });
+      break;
+    }
+    case 'src-only': {
+      service.addRule('imageSrcOnly', {
+        filter: 'img',
+        replacement: (_content, node) => {
+          const src = absolutize(node.getAttribute('src') ?? '', baseUrl);
+          return src ? `\n\n${src}\n\n` : '';
+        },
+      });
+      break;
+    }
+  }
+}

package/src/pipeline/urls.js

@@ -0,0 +1,14 @@
+// Shared by the turndown rules (image/anchor absolutization) and the
+// extract_links tool. Returns the input unchanged when either side is missing
+// or the URL constructor rejects the pair, so callers never throw on bad input.
+export function absolutize(src, baseUrl) {
+    if (!src || !baseUrl) {
+        return src;
+    }
+    try {
+        return new URL(src, baseUrl).href;
+    }
+    catch {
+        return src;
+    }
+}

package/src/pipeline/urls.ts

@@ -0,0 +1,13 @@
+// Shared by the turndown rules (image/anchor absolutization) and the
+// extract_links tool. Returns the input unchanged when either side is missing
+// or the URL constructor rejects the pair, so callers never throw on bad input.
+export function absolutize(src: string, baseUrl: string | undefined): string {
+  if (!src || !baseUrl) {
+    return src;
+  }
+  try {
+    return new URL(src, baseUrl).href;
+  } catch {
+    return src;
+  }
+}