@v1nvn/readability 0.43.0 → 0.45.0

package/src/tools/extract_metadata.ts

@@ -0,0 +1,102 @@
+import type { Metadata } from '../pipeline/context.ts';
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { normalizeDocument } from '../pipeline/normalize.ts';
+import { resolveMetadata } from '../policy/metadata.ts';
+import { fromLocalPath } from './html-source.ts';
+import { extractMetadataOutputShape } from './output-schema.ts';
+import {
+  type ExtractMetadataFromHtmlInput,
+  extractMetadataInputSchema,
+  extractMetadataInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+// Bibliographic fields only — wordCount/readingTimeMin/tokenEstimate are
+// meaningless without the extracted body, so a metadata-only caller never
+// sees wordCount: 0.
+const BIBLIOGRAPHIC_KEYS = [
+  'title',
+  'byline',
+  'siteName',
+  'lang',
+  'publishedTime',
+  'excerpt',
+  'canonical',
+  'baseUrl',
+] as const satisfies readonly (keyof Metadata)[];
+
+function pickBibliographic(
+  metadata: Readonly<Metadata>,
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const key of BIBLIOGRAPHIC_KEYS) {
+    const value = metadata[key];
+    if (value !== undefined) {
+      out[key] = value;
+    }
+  }
+  return out;
+}
+
+// One line per present field; never blank so content[0].text is always scannable.
+function renderMetadataLines(
+  metadata: Readonly<Record<string, string>>,
+): string {
+  const lines = Object.entries(metadata).map(
+    ([key, value]) => `${key}: ${value}`,
+  );
+  return lines.length > 0 ? lines.join('\n') : '(no metadata found)';
+}
+
+export function extractMetadataDocumentFromHtml(
+  input: Readonly<ExtractMetadataFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl } = input;
+
+  const { document } = buildDocument(html, baseUrl);
+  normalizeDocument(document);
+  const resolved = resolveMetadata({
+    document,
+    textContent: '',
+    wordCount: 0,
+    readingTimeMin: 0,
+    baseUrl,
+  });
+
+  const metadata = pickBibliographic(resolved);
+  const content = renderMetadataLines(metadata);
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      metadata,
+    },
+  };
+}
+
+export const EXTRACT_METADATA_TOOL_DESCRIPTION = `Return only the bibliographic metadata (title, byline, siteName, lang, publishedTime, excerpt, canonical, baseUrl) of already-rendered (post-JavaScript) HTML without running Readability/Turndown — a fast pre-check for crawlers and citation. Resolves the same metadata cascade as \`extract\` (JSON-LD → OpenGraph → Twitter → <meta> → <time> → <title>), plus <link rel="canonical"> → og:url. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
+
+export const extractMetadataHandler = tool(
+  'extract_metadata',
+  input => extractMetadataDocumentFromHtml(fromLocalPath(input)),
+  extractMetadataInputSchema,
+);
+
+export function registerExtractMetadataTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'extract_metadata',
+    {
+      title: 'Extract metadata only (no Readability)',
+      description: EXTRACT_METADATA_TOOL_DESCRIPTION,
+      inputSchema: extractMetadataInputShape,
+      outputSchema: extractMetadataOutputShape,
+    },
+    extractMetadataHandler,
+  );
+}

package/src/tools/extract_section.ts

@@ -0,0 +1,84 @@
+import type { ToolHandle } from '../server.ts';
+
+import { ExtractionError } from '../errors.ts';
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { normalizeDocument } from '../pipeline/normalize.ts';
+import { scopeToHeading } from '../policy/section.ts';
+import { extractArticleFromHtml } from './extract.ts';
+import { fromLocalPath } from './html-source.ts';
+import { outputSchemaShape } from './output-schema.ts';
+import {
+  type ExtractSectionFromHtmlInput,
+  extractSectionInputSchema,
+  extractSectionInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+const SECTION_SCOPE_SELECTOR = '[data-rdrm-section-scope]';
+
+export function extractSectionFromHtml(
+  input: Readonly<ExtractSectionFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, selector, heading } = input;
+
+  if (selector !== undefined) {
+    return extractArticleFromHtml({
+      html,
+      baseUrl,
+      selectors: { include: selector },
+      resolvePreset: false,
+    });
+  }
+  // The superRefine on the schema enforces selector/heading XOR, so reaching
+  // here means heading is set — but its declared type is still optional, so
+  // narrow explicitly rather than asserting.
+  if (heading === undefined) {
+    throw new ExtractionError(
+      'Provide exactly one of `selector` or `heading`.',
+    );
+  }
+
+  // Heading mode: wrap the matched subtree, re-serialize, and route through
+  // extractArticleFromHtml's selectors.include so this tool stays a thin
+  // resolver over the existing extraction pipeline — no forked extraction logic.
+  // The DOM is parsed+normalized twice: here to scope against a normalized
+  // DOM, and again inside the extract worker — which owns detectGating-before-
+  // normalize, resolveLazyImages, and applySelectors. Folding scoping into
+  // that pipeline would breach its invariants for a non-hot path; accepted.
+  const { document } = buildDocument(html, baseUrl);
+  normalizeDocument(document);
+  if (!scopeToHeading(document, heading)) {
+    throw new ExtractionError(`no heading matched: ${heading}`);
+  }
+  const scoped = `<!DOCTYPE html><html><head></head><body>${document.body.innerHTML}</body></html>`;
+  return extractArticleFromHtml({
+    html: scoped,
+    baseUrl,
+    selectors: { include: SECTION_SCOPE_SELECTOR },
+    resolvePreset: false,
+  });
+}
+
+export const EXTRACT_SECTION_TOOL_DESCRIPTION = `Extract one section of an already-rendered (post-JavaScript) HTML document and return its Markdown + metadata + diagnostics — a thin resolver over extract’s \`selectors.include\` path, not a new extractor. Pick the section by CSS \`selector\` (passed straight through) OR by \`heading\` text (case-insensitive, first match wins; the section spans from the matched heading to the next same-or-higher-level heading). Exactly one of \`selector\`/\`heading\` is required. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
+
+export const extractSectionHandler = tool(
+  'extract_section',
+  input => extractSectionFromHtml(fromLocalPath(input)),
+  extractSectionInputSchema,
+);
+
+export function registerExtractSectionTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'extract_section',
+    {
+      title: 'Extract one section by selector or heading',
+      description: EXTRACT_SECTION_TOOL_DESCRIPTION,
+      inputSchema: extractSectionInputShape,
+      outputSchema: outputSchemaShape,
+    },
+    extractSectionHandler,
+  );
+}

package/src/tools/extract_tables.ts

@@ -0,0 +1,101 @@
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { applySelectors } from '../pipeline/normalize.ts';
+import {
+  parseTableMatrix,
+  renderTable,
+  resolveHeaderKeys,
+} from '../pipeline/tables.ts';
+import { fromLocalPath, withSchemaDefaults } from './html-source.ts';
+import { extractTablesOutputShape } from './output-schema.ts';
+import {
+  type ExtractTablesFromHtmlInput,
+  extractTablesInputSchema,
+  extractTablesInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+interface ExtractedTable {
+  readonly cols: number;
+  readonly index: number;
+  readonly markdown: string;
+  readonly rows: number;
+}
+
+const NO_TABLES = '(no tables found)';
+
+export function extractTables(rawArgs: unknown): CallToolResult {
+  return extractTablesFromHtml(
+    fromLocalPath(extractTablesInputSchema.parse(rawArgs)),
+  );
+}
+
+export function extractTablesFromHtml(
+  input: Readonly<ExtractTablesFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, format, selectors } = withSchemaDefaults(
+    extractTablesInputSchema,
+    input,
+  );
+
+  // Skip normalizeDocument/Readability on purpose: this tool exists to reach
+  // tables that live outside the scored article (nav, aside, boilerplate), which
+  // those stages would discard. Table structure is static HTML, so the matrix
+  // walk is unaffected by unsanitized scripts/styles.
+  const { document } = buildDocument(html, baseUrl);
+  applySelectors(document, selectors);
+
+  const tables: ExtractedTable[] = [];
+  let index = 0;
+  for (const table of document.querySelectorAll('table')) {
+    const matrix = parseTableMatrix(table);
+    if (matrix.length === 0) {
+      continue;
+    }
+    const keys =
+      format === 'json' ? resolveHeaderKeys(table, matrix) : undefined;
+    const markdown = renderTable(matrix, format, keys);
+    tables.push({
+      index,
+      rows: matrix.length,
+      cols: matrix[0]?.length ?? 0,
+      markdown,
+    });
+    index++;
+  }
+
+  const content =
+    tables.length > 0
+      ? tables.map(entry => entry.markdown).join('\n\n')
+      : NO_TABLES;
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      tables,
+      metadata: { baseUrl, format, tableCount: tables.length },
+    },
+  };
+}
+
+export const EXTRACT_TABLES_TOOL_DESCRIPTION = `Extract every <table> on the page from already-rendered (post-JavaScript) HTML and return each as GFM / CSV / JSON (caller picks). Runs no Readability, Turndown, or sanitization — a page-wide \`querySelectorAll('table')\` walk in front of the same rowspan/colspan-aware matrix serializer used by the \`tables\` option on \`extract\`. Captures tables outside the article body (nav, aside, boilerplate) that the \`tables\` option never sees. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
+
+export const extractTablesHandler = tool('extract_tables', extractTables);
+
+export function registerExtractTablesTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'extract_tables',
+    {
+      title: 'Extract every table on the page',
+      description: EXTRACT_TABLES_TOOL_DESCRIPTION,
+      inputSchema: extractTablesInputShape,
+      outputSchema: extractTablesOutputShape,
+    },
+    extractTablesHandler,
+  );
+}

package/src/tools/html-source.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from 'node:fs';
+
+import type { FromHtmlInput } from './schemas.ts';
+
+export function readHtmlFile(localPath: string): string {
+  const raw = readFileSync(localPath, 'utf8');
+  const trimmed = raw.trim();
+  // A top-level JSON string scalar is a chrome-devtools capture (its JSON-encoded
+  // return value written to disk), not raw markup — unwrap it before parsing.
+  if (trimmed.length < 2 || trimmed.at(0) !== '"' || trimmed.at(-1) !== '"') {
+    return raw;
+  }
+  try {
+    const parsed: unknown = JSON.parse(trimmed);
+    return typeof parsed === 'string' ? parsed : raw;
+  } catch {
+    return raw;
+  }
+}
+
+export function fromLocalPath<T extends { localPath: string }>(
+  input: T,
+): FromHtmlInput<T> {
+  const { localPath, ...rest } = input;
+  return { ...rest, html: readHtmlFile(localPath) };
+}
+
+const schemaDefaults = new WeakMap<object, unknown>();
+
+export function withSchemaDefaults<
+  T extends { localPath: string },
+  I extends FromHtmlInput<T>,
+>(schema: { parse(raw: unknown): T }, input: I): I & T {
+  let defaults = schemaDefaults.get(schema);
+  if (defaults === undefined) {
+    defaults = schema.parse({ localPath: '' });
+    schemaDefaults.set(schema, defaults);
+  }
+  return { ...(defaults as T), ...input };
+}

package/src/tools/html_to_markdown.ts

@@ -0,0 +1,180 @@
+import type { SanitizationDiagnostics } from '../pipeline/context.ts';
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { formatPayload } from '../output/format.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import {
+  applySelectors,
+  normalizeDocument,
+  resolveLazyImages,
+} from '../pipeline/normalize.ts';
+import { sanitizeHtml } from '../pipeline/sanitize.ts';
+import { toMarkdown } from '../pipeline/turndown.ts';
+import { assembleDiagnostics, TraceCollector } from '../policy/diagnostics.ts';
+import { computeTextMetrics, nonEmpty } from '../policy/text.ts';
+import { truncateMarkdown } from '../policy/truncate.ts';
+import { fromLocalPath, withSchemaDefaults } from './html-source.ts';
+import { outputSchemaShape } from './output-schema.ts';
+import {
+  type HtmlToMarkdownFromHtmlInput,
+  htmlToMarkdownInputSchema,
+  htmlToMarkdownInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+const EXTRACTED_NODE = 'fragment';
+
+export function htmlToMarkdownFromHtml(
+  input: Readonly<HtmlToMarkdownFromHtmlInput>,
+): CallToolResult {
+  const {
+    html,
+    baseUrl,
+    selectors,
+    format,
+    metadataMode,
+    gfm,
+    headingStyle,
+    codeBlockStyle,
+    images,
+    sanitize: shouldSanitize,
+    maxChars,
+    wordsPerMinute,
+    cleanChrome,
+    tables,
+    debug,
+  } = withSchemaDefaults(htmlToMarkdownInputSchema, input);
+
+  const trace = new TraceCollector(debug);
+
+  const { document, window } = buildDocument(html, baseUrl);
+
+  const {
+    documentElementCount,
+    normalizeCounts,
+    imagesResolved,
+    textContent,
+    rawHtml,
+  } = trace.run('normalize', () => {
+    const documentElementCount = document.querySelectorAll('*').length;
+    const normalizeCounts = normalizeDocument(document, { cleanChrome });
+    const imagesResolved = resolveLazyImages(document);
+    applySelectors(document, selectors);
+    const body = document.body;
+    return {
+      documentElementCount,
+      imagesResolved,
+      normalizeCounts,
+      rawHtml: body.innerHTML,
+      textContent: body.textContent,
+    };
+  });
+
+  const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run(
+    'sanitize',
+    (): { counts: SanitizationDiagnostics; html: string } => {
+      if (!shouldSanitize) {
+        return { counts: { iframes: 0, scripts: 0 }, html: rawHtml };
+      }
+      const res = sanitizeHtml(rawHtml, window);
+      return {
+        counts: { iframes: res.iframesRemoved, scripts: res.scriptsRemoved },
+        html: res.html,
+      };
+    },
+  );
+  const markdown = trace.run('turndown', () =>
+    toMarkdown(sanitizedHtml, {
+      codeBlockStyle,
+      gfm,
+      headingStyle,
+      images,
+      tables,
+      baseUrl,
+    }),
+  );
+
+  const { metadata } = trace.run('metadata', () => {
+    const firstHeading = nonEmpty(
+      document.body.querySelector('h1, h2, h3, h4, h5, h6')?.textContent,
+    );
+    const metadata = {
+      title: firstHeading,
+      baseUrl,
+      ...computeTextMetrics(textContent, wordsPerMinute),
+    };
+    return { metadata };
+  });
+
+  const sanitization: SanitizationDiagnostics = {
+    iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
+    scripts: normalizeCounts.scripts + sanitizeCounts.scripts,
+  };
+  const baseDiagnostics = assembleDiagnostics({
+    articleHtml: sanitizedHtml,
+    boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+    chromeRemoved: normalizeCounts.chromeRemoved,
+    documentElementCount,
+    extractedNode: EXTRACTED_NODE,
+    fallbackUsed: true,
+    imagesResolved,
+    sanitization,
+    trace: trace.collect(),
+    truncated: false,
+    window,
+  });
+
+  let payload = formatPayload({
+    diagnostics: baseDiagnostics,
+    format,
+    markdown,
+    metadata,
+    metadataMode,
+    sanitizedHtml,
+    textContent,
+  });
+
+  let truncated = false;
+  if (maxChars !== undefined && (format === 'markdown' || format === 'text')) {
+    const res = truncateMarkdown(payload, maxChars);
+    payload = res.text;
+    truncated = res.truncated;
+  }
+  const diagnostics = truncated
+    ? { ...baseDiagnostics, truncated }
+    : baseDiagnostics;
+
+  return {
+    content: [{ text: payload, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content: payload,
+      metadata,
+      diagnostics,
+    },
+  };
+}
+
+export const HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
+
+export const htmlToMarkdownHandler = tool(
+  'html_to_markdown',
+  input => htmlToMarkdownFromHtml(fromLocalPath(input)),
+  htmlToMarkdownInputSchema,
+);
+
+export function registerHtmlToMarkdownTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'html_to_markdown',
+    {
+      title: 'Convert HTML fragment to Markdown',
+      description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
+      inputSchema: htmlToMarkdownInputShape,
+      outputSchema: outputSchemaShape,
+    },
+    htmlToMarkdownHandler,
+  );
+}

package/src/tools/outline.ts

@@ -0,0 +1,77 @@
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { applySelectors, normalizeDocument } from '../pipeline/normalize.ts';
+import { resolveOutline } from '../policy/outline.ts';
+import { fromLocalPath } from './html-source.ts';
+import { outlineOutputShape } from './output-schema.ts';
+import {
+  type OutlineFromHtmlInput,
+  outlineInputSchema,
+  outlineInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+// One line per heading, nested by depth; never blank so content[0].text is always scannable.
+function renderOutlineToc(
+  outline: readonly { level: number; text: string }[],
+): string {
+  if (outline.length === 0) {
+    return '(no headings found)';
+  }
+  return outline
+    .map(entry => `${'  '.repeat(entry.level - 1)}- ${entry.text}`)
+    .join('\n');
+}
+
+export function outlineDocumentFromHtml(
+  input: Readonly<OutlineFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, selectors } = input;
+
+  const { document } = buildDocument(html, baseUrl);
+  normalizeDocument(document);
+  applySelectors(document, selectors);
+  const outline = resolveOutline(document);
+
+  const title =
+    document.title.trim() ||
+    document.querySelector('h1')?.textContent.replace(/\s+/g, ' ').trim() ||
+    undefined;
+  const metadata = { title, baseUrl };
+
+  const content = renderOutlineToc(outline);
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      outline,
+      metadata,
+    },
+  };
+}
+
+export const OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
+
+export const outlineHandler = tool(
+  'outline',
+  input => outlineDocumentFromHtml(fromLocalPath(input)),
+  outlineInputSchema,
+);
+
+export function registerOutlineTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'outline',
+    {
+      title: 'Get document outline (heading TOC)',
+      description: OUTLINE_TOOL_DESCRIPTION,
+      inputSchema: outlineInputShape,
+      outputSchema: outlineOutputShape,
+    },
+    outlineHandler,
+  );
+}