@v1nvn/readability 0.44.0 → 0.45.1

package/src/tools/extract_tables.ts

@@ -0,0 +1,101 @@
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { applySelectors } from '../pipeline/normalize.ts';
+import {
+  parseTableMatrix,
+  renderTable,
+  resolveHeaderKeys,
+} from '../pipeline/tables.ts';
+import { fromLocalPath, withSchemaDefaults } from './html-source.ts';
+import { extractTablesOutputShape } from './output-schema.ts';
+import {
+  type ExtractTablesFromHtmlInput,
+  extractTablesInputSchema,
+  extractTablesInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+interface ExtractedTable {
+  readonly cols: number;
+  readonly index: number;
+  readonly markdown: string;
+  readonly rows: number;
+}
+
+const NO_TABLES = '(no tables found)';
+
+export function extractTables(rawArgs: unknown): CallToolResult {
+  return extractTablesFromHtml(
+    fromLocalPath(extractTablesInputSchema.parse(rawArgs)),
+  );
+}
+
+export function extractTablesFromHtml(
+  input: Readonly<ExtractTablesFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, format, selectors } = withSchemaDefaults(
+    extractTablesInputSchema,
+    input,
+  );
+
+  // Skip normalizeDocument/Readability on purpose: this tool exists to reach
+  // tables that live outside the scored article (nav, aside, boilerplate), which
+  // those stages would discard. Table structure is static HTML, so the matrix
+  // walk is unaffected by unsanitized scripts/styles.
+  const { document } = buildDocument(html, baseUrl);
+  applySelectors(document, selectors);
+
+  const tables: ExtractedTable[] = [];
+  let index = 0;
+  for (const table of document.querySelectorAll('table')) {
+    const matrix = parseTableMatrix(table);
+    if (matrix.length === 0) {
+      continue;
+    }
+    const keys =
+      format === 'json' ? resolveHeaderKeys(table, matrix) : undefined;
+    const markdown = renderTable(matrix, format, keys);
+    tables.push({
+      index,
+      rows: matrix.length,
+      cols: matrix[0]?.length ?? 0,
+      markdown,
+    });
+    index++;
+  }
+
+  const content =
+    tables.length > 0
+      ? tables.map(entry => entry.markdown).join('\n\n')
+      : NO_TABLES;
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      tables,
+      metadata: { baseUrl, format, tableCount: tables.length },
+    },
+  };
+}
+
+export const EXTRACT_TABLES_TOOL_DESCRIPTION = `Extract every <table> on the page from already-rendered (post-JavaScript) HTML and return each as GFM / CSV / JSON (caller picks). Runs no Readability, Turndown, or sanitization — a page-wide \`querySelectorAll('table')\` walk in front of the same rowspan/colspan-aware matrix serializer used by the \`tables\` option on \`extract\`. Captures tables outside the article body (nav, aside, boilerplate) that the \`tables\` option never sees. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only (never fetched).`;
+
+export const extractTablesHandler = tool('extract_tables', extractTables);
+
+export function registerExtractTablesTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'extract_tables',
+    {
+      title: 'Extract every table on the page',
+      description: EXTRACT_TABLES_TOOL_DESCRIPTION,
+      inputSchema: extractTablesInputShape,
+      outputSchema: extractTablesOutputShape,
+    },
+    extractTablesHandler,
+  );
+}

package/src/tools/html-source.js

@@ -0,0 +1,30 @@
+import { readFileSync } from 'node:fs';
+export function readHtmlFile(localPath) {
+    const raw = readFileSync(localPath, 'utf8');
+    const trimmed = raw.trim();
+    // A top-level JSON string scalar is a chrome-devtools capture (its JSON-encoded
+    // return value written to disk), not raw markup — unwrap it before parsing.
+    if (trimmed.length < 2 || trimmed.at(0) !== '"' || trimmed.at(-1) !== '"') {
+        return raw;
+    }
+    try {
+        const parsed = JSON.parse(trimmed);
+        return typeof parsed === 'string' ? parsed : raw;
+    }
+    catch {
+        return raw;
+    }
+}
+export function fromLocalPath(input) {
+    const { localPath, ...rest } = input;
+    return { ...rest, html: readHtmlFile(localPath) };
+}
+const schemaDefaults = new WeakMap();
+export function withSchemaDefaults(schema, input) {
+    let defaults = schemaDefaults.get(schema);
+    if (defaults === undefined) {
+        defaults = schema.parse({ localPath: '' });
+        schemaDefaults.set(schema, defaults);
+    }
+    return { ...defaults, ...input };
+}

package/src/tools/html-source.ts

@@ -0,0 +1,40 @@
+import { readFileSync } from 'node:fs';
+
+import type { FromHtmlInput } from './schemas.ts';
+
+export function readHtmlFile(localPath: string): string {
+  const raw = readFileSync(localPath, 'utf8');
+  const trimmed = raw.trim();
+  // A top-level JSON string scalar is a chrome-devtools capture (its JSON-encoded
+  // return value written to disk), not raw markup — unwrap it before parsing.
+  if (trimmed.length < 2 || trimmed.at(0) !== '"' || trimmed.at(-1) !== '"') {
+    return raw;
+  }
+  try {
+    const parsed: unknown = JSON.parse(trimmed);
+    return typeof parsed === 'string' ? parsed : raw;
+  } catch {
+    return raw;
+  }
+}
+
+export function fromLocalPath<T extends { localPath: string }>(
+  input: T,
+): FromHtmlInput<T> {
+  const { localPath, ...rest } = input;
+  return { ...rest, html: readHtmlFile(localPath) };
+}
+
+const schemaDefaults = new WeakMap<object, unknown>();
+
+export function withSchemaDefaults<
+  T extends { localPath: string },
+  I extends FromHtmlInput<T>,
+>(schema: { parse(raw: unknown): T }, input: I): I & T {
+  let defaults = schemaDefaults.get(schema);
+  if (defaults === undefined) {
+    defaults = schema.parse({ localPath: '' });
+    schemaDefaults.set(schema, defaults);
+  }
+  return { ...(defaults as T), ...input };
+}

package/src/tools/html_to_markdown.js

@@ -0,0 +1,113 @@
+import { tool } from "../handler.js";
+import { formatPayload } from "../output/format.js";
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument, resolveLazyImages, } from "../pipeline/normalize.js";
+import { sanitizeHtml } from "../pipeline/sanitize.js";
+import { toMarkdown } from "../pipeline/turndown.js";
+import { assembleDiagnostics, TraceCollector } from "../policy/diagnostics.js";
+import { computeTextMetrics, nonEmpty } from "../policy/text.js";
+import { truncateMarkdown } from "../policy/truncate.js";
+import { fromLocalPath, withSchemaDefaults } from "./html-source.js";
+import { outputSchemaShape } from "./output-schema.js";
+import { htmlToMarkdownInputSchema, htmlToMarkdownInputShape, } from "./schemas.js";
+const EXTRACTED_NODE = 'fragment';
+export function htmlToMarkdownFromHtml(input) {
+    const { html, baseUrl, selectors, format, metadataMode, gfm, headingStyle, codeBlockStyle, images, sanitize: shouldSanitize, maxChars, wordsPerMinute, cleanChrome, tables, debug, } = withSchemaDefaults(htmlToMarkdownInputSchema, input);
+    const trace = new TraceCollector(debug);
+    const { document, window } = buildDocument(html, baseUrl);
+    const { documentElementCount, normalizeCounts, imagesResolved, textContent, rawHtml, } = trace.run('normalize', () => {
+        const documentElementCount = document.querySelectorAll('*').length;
+        const normalizeCounts = normalizeDocument(document, { cleanChrome });
+        const imagesResolved = resolveLazyImages(document);
+        applySelectors(document, selectors);
+        const body = document.body;
+        return {
+            documentElementCount,
+            imagesResolved,
+            normalizeCounts,
+            rawHtml: body.innerHTML,
+            textContent: body.textContent,
+        };
+    });
+    const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run('sanitize', () => {
+        if (!shouldSanitize) {
+            return { counts: { iframes: 0, scripts: 0 }, html: rawHtml };
+        }
+        const res = sanitizeHtml(rawHtml, window);
+        return {
+            counts: { iframes: res.iframesRemoved, scripts: res.scriptsRemoved },
+            html: res.html,
+        };
+    });
+    const markdown = trace.run('turndown', () => toMarkdown(sanitizedHtml, {
+        codeBlockStyle,
+        gfm,
+        headingStyle,
+        images,
+        tables,
+        baseUrl,
+    }));
+    const { metadata } = trace.run('metadata', () => {
+        const firstHeading = nonEmpty(document.body.querySelector('h1, h2, h3, h4, h5, h6')?.textContent);
+        const metadata = {
+            title: firstHeading,
+            baseUrl,
+            ...computeTextMetrics(textContent, wordsPerMinute),
+        };
+        return { metadata };
+    });
+    const sanitization = {
+        iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
+        scripts: normalizeCounts.scripts + sanitizeCounts.scripts,
+    };
+    const baseDiagnostics = assembleDiagnostics({
+        articleHtml: sanitizedHtml,
+        boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+        chromeRemoved: normalizeCounts.chromeRemoved,
+        documentElementCount,
+        extractedNode: EXTRACTED_NODE,
+        fallbackUsed: true,
+        imagesResolved,
+        sanitization,
+        trace: trace.collect(),
+        truncated: false,
+        window,
+    });
+    let payload = formatPayload({
+        diagnostics: baseDiagnostics,
+        format,
+        markdown,
+        metadata,
+        metadataMode,
+        sanitizedHtml,
+        textContent,
+    });
+    let truncated = false;
+    if (maxChars !== undefined && (format === 'markdown' || format === 'text')) {
+        const res = truncateMarkdown(payload, maxChars);
+        payload = res.text;
+        truncated = res.truncated;
+    }
+    const diagnostics = truncated
+        ? { ...baseDiagnostics, truncated }
+        : baseDiagnostics;
+    return {
+        content: [{ text: payload, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content: payload,
+            metadata,
+            diagnostics,
+        },
+    };
+}
+export const HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
+export const htmlToMarkdownHandler = tool('html_to_markdown', input => htmlToMarkdownFromHtml(fromLocalPath(input)), htmlToMarkdownInputSchema);
+export function registerHtmlToMarkdownTool(server) {
+    return server.registerTool('html_to_markdown', {
+        title: 'Convert HTML fragment to Markdown',
+        description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
+        inputSchema: htmlToMarkdownInputShape,
+        outputSchema: outputSchemaShape,
+    }, htmlToMarkdownHandler);
+}

package/src/tools/html_to_markdown.ts

@@ -0,0 +1,180 @@
+import type { SanitizationDiagnostics } from '../pipeline/context.ts';
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { formatPayload } from '../output/format.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import {
+  applySelectors,
+  normalizeDocument,
+  resolveLazyImages,
+} from '../pipeline/normalize.ts';
+import { sanitizeHtml } from '../pipeline/sanitize.ts';
+import { toMarkdown } from '../pipeline/turndown.ts';
+import { assembleDiagnostics, TraceCollector } from '../policy/diagnostics.ts';
+import { computeTextMetrics, nonEmpty } from '../policy/text.ts';
+import { truncateMarkdown } from '../policy/truncate.ts';
+import { fromLocalPath, withSchemaDefaults } from './html-source.ts';
+import { outputSchemaShape } from './output-schema.ts';
+import {
+  type HtmlToMarkdownFromHtmlInput,
+  htmlToMarkdownInputSchema,
+  htmlToMarkdownInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+const EXTRACTED_NODE = 'fragment';
+
+export function htmlToMarkdownFromHtml(
+  input: Readonly<HtmlToMarkdownFromHtmlInput>,
+): CallToolResult {
+  const {
+    html,
+    baseUrl,
+    selectors,
+    format,
+    metadataMode,
+    gfm,
+    headingStyle,
+    codeBlockStyle,
+    images,
+    sanitize: shouldSanitize,
+    maxChars,
+    wordsPerMinute,
+    cleanChrome,
+    tables,
+    debug,
+  } = withSchemaDefaults(htmlToMarkdownInputSchema, input);
+
+  const trace = new TraceCollector(debug);
+
+  const { document, window } = buildDocument(html, baseUrl);
+
+  const {
+    documentElementCount,
+    normalizeCounts,
+    imagesResolved,
+    textContent,
+    rawHtml,
+  } = trace.run('normalize', () => {
+    const documentElementCount = document.querySelectorAll('*').length;
+    const normalizeCounts = normalizeDocument(document, { cleanChrome });
+    const imagesResolved = resolveLazyImages(document);
+    applySelectors(document, selectors);
+    const body = document.body;
+    return {
+      documentElementCount,
+      imagesResolved,
+      normalizeCounts,
+      rawHtml: body.innerHTML,
+      textContent: body.textContent,
+    };
+  });
+
+  const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run(
+    'sanitize',
+    (): { counts: SanitizationDiagnostics; html: string } => {
+      if (!shouldSanitize) {
+        return { counts: { iframes: 0, scripts: 0 }, html: rawHtml };
+      }
+      const res = sanitizeHtml(rawHtml, window);
+      return {
+        counts: { iframes: res.iframesRemoved, scripts: res.scriptsRemoved },
+        html: res.html,
+      };
+    },
+  );
+  const markdown = trace.run('turndown', () =>
+    toMarkdown(sanitizedHtml, {
+      codeBlockStyle,
+      gfm,
+      headingStyle,
+      images,
+      tables,
+      baseUrl,
+    }),
+  );
+
+  const { metadata } = trace.run('metadata', () => {
+    const firstHeading = nonEmpty(
+      document.body.querySelector('h1, h2, h3, h4, h5, h6')?.textContent,
+    );
+    const metadata = {
+      title: firstHeading,
+      baseUrl,
+      ...computeTextMetrics(textContent, wordsPerMinute),
+    };
+    return { metadata };
+  });
+
+  const sanitization: SanitizationDiagnostics = {
+    iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
+    scripts: normalizeCounts.scripts + sanitizeCounts.scripts,
+  };
+  const baseDiagnostics = assembleDiagnostics({
+    articleHtml: sanitizedHtml,
+    boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+    chromeRemoved: normalizeCounts.chromeRemoved,
+    documentElementCount,
+    extractedNode: EXTRACTED_NODE,
+    fallbackUsed: true,
+    imagesResolved,
+    sanitization,
+    trace: trace.collect(),
+    truncated: false,
+    window,
+  });
+
+  let payload = formatPayload({
+    diagnostics: baseDiagnostics,
+    format,
+    markdown,
+    metadata,
+    metadataMode,
+    sanitizedHtml,
+    textContent,
+  });
+
+  let truncated = false;
+  if (maxChars !== undefined && (format === 'markdown' || format === 'text')) {
+    const res = truncateMarkdown(payload, maxChars);
+    payload = res.text;
+    truncated = res.truncated;
+  }
+  const diagnostics = truncated
+    ? { ...baseDiagnostics, truncated }
+    : baseDiagnostics;
+
+  return {
+    content: [{ text: payload, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content: payload,
+      metadata,
+      diagnostics,
+    },
+  };
+}
+
+export const HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
+
+export const htmlToMarkdownHandler = tool(
+  'html_to_markdown',
+  input => htmlToMarkdownFromHtml(fromLocalPath(input)),
+  htmlToMarkdownInputSchema,
+);
+
+export function registerHtmlToMarkdownTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'html_to_markdown',
+    {
+      title: 'Convert HTML fragment to Markdown',
+      description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
+      inputSchema: htmlToMarkdownInputShape,
+      outputSchema: outputSchemaShape,
+    },
+    htmlToMarkdownHandler,
+  );
+}

package/src/tools/outline.js

@@ -0,0 +1,47 @@
+import { tool } from "../handler.js";
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument } from "../pipeline/normalize.js";
+import { resolveOutline } from "../policy/outline.js";
+import { fromLocalPath } from "./html-source.js";
+import { outlineOutputShape } from "./output-schema.js";
+import { outlineInputSchema, outlineInputShape, } from "./schemas.js";
+// One line per heading, nested by depth; never blank so content[0].text is always scannable.
+function renderOutlineToc(outline) {
+    if (outline.length === 0) {
+        return '(no headings found)';
+    }
+    return outline
+        .map(entry => `${'  '.repeat(entry.level - 1)}- ${entry.text}`)
+        .join('\n');
+}
+export function outlineDocumentFromHtml(input) {
+    const { html, baseUrl, selectors } = input;
+    const { document } = buildDocument(html, baseUrl);
+    normalizeDocument(document);
+    applySelectors(document, selectors);
+    const outline = resolveOutline(document);
+    const title = document.title.trim() ||
+        document.querySelector('h1')?.textContent.replace(/\s+/g, ' ').trim() ||
+        undefined;
+    const metadata = { title, baseUrl };
+    const content = renderOutlineToc(outline);
+    return {
+        content: [{ text: content, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content,
+            outline,
+            metadata,
+        },
+    };
+}
+export const OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
+export const outlineHandler = tool('outline', input => outlineDocumentFromHtml(fromLocalPath(input)), outlineInputSchema);
+export function registerOutlineTool(server) {
+    return server.registerTool('outline', {
+        title: 'Get document outline (heading TOC)',
+        description: OUTLINE_TOOL_DESCRIPTION,
+        inputSchema: outlineInputShape,
+        outputSchema: outlineOutputShape,
+    }, outlineHandler);
+}

package/src/tools/outline.ts

@@ -0,0 +1,77 @@
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildDocument } from '../pipeline/dom.ts';
+import { applySelectors, normalizeDocument } from '../pipeline/normalize.ts';
+import { resolveOutline } from '../policy/outline.ts';
+import { fromLocalPath } from './html-source.ts';
+import { outlineOutputShape } from './output-schema.ts';
+import {
+  type OutlineFromHtmlInput,
+  outlineInputSchema,
+  outlineInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+// One line per heading, nested by depth; never blank so content[0].text is always scannable.
+function renderOutlineToc(
+  outline: readonly { level: number; text: string }[],
+): string {
+  if (outline.length === 0) {
+    return '(no headings found)';
+  }
+  return outline
+    .map(entry => `${'  '.repeat(entry.level - 1)}- ${entry.text}`)
+    .join('\n');
+}
+
+export function outlineDocumentFromHtml(
+  input: Readonly<OutlineFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, selectors } = input;
+
+  const { document } = buildDocument(html, baseUrl);
+  normalizeDocument(document);
+  applySelectors(document, selectors);
+  const outline = resolveOutline(document);
+
+  const title =
+    document.title.trim() ||
+    document.querySelector('h1')?.textContent.replace(/\s+/g, ' ').trim() ||
+    undefined;
+  const metadata = { title, baseUrl };
+
+  const content = renderOutlineToc(outline);
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      outline,
+      metadata,
+    },
+  };
+}
+
+export const OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
+
+export const outlineHandler = tool(
+  'outline',
+  input => outlineDocumentFromHtml(fromLocalPath(input)),
+  outlineInputSchema,
+);
+
+export function registerOutlineTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'outline',
+    {
+      title: 'Get document outline (heading TOC)',
+      description: OUTLINE_TOOL_DESCRIPTION,
+      inputSchema: outlineInputShape,
+      outputSchema: outlineOutputShape,
+    },
+    outlineHandler,
+  );
+}