@v1nvn/readability 0.45.0 → 0.45.1

package/src/tools/html-source.js

@@ -0,0 +1,30 @@
+import { readFileSync } from 'node:fs';
+export function readHtmlFile(localPath) {
+    const raw = readFileSync(localPath, 'utf8');
+    const trimmed = raw.trim();
+    // A top-level JSON string scalar is a chrome-devtools capture (its JSON-encoded
+    // return value written to disk), not raw markup — unwrap it before parsing.
+    if (trimmed.length < 2 || trimmed.at(0) !== '"' || trimmed.at(-1) !== '"') {
+        return raw;
+    }
+    try {
+        const parsed = JSON.parse(trimmed);
+        return typeof parsed === 'string' ? parsed : raw;
+    }
+    catch {
+        return raw;
+    }
+}
+export function fromLocalPath(input) {
+    const { localPath, ...rest } = input;
+    return { ...rest, html: readHtmlFile(localPath) };
+}
+const schemaDefaults = new WeakMap();
+export function withSchemaDefaults(schema, input) {
+    let defaults = schemaDefaults.get(schema);
+    if (defaults === undefined) {
+        defaults = schema.parse({ localPath: '' });
+        schemaDefaults.set(schema, defaults);
+    }
+    return { ...defaults, ...input };
+}

package/src/tools/html_to_markdown.js

@@ -0,0 +1,113 @@
+import { tool } from "../handler.js";
+import { formatPayload } from "../output/format.js";
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument, resolveLazyImages, } from "../pipeline/normalize.js";
+import { sanitizeHtml } from "../pipeline/sanitize.js";
+import { toMarkdown } from "../pipeline/turndown.js";
+import { assembleDiagnostics, TraceCollector } from "../policy/diagnostics.js";
+import { computeTextMetrics, nonEmpty } from "../policy/text.js";
+import { truncateMarkdown } from "../policy/truncate.js";
+import { fromLocalPath, withSchemaDefaults } from "./html-source.js";
+import { outputSchemaShape } from "./output-schema.js";
+import { htmlToMarkdownInputSchema, htmlToMarkdownInputShape, } from "./schemas.js";
+const EXTRACTED_NODE = 'fragment';
+export function htmlToMarkdownFromHtml(input) {
+    const { html, baseUrl, selectors, format, metadataMode, gfm, headingStyle, codeBlockStyle, images, sanitize: shouldSanitize, maxChars, wordsPerMinute, cleanChrome, tables, debug, } = withSchemaDefaults(htmlToMarkdownInputSchema, input);
+    const trace = new TraceCollector(debug);
+    const { document, window } = buildDocument(html, baseUrl);
+    const { documentElementCount, normalizeCounts, imagesResolved, textContent, rawHtml, } = trace.run('normalize', () => {
+        const documentElementCount = document.querySelectorAll('*').length;
+        const normalizeCounts = normalizeDocument(document, { cleanChrome });
+        const imagesResolved = resolveLazyImages(document);
+        applySelectors(document, selectors);
+        const body = document.body;
+        return {
+            documentElementCount,
+            imagesResolved,
+            normalizeCounts,
+            rawHtml: body.innerHTML,
+            textContent: body.textContent,
+        };
+    });
+    const { html: sanitizedHtml, counts: sanitizeCounts } = trace.run('sanitize', () => {
+        if (!shouldSanitize) {
+            return { counts: { iframes: 0, scripts: 0 }, html: rawHtml };
+        }
+        const res = sanitizeHtml(rawHtml, window);
+        return {
+            counts: { iframes: res.iframesRemoved, scripts: res.scriptsRemoved },
+            html: res.html,
+        };
+    });
+    const markdown = trace.run('turndown', () => toMarkdown(sanitizedHtml, {
+        codeBlockStyle,
+        gfm,
+        headingStyle,
+        images,
+        tables,
+        baseUrl,
+    }));
+    const { metadata } = trace.run('metadata', () => {
+        const firstHeading = nonEmpty(document.body.querySelector('h1, h2, h3, h4, h5, h6')?.textContent);
+        const metadata = {
+            title: firstHeading,
+            baseUrl,
+            ...computeTextMetrics(textContent, wordsPerMinute),
+        };
+        return { metadata };
+    });
+    const sanitization = {
+        iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
+        scripts: normalizeCounts.scripts + sanitizeCounts.scripts,
+    };
+    const baseDiagnostics = assembleDiagnostics({
+        articleHtml: sanitizedHtml,
+        boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+        chromeRemoved: normalizeCounts.chromeRemoved,
+        documentElementCount,
+        extractedNode: EXTRACTED_NODE,
+        fallbackUsed: true,
+        imagesResolved,
+        sanitization,
+        trace: trace.collect(),
+        truncated: false,
+        window,
+    });
+    let payload = formatPayload({
+        diagnostics: baseDiagnostics,
+        format,
+        markdown,
+        metadata,
+        metadataMode,
+        sanitizedHtml,
+        textContent,
+    });
+    let truncated = false;
+    if (maxChars !== undefined && (format === 'markdown' || format === 'text')) {
+        const res = truncateMarkdown(payload, maxChars);
+        payload = res.text;
+        truncated = res.truncated;
+    }
+    const diagnostics = truncated
+        ? { ...baseDiagnostics, truncated }
+        : baseDiagnostics;
+    return {
+        content: [{ text: payload, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content: payload,
+            metadata,
+            diagnostics,
+        },
+    };
+}
+export const HTML_TO_MARKDOWN_TOOL_DESCRIPTION = `Convert an arbitrary HTML fragment to Markdown WITHOUT Readability article extraction (e.g. a snippet already isolated via chrome-devtools). Same Turndown + DOMPurify path as \`extract\`. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) absolutizes relative links.`;
+export const htmlToMarkdownHandler = tool('html_to_markdown', input => htmlToMarkdownFromHtml(fromLocalPath(input)), htmlToMarkdownInputSchema);
+export function registerHtmlToMarkdownTool(server) {
+    return server.registerTool('html_to_markdown', {
+        title: 'Convert HTML fragment to Markdown',
+        description: HTML_TO_MARKDOWN_TOOL_DESCRIPTION,
+        inputSchema: htmlToMarkdownInputShape,
+        outputSchema: outputSchemaShape,
+    }, htmlToMarkdownHandler);
+}

package/src/tools/outline.js

@@ -0,0 +1,47 @@
+import { tool } from "../handler.js";
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument } from "../pipeline/normalize.js";
+import { resolveOutline } from "../policy/outline.js";
+import { fromLocalPath } from "./html-source.js";
+import { outlineOutputShape } from "./output-schema.js";
+import { outlineInputSchema, outlineInputShape, } from "./schemas.js";
+// One line per heading, nested by depth; never blank so content[0].text is always scannable.
+function renderOutlineToc(outline) {
+    if (outline.length === 0) {
+        return '(no headings found)';
+    }
+    return outline
+        .map(entry => `${'  '.repeat(entry.level - 1)}- ${entry.text}`)
+        .join('\n');
+}
+export function outlineDocumentFromHtml(input) {
+    const { html, baseUrl, selectors } = input;
+    const { document } = buildDocument(html, baseUrl);
+    normalizeDocument(document);
+    applySelectors(document, selectors);
+    const outline = resolveOutline(document);
+    const title = document.title.trim() ||
+        document.querySelector('h1')?.textContent.replace(/\s+/g, ' ').trim() ||
+        undefined;
+    const metadata = { title, baseUrl };
+    const content = renderOutlineToc(outline);
+    return {
+        content: [{ text: content, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content,
+            outline,
+            metadata,
+        },
+    };
+}
+export const OUTLINE_TOOL_DESCRIPTION = `Return the document outline (h1-h6 headings with stable anchor ids) of already-rendered (post-JavaScript) HTML as a cheap pre-check before full extraction. No Readability scoring, no Turndown, no sanitization — a pure heading walk. The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` is origin context only (never fetched).`;
+export const outlineHandler = tool('outline', input => outlineDocumentFromHtml(fromLocalPath(input)), outlineInputSchema);
+export function registerOutlineTool(server) {
+    return server.registerTool('outline', {
+        title: 'Get document outline (heading TOC)',
+        description: OUTLINE_TOOL_DESCRIPTION,
+        inputSchema: outlineInputShape,
+        outputSchema: outlineOutputShape,
+    }, outlineHandler);
+}