@v1nvn/readability 0.44.0 → 0.45.1

package/src/tools/explain.ts

@@ -0,0 +1,299 @@
+import { z } from 'zod';
+
+import type { ExplainReport } from '../policy/explain.ts';
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { buildExplainReport } from '../policy/explain.ts';
+import { fromLocalPath, withSchemaDefaults } from './html-source.ts';
+import {
+  type FromHtmlInput,
+  localPathField,
+  selectorsSchema,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+// Schemas live with the tool rather than in schemas.ts/output-schema.ts to keep
+// the explain surface in one file (the report shape is explain-specific and not
+// shared with other tools). `selectorsSchema` is reused verbatim from extract so
+// the two tools agree on the include/exclude contract.
+const explainInputShape = {
+  localPath: localPathField,
+  baseUrl: z
+    .url()
+    .describe(
+      'Base URL for absolutizing relative links during pagination/gating detection. NEVER fetched — origin context only.',
+    )
+    .optional(),
+  selectors: selectorsSchema,
+  topN: z
+    .number()
+    .int()
+    .min(1)
+    .max(20)
+    .describe(
+      'Maximum number of scored candidate nodes to return (highest first). Default 5.',
+    )
+    .default(5),
+} as const;
+
+const explainInputSchema = z.object(explainInputShape);
+
+type ExplainInput = z.infer<typeof explainInputSchema>;
+type ExplainFromHtmlInput = FromHtmlInput<ExplainInput>;
+
+const candidateSchema = z
+  .object({
+    className: z
+      .string()
+      .describe(
+        "The candidate's class attribute (raw, unsplit). Empty string when absent.",
+      ),
+    id: z.string().describe("The candidate's id attribute, or empty string."),
+    score: z
+      .number()
+      .describe(
+        "Readability's actual contentScore for this node (link-density-scaled). Higher is better; the top entry is Readability's raw top candidate before its parent-walking/only-child adjustments.",
+      ),
+    selector: z
+      .string()
+      .describe(
+        "A CSS-ish hint (tag#id.class1.class2) for locating the node in the host DOM. NOT a unique locator — Readability's score lives on a JS expando invisible to CSS.",
+      ),
+    tag: z
+      .string()
+      .describe('Uppercase DOM tag name (e.g. "ARTICLE", "MAIN", "DIV").'),
+    textLength: z
+      .number()
+      .int()
+      .min(0)
+      .describe(
+        'Trimmed textContent length of the candidate node, for eyeballing content density.',
+      ),
+  })
+  .describe(
+    'One scored candidate node Readability considered, with its real contentScore.',
+  );
+
+const explainOutputShape = {
+  schemaVersion: z
+    .literal(1)
+    .describe(
+      'Structured-content schema version. Bumps only on breaking shape changes to this object.',
+    ),
+  content: z
+    .string()
+    .describe(
+      'Readable rendering of the report (chosen root, ranked candidates, removal counts, gating/pagination, snapshot head) so content[0].text is always scannable.',
+    ),
+  chosenRoot: candidateSchema
+    .nullable()
+    .describe(
+      'The highest-scoring candidate Readability computed — its raw top pick before parent-walking/only-child post-processing. Null when Readability scored nothing (e.g. empty input).',
+    ),
+  candidates: z
+    .array(candidateSchema)
+    .describe(
+      "Scored candidate nodes (highest first), capped at topN. These are Readability's real contentScore values, not a self-computed heuristic.",
+    ),
+  readerable: z
+    .boolean()
+    .describe(
+      'Readability isProbablyReaderable verdict on the normalized document.',
+    ),
+  parseSucceeded: z
+    .boolean()
+    .describe(
+      'True when reader.parse() returned article content. False signals that `extract` would fall back to its selector cascade.',
+    ),
+  fallbackUsed: z
+    .boolean()
+    .describe(
+      'Always false for explain — this tool runs only the Readability path it is diagnosing, never the fallback cascade.',
+    ),
+  gating: z
+    .object({
+      likely: z
+        .boolean()
+        .describe(
+          'True when heuristics strongly suggest the content is paywalled or truncated.',
+        ),
+      reason: z
+        .string()
+        .describe(
+          'Short label naming the detected signal (e.g. "paywall overlay").',
+        ),
+    })
+    .nullable()
+    .describe(
+      'Likely paywall / gating signal detected before normalization. Null when none.',
+    ),
+  pagination: z
+    .object({
+      type: z
+        .enum(['infinite', 'paginated'])
+        .describe('Kind of pagination signal detected.'),
+      nextUrl: z
+        .string()
+        .optional()
+        .describe(
+          'Absolute URL of the detected next page (paginated only). Never fetched.',
+        ),
+      selector: z
+        .string()
+        .optional()
+        .describe(
+          'CSS selector of the load-more / infinite-scroll sentinel (infinite only).',
+        ),
+    })
+    .nullable()
+    .describe('Detected pagination / infinite-scroll signal. Null when none.'),
+  removedNodes: z
+    .object({
+      boilerplate: z
+        .number()
+        .int()
+        .min(0)
+        .describe(
+          'Boilerplate blocks (related-posts, newsletter signup) stripped before Readability.',
+        ),
+      chrome: z
+        .number()
+        .int()
+        .min(0)
+        .describe(
+          'Browser-chrome nodes stripped (scrollbars, consent banners, overlays).',
+        ),
+      total: z
+        .number()
+        .int()
+        .min(0)
+        .describe(
+          'Net element count removed across the whole pipeline (delta vs. the parsed document).',
+        ),
+    })
+    .describe(
+      'Breakdown of nodes removed before Readability saw the document, reused from the extract diagnostics path.',
+    ),
+  snapshot: z
+    .object({
+      html: z
+        .string()
+        .describe(
+          'The sanitized-by-normalization (post chrome/boilerplate/script strip) HTML fed to Readability — "what Readability saw". Not DOMPurify-sanitized (that runs on Readability\'s output in `extract`), so it may still carry inline event handlers (`onerror`/`onclick`/…); it is diagnostic data — do not render verbatim.',
+        ),
+      truncated: z
+        .boolean()
+        .describe(
+          'True when the snapshot was cut at snapshotMaxChars (default 4000).',
+        ),
+    })
+    .describe('Pre-Readability HTML snapshot of the normalized document body.'),
+} as const;
+
+const explainOutput = z.object(explainOutputShape);
+
+function formatGating(report: ExplainReport): string {
+  const g = report.gating;
+  if (!g) {
+    return 'none';
+  }
+  return `${g.reason}${g.likely ? '' : ' (weak)'}`;
+}
+
+function formatPagination(report: ExplainReport): string {
+  const p = report.pagination;
+  if (!p) {
+    return 'none';
+  }
+  if (p.type === 'paginated') {
+    return `paginated -> ${p.nextUrl ?? '(no href)'}`;
+  }
+  return `infinite (${p.selector ?? 'sentinel'})`;
+}
+
+function renderText(report: ExplainReport): string {
+  const lines: string[] = [];
+  const root = report.chosenRoot;
+  lines.push(
+    `readerable: ${report.readerable ? 'yes' : 'no'}  parse: ${report.parseSucceeded ? 'ok' : 'fail'}  fallback: ${report.fallbackUsed ? 'yes' : 'no'}`,
+  );
+  lines.push(
+    `chosen root: ${root ? `${root.selector}  (score ${root.score.toFixed(2)}, ${root.textLength} chars)` : '(no candidate scored)'}`,
+  );
+  lines.push(`top candidates (${report.candidates.length}):`);
+  if (report.candidates.length === 0) {
+    lines.push('  (none)');
+  }
+  report.candidates.forEach((c, i) => {
+    lines.push(
+      `  ${i + 1}. ${c.selector}  score ${c.score.toFixed(2)}  (${c.textLength} chars)`,
+    );
+  });
+  const r = report.removedNodes;
+  lines.push(
+    `removed: total=${r.total} (chrome=${r.chrome}, boilerplate=${r.boilerplate})`,
+  );
+  lines.push(`gating: ${formatGating(report)}`);
+  lines.push(`pagination: ${formatPagination(report)}`);
+  lines.push(
+    `snapshot (${report.snapshot.html.length} chars${report.snapshot.truncated ? ', truncated' : ''}):`,
+  );
+  lines.push(report.snapshot.html);
+  return lines.join('\n');
+}
+
+export function explainFromHtml(
+  input: Readonly<ExplainFromHtmlInput>,
+): CallToolResult {
+  const { html, baseUrl, selectors, topN } = withSchemaDefaults(
+    explainInputSchema,
+    input,
+  );
+  const report = buildExplainReport({
+    html,
+    selectors,
+    topN,
+    baseUrl,
+  });
+  const content = renderText(report);
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      chosenRoot: report.chosenRoot,
+      candidates: report.candidates,
+      readerable: report.readerable,
+      parseSucceeded: report.parseSucceeded,
+      fallbackUsed: report.fallbackUsed,
+      gating: report.gating ?? null,
+      pagination: report.pagination ?? null,
+      removedNodes: report.removedNodes,
+      snapshot: report.snapshot,
+    },
+  };
+}
+
+export const EXPLAIN_TOOL_DESCRIPTION = `Post-mortem diagnostics for extraction: shows WHY Readability picked what it picked. Returns the chosen root, the ranked candidate nodes with their REAL Readability contentScore values (read off the DOM expando Readability stamps during scoring), a categorized removed-nodes breakdown, gating/pagination signals, and a snapshot of the normalized HTML fed to Readability. Runs the same normalize + Readability pipeline as \`extract\` (no fallback cascade, no Turndown). The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
+
+export const explainHandler = tool(
+  'explain',
+  input => explainFromHtml(fromLocalPath(input)),
+  explainInputSchema,
+);
+
+export function registerExplainTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'explain',
+    {
+      title: 'Explain why Readability picked what it picked',
+      description: EXPLAIN_TOOL_DESCRIPTION,
+      inputSchema: explainInputShape,
+      outputSchema: explainOutput,
+    },
+    explainHandler,
+  );
+}

package/src/tools/extract.js

@@ -0,0 +1,263 @@
+import { ExtractionError } from "../errors.js";
+import { tool } from "../handler.js";
+import { logger } from "../logger.js";
+import { formatPayload } from "../output/format.js";
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, canonicalizeCodeBlocks, normalizeDocument, resolveLazyImages, } from "../pipeline/normalize.js";
+import { isReaderable, parseArticle } from "../pipeline/readability.js";
+import { resolveReadabilityOptions } from "../pipeline/resolver.js";
+import { sanitizeHtml } from "../pipeline/sanitize.js";
+import { toMarkdown } from "../pipeline/turndown.js";
+import { chunkMarkdown } from "../policy/chunk.js";
+import { assembleDiagnostics, TraceCollector } from "../policy/diagnostics.js";
+import { extractViaFallback } from "../policy/fallback.js";
+import { detectGating } from "../policy/gating.js";
+import { collectImageInventory } from "../policy/images.js";
+import { resolveMetadata } from "../policy/metadata.js";
+import { detectPagination } from "../policy/pagination.js";
+import { resolvePreset } from "../policy/presets.js";
+import { computeTextMetrics } from "../policy/text.js";
+import { truncateMarkdown } from "../policy/truncate.js";
+import * as cache from "../resources.js";
+import { fromLocalPath, withSchemaDefaults } from "./html-source.js";
+import { outputSchemaShape } from "./output-schema.js";
+import { extractInputSchema, extractInputShape, } from "./schemas.js";
+// Names the extraction path, not a DOM tag — Readability doesn't expose the source node.
+const EXTRACTED_NODE = 'readability';
+export function extractArticle(rawArgs) {
+    return extractArticleFromHtml(fromLocalPath(extractInputSchema.parse(rawArgs)));
+}
+export function extractArticleFromHtml(input) {
+    const merged = withSchemaDefaults(extractInputSchema, input);
+    const { html, baseUrl, cache: useCache, selectors, extraction, minArticleLength, maxNodes, keepClasses, readabilityOverrides, format, metadataMode, gfm, headingStyle, codeBlockStyle, images, sanitize: shouldSanitize, maxChars, wordsPerMinute, cleanChrome, tables, chunk, imageInventory, debug, resolvePreset: presetsEnabled = true, } = merged;
+    // Cache hit short-circuits the pipeline. A clone is returned so the cache
+    // field can be overwritten to reflect *this* call's hashes (normalizedHash is
+    // stable across re-renders, but originalHash differs by nonce/CSP/…).
+    if (useCache) {
+        const hit = cache.lookup(html, merged);
+        if (hit) {
+            const cloned = JSON.parse(JSON.stringify(hit.entry.structuredContent));
+            const structuredContent = {
+                ...cloned,
+                diagnostics: {
+                    ...cloned.diagnostics,
+                    cache: {
+                        hit: true,
+                        normalizedHash: hit.normalizedHash,
+                        originalHash: hit.originalHash,
+                    },
+                },
+            };
+            return {
+                content: [{ text: hit.entry.contentText, type: 'text' }],
+                structuredContent,
+            };
+        }
+    }
+    // Stages are timed at the orchestrator boundary so they stay non-overlapping
+    // and sum to the pipeline's wall-clock — stripConsent/absolutize live inside
+    // normalize/turndown respectively and aren't carved out as their own stages.
+    const trace = new TraceCollector(debug);
+    const { document, window } = buildDocument(html, baseUrl);
+    // Paywall overlays are stripped by normalizeDocument's stripChrome (a Piano
+    // modal is role="dialog" + full-viewport fixed), so gating must be detected
+    // before normalization. Pagination chrome survives stripChrome and is detected after.
+    const { gating, documentElementCount, normalizeCounts, imagesResolved, pagination, preset, } = trace.run('normalize', () => {
+        const gating = detectGating(document);
+        const documentElementCount = document.querySelectorAll('*').length;
+        const normalizeCounts = normalizeDocument(document, { cleanChrome });
+        const imagesResolved = resolveLazyImages(document);
+        // Detect before applySelectors: a caller's selectors.include could scope the
+        // body and hide pagination chrome, but "more content exists" is still true.
+        const pagination = detectPagination(document, baseUrl);
+        const resolution = presetsEnabled
+            ? resolvePreset(document, baseUrl, selectors !== undefined)
+            : undefined;
+        applySelectors(document, resolution?.scope ?? selectors);
+        const codeBlocksCanonicalized = canonicalizeCodeBlocks(document);
+        if (codeBlocksCanonicalized > 0) {
+            logger.debug(`canonicalized ${codeBlocksCanonicalized} code-block language tag(s)`);
+        }
+        return {
+            documentElementCount,
+            gating,
+            imagesResolved,
+            normalizeCounts,
+            pagination,
+            preset: resolution?.signal,
+        };
+    });
+    const { readerable, article } = trace.run('readability', () => {
+        const readerable = isReaderable(document);
+        const readabilityOptions = resolveReadabilityOptions({
+            extraction,
+            keepClasses,
+            maxNodes,
+            minArticleLength,
+            readabilityOverrides,
+        });
+        const article = parseArticle(document, readabilityOptions);
+        return { article, readerable };
+    });
+    let markdown;
+    let sanitizedHtml;
+    let textContent;
+    let extractedNode;
+    let fallbackUsed;
+    let sanitizeCounts;
+    if (article?.content) {
+        extractedNode = EXTRACTED_NODE;
+        fallbackUsed = false;
+        textContent = article.textContent ?? '';
+        const articleHtml = article.content;
+        const sanitized = trace.run('sanitize', () => {
+            if (!shouldSanitize) {
+                return { counts: { iframes: 0, scripts: 0 }, html: articleHtml };
+            }
+            const res = sanitizeHtml(articleHtml, window);
+            return {
+                counts: { iframes: res.iframesRemoved, scripts: res.scriptsRemoved },
+                html: res.html,
+            };
+        });
+        sanitizedHtml = sanitized.html;
+        sanitizeCounts = sanitized.counts;
+        markdown = trace.run('turndown', () => toMarkdown(sanitizedHtml, {
+            codeBlockStyle,
+            gfm,
+            headingStyle,
+            images,
+            tables,
+            baseUrl,
+        }));
+    }
+    else {
+        // Cascade only on parse failure, not on isProbablyReaderable.
+        // Fallback owns sanitize + turndown internally, so the two stages collapse
+        // into one timed block here — keeping the trace non-overlapping.
+        const fallback = trace.run('fallback', () => extractViaFallback(document, {
+            codeBlockStyle,
+            gfm,
+            headingStyle,
+            images,
+            sanitize: shouldSanitize,
+            tables,
+            baseUrl,
+            window,
+        }));
+        if (!fallback) {
+            throw new ExtractionError('Readability returned no article and the selector cascade yielded no usable content.');
+        }
+        extractedNode = fallback.rootSelector;
+        fallbackUsed = true;
+        markdown = fallback.markdown;
+        sanitizedHtml = fallback.sanitizedHtml;
+        textContent = fallback.textContent;
+        sanitizeCounts = fallback.sanitization;
+    }
+    const { metadata } = trace.run('metadata', () => {
+        const { wordCount, readingTimeMin } = computeTextMetrics(textContent, wordsPerMinute);
+        const metadata = resolveMetadata({
+            document,
+            readability: article,
+            readingTimeMin,
+            textContent,
+            baseUrl,
+            wordCount,
+        });
+        return { metadata };
+    });
+    const sanitization = {
+        iframes: normalizeCounts.iframes + sanitizeCounts.iframes,
+        scripts: normalizeCounts.scripts + sanitizeCounts.scripts,
+    };
+    const baseDiagnostics = assembleDiagnostics({
+        articleHtml: sanitizedHtml,
+        boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+        chromeRemoved: normalizeCounts.chromeRemoved,
+        documentElementCount,
+        extractedNode,
+        fallbackUsed,
+        gated: gating,
+        imagesResolved,
+        pagination,
+        preset,
+        readerable,
+        sanitization,
+        trace: trace.collect(),
+        truncated: false,
+        window,
+    });
+    let payload = formatPayload({
+        diagnostics: baseDiagnostics,
+        format,
+        markdown,
+        metadata,
+        metadataMode,
+        sanitizedHtml,
+        textContent,
+    });
+    let truncated = false;
+    if (maxChars !== undefined && (format === 'markdown' || format === 'text')) {
+        const res = truncateMarkdown(payload, maxChars);
+        payload = res.text;
+        truncated = res.truncated;
+    }
+    const diagnostics = truncated
+        ? { ...baseDiagnostics, truncated }
+        : baseDiagnostics;
+    // Chunk the final payload (post-format, post-truncation) so token counts
+    // reflect what the host sees in content[0].text. HTML/JSON formats carry no
+    // markdown body to slice, so `chunks` stays unset for them.
+    const chunks = chunk && (format === 'markdown' || format === 'text')
+        ? chunkMarkdown(payload, chunk)
+        : undefined;
+    // Runs against sanitizedHtml so the inventory reflects exactly what the host
+    // sees in content[0].text — post-sanitization, post-lazy-resolution.
+    const imageInventoryEntries = imageInventory
+        ? collectImageInventory(sanitizedHtml, window, baseUrl)
+        : undefined;
+    const baseStructuredContent = {
+        schemaVersion: 1,
+        content: payload,
+        metadata,
+        diagnostics,
+        ...(chunks ? { chunks } : {}),
+        ...(imageInventoryEntries ? { images: imageInventoryEntries } : {}),
+    };
+    // On cache miss (when cache:true), persist a cache-agnostic copy so the
+    // stored entry never carries a stale hit/miss verdict, then surface the
+    // signal on the returned object only.
+    let structuredContent = baseStructuredContent;
+    if (useCache) {
+        const stored = cache.storeResult(html, merged, {
+            contentText: payload,
+            structuredContent: baseStructuredContent,
+        });
+        structuredContent = {
+            ...baseStructuredContent,
+            diagnostics: {
+                ...diagnostics,
+                cache: {
+                    hit: false,
+                    normalizedHash: stored.normalizedHash,
+                    originalHash: stored.originalHash,
+                },
+            },
+        };
+    }
+    return {
+        content: [{ text: payload, type: 'text' }],
+        structuredContent,
+    };
+}
+export const EXTRACT_TOOL_DESCRIPTION = `Extract the main article from already-rendered (post-JavaScript) HTML and return clean Markdown plus metadata and diagnostics. The server fetches nothing: \`localPath\` (a file holding the rendered HTML, e.g. \`document.documentElement.outerHTML\` written to disk by a browser/devtools capture) is the only source, and \`baseUrl\` (optional) is used solely to absolutize relative links.`;
+export const extractHandler = tool('extract', extractArticle);
+export function registerExtractTool(server) {
+    return server.registerTool('extract', {
+        title: 'Extract article to Markdown',
+        description: EXTRACT_TOOL_DESCRIPTION,
+        inputSchema: extractInputShape,
+        outputSchema: outputSchemaShape,
+    }, extractHandler);
+}