@v1nvn/readability 0.44.0 → 0.45.1

package/src/server.js

@@ -0,0 +1,73 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { loadConfig } from "./config.js";
+import { loadPresets } from "./preset-cache.js";
+import { registerResources } from "./resources.js";
+import { registerSamplingTools } from "./sampling.js";
+import { registerChunkTextTool } from "./tools/chunk_text.js";
+import { registerExplainTool } from "./tools/explain.js";
+import { registerExtractGridTool } from "./tools/extract_grid.js";
+import { registerExtractLinksTool } from "./tools/extract_links.js";
+import { registerExtractListTool } from "./tools/extract_list.js";
+import { registerExtractMetadataTool } from "./tools/extract_metadata.js";
+import { registerExtractSectionTool } from "./tools/extract_section.js";
+import { registerExtractTablesTool } from "./tools/extract_tables.js";
+import { registerExtractTool } from "./tools/extract.js";
+import { registerHtmlToMarkdownTool } from "./tools/html_to_markdown.js";
+import { registerOutlineTool } from "./tools/outline.js";
+// Re-exported so the dev hot-reload loop can import server.ts as a single
+// RuntimeModule and pick up every registration family (tools/resources) plus
+// the boot-time preset load.
+export { loadPresets, registerResources };
+export function createMcpServer() {
+    const { name, version, title, description, instructions } = loadConfig();
+    return new McpServer({ name, version, title, description }, { instructions });
+}
+// A tool earns its place only by skipping a pipeline stage (outline,
+// extract_metadata skip Readability) or returning a fundamentally different
+// shape (extract_list is a second engine). Alternate views of the one pipeline
+// — tables, images, structured data, code — are options or fields on `extract`,
+// not separate tools; that keeps the MCP surface small and the pipeline deep.
+export function registerTools(server) {
+    return [
+        registerChunkTextTool(server),
+        registerExplainTool(server),
+        registerExtractGridTool(server),
+        registerExtractLinksTool(server),
+        registerExtractListTool(server),
+        registerExtractTool(server),
+        registerExtractMetadataTool(server),
+        registerExtractSectionTool(server),
+        registerExtractTablesTool(server),
+        registerHtmlToMarkdownTool(server),
+        registerOutlineTool(server),
+    ];
+}
+// Capability-gated tools are registered AFTER the initialize handshake, not
+// eagerly with the families above. The MCP `tools` capability locks in on the
+// first pre-connect registration (registerCapabilities throws post-connect),
+// so the tool-list handlers are already live; adding to `_registeredTools`
+// later simply appears in the next `tools/list` and fires `listChanged`. The
+// low-level Server populates `getClientCapabilities()` from the client's
+// `initialize` request, exposed via `McpServer.server`.
+export function registerCapabilityGatedTools(server) {
+    const caps = server.server.getClientCapabilities();
+    if (!caps?.sampling) {
+        return [];
+    }
+    return registerSamplingTools(server);
+}
+export function createServer() {
+    const server = createMcpServer();
+    loadPresets();
+    registerTools(server);
+    registerResources(server);
+    // Capability-gated tools (sampling) need the client's advertised
+    // capabilities, which the low-level Server only populates after the
+    // initialize handshake. Hook the `initialized` notification — it fires once
+    // the client has connected and before any `tools/list`. `dev.ts` re-runs the
+    // gate directly on reload (the client is already past `initialized`).
+    server.server.oninitialized = () => {
+        registerCapabilityGatedTools(server);
+    };
+    return server;
+}

package/src/server.ts

@@ -0,0 +1,85 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+import { loadConfig } from './config.ts';
+import { loadPresets } from './preset-cache.ts';
+import { registerResources } from './resources.ts';
+import { registerSamplingTools } from './sampling.ts';
+import { registerChunkTextTool } from './tools/chunk_text.ts';
+import { registerExplainTool } from './tools/explain.ts';
+import { registerExtractGridTool } from './tools/extract_grid.ts';
+import { registerExtractLinksTool } from './tools/extract_links.ts';
+import { registerExtractListTool } from './tools/extract_list.ts';
+import { registerExtractMetadataTool } from './tools/extract_metadata.ts';
+import { registerExtractSectionTool } from './tools/extract_section.ts';
+import { registerExtractTablesTool } from './tools/extract_tables.ts';
+import { registerExtractTool } from './tools/extract.ts';
+import { registerHtmlToMarkdownTool } from './tools/html_to_markdown.ts';
+import { registerOutlineTool } from './tools/outline.ts';
+
+// Re-exported so the dev hot-reload loop can import server.ts as a single
+// RuntimeModule and pick up every registration family (tools/resources) plus
+// the boot-time preset load.
+export { loadPresets, registerResources };
+
+// `remove()` unregisters the tool and notifies the client; dev reload holds the
+// previous batch to remove before re-registering.
+export interface ToolHandle {
+  remove(): void;
+}
+
+export function createMcpServer(): McpServer {
+  const { name, version, title, description, instructions } = loadConfig();
+  return new McpServer({ name, version, title, description }, { instructions });
+}
+
+// A tool earns its place only by skipping a pipeline stage (outline,
+// extract_metadata skip Readability) or returning a fundamentally different
+// shape (extract_list is a second engine). Alternate views of the one pipeline
+// — tables, images, structured data, code — are options or fields on `extract`,
+// not separate tools; that keeps the MCP surface small and the pipeline deep.
+export function registerTools(server: McpServer): ToolHandle[] {
+  return [
+    registerChunkTextTool(server),
+    registerExplainTool(server),
+    registerExtractGridTool(server),
+    registerExtractLinksTool(server),
+    registerExtractListTool(server),
+    registerExtractTool(server),
+    registerExtractMetadataTool(server),
+    registerExtractSectionTool(server),
+    registerExtractTablesTool(server),
+    registerHtmlToMarkdownTool(server),
+    registerOutlineTool(server),
+  ];
+}
+
+// Capability-gated tools are registered AFTER the initialize handshake, not
+// eagerly with the families above. The MCP `tools` capability locks in on the
+// first pre-connect registration (registerCapabilities throws post-connect),
+// so the tool-list handlers are already live; adding to `_registeredTools`
+// later simply appears in the next `tools/list` and fires `listChanged`. The
+// low-level Server populates `getClientCapabilities()` from the client's
+// `initialize` request, exposed via `McpServer.server`.
+export function registerCapabilityGatedTools(server: McpServer): ToolHandle[] {
+  const caps = server.server.getClientCapabilities();
+  if (!caps?.sampling) {
+    return [];
+  }
+  return registerSamplingTools(server);
+}
+
+export function createServer(): McpServer {
+  const server = createMcpServer();
+  loadPresets();
+  registerTools(server);
+  registerResources(server);
+  // Capability-gated tools (sampling) need the client's advertised
+  // capabilities, which the low-level Server only populates after the
+  // initialize handshake. Hook the `initialized` notification — it fires once
+  // the client has connected and before any `tools/list`. `dev.ts` re-runs the
+  // gate directly on reload (the client is already past `initialized`).
+  server.server.oninitialized = () => {
+    registerCapabilityGatedTools(server);
+  };
+  return server;
+}

package/src/tools/chunk_text.js

@@ -0,0 +1,40 @@
+import { tool } from "../handler.js";
+import { chunkMarkdown } from "../policy/chunk.js";
+import { chunkTextOutputShape } from "./output-schema.js";
+import { chunkTextInputSchema, chunkTextInputShape, } from "./schemas.js";
+// One numbered section per chunk, heading context in brackets when present, so
+// content[0].text is always scannable without unpacking structuredContent.
+function renderChunkIndex(chunks) {
+    if (chunks.length === 0) {
+        return '(no chunks emitted — input had no non-whitespace content)';
+    }
+    return chunks
+        .map(chunk => {
+        const head = chunk.headingContext ? ` [${chunk.headingContext}]` : '';
+        return `## Chunk ${chunk.index}${head}\n\n${chunk.text}`;
+    })
+        .join('\n\n');
+}
+export function chunkTextDocument(args) {
+    const { text, maxTokens, overlap, strategy } = args;
+    const chunks = chunkMarkdown(text, { maxTokens, overlap, strategy });
+    const content = renderChunkIndex(chunks);
+    return {
+        content: [{ text: content, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content,
+            chunks,
+        },
+    };
+}
+export const CHUNK_TEXT_TOOL_DESCRIPTION = `Split already-extracted text into token-bounded chunks for embedding/RAG. Each chunk carries its index, tokenCount (chars/4), and the nearest preceding markdown heading as headingContext. Operates on any text — pair with the \`chunk\` option on \`extract\` when you want chunks inline with the extraction. The server fetches nothing: \`text\` is the only input.`;
+export const chunkTextHandler = tool('chunk_text', chunkTextDocument, chunkTextInputSchema);
+export function registerChunkTextTool(server) {
+    return server.registerTool('chunk_text', {
+        title: 'Chunk text for RAG/embedding',
+        description: CHUNK_TEXT_TOOL_DESCRIPTION,
+        inputSchema: chunkTextInputShape,
+        outputSchema: chunkTextOutputShape,
+    }, chunkTextHandler);
+}

package/src/tools/chunk_text.ts

@@ -0,0 +1,65 @@
+import type { Chunk } from '../policy/chunk.ts';
+import type { ToolHandle } from '../server.ts';
+
+import { tool } from '../handler.ts';
+import { chunkMarkdown } from '../policy/chunk.ts';
+import { chunkTextOutputShape } from './output-schema.ts';
+import {
+  type ChunkTextInput,
+  chunkTextInputSchema,
+  chunkTextInputShape,
+} from './schemas.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+// One numbered section per chunk, heading context in brackets when present, so
+// content[0].text is always scannable without unpacking structuredContent.
+function renderChunkIndex(chunks: readonly Chunk[]): string {
+  if (chunks.length === 0) {
+    return '(no chunks emitted — input had no non-whitespace content)';
+  }
+  return chunks
+    .map(chunk => {
+      const head = chunk.headingContext ? ` [${chunk.headingContext}]` : '';
+      return `## Chunk ${chunk.index}${head}\n\n${chunk.text}`;
+    })
+    .join('\n\n');
+}
+
+export function chunkTextDocument(
+  args: Readonly<ChunkTextInput>,
+): CallToolResult {
+  const { text, maxTokens, overlap, strategy } = args;
+  const chunks = chunkMarkdown(text, { maxTokens, overlap, strategy });
+  const content = renderChunkIndex(chunks);
+  return {
+    content: [{ text: content, type: 'text' }],
+    structuredContent: {
+      schemaVersion: 1,
+      content,
+      chunks,
+    },
+  };
+}
+
+export const CHUNK_TEXT_TOOL_DESCRIPTION = `Split already-extracted text into token-bounded chunks for embedding/RAG. Each chunk carries its index, tokenCount (chars/4), and the nearest preceding markdown heading as headingContext. Operates on any text — pair with the \`chunk\` option on \`extract\` when you want chunks inline with the extraction. The server fetches nothing: \`text\` is the only input.`;
+
+export const chunkTextHandler = tool(
+  'chunk_text',
+  chunkTextDocument,
+  chunkTextInputSchema,
+);
+
+export function registerChunkTextTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'chunk_text',
+    {
+      title: 'Chunk text for RAG/embedding',
+      description: CHUNK_TEXT_TOOL_DESCRIPTION,
+      inputSchema: chunkTextInputShape,
+      outputSchema: chunkTextOutputShape,
+    },
+    chunkTextHandler,
+  );
+}

package/src/tools/explain.js

@@ -0,0 +1,200 @@
+import { z } from 'zod';
+import { tool } from "../handler.js";
+import { buildExplainReport } from "../policy/explain.js";
+import { fromLocalPath, withSchemaDefaults } from "./html-source.js";
+import { localPathField, selectorsSchema, } from "./schemas.js";
+// Schemas live with the tool rather than in schemas.ts/output-schema.ts to keep
+// the explain surface in one file (the report shape is explain-specific and not
+// shared with other tools). `selectorsSchema` is reused verbatim from extract so
+// the two tools agree on the include/exclude contract.
+const explainInputShape = {
+    localPath: localPathField,
+    baseUrl: z
+        .url()
+        .describe('Base URL for absolutizing relative links during pagination/gating detection. NEVER fetched — origin context only.')
+        .optional(),
+    selectors: selectorsSchema,
+    topN: z
+        .number()
+        .int()
+        .min(1)
+        .max(20)
+        .describe('Maximum number of scored candidate nodes to return (highest first). Default 5.')
+        .default(5),
+};
+const explainInputSchema = z.object(explainInputShape);
+const candidateSchema = z
+    .object({
+    className: z
+        .string()
+        .describe("The candidate's class attribute (raw, unsplit). Empty string when absent."),
+    id: z.string().describe("The candidate's id attribute, or empty string."),
+    score: z
+        .number()
+        .describe("Readability's actual contentScore for this node (link-density-scaled). Higher is better; the top entry is Readability's raw top candidate before its parent-walking/only-child adjustments."),
+    selector: z
+        .string()
+        .describe("A CSS-ish hint (tag#id.class1.class2) for locating the node in the host DOM. NOT a unique locator — Readability's score lives on a JS expando invisible to CSS."),
+    tag: z
+        .string()
+        .describe('Uppercase DOM tag name (e.g. "ARTICLE", "MAIN", "DIV").'),
+    textLength: z
+        .number()
+        .int()
+        .min(0)
+        .describe('Trimmed textContent length of the candidate node, for eyeballing content density.'),
+})
+    .describe('One scored candidate node Readability considered, with its real contentScore.');
+const explainOutputShape = {
+    schemaVersion: z
+        .literal(1)
+        .describe('Structured-content schema version. Bumps only on breaking shape changes to this object.'),
+    content: z
+        .string()
+        .describe('Readable rendering of the report (chosen root, ranked candidates, removal counts, gating/pagination, snapshot head) so content[0].text is always scannable.'),
+    chosenRoot: candidateSchema
+        .nullable()
+        .describe('The highest-scoring candidate Readability computed — its raw top pick before parent-walking/only-child post-processing. Null when Readability scored nothing (e.g. empty input).'),
+    candidates: z
+        .array(candidateSchema)
+        .describe("Scored candidate nodes (highest first), capped at topN. These are Readability's real contentScore values, not a self-computed heuristic."),
+    readerable: z
+        .boolean()
+        .describe('Readability isProbablyReaderable verdict on the normalized document.'),
+    parseSucceeded: z
+        .boolean()
+        .describe('True when reader.parse() returned article content. False signals that `extract` would fall back to its selector cascade.'),
+    fallbackUsed: z
+        .boolean()
+        .describe('Always false for explain — this tool runs only the Readability path it is diagnosing, never the fallback cascade.'),
+    gating: z
+        .object({
+        likely: z
+            .boolean()
+            .describe('True when heuristics strongly suggest the content is paywalled or truncated.'),
+        reason: z
+            .string()
+            .describe('Short label naming the detected signal (e.g. "paywall overlay").'),
+    })
+        .nullable()
+        .describe('Likely paywall / gating signal detected before normalization. Null when none.'),
+    pagination: z
+        .object({
+        type: z
+            .enum(['infinite', 'paginated'])
+            .describe('Kind of pagination signal detected.'),
+        nextUrl: z
+            .string()
+            .optional()
+            .describe('Absolute URL of the detected next page (paginated only). Never fetched.'),
+        selector: z
+            .string()
+            .optional()
+            .describe('CSS selector of the load-more / infinite-scroll sentinel (infinite only).'),
+    })
+        .nullable()
+        .describe('Detected pagination / infinite-scroll signal. Null when none.'),
+    removedNodes: z
+        .object({
+        boilerplate: z
+            .number()
+            .int()
+            .min(0)
+            .describe('Boilerplate blocks (related-posts, newsletter signup) stripped before Readability.'),
+        chrome: z
+            .number()
+            .int()
+            .min(0)
+            .describe('Browser-chrome nodes stripped (scrollbars, consent banners, overlays).'),
+        total: z
+            .number()
+            .int()
+            .min(0)
+            .describe('Net element count removed across the whole pipeline (delta vs. the parsed document).'),
+    })
+        .describe('Breakdown of nodes removed before Readability saw the document, reused from the extract diagnostics path.'),
+    snapshot: z
+        .object({
+        html: z
+            .string()
+            .describe('The sanitized-by-normalization (post chrome/boilerplate/script strip) HTML fed to Readability — "what Readability saw". Not DOMPurify-sanitized (that runs on Readability\'s output in `extract`), so it may still carry inline event handlers (`onerror`/`onclick`/…); it is diagnostic data — do not render verbatim.'),
+        truncated: z
+            .boolean()
+            .describe('True when the snapshot was cut at snapshotMaxChars (default 4000).'),
+    })
+        .describe('Pre-Readability HTML snapshot of the normalized document body.'),
+};
+const explainOutput = z.object(explainOutputShape);
+function formatGating(report) {
+    const g = report.gating;
+    if (!g) {
+        return 'none';
+    }
+    return `${g.reason}${g.likely ? '' : ' (weak)'}`;
+}
+function formatPagination(report) {
+    const p = report.pagination;
+    if (!p) {
+        return 'none';
+    }
+    if (p.type === 'paginated') {
+        return `paginated -> ${p.nextUrl ?? '(no href)'}`;
+    }
+    return `infinite (${p.selector ?? 'sentinel'})`;
+}
+function renderText(report) {
+    const lines = [];
+    const root = report.chosenRoot;
+    lines.push(`readerable: ${report.readerable ? 'yes' : 'no'}  parse: ${report.parseSucceeded ? 'ok' : 'fail'}  fallback: ${report.fallbackUsed ? 'yes' : 'no'}`);
+    lines.push(`chosen root: ${root ? `${root.selector}  (score ${root.score.toFixed(2)}, ${root.textLength} chars)` : '(no candidate scored)'}`);
+    lines.push(`top candidates (${report.candidates.length}):`);
+    if (report.candidates.length === 0) {
+        lines.push('  (none)');
+    }
+    report.candidates.forEach((c, i) => {
+        lines.push(`  ${i + 1}. ${c.selector}  score ${c.score.toFixed(2)}  (${c.textLength} chars)`);
+    });
+    const r = report.removedNodes;
+    lines.push(`removed: total=${r.total} (chrome=${r.chrome}, boilerplate=${r.boilerplate})`);
+    lines.push(`gating: ${formatGating(report)}`);
+    lines.push(`pagination: ${formatPagination(report)}`);
+    lines.push(`snapshot (${report.snapshot.html.length} chars${report.snapshot.truncated ? ', truncated' : ''}):`);
+    lines.push(report.snapshot.html);
+    return lines.join('\n');
+}
+export function explainFromHtml(input) {
+    const { html, baseUrl, selectors, topN } = withSchemaDefaults(explainInputSchema, input);
+    const report = buildExplainReport({
+        html,
+        selectors,
+        topN,
+        baseUrl,
+    });
+    const content = renderText(report);
+    return {
+        content: [{ text: content, type: 'text' }],
+        structuredContent: {
+            schemaVersion: 1,
+            content,
+            chosenRoot: report.chosenRoot,
+            candidates: report.candidates,
+            readerable: report.readerable,
+            parseSucceeded: report.parseSucceeded,
+            fallbackUsed: report.fallbackUsed,
+            gating: report.gating ?? null,
+            pagination: report.pagination ?? null,
+            removedNodes: report.removedNodes,
+            snapshot: report.snapshot,
+        },
+    };
+}
+export const EXPLAIN_TOOL_DESCRIPTION = `Post-mortem diagnostics for extraction: shows WHY Readability picked what it picked. Returns the chosen root, the ranked candidate nodes with their REAL Readability contentScore values (read off the DOM expando Readability stamps during scoring), a categorized removed-nodes breakdown, gating/pagination signals, and a snapshot of the normalized HTML fed to Readability. Runs the same normalize + Readability pipeline as \`extract\` (no fallback cascade, no Turndown). The server fetches nothing: \`localPath\` is the only source, and \`baseUrl\` (optional) is origin context only.`;
+export const explainHandler = tool('explain', input => explainFromHtml(fromLocalPath(input)), explainInputSchema);
+export function registerExplainTool(server) {
+    return server.registerTool('explain', {
+        title: 'Explain why Readability picked what it picked',
+        description: EXPLAIN_TOOL_DESCRIPTION,
+        inputSchema: explainInputShape,
+        outputSchema: explainOutput,
+    }, explainHandler);
+}