@v1nvn/readability 0.43.0 → 0.45.0

package/README.md

@@ -10,9 +10,9 @@ User-facing docs: [root README](../../README.md).
 ## Install
 
 ```bash
-npm install @v1nvn/readability@0.43.0
+npm install @v1nvn/readability@0.45.0
 # or run on demand:
-npx @v1nvn/readability@0.43.0
+npx @v1nvn/readability@0.45.0
 ```
 
 Requires Node >= 22.22.2 (jsdom 30's floor). Build from source:
@@ -58,7 +58,7 @@ Add to your MCP client config (Claude Code, Claude Desktop, etc.):
   "mcpServers": {
     "readability": {
       "command": "npx",
-      "args": ["-y", "@v1nvn/readability@0.43.0"],
+      "args": ["-y", "@v1nvn/readability@0.45.0"],
     },
   },
 }

package/package.json

@@ -1,15 +1,14 @@
 {
   "name": "@v1nvn/readability",
-  "version": "0.43.0",
+  "version": "0.45.0",
   "description": "MCP server that turns rendered (post-JS) HTML into clean Markdown + metadata via Readability, Turndown, and DOMPurify.",
   "type": "module",
-  "main": "dist/index.js",
-  "bin": "dist/index.js",
+  "main": "./src/index.ts",
+  "bin": "./src/index.ts",
   "scripts": {
-    "build": "vite build",
     "bench": "vite-node test/bench/run.ts",
     "dev": "vite-node src/dev.ts",
-    "start": "node dist/index.js",
+    "start": "node src/index.ts",
     "test": "vitest run",
     "test:watch": "vitest",
     "test:update-goldens": "UPDATE_GOLDENS=1 vitest run",
@@ -33,19 +32,18 @@
     "directory": "packages/readability"
   },
   "files": [
-    "dist"
+    "src"
   ],
   "publishConfig": {
     "access": "public"
   },
   "engines": {
-    "node": ">=22.22.2"
+    "node": ">=23.6"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@mozilla/readability": "^0.6.0",
-    "@v1nvn/agentic-core": "^0.43.0",
-    "commander": "^15.0.0",
+    "@v1nvn/agentic-core": "^0.45.0",
     "dompurify": "^3.4.12",
     "jsdom": "^30.0.1",
     "remark-gfm": "^4.0.1",

package/src/cli.ts

@@ -0,0 +1,134 @@
+import {
+  type CliSpec,
+  describeError,
+  HelpRequested,
+  parseCli,
+  runCli,
+} from '@v1nvn/agentic-core';
+
+import { extractArticleFromHtml } from './tools/extract.ts';
+import { readHtmlFile } from './tools/html-source.ts';
+
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+import type { Readable } from 'node:stream';
+
+type CliFormat = 'html' | 'json' | 'md';
+
+export interface ParsedArgs {
+  readonly file: string | undefined;
+  readonly format: CliFormat;
+  readonly maxChars: number | undefined;
+}
+
+const FORMATS: readonly CliFormat[] = ['html', 'json', 'md'];
+
+function parseMaxChars(value: string): number {
+  const n = Number(value);
+  if (!Number.isInteger(n)) {
+    throw new Error('must be an integer');
+  }
+  return n;
+}
+
+const spec: CliSpec = {
+  name: 'readability',
+  usageExit: 2,
+  commands: [
+    {
+      name: 'extract',
+      description: 'extract the readable article from an HTML file',
+      argument: {
+        flags: '[file]',
+        description: 'HTML file; stdin when omitted',
+      },
+      options: [
+        {
+          flags: '--format <fmt>',
+          description: 'output format',
+          choices: FORMATS,
+          default: 'md',
+        },
+        {
+          flags: '--max-chars <n>',
+          description: 'truncate the output',
+          parse: parseMaxChars,
+        },
+      ],
+      run: async (options, args) => {
+        try {
+          const html = await readHtml(args.at(0), process.stdin);
+          // json reuses the markdown pipeline; the structured object is serialized below.
+          const pipelineFormat =
+            options.format === 'html' ? 'html' : 'markdown';
+          const result = extractArticleFromHtml({
+            html,
+            format: pipelineFormat,
+            ...(typeof options.maxChars === 'number'
+              ? { maxChars: options.maxChars }
+              : {}),
+          });
+
+          if (result.isError) {
+            process.stderr.write(`${payloadText(result)}\n`);
+            return 1;
+          }
+
+          if (options.format === 'json') {
+            process.stdout.write(
+              `${JSON.stringify(result.structuredContent, null, 2)}\n`,
+            );
+          } else {
+            process.stdout.write(`${payloadText(result)}\n`);
+          }
+          return 0;
+        } catch (err) {
+          process.stderr.write(`${describeError(err)}\n`);
+          return 1;
+        }
+      },
+    },
+  ],
+};
+
+export function parseArgs(
+  argv: readonly string[],
+): HelpRequested | ParsedArgs | undefined {
+  const parsed = parseCli(spec, argv);
+  if (parsed === undefined || parsed instanceof HelpRequested) {
+    return parsed;
+  }
+  const { format, maxChars } = parsed.options as {
+    format: CliFormat;
+    maxChars: number | undefined;
+  };
+  return { file: parsed.args.at(0), format, maxChars };
+}
+
+// The stream is injected rather than reading process.stdin directly so the
+// path is testable; chunks may be Buffer or string.
+export async function readHtml(
+  file: string | undefined,
+  stream: Readable,
+): Promise<string> {
+  if (file !== undefined) {
+    return readHtmlFile(file);
+  }
+  const chunks: string[] = [];
+  for await (const chunk of stream) {
+    if (typeof chunk === 'string') {
+      chunks.push(chunk);
+    } else {
+      chunks.push(Buffer.from(chunk as Uint8Array).toString('utf8'));
+    }
+  }
+  return chunks.join('');
+}
+
+function payloadText(result: CallToolResult): string {
+  const first = result.content.at(0);
+  return first !== undefined && 'text' in first ? first.text : '';
+}
+
+export function runExtract(argv: readonly string[]): Promise<number> {
+  return runCli(spec, argv);
+}

package/src/config.ts

@@ -0,0 +1,42 @@
+import pkg from '../package.json' with { type: 'json' };
+
+export interface ServerConfig {
+  readonly description: string;
+  readonly instructions: string;
+  readonly name: 'readability';
+  readonly title: string;
+  readonly version: string;
+}
+
+const SERVER_TITLE = 'Readability MCP';
+
+const SERVER_DESCRIPTION =
+  'Turn already-rendered (post-JavaScript) HTML into clean, LLM-friendly Markdown plus metadata, via Mozilla Readability, Turndown, and DOMPurify. Makes no outbound requests — input is the rendered HTML, read from a file path (localPath) so the page bytes never enter the model context.';
+
+const SERVER_INSTRUCTIONS = `Every tool reads already-rendered HTML from a file path (\`localPath\`) — e.g. \`document.documentElement.outerHTML\` written to disk by a browser/devtools capture — so the page bytes never enter the model context; \`chunk_text\` alone takes text. The server never fetches URLs: \`baseUrl\` only absolutizes links.
+
+Pick the tool by page shape:
+- extract: the default for article-like pages → Markdown + metadata + diagnostics. \`cache: true\` exposes the result as a \`readability://page/{hash}\` resource; \`chunk\` adds token-bounded chunks.
+- extract_list: feed, index, search, or HN-style pages Readability cannot turn into one article.
+- extract_tables: every \`<table>\` on the page, including ones outside the article body.
+- extract_grid: div-rendered tables in SPAs.
+- extract_section: one section, by CSS selector or heading text.
+- extract_links: anchors from the raw DOM, for crawl and navigation decisions.
+- extract_metadata, outline: cheap pre-checks (bibliographic metadata; h1-h6 heading TOC) before full extraction.
+- html_to_markdown: a fragment already isolated, converted with no Readability scoring.
+- explain: post-mortem when \`extract\` picked the wrong root.
+- chunk_text: split already-extracted text for embedding/RAG.
+
+When the client advertises MCP \`sampling\`, two more tools appear: \`summarize\` (the host's model summarizes text) and \`suggest_preset\` (after \`extract\` reports gating, a fallback, a near-empty result, or debris, the host's model proposes site selectors; the server verifies them and caches a per-site preset).
+
+Failures come back as \`isError: true\` results, never thrown.`;
+
+export function loadConfig(): ServerConfig {
+  return {
+    name: 'readability',
+    version: pkg.version,
+    title: SERVER_TITLE,
+    description: SERVER_DESCRIPTION,
+    instructions: SERVER_INSTRUCTIONS,
+  };
+}

package/src/dev.ts

@@ -0,0 +1,77 @@
+// Dev-only hot reload: tool implementations reload on file change without
+// restarting the process; the harness in core owns the vite/runner/watch loop.
+// Never bundled into dist (build entry is src/index.ts). Diagnostics to stderr.
+
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { describeError } from '@v1nvn/agentic-core';
+import { runDevServer } from '@v1nvn/agentic-core/vite';
+import { dirname, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import type { PresetCacheReport } from './preset-cache.ts';
+
+import { logger } from './logger.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+interface Handle {
+  remove(): void;
+}
+
+// Loaded through Vite (not imported statically) so the runner controls its
+// cache and can re-evaluate it on change.
+interface RuntimeModule {
+  createMcpServer(): McpServer;
+  loadPresets(env?: NodeJS.ProcessEnv): PresetCacheReport | undefined;
+  registerCapabilityGatedTools(server: McpServer): Handle[];
+  registerResources(server: McpServer): Handle[];
+  registerTools(server: McpServer): Handle[];
+}
+
+const here = dirname(fileURLToPath(import.meta.url));
+
+runDevServer<RuntimeModule>({
+  entry: resolve(here, 'server.ts'),
+  log: logger,
+  start: async mod => {
+    const server = mod.createMcpServer();
+    mod.loadPresets();
+    let handles = [
+      ...mod.registerTools(server),
+      ...mod.registerResources(server),
+    ];
+    // Capability-gated tools (sampling) defer to the `initialized`
+    // notification: `connect()` returns before the client advertises
+    // capabilities, so a direct call here would observe
+    // `getClientCapabilities() === undefined`. The hook fires once per
+    // connection, after the initialize handshake but before any `tools/list`.
+    let gatedHandles: Handle[] = [];
+    server.server.oninitialized = () => {
+      gatedHandles = mod.registerCapabilityGatedTools(server);
+    };
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    return {
+      closers: [server],
+      reload: next => {
+        for (const handle of [...handles, ...gatedHandles]) {
+          handle.remove();
+        }
+        handles = [
+          ...next.registerTools(server),
+          ...next.registerResources(server),
+        ];
+        // The re-imported module graph starts with an empty preset store;
+        // refill it alongside the re-registrations. The client is already
+        // past `initialized` by the time a file change fires, so the gate
+        // can re-evaluate directly.
+        next.loadPresets();
+        gatedHandles = next.registerCapabilityGatedTools(server);
+      },
+    };
+  },
+}).catch((error: unknown) => {
+  logger.error(`dev failed: ${describeError(error)}`);
+  // eslint-disable-next-line n/no-process-exit
+  process.exit(1);
+});

package/src/errors.ts

@@ -0,0 +1,19 @@
+import { describeError, type ErrorResult } from '@v1nvn/agentic-core';
+
+export class ExtractionError extends Error {
+  public override readonly cause: unknown;
+
+  public constructor(message: string, options: { cause?: unknown } = {}) {
+    super(message);
+    this.name = 'ExtractionError';
+    this.cause = options.cause;
+  }
+}
+
+export function toErrorResult(err: unknown): ErrorResult {
+  const label = err instanceof ExtractionError ? err.name : 'Error';
+  return {
+    isError: true,
+    content: [{ type: 'text', text: `${label}: ${describeError(err)}` }],
+  };
+}

package/src/handler.ts

@@ -0,0 +1,8 @@
+import { toolHandler } from '@v1nvn/agentic-core';
+
+import { toErrorResult } from './errors.ts';
+import { logger } from './logger.ts';
+
+import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';
+
+export const tool = toolHandler<CallToolResult>(logger, toErrorResult);

package/src/host-sampling.ts

@@ -0,0 +1,86 @@
+// The one seam to the HOST's model via MCP `sampling/createMessage`
+// (server→client request). The server never embeds a model and never calls a
+// provider directly — every LLM call is delegated to the connected client,
+// which picks the model and may prompt the user first. Gating on the
+// capability happens at registration (server.ts), not here.
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+// Host models — and the human-in-the-loop approval MCP allows before each
+// call — routinely take longer than the SDK's 60s request default.
+const SAMPLING_TIMEOUT_MS = 300_000;
+
+export interface HostSampleInput {
+  readonly maxTokens: number;
+  readonly systemPrompt: string;
+  readonly userText: string;
+}
+
+export async function sampleText(
+  server: McpServer,
+  args: HostSampleInput,
+): Promise<string> {
+  const result = await server.server.createMessage(
+    {
+      messages: [
+        {
+          role: 'user',
+          content: { type: 'text', text: args.userText },
+        },
+      ],
+      systemPrompt: args.systemPrompt,
+      maxTokens: args.maxTokens,
+    },
+    { timeout: SAMPLING_TIMEOUT_MS },
+  );
+  if (result.content.type !== 'text') {
+    throw new Error(
+      `host sampling returned non-text content (${result.content.type})`,
+    );
+  }
+  return result.content.text;
+}
+
+// createMessage has no structured-output field, so JSON proposals are
+// negotiated in prose: the prompt demands strict JSON, and the reply is
+// tolerated past code fences before the caller's schema takes over.
+export class SuggestParseError extends Error {
+  readonly rawText: string;
+
+  constructor(message: string, rawText: string) {
+    super(message);
+    this.rawText = rawText;
+  }
+}
+
+export function extractJsonReply(text: string): unknown {
+  const withoutFences = text.replace(
+    /^[\s\S]*?```(?:json)?\s*\n?([\s\S]*?)\n?```[\s\S]*$/,
+    '$1',
+  );
+  const trimmed = withoutFences.trim();
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const start = trimmed.indexOf('{');
+    const end = trimmed.lastIndexOf('}');
+    if (start !== -1 && end > start) {
+      try {
+        return JSON.parse(trimmed.slice(start, end + 1));
+      } catch {
+        // Fall through to the error below.
+      }
+    }
+    throw new SuggestParseError(
+      `host sampling returned text that does not parse as JSON (${trimmed.slice(0, 80)}…)`,
+      trimmed,
+    );
+  }
+}
+
+export async function sampleJson(
+  server: McpServer,
+  args: HostSampleInput,
+): Promise<unknown> {
+  return extractJsonReply(await sampleText(server, args));
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { describeError, shutdownOnSignals } from '@v1nvn/agentic-core';
+
+import { createServer } from './server.ts';
+
+if (process.argv[2] === 'extract') {
+  void import('./cli.ts')
+    .then(m => m.runExtract(process.argv.slice(2)))
+    .then(code => {
+      // eslint-disable-next-line n/no-process-exit
+      process.exit(code);
+    })
+    .catch((err: unknown) => {
+      process.stderr.write(`${describeError(err)}\n`);
+      // eslint-disable-next-line n/no-process-exit
+      process.exit(1);
+    });
+} else {
+  const server: McpServer = createServer();
+  const transport = new StdioServerTransport();
+
+  await server.connect(transport);
+
+  shutdownOnSignals([server]);
+}

package/src/logger.ts

@@ -0,0 +1,3 @@
+import { loggerFromEnv } from '@v1nvn/agentic-core';
+
+export const logger = loggerFromEnv(process.env, 'READABILITY_LOG_LEVEL');

package/src/output/format.ts

@@ -0,0 +1,132 @@
+import { stringify as stringifyYaml } from 'yaml';
+
+import type { Diagnostics, Metadata } from '../pipeline/context.ts';
+
+import { headingText, parseBlocks } from '../policy/markdown.ts';
+
+export type Format = 'html' | 'json' | 'markdown' | 'text';
+export type MetadataMode = 'json' | 'none' | 'yaml';
+
+export interface FormatInput {
+  readonly diagnostics: Diagnostics;
+  readonly format: Format;
+  readonly markdown: string;
+  readonly metadata: Metadata;
+  readonly metadataMode: MetadataMode;
+  readonly sanitizedHtml: string;
+  readonly textContent: string;
+}
+
+interface JsonObject {
+  readonly content: string;
+  readonly diagnostics: Diagnostics;
+  readonly metadata: Metadata;
+}
+
+// Readability demotes the article <h1> to <h2> inside `content` and mirrors its
+// text into the title; drop the echoed sub-heading so the title prints once.
+function dropEchoedTitle(body: string, title: string): string {
+  const blocks = parseBlocks(body);
+  if (blocks.length === 0) {
+    return body;
+  }
+  const first = blocks[0];
+  if (
+    first.kind === 'heading' &&
+    headingText(body.slice(first.start, first.end)) === title.trim()
+  ) {
+    return body.slice(first.end).replace(/^\n+/, '');
+  }
+  return body;
+}
+
+function renderMarkdown(input: Readonly<FormatInput>): string {
+  const title = input.metadata.title?.trim();
+  let body = input.markdown;
+  if (title) {
+    body = dropEchoedTitle(body, title);
+    return `# ${title}\n\n${body}`.replace(/\n+$/, '\n');
+  }
+  return body.replace(/\n+$/, '\n');
+}
+
+// Stable key order for deterministic frontmatter. `structured` is deliberately
+// absent: it is a nested object, not a scalar, and would dump a whole JSON-LD
+// graph into frontmatter. It reaches the host via format=json and
+// structuredContent.metadata, which serialize the full metadata object.
+const METADATA_KEYS = [
+  'title',
+  'byline',
+  'siteName',
+  'lang',
+  'publishedTime',
+  'excerpt',
+  'canonical',
+  'baseUrl',
+  'wordCount',
+  'readingTimeMin',
+  'tokenEstimate',
+  'estimator',
+] as const satisfies readonly (keyof Metadata)[];
+
+function pickMetadata(
+  metadata: Readonly<Metadata>,
+): Record<string, number | string> {
+  const picked: Record<string, number | string> = {};
+  for (const key of METADATA_KEYS) {
+    const value = metadata[key];
+    if (value !== undefined) {
+      picked[key] = value;
+    }
+  }
+  return picked;
+}
+
+// lineWidth: 0 disables folding so each field stays on one line.
+function yamlFrontmatter(metadata: Readonly<Metadata>): string {
+  return `---\n${stringifyYaml(pickMetadata(metadata), { lineWidth: 0 })}---\n`;
+}
+
+function jsonFrontmatter(metadata: Readonly<Metadata>): string {
+  return (
+    '```json\n' + JSON.stringify(pickMetadata(metadata), null, 2) + '\n```\n'
+  );
+}
+
+function withFrontmatter(
+  payload: string,
+  mode: MetadataMode,
+  metadata: Readonly<Metadata>,
+): string {
+  if (mode === 'yaml') {
+    return `${yamlFrontmatter(metadata)}${payload}`;
+  }
+  if (mode === 'json') {
+    return `${jsonFrontmatter(metadata)}\n${payload}`;
+  }
+  return payload;
+}
+
+export function formatPayload(input: Readonly<FormatInput>): string {
+  let payload: string;
+  switch (input.format) {
+    case 'html':
+      return input.sanitizedHtml;
+    case 'json': {
+      const body: JsonObject = {
+        metadata: input.metadata,
+        content: input.markdown,
+        diagnostics: input.diagnostics,
+      };
+      return JSON.stringify(body, null, 2);
+    }
+    case 'text':
+      payload = input.textContent;
+      break;
+    case 'markdown':
+    default:
+      payload = renderMarkdown(input);
+      break;
+  }
+  return withFrontmatter(payload, input.metadataMode, input.metadata);
+}

package/src/pipeline/cell-text.ts

@@ -0,0 +1,56 @@
+// Resolve a table/grid cell element to the string that represents it in the
+// matrix. Shared by tables.ts (the <table> serializer) and
+// policy/grid-detector.ts (the div-grid detector) so both paths project a
+// cell the same way: whitespace-normalized visible text, with two fallbacks.
+//
+// 1. Tooltip/badge/aria wrappers carry chrome, not data — they are stripped
+//    from a clone so a label cell reads "Market Capitalization" rather than
+//    concatenating every descendant. But stripping must never empty a cell: a
+//    badge/aria element that holds a cell's only text is data, not chrome, so
+//    when stripping would blank the cell the full text is kept.
+// 2. A cell whose only content is an icon link has no visible text at all; its
+//    value lives in the anchor's href, which is surfaced as a last resort.
+const CELL_CHROME_SELECTOR =
+  '[aria-label], [data-tooltip], [data-toggle="tooltip"], .tooltip, .badge';
+
+function normalize(text: string): string {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+// All non-empty hrefs reachable from the cell, including the cell itself when it
+// is an anchor (a div-grid "cell" can be an <a>). Descendant-only search would
+// miss a link-cell whose element IS the anchor.
+function nonEmptyHrefs(cell: Element): string[] {
+  const anchors: Element[] =
+    cell.tagName === 'A'
+      ? [cell, ...Array.from(cell.querySelectorAll('a[href]'))]
+      : Array.from(cell.querySelectorAll('a[href]'));
+  const hrefs: string[] = [];
+  for (const a of anchors) {
+    const href = (a.getAttribute('href') ?? '').trim();
+    if (href !== '') {
+      hrefs.push(href);
+    }
+  }
+  return hrefs;
+}
+
+export function resolveCellText(cell: Element): string {
+  const full = normalize(cell.textContent);
+  let resolved = full;
+  if (full !== '') {
+    const clone = cell.cloneNode(true) as Element;
+    clone.querySelectorAll(CELL_CHROME_SELECTOR).forEach(el => {
+      el.remove();
+    });
+    const stripped = normalize(clone.textContent);
+    resolved = stripped !== '' ? stripped : full;
+  }
+  if (resolved === '') {
+    const hrefs = nonEmptyHrefs(cell);
+    if (hrefs.length > 0) {
+      return hrefs.join(' ');
+    }
+  }
+  return resolved;
+}