@v1nvn/readability 0.44.0 → 0.45.1

package/src/host-sampling.ts

@@ -0,0 +1,86 @@
+// The one seam to the HOST's model via MCP `sampling/createMessage`
+// (server→client request). The server never embeds a model and never calls a
+// provider directly — every LLM call is delegated to the connected client,
+// which picks the model and may prompt the user first. Gating on the
+// capability happens at registration (server.ts), not here.
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+// Host models — and the human-in-the-loop approval MCP allows before each
+// call — routinely take longer than the SDK's 60s request default.
+const SAMPLING_TIMEOUT_MS = 300_000;
+
+export interface HostSampleInput {
+  readonly maxTokens: number;
+  readonly systemPrompt: string;
+  readonly userText: string;
+}
+
+export async function sampleText(
+  server: McpServer,
+  args: HostSampleInput,
+): Promise<string> {
+  const result = await server.server.createMessage(
+    {
+      messages: [
+        {
+          role: 'user',
+          content: { type: 'text', text: args.userText },
+        },
+      ],
+      systemPrompt: args.systemPrompt,
+      maxTokens: args.maxTokens,
+    },
+    { timeout: SAMPLING_TIMEOUT_MS },
+  );
+  if (result.content.type !== 'text') {
+    throw new Error(
+      `host sampling returned non-text content (${result.content.type})`,
+    );
+  }
+  return result.content.text;
+}
+
+// createMessage has no structured-output field, so JSON proposals are
+// negotiated in prose: the prompt demands strict JSON, and the reply is
+// tolerated past code fences before the caller's schema takes over.
+export class SuggestParseError extends Error {
+  readonly rawText: string;
+
+  constructor(message: string, rawText: string) {
+    super(message);
+    this.rawText = rawText;
+  }
+}
+
+export function extractJsonReply(text: string): unknown {
+  const withoutFences = text.replace(
+    /^[\s\S]*?```(?:json)?\s*\n?([\s\S]*?)\n?```[\s\S]*$/,
+    '$1',
+  );
+  const trimmed = withoutFences.trim();
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    const start = trimmed.indexOf('{');
+    const end = trimmed.lastIndexOf('}');
+    if (start !== -1 && end > start) {
+      try {
+        return JSON.parse(trimmed.slice(start, end + 1));
+      } catch {
+        // Fall through to the error below.
+      }
+    }
+    throw new SuggestParseError(
+      `host sampling returned text that does not parse as JSON (${trimmed.slice(0, 80)}…)`,
+      trimmed,
+    );
+  }
+}
+
+export async function sampleJson(
+  server: McpServer,
+  args: HostSampleInput,
+): Promise<unknown> {
+  return extractJsonReply(await sampleText(server, args));
+}

package/src/index.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { describeError, shutdownOnSignals } from '@v1nvn/agentic-core';
+import { createServer } from "./server.js";
+if (process.argv[2] === 'extract') {
+    void import("./cli.js")
+        .then(m => m.runExtract(process.argv.slice(2)))
+        .then(code => {
+        // eslint-disable-next-line n/no-process-exit
+        process.exit(code);
+    })
+        .catch((err) => {
+        process.stderr.write(`${describeError(err)}\n`);
+        // eslint-disable-next-line n/no-process-exit
+        process.exit(1);
+    });
+}
+else {
+    const server = createServer();
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    shutdownOnSignals([server]);
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+#!/usr/bin/env node
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { describeError, shutdownOnSignals } from '@v1nvn/agentic-core';
+
+import { createServer } from './server.ts';
+
+if (process.argv[2] === 'extract') {
+  void import('./cli.ts')
+    .then(m => m.runExtract(process.argv.slice(2)))
+    .then(code => {
+      // eslint-disable-next-line n/no-process-exit
+      process.exit(code);
+    })
+    .catch((err: unknown) => {
+      process.stderr.write(`${describeError(err)}\n`);
+      // eslint-disable-next-line n/no-process-exit
+      process.exit(1);
+    });
+} else {
+  const server: McpServer = createServer();
+  const transport = new StdioServerTransport();
+
+  await server.connect(transport);
+
+  shutdownOnSignals([server]);
+}

package/src/logger.js

@@ -0,0 +1,2 @@
+import { loggerFromEnv } from '@v1nvn/agentic-core';
+export const logger = loggerFromEnv(process.env, 'READABILITY_LOG_LEVEL');

package/src/logger.ts

@@ -0,0 +1,3 @@
+import { loggerFromEnv } from '@v1nvn/agentic-core';
+
+export const logger = loggerFromEnv(process.env, 'READABILITY_LOG_LEVEL');

package/src/output/format.js

@@ -0,0 +1,92 @@
+import { stringify as stringifyYaml } from 'yaml';
+import { headingText, parseBlocks } from "../policy/markdown.js";
+// Readability demotes the article <h1> to <h2> inside `content` and mirrors its
+// text into the title; drop the echoed sub-heading so the title prints once.
+function dropEchoedTitle(body, title) {
+    const blocks = parseBlocks(body);
+    if (blocks.length === 0) {
+        return body;
+    }
+    const first = blocks[0];
+    if (first.kind === 'heading' &&
+        headingText(body.slice(first.start, first.end)) === title.trim()) {
+        return body.slice(first.end).replace(/^\n+/, '');
+    }
+    return body;
+}
+function renderMarkdown(input) {
+    const title = input.metadata.title?.trim();
+    let body = input.markdown;
+    if (title) {
+        body = dropEchoedTitle(body, title);
+        return `# ${title}\n\n${body}`.replace(/\n+$/, '\n');
+    }
+    return body.replace(/\n+$/, '\n');
+}
+// Stable key order for deterministic frontmatter. `structured` is deliberately
+// absent: it is a nested object, not a scalar, and would dump a whole JSON-LD
+// graph into frontmatter. It reaches the host via format=json and
+// structuredContent.metadata, which serialize the full metadata object.
+const METADATA_KEYS = [
+    'title',
+    'byline',
+    'siteName',
+    'lang',
+    'publishedTime',
+    'excerpt',
+    'canonical',
+    'baseUrl',
+    'wordCount',
+    'readingTimeMin',
+    'tokenEstimate',
+    'estimator',
+];
+function pickMetadata(metadata) {
+    const picked = {};
+    for (const key of METADATA_KEYS) {
+        const value = metadata[key];
+        if (value !== undefined) {
+            picked[key] = value;
+        }
+    }
+    return picked;
+}
+// lineWidth: 0 disables folding so each field stays on one line.
+function yamlFrontmatter(metadata) {
+    return `---\n${stringifyYaml(pickMetadata(metadata), { lineWidth: 0 })}---\n`;
+}
+function jsonFrontmatter(metadata) {
+    return ('```json\n' + JSON.stringify(pickMetadata(metadata), null, 2) + '\n```\n');
+}
+function withFrontmatter(payload, mode, metadata) {
+    if (mode === 'yaml') {
+        return `${yamlFrontmatter(metadata)}${payload}`;
+    }
+    if (mode === 'json') {
+        return `${jsonFrontmatter(metadata)}\n${payload}`;
+    }
+    return payload;
+}
+export function formatPayload(input) {
+    let payload;
+    switch (input.format) {
+        case 'html':
+            return input.sanitizedHtml;
+        case 'json': {
+            const body = {
+                metadata: input.metadata,
+                content: input.markdown,
+                diagnostics: input.diagnostics,
+            };
+            return JSON.stringify(body, null, 2);
+        }
+        case 'text':
+            payload = input.textContent;
+            break;
+        case 'markdown':
+        default:
+            payload = renderMarkdown(input);
+            break;
+    }
+    return withFrontmatter(payload, input.metadataMode, input.metadata);
+}

package/src/output/format.ts

@@ -0,0 +1,132 @@
+import { stringify as stringifyYaml } from 'yaml';
+
+import type { Diagnostics, Metadata } from '../pipeline/context.ts';
+
+import { headingText, parseBlocks } from '../policy/markdown.ts';
+
+export type Format = 'html' | 'json' | 'markdown' | 'text';
+export type MetadataMode = 'json' | 'none' | 'yaml';
+
+export interface FormatInput {
+  readonly diagnostics: Diagnostics;
+  readonly format: Format;
+  readonly markdown: string;
+  readonly metadata: Metadata;
+  readonly metadataMode: MetadataMode;
+  readonly sanitizedHtml: string;
+  readonly textContent: string;
+}
+
+interface JsonObject {
+  readonly content: string;
+  readonly diagnostics: Diagnostics;
+  readonly metadata: Metadata;
+}
+
+// Readability demotes the article <h1> to <h2> inside `content` and mirrors its
+// text into the title; drop the echoed sub-heading so the title prints once.
+function dropEchoedTitle(body: string, title: string): string {
+  const blocks = parseBlocks(body);
+  if (blocks.length === 0) {
+    return body;
+  }
+  const first = blocks[0];
+  if (
+    first.kind === 'heading' &&
+    headingText(body.slice(first.start, first.end)) === title.trim()
+  ) {
+    return body.slice(first.end).replace(/^\n+/, '');
+  }
+  return body;
+}
+
+function renderMarkdown(input: Readonly<FormatInput>): string {
+  const title = input.metadata.title?.trim();
+  let body = input.markdown;
+  if (title) {
+    body = dropEchoedTitle(body, title);
+    return `# ${title}\n\n${body}`.replace(/\n+$/, '\n');
+  }
+  return body.replace(/\n+$/, '\n');
+}
+
+// Stable key order for deterministic frontmatter. `structured` is deliberately
+// absent: it is a nested object, not a scalar, and would dump a whole JSON-LD
+// graph into frontmatter. It reaches the host via format=json and
+// structuredContent.metadata, which serialize the full metadata object.
+const METADATA_KEYS = [
+  'title',
+  'byline',
+  'siteName',
+  'lang',
+  'publishedTime',
+  'excerpt',
+  'canonical',
+  'baseUrl',
+  'wordCount',
+  'readingTimeMin',
+  'tokenEstimate',
+  'estimator',
+] as const satisfies readonly (keyof Metadata)[];
+
+function pickMetadata(
+  metadata: Readonly<Metadata>,
+): Record<string, number | string> {
+  const picked: Record<string, number | string> = {};
+  for (const key of METADATA_KEYS) {
+    const value = metadata[key];
+    if (value !== undefined) {
+      picked[key] = value;
+    }
+  }
+  return picked;
+}
+
+// lineWidth: 0 disables folding so each field stays on one line.
+function yamlFrontmatter(metadata: Readonly<Metadata>): string {
+  return `---\n${stringifyYaml(pickMetadata(metadata), { lineWidth: 0 })}---\n`;
+}
+
+function jsonFrontmatter(metadata: Readonly<Metadata>): string {
+  return (
+    '```json\n' + JSON.stringify(pickMetadata(metadata), null, 2) + '\n```\n'
+  );
+}
+
+function withFrontmatter(
+  payload: string,
+  mode: MetadataMode,
+  metadata: Readonly<Metadata>,
+): string {
+  if (mode === 'yaml') {
+    return `${yamlFrontmatter(metadata)}${payload}`;
+  }
+  if (mode === 'json') {
+    return `${jsonFrontmatter(metadata)}\n${payload}`;
+  }
+  return payload;
+}
+
+export function formatPayload(input: Readonly<FormatInput>): string {
+  let payload: string;
+  switch (input.format) {
+    case 'html':
+      return input.sanitizedHtml;
+    case 'json': {
+      const body: JsonObject = {
+        metadata: input.metadata,
+        content: input.markdown,
+        diagnostics: input.diagnostics,
+      };
+      return JSON.stringify(body, null, 2);
+    }
+    case 'text':
+      payload = input.textContent;
+      break;
+    case 'markdown':
+    default:
+      payload = renderMarkdown(input);
+      break;
+  }
+  return withFrontmatter(payload, input.metadataMode, input.metadata);
+}

package/src/pipeline/cell-text.js

@@ -0,0 +1,51 @@
+// Resolve a table/grid cell element to the string that represents it in the
+// matrix. Shared by tables.ts (the <table> serializer) and
+// policy/grid-detector.ts (the div-grid detector) so both paths project a
+// cell the same way: whitespace-normalized visible text, with two fallbacks.
+//
+// 1. Tooltip/badge/aria wrappers carry chrome, not data — they are stripped
+//    from a clone so a label cell reads "Market Capitalization" rather than
+//    concatenating every descendant. But stripping must never empty a cell: a
+//    badge/aria element that holds a cell's only text is data, not chrome, so
+//    when stripping would blank the cell the full text is kept.
+// 2. A cell whose only content is an icon link has no visible text at all; its
+//    value lives in the anchor's href, which is surfaced as a last resort.
+const CELL_CHROME_SELECTOR = '[aria-label], [data-tooltip], [data-toggle="tooltip"], .tooltip, .badge';
+function normalize(text) {
+    return text.replace(/\s+/g, ' ').trim();
+}
+// All non-empty hrefs reachable from the cell, including the cell itself when it
+// is an anchor (a div-grid "cell" can be an <a>). Descendant-only search would
+// miss a link-cell whose element IS the anchor.
+function nonEmptyHrefs(cell) {
+    const anchors = cell.tagName === 'A'
+        ? [cell, ...Array.from(cell.querySelectorAll('a[href]'))]
+        : Array.from(cell.querySelectorAll('a[href]'));
+    const hrefs = [];
+    for (const a of anchors) {
+        const href = (a.getAttribute('href') ?? '').trim();
+        if (href !== '') {
+            hrefs.push(href);
+        }
+    }
+    return hrefs;
+}
+export function resolveCellText(cell) {
+    const full = normalize(cell.textContent);
+    let resolved = full;
+    if (full !== '') {
+        const clone = cell.cloneNode(true);
+        clone.querySelectorAll(CELL_CHROME_SELECTOR).forEach(el => {
+            el.remove();
+        });
+        const stripped = normalize(clone.textContent);
+        resolved = stripped !== '' ? stripped : full;
+    }
+    if (resolved === '') {
+        const hrefs = nonEmptyHrefs(cell);
+        if (hrefs.length > 0) {
+            return hrefs.join(' ');
+        }
+    }
+    return resolved;
+}

package/src/pipeline/cell-text.ts

@@ -0,0 +1,56 @@
+// Resolve a table/grid cell element to the string that represents it in the
+// matrix. Shared by tables.ts (the <table> serializer) and
+// policy/grid-detector.ts (the div-grid detector) so both paths project a
+// cell the same way: whitespace-normalized visible text, with two fallbacks.
+//
+// 1. Tooltip/badge/aria wrappers carry chrome, not data — they are stripped
+//    from a clone so a label cell reads "Market Capitalization" rather than
+//    concatenating every descendant. But stripping must never empty a cell: a
+//    badge/aria element that holds a cell's only text is data, not chrome, so
+//    when stripping would blank the cell the full text is kept.
+// 2. A cell whose only content is an icon link has no visible text at all; its
+//    value lives in the anchor's href, which is surfaced as a last resort.
+const CELL_CHROME_SELECTOR =
+  '[aria-label], [data-tooltip], [data-toggle="tooltip"], .tooltip, .badge';
+
+function normalize(text: string): string {
+  return text.replace(/\s+/g, ' ').trim();
+}
+
+// All non-empty hrefs reachable from the cell, including the cell itself when it
+// is an anchor (a div-grid "cell" can be an <a>). Descendant-only search would
+// miss a link-cell whose element IS the anchor.
+function nonEmptyHrefs(cell: Element): string[] {
+  const anchors: Element[] =
+    cell.tagName === 'A'
+      ? [cell, ...Array.from(cell.querySelectorAll('a[href]'))]
+      : Array.from(cell.querySelectorAll('a[href]'));
+  const hrefs: string[] = [];
+  for (const a of anchors) {
+    const href = (a.getAttribute('href') ?? '').trim();
+    if (href !== '') {
+      hrefs.push(href);
+    }
+  }
+  return hrefs;
+}
+
+export function resolveCellText(cell: Element): string {
+  const full = normalize(cell.textContent);
+  let resolved = full;
+  if (full !== '') {
+    const clone = cell.cloneNode(true) as Element;
+    clone.querySelectorAll(CELL_CHROME_SELECTOR).forEach(el => {
+      el.remove();
+    });
+    const stripped = normalize(clone.textContent);
+    resolved = stripped !== '' ? stripped : full;
+  }
+  if (resolved === '') {
+    const hrefs = nonEmptyHrefs(cell);
+    if (hrefs.length > 0) {
+      return hrefs.join(' ');
+    }
+  }
+  return resolved;
+}

package/src/pipeline/context.js

@@ -0,0 +1 @@
+export {};

package/src/pipeline/context.ts

@@ -0,0 +1,88 @@
+import type { ReadabilityParseResult } from './readability.ts';
+
+interface GatingSignal {
+  readonly likely: boolean;
+  readonly reason: string;
+}
+
+interface PaginationSignal {
+  readonly nextUrl?: string;
+  readonly selector?: string;
+  readonly type: 'infinite' | 'paginated';
+}
+
+interface PresetSignal {
+  readonly applied: boolean;
+  readonly reason?: 'detectors-missed' | 'overridden';
+  readonly site: string;
+}
+
+export interface StructuredData {
+  readonly '@type'?: string;
+  readonly [key: string]: unknown;
+}
+
+export interface Metadata {
+  readonly baseUrl?: string;
+  readonly byline?: string;
+  readonly canonical?: string;
+  readonly estimator?: string;
+  readonly excerpt?: string;
+  readonly lang?: string;
+  readonly publishedTime?: string;
+  readonly readingTimeMin?: number;
+  readonly siteName?: string;
+  readonly structured?: StructuredData;
+  readonly title?: string;
+  readonly tokenEstimate?: number;
+  readonly wordCount?: number;
+}
+
+export interface SanitizationDiagnostics {
+  readonly iframes: number;
+  readonly scripts: number;
+}
+
+export interface TraceStage {
+  readonly ms: number;
+  readonly stage: string;
+}
+
+export interface CacheSignal {
+  readonly hit: boolean;
+  readonly normalizedHash: string;
+  readonly originalHash: string;
+}
+
+export interface Diagnostics {
+  readonly boilerplateRemoved?: number;
+  readonly cache?: CacheSignal;
+  readonly chromeRemoved?: number;
+  readonly extractedNode?: string;
+  readonly fallbackUsed: boolean;
+  readonly gated?: GatingSignal;
+  readonly imagesResolved?: number;
+  readonly pagination?: PaginationSignal;
+  readonly preset?: PresetSignal;
+  readonly readerable?: boolean;
+  readonly removedNodes?: number;
+  readonly sanitization?: SanitizationDiagnostics;
+  readonly trace?: readonly TraceStage[];
+  readonly truncated: boolean;
+}
+
+export interface ExtractionContext {
+  readonly article?: null | Readonly<ReadabilityParseResult>;
+  readonly baseUrl?: string;
+  readonly diagnostics?: Diagnostics;
+  readonly document?: Document;
+  readonly documentElementCount?: number;
+  readonly html: string;
+  readonly markdown?: string;
+  readonly metadata?: Metadata;
+  readonly readerable?: boolean;
+  readonly rootSelector?: string;
+  readonly sanitizedHtml?: string;
+  // Must be the same window the document came from — DOMPurify is bound to it.
+  readonly window?: Window;
+}

package/src/pipeline/dom.js

@@ -0,0 +1,16 @@
+import { JSDOM } from 'jsdom';
+// Built only from caller-supplied `html`; `baseUrl` is origin context for
+// absolutization, never fetched. Parse and the downstream Readability walk run
+// synchronously on the event loop — no worker isolation or wall-clock timeout
+// guards them. A worker pool was deferred rather than rejected: the synchronous
+// parse can't be interrupted in-process, and `worker_threads` can't load the dev
+// hot-reload loop's in-memory Vite modules. `maxNodes` is the only input-size
+// bound until a holistic worker strategy lands; a standalone `timeout` option
+// would mislead (it can't interrupt the parse), so it ships with the worker.
+export function buildDocument(html, baseUrl) {
+    const dom = new JSDOM(html, { url: baseUrl });
+    return { document: dom.window.document, window: dom.window };
+}
+export function isElement(node) {
+    return node.nodeType === 1;
+}

package/src/pipeline/dom.ts

@@ -0,0 +1,23 @@
+import { JSDOM } from 'jsdom';
+
+export interface BuiltDocument {
+  readonly document: Document;
+  readonly window: Window;
+}
+
+// Built only from caller-supplied `html`; `baseUrl` is origin context for
+// absolutization, never fetched. Parse and the downstream Readability walk run
+// synchronously on the event loop — no worker isolation or wall-clock timeout
+// guards them. A worker pool was deferred rather than rejected: the synchronous
+// parse can't be interrupted in-process, and `worker_threads` can't load the dev
+// hot-reload loop's in-memory Vite modules. `maxNodes` is the only input-size
+// bound until a holistic worker strategy lands; a standalone `timeout` option
+// would mislead (it can't interrupt the parse), so it ships with the worker.
+export function buildDocument(html: string, baseUrl?: string): BuiltDocument {
+  const dom = new JSDOM(html, { url: baseUrl });
+  return { document: dom.window.document, window: dom.window };
+}
+
+export function isElement(node: Node): node is Element {
+  return node.nodeType === 1;
+}