@v1nvn/readability 0.44.0 → 0.45.0

package/src/policy/truncate.ts

@@ -0,0 +1,59 @@
+import { hardSplitLines, parseBlocks } from './markdown.ts';
+
+export interface TruncateResult {
+  readonly text: string;
+  readonly truncated: boolean;
+}
+
+const TRUNCATION_MARKER = '\n\n…[truncated]';
+
+// Cuts at a top-level block boundary. A fenced code block is a single mdast
+// node, so a boundary cut never lands inside one — no open-fence rollback is
+// needed. The kept span is sliced from the original string so the payload is a
+// byte-for-byte prefix of the Turndown output.
+export function truncateMarkdown(
+  markdown: string,
+  maxChars: number,
+): TruncateResult {
+  if (markdown.length <= maxChars) {
+    return { text: markdown, truncated: false };
+  }
+
+  const blocks = parseBlocks(markdown);
+  let start = -1;
+  let end = -1;
+  let truncated = false;
+  for (const block of blocks) {
+    const from = start === -1 ? block.start : start;
+    if (block.end - from > maxChars) {
+      // First non-code block alone overflows: take a line-aware prefix instead
+      // of dropping everything. A code block is never split — a half-open fence
+      // is unrecoverable — so an oversized first code block stays dropped.
+      if (start === -1 && block.kind !== 'code') {
+        const firstPiece =
+          hardSplitLines(markdown.slice(block.start, block.end), maxChars)[0] ??
+          '';
+        return {
+          text: firstPiece.replace(/\s+$/, '') + TRUNCATION_MARKER,
+          truncated: true,
+        };
+      }
+      truncated = true;
+      break;
+    }
+    if (start === -1) {
+      start = block.start;
+    }
+    end = block.end;
+  }
+
+  if (!truncated) {
+    return { text: markdown, truncated: false };
+  }
+
+  const kept = start === -1 ? '' : markdown.slice(start, end);
+  return {
+    text: kept.replace(/\s+$/, '') + TRUNCATION_MARKER,
+    truncated: true,
+  };
+}

package/src/preset-cache.ts

@@ -0,0 +1,221 @@
+// Disk side of the preset store: one <site>.json file per preset, read into
+// the in-memory store at boot. Local to the user's machine, ordinary files;
+// the server never fetches and nothing expires by clock — staleness is
+// detector-checked per page (policy/presets.ts), so eviction here is disk
+// hygiene only.
+
+import { describeError } from '@v1nvn/agentic-core';
+import {
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { z } from 'zod';
+
+import type { SitePreset } from './policy/presets.ts';
+
+import { logger } from './logger.ts';
+import { addPreset, normalizeSiteKey } from './policy/presets.ts';
+
+export const PRESETS_DIR_ENV = 'READABILITY_PRESETS_DIR';
+
+export const MAX_PRESET_FILES = 64;
+
+// An empty scope applies nothing while reporting applied:true, and detectors
+// only mean something as a non-empty set — reject both at the file boundary.
+const presetFileSchema = z.object({
+  site: z.string().min(1),
+  detectors: z.array(z.string().min(1)).min(1),
+  scope: z
+    .object({
+      include: z.string().min(1).optional(),
+      exclude: z.array(z.string().min(1)).min(1).optional(),
+    })
+    .refine(
+      scope => scope.include !== undefined || scope.exclude !== undefined,
+      {
+        message: 'scope must carry include or exclude',
+      },
+    ),
+});
+
+export interface PresetCacheReport {
+  readonly loaded: number;
+  readonly pruned: number;
+  readonly skipped: number;
+}
+
+// Empty string disables preset loading; an absolute path wins over the
+// platform default.
+export function resolvePresetsDir(
+  env: NodeJS.ProcessEnv = process.env,
+): string | undefined {
+  const override = env[PRESETS_DIR_ENV];
+  if (override !== undefined) {
+    return override.trim() === '' ? undefined : override;
+  }
+  const root =
+    env.XDG_CACHE_HOME ||
+    join(
+      homedir(),
+      process.platform === 'darwin' ? 'Library/Caches' : '.cache',
+    );
+  return join(root, 'readability', 'presets');
+}
+
+// A preset that cannot load is a warning, never a boot failure — the pipeline
+// is designed to run without one.
+export function loadPresetDir(dir: string): PresetCacheReport {
+  let names: string[];
+  try {
+    names = readdirSync(dir)
+      .filter(name => name.endsWith('.json'))
+      .sort();
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') {
+      logger.debug(`no preset directory at ${dir}`);
+    } else {
+      logger.warn(`preset directory unreadable: ${dir}`);
+    }
+    return { loaded: 0, pruned: 0, skipped: 0 };
+  }
+
+  const ranked = rankPresetFiles(dir, names);
+
+  const pruned = pruneRanked(dir, ranked.slice(MAX_PRESET_FILES));
+
+  let loaded = 0;
+  let skipped = 0;
+  for (const { name } of ranked.slice(0, MAX_PRESET_FILES)) {
+    const path = join(dir, name);
+    if (loadPresetFile(path)) {
+      loaded++;
+    } else {
+      skipped++;
+    }
+  }
+  if (loaded > 0) {
+    logger.info(`loaded ${loaded} site preset(s) from ${dir}`);
+  }
+  if (pruned > 0) {
+    logger.info(
+      `pruned ${pruned} preset file(s) beyond the ${MAX_PRESET_FILES}-file bound`,
+    );
+  }
+  return { loaded, pruned, skipped };
+}
+
+// Env-resolving entry point for the boot paths; undefined means disabled.
+export function loadPresets(
+  env: NodeJS.ProcessEnv = process.env,
+): PresetCacheReport | undefined {
+  const dir = resolvePresetsDir(env);
+  return dir ? loadPresetDir(dir) : undefined;
+}
+
+function rankPresetFiles(
+  dir: string,
+  names: string[],
+): { mtimeMs: number; name: string }[] {
+  const ranked: { mtimeMs: number; name: string }[] = [];
+  for (const name of names) {
+    try {
+      ranked.push({ name, mtimeMs: statSync(join(dir, name)).mtimeMs });
+    } catch {
+      // Vanished between readdir and stat — nothing to load or prune.
+    }
+  }
+  ranked.sort((a, b) => b.mtimeMs - a.mtimeMs);
+  return ranked;
+}
+
+function pruneRanked(
+  dir: string,
+  ranked: { mtimeMs: number; name: string }[],
+): number {
+  let pruned = 0;
+  for (const { name } of ranked) {
+    try {
+      unlinkSync(join(dir, name));
+      pruned++;
+    } catch (err) {
+      logger.warn(`could not prune preset file ${name}: ${describeError(err)}`);
+    }
+  }
+  return pruned;
+}
+
+export interface PersistReport {
+  readonly path?: string;
+  readonly persisted: boolean;
+  readonly reason?: string;
+}
+
+// The writer the loader was waiting for: the suggest loop saves an accepted
+// preset so the next server start loads it like any hand-placed file. A
+// failure leaves the in-memory preset working — persistence is best-effort,
+// exactly like loading.
+export function savePreset(
+  preset: SitePreset,
+  env: NodeJS.ProcessEnv = process.env,
+): PersistReport {
+  const dir = resolvePresetsDir(env);
+  if (!dir) {
+    return { persisted: false, reason: 'preset-directory-disabled' };
+  }
+  const shape = presetFileSchema.safeParse(preset);
+  if (!shape.success) {
+    return {
+      persisted: false,
+      reason: shape.error.issues[0]?.message ?? 'invalid preset shape',
+    };
+  }
+  const key = normalizeSiteKey(preset.site);
+  if (!key) {
+    return { persisted: false, reason: 'site-is-not-a-hostname' };
+  }
+  const path = join(dir, `${key}.json`);
+  try {
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(path, `${JSON.stringify(shape.data, null, 2)}\n`);
+  } catch (err) {
+    return {
+      persisted: false,
+      reason: describeError(err),
+    };
+  }
+  const over = rankPresetFiles(
+    dir,
+    readdirSync(dir).filter(name => name.endsWith('.json')),
+  );
+  pruneRanked(dir, over.slice(MAX_PRESET_FILES));
+  return { path, persisted: true };
+}
+
+function loadPresetFile(path: string): boolean {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(readFileSync(path, 'utf8')) as unknown;
+  } catch (err) {
+    logger.warn(
+      `${path}: not readable JSON (${describeError(err)}), preset skipped`,
+    );
+    return false;
+  }
+  const result = presetFileSchema.safeParse(parsed);
+  if (!result.success) {
+    logger.warn(`${path}: ${result.error.issues[0]?.message}, preset skipped`);
+    return false;
+  }
+  if (!normalizeSiteKey(result.data.site)) {
+    logger.warn(`${path}: site is not a hostname, preset skipped`);
+    return false;
+  }
+  addPreset(result.data);
+  return true;
+}

package/src/resources.ts

@@ -0,0 +1,317 @@
+// In-memory cache of extract results keyed by a volatility-normalized hash of
+// the HTML plus an args fingerprint, exposed to clients as addressable MCP
+// Resources (`readability://page/{hash}`). Re-renders that differ only in
+// nonce/CSP/generated ids collapse to the same key; the original hash rides
+// alongside each entry so a should-have-hit-but-didn't miss points at a
+// normalizer bug rather than a genuinely different page.
+
+import { ResourceTemplate } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { createHash } from 'node:crypto';
+
+import type { ToolHandle } from './server.ts';
+import type { ExtractFromHtmlInput } from './tools/schemas.ts';
+
+import { presetGeneration } from './policy/presets.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type {
+  ListResourcesResult,
+  ReadResourceResult,
+} from '@modelcontextprotocol/sdk/types.js';
+
+const MAX_ENTRIES = 256;
+const TTL_MS = 30 * 60 * 1000;
+const RESOURCE_SCHEME = 'readability://page/';
+
+interface CacheEntry {
+  readonly argsFingerprint: string;
+  readonly cacheKey: string;
+  readonly contentText: string;
+  readonly expiresAt: number;
+  readonly normalizedHash: string;
+  readonly originalHash: string;
+  // Held opaquely: cloned on hit and returned as CallToolResult.structuredContent
+  // without re-validation, so a wider element type than the strict output schema
+  // (e.g. readonly trace arrays) is fine here.
+  readonly structuredContent: unknown;
+}
+
+interface CacheLookup {
+  readonly entry: CacheEntry;
+  readonly normalizedHash: string;
+  readonly originalHash: string;
+}
+
+const entries = new Map<string, CacheEntry>();
+
+export function resetCache(): void {
+  entries.clear();
+}
+
+function evictExpired(now: number): void {
+  for (const [key, entry] of entries) {
+    if (entry.expiresAt <= now) {
+      entries.delete(key);
+    }
+  }
+}
+
+// Bounded LRU: Map preserves insertion order, so re-insert on access to surface
+// recently-used entries to the end and drop the oldest when over capacity.
+function touch(key: string, entry: CacheEntry): void {
+  entries.delete(key);
+  entries.set(key, entry);
+}
+
+function pruneToMax(): void {
+  while (entries.size > MAX_ENTRIES) {
+    const oldest = entries.keys().next();
+    if (oldest.done) {
+      break;
+    }
+    entries.delete(oldest.value);
+  }
+}
+
+// --- Hashing ---------------------------------------------------------------
+
+// Conservative volatility normalization — over-stripping collides distinct
+// pages. Each rule targets a specific known-volatile source.
+function normalizeForHash(html: string): string {
+  let s = html;
+  // Non-JSON-LD scripts carry CSP nonces, build hashes, A/B test buckets —
+  // same page, different bytes on every render — so strip them. Preserve
+  // <script type="application/ld+json">: structured metadata is content, so
+  // a changed datePublished must bust the cache. Mirrors the extraction
+  // normalizer in pipeline/normalize.ts.
+  s = s.replace(
+    /<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*>[\s\S]*?<\/script>/gi,
+    '',
+  );
+  s = s.replace(
+    /<script\b(?![^>]*type\s*=\s*["']application\/ld\+json["'])[^>]*\/?>/gi,
+    '',
+  );
+  // CSP <meta http-equiv="Content-Security-Policy"> carries the per-response
+  // nonce directive — strip the whole tag.
+  s = s.replace(
+    /<meta\b[^>]*http-equiv=["']?content-security-policy["']?[^>]*>/gi,
+    '',
+  );
+  // Per-render nonce attributes on any tag (script, link, style, …).
+  s = s.replace(/\snonce\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/gi, '');
+  // Build-tool generated attribute names: Vue `data-v-1a2b3c4d`,
+  // CSS-modules `data-css-1a2b3c4d`, Svelte `data-svelte-1a2b3c4d`. The
+  // value-part is optional — Vue's compiler emits the bare `data-v-<hash>`
+  // form with no `="..."`.
+  s = s.replace(
+    /\sdata-(?:v|css|svelte|h)-[a-z0-9]{6,}(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]+))?/gi,
+    '',
+  );
+  // React useId / RSC / Next.js generated ids: id=":R1:", ':r1:',
+  // __next_internal_action_…, react internal identifiers.
+  s = s.replace(
+    /\sid\s*=\s*(":[A-Za-z0-9_-]+:"|':[A-Za-z0-9_-]+:'|__next_[A-Za-z0-9_-]+)/g,
+    '',
+  );
+  s = s.replace(
+    /\sid\s*=\s*("react[A-Z]_[A-Za-z0-9_]+"|'react[A-Z]_[A-Za-z0-9_]+')/gi,
+    '',
+  );
+  // Whitespace runs and indentation differ by minifier/pretty-printer.
+  s = s.replace(/\s+/g, ' ');
+  return s.trim();
+}
+
+function sha256(text: string): string {
+  return createHash('sha256').update(text).digest('hex');
+}
+
+export function normalizedHashOf(html: string): string {
+  return sha256(normalizeForHash(html));
+}
+
+export function originalHashOf(html: string): string {
+  return sha256(html.trim());
+}
+
+// Output-affecting options participate in the fingerprint so the same page with
+// format:'html' vs format:'markdown' does not collide. `baseUrl` absolutizes
+// links in the output, so it must participate too. The preset generation rides
+// in the fingerprint rather than the resolved scope: an entry extracted before
+// a preset landed must never be served after it, and hashing the scope would
+// need a second document parse while conflating applied and missed runs under
+// one key.
+function buildArgsFingerprint(args: ExtractFromHtmlInput): string {
+  const sel = args.selectors
+    ? {
+        exclude: args.selectors.exclude ?? [],
+        include: args.selectors.include ?? '',
+      }
+    : null;
+  const chunk = args.chunk
+    ? {
+        maxTokens: args.chunk.maxTokens,
+        overlap: args.chunk.overlap,
+        strategy: args.chunk.strategy,
+      }
+    : null;
+  const fp = {
+    chunk,
+    cleanChrome: args.cleanChrome,
+    codeBlockStyle: args.codeBlockStyle,
+    debug: args.debug,
+    extraction: args.extraction,
+    format: args.format,
+    gfm: args.gfm,
+    headingStyle: args.headingStyle,
+    imageInventory: args.imageInventory,
+    images: args.images,
+    keepClasses: args.keepClasses,
+    maxChars: args.maxChars ?? null,
+    maxNodes: args.maxNodes ?? null,
+    metadataMode: args.metadataMode,
+    minArticleLength: args.minArticleLength ?? null,
+    presetGeneration: presetGeneration(),
+    readabilityOverrides: args.readabilityOverrides ?? null,
+    sanitize: args.sanitize,
+    selectors: sel,
+    tables: args.tables ?? null,
+    baseUrl: args.baseUrl ?? null,
+    wordsPerMinute: args.wordsPerMinute,
+  };
+  return JSON.stringify(fp);
+}
+
+function computeHashes(html: string): {
+  normalizedHash: string;
+  originalHash: string;
+} {
+  return {
+    normalizedHash: normalizedHashOf(html),
+    originalHash: originalHashOf(html),
+  };
+}
+
+function combineKey(normalizedHash: string, argsFingerprint: string): string {
+  return sha256(`${normalizedHash}:${argsFingerprint}`);
+}
+
+// --- Store API -------------------------------------------------------------
+
+export function lookup(
+  html: string,
+  args: ExtractFromHtmlInput,
+): CacheLookup | undefined {
+  const now = Date.now();
+  evictExpired(now);
+  const { normalizedHash, originalHash } = computeHashes(html);
+  const cacheKey = combineKey(normalizedHash, buildArgsFingerprint(args));
+  const entry = entries.get(cacheKey);
+  if (!entry || entry.expiresAt <= now) {
+    if (entry) {
+      entries.delete(cacheKey);
+    }
+    return undefined;
+  }
+  touch(cacheKey, entry);
+  return { entry, normalizedHash, originalHash };
+}
+
+export function storeResult(
+  html: string,
+  args: ExtractFromHtmlInput,
+  result: {
+    contentText: string;
+    structuredContent: unknown;
+  },
+): { normalizedHash: string; originalHash: string } {
+  const now = Date.now();
+  evictExpired(now);
+  const { normalizedHash, originalHash } = computeHashes(html);
+  const argsFingerprint = buildArgsFingerprint(args);
+  const cacheKey = combineKey(normalizedHash, argsFingerprint);
+  const entry: CacheEntry = {
+    argsFingerprint,
+    cacheKey,
+    contentText: result.contentText,
+    expiresAt: now + TTL_MS,
+    normalizedHash,
+    originalHash,
+    structuredContent: result.structuredContent,
+  };
+  entries.set(cacheKey, entry);
+  pruneToMax();
+  return { normalizedHash, originalHash };
+}
+
+export function getEntryByHash(hash: string): CacheEntry | undefined {
+  for (const entry of entries.values()) {
+    if (entry.normalizedHash === hash || entry.cacheKey === hash) {
+      return entry;
+    }
+  }
+  return undefined;
+}
+
+export function listEntries(): readonly CacheEntry[] {
+  const now = Date.now();
+  evictExpired(now);
+  return [...entries.values()];
+}
+
+// --- Resource registration -------------------------------------------------
+
+// "page-cache" names the registered template; the readability:// URI scheme
+// identifies the cache namespace and {hash} is the combined cache key.
+const PAGE_CACHE_TEMPLATE = new ResourceTemplate('readability://page/{hash}', {
+  // Dynamic set: the cache comes and goes, so we enumerate current entries.
+  list: (): ListResourcesResult => ({
+    resources: listEntries().map(entry => ({
+      description: `Cached extract for normalized hash ${entry.normalizedHash.slice(0, 12)}…`,
+      mimeType: 'text/markdown',
+      name: entry.normalizedHash,
+      uri: `${RESOURCE_SCHEME}${entry.cacheKey}`,
+    })),
+  }),
+});
+
+export function registerCacheResources(server: McpServer): ToolHandle {
+  return server.registerResource(
+    'page-cache',
+    PAGE_CACHE_TEMPLATE,
+    {
+      title: 'Cached page extractions',
+      description:
+        'Addressable cache of `extract` results called with cache:true. Each entry is keyed by the volatility-normalized hash of the HTML plus the output options; the URI path segment is the combined cache key.',
+      mimeType: 'text/markdown',
+    },
+    (uri, variables): ReadResourceResult => {
+      const hash = String(variables.hash);
+      const entry = getEntryByHash(hash);
+      if (!entry) {
+        return {
+          contents: [
+            {
+              uri: uri.href,
+              text: '',
+            },
+          ],
+        };
+      }
+      return {
+        contents: [
+          {
+            uri: uri.href,
+            mimeType: 'text/markdown',
+            text: entry.contentText,
+          },
+        ],
+      };
+    },
+  );
+}
+
+export function registerResources(server: McpServer): ToolHandle[] {
+  return [registerCacheResources(server)];
+}

package/src/sampling.ts

@@ -0,0 +1,74 @@
+// Optional, capability-gated tools backed by the HOST's model. The sampling
+// seam itself lives in host-sampling.ts; this file owns the family's tool
+// registrations.
+
+import { z } from 'zod';
+
+import type { ToolHandle } from './server.ts';
+
+import { tool } from './handler.ts';
+import { sampleText } from './host-sampling.ts';
+import { registerSuggestPresetTool } from './tools/suggest-preset.ts';
+
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+const SUMMARIZE_SYSTEM_PROMPT =
+  'Summarize the user-supplied text concisely while preserving its key points, entities, and any decisive conclusions. Output only the summary prose — no preamble, no headings unless the source had them.';
+
+const summarizeInputShape = {
+  text: z
+    .string()
+    .describe(
+      'The markdown or text to summarize — typically the output of `extract`, `extract_section`, `html_to_markdown`, or `chunk_text`. Passed through to the host model verbatim; the server does not parse or modify it.',
+    ),
+  maxTokens: z
+    .number()
+    .int()
+    .min(1)
+    .describe(
+      'Ceiling on tokens the host model may generate, forwarded as `sampling/createMessage` maxTokens. Thinking counts toward it on thinking models, so a low value truncates the summary; the system prompt already asks for a concise one.',
+    )
+    .default(4096),
+} as const;
+
+const summarizeInputSchema = z.object(summarizeInputShape);
+
+export const SUMMARIZE_TOOL_DESCRIPTION = `Summarize text using the HOST's model via MCP \`sampling/createMessage\` — the server embeds no model and calls no provider directly. Hand it the output of \`extract\`, \`extract_section\`, \`html_to_markdown\`, or any markdown/text string; the host picks the model and may ask the user to approve the sampling request (human-in-the-loop per MCP). The tool is only listed when the connected client advertises the sampling capability.`;
+
+async function summarizeWithHost(
+  server: McpServer,
+  args: { maxTokens: number; text: string },
+): Promise<string> {
+  return sampleText(server, {
+    maxTokens: args.maxTokens,
+    systemPrompt: SUMMARIZE_SYSTEM_PROMPT,
+    userText: args.text,
+  });
+}
+
+export function registerSummarizeTool(server: McpServer): ToolHandle {
+  return server.registerTool(
+    'summarize',
+    {
+      title: 'Summarize text using the host model',
+      description: SUMMARIZE_TOOL_DESCRIPTION,
+      inputSchema: summarizeInputShape,
+    },
+    tool(
+      'summarize',
+      async args => {
+        const summary = await summarizeWithHost(server, args);
+        return {
+          content: [{ type: 'text', text: summary }],
+        };
+      },
+      summarizeInputSchema,
+    ),
+  );
+}
+
+// Mirrors registerTools/registerResources so the dev reload
+// loop and capability gate can treat sampling as one registration family.
+export function registerSamplingTools(server: McpServer): ToolHandle[] {
+  return [registerSummarizeTool(server), registerSuggestPresetTool(server)];
+}