@v1nvn/readability 0.44.0 → 0.45.1

package/src/policy/chunk.ts

@@ -0,0 +1,482 @@
+// Token-bounded chunking of extracted markdown for RAG/embedding. Chunk
+// `tokenCount` uses the shared chars/4 estimator (policy/text.ts), keeping it
+// directly comparable to `metadata.tokenEstimate`. This is the server-side
+// chunking path for large documents; transport-level streaming was rejected —
+// MCP tool-result progress isn't universally host-supported, and this option
+// already keeps a multi-MB page from returning as one payload.
+
+import { hardSplitLines, headingText, parseBlocks } from './markdown.ts';
+import { estimateTokens } from './text.ts';
+
+export interface Chunk {
+  readonly headingContext: string;
+  readonly index: number;
+  readonly text: string;
+  readonly tokenCount: number;
+}
+
+export type ChunkStrategy = 'char' | 'semantic';
+
+export interface ChunkOptions {
+  readonly maxTokens: number;
+  readonly overlap: number;
+  readonly strategy: ChunkStrategy;
+}
+
+interface Block {
+  readonly headingContext: string;
+  readonly text: string;
+}
+
+// Character offsets of a contributing unit within the in-progress chunk text;
+// used to recover the heading context at the start of an overlap tail.
+interface Span {
+  readonly end: number;
+  readonly headingContext: string;
+  readonly start: number;
+}
+
+const HEADING_FIRST_LINE = /^#{1,6}\s/;
+
+function splitBlocks(markdown: string): Block[] {
+  const parts = markdown.split(/\n{2,}/);
+  const blocks: Block[] = [];
+  let heading = '';
+  for (const part of parts) {
+    const text = part.trim();
+    if (!text) {
+      continue;
+    }
+    if (HEADING_FIRST_LINE.test(text)) {
+      heading = headingText(text);
+    }
+    blocks.push({ headingContext: heading, text });
+  }
+  return blocks;
+}
+
+// Char strategy may break a code block — the semantic strategy avoids that.
+function splitOversizedBlock(block: Block, maxChars: number): Block[] {
+  return hardSplitLines(block.text, maxChars).map(text => ({
+    headingContext: block.headingContext,
+    text,
+  }));
+}
+
+function toUnits(blocks: readonly Block[], maxChars: number): Block[] {
+  const units: Block[] = [];
+  for (const block of blocks) {
+    if (block.text.length <= maxChars) {
+      units.push(block);
+    } else {
+      units.push(...splitOversizedBlock(block, maxChars));
+    }
+  }
+  return units;
+}
+
+function chunkMarkdownChar(
+  markdown: string,
+  options: Readonly<ChunkOptions>,
+): Chunk[] {
+  if (!markdown) {
+    return [];
+  }
+  const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+  const maxChars = maxTokens * 4;
+  const overlapTokens = Math.max(0, Math.floor(options.overlap));
+  // Overlap must be strictly less than maxChars — otherwise a chunk's tail
+  // could consume the entire budget and we'd loop forever re-emitting it.
+  const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+
+  const blocks = splitBlocks(markdown);
+  if (blocks.length === 0) {
+    return [];
+  }
+  const units = toUnits(blocks, maxChars);
+  if (units.length === 0) {
+    return [];
+  }
+
+  const chunks: Chunk[] = [];
+  let i = 0;
+  let overlapText = '';
+  let overlapHeading = '';
+
+  while (i < units.length) {
+    const spans: Span[] = [];
+    let chunkText = '';
+
+    if (overlapText) {
+      chunkText = overlapText;
+      spans.push({
+        end: overlapText.length,
+        headingContext: overlapHeading,
+        start: 0,
+      });
+    }
+
+    // Force at least one new unit per chunk so the loop terminates. If carrying
+    // the overlap would push the first new unit past maxChars, drop the overlap
+    // for this chunk (the unit was pre-split to <= maxChars so it always fits alone).
+    const firstUnit = units[i];
+    const sepLen = chunkText ? 2 : 0;
+    if (
+      chunkText &&
+      chunkText.length + sepLen + firstUnit.text.length > maxChars
+    ) {
+      chunkText = '';
+      spans.length = 0;
+    }
+
+    {
+      const sep = chunkText ? '\n\n' : '';
+      const start = chunkText.length + sep.length;
+      chunkText = chunkText + sep + firstUnit.text;
+      spans.push({
+        end: chunkText.length,
+        headingContext: firstUnit.headingContext,
+        start,
+      });
+      i += 1;
+    }
+
+    while (i < units.length) {
+      const unit = units[i];
+      const candidate = `${chunkText}\n\n${unit.text}`;
+      if (candidate.length > maxChars) {
+        break;
+      }
+      const start = chunkText.length + 2;
+      chunkText = candidate;
+      spans.push({
+        end: chunkText.length,
+        headingContext: unit.headingContext,
+        start,
+      });
+      i += 1;
+    }
+
+    const text = chunkText.trim();
+    if (text) {
+      chunks.push({
+        headingContext: spans[0]?.headingContext ?? '',
+        index: chunks.length,
+        text,
+        tokenCount: estimateTokens(text).tokenEstimate,
+      });
+    }
+
+    if (overlapChars > 0 && i < units.length) {
+      const overlapStart = Math.max(0, chunkText.length - overlapChars);
+      const carrier =
+        spans.find(
+          span => overlapStart >= span.start && overlapStart < span.end,
+        ) ?? spans.find(span => span.start >= overlapStart);
+      overlapHeading = carrier?.headingContext ?? '';
+      overlapText = chunkText.slice(overlapStart).replace(/^\n+/, '');
+    } else {
+      overlapText = '';
+      overlapHeading = '';
+    }
+  }
+
+  return chunks;
+}
+
+// --- Semantic strategy ------------------------------------------------------
+//
+// Break on heading/section boundaries and never split a fenced code block. Each
+// top-level mdast node is one unit; a fenced code block is a single `code` node
+// so it stays atomic for free.
+
+type SemanticUnitKind = 'code' | 'heading' | 'text';
+
+interface SemanticUnit {
+  readonly headingContext: string;
+  readonly kind: SemanticUnitKind;
+  readonly text: string;
+}
+
+interface SemanticSection {
+  readonly headingContext: string;
+  readonly units: readonly SemanticUnit[];
+}
+
+// A group is the unit-list slice that becomes one chunk's body, plus the
+// heading-context of its first non-carrier unit (so prepending an overlap
+// carrier cannot steal the heading's context).
+interface SemanticGroup {
+  readonly headingContext: string;
+  readonly units: readonly SemanticUnit[];
+}
+
+function parseSemanticUnits(markdown: string): SemanticUnit[] {
+  const units: SemanticUnit[] = [];
+  const stack: { level: number; text: string }[] = [];
+
+  function context(): string {
+    return stack.map(h => h.text).join(' > ');
+  }
+
+  for (const block of parseBlocks(markdown)) {
+    const text = markdown.slice(block.start, block.end);
+    if (block.kind === 'heading') {
+      while (stack.length > 0 && stack[stack.length - 1].level >= block.depth) {
+        stack.pop();
+      }
+      stack.push({ level: block.depth, text: headingText(text) });
+      units.push({ headingContext: context(), kind: 'heading', text });
+      continue;
+    }
+    units.push({
+      headingContext: context(),
+      kind: block.kind === 'code' ? 'code' : 'text',
+      text,
+    });
+  }
+  return units;
+}
+
+// A section = a heading unit plus its following non-heading units until the
+// next heading. Pre-first-heading content forms its own section with an empty
+// hierarchy path.
+function groupSections(units: readonly SemanticUnit[]): SemanticSection[] {
+  const sections: SemanticSection[] = [];
+  let current: SemanticUnit[] = [];
+  let currentContext = '';
+
+  function flush(): void {
+    if (current.length > 0) {
+      sections.push({ headingContext: currentContext, units: current });
+      current = [];
+    }
+  }
+
+  for (const unit of units) {
+    if (unit.kind === 'heading') {
+      flush();
+      current = [unit];
+      currentContext = unit.headingContext;
+    } else {
+      if (current.length === 0) {
+        currentContext = unit.headingContext;
+      }
+      current.push(unit);
+    }
+  }
+  flush();
+  return sections;
+}
+
+function joinedLength(units: readonly SemanticUnit[]): number {
+  if (units.length === 0) {
+    return 0;
+  }
+  let total = units[0].text.length;
+  for (let i = 1; i < units.length; i++) {
+    total += 2 + units[i].text.length;
+  }
+  return total;
+}
+
+// Hard-split a text unit that alone exceeds the budget — line-aware first, then
+// hard-split any oversized line. Only ever applied to paragraphs; code blocks
+// are never passed through here.
+function splitOversizedTextUnit(
+  unit: SemanticUnit,
+  maxChars: number,
+): SemanticUnit[] {
+  return hardSplitLines(unit.text, maxChars).map(text => ({
+    headingContext: unit.headingContext,
+    kind: 'text',
+    text,
+  }));
+}
+
+// Pack sections into chunk groups <= maxChars, preferring heading boundaries.
+// A section that fits is never split; an oversized section is split into
+// sub-chunks by body unit (paragraphs may split; code blocks never do).
+function buildBaseGroups(
+  sections: readonly SemanticSection[],
+  maxChars: number,
+): SemanticUnit[][] {
+  const groups: SemanticUnit[][] = [];
+  let current: SemanticUnit[] = [];
+  let currentLen = 0;
+
+  function emit(): void {
+    if (current.length > 0) {
+      groups.push(current);
+      current = [];
+      currentLen = 0;
+    }
+  }
+
+  function append(unit: SemanticUnit): void {
+    const sep = current.length > 0 ? 2 : 0;
+    current.push(unit);
+    currentLen += sep + unit.text.length;
+  }
+
+  for (const section of sections) {
+    const sectionLen = joinedLength(section.units);
+
+    const sep = current.length > 0 ? 2 : 0;
+    if (currentLen + sep + sectionLen <= maxChars) {
+      for (const unit of section.units) {
+        append(unit);
+      }
+      continue;
+    }
+
+    // Doesn't fit merged — close the current group and retry the section alone
+    // so it has first dibs on the next group (it may still merge with a later
+    // section that fits behind it).
+    emit();
+
+    if (sectionLen <= maxChars) {
+      for (const unit of section.units) {
+        append(unit);
+      }
+      continue;
+    }
+
+    // Oversized section: pack body units under the heading; split oversized
+    // paragraphs, emit oversized code blocks whole.
+    const [headingUnit, ...body] = section.units;
+    current.push(headingUnit);
+    currentLen = headingUnit.text.length;
+    for (const unit of body) {
+      if (unit.kind === 'code' && unit.text.length > maxChars) {
+        // Semantic never splits a code fence: a fence cut mid-way is
+        // unrecoverable, so the block is emitted as its own chunk even though
+        // it exceeds the token budget. Deliberate tradeoff vs. the char cap.
+        emit();
+        groups.push([unit]);
+        continue;
+      }
+      const sepNow = current.length > 0 ? 2 : 0;
+      if (
+        unit.text.length <= maxChars &&
+        currentLen + sepNow + unit.text.length <= maxChars
+      ) {
+        append(unit);
+        continue;
+      }
+      const pieces =
+        unit.text.length > maxChars
+          ? splitOversizedTextUnit(unit, maxChars)
+          : [unit];
+      for (const piece of pieces) {
+        const pieceSep = current.length > 0 ? 2 : 0;
+        if (
+          current.length > 0 &&
+          currentLen + pieceSep + piece.text.length > maxChars
+        ) {
+          emit();
+        }
+        append(piece);
+      }
+    }
+    // Leave the trailing sub-chunk in `current` so a following section can
+    // merge into it if it fits.
+  }
+  emit();
+  return groups;
+}
+
+// Carry the trailing text (non-code) units from the previous group as the
+// prefix of the next, up to overlapChars. Never carry into or out of a code
+// block: a trailing code run is skipped, then contiguous text units before it
+// are carried (stop at the first heading or code block encountered).
+function applySemanticOverlap(
+  groups: readonly SemanticUnit[][],
+  overlapChars: number,
+): SemanticGroup[] {
+  const result: SemanticGroup[] = [];
+  for (let i = 0; i < groups.length; i++) {
+    const group = groups[i];
+    const carrier: SemanticUnit[] = [];
+    if (i > 0 && overlapChars > 0) {
+      const prev = groups[i - 1];
+      let end = prev.length;
+      while (end > 0 && prev[end - 1].kind === 'code') {
+        end -= 1;
+      }
+      let carrierLen = 0;
+      for (let j = end - 1; j >= 0; j--) {
+        const unit = prev[j];
+        if (unit.kind !== 'text') {
+          break;
+        }
+        const sep = carrier.length > 0 ? 2 : 0;
+        if (carrierLen + sep + unit.text.length > overlapChars) {
+          break;
+        }
+        carrier.unshift(unit);
+        carrierLen += sep + unit.text.length;
+      }
+    }
+    const units = carrier.length === 0 ? group : [...carrier, ...group];
+    result.push({
+      // The chunk's heading context follows its first non-carrier unit, so a
+      // heading-leading chunk keeps its heading's path even with overlap.
+      headingContext: group[0]?.headingContext ?? '',
+      units,
+    });
+  }
+  return result;
+}
+
+function chunkMarkdownSemantic(
+  markdown: string,
+  options: Readonly<ChunkOptions>,
+): Chunk[] {
+  if (!markdown) {
+    return [];
+  }
+  const maxTokens = Math.max(1, Math.floor(options.maxTokens));
+  const maxChars = maxTokens * 4;
+  const overlapTokens = Math.max(0, Math.floor(options.overlap));
+  const overlapChars = Math.min(overlapTokens * 4, maxChars - 1);
+
+  const units = parseSemanticUnits(markdown);
+  if (units.length === 0) {
+    return [];
+  }
+  const sections = groupSections(units);
+  const baseGroups = buildBaseGroups(sections, maxChars);
+  const groups = applySemanticOverlap(baseGroups, overlapChars);
+
+  const chunks: Chunk[] = [];
+  for (const group of groups) {
+    if (group.units.length === 0) {
+      continue;
+    }
+    const text = group.units
+      .map(unit => unit.text)
+      .join('\n\n')
+      .trim();
+    if (!text) {
+      continue;
+    }
+    chunks.push({
+      headingContext: group.headingContext,
+      index: chunks.length,
+      text,
+      tokenCount: estimateTokens(text).tokenEstimate,
+    });
+  }
+  return chunks;
+}
+
+export function chunkMarkdown(
+  markdown: string,
+  options: Readonly<ChunkOptions>,
+): Chunk[] {
+  if (options.strategy === 'semantic') {
+    return chunkMarkdownSemantic(markdown, options);
+  }
+  return chunkMarkdownChar(markdown, options);
+}

package/src/policy/diagnostics.js

@@ -0,0 +1,53 @@
+function countElements(html, window) {
+    if (!window || !html) {
+        return 0;
+    }
+    const template = window.document.createElement('div');
+    template.innerHTML = html;
+    return template.querySelectorAll('*').length;
+}
+export function assembleDiagnostics(input) {
+    const articleElementCount = countElements(input.articleHtml ?? '', input.window);
+    const removedNodes = Math.max(0, (input.documentElementCount ?? 0) - articleElementCount);
+    return {
+        readerable: input.readerable,
+        extractedNode: input.extractedNode,
+        fallbackUsed: input.fallbackUsed ?? false,
+        gated: input.gated,
+        imagesResolved: input.imagesResolved,
+        pagination: input.pagination,
+        removedNodes,
+        boilerplateRemoved: input.boilerplateRemoved,
+        chromeRemoved: input.chromeRemoved,
+        sanitization: input.sanitization,
+        trace: input.trace,
+        truncated: input.truncated ?? false,
+        ...(input.cache ? { cache: input.cache } : {}),
+        ...(input.preset ? { preset: input.preset } : {}),
+    };
+}
+// Emits trace entries only when enabled, so the debug-off hot path allocates
+// nothing (the fn runs bare). Used by the orchestrator tools to time pipeline
+// stages without threading timers into lower layers.
+export class TraceCollector {
+    enabled;
+    entries = [];
+    constructor(enabled) {
+        this.enabled = enabled;
+    }
+    collect() {
+        return this.enabled ? this.entries : undefined;
+    }
+    run(stage, fn) {
+        if (!this.enabled) {
+            return fn();
+        }
+        const start = performance.now();
+        try {
+            return fn();
+        }
+        finally {
+            this.entries.push({ ms: performance.now() - start, stage });
+        }
+    }
+}

package/src/policy/diagnostics.ts

@@ -0,0 +1,96 @@
+import type {
+  CacheSignal,
+  Diagnostics,
+  SanitizationDiagnostics,
+  TraceStage,
+} from '../pipeline/context.ts';
+import type { GatingSignal } from './gating.ts';
+import type { PaginationSignal } from './pagination.ts';
+import type { PresetSignal } from './presets.ts';
+
+export interface DiagnosticsInput {
+  readonly articleHtml?: string;
+  readonly boilerplateRemoved?: number;
+  readonly cache?: CacheSignal;
+  readonly chromeRemoved?: number;
+  readonly document?: Document;
+  readonly documentElementCount?: number;
+  readonly extractedNode?: string;
+  readonly fallbackUsed?: boolean;
+  readonly gated?: GatingSignal;
+  readonly imagesResolved?: number;
+  readonly pagination?: PaginationSignal;
+  readonly preset?: PresetSignal;
+  readonly readerable?: boolean;
+  readonly sanitization?: SanitizationDiagnostics;
+  readonly trace?: readonly TraceStage[];
+  readonly truncated?: boolean;
+  readonly window?: Window;
+}
+
+function countElements(html: string, window?: Window): number {
+  if (!window || !html) {
+    return 0;
+  }
+  const template = window.document.createElement('div');
+  template.innerHTML = html;
+  return template.querySelectorAll('*').length;
+}
+
+export function assembleDiagnostics(
+  input: Readonly<DiagnosticsInput>,
+): Diagnostics {
+  const articleElementCount = countElements(
+    input.articleHtml ?? '',
+    input.window,
+  );
+  const removedNodes = Math.max(
+    0,
+    (input.documentElementCount ?? 0) - articleElementCount,
+  );
+
+  return {
+    readerable: input.readerable,
+    extractedNode: input.extractedNode,
+    fallbackUsed: input.fallbackUsed ?? false,
+    gated: input.gated,
+    imagesResolved: input.imagesResolved,
+    pagination: input.pagination,
+    removedNodes,
+    boilerplateRemoved: input.boilerplateRemoved,
+    chromeRemoved: input.chromeRemoved,
+    sanitization: input.sanitization,
+    trace: input.trace,
+    truncated: input.truncated ?? false,
+    ...(input.cache ? { cache: input.cache } : {}),
+    ...(input.preset ? { preset: input.preset } : {}),
+  };
+}
+
+// Emits trace entries only when enabled, so the debug-off hot path allocates
+// nothing (the fn runs bare). Used by the orchestrator tools to time pipeline
+// stages without threading timers into lower layers.
+export class TraceCollector {
+  private readonly enabled: boolean;
+  private readonly entries: TraceStage[] = [];
+
+  constructor(enabled: boolean) {
+    this.enabled = enabled;
+  }
+
+  collect(): readonly TraceStage[] | undefined {
+    return this.enabled ? this.entries : undefined;
+  }
+
+  run<T>(stage: string, fn: () => T): T {
+    if (!this.enabled) {
+      return fn();
+    }
+    const start = performance.now();
+    try {
+      return fn();
+    } finally {
+      this.entries.push({ ms: performance.now() - start, stage });
+    }
+  }
+}