@v1nvn/readability 0.44.0 → 0.45.0

package/src/policy/explain.ts

@@ -0,0 +1,182 @@
+import { Readability } from '@mozilla/readability';
+
+import type { GatingSignal } from './gating.ts';
+import type { PaginationSignal } from './pagination.ts';
+
+import { buildDocument } from '../pipeline/dom.ts';
+import {
+  applySelectors,
+  normalizeDocument,
+  resolveLazyImages,
+  type SelectorScope,
+} from '../pipeline/normalize.ts';
+import { isReaderable } from '../pipeline/readability.ts';
+import { resolveReadabilityOptions } from '../pipeline/resolver.ts';
+import { assembleDiagnostics } from './diagnostics.ts';
+import { detectGating } from './gating.ts';
+import { detectPagination } from './pagination.ts';
+import { describeSelector } from './sibling-scan.ts';
+
+// Readability stamps `{ contentScore }` on candidate DOM nodes under a
+// `readability` expando (Readability.js:894/1272/1288). The property is
+// untyped upstream, so reach it through `unknown` rather than a guessed shape.
+interface ReadabilityExpando {
+  readonly contentScore?: number;
+}
+
+export interface ExplainCandidate {
+  readonly className: string;
+  readonly id: string;
+  readonly score: number;
+  readonly selector: string;
+  readonly tag: string;
+  readonly textLength: number;
+}
+
+export interface ExplainRemovedNodes {
+  readonly boilerplate: number;
+  readonly chrome: number;
+  readonly total: number;
+}
+
+export interface ExplainSnapshot {
+  readonly html: string;
+  readonly truncated: boolean;
+}
+
+export interface ExplainReport {
+  readonly candidates: readonly ExplainCandidate[];
+  readonly chosenRoot: ExplainCandidate | null;
+  readonly fallbackUsed: boolean;
+  readonly gating: GatingSignal | undefined;
+  readonly pagination: PaginationSignal | undefined;
+  readonly parseSucceeded: boolean;
+  readonly readerable: boolean;
+  readonly removedNodes: ExplainRemovedNodes;
+  readonly snapshot: ExplainSnapshot;
+}
+
+export interface BuildExplainOptions {
+  readonly baseUrl?: string;
+  readonly html: string;
+  readonly selectors?: Readonly<SelectorScope>;
+  readonly snapshotMaxChars?: number;
+  readonly topN?: number;
+}
+
+const DEFAULT_SNAPSHOT_MAX = 4000;
+const DEFAULT_TOP_N = 5;
+
+function readScored(el: Element): ExplainCandidate | null {
+  const stamp = (el as unknown as { readability?: ReadabilityExpando })
+    .readability;
+  const score = stamp?.contentScore;
+  if (typeof score !== 'number') {
+    return null;
+  }
+  const className = typeof el.className === 'string' ? el.className : '';
+  const candidate = {
+    className,
+    id: el.id,
+    score,
+    tag: el.tagName,
+    textLength: el.textContent.trim().length,
+  };
+  // Two classes cap the hint's width — Readability's expando is invisible to
+  // CSS and an nth-child chain would be brittle against the host's live DOM,
+  // so tag + id + a couple of classes is all a human needs to pick the node
+  // out of a small candidate list.
+  return { ...candidate, selector: describeSelector(el, 2) };
+}
+
+function truncateSnapshot(html: string, max: number): ExplainSnapshot {
+  if (html.length <= max) {
+    return { html, truncated: false };
+  }
+  return { html: html.slice(0, max), truncated: true };
+}
+
+export function buildExplainReport(
+  options: Readonly<BuildExplainOptions>,
+): ExplainReport {
+  const {
+    html,
+    selectors,
+    snapshotMaxChars = DEFAULT_SNAPSHOT_MAX,
+    topN = DEFAULT_TOP_N,
+    baseUrl,
+  } = options;
+
+  const { document, window } = buildDocument(html, baseUrl);
+
+  // Mirror the extract pipeline's ordering: gating detection must precede
+  // normalization (chrome stripping would remove the overlay), and pagination
+  // detection runs before applySelectors (a caller's include could hide the
+  // sentinel but not the "more content exists" signal).
+  const gating = detectGating(document);
+  const documentElementCount = document.querySelectorAll('*').length;
+  const normalizeCounts = normalizeDocument(document);
+  resolveLazyImages(document);
+  const pagination = detectPagination(document, baseUrl);
+  applySelectors(document, selectors);
+
+  const snapshot = truncateSnapshot(document.body.innerHTML, snapshotMaxChars);
+
+  const readerable = isReaderable(document);
+  const readabilityOptions = resolveReadabilityOptions({});
+
+  // Clone so the normalized doc (and the snapshot above) is preserved untouched
+  // — Readability restructures its input during parse.
+  const clone = document.cloneNode(true) as Document;
+  // Grab node references before parse: Readability detaches scored candidates
+  // from the live tree while restructuring, so a post-parse querySelectorAll
+  // finds almost nothing. The object refs held here keep their `readability`
+  // stamps after being detached — that retention is what surfaces the real
+  // per-candidate scores without forking the library or parsing its debug log.
+  const heldNodes = Array.from(clone.querySelectorAll('*'));
+
+  const reader = new Readability(clone, readabilityOptions);
+  const article = reader.parse();
+  const parseSucceeded = !!article?.content;
+
+  const scored: ExplainCandidate[] = [];
+  for (const el of heldNodes) {
+    const entry = readScored(el);
+    if (entry) {
+      scored.push(entry);
+    }
+  }
+  scored.sort((a, b) => b.score - a.score);
+
+  const candidates = scored.slice(0, Math.max(0, topN));
+  const chosenRoot = scored[0] ?? null;
+
+  const diagnostics = assembleDiagnostics({
+    articleHtml: article?.content ?? '',
+    boilerplateRemoved: normalizeCounts.boilerplateRemoved,
+    chromeRemoved: normalizeCounts.chromeRemoved,
+    documentElementCount,
+    extractedNode: 'readability',
+    fallbackUsed: false,
+    gated: gating,
+    pagination,
+    readerable,
+    window,
+  });
+
+  return {
+    candidates,
+    chosenRoot,
+    fallbackUsed: diagnostics.fallbackUsed,
+    gating: diagnostics.gated,
+    pagination: diagnostics.pagination,
+    parseSucceeded,
+    readerable: diagnostics.readerable ?? false,
+    removedNodes: {
+      boilerplate: diagnostics.boilerplateRemoved ?? 0,
+      chrome: diagnostics.chromeRemoved ?? 0,
+      total: diagnostics.removedNodes ?? 0,
+    },
+    snapshot,
+  };
+}

package/src/policy/fallback.ts

@@ -0,0 +1,110 @@
+import type { SanitizationDiagnostics } from '../pipeline/context.ts';
+import type { ImageMode, TableFormat } from '../pipeline/turndown.ts';
+
+import { sanitizeHtml } from '../pipeline/sanitize.ts';
+import { toMarkdown } from '../pipeline/turndown.ts';
+
+export interface FallbackOptions {
+  readonly baseUrl?: string;
+  readonly codeBlockStyle?: 'fenced' | 'indented';
+  readonly gfm?: boolean;
+  readonly headingStyle?: 'atx' | 'setext';
+  readonly images?: ImageMode;
+  readonly sanitize: boolean;
+  readonly tables?: TableFormat;
+  readonly window: Window;
+}
+
+export interface FallbackResult {
+  readonly markdown: string;
+  readonly rootSelector: string;
+  readonly sanitization: SanitizationDiagnostics;
+  readonly sanitizedHtml: string;
+  readonly textContent: string;
+}
+
+// Below this a block is treated as not text-dense (nav/boilerplate-ish).
+const MIN_DENSE_BLOCK_CHARS = 200;
+
+function convert(
+  element: Element,
+  options: Readonly<FallbackOptions>,
+):
+  | undefined
+  | {
+      markdown: string;
+      sanitization: SanitizationDiagnostics;
+      sanitizedHtml: string;
+      textContent: string;
+    } {
+  const rawHtml = element.outerHTML;
+  let sanitizedHtml = rawHtml;
+  let sanitization: SanitizationDiagnostics = { iframes: 0, scripts: 0 };
+  if (options.sanitize) {
+    const res = sanitizeHtml(rawHtml, options.window);
+    sanitizedHtml = res.html;
+    sanitization = { iframes: res.iframesRemoved, scripts: res.scriptsRemoved };
+  }
+  const markdown = toMarkdown(sanitizedHtml, {
+    codeBlockStyle: options.codeBlockStyle,
+    gfm: options.gfm,
+    headingStyle: options.headingStyle,
+    images: options.images,
+    tables: options.tables,
+    baseUrl: options.baseUrl,
+  });
+  const textContent = element.textContent;
+  if (markdown.trim().length === 0) {
+    return undefined;
+  }
+  return { markdown, sanitization, sanitizedHtml, textContent };
+}
+
+function largestTextDenseBlock(document: Document): Element | undefined {
+  let best: Element | undefined;
+  let bestLen = 0;
+  const candidates = document.querySelectorAll('div, section');
+  candidates.forEach(el => {
+    const len = el.textContent.trim().length;
+    if (len > bestLen && len >= MIN_DENSE_BLOCK_CHARS) {
+      best = el;
+      bestLen = len;
+    }
+  });
+  return best;
+}
+
+export function extractViaFallback(
+  document: Document,
+  options: Readonly<FallbackOptions>,
+): FallbackResult | null {
+  const orderedSelectors: readonly string[] = [
+    'article',
+    'main',
+    '[role=main]',
+  ];
+  for (const selector of orderedSelectors) {
+    const root = document.querySelector(selector);
+    if (root) {
+      const converted = convert(root, options);
+      if (converted) {
+        return { ...converted, rootSelector: selector };
+      }
+    }
+  }
+
+  const dense = largestTextDenseBlock(document);
+  if (dense) {
+    const converted = convert(dense, options);
+    if (converted) {
+      return { ...converted, rootSelector: 'largest-block' };
+    }
+  }
+
+  const converted = convert(document.body, options);
+  if (converted) {
+    return { ...converted, rootSelector: 'body' };
+  }
+
+  return null;
+}

package/src/policy/gating.ts

@@ -0,0 +1,106 @@
+export interface GatingSignal {
+  readonly likely: boolean;
+  readonly reason: string;
+}
+
+// Curated vendor/structural paywall selectors — intentionally NOT a greedy
+// `[class*="subscribe"]` (that catches newsletter CTAs on clean articles and
+// false-positives). Each entry names a known paywall surface.
+const PAYWALL_SELECTORS = [
+  '.piano',
+  '#piano',
+  '.tp-modal',
+  '.tp-active',
+  '[class*="subscribe-wall"]',
+  '[id*="subscribe-wall"]',
+  '[class*="metered-wall"]',
+  '[id*="metered-wall"]',
+  '.leaky-paywall',
+] as const;
+
+// Class/id substring hits are verdicts, not signals. Measured on real captures:
+// Daily Mail free articles carry `<html class="… paywall-ineligible">` plus ~240
+// `is-paywalled` / `is-paywall-processed` feed badges — state about other
+// articles or processing markers, all matching `[class*="paywall"]`; Corriere's
+// free articles carry an `offer-header-piano` subscription header matching
+// `[class*="piano"]`. Real surfaces name the gate as the head noun (WIRED's
+// `paywall-modal`, the camel-cased `PaywallModalWrapper`).
+const PAYWALL_ATTR_CANDIDATES =
+  '[class*="paywall"], [id*="paywall"], [class*="piano"], [id*="piano"]';
+
+const NEGATION_SEGMENTS = new Set([
+  'bypass',
+  'disabled',
+  'exempt',
+  'free',
+  'ineligible',
+  'no',
+  'non',
+  'not',
+  'off',
+  'optout',
+  'without',
+]);
+
+function namesGateSurface(classAndId: string, noun: string): boolean {
+  for (const token of classAndId.split(/\s+/)) {
+    const segments = token
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .toLowerCase()
+      .split(/[^a-z]+/)
+      .filter(Boolean);
+    if (segments[0] !== noun) {
+      continue;
+    }
+    if (segments.some(segment => NEGATION_SEGMENTS.has(segment))) {
+      continue;
+    }
+    return true;
+  }
+  return false;
+}
+
+// Phrases that essentially never appear on a fully-unlocked article. Bare
+// "Subscribe" nav links / newsletter CTAs are intentionally excluded — they
+// are ubiquitous and would mislead the host into discarding complete content.
+const METERED_TEXT_RE =
+  /(\d+)\s*(?:free\s*)?(?:articles?|stories?)\s*(?:left|remaining)|you\s+have\s+reached\s+(?:your\s+)?(?:free\s+)?(?:article\s+|story\s+)?limit|subscribe\s+to\s+(?:continue\s+)?reading|read\s+the\s+full\s+(?:article|story)|unlock\s+(?:this|full|all)\s+(?:article|story|content)|keep\s+reading\s+with/i;
+
+function findPaywallOverlay(document: Document): GatingSignal | undefined {
+  for (const el of document.querySelectorAll(PAYWALL_ATTR_CANDIDATES)) {
+    const classAndId = `${el.getAttribute('class') ?? ''} ${el.getAttribute('id') ?? ''}`;
+    if (!el.isConnected) {
+      continue;
+    }
+    if (
+      namesGateSurface(classAndId, 'paywall') ||
+      namesGateSurface(classAndId, 'piano')
+    ) {
+      return { likely: true, reason: 'paywall overlay' };
+    }
+  }
+  for (const selector of PAYWALL_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (el?.isConnected) {
+      return { likely: true, reason: 'paywall overlay' };
+    }
+  }
+  return undefined;
+}
+
+// One textContent read is cheaper than a per-element scan of a large doc and
+// avoids ordering a query per selector. jsdom computes body.textContent once.
+function findMeteredMessage(document: Document): GatingSignal | undefined {
+  const text = document.body.textContent;
+  if (METERED_TEXT_RE.test(text)) {
+    return { likely: true, reason: 'metered paywall message' };
+  }
+  return undefined;
+}
+
+// Reads the DOM only — never fetches, authenticates, or mutates the document.
+// Conservative by design: a false positive misleads the host into treating
+// complete content as truncated, which is worse than a miss.
+export function detectGating(document: Document): GatingSignal | undefined {
+  return findPaywallOverlay(document) ?? findMeteredMessage(document);
+}