@v1nvn/readability 0.43.0 → 0.45.0

package/src/policy/pagination.ts

@@ -0,0 +1,104 @@
+import { absolutize } from '../pipeline/urls.ts';
+
+export interface PaginationSignal {
+  readonly nextUrl?: string;
+  readonly selector?: string;
+  readonly type: 'infinite' | 'paginated';
+}
+
+// Short pagination-style link text only; matching prose like "next, we cover
+// rendering" would burn an article's internal link as a next-page signal.
+// `next ›/»/→` and bare arrows cover the common next-page glyphs in one place.
+const NEXT_LINK_TEXT_RE =
+  /^(next(\s+page)?|older(\s+posts?)?|[›»→]|next\s*[›»→]|older\s*[›»→])$/i;
+
+function usableHref(href: null | string): string | undefined {
+  if (!href || href === '#') {
+    return undefined;
+  }
+  return href;
+}
+
+function findPaginated(
+  document: Document,
+  baseUrl: string | undefined,
+): PaginationSignal | undefined {
+  const linkNext = document.querySelector('link[rel="next"][href]');
+  if (linkNext) {
+    const href = usableHref(linkNext.getAttribute('href'));
+    if (href) {
+      return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+  }
+
+  const aRelNext = document.querySelector('a[rel="next"][href]');
+  if (aRelNext) {
+    const href = usableHref(aRelNext.getAttribute('href'));
+    if (href) {
+      return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+  }
+
+  for (const anchor of document.querySelectorAll('a[href]')) {
+    const text = anchor.textContent.trim();
+    if (!text || !NEXT_LINK_TEXT_RE.test(text)) {
+      continue;
+    }
+    const href = usableHref(anchor.getAttribute('href'));
+    if (!href) {
+      continue;
+    }
+    return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+  }
+
+  return undefined;
+}
+
+const INFINITE_ATTR_SELECTORS = [
+  '[data-load-more]',
+  '[data-infinite-scroll]',
+  '[data-pagination]',
+  '[infinite-scroll]',
+] as const;
+
+const INFINITE_SUBSTRING_SELECTORS = [
+  '[class*="load-more"]',
+  '[class*="loadmore"]',
+  '[class*="infinite"]',
+  '[id*="load-more"]',
+  '[class*="sentinel"]',
+] as const;
+
+const LOAD_MORE_BUTTON_RE =
+  /^(load more|show more|view more|more results|load more comments)$/i;
+
+function findInfinite(document: Document): PaginationSignal | undefined {
+  for (const selector of INFINITE_ATTR_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (el?.isConnected) {
+      return { selector, type: 'infinite' };
+    }
+  }
+  for (const selector of INFINITE_SUBSTRING_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (el?.isConnected) {
+      return { selector, type: 'infinite' };
+    }
+  }
+  for (const button of document.querySelectorAll('button')) {
+    const text = button.textContent.trim();
+    if (text && LOAD_MORE_BUTTON_RE.test(text)) {
+      return { selector: 'button', type: 'infinite' };
+    }
+  }
+  return undefined;
+}
+
+// Paginated beats infinite: an explicit "next" link is a stronger signal than
+// a load-more sentinel. Read-only — never follows or fetches.
+export function detectPagination(
+  document: Document,
+  baseUrl?: string,
+): PaginationSignal | undefined {
+  return findPaginated(document, baseUrl) ?? findInfinite(document);
+}

package/src/policy/presets.ts

@@ -0,0 +1,142 @@
+// Structural twin of the pipeline's SelectorScope — policy cannot import upward.
+export interface PresetScope {
+  readonly exclude?: readonly string[];
+  readonly include?: string;
+}
+
+export interface SitePreset {
+  readonly detectors: readonly string[];
+  readonly scope: PresetScope;
+  readonly site: string;
+}
+
+export type PresetRejection = 'detectors-missed' | 'overridden';
+
+export interface PresetSignal {
+  readonly applied: boolean;
+  readonly reason?: PresetRejection;
+  readonly site: string;
+}
+
+export interface PresetResolution {
+  readonly scope?: PresetScope;
+  readonly signal: PresetSignal;
+}
+
+const presets = new Map<string, SitePreset>();
+
+// Bumped on every store mutation so cache keys can fold in preset state
+// without hashing the resolved scope: an entry extracted before a preset
+// landed must never be served after it.
+let generation = 0;
+
+export function presetGeneration(): number {
+  return generation;
+}
+
+export function resetPresets(): void {
+  presets.clear();
+  generation += 1;
+}
+
+export function addPreset(preset: SitePreset): void {
+  const key = normalizeSiteKey(preset.site);
+  if (!key) {
+    throw new Error(`invalid preset site: ${preset.site}`);
+  }
+  presets.set(key, preset);
+  generation += 1;
+}
+
+export function removePreset(site: string): boolean {
+  const key = normalizeSiteKey(site);
+  const removed = key !== undefined && presets.delete(key);
+  if (removed) {
+    generation += 1;
+  }
+  return removed;
+}
+
+export function presetForSite(
+  site: string | undefined,
+): SitePreset | undefined {
+  const key = normalizeSiteKey(site);
+  return key ? presets.get(key) : undefined;
+}
+
+// Accepts a full URL or a bare host; `www.` is the one prefix every site's own
+// pages agree on, so exactly one strip — `www.www.com` keys as `www.com`.
+export function normalizeSiteKey(
+  value: string | undefined,
+): string | undefined {
+  const raw = value?.trim();
+  if (!raw) {
+    return undefined;
+  }
+  let host: string;
+  try {
+    host = new URL(raw).hostname.toLowerCase();
+  } catch {
+    host = raw.toLowerCase();
+  }
+  const key = host.startsWith('www.') ? host.slice('www.'.length) : host;
+  return /^[a-z0-9.-]+$/.test(key) ? key : undefined;
+}
+
+// A selector that throws cannot match anything, so a malformed preset selector
+// is a miss by the same predicate as a non-matching one — never a crash.
+export function selectorMisses(scope: ParentNode, selector: string): boolean {
+  try {
+    return scope.querySelector(selector) === null;
+  } catch {
+    return true;
+  }
+}
+
+function presetMatches(document: Document, preset: SitePreset): boolean {
+  for (const detector of preset.detectors) {
+    if (selectorMisses(document, detector)) {
+      return false;
+    }
+  }
+  // The include must still match: an unmatched include is applySelectors'
+  // silent no-op, and reporting applied:true over baseline output is a lie.
+  if (
+    preset.scope.include &&
+    selectorMisses(document.body, preset.scope.include)
+  ) {
+    return false;
+  }
+  // Excludes must execute but need not match — the debris they name is absent
+  // from some pages of the site, and that is not staleness.
+  for (const selector of preset.scope.exclude ?? []) {
+    try {
+      document.querySelectorAll(selector);
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function resolvePreset(
+  document: Document,
+  baseUrl: string | undefined,
+  overridden: boolean,
+): PresetResolution | undefined {
+  const site = normalizeSiteKey(baseUrl);
+  if (!site) {
+    return undefined;
+  }
+  const preset = presets.get(site);
+  if (!preset) {
+    return undefined;
+  }
+  if (overridden) {
+    return { signal: { applied: false, reason: 'overridden', site } };
+  }
+  if (!presetMatches(document, preset)) {
+    return { signal: { applied: false, reason: 'detectors-missed', site } };
+  }
+  return { scope: preset.scope, signal: { applied: true, site } };
+}

package/src/policy/section.ts

@@ -0,0 +1,127 @@
+import { isElement } from '../pipeline/dom.ts';
+import { normalizeHeadingText } from './outline.ts';
+
+interface HeadingMatch {
+  readonly heading: Element;
+  readonly level: number;
+}
+
+// Match the first heading by normalized text. Exact equality wins; otherwise
+// fall back to the first substring match so a short query ("auth") still
+// lands on a longer heading ("Authentication") without forcing an exact match.
+function findHeading(
+  document: Document,
+  query: string,
+): HeadingMatch | undefined {
+  const needle = normalizeHeadingText(query).toLowerCase();
+  if (!needle) {
+    return undefined;
+  }
+  const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+  let substring: HeadingMatch | undefined;
+  for (const heading of headings) {
+    const text = normalizeHeadingText(heading.textContent).toLowerCase();
+    if (!text) {
+      continue;
+    }
+    const level = Number.parseInt(heading.tagName.slice(1), 10);
+    if (text === needle) {
+      return { heading, level };
+    }
+    if (!substring && text.includes(needle)) {
+      substring = { heading, level };
+    }
+  }
+  return substring;
+}
+
+// Elements whose direct children form the section flow. The matched heading's
+// nearest such ancestor is the level at which the section's content lives as
+// siblings: on docs sites (GitHub markdown, etc.) the heading is wrapped in
+// <div class="markdown-heading">, so the body <p> is a sibling of the wrapper
+// div, not of the <h2>. Walking the <h2>'s own siblings would capture only the
+// permalink anchor and miss the body.
+const FLOW_CONTAINERS = new Set([
+  'ARTICLE',
+  'ASIDE',
+  'BLOCKQUOTE',
+  'BODY',
+  'DD',
+  'DETAILS',
+  'DT',
+  'FOOTER',
+  'HEADER',
+  'LI',
+  'MAIN',
+  'NAV',
+  'SECTION',
+  'TD',
+]);
+
+function findFlowContainer(heading: Element): Element {
+  let ancestor: Element | null = heading.parentElement;
+  while (ancestor) {
+    if (FLOW_CONTAINERS.has(ancestor.tagName)) {
+      return ancestor;
+    }
+    ancestor = ancestor.parentElement;
+  }
+  // BODY is in FLOW_CONTAINERS, so the loop reaches it for any attached
+  // heading; this fallback only covers a detached heading.
+  return heading.ownerDocument.body;
+}
+
+// Level of the first heading at or inside `el`, or undefined when `el` is
+// neither a heading nor a wrapper around one. Used to test whether a following
+// sibling terminates the section.
+function firstHeadingLevel(el: Element): number | undefined {
+  const direct = /^H([1-6])$/.exec(el.tagName);
+  if (direct) {
+    return Number.parseInt(direct[1], 10);
+  }
+  const inner = el.querySelector('h1, h2, h3, h4, h5, h6');
+  if (inner) {
+    return Number.parseInt(inner.tagName.slice(1), 10);
+  }
+  return undefined;
+}
+
+// Wraps the matched heading and its subtree in a `<section data-rdrm-section-scope>`
+// so the existing selectors.include path can isolate it. Returns false (and
+// leaves the document untouched) when no heading matches.
+export function scopeToHeading(
+  document: Document,
+  headingText: string,
+): boolean {
+  const match = findHeading(document, headingText);
+  if (!match) {
+    return false;
+  }
+  const { heading, level } = match;
+  const container = findFlowContainer(heading);
+  let startChild: Element = heading;
+  while (startChild.parentElement && startChild.parentElement !== container) {
+    startChild = startChild.parentElement;
+  }
+  const wrap = document.createElement('section');
+  wrap.setAttribute('data-rdrm-section-scope', '');
+  container.insertBefore(wrap, startChild);
+  let node: Node | null = startChild;
+  while (node) {
+    const next: Node | null = node.nextSibling;
+    wrap.appendChild(node);
+    // A section ends at the next same-or-higher-level heading (level <= L):
+    // deeper headings (h3 under an h2) belong to this section, peers and
+    // shallower headings start the next one. The terminating sibling may be a
+    // wrapper around the heading (e.g. <div class="markdown-heading">), so the
+    // check looks for a heading at OR inside the sibling.
+    if (next !== null && isElement(next)) {
+      const nextLevel = firstHeadingLevel(next);
+      if (nextLevel !== undefined && nextLevel <= level) {
+        break;
+      }
+    }
+    node = next;
+  }
+  return true;
+}

package/src/policy/selector-lint.ts

@@ -0,0 +1,186 @@
+import { isGeneratedIdentifier } from './identifiers.ts';
+import { selectorMisses } from './presets.ts';
+
+export type SelectorViolationKind =
+  | 'contains-pseudo'
+  | 'exclude-shadows-include'
+  | 'generated-identifier'
+  | 'no-match'
+  | 'positional-pseudo'
+  | 'unparseable';
+
+export interface SelectorViolation {
+  readonly detail: string;
+  readonly kind: SelectorViolationKind;
+  readonly selector: string;
+}
+
+export interface ProposalInput {
+  readonly detectors: readonly string[];
+  readonly document: Document;
+  readonly scope: {
+    readonly exclude?: readonly string[];
+    readonly include?: string;
+  };
+}
+
+// nwsapi/jsdom accept `:contains` and silently apply it, so an engine error can
+// reject nothing — the rejection has to happen here, before any selector
+// reaches querySelector. Positional pseudos are rejected as a family: they
+// describe this render's child order, not the site's structure.
+const POSITIONAL_PSEUDO_RE =
+  /:(?:nth(?:-last)?(?:-child|-of-type)|first(?:-child|-of-type)|last(?:-child|-of-type)|only(?:-child|-of-type))\b/i;
+const CONTAINS_PSEUDO_RE = /:contains\s*\(/i;
+
+function quotedSegmentsOut(selector: string): string {
+  return selector.replace(/(["'])(?:\\.|(?!\1)[\s\S])*\1/g, '""');
+}
+
+// `.class` and `#id` tokens are identifiers; `[attr="value"]` values are
+// substring matches against whatever the site emitted and are exempt.
+function identifierTokens(selector: string): string[] {
+  return [...quotedSegmentsOut(selector).matchAll(/[.#]([A-Za-z0-9_-]+)/g)].map(
+    match => match[1],
+  );
+}
+
+export function lintSelectorText(
+  selector: string,
+): SelectorViolation | undefined {
+  if (CONTAINS_PSEUDO_RE.test(selector)) {
+    return {
+      kind: 'contains-pseudo',
+      selector,
+      detail: ':contains is not standard CSS and changes the result silently',
+    };
+  }
+  const positional = POSITIONAL_PSEUDO_RE.exec(selector);
+  if (positional) {
+    return {
+      kind: 'positional-pseudo',
+      selector,
+      detail: `${positional[0]} depends on this page's child order, not the site's layout`,
+    };
+  }
+  for (const token of identifierTokens(selector)) {
+    if (isGeneratedIdentifier(token)) {
+      return {
+        kind: 'generated-identifier',
+        selector,
+        detail: `"${token}" reads as a generated hash that changes on deploy`,
+      };
+    }
+  }
+  return undefined;
+}
+
+// Propose-time validation for a preset a suggester wants stored. Stricter than
+// presetMatches: an exclude that matches nothing is tolerated at runtime (the
+// debris is absent from some pages) but at propose time it is a mistake — the
+// proposal names debris that is not there.
+export function lintProposal({
+  document,
+  detectors,
+  scope,
+}: ProposalInput): readonly SelectorViolation[] {
+  const violations: SelectorViolation[] = [];
+  const selectors = [
+    ...detectors,
+    ...(scope.include ? [scope.include] : []),
+    ...(scope.exclude ?? []),
+  ];
+  for (const selector of selectors) {
+    const violation = lintSelectorText(selector);
+    if (violation) {
+      violations.push(violation);
+    }
+  }
+
+  let includeRoot: Element | undefined;
+  if (
+    scope.include &&
+    !violations.some(violation => violation.selector === scope.include)
+  ) {
+    try {
+      includeRoot = document.body.querySelector(scope.include) ?? undefined;
+    } catch {
+      violations.push({
+        kind: 'unparseable',
+        selector: scope.include,
+        detail: 'the selector engine rejects this selector',
+      });
+    }
+  }
+
+  for (const detector of detectors) {
+    if (violations.some(violation => violation.selector === detector)) {
+      continue;
+    }
+    if (selectorMisses(document, detector)) {
+      violations.push({
+        kind: 'no-match',
+        selector: detector,
+        detail: 'detector matches nothing on this page',
+      });
+    }
+  }
+  if (
+    scope.include &&
+    !includeRoot &&
+    !violations.some(violation => violation.selector === scope.include)
+  ) {
+    violations.push({
+      kind: 'no-match',
+      selector: scope.include,
+      detail:
+        'include matches nothing inside <body>, where applySelectors searches',
+    });
+  }
+  for (const selector of scope.exclude ?? []) {
+    if (violations.some(violation => violation.selector === selector)) {
+      continue;
+    }
+    try {
+      if (document.querySelectorAll(selector).length === 0) {
+        violations.push({
+          kind: 'no-match',
+          selector,
+          detail: 'exclude matches nothing on this page',
+        });
+      }
+    } catch {
+      violations.push({
+        kind: 'unparseable',
+        selector,
+        detail: 'the selector engine rejects this selector',
+      });
+    }
+  }
+
+  // Excludes are removed document-wide before the include is applied, so an
+  // exclude matching an ancestor of the include root (or the root itself)
+  // deletes it, and the include then quietly no-ops over a damaged body.
+  if (includeRoot) {
+    for (const selector of scope.exclude ?? []) {
+      if (violations.some(violation => violation.selector === selector)) {
+        continue;
+      }
+      try {
+        for (const match of document.querySelectorAll(selector)) {
+          if (match.contains(includeRoot)) {
+            violations.push({
+              kind: 'exclude-shadows-include',
+              selector,
+              detail: 'this exclude removes the include root itself',
+            });
+            break;
+          }
+        }
+      } catch {
+        // Already reported as unparseable.
+      }
+    }
+  }
+
+  return violations;
+}

package/src/policy/sibling-scan.ts

@@ -0,0 +1,99 @@
+import { isElement } from '../pipeline/dom.ts';
+
+// Shared pre-pass for the sibling-cluster detectors (list-detector,
+// grid-detector). Both walk the document for a container whose direct children
+// form a same-shape sibling group; the constants and helpers here are the
+// heuristic-agnostic infrastructure that scan shares. Detector-specific scoring
+// (anchor/pathname signals vs data-cell text) stays in each detector.
+
+// Direct children are scanned only at these container levels. <tbody> covers
+// HN's <table>-based layout; the rest cover semantic (<main>/<article>) and
+// generic (<div>/<section>) wrappers. <table> is included for HTML that skips
+// <tbody>; jsdom synthesizes one anyway, so this is mostly defensive.
+export const CONTAINER_TAGS = new Set([
+  'ARTICLE',
+  'DIV',
+  'MAIN',
+  'OL',
+  'SECTION',
+  'TABLE',
+  'TBODY',
+  'UL',
+]);
+
+// Subtrees that never carry a page's primary repeated structure. Stripping them
+// is the false-positive guard: without it, an article's <nav> menu or footer
+// link list would look identical to a feed/grid and the detectors would mis-fire
+// on every page with chrome. Distinct from normalize.ts's stripChrome (which
+// removes consent/overlay banners) — this is landmark chrome only.
+const LANDMARK_CHROME_SELECTOR =
+  'nav, header, footer, aside, [role="navigation"], [role="banner"], ' +
+  '[role="contentinfo"], [role="complementary"], [role="search"], ' +
+  '[role="menu"], [role="menubar"]';
+
+export function stripLandmarkChrome(document: Document): void {
+  for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) {
+    el.remove();
+  }
+  for (const el of document.querySelectorAll('script, style, template')) {
+    el.remove();
+  }
+}
+
+// Composite of tag + class signature so mixed siblings — e.g. HN's
+// <tr class="athing"> title row vs classless <tr> subtext row, or a grid's
+// header row vs data rows — split into separate candidate groups instead of
+// blending into one ragged group.
+export function shapeKey(el: Element): string {
+  const raw = el.getAttribute('class');
+  if (!raw) {
+    return el.tagName;
+  }
+  const normalized = raw.trim().split(/\s+/).sort().join(' ');
+  return normalized ? `${el.tagName}|${normalized}` : el.tagName;
+}
+
+// Group a container's direct element-children by shape so a homogeneous
+// cluster surfaces as one candidate and mixed-shape siblings (header row vs
+// data rows, HN's athing + subtext) split apart rather than blending.
+export function groupChildrenByShape(container: Node): Map<string, Element[]> {
+  const groups = new Map<string, Element[]>();
+  for (const child of Array.from(container.childNodes)) {
+    if (!isElement(child)) {
+      continue;
+    }
+    const key = shapeKey(child);
+    const bucket = groups.get(key);
+    if (bucket) {
+      bucket.push(child);
+    } else {
+      groups.set(key, [child]);
+    }
+  }
+  return groups;
+}
+
+// tag#id.class hint for the winning container. Best-effort, not a unique
+// locator — used only for diagnostics so a human can see which subtree won.
+// `maxClasses` caps the class list for compact diagnostic output.
+export function describeSelector(el: Element, maxClasses?: number): string {
+  const parts = [el.tagName.toLowerCase()];
+  const id = el.getAttribute('id');
+  if (id) {
+    parts.push(`#${id}`);
+  }
+  const cls = el.getAttribute('class');
+  if (cls) {
+    let used = 0;
+    for (const token of cls.trim().split(/\s+/)) {
+      if (token) {
+        parts.push(`.${token}`);
+        used += 1;
+      }
+      if (maxClasses !== undefined && used >= maxClasses) {
+        break;
+      }
+    }
+  }
+  return parts.join('');
+}

package/src/policy/text.ts

@@ -0,0 +1,36 @@
+export const TOKEN_ESTIMATOR = 'chars/4';
+
+export function estimateTokens(textContent: string): {
+  estimator: string;
+  tokenEstimate: number;
+} {
+  return {
+    tokenEstimate: Math.round(textContent.length / 4),
+    estimator: TOKEN_ESTIMATOR,
+  };
+}
+
+export function countWords(text: string): number {
+  return (text.match(/\S+/g) ?? []).length;
+}
+
+export function nonEmpty(value: string | undefined): string | undefined {
+  return value?.trim() ? value : undefined;
+}
+
+export interface TextMetrics {
+  readonly estimator: string;
+  readonly readingTimeMin: number;
+  readonly tokenEstimate: number;
+  readonly wordCount: number;
+}
+
+export function computeTextMetrics(
+  text: string,
+  wordsPerMinute: number,
+): TextMetrics {
+  const wordCount = countWords(text);
+  const readingTimeMin =
+    wordCount === 0 ? 0 : Math.max(1, Math.round(wordCount / wordsPerMinute));
+  return { wordCount, readingTimeMin, ...estimateTokens(text) };
+}