@v1nvn/readability 0.44.0 → 0.45.1

package/src/policy/section.ts

@@ -0,0 +1,127 @@
+import { isElement } from '../pipeline/dom.ts';
+import { normalizeHeadingText } from './outline.ts';
+
+interface HeadingMatch {
+  readonly heading: Element;
+  readonly level: number;
+}
+
+// Match the first heading by normalized text. Exact equality wins; otherwise
+// fall back to the first substring match so a short query ("auth") still
+// lands on a longer heading ("Authentication") without forcing an exact match.
+function findHeading(
+  document: Document,
+  query: string,
+): HeadingMatch | undefined {
+  const needle = normalizeHeadingText(query).toLowerCase();
+  if (!needle) {
+    return undefined;
+  }
+  const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+  let substring: HeadingMatch | undefined;
+  for (const heading of headings) {
+    const text = normalizeHeadingText(heading.textContent).toLowerCase();
+    if (!text) {
+      continue;
+    }
+    const level = Number.parseInt(heading.tagName.slice(1), 10);
+    if (text === needle) {
+      return { heading, level };
+    }
+    if (!substring && text.includes(needle)) {
+      substring = { heading, level };
+    }
+  }
+  return substring;
+}
+
+// Elements whose direct children form the section flow. The matched heading's
+// nearest such ancestor is the level at which the section's content lives as
+// siblings: on docs sites (GitHub markdown, etc.) the heading is wrapped in
+// <div class="markdown-heading">, so the body <p> is a sibling of the wrapper
+// div, not of the <h2>. Walking the <h2>'s own siblings would capture only the
+// permalink anchor and miss the body.
+const FLOW_CONTAINERS = new Set([
+  'ARTICLE',
+  'ASIDE',
+  'BLOCKQUOTE',
+  'BODY',
+  'DD',
+  'DETAILS',
+  'DT',
+  'FOOTER',
+  'HEADER',
+  'LI',
+  'MAIN',
+  'NAV',
+  'SECTION',
+  'TD',
+]);
+
+function findFlowContainer(heading: Element): Element {
+  let ancestor: Element | null = heading.parentElement;
+  while (ancestor) {
+    if (FLOW_CONTAINERS.has(ancestor.tagName)) {
+      return ancestor;
+    }
+    ancestor = ancestor.parentElement;
+  }
+  // BODY is in FLOW_CONTAINERS, so the loop reaches it for any attached
+  // heading; this fallback only covers a detached heading.
+  return heading.ownerDocument.body;
+}
+
+// Level of the first heading at or inside `el`, or undefined when `el` is
+// neither a heading nor a wrapper around one. Used to test whether a following
+// sibling terminates the section.
+function firstHeadingLevel(el: Element): number | undefined {
+  const direct = /^H([1-6])$/.exec(el.tagName);
+  if (direct) {
+    return Number.parseInt(direct[1], 10);
+  }
+  const inner = el.querySelector('h1, h2, h3, h4, h5, h6');
+  if (inner) {
+    return Number.parseInt(inner.tagName.slice(1), 10);
+  }
+  return undefined;
+}
+
+// Wraps the matched heading and its subtree in a `<section data-rdrm-section-scope>`
+// so the existing selectors.include path can isolate it. Returns false (and
+// leaves the document untouched) when no heading matches.
+export function scopeToHeading(
+  document: Document,
+  headingText: string,
+): boolean {
+  const match = findHeading(document, headingText);
+  if (!match) {
+    return false;
+  }
+  const { heading, level } = match;
+  const container = findFlowContainer(heading);
+  let startChild: Element = heading;
+  while (startChild.parentElement && startChild.parentElement !== container) {
+    startChild = startChild.parentElement;
+  }
+  const wrap = document.createElement('section');
+  wrap.setAttribute('data-rdrm-section-scope', '');
+  container.insertBefore(wrap, startChild);
+  let node: Node | null = startChild;
+  while (node) {
+    const next: Node | null = node.nextSibling;
+    wrap.appendChild(node);
+    // A section ends at the next same-or-higher-level heading (level <= L):
+    // deeper headings (h3 under an h2) belong to this section, peers and
+    // shallower headings start the next one. The terminating sibling may be a
+    // wrapper around the heading (e.g. <div class="markdown-heading">), so the
+    // check looks for a heading at OR inside the sibling.
+    if (next !== null && isElement(next)) {
+      const nextLevel = firstHeadingLevel(next);
+      if (nextLevel !== undefined && nextLevel <= level) {
+        break;
+      }
+    }
+    node = next;
+  }
+  return true;
+}

package/src/policy/selector-lint.js

@@ -0,0 +1,143 @@
+import { isGeneratedIdentifier } from "./identifiers.js";
+import { selectorMisses } from "./presets.js";
+// nwsapi/jsdom accept `:contains` and silently apply it, so an engine error can
+// reject nothing — the rejection has to happen here, before any selector
+// reaches querySelector. Positional pseudos are rejected as a family: they
+// describe this render's child order, not the site's structure.
+const POSITIONAL_PSEUDO_RE = /:(?:nth(?:-last)?(?:-child|-of-type)|first(?:-child|-of-type)|last(?:-child|-of-type)|only(?:-child|-of-type))\b/i;
+const CONTAINS_PSEUDO_RE = /:contains\s*\(/i;
+function quotedSegmentsOut(selector) {
+    return selector.replace(/(["'])(?:\\.|(?!\1)[\s\S])*\1/g, '""');
+}
+// `.class` and `#id` tokens are identifiers; `[attr="value"]` values are
+// substring matches against whatever the site emitted and are exempt.
+function identifierTokens(selector) {
+    return [...quotedSegmentsOut(selector).matchAll(/[.#]([A-Za-z0-9_-]+)/g)].map(match => match[1]);
+}
+export function lintSelectorText(selector) {
+    if (CONTAINS_PSEUDO_RE.test(selector)) {
+        return {
+            kind: 'contains-pseudo',
+            selector,
+            detail: ':contains is not standard CSS and changes the result silently',
+        };
+    }
+    const positional = POSITIONAL_PSEUDO_RE.exec(selector);
+    if (positional) {
+        return {
+            kind: 'positional-pseudo',
+            selector,
+            detail: `${positional[0]} depends on this page's child order, not the site's layout`,
+        };
+    }
+    for (const token of identifierTokens(selector)) {
+        if (isGeneratedIdentifier(token)) {
+            return {
+                kind: 'generated-identifier',
+                selector,
+                detail: `"${token}" reads as a generated hash that changes on deploy`,
+            };
+        }
+    }
+    return undefined;
+}
+// Propose-time validation for a preset a suggester wants stored. Stricter than
+// presetMatches: an exclude that matches nothing is tolerated at runtime (the
+// debris is absent from some pages) but at propose time it is a mistake — the
+// proposal names debris that is not there.
+export function lintProposal({ document, detectors, scope, }) {
+    const violations = [];
+    const selectors = [
+        ...detectors,
+        ...(scope.include ? [scope.include] : []),
+        ...(scope.exclude ?? []),
+    ];
+    for (const selector of selectors) {
+        const violation = lintSelectorText(selector);
+        if (violation) {
+            violations.push(violation);
+        }
+    }
+    let includeRoot;
+    if (scope.include &&
+        !violations.some(violation => violation.selector === scope.include)) {
+        try {
+            includeRoot = document.body.querySelector(scope.include) ?? undefined;
+        }
+        catch {
+            violations.push({
+                kind: 'unparseable',
+                selector: scope.include,
+                detail: 'the selector engine rejects this selector',
+            });
+        }
+    }
+    for (const detector of detectors) {
+        if (violations.some(violation => violation.selector === detector)) {
+            continue;
+        }
+        if (selectorMisses(document, detector)) {
+            violations.push({
+                kind: 'no-match',
+                selector: detector,
+                detail: 'detector matches nothing on this page',
+            });
+        }
+    }
+    if (scope.include &&
+        !includeRoot &&
+        !violations.some(violation => violation.selector === scope.include)) {
+        violations.push({
+            kind: 'no-match',
+            selector: scope.include,
+            detail: 'include matches nothing inside <body>, where applySelectors searches',
+        });
+    }
+    for (const selector of scope.exclude ?? []) {
+        if (violations.some(violation => violation.selector === selector)) {
+            continue;
+        }
+        try {
+            if (document.querySelectorAll(selector).length === 0) {
+                violations.push({
+                    kind: 'no-match',
+                    selector,
+                    detail: 'exclude matches nothing on this page',
+                });
+            }
+        }
+        catch {
+            violations.push({
+                kind: 'unparseable',
+                selector,
+                detail: 'the selector engine rejects this selector',
+            });
+        }
+    }
+    // Excludes are removed document-wide before the include is applied, so an
+    // exclude matching an ancestor of the include root (or the root itself)
+    // deletes it, and the include then quietly no-ops over a damaged body.
+    if (includeRoot) {
+        for (const selector of scope.exclude ?? []) {
+            if (violations.some(violation => violation.selector === selector)) {
+                continue;
+            }
+            try {
+                for (const match of document.querySelectorAll(selector)) {
+                    if (match.contains(includeRoot)) {
+                        violations.push({
+                            kind: 'exclude-shadows-include',
+                            selector,
+                            detail: 'this exclude removes the include root itself',
+                        });
+                        break;
+                    }
+                }
+            }
+            catch {
+                // Already reported as unparseable.
+            }
+        }
+    }
+    return violations;
+}

package/src/policy/selector-lint.ts

@@ -0,0 +1,186 @@
+import { isGeneratedIdentifier } from './identifiers.ts';
+import { selectorMisses } from './presets.ts';
+
+export type SelectorViolationKind =
+  | 'contains-pseudo'
+  | 'exclude-shadows-include'
+  | 'generated-identifier'
+  | 'no-match'
+  | 'positional-pseudo'
+  | 'unparseable';
+
+export interface SelectorViolation {
+  readonly detail: string;
+  readonly kind: SelectorViolationKind;
+  readonly selector: string;
+}
+
+export interface ProposalInput {
+  readonly detectors: readonly string[];
+  readonly document: Document;
+  readonly scope: {
+    readonly exclude?: readonly string[];
+    readonly include?: string;
+  };
+}
+
+// nwsapi/jsdom accept `:contains` and silently apply it, so an engine error can
+// reject nothing — the rejection has to happen here, before any selector
+// reaches querySelector. Positional pseudos are rejected as a family: they
+// describe this render's child order, not the site's structure.
+const POSITIONAL_PSEUDO_RE =
+  /:(?:nth(?:-last)?(?:-child|-of-type)|first(?:-child|-of-type)|last(?:-child|-of-type)|only(?:-child|-of-type))\b/i;
+const CONTAINS_PSEUDO_RE = /:contains\s*\(/i;
+
+function quotedSegmentsOut(selector: string): string {
+  return selector.replace(/(["'])(?:\\.|(?!\1)[\s\S])*\1/g, '""');
+}
+
+// `.class` and `#id` tokens are identifiers; `[attr="value"]` values are
+// substring matches against whatever the site emitted and are exempt.
+function identifierTokens(selector: string): string[] {
+  return [...quotedSegmentsOut(selector).matchAll(/[.#]([A-Za-z0-9_-]+)/g)].map(
+    match => match[1],
+  );
+}
+
+export function lintSelectorText(
+  selector: string,
+): SelectorViolation | undefined {
+  if (CONTAINS_PSEUDO_RE.test(selector)) {
+    return {
+      kind: 'contains-pseudo',
+      selector,
+      detail: ':contains is not standard CSS and changes the result silently',
+    };
+  }
+  const positional = POSITIONAL_PSEUDO_RE.exec(selector);
+  if (positional) {
+    return {
+      kind: 'positional-pseudo',
+      selector,
+      detail: `${positional[0]} depends on this page's child order, not the site's layout`,
+    };
+  }
+  for (const token of identifierTokens(selector)) {
+    if (isGeneratedIdentifier(token)) {
+      return {
+        kind: 'generated-identifier',
+        selector,
+        detail: `"${token}" reads as a generated hash that changes on deploy`,
+      };
+    }
+  }
+  return undefined;
+}
+
+// Propose-time validation for a preset a suggester wants stored. Stricter than
+// presetMatches: an exclude that matches nothing is tolerated at runtime (the
+// debris is absent from some pages) but at propose time it is a mistake — the
+// proposal names debris that is not there.
+export function lintProposal({
+  document,
+  detectors,
+  scope,
+}: ProposalInput): readonly SelectorViolation[] {
+  const violations: SelectorViolation[] = [];
+  const selectors = [
+    ...detectors,
+    ...(scope.include ? [scope.include] : []),
+    ...(scope.exclude ?? []),
+  ];
+  for (const selector of selectors) {
+    const violation = lintSelectorText(selector);
+    if (violation) {
+      violations.push(violation);
+    }
+  }
+
+  let includeRoot: Element | undefined;
+  if (
+    scope.include &&
+    !violations.some(violation => violation.selector === scope.include)
+  ) {
+    try {
+      includeRoot = document.body.querySelector(scope.include) ?? undefined;
+    } catch {
+      violations.push({
+        kind: 'unparseable',
+        selector: scope.include,
+        detail: 'the selector engine rejects this selector',
+      });
+    }
+  }
+
+  for (const detector of detectors) {
+    if (violations.some(violation => violation.selector === detector)) {
+      continue;
+    }
+    if (selectorMisses(document, detector)) {
+      violations.push({
+        kind: 'no-match',
+        selector: detector,
+        detail: 'detector matches nothing on this page',
+      });
+    }
+  }
+  if (
+    scope.include &&
+    !includeRoot &&
+    !violations.some(violation => violation.selector === scope.include)
+  ) {
+    violations.push({
+      kind: 'no-match',
+      selector: scope.include,
+      detail:
+        'include matches nothing inside <body>, where applySelectors searches',
+    });
+  }
+  for (const selector of scope.exclude ?? []) {
+    if (violations.some(violation => violation.selector === selector)) {
+      continue;
+    }
+    try {
+      if (document.querySelectorAll(selector).length === 0) {
+        violations.push({
+          kind: 'no-match',
+          selector,
+          detail: 'exclude matches nothing on this page',
+        });
+      }
+    } catch {
+      violations.push({
+        kind: 'unparseable',
+        selector,
+        detail: 'the selector engine rejects this selector',
+      });
+    }
+  }
+
+  // Excludes are removed document-wide before the include is applied, so an
+  // exclude matching an ancestor of the include root (or the root itself)
+  // deletes it, and the include then quietly no-ops over a damaged body.
+  if (includeRoot) {
+    for (const selector of scope.exclude ?? []) {
+      if (violations.some(violation => violation.selector === selector)) {
+        continue;
+      }
+      try {
+        for (const match of document.querySelectorAll(selector)) {
+          if (match.contains(includeRoot)) {
+            violations.push({
+              kind: 'exclude-shadows-include',
+              selector,
+              detail: 'this exclude removes the include root itself',
+            });
+            break;
+          }
+        }
+      } catch {
+        // Already reported as unparseable.
+      }
+    }
+  }
+
+  return violations;
+}

package/src/policy/sibling-scan.js

@@ -0,0 +1,92 @@
+import { isElement } from "../pipeline/dom.js";
+// Shared pre-pass for the sibling-cluster detectors (list-detector,
+// grid-detector). Both walk the document for a container whose direct children
+// form a same-shape sibling group; the constants and helpers here are the
+// heuristic-agnostic infrastructure that scan shares. Detector-specific scoring
+// (anchor/pathname signals vs data-cell text) stays in each detector.
+// Direct children are scanned only at these container levels. <tbody> covers
+// HN's <table>-based layout; the rest cover semantic (<main>/<article>) and
+// generic (<div>/<section>) wrappers. <table> is included for HTML that skips
+// <tbody>; jsdom synthesizes one anyway, so this is mostly defensive.
+export const CONTAINER_TAGS = new Set([
+    'ARTICLE',
+    'DIV',
+    'MAIN',
+    'OL',
+    'SECTION',
+    'TABLE',
+    'TBODY',
+    'UL',
+]);
+// Subtrees that never carry a page's primary repeated structure. Stripping them
+// is the false-positive guard: without it, an article's <nav> menu or footer
+// link list would look identical to a feed/grid and the detectors would mis-fire
+// on every page with chrome. Distinct from normalize.ts's stripChrome (which
+// removes consent/overlay banners) — this is landmark chrome only.
+const LANDMARK_CHROME_SELECTOR = 'nav, header, footer, aside, [role="navigation"], [role="banner"], ' +
+    '[role="contentinfo"], [role="complementary"], [role="search"], ' +
+    '[role="menu"], [role="menubar"]';
+export function stripLandmarkChrome(document) {
+    for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) {
+        el.remove();
+    }
+    for (const el of document.querySelectorAll('script, style, template')) {
+        el.remove();
+    }
+}
+// Composite of tag + class signature so mixed siblings — e.g. HN's
+// <tr class="athing"> title row vs classless <tr> subtext row, or a grid's
+// header row vs data rows — split into separate candidate groups instead of
+// blending into one ragged group.
+export function shapeKey(el) {
+    const raw = el.getAttribute('class');
+    if (!raw) {
+        return el.tagName;
+    }
+    const normalized = raw.trim().split(/\s+/).sort().join(' ');
+    return normalized ? `${el.tagName}|${normalized}` : el.tagName;
+}
+// Group a container's direct element-children by shape so a homogeneous
+// cluster surfaces as one candidate and mixed-shape siblings (header row vs
+// data rows, HN's athing + subtext) split apart rather than blending.
+export function groupChildrenByShape(container) {
+    const groups = new Map();
+    for (const child of Array.from(container.childNodes)) {
+        if (!isElement(child)) {
+            continue;
+        }
+        const key = shapeKey(child);
+        const bucket = groups.get(key);
+        if (bucket) {
+            bucket.push(child);
+        }
+        else {
+            groups.set(key, [child]);
+        }
+    }
+    return groups;
+}
+// tag#id.class hint for the winning container. Best-effort, not a unique
+// locator — used only for diagnostics so a human can see which subtree won.
+// `maxClasses` caps the class list for compact diagnostic output.
+export function describeSelector(el, maxClasses) {
+    const parts = [el.tagName.toLowerCase()];
+    const id = el.getAttribute('id');
+    if (id) {
+        parts.push(`#${id}`);
+    }
+    const cls = el.getAttribute('class');
+    if (cls) {
+        let used = 0;
+        for (const token of cls.trim().split(/\s+/)) {
+            if (token) {
+                parts.push(`.${token}`);
+                used += 1;
+            }
+            if (maxClasses !== undefined && used >= maxClasses) {
+                break;
+            }
+        }
+    }
+    return parts.join('');
+}

package/src/policy/sibling-scan.ts

@@ -0,0 +1,99 @@
+import { isElement } from '../pipeline/dom.ts';
+
+// Shared pre-pass for the sibling-cluster detectors (list-detector,
+// grid-detector). Both walk the document for a container whose direct children
+// form a same-shape sibling group; the constants and helpers here are the
+// heuristic-agnostic infrastructure that scan shares. Detector-specific scoring
+// (anchor/pathname signals vs data-cell text) stays in each detector.
+
+// Direct children are scanned only at these container levels. <tbody> covers
+// HN's <table>-based layout; the rest cover semantic (<main>/<article>) and
+// generic (<div>/<section>) wrappers. <table> is included for HTML that skips
+// <tbody>; jsdom synthesizes one anyway, so this is mostly defensive.
+export const CONTAINER_TAGS = new Set([
+  'ARTICLE',
+  'DIV',
+  'MAIN',
+  'OL',
+  'SECTION',
+  'TABLE',
+  'TBODY',
+  'UL',
+]);
+
+// Subtrees that never carry a page's primary repeated structure. Stripping them
+// is the false-positive guard: without it, an article's <nav> menu or footer
+// link list would look identical to a feed/grid and the detectors would mis-fire
+// on every page with chrome. Distinct from normalize.ts's stripChrome (which
+// removes consent/overlay banners) — this is landmark chrome only.
+const LANDMARK_CHROME_SELECTOR =
+  'nav, header, footer, aside, [role="navigation"], [role="banner"], ' +
+  '[role="contentinfo"], [role="complementary"], [role="search"], ' +
+  '[role="menu"], [role="menubar"]';
+
+export function stripLandmarkChrome(document: Document): void {
+  for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) {
+    el.remove();
+  }
+  for (const el of document.querySelectorAll('script, style, template')) {
+    el.remove();
+  }
+}
+
+// Composite of tag + class signature so mixed siblings — e.g. HN's
+// <tr class="athing"> title row vs classless <tr> subtext row, or a grid's
+// header row vs data rows — split into separate candidate groups instead of
+// blending into one ragged group.
+export function shapeKey(el: Element): string {
+  const raw = el.getAttribute('class');
+  if (!raw) {
+    return el.tagName;
+  }
+  const normalized = raw.trim().split(/\s+/).sort().join(' ');
+  return normalized ? `${el.tagName}|${normalized}` : el.tagName;
+}
+
+// Group a container's direct element-children by shape so a homogeneous
+// cluster surfaces as one candidate and mixed-shape siblings (header row vs
+// data rows, HN's athing + subtext) split apart rather than blending.
+export function groupChildrenByShape(container: Node): Map<string, Element[]> {
+  const groups = new Map<string, Element[]>();
+  for (const child of Array.from(container.childNodes)) {
+    if (!isElement(child)) {
+      continue;
+    }
+    const key = shapeKey(child);
+    const bucket = groups.get(key);
+    if (bucket) {
+      bucket.push(child);
+    } else {
+      groups.set(key, [child]);
+    }
+  }
+  return groups;
+}
+
+// tag#id.class hint for the winning container. Best-effort, not a unique
+// locator — used only for diagnostics so a human can see which subtree won.
+// `maxClasses` caps the class list for compact diagnostic output.
+export function describeSelector(el: Element, maxClasses?: number): string {
+  const parts = [el.tagName.toLowerCase()];
+  const id = el.getAttribute('id');
+  if (id) {
+    parts.push(`#${id}`);
+  }
+  const cls = el.getAttribute('class');
+  if (cls) {
+    let used = 0;
+    for (const token of cls.trim().split(/\s+/)) {
+      if (token) {
+        parts.push(`.${token}`);
+        used += 1;
+      }
+      if (maxClasses !== undefined && used >= maxClasses) {
+        break;
+      }
+    }
+  }
+  return parts.join('');
+}

package/src/policy/text.js

@@ -0,0 +1,18 @@
+export const TOKEN_ESTIMATOR = 'chars/4';
+export function estimateTokens(textContent) {
+    return {
+        tokenEstimate: Math.round(textContent.length / 4),
+        estimator: TOKEN_ESTIMATOR,
+    };
+}
+export function countWords(text) {
+    return (text.match(/\S+/g) ?? []).length;
+}
+export function nonEmpty(value) {
+    return value?.trim() ? value : undefined;
+}
+export function computeTextMetrics(text, wordsPerMinute) {
+    const wordCount = countWords(text);
+    const readingTimeMin = wordCount === 0 ? 0 : Math.max(1, Math.round(wordCount / wordsPerMinute));
+    return { wordCount, readingTimeMin, ...estimateTokens(text) };
+}