@v1nvn/readability 0.44.0 → 0.45.1

package/src/policy/outline.js

@@ -0,0 +1,71 @@
+// Collapse whitespace, trim, then strip leading/trailing `#` (GitHub permalinks
+// append `<a>#</a>`, so heading textContent ends with `#`).
+export function normalizeHeadingText(raw) {
+    const collapsed = raw.replace(/\s+/g, ' ').trim();
+    return collapsed.replace(/^#+|#+$/g, '').trim();
+}
+// GFM anchor algorithm (comrak `anchorize`): keep Unicode letters, marks,
+// numbers, and connector punctuation; map spaces to hyphens. No hyphen collapse
+// or trim, so anchors match GitHub and non-Latin scripts survive. An all-symbol
+// heading yields an empty slug; fall back to `section` so anchors stay non-empty.
+function slugify(text) {
+    const cleaned = text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, '');
+    return cleaned.replace(/ /g, '-') || 'section';
+}
+// `href="#"` (empty fragment) does not win — fall through to slugify.
+function linkAnchor(heading) {
+    for (const link of heading.querySelectorAll('a[href^="#"]')) {
+        const href = link.getAttribute('href');
+        if (!href) {
+            continue;
+        }
+        const fragment = href.slice(1);
+        if (fragment) {
+            return fragment;
+        }
+    }
+    return undefined;
+}
+function resolveCandidate(heading, text) {
+    const id = heading.getAttribute('id');
+    if (id) {
+        return { anchor: id, explicit: true };
+    }
+    const link = linkAnchor(heading);
+    if (link) {
+        return { anchor: link, explicit: true };
+    }
+    return { anchor: slugify(text), explicit: false };
+}
+// Explicit anchors are kept verbatim; generated slugs are collision-suffixed
+// (`-1`, `-2`, …), first occurrence bare.
+function dedupe(candidate, explicit, used) {
+    if (explicit || !used.has(candidate)) {
+        return candidate;
+    }
+    let suffix = 1;
+    while (used.has(`${candidate}-${suffix}`)) {
+        suffix++;
+    }
+    return `${candidate}-${suffix}`;
+}
+export function resolveOutline(document) {
+    const entries = [];
+    const used = new Set();
+    const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+    headings.forEach(heading => {
+        const rawText = heading.textContent;
+        if (!rawText.trim()) {
+            return;
+        }
+        const text = normalizeHeadingText(rawText);
+        const level = Number.parseInt(heading.tagName.slice(1), 10);
+        const { anchor: candidate, explicit } = resolveCandidate(heading, text);
+        const anchor = dedupe(candidate, explicit, used);
+        used.add(anchor);
+        entries.push({ anchor, level, text });
+    });
+    return entries;
+}

package/src/policy/outline.ts

@@ -0,0 +1,89 @@
+export interface OutlineEntry {
+  readonly anchor: string;
+  readonly level: number;
+  readonly text: string;
+}
+
+// Collapse whitespace, trim, then strip leading/trailing `#` (GitHub permalinks
+// append `<a>#</a>`, so heading textContent ends with `#`).
+export function normalizeHeadingText(raw: string): string {
+  const collapsed = raw.replace(/\s+/g, ' ').trim();
+  return collapsed.replace(/^#+|#+$/g, '').trim();
+}
+
+// GFM anchor algorithm (comrak `anchorize`): keep Unicode letters, marks,
+// numbers, and connector punctuation; map spaces to hyphens. No hyphen collapse
+// or trim, so anchors match GitHub and non-Latin scripts survive. An all-symbol
+// heading yields an empty slug; fall back to `section` so anchors stay non-empty.
+function slugify(text: string): string {
+  const cleaned = text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, '');
+  return cleaned.replace(/ /g, '-') || 'section';
+}
+
+// `href="#"` (empty fragment) does not win — fall through to slugify.
+function linkAnchor(heading: Element): string | undefined {
+  for (const link of heading.querySelectorAll('a[href^="#"]')) {
+    const href = link.getAttribute('href');
+    if (!href) {
+      continue;
+    }
+    const fragment = href.slice(1);
+    if (fragment) {
+      return fragment;
+    }
+  }
+  return undefined;
+}
+
+function resolveCandidate(
+  heading: Element,
+  text: string,
+): { readonly anchor: string; readonly explicit: boolean } {
+  const id = heading.getAttribute('id');
+  if (id) {
+    return { anchor: id, explicit: true };
+  }
+  const link = linkAnchor(heading);
+  if (link) {
+    return { anchor: link, explicit: true };
+  }
+  return { anchor: slugify(text), explicit: false };
+}
+
+// Explicit anchors are kept verbatim; generated slugs are collision-suffixed
+// (`-1`, `-2`, …), first occurrence bare.
+function dedupe(
+  candidate: string,
+  explicit: boolean,
+  used: Set<string>,
+): string {
+  if (explicit || !used.has(candidate)) {
+    return candidate;
+  }
+  let suffix = 1;
+  while (used.has(`${candidate}-${suffix}`)) {
+    suffix++;
+  }
+  return `${candidate}-${suffix}`;
+}
+
+export function resolveOutline(document: Document): OutlineEntry[] {
+  const entries: OutlineEntry[] = [];
+  const used = new Set<string>();
+  const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+  headings.forEach(heading => {
+    const rawText = heading.textContent;
+    if (!rawText.trim()) {
+      return;
+    }
+    const text = normalizeHeadingText(rawText);
+    const level = Number.parseInt(heading.tagName.slice(1), 10);
+    const { anchor: candidate, explicit } = resolveCandidate(heading, text);
+    const anchor = dedupe(candidate, explicit, used);
+    used.add(anchor);
+    entries.push({ anchor, level, text });
+  });
+  return entries;
+}

package/src/policy/pagination.js

@@ -0,0 +1,79 @@
+import { absolutize } from "../pipeline/urls.js";
+// Short pagination-style link text only; matching prose like "next, we cover
+// rendering" would burn an article's internal link as a next-page signal.
+// `next ›/»/→` and bare arrows cover the common next-page glyphs in one place.
+const NEXT_LINK_TEXT_RE = /^(next(\s+page)?|older(\s+posts?)?|[›»→]|next\s*[›»→]|older\s*[›»→])$/i;
+function usableHref(href) {
+    if (!href || href === '#') {
+        return undefined;
+    }
+    return href;
+}
+function findPaginated(document, baseUrl) {
+    const linkNext = document.querySelector('link[rel="next"][href]');
+    if (linkNext) {
+        const href = usableHref(linkNext.getAttribute('href'));
+        if (href) {
+            return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+        }
+    }
+    const aRelNext = document.querySelector('a[rel="next"][href]');
+    if (aRelNext) {
+        const href = usableHref(aRelNext.getAttribute('href'));
+        if (href) {
+            return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+        }
+    }
+    for (const anchor of document.querySelectorAll('a[href]')) {
+        const text = anchor.textContent.trim();
+        if (!text || !NEXT_LINK_TEXT_RE.test(text)) {
+            continue;
+        }
+        const href = usableHref(anchor.getAttribute('href'));
+        if (!href) {
+            continue;
+        }
+        return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+    return undefined;
+}
+const INFINITE_ATTR_SELECTORS = [
+    '[data-load-more]',
+    '[data-infinite-scroll]',
+    '[data-pagination]',
+    '[infinite-scroll]',
+];
+const INFINITE_SUBSTRING_SELECTORS = [
+    '[class*="load-more"]',
+    '[class*="loadmore"]',
+    '[class*="infinite"]',
+    '[id*="load-more"]',
+    '[class*="sentinel"]',
+];
+const LOAD_MORE_BUTTON_RE = /^(load more|show more|view more|more results|load more comments)$/i;
+function findInfinite(document) {
+    for (const selector of INFINITE_ATTR_SELECTORS) {
+        const el = document.querySelector(selector);
+        if (el?.isConnected) {
+            return { selector, type: 'infinite' };
+        }
+    }
+    for (const selector of INFINITE_SUBSTRING_SELECTORS) {
+        const el = document.querySelector(selector);
+        if (el?.isConnected) {
+            return { selector, type: 'infinite' };
+        }
+    }
+    for (const button of document.querySelectorAll('button')) {
+        const text = button.textContent.trim();
+        if (text && LOAD_MORE_BUTTON_RE.test(text)) {
+            return { selector: 'button', type: 'infinite' };
+        }
+    }
+    return undefined;
+}
+// Paginated beats infinite: an explicit "next" link is a stronger signal than
+// a load-more sentinel. Read-only — never follows or fetches.
+export function detectPagination(document, baseUrl) {
+    return findPaginated(document, baseUrl) ?? findInfinite(document);
+}

package/src/policy/pagination.ts

@@ -0,0 +1,104 @@
+import { absolutize } from '../pipeline/urls.ts';
+
+export interface PaginationSignal {
+  readonly nextUrl?: string;
+  readonly selector?: string;
+  readonly type: 'infinite' | 'paginated';
+}
+
+// Short pagination-style link text only; matching prose like "next, we cover
+// rendering" would burn an article's internal link as a next-page signal.
+// `next ›/»/→` and bare arrows cover the common next-page glyphs in one place.
+const NEXT_LINK_TEXT_RE =
+  /^(next(\s+page)?|older(\s+posts?)?|[›»→]|next\s*[›»→]|older\s*[›»→])$/i;
+
+function usableHref(href: null | string): string | undefined {
+  if (!href || href === '#') {
+    return undefined;
+  }
+  return href;
+}
+
+function findPaginated(
+  document: Document,
+  baseUrl: string | undefined,
+): PaginationSignal | undefined {
+  const linkNext = document.querySelector('link[rel="next"][href]');
+  if (linkNext) {
+    const href = usableHref(linkNext.getAttribute('href'));
+    if (href) {
+      return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+  }
+
+  const aRelNext = document.querySelector('a[rel="next"][href]');
+  if (aRelNext) {
+    const href = usableHref(aRelNext.getAttribute('href'));
+    if (href) {
+      return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+  }
+
+  for (const anchor of document.querySelectorAll('a[href]')) {
+    const text = anchor.textContent.trim();
+    if (!text || !NEXT_LINK_TEXT_RE.test(text)) {
+      continue;
+    }
+    const href = usableHref(anchor.getAttribute('href'));
+    if (!href) {
+      continue;
+    }
+    return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+  }
+
+  return undefined;
+}
+
+const INFINITE_ATTR_SELECTORS = [
+  '[data-load-more]',
+  '[data-infinite-scroll]',
+  '[data-pagination]',
+  '[infinite-scroll]',
+] as const;
+
+const INFINITE_SUBSTRING_SELECTORS = [
+  '[class*="load-more"]',
+  '[class*="loadmore"]',
+  '[class*="infinite"]',
+  '[id*="load-more"]',
+  '[class*="sentinel"]',
+] as const;
+
+const LOAD_MORE_BUTTON_RE =
+  /^(load more|show more|view more|more results|load more comments)$/i;
+
+function findInfinite(document: Document): PaginationSignal | undefined {
+  for (const selector of INFINITE_ATTR_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (el?.isConnected) {
+      return { selector, type: 'infinite' };
+    }
+  }
+  for (const selector of INFINITE_SUBSTRING_SELECTORS) {
+    const el = document.querySelector(selector);
+    if (el?.isConnected) {
+      return { selector, type: 'infinite' };
+    }
+  }
+  for (const button of document.querySelectorAll('button')) {
+    const text = button.textContent.trim();
+    if (text && LOAD_MORE_BUTTON_RE.test(text)) {
+      return { selector: 'button', type: 'infinite' };
+    }
+  }
+  return undefined;
+}
+
+// Paginated beats infinite: an explicit "next" link is a stronger signal than
+// a load-more sentinel. Read-only — never follows or fetches.
+export function detectPagination(
+  document: Document,
+  baseUrl?: string,
+): PaginationSignal | undefined {
+  return findPaginated(document, baseUrl) ?? findInfinite(document);
+}

package/src/policy/presets.js

@@ -0,0 +1,100 @@
+const presets = new Map();
+// Bumped on every store mutation so cache keys can fold in preset state
+// without hashing the resolved scope: an entry extracted before a preset
+// landed must never be served after it.
+let generation = 0;
+export function presetGeneration() {
+    return generation;
+}
+export function resetPresets() {
+    presets.clear();
+    generation += 1;
+}
+export function addPreset(preset) {
+    const key = normalizeSiteKey(preset.site);
+    if (!key) {
+        throw new Error(`invalid preset site: ${preset.site}`);
+    }
+    presets.set(key, preset);
+    generation += 1;
+}
+export function removePreset(site) {
+    const key = normalizeSiteKey(site);
+    const removed = key !== undefined && presets.delete(key);
+    if (removed) {
+        generation += 1;
+    }
+    return removed;
+}
+export function presetForSite(site) {
+    const key = normalizeSiteKey(site);
+    return key ? presets.get(key) : undefined;
+}
+// Accepts a full URL or a bare host; `www.` is the one prefix every site's own
+// pages agree on, so exactly one strip — `www.www.com` keys as `www.com`.
+export function normalizeSiteKey(value) {
+    const raw = value?.trim();
+    if (!raw) {
+        return undefined;
+    }
+    let host;
+    try {
+        host = new URL(raw).hostname.toLowerCase();
+    }
+    catch {
+        host = raw.toLowerCase();
+    }
+    const key = host.startsWith('www.') ? host.slice('www.'.length) : host;
+    return /^[a-z0-9.-]+$/.test(key) ? key : undefined;
+}
+// A selector that throws cannot match anything, so a malformed preset selector
+// is a miss by the same predicate as a non-matching one — never a crash.
+export function selectorMisses(scope, selector) {
+    try {
+        return scope.querySelector(selector) === null;
+    }
+    catch {
+        return true;
+    }
+}
+function presetMatches(document, preset) {
+    for (const detector of preset.detectors) {
+        if (selectorMisses(document, detector)) {
+            return false;
+        }
+    }
+    // The include must still match: an unmatched include is applySelectors'
+    // silent no-op, and reporting applied:true over baseline output is a lie.
+    if (preset.scope.include &&
+        selectorMisses(document.body, preset.scope.include)) {
+        return false;
+    }
+    // Excludes must execute but need not match — the debris they name is absent
+    // from some pages of the site, and that is not staleness.
+    for (const selector of preset.scope.exclude ?? []) {
+        try {
+            document.querySelectorAll(selector);
+        }
+        catch {
+            return false;
+        }
+    }
+    return true;
+}
+export function resolvePreset(document, baseUrl, overridden) {
+    const site = normalizeSiteKey(baseUrl);
+    if (!site) {
+        return undefined;
+    }
+    const preset = presets.get(site);
+    if (!preset) {
+        return undefined;
+    }
+    if (overridden) {
+        return { signal: { applied: false, reason: 'overridden', site } };
+    }
+    if (!presetMatches(document, preset)) {
+        return { signal: { applied: false, reason: 'detectors-missed', site } };
+    }
+    return { scope: preset.scope, signal: { applied: true, site } };
+}

package/src/policy/presets.ts

@@ -0,0 +1,142 @@
+// Structural twin of the pipeline's SelectorScope — policy cannot import upward.
+export interface PresetScope {
+  readonly exclude?: readonly string[];
+  readonly include?: string;
+}
+
+export interface SitePreset {
+  readonly detectors: readonly string[];
+  readonly scope: PresetScope;
+  readonly site: string;
+}
+
+export type PresetRejection = 'detectors-missed' | 'overridden';
+
+export interface PresetSignal {
+  readonly applied: boolean;
+  readonly reason?: PresetRejection;
+  readonly site: string;
+}
+
+export interface PresetResolution {
+  readonly scope?: PresetScope;
+  readonly signal: PresetSignal;
+}
+
+const presets = new Map<string, SitePreset>();
+
+// Bumped on every store mutation so cache keys can fold in preset state
+// without hashing the resolved scope: an entry extracted before a preset
+// landed must never be served after it.
+let generation = 0;
+
+export function presetGeneration(): number {
+  return generation;
+}
+
+export function resetPresets(): void {
+  presets.clear();
+  generation += 1;
+}
+
+export function addPreset(preset: SitePreset): void {
+  const key = normalizeSiteKey(preset.site);
+  if (!key) {
+    throw new Error(`invalid preset site: ${preset.site}`);
+  }
+  presets.set(key, preset);
+  generation += 1;
+}
+
+export function removePreset(site: string): boolean {
+  const key = normalizeSiteKey(site);
+  const removed = key !== undefined && presets.delete(key);
+  if (removed) {
+    generation += 1;
+  }
+  return removed;
+}
+
+export function presetForSite(
+  site: string | undefined,
+): SitePreset | undefined {
+  const key = normalizeSiteKey(site);
+  return key ? presets.get(key) : undefined;
+}
+
+// Accepts a full URL or a bare host; `www.` is the one prefix every site's own
+// pages agree on, so exactly one strip — `www.www.com` keys as `www.com`.
+export function normalizeSiteKey(
+  value: string | undefined,
+): string | undefined {
+  const raw = value?.trim();
+  if (!raw) {
+    return undefined;
+  }
+  let host: string;
+  try {
+    host = new URL(raw).hostname.toLowerCase();
+  } catch {
+    host = raw.toLowerCase();
+  }
+  const key = host.startsWith('www.') ? host.slice('www.'.length) : host;
+  return /^[a-z0-9.-]+$/.test(key) ? key : undefined;
+}
+
+// A selector that throws cannot match anything, so a malformed preset selector
+// is a miss by the same predicate as a non-matching one — never a crash.
+export function selectorMisses(scope: ParentNode, selector: string): boolean {
+  try {
+    return scope.querySelector(selector) === null;
+  } catch {
+    return true;
+  }
+}
+
+function presetMatches(document: Document, preset: SitePreset): boolean {
+  for (const detector of preset.detectors) {
+    if (selectorMisses(document, detector)) {
+      return false;
+    }
+  }
+  // The include must still match: an unmatched include is applySelectors'
+  // silent no-op, and reporting applied:true over baseline output is a lie.
+  if (
+    preset.scope.include &&
+    selectorMisses(document.body, preset.scope.include)
+  ) {
+    return false;
+  }
+  // Excludes must execute but need not match — the debris they name is absent
+  // from some pages of the site, and that is not staleness.
+  for (const selector of preset.scope.exclude ?? []) {
+    try {
+      document.querySelectorAll(selector);
+    } catch {
+      return false;
+    }
+  }
+  return true;
+}
+
+export function resolvePreset(
+  document: Document,
+  baseUrl: string | undefined,
+  overridden: boolean,
+): PresetResolution | undefined {
+  const site = normalizeSiteKey(baseUrl);
+  if (!site) {
+    return undefined;
+  }
+  const preset = presets.get(site);
+  if (!preset) {
+    return undefined;
+  }
+  if (overridden) {
+    return { signal: { applied: false, reason: 'overridden', site } };
+  }
+  if (!presetMatches(document, preset)) {
+    return { signal: { applied: false, reason: 'detectors-missed', site } };
+  }
+  return { scope: preset.scope, signal: { applied: true, site } };
+}

package/src/policy/section.js

@@ -0,0 +1,111 @@
+import { isElement } from "../pipeline/dom.js";
+import { normalizeHeadingText } from "./outline.js";
+// Match the first heading by normalized text. Exact equality wins; otherwise
+// fall back to the first substring match so a short query ("auth") still
+// lands on a longer heading ("Authentication") without forcing an exact match.
+function findHeading(document, query) {
+    const needle = normalizeHeadingText(query).toLowerCase();
+    if (!needle) {
+        return undefined;
+    }
+    const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+    let substring;
+    for (const heading of headings) {
+        const text = normalizeHeadingText(heading.textContent).toLowerCase();
+        if (!text) {
+            continue;
+        }
+        const level = Number.parseInt(heading.tagName.slice(1), 10);
+        if (text === needle) {
+            return { heading, level };
+        }
+        if (!substring && text.includes(needle)) {
+            substring = { heading, level };
+        }
+    }
+    return substring;
+}
+// Elements whose direct children form the section flow. The matched heading's
+// nearest such ancestor is the level at which the section's content lives as
+// siblings: on docs sites (GitHub markdown, etc.) the heading is wrapped in
+// <div class="markdown-heading">, so the body <p> is a sibling of the wrapper
+// div, not of the <h2>. Walking the <h2>'s own siblings would capture only the
+// permalink anchor and miss the body.
+const FLOW_CONTAINERS = new Set([
+    'ARTICLE',
+    'ASIDE',
+    'BLOCKQUOTE',
+    'BODY',
+    'DD',
+    'DETAILS',
+    'DT',
+    'FOOTER',
+    'HEADER',
+    'LI',
+    'MAIN',
+    'NAV',
+    'SECTION',
+    'TD',
+]);
+function findFlowContainer(heading) {
+    let ancestor = heading.parentElement;
+    while (ancestor) {
+        if (FLOW_CONTAINERS.has(ancestor.tagName)) {
+            return ancestor;
+        }
+        ancestor = ancestor.parentElement;
+    }
+    // BODY is in FLOW_CONTAINERS, so the loop reaches it for any attached
+    // heading; this fallback only covers a detached heading.
+    return heading.ownerDocument.body;
+}
+// Level of the first heading at or inside `el`, or undefined when `el` is
+// neither a heading nor a wrapper around one. Used to test whether a following
+// sibling terminates the section.
+function firstHeadingLevel(el) {
+    const direct = /^H([1-6])$/.exec(el.tagName);
+    if (direct) {
+        return Number.parseInt(direct[1], 10);
+    }
+    const inner = el.querySelector('h1, h2, h3, h4, h5, h6');
+    if (inner) {
+        return Number.parseInt(inner.tagName.slice(1), 10);
+    }
+    return undefined;
+}
+// Wraps the matched heading and its subtree in a `<section data-rdrm-section-scope>`
+// so the existing selectors.include path can isolate it. Returns false (and
+// leaves the document untouched) when no heading matches.
+export function scopeToHeading(document, headingText) {
+    const match = findHeading(document, headingText);
+    if (!match) {
+        return false;
+    }
+    const { heading, level } = match;
+    const container = findFlowContainer(heading);
+    let startChild = heading;
+    while (startChild.parentElement && startChild.parentElement !== container) {
+        startChild = startChild.parentElement;
+    }
+    const wrap = document.createElement('section');
+    wrap.setAttribute('data-rdrm-section-scope', '');
+    container.insertBefore(wrap, startChild);
+    let node = startChild;
+    while (node) {
+        const next = node.nextSibling;
+        wrap.appendChild(node);
+        // A section ends at the next same-or-higher-level heading (level <= L):
+        // deeper headings (h3 under an h2) belong to this section, peers and
+        // shallower headings start the next one. The terminating sibling may be a
+        // wrapper around the heading (e.g. <div class="markdown-heading">), so the
+        // check looks for a heading at OR inside the sibling.
+        if (next !== null && isElement(next)) {
+            const nextLevel = firstHeadingLevel(next);
+            if (nextLevel !== undefined && nextLevel <= level) {
+                break;
+            }
+        }
+        node = next;
+    }
+    return true;
+}