@v1nvn/readability 0.45.0 → 0.45.1

package/src/policy/section.js

@@ -0,0 +1,111 @@
+import { isElement } from "../pipeline/dom.js";
+import { normalizeHeadingText } from "./outline.js";
+// Match the first heading by normalized text. Exact equality wins; otherwise
+// fall back to the first substring match so a short query ("auth") still
+// lands on a longer heading ("Authentication") without forcing an exact match.
+function findHeading(document, query) {
+    const needle = normalizeHeadingText(query).toLowerCase();
+    if (!needle) {
+        return undefined;
+    }
+    const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+    let substring;
+    for (const heading of headings) {
+        const text = normalizeHeadingText(heading.textContent).toLowerCase();
+        if (!text) {
+            continue;
+        }
+        const level = Number.parseInt(heading.tagName.slice(1), 10);
+        if (text === needle) {
+            return { heading, level };
+        }
+        if (!substring && text.includes(needle)) {
+            substring = { heading, level };
+        }
+    }
+    return substring;
+}
+// Elements whose direct children form the section flow. The matched heading's
+// nearest such ancestor is the level at which the section's content lives as
+// siblings: on docs sites (GitHub markdown, etc.) the heading is wrapped in
+// <div class="markdown-heading">, so the body <p> is a sibling of the wrapper
+// div, not of the <h2>. Walking the <h2>'s own siblings would capture only the
+// permalink anchor and miss the body.
+const FLOW_CONTAINERS = new Set([
+    'ARTICLE',
+    'ASIDE',
+    'BLOCKQUOTE',
+    'BODY',
+    'DD',
+    'DETAILS',
+    'DT',
+    'FOOTER',
+    'HEADER',
+    'LI',
+    'MAIN',
+    'NAV',
+    'SECTION',
+    'TD',
+]);
+function findFlowContainer(heading) {
+    let ancestor = heading.parentElement;
+    while (ancestor) {
+        if (FLOW_CONTAINERS.has(ancestor.tagName)) {
+            return ancestor;
+        }
+        ancestor = ancestor.parentElement;
+    }
+    // BODY is in FLOW_CONTAINERS, so the loop reaches it for any attached
+    // heading; this fallback only covers a detached heading.
+    return heading.ownerDocument.body;
+}
+// Level of the first heading at or inside `el`, or undefined when `el` is
+// neither a heading nor a wrapper around one. Used to test whether a following
+// sibling terminates the section.
+function firstHeadingLevel(el) {
+    const direct = /^H([1-6])$/.exec(el.tagName);
+    if (direct) {
+        return Number.parseInt(direct[1], 10);
+    }
+    const inner = el.querySelector('h1, h2, h3, h4, h5, h6');
+    if (inner) {
+        return Number.parseInt(inner.tagName.slice(1), 10);
+    }
+    return undefined;
+}
+// Wraps the matched heading and its subtree in a `<section data-rdrm-section-scope>`
+// so the existing selectors.include path can isolate it. Returns false (and
+// leaves the document untouched) when no heading matches.
+export function scopeToHeading(document, headingText) {
+    const match = findHeading(document, headingText);
+    if (!match) {
+        return false;
+    }
+    const { heading, level } = match;
+    const container = findFlowContainer(heading);
+    let startChild = heading;
+    while (startChild.parentElement && startChild.parentElement !== container) {
+        startChild = startChild.parentElement;
+    }
+    const wrap = document.createElement('section');
+    wrap.setAttribute('data-rdrm-section-scope', '');
+    container.insertBefore(wrap, startChild);
+    let node = startChild;
+    while (node) {
+        const next = node.nextSibling;
+        wrap.appendChild(node);
+        // A section ends at the next same-or-higher-level heading (level <= L):
+        // deeper headings (h3 under an h2) belong to this section, peers and
+        // shallower headings start the next one. The terminating sibling may be a
+        // wrapper around the heading (e.g. <div class="markdown-heading">), so the
+        // check looks for a heading at OR inside the sibling.
+        if (next !== null && isElement(next)) {
+            const nextLevel = firstHeadingLevel(next);
+            if (nextLevel !== undefined && nextLevel <= level) {
+                break;
+            }
+        }
+        node = next;
+    }
+    return true;
+}

package/src/policy/selector-lint.js

@@ -0,0 +1,143 @@
+import { isGeneratedIdentifier } from "./identifiers.js";
+import { selectorMisses } from "./presets.js";
+// nwsapi/jsdom accept `:contains` and silently apply it, so an engine error can
+// reject nothing — the rejection has to happen here, before any selector
+// reaches querySelector. Positional pseudos are rejected as a family: they
+// describe this render's child order, not the site's structure.
+const POSITIONAL_PSEUDO_RE = /:(?:nth(?:-last)?(?:-child|-of-type)|first(?:-child|-of-type)|last(?:-child|-of-type)|only(?:-child|-of-type))\b/i;
+const CONTAINS_PSEUDO_RE = /:contains\s*\(/i;
+function quotedSegmentsOut(selector) {
+    return selector.replace(/(["'])(?:\\.|(?!\1)[\s\S])*\1/g, '""');
+}
+// `.class` and `#id` tokens are identifiers; `[attr="value"]` values are
+// substring matches against whatever the site emitted and are exempt.
+function identifierTokens(selector) {
+    return [...quotedSegmentsOut(selector).matchAll(/[.#]([A-Za-z0-9_-]+)/g)].map(match => match[1]);
+}
+export function lintSelectorText(selector) {
+    if (CONTAINS_PSEUDO_RE.test(selector)) {
+        return {
+            kind: 'contains-pseudo',
+            selector,
+            detail: ':contains is not standard CSS and changes the result silently',
+        };
+    }
+    const positional = POSITIONAL_PSEUDO_RE.exec(selector);
+    if (positional) {
+        return {
+            kind: 'positional-pseudo',
+            selector,
+            detail: `${positional[0]} depends on this page's child order, not the site's layout`,
+        };
+    }
+    for (const token of identifierTokens(selector)) {
+        if (isGeneratedIdentifier(token)) {
+            return {
+                kind: 'generated-identifier',
+                selector,
+                detail: `"${token}" reads as a generated hash that changes on deploy`,
+            };
+        }
+    }
+    return undefined;
+}
+// Propose-time validation for a preset a suggester wants stored. Stricter than
+// presetMatches: an exclude that matches nothing is tolerated at runtime (the
+// debris is absent from some pages) but at propose time it is a mistake — the
+// proposal names debris that is not there.
+export function lintProposal({ document, detectors, scope, }) {
+    const violations = [];
+    const selectors = [
+        ...detectors,
+        ...(scope.include ? [scope.include] : []),
+        ...(scope.exclude ?? []),
+    ];
+    for (const selector of selectors) {
+        const violation = lintSelectorText(selector);
+        if (violation) {
+            violations.push(violation);
+        }
+    }
+    let includeRoot;
+    if (scope.include &&
+        !violations.some(violation => violation.selector === scope.include)) {
+        try {
+            includeRoot = document.body.querySelector(scope.include) ?? undefined;
+        }
+        catch {
+            violations.push({
+                kind: 'unparseable',
+                selector: scope.include,
+                detail: 'the selector engine rejects this selector',
+            });
+        }
+    }
+    for (const detector of detectors) {
+        if (violations.some(violation => violation.selector === detector)) {
+            continue;
+        }
+        if (selectorMisses(document, detector)) {
+            violations.push({
+                kind: 'no-match',
+                selector: detector,
+                detail: 'detector matches nothing on this page',
+            });
+        }
+    }
+    if (scope.include &&
+        !includeRoot &&
+        !violations.some(violation => violation.selector === scope.include)) {
+        violations.push({
+            kind: 'no-match',
+            selector: scope.include,
+            detail: 'include matches nothing inside <body>, where applySelectors searches',
+        });
+    }
+    for (const selector of scope.exclude ?? []) {
+        if (violations.some(violation => violation.selector === selector)) {
+            continue;
+        }
+        try {
+            if (document.querySelectorAll(selector).length === 0) {
+                violations.push({
+                    kind: 'no-match',
+                    selector,
+                    detail: 'exclude matches nothing on this page',
+                });
+            }
+        }
+        catch {
+            violations.push({
+                kind: 'unparseable',
+                selector,
+                detail: 'the selector engine rejects this selector',
+            });
+        }
+    }
+    // Excludes are removed document-wide before the include is applied, so an
+    // exclude matching an ancestor of the include root (or the root itself)
+    // deletes it, and the include then quietly no-ops over a damaged body.
+    if (includeRoot) {
+        for (const selector of scope.exclude ?? []) {
+            if (violations.some(violation => violation.selector === selector)) {
+                continue;
+            }
+            try {
+                for (const match of document.querySelectorAll(selector)) {
+                    if (match.contains(includeRoot)) {
+                        violations.push({
+                            kind: 'exclude-shadows-include',
+                            selector,
+                            detail: 'this exclude removes the include root itself',
+                        });
+                        break;
+                    }
+                }
+            }
+            catch {
+                // Already reported as unparseable.
+            }
+        }
+    }
+    return violations;
+}

package/src/policy/sibling-scan.js

@@ -0,0 +1,92 @@
+import { isElement } from "../pipeline/dom.js";
+// Shared pre-pass for the sibling-cluster detectors (list-detector,
+// grid-detector). Both walk the document for a container whose direct children
+// form a same-shape sibling group; the constants and helpers here are the
+// heuristic-agnostic infrastructure that scan shares. Detector-specific scoring
+// (anchor/pathname signals vs data-cell text) stays in each detector.
+// Direct children are scanned only at these container levels. <tbody> covers
+// HN's <table>-based layout; the rest cover semantic (<main>/<article>) and
+// generic (<div>/<section>) wrappers. <table> is included for HTML that skips
+// <tbody>; jsdom synthesizes one anyway, so this is mostly defensive.
+export const CONTAINER_TAGS = new Set([
+    'ARTICLE',
+    'DIV',
+    'MAIN',
+    'OL',
+    'SECTION',
+    'TABLE',
+    'TBODY',
+    'UL',
+]);
+// Subtrees that never carry a page's primary repeated structure. Stripping them
+// is the false-positive guard: without it, an article's <nav> menu or footer
+// link list would look identical to a feed/grid and the detectors would mis-fire
+// on every page with chrome. Distinct from normalize.ts's stripChrome (which
+// removes consent/overlay banners) — this is landmark chrome only.
+const LANDMARK_CHROME_SELECTOR = 'nav, header, footer, aside, [role="navigation"], [role="banner"], ' +
+    '[role="contentinfo"], [role="complementary"], [role="search"], ' +
+    '[role="menu"], [role="menubar"]';
+export function stripLandmarkChrome(document) {
+    for (const el of document.querySelectorAll(LANDMARK_CHROME_SELECTOR)) {
+        el.remove();
+    }
+    for (const el of document.querySelectorAll('script, style, template')) {
+        el.remove();
+    }
+}
+// Composite of tag + class signature so mixed siblings — e.g. HN's
+// <tr class="athing"> title row vs classless <tr> subtext row, or a grid's
+// header row vs data rows — split into separate candidate groups instead of
+// blending into one ragged group.
+export function shapeKey(el) {
+    const raw = el.getAttribute('class');
+    if (!raw) {
+        return el.tagName;
+    }
+    const normalized = raw.trim().split(/\s+/).sort().join(' ');
+    return normalized ? `${el.tagName}|${normalized}` : el.tagName;
+}
+// Group a container's direct element-children by shape so a homogeneous
+// cluster surfaces as one candidate and mixed-shape siblings (header row vs
+// data rows, HN's athing + subtext) split apart rather than blending.
+export function groupChildrenByShape(container) {
+    const groups = new Map();
+    for (const child of Array.from(container.childNodes)) {
+        if (!isElement(child)) {
+            continue;
+        }
+        const key = shapeKey(child);
+        const bucket = groups.get(key);
+        if (bucket) {
+            bucket.push(child);
+        }
+        else {
+            groups.set(key, [child]);
+        }
+    }
+    return groups;
+}
+// tag#id.class hint for the winning container. Best-effort, not a unique
+// locator — used only for diagnostics so a human can see which subtree won.
+// `maxClasses` caps the class list for compact diagnostic output.
+export function describeSelector(el, maxClasses) {
+    const parts = [el.tagName.toLowerCase()];
+    const id = el.getAttribute('id');
+    if (id) {
+        parts.push(`#${id}`);
+    }
+    const cls = el.getAttribute('class');
+    if (cls) {
+        let used = 0;
+        for (const token of cls.trim().split(/\s+/)) {
+            if (token) {
+                parts.push(`.${token}`);
+                used += 1;
+            }
+            if (maxClasses !== undefined && used >= maxClasses) {
+                break;
+            }
+        }
+    }
+    return parts.join('');
+}

package/src/policy/text.js

@@ -0,0 +1,18 @@
+export const TOKEN_ESTIMATOR = 'chars/4';
+export function estimateTokens(textContent) {
+    return {
+        tokenEstimate: Math.round(textContent.length / 4),
+        estimator: TOKEN_ESTIMATOR,
+    };
+}
+export function countWords(text) {
+    return (text.match(/\S+/g) ?? []).length;
+}
+export function nonEmpty(value) {
+    return value?.trim() ? value : undefined;
+}
+export function computeTextMetrics(text, wordsPerMinute) {
+    const wordCount = countWords(text);
+    const readingTimeMin = wordCount === 0 ? 0 : Math.max(1, Math.round(wordCount / wordsPerMinute));
+    return { wordCount, readingTimeMin, ...estimateTokens(text) };
+}

package/src/policy/truncate.js

@@ -0,0 +1,45 @@
+import { hardSplitLines, parseBlocks } from "./markdown.js";
+const TRUNCATION_MARKER = '\n\n…[truncated]';
+// Cuts at a top-level block boundary. A fenced code block is a single mdast
+// node, so a boundary cut never lands inside one — no open-fence rollback is
+// needed. The kept span is sliced from the original string so the payload is a
+// byte-for-byte prefix of the Turndown output.
+export function truncateMarkdown(markdown, maxChars) {
+    if (markdown.length <= maxChars) {
+        return { text: markdown, truncated: false };
+    }
+    const blocks = parseBlocks(markdown);
+    let start = -1;
+    let end = -1;
+    let truncated = false;
+    for (const block of blocks) {
+        const from = start === -1 ? block.start : start;
+        if (block.end - from > maxChars) {
+            // First non-code block alone overflows: take a line-aware prefix instead
+            // of dropping everything. A code block is never split — a half-open fence
+            // is unrecoverable — so an oversized first code block stays dropped.
+            if (start === -1 && block.kind !== 'code') {
+                const firstPiece = hardSplitLines(markdown.slice(block.start, block.end), maxChars)[0] ??
+                    '';
+                return {
+                    text: firstPiece.replace(/\s+$/, '') + TRUNCATION_MARKER,
+                    truncated: true,
+                };
+            }
+            truncated = true;
+            break;
+        }
+        if (start === -1) {
+            start = block.start;
+        }
+        end = block.end;
+    }
+    if (!truncated) {
+        return { text: markdown, truncated: false };
+    }
+    const kept = start === -1 ? '' : markdown.slice(start, end);
+    return {
+        text: kept.replace(/\s+$/, '') + TRUNCATION_MARKER,
+        truncated: true,
+    };
+}

package/src/preset-cache.js

@@ -0,0 +1,165 @@
+// Disk side of the preset store: one <site>.json file per preset, read into
+// the in-memory store at boot. Local to the user's machine, ordinary files;
+// the server never fetches and nothing expires by clock — staleness is
+// detector-checked per page (policy/presets.ts), so eviction here is disk
+// hygiene only.
+import { describeError } from '@v1nvn/agentic-core';
+import { mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, writeFileSync, } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { z } from 'zod';
+import { logger } from "./logger.js";
+import { addPreset, normalizeSiteKey } from "./policy/presets.js";
+export const PRESETS_DIR_ENV = 'READABILITY_PRESETS_DIR';
+export const MAX_PRESET_FILES = 64;
+// An empty scope applies nothing while reporting applied:true, and detectors
+// only mean something as a non-empty set — reject both at the file boundary.
+const presetFileSchema = z.object({
+    site: z.string().min(1),
+    detectors: z.array(z.string().min(1)).min(1),
+    scope: z
+        .object({
+        include: z.string().min(1).optional(),
+        exclude: z.array(z.string().min(1)).min(1).optional(),
+    })
+        .refine(scope => scope.include !== undefined || scope.exclude !== undefined, {
+        message: 'scope must carry include or exclude',
+    }),
+});
+// Empty string disables preset loading; an absolute path wins over the
+// platform default.
+export function resolvePresetsDir(env = process.env) {
+    const override = env[PRESETS_DIR_ENV];
+    if (override !== undefined) {
+        return override.trim() === '' ? undefined : override;
+    }
+    const root = env.XDG_CACHE_HOME ||
+        join(homedir(), process.platform === 'darwin' ? 'Library/Caches' : '.cache');
+    return join(root, 'readability', 'presets');
+}
+// A preset that cannot load is a warning, never a boot failure — the pipeline
+// is designed to run without one.
+export function loadPresetDir(dir) {
+    let names;
+    try {
+        names = readdirSync(dir)
+            .filter(name => name.endsWith('.json'))
+            .sort();
+    }
+    catch (err) {
+        if (err.code === 'ENOENT') {
+            logger.debug(`no preset directory at ${dir}`);
+        }
+        else {
+            logger.warn(`preset directory unreadable: ${dir}`);
+        }
+        return { loaded: 0, pruned: 0, skipped: 0 };
+    }
+    const ranked = rankPresetFiles(dir, names);
+    const pruned = pruneRanked(dir, ranked.slice(MAX_PRESET_FILES));
+    let loaded = 0;
+    let skipped = 0;
+    for (const { name } of ranked.slice(0, MAX_PRESET_FILES)) {
+        const path = join(dir, name);
+        if (loadPresetFile(path)) {
+            loaded++;
+        }
+        else {
+            skipped++;
+        }
+    }
+    if (loaded > 0) {
+        logger.info(`loaded ${loaded} site preset(s) from ${dir}`);
+    }
+    if (pruned > 0) {
+        logger.info(`pruned ${pruned} preset file(s) beyond the ${MAX_PRESET_FILES}-file bound`);
+    }
+    return { loaded, pruned, skipped };
+}
+// Env-resolving entry point for the boot paths; undefined means disabled.
+export function loadPresets(env = process.env) {
+    const dir = resolvePresetsDir(env);
+    return dir ? loadPresetDir(dir) : undefined;
+}
+function rankPresetFiles(dir, names) {
+    const ranked = [];
+    for (const name of names) {
+        try {
+            ranked.push({ name, mtimeMs: statSync(join(dir, name)).mtimeMs });
+        }
+        catch {
+            // Vanished between readdir and stat — nothing to load or prune.
+        }
+    }
+    ranked.sort((a, b) => b.mtimeMs - a.mtimeMs);
+    return ranked;
+}
+function pruneRanked(dir, ranked) {
+    let pruned = 0;
+    for (const { name } of ranked) {
+        try {
+            unlinkSync(join(dir, name));
+            pruned++;
+        }
+        catch (err) {
+            logger.warn(`could not prune preset file ${name}: ${describeError(err)}`);
+        }
+    }
+    return pruned;
+}
+// The writer the loader was waiting for: the suggest loop saves an accepted
+// preset so the next server start loads it like any hand-placed file. A
+// failure leaves the in-memory preset working — persistence is best-effort,
+// exactly like loading.
+export function savePreset(preset, env = process.env) {
+    const dir = resolvePresetsDir(env);
+    if (!dir) {
+        return { persisted: false, reason: 'preset-directory-disabled' };
+    }
+    const shape = presetFileSchema.safeParse(preset);
+    if (!shape.success) {
+        return {
+            persisted: false,
+            reason: shape.error.issues[0]?.message ?? 'invalid preset shape',
+        };
+    }
+    const key = normalizeSiteKey(preset.site);
+    if (!key) {
+        return { persisted: false, reason: 'site-is-not-a-hostname' };
+    }
+    const path = join(dir, `${key}.json`);
+    try {
+        mkdirSync(dir, { recursive: true });
+        writeFileSync(path, `${JSON.stringify(shape.data, null, 2)}\n`);
+    }
+    catch (err) {
+        return {
+            persisted: false,
+            reason: describeError(err),
+        };
+    }
+    const over = rankPresetFiles(dir, readdirSync(dir).filter(name => name.endsWith('.json')));
+    pruneRanked(dir, over.slice(MAX_PRESET_FILES));
+    return { path, persisted: true };
+}
+function loadPresetFile(path) {
+    let parsed;
+    try {
+        parsed = JSON.parse(readFileSync(path, 'utf8'));
+    }
+    catch (err) {
+        logger.warn(`${path}: not readable JSON (${describeError(err)}), preset skipped`);
+        return false;
+    }
+    const result = presetFileSchema.safeParse(parsed);
+    if (!result.success) {
+        logger.warn(`${path}: ${result.error.issues[0]?.message}, preset skipped`);
+        return false;
+    }
+    if (!normalizeSiteKey(result.data.site)) {
+        logger.warn(`${path}: site is not a hostname, preset skipped`);
+        return false;
+    }
+    addPreset(result.data);
+    return true;
+}