@v1nvn/readability 0.45.0 → 0.45.1

package/src/policy/metadata.js

@@ -0,0 +1,216 @@
+import { estimateTokens, nonEmpty } from "./text.js";
+const ARTICLE_TYPES = new Set([
+    'Article',
+    'BlogPosting',
+    'NewsArticle',
+    'Report',
+    'ScholarlyArticle',
+    'SocialMediaPosting',
+    'TechArticle',
+    'WebPage',
+]);
+function first(...values) {
+    for (const value of values) {
+        const picked = nonEmpty(value);
+        if (picked) {
+            return picked;
+        }
+    }
+    return undefined;
+}
+function metaProperty(document, property) {
+    return nonEmpty(document
+        .querySelector(`meta[property="${property}"]`)
+        ?.getAttribute('content') ?? undefined);
+}
+function metaName(document, name) {
+    return nonEmpty(document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ??
+        undefined);
+}
+// Flatten ld+json scripts (a script may hold an array or a @graph array).
+// Invalid JSON is skipped — never crash extraction over malformed metadata.
+function parseJsonLd(document) {
+    const nodes = document.querySelectorAll('script[type="application/ld+json"]');
+    const out = [];
+    nodes.forEach(node => {
+        const raw = node.textContent;
+        if (!raw.trim()) {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        collectObjects(parsed, out);
+    });
+    return out;
+}
+function collectObjects(value, out) {
+    if (!value || typeof value !== 'object') {
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            collectObjects(item, out);
+        }
+        return;
+    }
+    // `in` guard: the index signature types values as JsonLdValue (no undefined),
+    // so a `!== undefined` check has no overlap.
+    if ('@graph' in value) {
+        collectObjects(value['@graph'], out);
+    }
+    out.push(value);
+}
+function typeMatches(type) {
+    const types = Array.isArray(type) ? type : [type];
+    return types.some(t => typeof t === 'string' && ARTICLE_TYPES.has(t));
+}
+function pickArticleNode(candidates) {
+    if (candidates.length === 0) {
+        return undefined;
+    }
+    return candidates.find(node => typeMatches(node['@type'])) ?? candidates[0];
+}
+// Preference order for the primary structured object: richest schema.org
+// main-entity first. Article types are the tail fallback — Readability already
+// captured the article body, so they are low-value-but-recognizable. Generic
+// graph scaffolding (WebSite/Organization/BreadcrumbList) is intentionally NOT
+// listed: emitting it as `structured` would be noise, and we prefer undefined
+// over a misleading trivial node.
+const STRUCTURED_PRIORITY = [
+    'Recipe',
+    'Product',
+    'Event',
+    'HowTo',
+    'Course',
+    'Movie',
+    'Book',
+    'MusicRecording',
+    'JobPosting',
+    'FAQPage',
+    ...ARTICLE_TYPES,
+];
+function nodeTypeList(node) {
+    const type = node['@type'];
+    if (Array.isArray(type)) {
+        return type.filter((t) => typeof t === 'string');
+    }
+    return typeof type === 'string' ? [type] : [];
+}
+// Iterate priority types in order so a Recipe wins over an earlier Article
+// node, and document order wins among candidates of the same type.
+function pickStructuredObject(candidates) {
+    for (const priorityType of STRUCTURED_PRIORITY) {
+        const hit = candidates.find(node => nodeTypeList(node).includes(priorityType));
+        if (hit) {
+            return hit;
+        }
+    }
+    return undefined;
+}
+// Strip @context (always https://schema.org, pure noise) and normalize a
+// multi-typed node to a "+"-joined string so the host sees a single type label.
+// Data fields are kept verbatim — the host wants the real schema.org object.
+function cleanStructured(obj) {
+    const out = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (key === '@context') {
+            continue;
+        }
+        if (key === '@type') {
+            const types = Array.isArray(value)
+                ? value.filter((t) => typeof t === 'string')
+                : typeof value === 'string'
+                    ? [value]
+                    : [];
+            out[key] = types.join('+');
+            continue;
+        }
+        out[key] = value;
+    }
+    return out;
+}
+// Author may be a string, a Person/Organization object, an array, or a
+// {@list: [...]} container; reduce to a comma-joined byline.
+function resolveJsonLdAuthor(author) {
+    const names = [];
+    function visit(value) {
+        if (typeof value === 'string') {
+            const trimmed = value.trim();
+            if (trimmed) {
+                names.push(trimmed);
+            }
+            return;
+        }
+        if (!value || typeof value !== 'object') {
+            return;
+        }
+        if (Array.isArray(value)) {
+            value.forEach(visit);
+            return;
+        }
+        // {@list: [...]} is an ordering wrapper; unwrap it.
+        const list = value['@list'];
+        if (Array.isArray(list)) {
+            list.forEach(visit);
+            return;
+        }
+        const name = value.name;
+        if (typeof name === 'string') {
+            const trimmed = name.trim();
+            if (trimmed) {
+                names.push(trimmed);
+            }
+        }
+    }
+    visit(author);
+    return names.length > 0 ? names.join(', ') : undefined;
+}
+function asString(value) {
+    return typeof value === 'string' ? nonEmpty(value) : undefined;
+}
+function field(obj, key) {
+    if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+        return undefined;
+    }
+    return obj[key];
+}
+export function resolveMetadata(input) {
+    const { document, readability } = input;
+    const jsonLdObjects = parseJsonLd(document);
+    const jsonLd = pickArticleNode(jsonLdObjects);
+    const structuredRaw = pickStructuredObject(jsonLdObjects);
+    const htmlLang = nonEmpty(document.documentElement.getAttribute('lang') ?? undefined);
+    const titleFromTitleTag = nonEmpty(document.title);
+    const title = first(asString(jsonLd?.headline), metaProperty(document, 'og:title'), metaName(document, 'twitter:title'), readability?.title ?? undefined, titleFromTitleTag);
+    const byline = first(jsonLd ? resolveJsonLdAuthor(jsonLd.author) : undefined, metaProperty(document, 'article:author'), metaName(document, 'author'), readability?.byline ?? undefined);
+    const siteName = first(asString(field(field(jsonLd, 'publisher'), 'name')), metaProperty(document, 'og:site_name'), readability?.siteName ?? undefined);
+    const lang = first(asString(jsonLd?.inLanguage), htmlLang, readability?.lang ?? undefined);
+    const publishedTime = first(asString(jsonLd?.datePublished), metaProperty(document, 'article:published_time'), 
+    // First <time datetime> is the canonical publish time when rendered inline.
+    nonEmpty(document.querySelector('time[datetime]')?.getAttribute('datetime') ??
+        undefined), readability?.publishedTime ?? undefined);
+    const excerpt = first(asString(jsonLd?.description), metaProperty(document, 'og:description'), metaName(document, 'twitter:description'), metaName(document, 'description'), readability?.excerpt ?? undefined);
+    // canonical/og:url are absolute by spec; returned raw so callers can tell
+    // the declared canonical apart from the input baseUrl (origin context only).
+    const canonical = first(nonEmpty(document.querySelector('link[rel="canonical"]')?.getAttribute('href') ??
+        undefined), metaProperty(document, 'og:url'));
+    return {
+        title,
+        byline,
+        siteName,
+        lang,
+        publishedTime,
+        excerpt,
+        canonical,
+        baseUrl: input.baseUrl,
+        wordCount: input.wordCount,
+        readingTimeMin: input.readingTimeMin,
+        ...estimateTokens(input.textContent),
+        ...(structuredRaw ? { structured: cleanStructured(structuredRaw) } : {}),
+    };
+}

package/src/policy/outline-chains.js

@@ -0,0 +1,165 @@
+import { buildDocument } from "../pipeline/dom.js";
+import { applySelectors, normalizeDocument, resolveLazyImages, } from "../pipeline/normalize.js";
+import { isGeneratedIdentifier } from "./identifiers.js";
+// Step-0 measured thresholds: 200 own chars is the chain view that produced a
+// working proposal at ~630 tokens, and the debris that view misses (video
+// controls, captions) holds under 200 own chars — the round-two sweep runs at
+// 40 to surface it.
+export const OUTLINE_OWN_TEXT_MIN = 200;
+export const BLOCK_OWN_TEXT_MIN = 40;
+export const OUTLINE_MAX_CHARS = 6000;
+export const OUTLINE_MAX_CHAINS = 60;
+const SAMPLE_MAX_CHARS = 80;
+const NON_CONTENT_SELECTOR = 'script, style, noscript, template';
+const HOP_ATTRIBUTE_WHITELIST = ['itemprop', 'role', 'data-testid'];
+// The document the outline renders is normalized exactly as extraction
+// normalizes it, so a selector proposed from this material validates and
+// applies against the same tree the pipeline will run it on.
+export function buildChainOutline(input) {
+    const { document } = buildDocument(input.html, input.baseUrl);
+    normalizeDocument(document, { cleanChrome: input.cleanChrome });
+    resolveLazyImages(document);
+    const scope = input.mode === 'page' ? undefined : input.mode.scope;
+    if (scope) {
+        applySelectors(document, scope);
+    }
+    const scopeRoot = scope?.include
+        ? (document.body.querySelector(scope.include) ?? undefined)
+        : undefined;
+    const min = scope ? BLOCK_OWN_TEXT_MIN : OUTLINE_OWN_TEXT_MIN;
+    const candidates = [];
+    for (const element of (scopeRoot ?? document.body).querySelectorAll('*')) {
+        if (element.closest(NON_CONTENT_SELECTOR)) {
+            continue;
+        }
+        const own = ownTextLength(element);
+        if (own >= min) {
+            candidates.push({ element, own });
+        }
+    }
+    return scope
+        ? groupedOutline(candidates, scopeRoot)
+        : flatOutline(candidates);
+}
+// Round one: one full ancestor chain per qualifying node, step-0 format.
+function flatOutline(candidates) {
+    const lines = [];
+    let truncated = false;
+    for (const candidate of [...candidates].sort((a, b) => b.own - a.own)) {
+        if (lines.length >= OUTLINE_MAX_CHAINS) {
+            truncated = true;
+            break;
+        }
+        const hops = ancestorsUpTo(candidate.element, undefined).map(element => hopWithSizes(element));
+        const line = hops.join(' > ');
+        if (lines.join('\n---\n').length + line.length > OUTLINE_MAX_CHARS) {
+            truncated = true;
+            break;
+        }
+        lines.push(line);
+    }
+    return { chainCount: lines.length, text: lines.join('\n---\n'), truncated };
+}
+// Round two: the included subtree's remaining text blocks, grouped by
+// structural chain so dozens of prose paragraphs collapse into one line and
+// the odd control-text/caption container stands alone. Smallest own text
+// first — that is where the debris lives.
+function groupedOutline(candidates, scopeRoot) {
+    const groups = new Map();
+    for (const candidate of candidates) {
+        const chain = ancestorsUpTo(candidate.element, scopeRoot).map(element => renderHop(element));
+        if (scopeRoot) {
+            chain.push(renderHop(scopeRoot));
+        }
+        const key = chain.join(' > ');
+        const existing = groups.get(key);
+        if (existing) {
+            existing.count += 1;
+            existing.min = Math.min(existing.min, candidate.own);
+            existing.max = Math.max(existing.max, candidate.own);
+        }
+        else {
+            groups.set(key, {
+                chain: key,
+                count: 1,
+                min: candidate.own,
+                max: candidate.own,
+                sample: sampleText(candidate.element),
+            });
+        }
+    }
+    const lines = [];
+    let truncated = false;
+    const sorted = [...groups.values()].sort((a, b) => a.min - b.min);
+    for (const group of sorted) {
+        if (lines.length >= OUTLINE_MAX_CHAINS) {
+            truncated = true;
+            break;
+        }
+        const line = `${group.chain} (own:${group.min}..${group.max}, ×${group.count}) sample: "${group.sample}"`;
+        if (lines.join('\n').length + line.length > OUTLINE_MAX_CHARS) {
+            truncated = true;
+            break;
+        }
+        lines.push(line);
+    }
+    return { chainCount: lines.length, text: lines.join('\n'), truncated };
+}
+// Nearest-to-leaf first; the walk stops below <body> (the root applySelectors
+// never matches) and below the include root when one is given — the caller
+// prepends that root's own hop where it is wanted.
+function ancestorsUpTo(element, stop) {
+    const hops = [];
+    let node = element;
+    const body = element.ownerDocument.body;
+    while (node && node !== stop && node !== body) {
+        hops.push(node);
+        node = node.parentElement;
+    }
+    return hops;
+}
+function hopWithSizes(element) {
+    const own = ownTextLength(element);
+    const all = element.textContent.length;
+    return `${renderHop(element)} (own:${own}, all:${all})`;
+}
+function ownTextLength(element) {
+    let own = 0;
+    for (const node of element.childNodes) {
+        if (node.nodeType === node.TEXT_NODE) {
+            own += (node.textContent ?? '').length;
+        }
+    }
+    return own;
+}
+// Copy-safety: what renders here is what the suggester may type back. Every
+// token is real CSS — real attribute names and values, and classes/ids that
+// survive the generated-identifier lint, so a hash class can never be copied
+// into a proposal because it was never shown.
+function renderHop(element) {
+    let hop = element.tagName.toLowerCase();
+    const id = element.id;
+    if (id && !isGeneratedIdentifier(id)) {
+        hop += `#${id}`;
+    }
+    const classes = (element.getAttribute('class') ?? '')
+        .split(/\s+/)
+        .filter(Boolean)
+        .filter(className => !isGeneratedIdentifier(className));
+    if (classes.length > 0) {
+        hop += `.${classes.join('.')}`;
+    }
+    for (const attribute of HOP_ATTRIBUTE_WHITELIST) {
+        const value = element.getAttribute(attribute);
+        if (value) {
+            hop += `[${attribute}="${value}"]`;
+        }
+    }
+    return hop;
+}
+function sampleText(element) {
+    const text = element.textContent.trim().replace(/\s+/g, ' ');
+    return text.length > SAMPLE_MAX_CHARS
+        ? `${text.slice(0, SAMPLE_MAX_CHARS)}…`
+        : text;
+}

package/src/policy/outline.js

@@ -0,0 +1,71 @@
+// Collapse whitespace, trim, then strip leading/trailing `#` (GitHub permalinks
+// append `<a>#</a>`, so heading textContent ends with `#`).
+export function normalizeHeadingText(raw) {
+    const collapsed = raw.replace(/\s+/g, ' ').trim();
+    return collapsed.replace(/^#+|#+$/g, '').trim();
+}
+// GFM anchor algorithm (comrak `anchorize`): keep Unicode letters, marks,
+// numbers, and connector punctuation; map spaces to hyphens. No hyphen collapse
+// or trim, so anchors match GitHub and non-Latin scripts survive. An all-symbol
+// heading yields an empty slug; fall back to `section` so anchors stay non-empty.
+function slugify(text) {
+    const cleaned = text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, '');
+    return cleaned.replace(/ /g, '-') || 'section';
+}
+// `href="#"` (empty fragment) does not win — fall through to slugify.
+function linkAnchor(heading) {
+    for (const link of heading.querySelectorAll('a[href^="#"]')) {
+        const href = link.getAttribute('href');
+        if (!href) {
+            continue;
+        }
+        const fragment = href.slice(1);
+        if (fragment) {
+            return fragment;
+        }
+    }
+    return undefined;
+}
+function resolveCandidate(heading, text) {
+    const id = heading.getAttribute('id');
+    if (id) {
+        return { anchor: id, explicit: true };
+    }
+    const link = linkAnchor(heading);
+    if (link) {
+        return { anchor: link, explicit: true };
+    }
+    return { anchor: slugify(text), explicit: false };
+}
+// Explicit anchors are kept verbatim; generated slugs are collision-suffixed
+// (`-1`, `-2`, …), first occurrence bare.
+function dedupe(candidate, explicit, used) {
+    if (explicit || !used.has(candidate)) {
+        return candidate;
+    }
+    let suffix = 1;
+    while (used.has(`${candidate}-${suffix}`)) {
+        suffix++;
+    }
+    return `${candidate}-${suffix}`;
+}
+export function resolveOutline(document) {
+    const entries = [];
+    const used = new Set();
+    const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+    headings.forEach(heading => {
+        const rawText = heading.textContent;
+        if (!rawText.trim()) {
+            return;
+        }
+        const text = normalizeHeadingText(rawText);
+        const level = Number.parseInt(heading.tagName.slice(1), 10);
+        const { anchor: candidate, explicit } = resolveCandidate(heading, text);
+        const anchor = dedupe(candidate, explicit, used);
+        used.add(anchor);
+        entries.push({ anchor, level, text });
+    });
+    return entries;
+}

package/src/policy/pagination.js

@@ -0,0 +1,79 @@
+import { absolutize } from "../pipeline/urls.js";
+// Short pagination-style link text only; matching prose like "next, we cover
+// rendering" would burn an article's internal link as a next-page signal.
+// `next ›/»/→` and bare arrows cover the common next-page glyphs in one place.
+const NEXT_LINK_TEXT_RE = /^(next(\s+page)?|older(\s+posts?)?|[›»→]|next\s*[›»→]|older\s*[›»→])$/i;
+function usableHref(href) {
+    if (!href || href === '#') {
+        return undefined;
+    }
+    return href;
+}
+function findPaginated(document, baseUrl) {
+    const linkNext = document.querySelector('link[rel="next"][href]');
+    if (linkNext) {
+        const href = usableHref(linkNext.getAttribute('href'));
+        if (href) {
+            return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+        }
+    }
+    const aRelNext = document.querySelector('a[rel="next"][href]');
+    if (aRelNext) {
+        const href = usableHref(aRelNext.getAttribute('href'));
+        if (href) {
+            return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+        }
+    }
+    for (const anchor of document.querySelectorAll('a[href]')) {
+        const text = anchor.textContent.trim();
+        if (!text || !NEXT_LINK_TEXT_RE.test(text)) {
+            continue;
+        }
+        const href = usableHref(anchor.getAttribute('href'));
+        if (!href) {
+            continue;
+        }
+        return { type: 'paginated', nextUrl: absolutize(href, baseUrl) };
+    }
+    return undefined;
+}
+const INFINITE_ATTR_SELECTORS = [
+    '[data-load-more]',
+    '[data-infinite-scroll]',
+    '[data-pagination]',
+    '[infinite-scroll]',
+];
+const INFINITE_SUBSTRING_SELECTORS = [
+    '[class*="load-more"]',
+    '[class*="loadmore"]',
+    '[class*="infinite"]',
+    '[id*="load-more"]',
+    '[class*="sentinel"]',
+];
+const LOAD_MORE_BUTTON_RE = /^(load more|show more|view more|more results|load more comments)$/i;
+function findInfinite(document) {
+    for (const selector of INFINITE_ATTR_SELECTORS) {
+        const el = document.querySelector(selector);
+        if (el?.isConnected) {
+            return { selector, type: 'infinite' };
+        }
+    }
+    for (const selector of INFINITE_SUBSTRING_SELECTORS) {
+        const el = document.querySelector(selector);
+        if (el?.isConnected) {
+            return { selector, type: 'infinite' };
+        }
+    }
+    for (const button of document.querySelectorAll('button')) {
+        const text = button.textContent.trim();
+        if (text && LOAD_MORE_BUTTON_RE.test(text)) {
+            return { selector: 'button', type: 'infinite' };
+        }
+    }
+    return undefined;
+}
+// Paginated beats infinite: an explicit "next" link is a stronger signal than
+// a load-more sentinel. Read-only — never follows or fetches.
+export function detectPagination(document, baseUrl) {
+    return findPaginated(document, baseUrl) ?? findInfinite(document);
+}

package/src/policy/presets.js

@@ -0,0 +1,100 @@
+const presets = new Map();
+// Bumped on every store mutation so cache keys can fold in preset state
+// without hashing the resolved scope: an entry extracted before a preset
+// landed must never be served after it.
+let generation = 0;
+export function presetGeneration() {
+    return generation;
+}
+export function resetPresets() {
+    presets.clear();
+    generation += 1;
+}
+export function addPreset(preset) {
+    const key = normalizeSiteKey(preset.site);
+    if (!key) {
+        throw new Error(`invalid preset site: ${preset.site}`);
+    }
+    presets.set(key, preset);
+    generation += 1;
+}
+export function removePreset(site) {
+    const key = normalizeSiteKey(site);
+    const removed = key !== undefined && presets.delete(key);
+    if (removed) {
+        generation += 1;
+    }
+    return removed;
+}
+export function presetForSite(site) {
+    const key = normalizeSiteKey(site);
+    return key ? presets.get(key) : undefined;
+}
+// Accepts a full URL or a bare host; `www.` is the one prefix every site's own
+// pages agree on, so exactly one strip — `www.www.com` keys as `www.com`.
+export function normalizeSiteKey(value) {
+    const raw = value?.trim();
+    if (!raw) {
+        return undefined;
+    }
+    let host;
+    try {
+        host = new URL(raw).hostname.toLowerCase();
+    }
+    catch {
+        host = raw.toLowerCase();
+    }
+    const key = host.startsWith('www.') ? host.slice('www.'.length) : host;
+    return /^[a-z0-9.-]+$/.test(key) ? key : undefined;
+}
+// A selector that throws cannot match anything, so a malformed preset selector
+// is a miss by the same predicate as a non-matching one — never a crash.
+export function selectorMisses(scope, selector) {
+    try {
+        return scope.querySelector(selector) === null;
+    }
+    catch {
+        return true;
+    }
+}
+function presetMatches(document, preset) {
+    for (const detector of preset.detectors) {
+        if (selectorMisses(document, detector)) {
+            return false;
+        }
+    }
+    // The include must still match: an unmatched include is applySelectors'
+    // silent no-op, and reporting applied:true over baseline output is a lie.
+    if (preset.scope.include &&
+        selectorMisses(document.body, preset.scope.include)) {
+        return false;
+    }
+    // Excludes must execute but need not match — the debris they name is absent
+    // from some pages of the site, and that is not staleness.
+    for (const selector of preset.scope.exclude ?? []) {
+        try {
+            document.querySelectorAll(selector);
+        }
+        catch {
+            return false;
+        }
+    }
+    return true;
+}
+export function resolvePreset(document, baseUrl, overridden) {
+    const site = normalizeSiteKey(baseUrl);
+    if (!site) {
+        return undefined;
+    }
+    const preset = presets.get(site);
+    if (!preset) {
+        return undefined;
+    }
+    if (overridden) {
+        return { signal: { applied: false, reason: 'overridden', site } };
+    }
+    if (!presetMatches(document, preset)) {
+        return { signal: { applied: false, reason: 'detectors-missed', site } };
+    }
+    return { scope: preset.scope, signal: { applied: true, site } };
+}