@v1nvn/readability 0.44.0 → 0.45.1

package/src/policy/lost-signal.ts

@@ -0,0 +1,69 @@
+import type { GatingSignal } from './gating.ts';
+
+import { countWords } from './text.ts';
+
+// Below this the extraction cannot be the article a reader came for. The
+// measured corpus floor: the smallest healthy extract is gatwick at 435 words;
+// the smallest known-lost one (WIRED's metered barrier) still carries 180.
+export const NEAR_EMPTY_WORDS = 120;
+
+// Curated junk signatures over the EXTRACTED text, not the page — gating's
+// METERED_TEXT_RE reads the whole document, which is a different question.
+// Each entry is measured on a real capture: the first two are the Daily Mail
+// player controls that survive inside itemprop="articleBody" (dailymail-a66),
+// the third is the metered barrier WIRED prints when the article itself was
+// truncated server-side.
+const DEBRIS_PROBES: readonly {
+  readonly label: string;
+  readonly pattern: RegExp;
+}[] = [
+  { label: 'player-controls', pattern: /Loaded:\s*\d+%/ },
+  { label: 'player-controls', pattern: /Duration Time \d+:\d\d/ },
+  {
+    label: 'metered-barrier',
+    pattern: /read your last free article|subscribe to continue reading/i,
+  },
+];
+
+export interface LostEvidence {
+  readonly debrisProbes: readonly string[];
+  readonly fallbackUsed: boolean;
+  readonly gatedReason?: string;
+  readonly nearEmpty: boolean;
+  readonly reasons: readonly string[];
+  readonly wordCount: number;
+}
+
+// The verdict the suggest loop fires on. Gating alone never fires it — vendor
+// SDK classes (paywall-ineligible feed badges, piano offer headers) report on
+// free articles, and the pairing is enforced by construction: `gatedReason` is
+// carried as evidence for the prompt and the report, and only
+// fallback/near-empty/debris count as reasons.
+export function assessLostSignal(input: {
+  contentText: string;
+  fallbackUsed: boolean;
+  gated?: GatingSignal;
+}): LostEvidence {
+  const wordCount = countWords(input.contentText);
+  const nearEmpty = wordCount < NEAR_EMPTY_WORDS;
+  const debrisProbes = [
+    ...new Set(
+      DEBRIS_PROBES.filter(probe => probe.pattern.test(input.contentText)).map(
+        probe => probe.label,
+      ),
+    ),
+  ];
+  const reasons = [
+    ...(input.fallbackUsed ? ['fallback-used'] : []),
+    ...(nearEmpty ? ['near-empty'] : []),
+    ...debrisProbes.map(label => `debris:${label}`),
+  ];
+  return {
+    debrisProbes,
+    fallbackUsed: input.fallbackUsed,
+    gatedReason: input.gated?.likely ? input.gated.reason : undefined,
+    nearEmpty,
+    reasons,
+    wordCount,
+  };
+}

package/src/policy/markdown.js

@@ -0,0 +1,58 @@
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+const processor = unified().use(remarkParse).use(remarkGfm);
+const HEADING_MARKERS = /^#{1,6}\s+/;
+export function parseBlocks(source) {
+    const tree = processor.parse(source);
+    const blocks = [];
+    for (const node of tree.children) {
+        const start = node.position?.start.offset;
+        const end = node.position?.end.offset;
+        if (start === undefined || end === undefined) {
+            continue;
+        }
+        const kind = node.type === 'code'
+            ? 'code'
+            : node.type === 'heading'
+                ? 'heading'
+                : 'other';
+        const depth = node.type === 'heading' ? node.depth : 0;
+        blocks.push({ depth, end, kind, start });
+    }
+    return blocks;
+}
+export function headingText(raw) {
+    return raw.replace(HEADING_MARKERS, '').trim();
+}
+// Hard-cap a run to <= maxChars pieces: pack whole lines, then hard-split any
+// single line that alone exceeds the budget.
+export function hardSplitLines(text, maxChars) {
+    const pieces = [];
+    let buffer = '';
+    function flush() {
+        if (buffer) {
+            pieces.push(buffer);
+            buffer = '';
+        }
+    }
+    for (const line of text.split('\n')) {
+        if (line.length > maxChars) {
+            flush();
+            for (let i = 0; i < line.length; i += maxChars) {
+                pieces.push(line.slice(i, i + maxChars));
+            }
+            continue;
+        }
+        const candidate = buffer ? `${buffer}\n${line}` : line;
+        if (candidate.length > maxChars) {
+            flush();
+            buffer = line;
+        }
+        else {
+            buffer = candidate;
+        }
+    }
+    flush();
+    return pieces;
+}

package/src/policy/markdown.ts

@@ -0,0 +1,76 @@
+import remarkGfm from 'remark-gfm';
+import remarkParse from 'remark-parse';
+import { unified } from 'unified';
+
+export type MarkdownBlockKind = 'code' | 'heading' | 'other';
+
+// A top-level CommonMark block located by source offset. remark is used as a
+// boundary finder, never a serializer: callers slice the *original* string at
+// these offsets so truncate/chunk return byte-for-byte slices of the Turndown
+// output rather than a reflowed re-stringification.
+export interface MarkdownBlock {
+  readonly depth: number;
+  readonly end: number;
+  readonly kind: MarkdownBlockKind;
+  readonly start: number;
+}
+
+const processor = unified().use(remarkParse).use(remarkGfm);
+
+const HEADING_MARKERS = /^#{1,6}\s+/;
+
+export function parseBlocks(source: string): MarkdownBlock[] {
+  const tree = processor.parse(source);
+  const blocks: MarkdownBlock[] = [];
+  for (const node of tree.children) {
+    const start = node.position?.start.offset;
+    const end = node.position?.end.offset;
+    if (start === undefined || end === undefined) {
+      continue;
+    }
+    const kind: MarkdownBlockKind =
+      node.type === 'code'
+        ? 'code'
+        : node.type === 'heading'
+          ? 'heading'
+          : 'other';
+    const depth = node.type === 'heading' ? node.depth : 0;
+    blocks.push({ depth, end, kind, start });
+  }
+  return blocks;
+}
+
+export function headingText(raw: string): string {
+  return raw.replace(HEADING_MARKERS, '').trim();
+}
+
+// Hard-cap a run to <= maxChars pieces: pack whole lines, then hard-split any
+// single line that alone exceeds the budget.
+export function hardSplitLines(text: string, maxChars: number): string[] {
+  const pieces: string[] = [];
+  let buffer = '';
+  function flush(): void {
+    if (buffer) {
+      pieces.push(buffer);
+      buffer = '';
+    }
+  }
+  for (const line of text.split('\n')) {
+    if (line.length > maxChars) {
+      flush();
+      for (let i = 0; i < line.length; i += maxChars) {
+        pieces.push(line.slice(i, i + maxChars));
+      }
+      continue;
+    }
+    const candidate = buffer ? `${buffer}\n${line}` : line;
+    if (candidate.length > maxChars) {
+      flush();
+      buffer = line;
+    } else {
+      buffer = candidate;
+    }
+  }
+  flush();
+  return pieces;
+}

package/src/policy/metadata.js

@@ -0,0 +1,216 @@
+import { estimateTokens, nonEmpty } from "./text.js";
+const ARTICLE_TYPES = new Set([
+    'Article',
+    'BlogPosting',
+    'NewsArticle',
+    'Report',
+    'ScholarlyArticle',
+    'SocialMediaPosting',
+    'TechArticle',
+    'WebPage',
+]);
+function first(...values) {
+    for (const value of values) {
+        const picked = nonEmpty(value);
+        if (picked) {
+            return picked;
+        }
+    }
+    return undefined;
+}
+function metaProperty(document, property) {
+    return nonEmpty(document
+        .querySelector(`meta[property="${property}"]`)
+        ?.getAttribute('content') ?? undefined);
+}
+function metaName(document, name) {
+    return nonEmpty(document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ??
+        undefined);
+}
+// Flatten ld+json scripts (a script may hold an array or a @graph array).
+// Invalid JSON is skipped — never crash extraction over malformed metadata.
+function parseJsonLd(document) {
+    const nodes = document.querySelectorAll('script[type="application/ld+json"]');
+    const out = [];
+    nodes.forEach(node => {
+        const raw = node.textContent;
+        if (!raw.trim()) {
+            return;
+        }
+        let parsed;
+        try {
+            parsed = JSON.parse(raw);
+        }
+        catch {
+            return;
+        }
+        collectObjects(parsed, out);
+    });
+    return out;
+}
+function collectObjects(value, out) {
+    if (!value || typeof value !== 'object') {
+        return;
+    }
+    if (Array.isArray(value)) {
+        for (const item of value) {
+            collectObjects(item, out);
+        }
+        return;
+    }
+    // `in` guard: the index signature types values as JsonLdValue (no undefined),
+    // so a `!== undefined` check has no overlap.
+    if ('@graph' in value) {
+        collectObjects(value['@graph'], out);
+    }
+    out.push(value);
+}
+function typeMatches(type) {
+    const types = Array.isArray(type) ? type : [type];
+    return types.some(t => typeof t === 'string' && ARTICLE_TYPES.has(t));
+}
+function pickArticleNode(candidates) {
+    if (candidates.length === 0) {
+        return undefined;
+    }
+    return candidates.find(node => typeMatches(node['@type'])) ?? candidates[0];
+}
+// Preference order for the primary structured object: richest schema.org
+// main-entity first. Article types are the tail fallback — Readability already
+// captured the article body, so they are low-value-but-recognizable. Generic
+// graph scaffolding (WebSite/Organization/BreadcrumbList) is intentionally NOT
+// listed: emitting it as `structured` would be noise, and we prefer undefined
+// over a misleading trivial node.
+const STRUCTURED_PRIORITY = [
+    'Recipe',
+    'Product',
+    'Event',
+    'HowTo',
+    'Course',
+    'Movie',
+    'Book',
+    'MusicRecording',
+    'JobPosting',
+    'FAQPage',
+    ...ARTICLE_TYPES,
+];
+function nodeTypeList(node) {
+    const type = node['@type'];
+    if (Array.isArray(type)) {
+        return type.filter((t) => typeof t === 'string');
+    }
+    return typeof type === 'string' ? [type] : [];
+}
+// Iterate priority types in order so a Recipe wins over an earlier Article
+// node, and document order wins among candidates of the same type.
+function pickStructuredObject(candidates) {
+    for (const priorityType of STRUCTURED_PRIORITY) {
+        const hit = candidates.find(node => nodeTypeList(node).includes(priorityType));
+        if (hit) {
+            return hit;
+        }
+    }
+    return undefined;
+}
+// Strip @context (always https://schema.org, pure noise) and normalize a
+// multi-typed node to a "+"-joined string so the host sees a single type label.
+// Data fields are kept verbatim — the host wants the real schema.org object.
+function cleanStructured(obj) {
+    const out = {};
+    for (const [key, value] of Object.entries(obj)) {
+        if (key === '@context') {
+            continue;
+        }
+        if (key === '@type') {
+            const types = Array.isArray(value)
+                ? value.filter((t) => typeof t === 'string')
+                : typeof value === 'string'
+                    ? [value]
+                    : [];
+            out[key] = types.join('+');
+            continue;
+        }
+        out[key] = value;
+    }
+    return out;
+}
+// Author may be a string, a Person/Organization object, an array, or a
+// {@list: [...]} container; reduce to a comma-joined byline.
+function resolveJsonLdAuthor(author) {
+    const names = [];
+    function visit(value) {
+        if (typeof value === 'string') {
+            const trimmed = value.trim();
+            if (trimmed) {
+                names.push(trimmed);
+            }
+            return;
+        }
+        if (!value || typeof value !== 'object') {
+            return;
+        }
+        if (Array.isArray(value)) {
+            value.forEach(visit);
+            return;
+        }
+        // {@list: [...]} is an ordering wrapper; unwrap it.
+        const list = value['@list'];
+        if (Array.isArray(list)) {
+            list.forEach(visit);
+            return;
+        }
+        const name = value.name;
+        if (typeof name === 'string') {
+            const trimmed = name.trim();
+            if (trimmed) {
+                names.push(trimmed);
+            }
+        }
+    }
+    visit(author);
+    return names.length > 0 ? names.join(', ') : undefined;
+}
+function asString(value) {
+    return typeof value === 'string' ? nonEmpty(value) : undefined;
+}
+function field(obj, key) {
+    if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+        return undefined;
+    }
+    return obj[key];
+}
+export function resolveMetadata(input) {
+    const { document, readability } = input;
+    const jsonLdObjects = parseJsonLd(document);
+    const jsonLd = pickArticleNode(jsonLdObjects);
+    const structuredRaw = pickStructuredObject(jsonLdObjects);
+    const htmlLang = nonEmpty(document.documentElement.getAttribute('lang') ?? undefined);
+    const titleFromTitleTag = nonEmpty(document.title);
+    const title = first(asString(jsonLd?.headline), metaProperty(document, 'og:title'), metaName(document, 'twitter:title'), readability?.title ?? undefined, titleFromTitleTag);
+    const byline = first(jsonLd ? resolveJsonLdAuthor(jsonLd.author) : undefined, metaProperty(document, 'article:author'), metaName(document, 'author'), readability?.byline ?? undefined);
+    const siteName = first(asString(field(field(jsonLd, 'publisher'), 'name')), metaProperty(document, 'og:site_name'), readability?.siteName ?? undefined);
+    const lang = first(asString(jsonLd?.inLanguage), htmlLang, readability?.lang ?? undefined);
+    const publishedTime = first(asString(jsonLd?.datePublished), metaProperty(document, 'article:published_time'), 
+    // First <time datetime> is the canonical publish time when rendered inline.
+    nonEmpty(document.querySelector('time[datetime]')?.getAttribute('datetime') ??
+        undefined), readability?.publishedTime ?? undefined);
+    const excerpt = first(asString(jsonLd?.description), metaProperty(document, 'og:description'), metaName(document, 'twitter:description'), metaName(document, 'description'), readability?.excerpt ?? undefined);
+    // canonical/og:url are absolute by spec; returned raw so callers can tell
+    // the declared canonical apart from the input baseUrl (origin context only).
+    const canonical = first(nonEmpty(document.querySelector('link[rel="canonical"]')?.getAttribute('href') ??
+        undefined), metaProperty(document, 'og:url'));
+    return {
+        title,
+        byline,
+        siteName,
+        lang,
+        publishedTime,
+        excerpt,
+        canonical,
+        baseUrl: input.baseUrl,
+        wordCount: input.wordCount,
+        readingTimeMin: input.readingTimeMin,
+        ...estimateTokens(input.textContent),
+        ...(structuredRaw ? { structured: cleanStructured(structuredRaw) } : {}),
+    };
+}