@v1nvn/readability 0.44.0 → 0.45.0

package/src/policy/metadata.ts

@@ -0,0 +1,312 @@
+import type { Metadata, StructuredData } from '../pipeline/context.ts';
+import type { ReadabilityParseResult } from '../pipeline/readability.ts';
+
+import { estimateTokens, nonEmpty } from './text.ts';
+
+const ARTICLE_TYPES = new Set([
+  'Article',
+  'BlogPosting',
+  'NewsArticle',
+  'Report',
+  'ScholarlyArticle',
+  'SocialMediaPosting',
+  'TechArticle',
+  'WebPage',
+]);
+
+type JsonLdValue =
+  boolean | JsonLdObject | JsonLdValue[] | null | number | string;
+interface JsonLdObject {
+  readonly [key: string]: JsonLdValue;
+}
+
+function first(...values: readonly (string | undefined)[]): string | undefined {
+  for (const value of values) {
+    const picked = nonEmpty(value);
+    if (picked) {
+      return picked;
+    }
+  }
+  return undefined;
+}
+
+function metaProperty(
+  document: Document,
+  property: string,
+): string | undefined {
+  return nonEmpty(
+    document
+      .querySelector(`meta[property="${property}"]`)
+      ?.getAttribute('content') ?? undefined,
+  );
+}
+
+function metaName(document: Document, name: string): string | undefined {
+  return nonEmpty(
+    document.querySelector(`meta[name="${name}"]`)?.getAttribute('content') ??
+      undefined,
+  );
+}
+
+// Flatten ld+json scripts (a script may hold an array or a @graph array).
+// Invalid JSON is skipped — never crash extraction over malformed metadata.
+function parseJsonLd(document: Document): JsonLdObject[] {
+  const nodes = document.querySelectorAll('script[type="application/ld+json"]');
+  const out: JsonLdObject[] = [];
+  nodes.forEach(node => {
+    const raw = node.textContent;
+    if (!raw.trim()) {
+      return;
+    }
+    let parsed: JsonLdValue;
+    try {
+      parsed = JSON.parse(raw) as JsonLdValue;
+    } catch {
+      return;
+    }
+    collectObjects(parsed, out);
+  });
+  return out;
+}
+
+function collectObjects(value: JsonLdValue, out: JsonLdObject[]): void {
+  if (!value || typeof value !== 'object') {
+    return;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      collectObjects(item, out);
+    }
+    return;
+  }
+  // `in` guard: the index signature types values as JsonLdValue (no undefined),
+  // so a `!== undefined` check has no overlap.
+  if ('@graph' in value) {
+    collectObjects(value['@graph'], out);
+  }
+  out.push(value);
+}
+
+function typeMatches(type: JsonLdValue): boolean {
+  const types = Array.isArray(type) ? type : [type];
+  return types.some(t => typeof t === 'string' && ARTICLE_TYPES.has(t));
+}
+
+function pickArticleNode(
+  candidates: readonly JsonLdObject[],
+): JsonLdObject | undefined {
+  if (candidates.length === 0) {
+    return undefined;
+  }
+  return candidates.find(node => typeMatches(node['@type'])) ?? candidates[0];
+}
+
+// Preference order for the primary structured object: richest schema.org
+// main-entity first. Article types are the tail fallback — Readability already
+// captured the article body, so they are low-value-but-recognizable. Generic
+// graph scaffolding (WebSite/Organization/BreadcrumbList) is intentionally NOT
+// listed: emitting it as `structured` would be noise, and we prefer undefined
+// over a misleading trivial node.
+const STRUCTURED_PRIORITY = [
+  'Recipe',
+  'Product',
+  'Event',
+  'HowTo',
+  'Course',
+  'Movie',
+  'Book',
+  'MusicRecording',
+  'JobPosting',
+  'FAQPage',
+  ...ARTICLE_TYPES,
+] as const;
+
+function nodeTypeList(node: JsonLdObject): readonly string[] {
+  const type = node['@type'];
+  if (Array.isArray(type)) {
+    return type.filter((t): t is string => typeof t === 'string');
+  }
+  return typeof type === 'string' ? [type] : [];
+}
+
+// Iterate priority types in order so a Recipe wins over an earlier Article
+// node, and document order wins among candidates of the same type.
+function pickStructuredObject(
+  candidates: readonly JsonLdObject[],
+): JsonLdObject | undefined {
+  for (const priorityType of STRUCTURED_PRIORITY) {
+    const hit = candidates.find(node =>
+      nodeTypeList(node).includes(priorityType),
+    );
+    if (hit) {
+      return hit;
+    }
+  }
+  return undefined;
+}
+
+// Strip @context (always https://schema.org, pure noise) and normalize a
+// multi-typed node to a "+"-joined string so the host sees a single type label.
+// Data fields are kept verbatim — the host wants the real schema.org object.
+function cleanStructured(obj: JsonLdObject): StructuredData {
+  const out: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(obj)) {
+    if (key === '@context') {
+      continue;
+    }
+    if (key === '@type') {
+      const types = Array.isArray(value)
+        ? value.filter((t): t is string => typeof t === 'string')
+        : typeof value === 'string'
+          ? [value]
+          : [];
+      out[key] = types.join('+');
+      continue;
+    }
+    out[key] = value;
+  }
+  return out;
+}
+
+// Author may be a string, a Person/Organization object, an array, or a
+// {@list: [...]} container; reduce to a comma-joined byline.
+function resolveJsonLdAuthor(author: JsonLdValue): string | undefined {
+  const names: string[] = [];
+  function visit(value: JsonLdValue): void {
+    if (typeof value === 'string') {
+      const trimmed = value.trim();
+      if (trimmed) {
+        names.push(trimmed);
+      }
+      return;
+    }
+    if (!value || typeof value !== 'object') {
+      return;
+    }
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    // {@list: [...]} is an ordering wrapper; unwrap it.
+    const list = value['@list'];
+    if (Array.isArray(list)) {
+      list.forEach(visit);
+      return;
+    }
+    const name = value.name;
+    if (typeof name === 'string') {
+      const trimmed = name.trim();
+      if (trimmed) {
+        names.push(trimmed);
+      }
+    }
+  }
+  visit(author);
+  return names.length > 0 ? names.join(', ') : undefined;
+}
+
+function asString(value: JsonLdValue | undefined): string | undefined {
+  return typeof value === 'string' ? nonEmpty(value) : undefined;
+}
+
+function field(
+  obj: JsonLdValue | undefined,
+  key: string,
+): JsonLdValue | undefined {
+  if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
+    return undefined;
+  }
+  return obj[key];
+}
+
+export interface MetadataInput {
+  readonly baseUrl?: string;
+  readonly document: Document;
+  readonly readability?: null | ReadabilityParseResult;
+  readonly readingTimeMin: number;
+  readonly textContent: string;
+  readonly wordCount: number;
+}
+
+export function resolveMetadata(input: Readonly<MetadataInput>): Metadata {
+  const { document, readability } = input;
+
+  const jsonLdObjects = parseJsonLd(document);
+  const jsonLd = pickArticleNode(jsonLdObjects);
+  const structuredRaw = pickStructuredObject(jsonLdObjects);
+  const htmlLang = nonEmpty(
+    document.documentElement.getAttribute('lang') ?? undefined,
+  );
+  const titleFromTitleTag = nonEmpty(document.title);
+
+  const title = first(
+    asString(jsonLd?.headline),
+    metaProperty(document, 'og:title'),
+    metaName(document, 'twitter:title'),
+    readability?.title ?? undefined,
+    titleFromTitleTag,
+  );
+
+  const byline = first(
+    jsonLd ? resolveJsonLdAuthor(jsonLd.author) : undefined,
+    metaProperty(document, 'article:author'),
+    metaName(document, 'author'),
+    readability?.byline ?? undefined,
+  );
+
+  const siteName = first(
+    asString(field(field(jsonLd, 'publisher'), 'name')),
+    metaProperty(document, 'og:site_name'),
+    readability?.siteName ?? undefined,
+  );
+
+  const lang = first(
+    asString(jsonLd?.inLanguage),
+    htmlLang,
+    readability?.lang ?? undefined,
+  );
+
+  const publishedTime = first(
+    asString(jsonLd?.datePublished),
+    metaProperty(document, 'article:published_time'),
+    // First <time datetime> is the canonical publish time when rendered inline.
+    nonEmpty(
+      document.querySelector('time[datetime]')?.getAttribute('datetime') ??
+        undefined,
+    ),
+    readability?.publishedTime ?? undefined,
+  );
+
+  const excerpt = first(
+    asString(jsonLd?.description),
+    metaProperty(document, 'og:description'),
+    metaName(document, 'twitter:description'),
+    metaName(document, 'description'),
+    readability?.excerpt ?? undefined,
+  );
+
+  // canonical/og:url are absolute by spec; returned raw so callers can tell
+  // the declared canonical apart from the input baseUrl (origin context only).
+  const canonical = first(
+    nonEmpty(
+      document.querySelector('link[rel="canonical"]')?.getAttribute('href') ??
+        undefined,
+    ),
+    metaProperty(document, 'og:url'),
+  );
+
+  return {
+    title,
+    byline,
+    siteName,
+    lang,
+    publishedTime,
+    excerpt,
+    canonical,
+    baseUrl: input.baseUrl,
+    wordCount: input.wordCount,
+    readingTimeMin: input.readingTimeMin,
+    ...estimateTokens(input.textContent),
+    ...(structuredRaw ? { structured: cleanStructured(structuredRaw) } : {}),
+  };
+}

package/src/policy/outline-chains.ts

@@ -0,0 +1,215 @@
+import type { SelectorScope } from '../pipeline/normalize.ts';
+
+import { buildDocument } from '../pipeline/dom.ts';
+import {
+  applySelectors,
+  normalizeDocument,
+  resolveLazyImages,
+} from '../pipeline/normalize.ts';
+import { isGeneratedIdentifier } from './identifiers.ts';
+
+// Step-0 measured thresholds: 200 own chars is the chain view that produced a
+// working proposal at ~630 tokens, and the debris that view misses (video
+// controls, captions) holds under 200 own chars — the round-two sweep runs at
+// 40 to surface it.
+export const OUTLINE_OWN_TEXT_MIN = 200;
+export const BLOCK_OWN_TEXT_MIN = 40;
+export const OUTLINE_MAX_CHARS = 6000;
+export const OUTLINE_MAX_CHAINS = 60;
+const SAMPLE_MAX_CHARS = 80;
+
+const NON_CONTENT_SELECTOR = 'script, style, noscript, template';
+
+const HOP_ATTRIBUTE_WHITELIST = ['itemprop', 'role', 'data-testid'] as const;
+
+export interface OutlineResult {
+  readonly chainCount: number;
+  readonly text: string;
+  readonly truncated: boolean;
+}
+
+export interface OutlineInput {
+  readonly baseUrl?: string;
+  readonly cleanChrome: boolean;
+  readonly html: string;
+  // `page` renders the round-one view of the whole normalized document; a
+  // scope renders the round-two view of the included subtree after the scope
+  // has been applied for real.
+  readonly mode: 'page' | { readonly scope: SelectorScope };
+}
+
+interface Candidate {
+  readonly element: Element;
+  readonly own: number;
+}
+
+// The document the outline renders is normalized exactly as extraction
+// normalizes it, so a selector proposed from this material validates and
+// applies against the same tree the pipeline will run it on.
+export function buildChainOutline(input: OutlineInput): OutlineResult {
+  const { document } = buildDocument(input.html, input.baseUrl);
+  normalizeDocument(document, { cleanChrome: input.cleanChrome });
+  resolveLazyImages(document);
+  const scope = input.mode === 'page' ? undefined : input.mode.scope;
+  if (scope) {
+    applySelectors(document, scope);
+  }
+  const scopeRoot = scope?.include
+    ? (document.body.querySelector(scope.include) ?? undefined)
+    : undefined;
+
+  const min = scope ? BLOCK_OWN_TEXT_MIN : OUTLINE_OWN_TEXT_MIN;
+  const candidates: Candidate[] = [];
+  for (const element of (scopeRoot ?? document.body).querySelectorAll('*')) {
+    if (element.closest(NON_CONTENT_SELECTOR)) {
+      continue;
+    }
+    const own = ownTextLength(element);
+    if (own >= min) {
+      candidates.push({ element, own });
+    }
+  }
+
+  return scope
+    ? groupedOutline(candidates, scopeRoot)
+    : flatOutline(candidates);
+}
+
+// Round one: one full ancestor chain per qualifying node, step-0 format.
+function flatOutline(candidates: Candidate[]): OutlineResult {
+  const lines: string[] = [];
+  let truncated = false;
+  for (const candidate of [...candidates].sort((a, b) => b.own - a.own)) {
+    if (lines.length >= OUTLINE_MAX_CHAINS) {
+      truncated = true;
+      break;
+    }
+    const hops = ancestorsUpTo(candidate.element, undefined).map(element =>
+      hopWithSizes(element),
+    );
+    const line = hops.join(' > ');
+    if (lines.join('\n---\n').length + line.length > OUTLINE_MAX_CHARS) {
+      truncated = true;
+      break;
+    }
+    lines.push(line);
+  }
+  return { chainCount: lines.length, text: lines.join('\n---\n'), truncated };
+}
+
+// Round two: the included subtree's remaining text blocks, grouped by
+// structural chain so dozens of prose paragraphs collapse into one line and
+// the odd control-text/caption container stands alone. Smallest own text
+// first — that is where the debris lives.
+function groupedOutline(
+  candidates: Candidate[],
+  scopeRoot: Element | undefined,
+): OutlineResult {
+  const groups = new Map<
+    string,
+    { chain: string; count: number; max: number; min: number; sample: string }
+  >();
+  for (const candidate of candidates) {
+    const chain = ancestorsUpTo(candidate.element, scopeRoot).map(element =>
+      renderHop(element),
+    );
+    if (scopeRoot) {
+      chain.push(renderHop(scopeRoot));
+    }
+    const key = chain.join(' > ');
+    const existing = groups.get(key);
+    if (existing) {
+      existing.count += 1;
+      existing.min = Math.min(existing.min, candidate.own);
+      existing.max = Math.max(existing.max, candidate.own);
+    } else {
+      groups.set(key, {
+        chain: key,
+        count: 1,
+        min: candidate.own,
+        max: candidate.own,
+        sample: sampleText(candidate.element),
+      });
+    }
+  }
+
+  const lines: string[] = [];
+  let truncated = false;
+  const sorted = [...groups.values()].sort((a, b) => a.min - b.min);
+  for (const group of sorted) {
+    if (lines.length >= OUTLINE_MAX_CHAINS) {
+      truncated = true;
+      break;
+    }
+    const line = `${group.chain} (own:${group.min}..${group.max}, ×${group.count}) sample: "${group.sample}"`;
+    if (lines.join('\n').length + line.length > OUTLINE_MAX_CHARS) {
+      truncated = true;
+      break;
+    }
+    lines.push(line);
+  }
+  return { chainCount: lines.length, text: lines.join('\n'), truncated };
+}
+
+// Nearest-to-leaf first; the walk stops below <body> (the root applySelectors
+// never matches) and below the include root when one is given — the caller
+// prepends that root's own hop where it is wanted.
+function ancestorsUpTo(element: Element, stop: Element | undefined): Element[] {
+  const hops: Element[] = [];
+  let node: Element | null = element;
+  const body = element.ownerDocument.body;
+  while (node && node !== stop && node !== body) {
+    hops.push(node);
+    node = node.parentElement;
+  }
+  return hops;
+}
+
+function hopWithSizes(element: Element): string {
+  const own = ownTextLength(element);
+  const all = element.textContent.length;
+  return `${renderHop(element)} (own:${own}, all:${all})`;
+}
+
+function ownTextLength(element: Element): number {
+  let own = 0;
+  for (const node of element.childNodes) {
+    if (node.nodeType === node.TEXT_NODE) {
+      own += (node.textContent ?? '').length;
+    }
+  }
+  return own;
+}
+
+// Copy-safety: what renders here is what the suggester may type back. Every
+// token is real CSS — real attribute names and values, and classes/ids that
+// survive the generated-identifier lint, so a hash class can never be copied
+// into a proposal because it was never shown.
+function renderHop(element: Element): string {
+  let hop = element.tagName.toLowerCase();
+  const id = element.id;
+  if (id && !isGeneratedIdentifier(id)) {
+    hop += `#${id}`;
+  }
+  const classes = (element.getAttribute('class') ?? '')
+    .split(/\s+/)
+    .filter(Boolean)
+    .filter(className => !isGeneratedIdentifier(className));
+  if (classes.length > 0) {
+    hop += `.${classes.join('.')}`;
+  }
+  for (const attribute of HOP_ATTRIBUTE_WHITELIST) {
+    const value = element.getAttribute(attribute);
+    if (value) {
+      hop += `[${attribute}="${value}"]`;
+    }
+  }
+  return hop;
+}
+
+function sampleText(element: Element): string {
+  const text = element.textContent.trim().replace(/\s+/g, ' ');
+  return text.length > SAMPLE_MAX_CHARS
+    ? `${text.slice(0, SAMPLE_MAX_CHARS)}…`
+    : text;
+}

package/src/policy/outline.ts

@@ -0,0 +1,89 @@
+export interface OutlineEntry {
+  readonly anchor: string;
+  readonly level: number;
+  readonly text: string;
+}
+
+// Collapse whitespace, trim, then strip leading/trailing `#` (GitHub permalinks
+// append `<a>#</a>`, so heading textContent ends with `#`).
+export function normalizeHeadingText(raw: string): string {
+  const collapsed = raw.replace(/\s+/g, ' ').trim();
+  return collapsed.replace(/^#+|#+$/g, '').trim();
+}
+
+// GFM anchor algorithm (comrak `anchorize`): keep Unicode letters, marks,
+// numbers, and connector punctuation; map spaces to hyphens. No hyphen collapse
+// or trim, so anchors match GitHub and non-Latin scripts survive. An all-symbol
+// heading yields an empty slug; fall back to `section` so anchors stay non-empty.
+function slugify(text: string): string {
+  const cleaned = text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{M}\p{N}\p{Pc} -]/gu, '');
+  return cleaned.replace(/ /g, '-') || 'section';
+}
+
+// `href="#"` (empty fragment) does not win — fall through to slugify.
+function linkAnchor(heading: Element): string | undefined {
+  for (const link of heading.querySelectorAll('a[href^="#"]')) {
+    const href = link.getAttribute('href');
+    if (!href) {
+      continue;
+    }
+    const fragment = href.slice(1);
+    if (fragment) {
+      return fragment;
+    }
+  }
+  return undefined;
+}
+
+function resolveCandidate(
+  heading: Element,
+  text: string,
+): { readonly anchor: string; readonly explicit: boolean } {
+  const id = heading.getAttribute('id');
+  if (id) {
+    return { anchor: id, explicit: true };
+  }
+  const link = linkAnchor(heading);
+  if (link) {
+    return { anchor: link, explicit: true };
+  }
+  return { anchor: slugify(text), explicit: false };
+}
+
+// Explicit anchors are kept verbatim; generated slugs are collision-suffixed
+// (`-1`, `-2`, …), first occurrence bare.
+function dedupe(
+  candidate: string,
+  explicit: boolean,
+  used: Set<string>,
+): string {
+  if (explicit || !used.has(candidate)) {
+    return candidate;
+  }
+  let suffix = 1;
+  while (used.has(`${candidate}-${suffix}`)) {
+    suffix++;
+  }
+  return `${candidate}-${suffix}`;
+}
+
+export function resolveOutline(document: Document): OutlineEntry[] {
+  const entries: OutlineEntry[] = [];
+  const used = new Set<string>();
+  const headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
+  headings.forEach(heading => {
+    const rawText = heading.textContent;
+    if (!rawText.trim()) {
+      return;
+    }
+    const text = normalizeHeadingText(rawText);
+    const level = Number.parseInt(heading.tagName.slice(1), 10);
+    const { anchor: candidate, explicit } = resolveCandidate(heading, text);
+    const anchor = dedupe(candidate, explicit, used);
+    used.add(anchor);
+    entries.push({ anchor, level, text });
+  });
+  return entries;
+}