page-markdown 0.0.0-stage → 0.1.3

package/src/convert.js

@@ -0,0 +1,219 @@
+// Stage 4: HTML to Markdown with Turndown plus GitHub-flavoured rules:
+// strikethrough, task-list checkboxes and tables. Every data table becomes a
+// GFM table, never raw HTML.
+import TurndownService from "turndown";
+
+export const TURNDOWN_OPTIONS = Object.freeze({
+  headingStyle: "atx",
+  bulletListMarker: "-",
+  codeBlockStyle: "fenced",
+  emDelimiter: "_",
+  strongDelimiter: "**",
+  linkStyle: "inlined",
+  hr: "---",
+});
+
+function gfmInlineRules(turndown) {
+  turndown.addRule("pmStrikethrough", {
+    filter: ["del", "s", "strike"],
+    replacement: (content) => `~~${content}~~`,
+  });
+  // A checkbox at the start of a list item (directly, or in a label or span).
+  turndown.addRule("pmTaskListItem", {
+    filter(node) {
+      const isCheckbox = (node.nodeName === "INPUT" && String(node.getAttribute("type")).toLowerCase() === "checkbox")
+        || node.getAttribute?.("role") === "checkbox";
+      if (!isCheckbox) return false;
+      const parent = node.parentNode;
+      return parent?.nodeName === "LI"
+        || (["LABEL", "SPAN"].includes(parent?.nodeName) && parent.parentNode?.nodeName === "LI");
+    },
+    replacement(content, node) {
+      const checked = node.nodeName === "INPUT"
+        ? Boolean(node.checked) || node.hasAttribute?.("checked")
+        : node.getAttribute("aria-checked") === "true";
+      return checked ? "[x] " : "[ ] ";
+    },
+  });
+}
+
+function closest(node, name) {
+  for (let parent = node.parentNode; parent; parent = parent.parentNode) {
+    if (parent.nodeName === name) return parent;
+  }
+  return null;
+}
+
+function tableRows(table) {
+  const rows = [];
+  const walk = (node) => {
+    for (const child of Array.from(node.childNodes || [])) {
+      if (child.nodeName === "TR") rows.push(child);
+      else if (["THEAD", "TBODY", "TFOOT"].includes(child.nodeName)) walk(child);
+    }
+  };
+  walk(table);
+  return rows;
+}
+
+function rowCells(row) {
+  return Array.from(row.childNodes || []).filter((node) => node.nodeName === "TD" || node.nodeName === "TH");
+}
+
+function spanOf(cell) {
+  const span = Number.parseInt(cell.getAttribute?.("colspan") || "1", 10);
+  return Number.isFinite(span) && span > 1 ? Math.min(span, 50) : 1;
+}
+
+// Layout tables (nesting, a single cell, a single column) read better as
+// blocks than as a grid.
+function isLayoutTable(table) {
+  if (table.querySelector?.("table")) return true;
+  const rows = tableRows(table);
+  if (!rows.length) return true;
+  const widest = Math.max(...rows.map((row) => rowCells(row).reduce((sum, cell) => sum + spanOf(cell), 0)));
+  return widest <= 1;
+}
+
+// A GFM cell is one line, so a data table holding code blocks is rendered as
+// blocks too, each cell under its column's header: the code stays fenced.
+function isBlockTable(table) {
+  return isLayoutTable(table) || Boolean(table.querySelector?.("pre"));
+}
+
+function isBlockCell(node) {
+  const table = closest(node, "TABLE");
+  return !table || isBlockTable(table);
+}
+
+function labelText(cell) {
+  return String(cell.textContent || "").replace(/\s+/g, " ").trim();
+}
+
+// A first row of two or more <th>: column headers. (A single spanning <th>
+// is a title, not a header.)
+function columnHeaders(table) {
+  const first = tableRows(table)[0];
+  const cells = first ? rowCells(first) : [];
+  return cells.length > 1 && cells.every((cell) => cell.nodeName === "TH") ? first : null;
+}
+
+function rowHeader(row) {
+  const [first, ...rest] = rowCells(row);
+  return first?.nodeName === "TH" && rest.length ? first : null;
+}
+
+// The label for a cell of a block-rendered data table: its column's header
+// (colspans counted), else its row's header. null when the cell is itself a
+// label; "" for layout tables and unlabelled cells.
+function cellLabel(cell) {
+  const table = closest(cell, "TABLE");
+  if (!table || isLayoutTable(table)) return "";
+  const row = cell.parentNode;
+  const header = columnHeaders(table);
+  if (header) {
+    if (row === header) return null;
+    let column = 0;
+    for (const other of rowCells(row)) {
+      if (other === cell) break;
+      column += spanOf(other);
+    }
+    let start = 0;
+    for (const th of rowCells(header)) {
+      if (column < start + spanOf(th)) return labelText(th);
+      start += spanOf(th);
+    }
+    return "";
+  }
+  const th = rowHeader(row);
+  if (!th) return "";
+  return cell === th ? null : labelText(th);
+}
+
+// A GFM cell is one line: collapse breaks and block structure to spaces,
+// and escape pipes.
+function cellText(content) {
+  const text = String(content)
+    .replace(/\n+/g, " ")
+    .replace(/\s{2,}/g, " ")
+    .trim();
+  // Every pipe: Turndown escapes a literal backslash as "\\", so "\|" in the
+  // content is not an escaped pipe.
+  return text.replace(/\|/g, "\\|");
+}
+
+function tableRules(turndown) {
+  turndown.addRule("pmTableCell", {
+    filter: ["th", "td"],
+    replacement(content, node) {
+      if (isBlockCell(node)) {
+        // Header cells become the labels on the other cells.
+        const label = cellLabel(node);
+        if (label === null) return "";
+        return label ? `\n\n**${createTurndown().escape(label)}**\n\n${content}\n\n` : `\n\n${content}\n\n`;
+      }
+      const pad = " |   ".repeat(spanOf(node) - 1);
+      return ` ${cellText(content)}${pad} |`;
+    },
+  });
+  turndown.addRule("pmTableRow", {
+    filter: "tr",
+    replacement(content, node) {
+      if (isBlockCell(node)) return `\n${content}\n`;
+      return `\n|${content}`;
+    },
+  });
+  turndown.addRule("pmTableSection", {
+    filter: ["thead", "tbody", "tfoot"],
+    replacement(content) {
+      return content;
+    },
+  });
+  turndown.addRule("pmTable", {
+    filter: "table",
+    replacement(content, node) {
+      if (isBlockTable(node)) return `\n\n${content}\n\n`;
+      const lines = content.split("\n").filter((line) => line.startsWith("|"));
+      if (!lines.length) return "\n\n";
+      const width = Math.max(...lines.map((line) => line.split(/(?<!\\)\|/).length - 2));
+      const padded = lines.map((line) => {
+        const cells = line.split(/(?<!\\)\|/).length - 2;
+        return cells < width ? `${line}${" |".repeat(width - cells)}` : line;
+      });
+      // GFM needs a header row; the first row serves when the table has none.
+      const separator = `|${" --- |".repeat(width)}`;
+      const caption = node.querySelector?.("caption")?.textContent?.trim();
+      const body = [padded[0], separator, ...padded.slice(1)].join("\n");
+      return `\n\n${caption ? `${cellText(caption)}\n\n` : ""}${body}\n\n`;
+    },
+  });
+  turndown.addRule("pmTableCaption", {
+    filter: "caption",
+    replacement() {
+      return "";
+    },
+  });
+}
+
+let shared = null;
+
+/** A configured Turndown instance (cached). */
+export function createTurndown() {
+  if (shared) return shared;
+  const turndown = new TurndownService({ ...TURNDOWN_OPTIONS });
+  gfmInlineRules(turndown);
+  tableRules(turndown);
+  // Elements that never carry readable text. (Buttons stay: accordions put
+  // headings in them.)
+  turndown.remove(["script", "style", "noscript", "template", "iframe", "svg", "canvas"]);
+  shared = turndown;
+  return turndown;
+}
+
+/**
+ * @param {string} html
+ * @returns {string}
+ */
+export function htmlToMarkdown(html) {
+  return createTurndown().turndown(String(html || ""));
+}

package/src/extract.js

@@ -0,0 +1,70 @@
+// Stage 3: pick the readable content. Default: Mozilla Readability on a copy
+// of the standardized clone, falling back to the whole body when it finds
+// nothing or too little.
+// The class only: skips isProbablyReaderable, which the pipeline does not use.
+import Readability from "@mozilla/readability/Readability.js";
+
+export const DEFAULT_MIN_EXTRACT_CHARS = 100;
+
+function textLength(html = "") {
+  return String(html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().length;
+}
+
+function bodyHtml(doc) {
+  return (doc.body || doc.documentElement)?.innerHTML || "";
+}
+
+function copyDocument(doc) {
+  const copy = doc.cloneNode(true);
+  if (copy && copy.nodeType === 9 && copy.documentElement) return copy;
+  // Implementations whose Document.cloneNode is shallow: reparse.
+  const Parser = doc.defaultView?.DOMParser || globalThis.DOMParser;
+  return new Parser().parseFromString(doc.documentElement.outerHTML, "text/html");
+}
+
+/**
+ * @typedef {{ html: string, method: string, fallbackReason?: string, title?: string,
+ *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string }} Extraction
+ */
+
+/**
+ * @param {Document} doc standardized inert document (not mutated)
+ * @param {{ extract?: "readable"|"full"|((doc: Document) => Extraction|null),
+ *   minExtractChars?: number, readability?: object }} [options]
+ * @returns {Extraction}
+ */
+export function extractContent(doc, options = {}) {
+  const mode = options.extract ?? "readable";
+  const full = () => ({ html: bodyHtml(doc), method: "full_page" });
+
+  if (mode === "full") return full();
+
+  const minChars = options.minExtractChars ?? DEFAULT_MIN_EXTRACT_CHARS;
+  let result = null;
+  let failure = "";
+  try {
+    if (typeof mode === "function") {
+      result = mode(copyDocument(doc));
+    } else {
+      const article = new Readability(copyDocument(doc), { keepClasses: true, ...options.readability }).parse();
+      result = article && {
+        html: article.content || "",
+        method: "readability",
+        title: article.title || "",
+        byline: article.byline || "",
+        published: article.publishedTime || "",
+        siteName: article.siteName || "",
+        lang: article.lang || "",
+        excerpt: article.excerpt || "",
+      };
+    }
+  } catch (error) {
+    failure = `extractor_error: ${error?.message || error}`;
+  }
+
+  if (!result?.html) return { ...full(), fallbackReason: failure || "extractor_found_nothing" };
+  if (textLength(result.html) < minChars) {
+    return { ...full(), fallbackReason: "extracted_text_below_minimum", title: result.title };
+  }
+  return result;
+}

package/src/facts.js

@@ -0,0 +1,96 @@
+// Short label/value facts the extractor left out: package sidebars (version,
+// license, downloads), price and spec boxes, definition lists, two-column
+// tables. Readability keeps the article and drops these asides, which is right
+// for reading and wrong when the question is "which version?". Opt-in.
+
+const MAX_FACTS = 24;
+const MAX_LABEL_CHARS = 40;
+const MAX_VALUE_CHARS = 100;
+const CHROME = "nav, footer, header, form, [role='navigation'], [role='contentinfo'], [role='banner'], [role='search']";
+
+function squash(text) {
+  return String(text || "").replace(/\s+/g, " ").trim();
+}
+
+// A sentence is prose, not a fact: "Returns a signal that aborts later."
+function isSentence(text) {
+  return /[.!?]$/.test(text) && text.split(" ").length >= 4;
+}
+
+function shortValue(el) {
+  if (!el || el.querySelector("h1, h2, h3, h4, h5, h6, table, pre")) return "";
+  // A list of links is navigation, not a fact.
+  const links = el.tagName === "A" ? [el] : el.querySelectorAll("a");
+  if (links.length > 1) return "";
+  const text = squash(el.textContent);
+  // So is a lone link, unless its text is data: a domain, path or number.
+  if (links.length === 1 && squash(links[0].textContent) === text && !/[\d./@]/.test(text)) return "";
+  return text.length <= MAX_VALUE_CHARS && !isSentence(text) ? text : "";
+}
+
+function* candidatePairs(doc) {
+  // Heading followed by a short value: <h3>Version</h3><p>1.2.3</p>.
+  for (const h of doc.querySelectorAll("h2, h3, h4, h5, h6")) {
+    yield [h.textContent, shortValue(h.nextElementSibling), h];
+  }
+  for (const dt of doc.querySelectorAll("dt")) {
+    const dd = dt.nextElementSibling;
+    if (dd?.tagName === "DD") yield [dt.textContent, shortValue(dd), dt];
+  }
+  // Two-cell rows: | Label | Value |.
+  for (const row of doc.querySelectorAll("tr")) {
+    const cells = Array.from(row.children).filter((c) => c.tagName === "TH" || c.tagName === "TD");
+    if (cells.length === 2) yield [cells[0].textContent, shortValue(cells[1]), row];
+  }
+}
+
+// "1.0.1 December 17, 2004 [5]" and "1.0.1December 17, 2004" compare equal.
+function compact(text) {
+  return text.replace(/\[\d+\]/g, "").replace(/\s+/g, "");
+}
+
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * @param {Document} doc standardized clone
+ * @param {string} articleHtml the extracted article
+ * @returns {Array<{ label: string, value: string }>} facts absent from the article
+ */
+export function collectPageFacts(doc, articleHtml) {
+  const article = squash(String(articleHtml || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ")
+    .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&amp;/g, "&"));
+  const articleCompact = compact(article);
+  const pairs = Array.from(candidatePairs(doc), ([label, value, el]) => [squash(label).replace(/\s*[:：¶#§]\s*$/, ""), value, el]);
+  // A "value" that is the next section's label is a heading run, not a fact.
+  const labels = new Set(pairs.map(([label]) => label));
+  const facts = [];
+  const seen = new Set();
+  for (const [label, value, el] of pairs) {
+    if (!label || !value || label.length > MAX_LABEL_CHARS || label === value || labels.has(value)) continue;
+    if (el.closest(CHROME)) continue;
+    const key = `${label}\u0000${value}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    // Already in the article: short values as a whole word, longer ones
+    // ignoring spacing and citation marks.
+    const bare = compact(value);
+    if (bare.length >= 6 ? articleCompact.includes(bare)
+      : new RegExp(`(^|[^\\w])${escapeRegExp(value)}($|[^\\w])`).test(article)) continue;
+    facts.push({ label, value });
+    if (facts.length >= MAX_FACTS) break;
+  }
+  return facts;
+}
+
+function escapeMarkdown(text) {
+  return text.replace(/([\\`*_[\]<>|])/g, "\\$1");
+}
+
+/** "## Page facts" list, or "" when there are none. */
+export function pageFactsMarkdown(facts) {
+  if (!facts.length) return "";
+  const lines = facts.map(({ label, value }) => `- ${escapeMarkdown(label)}: ${escapeMarkdown(value)}`);
+  return `## Page facts\n\n${lines.join("\n")}`;
+}

package/src/hidden.js

@@ -0,0 +1,44 @@
+// Optional last stage: remove characters that hide text from people but not
+// from models (tag characters, variation-selector runs, invisible operators),
+// found by text-sanctifier's run finder. Legitimate sequences are kept: the
+// England, Scotland and Wales flags and a lone emoji or text presentation
+// selector. Unlike the clean stage, this also applies inside fenced code.
+import { findHiddenRuns } from "text-sanctifier/hidden";
+
+const NOTES = {
+  tag: "Hidden tag-character text removed",
+  variation_selector: "Variation selector runs removed",
+  invisible_operator: "Invisible operator runs removed",
+};
+
+/**
+ * @param {string} text
+ * @returns {{ text: string, runs: Array<{ kind: string, count: number }> }}
+ */
+export function removeHiddenText(text) {
+  const source = String(text ?? "");
+  const { runs } = findHiddenRuns(source);
+  if (!runs.length) return { text: source, runs };
+  let out = "";
+  let at = 0;
+  for (const run of runs) {
+    out += source.slice(at, run.offset);
+    at = run.end;
+  }
+  return { text: out + source.slice(at), runs };
+}
+
+/**
+ * One omission per kind of run: `count` runs, `chars` code points.
+ * @param {Array<{ kind: string, count: number }>} runs
+ */
+export function hiddenTextOmissions(runs) {
+  const byKind = new Map();
+  for (const run of runs) {
+    const entry = byKind.get(run.kind) || { kind: `hidden_${run.kind}`, count: 0, chars: 0, note: NOTES[run.kind] };
+    entry.count += 1;
+    entry.chars += run.count;
+    byKind.set(run.kind, entry);
+  }
+  return Array.from(byKind.values());
+}

package/src/index.js

@@ -0,0 +1,162 @@
+// page-markdown: a page's DOM in, clean Markdown and metadata out.
+// sanitize → standardize → extract → convert → clean (→ mode) (→ hidden text)
+import { sanitizeToInertDocument } from "./sanitize.js";
+import { standardizeDocument } from "./standardize.js";
+import { extractContent } from "./extract.js";
+import { htmlToMarkdown } from "./convert.js";
+import { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
+import { collectMetadata } from "./metadata.js";
+import { recoverLeadParagraph, chooseHeadline } from "./refine.js";
+import { collectPageFacts, pageFactsMarkdown } from "./facts.js";
+import { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
+
+export const VERSION = "0.1.3";
+
+export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
+export { standardizeDocument } from "./standardize.js";
+export { extractContent, DEFAULT_MIN_EXTRACT_CHARS } from "./extract.js";
+export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
+export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
+export { collectMetadata } from "./metadata.js";
+export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
+export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
+export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
+
+function documentOf(input) {
+  if (input?.nodeType === 9) return input;
+  if (input?.nodeType === 1) return input.ownerDocument;
+  throw new TypeError("page-markdown: expected a Document or Element");
+}
+
+function toMarkdown(html, mode) {
+  const converted = htmlToMarkdown(html);
+  const { markdown, removedJsonLines } = cleanMarkdown(converted);
+  return { markdown: applyMode(markdown, mode), removedJsonLines, convertedChars: converted.length };
+}
+
+// A top-level heading in the prose; a "# comment" line in fenced code is not one.
+function hasTopHeading(markdown) {
+  let found = false;
+  mapOutsideFences(markdown, (prose) => {
+    found ||= /^# \S/m.test(prose);
+    return prose;
+  });
+  return found;
+}
+
+/**
+ * @typedef {object} PageMarkdownOptions
+ * @property {string} [url] page URL for absolute links (default: the document's URL)
+ * @property {"full"|"no_links"|"text_only"} [mode="full"]
+ * @property {"readable"|"full"|((doc: Document) => object|null)} [extract="readable"]
+ *   "readable": Readability with full-page fallback; "full": the whole body;
+ *   or a custom extractor receiving a copy of the standardized document.
+ * @property {number} [minExtractChars=100] below this, fall back to the full page
+ * @property {boolean} [includeFullPage=false] also return the full-page Markdown
+ * @property {boolean} [includeTitle=true] start readable output with "# <headline>"
+ * @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
+ * @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
+ *   (sidebar versions, prices, spec rows) that the extracted article left out
+ * @property {boolean} [removeHiddenText=false] remove tag-character text,
+ *   variation-selector runs and invisible operators from the Markdown (fenced
+ *   code included), fullPageMarkdown and metadata; omissions count the runs
+ *   removed from the Markdown
+ * @property {object} [sanitize] see DEFAULT_SANITIZE_OPTIONS
+ * @property {object} [readability] options passed to Readability
+ */
+
+/**
+ * Converts a page (or an element of it) to Markdown.
+ * Pure with respect to the input: the live document is only read.
+ *
+ * @param {Document|Element} input
+ * @param {PageMarkdownOptions} [options]
+ */
+export function pageToMarkdown(input, options = {}) {
+  const doc = documentOf(input);
+  const url = options.url || doc.URL || doc.location?.href || "";
+  const mode = options.mode || "full";
+  const warnings = [];
+
+  const { document: inert, omissions } = sanitizeToInertDocument(input, options.sanitize);
+  const standardization = standardizeDocument(inert, { url });
+  const extraction = extractContent(inert, {
+    extract: options.extract,
+    minExtractChars: options.minExtractChars,
+    readability: options.readability,
+  });
+  if (extraction.fallbackReason) warnings.push(`extraction fell back to the full page: ${extraction.fallbackReason}`);
+
+  const pageMetadata = collectMetadata(doc, { url });
+  let extractedHtml = extraction.html;
+  let leadRecovered = false;
+  if (extraction.method !== "full_page" && options.recoverLead !== false) {
+    ({ html: extractedHtml, recovered: leadRecovered } = recoverLeadParagraph(extractedHtml, inert, pageMetadata.description));
+  }
+  const headline = chooseHeadline(inert, { ...pageMetadata, extractedTitle: extraction.title });
+
+  const main = toMarkdown(extractedHtml, mode);
+  // Readability lifts the article's H1 out as its title; put a clean heading
+  // back so the capture stands alone, unless the body still has a top-level one.
+  if (options.includeTitle !== false && extraction.method === "readability" && mode !== "text_only") {
+    if (headline && !hasTopHeading(main.markdown)) main.markdown = `# ${headline}\n\n${main.markdown}`;
+  }
+  let pageFacts = [];
+  if (options.pageFacts && extraction.method !== "full_page" && mode !== "text_only") {
+    pageFacts = collectPageFacts(inert, extractedHtml);
+    if (pageFacts.length) main.markdown = `${main.markdown}\n\n${pageFactsMarkdown(pageFacts)}`;
+  }
+  const omissionList = [];
+  if (omissions.shadowHosts) {
+    omissionList.push({ kind: "shadow_content", count: omissions.shadowHosts, note: "Open shadow content omitted: capture limit" });
+  }
+  if (main.removedJsonLines) {
+    omissionList.push({ kind: "json_line", count: main.removedJsonLines, note: "Large single-line JSON documents removed" });
+  }
+
+  const result = {
+    version: VERSION,
+    markdown: main.markdown,
+    metadata: {
+      ...pageMetadata,
+      ...(headline ? { headline } : {}),
+      ...(extraction.title ? { extractedTitle: extraction.title } : {}),
+      ...(extraction.byline ? { byline: extraction.byline } : {}),
+    },
+    extraction: {
+      method: extraction.method,
+      ...(extraction.fallbackReason ? { fallbackReason: extraction.fallbackReason } : {}),
+      ...(leadRecovered ? { leadRecovered: true } : {}),
+      ...(pageFacts.length ? { pageFacts: pageFacts.length } : {}),
+    },
+    omissions: omissionList,
+    warnings,
+    stats: {
+      mode,
+      extractedHtmlChars: extraction.html.length,
+      convertedChars: main.convertedChars,
+      markdownChars: main.markdown.length,
+      standardization,
+    },
+  };
+
+  if (options.includeFullPage) {
+    const full = extraction.method === "full_page"
+      ? main
+      : toMarkdown((inert.body || inert.documentElement).innerHTML || "", mode);
+    result.fullPageMarkdown = full.markdown;
+  }
+  if (options.removeHiddenText) {
+    const cleaned = removeHiddenText(result.markdown);
+    result.markdown = cleaned.text;
+    result.omissions.push(...hiddenTextOmissions(cleaned.runs));
+    if (result.fullPageMarkdown !== undefined) result.fullPageMarkdown = removeHiddenText(result.fullPageMarkdown).text;
+    for (const [key, value] of Object.entries(result.metadata)) {
+      if (typeof value === "string") result.metadata[key] = removeHiddenText(value).text;
+    }
+  }
+  if (omissions.shadowHosts) {
+    result.markdown += `\n\n_Open shadow content omitted: capture limit (${omissions.shadowHosts} host(s))._`;
+  }
+  return result;
+}

package/src/metadata.js

@@ -0,0 +1,78 @@
+// Page metadata from the original document (read-only): meta tags, JSON-LD,
+// canonical link and language. The sanitized clone has no <meta> or scripts.
+
+function clean(value, limit = 500) {
+  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
+}
+
+function meta(doc, selectors) {
+  for (const selector of selectors) {
+    const value = doc.querySelector?.(selector)?.getAttribute?.("content");
+    if (value && value.trim()) return clean(value);
+  }
+  return "";
+}
+
+function absolute(value, base) {
+  const raw = clean(value, 2000);
+  if (!raw) return "";
+  try {
+    return base ? new URL(raw, base).href : raw;
+  } catch {
+    return raw;
+  }
+}
+
+function jsonLdNodes(doc) {
+  const nodes = [];
+  for (const script of Array.from(doc.querySelectorAll?.('script[type="application/ld+json"]') || [])) {
+    try {
+      const data = JSON.parse(script.textContent || "");
+      const queue = Array.isArray(data) ? [...data] : [data];
+      while (queue.length) {
+        const node = queue.shift();
+        if (!node || typeof node !== "object") continue;
+        nodes.push(node);
+        if (Array.isArray(node["@graph"])) queue.push(...node["@graph"]);
+      }
+    } catch {
+      // Malformed JSON-LD is common; ignore it.
+    }
+  }
+  return nodes;
+}
+
+function nameOf(value) {
+  if (!value) return "";
+  if (typeof value === "string") return clean(value);
+  if (Array.isArray(value)) return value.map(nameOf).filter(Boolean).join(", ");
+  return clean(value.name || "");
+}
+
+const ARTICLE_TYPES = /Article|BlogPosting|NewsArticle|Report|TechArticle|WebPage|ScholarlyArticle/i;
+
+/**
+ * @param {Document} doc the original page document
+ * @param {{ url?: string }} [options]
+ */
+export function collectMetadata(doc, options = {}) {
+  const url = options.url || doc.URL || doc.location?.href || "";
+  const ld = jsonLdNodes(doc);
+  const article = ld.find((node) => ARTICLE_TYPES.test(String(node["@type"] || ""))) || {};
+  const canonicalHref = doc.querySelector?.("link[rel='canonical']")?.getAttribute?.("href") || "";
+  let hostname = "";
+  try { hostname = url ? new URL(url).hostname : ""; } catch { /* keep empty */ }
+
+  return {
+    url,
+    canonicalUrl: absolute(canonicalHref, url) || url,
+    documentTitle: clean(doc.title || doc.querySelector?.("title")?.textContent || ""),
+    title: meta(doc, ["meta[property='og:title']", "meta[name='twitter:title']"]) || clean(article.headline || "") || clean(doc.title || ""),
+    description: meta(doc, ["meta[name='description']", "meta[property='og:description']", "meta[name='twitter:description']"]),
+    author: meta(doc, ["meta[name='author']", "meta[property='article:author']", "meta[name='byl']"]) || nameOf(article.author),
+    published: meta(doc, ["meta[property='article:published_time']", "meta[name='date']", "meta[itemprop='datePublished']"]) || clean(article.datePublished || ""),
+    modified: meta(doc, ["meta[property='article:modified_time']", "meta[itemprop='dateModified']", "meta[name='last-modified']"]) || clean(article.dateModified || ""),
+    siteName: meta(doc, ["meta[property='og:site_name']", "meta[name='application-name']"]) || nameOf(article.publisher) || hostname,
+    lang: clean(doc.documentElement?.getAttribute?.("lang") || "", 40),
+  };
+}