page-markdown 0.1.4 → 0.2.0

package/src/index.js

@@ -1,162 +1,200 @@
-// page-markdown: a page's DOM in, clean Markdown and metadata out.
-// sanitize → standardize → extract → convert → clean (→ mode) (→ hidden text)
-import { sanitizeToInertDocument } from "./sanitize.js";
-import { standardizeDocument } from "./standardize.js";
-import { extractContent } from "./extract.js";
-import { htmlToMarkdown } from "./convert.js";
-import { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
-import { collectMetadata } from "./metadata.js";
-import { recoverLeadParagraph, chooseHeadline } from "./refine.js";
-import { collectPageFacts, pageFactsMarkdown } from "./facts.js";
-import { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
-
-export const VERSION = "0.1.4";
-
-export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
-export { standardizeDocument } from "./standardize.js";
-export { extractContent, DEFAULT_MIN_EXTRACT_CHARS } from "./extract.js";
-export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
-export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
-export { collectMetadata } from "./metadata.js";
-export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
-export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
-export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
-
-function documentOf(input) {
-  if (input?.nodeType === 9) return input;
-  if (input?.nodeType === 1) return input.ownerDocument;
-  throw new TypeError("page-markdown: expected a Document or Element");
-}
-
-function toMarkdown(html, mode) {
-  const converted = htmlToMarkdown(html);
-  const { markdown, removedJsonLines } = cleanMarkdown(converted);
-  return { markdown: applyMode(markdown, mode), removedJsonLines, convertedChars: converted.length };
-}
-
-// A top-level heading in the prose; a "# comment" line in fenced code is not one.
-function hasTopHeading(markdown) {
-  let found = false;
-  mapOutsideFences(markdown, (prose) => {
-    found ||= /^# \S/m.test(prose);
-    return prose;
-  });
-  return found;
-}
-
-/**
- * @typedef {object} PageMarkdownOptions
- * @property {string} [url] page URL for absolute links (default: the document's URL)
- * @property {"full"|"no_links"|"text_only"} [mode="full"]
- * @property {"readable"|"full"|((doc: Document) => object|null)} [extract="readable"]
- *   "readable": Readability with full-page fallback; "full": the whole body;
- *   or a custom extractor receiving a copy of the standardized document.
- * @property {number} [minExtractChars=100] below this, fall back to the full page
- * @property {boolean} [includeFullPage=false] also return the full-page Markdown
- * @property {boolean} [includeTitle=true] start readable output with "# <headline>"
- * @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
- * @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
- *   (sidebar versions, prices, spec rows) that the extracted article left out
- * @property {boolean} [removeHiddenText=false] remove tag-character text,
- *   variation-selector runs and invisible operators from the Markdown (fenced
- *   code included), fullPageMarkdown and metadata; omissions count the runs
- *   removed from the Markdown
- * @property {object} [sanitize] see DEFAULT_SANITIZE_OPTIONS
- * @property {object} [readability] options passed to Readability
- */
-
-/**
- * Converts a page (or an element of it) to Markdown.
- * Pure with respect to the input: the live document is only read.
- *
- * @param {Document|Element} input
- * @param {PageMarkdownOptions} [options]
- */
-export function pageToMarkdown(input, options = {}) {
-  const doc = documentOf(input);
-  const url = options.url || doc.URL || doc.location?.href || "";
-  const mode = options.mode || "full";
-  const warnings = [];
-
-  const { document: inert, omissions } = sanitizeToInertDocument(input, options.sanitize);
-  const standardization = standardizeDocument(inert, { url });
-  const extraction = extractContent(inert, {
-    extract: options.extract,
-    minExtractChars: options.minExtractChars,
-    readability: options.readability,
-  });
-  if (extraction.fallbackReason) warnings.push(`extraction fell back to the full page: ${extraction.fallbackReason}`);
-
-  const pageMetadata = collectMetadata(doc, { url });
-  let extractedHtml = extraction.html;
-  let leadRecovered = false;
-  if (extraction.method !== "full_page" && options.recoverLead !== false) {
-    ({ html: extractedHtml, recovered: leadRecovered } = recoverLeadParagraph(extractedHtml, inert, pageMetadata.description));
-  }
-  const headline = chooseHeadline(inert, { ...pageMetadata, extractedTitle: extraction.title });
-
-  const main = toMarkdown(extractedHtml, mode);
-  // Readability lifts the article's H1 out as its title; put a clean heading
-  // back so the capture stands alone, unless the body still has a top-level one.
-  if (options.includeTitle !== false && extraction.method === "readability" && mode !== "text_only") {
-    if (headline && !hasTopHeading(main.markdown)) main.markdown = `# ${headline}\n\n${main.markdown}`;
-  }
-  let pageFacts = [];
-  if (options.pageFacts && extraction.method !== "full_page" && mode !== "text_only") {
-    pageFacts = collectPageFacts(inert, extractedHtml);
-    if (pageFacts.length) main.markdown = `${main.markdown}\n\n${pageFactsMarkdown(pageFacts)}`;
-  }
-  const omissionList = [];
-  if (omissions.shadowHosts) {
-    omissionList.push({ kind: "shadow_content", count: omissions.shadowHosts, note: "Open shadow content omitted: capture limit" });
-  }
-  if (main.removedJsonLines) {
-    omissionList.push({ kind: "json_line", count: main.removedJsonLines, note: "Large single-line JSON documents removed" });
-  }
-
-  const result = {
-    version: VERSION,
-    markdown: main.markdown,
-    metadata: {
-      ...pageMetadata,
-      ...(headline ? { headline } : {}),
-      ...(extraction.title ? { extractedTitle: extraction.title } : {}),
-      ...(extraction.byline ? { byline: extraction.byline } : {}),
-    },
-    extraction: {
-      method: extraction.method,
-      ...(extraction.fallbackReason ? { fallbackReason: extraction.fallbackReason } : {}),
-      ...(leadRecovered ? { leadRecovered: true } : {}),
-      ...(pageFacts.length ? { pageFacts: pageFacts.length } : {}),
-    },
-    omissions: omissionList,
-    warnings,
-    stats: {
-      mode,
-      extractedHtmlChars: extraction.html.length,
-      convertedChars: main.convertedChars,
-      markdownChars: main.markdown.length,
-      standardization,
-    },
-  };
-
-  if (options.includeFullPage) {
-    const full = extraction.method === "full_page"
-      ? main
-      : toMarkdown((inert.body || inert.documentElement).innerHTML || "", mode);
-    result.fullPageMarkdown = full.markdown;
-  }
-  if (options.removeHiddenText) {
-    const cleaned = removeHiddenText(result.markdown);
-    result.markdown = cleaned.text;
-    result.omissions.push(...hiddenTextOmissions(cleaned.runs));
-    if (result.fullPageMarkdown !== undefined) result.fullPageMarkdown = removeHiddenText(result.fullPageMarkdown).text;
-    for (const [key, value] of Object.entries(result.metadata)) {
-      if (typeof value === "string") result.metadata[key] = removeHiddenText(value).text;
-    }
-  }
-  if (omissions.shadowHosts) {
-    result.markdown += `\n\n_Open shadow content omitted: capture limit (${omissions.shadowHosts} host(s))._`;
-  }
-  return result;
-}
+// page-markdown: a page's DOM in, clean Markdown and metadata out.
+// sanitize → standardize → extract → convert → clean (→ mode) (→ hidden text)
+import { sanitizeToInertDocument } from "./sanitize.js";
+import { standardizeDocument } from "./standardize.js";
+import { extractContent } from "./extract.js";
+import { htmlToMarkdown } from "./convert.js";
+import { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
+import { collectMetadata } from "./metadata.js";
+import { recoverLeadParagraph, chooseHeadline } from "./refine.js";
+import { collectPageFacts, pageFactsMarkdown, escapeMarkdown } from "./facts.js";
+import { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
+
+export const VERSION = "0.2.0";
+
+export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
+export { standardizeDocument } from "./standardize.js";
+export { extractContent, DEFAULT_MIN_EXTRACT_CHARS, DEFAULT_MIN_EXTRACT_SHARE } from "./extract.js";
+export { htmlToMarkdown, createTurndown, TURNDOWN_OPTIONS } from "./convert.js";
+export { cleanMarkdown, applyMode, mapOutsideFences } from "./clean.js";
+export { collectMetadata } from "./metadata.js";
+export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
+export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
+export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
+export * from "./types.js";
+
+function documentOf(input) {
+  if (input?.nodeType === 9) return input;
+  if (input?.nodeType === 1) return input.ownerDocument;
+  throw new TypeError("page-markdown: expected a Document or Element");
+}
+
+function toMarkdown(html, mode) {
+  const converted = htmlToMarkdown(html);
+  const { markdown, removedJsonLines } = cleanMarkdown(converted);
+  return { markdown: applyMode(markdown, mode), removedJsonLines, convertedChars: converted.length };
+}
+
+// What relative links resolve against: the page's <base href>, else its URL.
+function baseUrl(doc, url) {
+  const href = doc.querySelector?.("base[href]")?.getAttribute("href");
+  try {
+    return href ? new URL(href, url || undefined).href : url;
+  } catch {
+    return url;
+  }
+}
+
+// A top-level heading in the prose; a "# comment" line in fenced code is not one.
+function hasTopHeading(markdown) {
+  let found = false;
+  mapOutsideFences(markdown, (prose) => {
+    found ||= /^# \S/m.test(prose);
+    return prose;
+  });
+  return found;
+}
+
+/**
+ * @typedef {object} PageMarkdownOptions
+ * @property {string} [url] page URL for absolute links (default: the document's URL)
+ * @property {"full"|"no_links"|"text_only"} [mode="full"]
+ * @property {"readable"|"full"|import("./extract.js").Extractor} [extract="readable"]
+ *   "readable": Readability with full-page fallback; "full": the whole body;
+ *   or a custom extractor receiving a copy of the standardized document.
+ * @property {number} [minExtractChars=100] below this, fall back to the page's <main>
+ *   if it marks one and it holds enough, else to the full page
+ * @property {number} [minExtractShare=0.4] when Readability keeps less than this
+ *   share of the page's main content, use the main content instead; 0 turns it off
+ * @property {boolean} [includeFullPage=false] also return the full-page Markdown
+ * @property {boolean} [includeTitle=true] start readable output with "# <headline>"
+ * @property {boolean} [recoverLead=true] restore a dropped lead paragraph that matches the meta description
+ * @property {boolean} [pageFacts=false] append "## Page facts": short label/value pairs
+ *   (sidebar versions, prices, spec rows) that the extracted article left out
+ * @property {boolean} [removeHiddenText=false] remove tag-character text,
+ *   variation-selector runs and invisible operators from the Markdown (fenced
+ *   code included), fullPageMarkdown and metadata; omissions count the runs
+ *   removed from the Markdown
+ * @property {import("./types.js").SanitizeOptions} [sanitize] see DEFAULT_SANITIZE_OPTIONS
+ * @property {import("./types.js").ReadabilityOptions} [readability] options passed to Readability
+ */
+
+/**
+ * Converts a page (or an element of it) to Markdown.
+ * Pure with respect to the input: the live document is only read.
+ *
+ * @param {Document|Element} input
+ * @param {PageMarkdownOptions} [options]
+ * @returns {import("./types.js").PageMarkdownResult}
+ */
+export function pageToMarkdown(input, options = {}) {
+  const doc = documentOf(input);
+  const url = options.url || doc.URL || doc.location?.href || "";
+  const mode = options.mode || "full";
+  /** @type {string[]} */
+  const warnings = [];
+
+  const { document: inert, omissions } = sanitizeToInertDocument(input, options.sanitize);
+  const standardization = standardizeDocument(inert, { url: baseUrl(doc, url) });
+  const extraction = extractContent(inert, {
+    extract: options.extract,
+    minExtractChars: options.minExtractChars,
+    minExtractShare: options.minExtractShare,
+    readability: options.readability,
+  });
+  const fallbackTarget = extraction.method === "main_content" ? "the main content" : "the full page";
+  if (extraction.fallbackReason) {
+    warnings.push(`extraction fell back to ${fallbackTarget}: ${extraction.fallbackReason}`);
+  }
+  if (omissions.shadowHosts) warnings.push(`shadow content cut off at the capture limit in ${omissions.shadowHosts} host(s); the page's main content may be missing`);
+
+  const pageMetadata = collectMetadata(doc, { url });
+  let extractedHtml = extraction.html;
+  let leadRecovered = false;
+  if (extraction.method !== "full_page" && options.recoverLead !== false) {
+    ({ html: extractedHtml, recovered: leadRecovered } = recoverLeadParagraph(extractedHtml, inert, pageMetadata.description));
+  }
+  const headline = chooseHeadline(inert, { ...pageMetadata, extractedTitle: extraction.title });
+
+  const main = toMarkdown(extractedHtml, mode);
+  // Readability lifts the article's H1 out as its title; put a clean heading
+  // back so the capture stands alone, unless the body still has a top-level one.
+  if (options.includeTitle !== false && ["readability", "main_content"].includes(extraction.method) && mode !== "text_only") {
+    if (headline && !hasTopHeading(main.markdown)) main.markdown = `# ${headline}\n\n${main.markdown}`;
+  }
+  let pageFacts = [];
+  if (options.pageFacts && extraction.method !== "full_page" && mode !== "text_only") {
+    pageFacts = collectPageFacts(inert, extractedHtml);
+    if (pageFacts.length) main.markdown = `${main.markdown}\n\n${pageFactsMarkdown(pageFacts)}`;
+  }
+  /** @type {import("./types.js").PageMarkdownOmission[]} */
+  const omissionList = [];
+  if (omissions.shadowHosts) {
+    omissionList.push({ kind: "shadow_content", count: omissions.shadowHosts, note: "Open shadow content omitted: capture limit" });
+  }
+  if (main.removedJsonLines) {
+    omissionList.push({ kind: "json_line", count: main.removedJsonLines, note: "Large single-line JSON documents removed" });
+  }
+  if (standardization.omittedTabs.length) {
+    const count = standardization.omittedTabs.length;
+    const labels = standardization.omittedTabs.filter(Boolean).map(escapeMarkdown).join(", ");
+    omissionList.push({ kind: "inactive_tabs", count, note: labels ? `Other tabs not included: ${labels}` : `${count} other tab(s) not included` });
+  }
+  if (extraction.commentsOmitted) {
+    omissionList.push({ kind: "comments", count: 1, note: "Comment section not included" });
+  }
+
+  /** @type {import("./types.js").PageMarkdownResult} */
+  const result = {
+    version: VERSION,
+    markdown: main.markdown,
+    metadata: {
+      ...pageMetadata,
+      ...(headline ? { headline } : {}),
+      ...(extraction.title ? { extractedTitle: extraction.title } : {}),
+      ...(extraction.byline ? { byline: extraction.byline } : {}),
+    },
+    extraction: {
+      method: extraction.method,
+      ...(extraction.fallbackReason ? { fallbackReason: extraction.fallbackReason } : {}),
+      ...(leadRecovered ? { leadRecovered: true } : {}),
+      ...(pageFacts.length ? { pageFacts: pageFacts.length } : {}),
+    },
+    omissions: omissionList,
+    warnings,
+    stats: {
+      mode,
+      extractedHtmlChars: extraction.html.length,
+      convertedChars: main.convertedChars,
+      markdownChars: main.markdown.length,
+      pageTextChars: (inert.body?.textContent || "").replace(/\s+/g, " ").trim().length,
+      standardization,
+    },
+  };
+
+  if (options.includeFullPage) {
+    const full = extraction.method === "full_page"
+      ? main
+      : toMarkdown((inert.body || inert.documentElement).innerHTML || "", mode);
+    result.fullPageMarkdown = full.markdown;
+  }
+  if (options.removeHiddenText) {
+    const cleaned = removeHiddenText(result.markdown);
+    result.markdown = cleaned.text;
+    result.omissions.push(...hiddenTextOmissions(cleaned.runs));
+    if (result.fullPageMarkdown !== undefined) result.fullPageMarkdown = removeHiddenText(result.fullPageMarkdown).text;
+    for (const [key, value] of Object.entries(result.metadata)) {
+      if (typeof value === "string") result.metadata[key] = removeHiddenText(value).text;
+    }
+  }
+  // A caller may pass on only the Markdown, so what changes the content says so
+  // in it too. Tab labels are page text: hidden-text removal applies to them.
+  const notices = [
+    ...(extraction.fallbackReason ? [`Captured ${fallbackTarget} instead of the article (\`${extraction.fallbackReason}\`)`] : []),
+    ...omissionList.filter(({ kind }) => ["json_line", "inactive_tabs", "comments"].includes(kind)).map((omission) => omission.note),
+    ...(omissions.shadowHosts ? [`Open shadow content omitted: capture limit (${omissions.shadowHosts} host(s))`] : []),
+  ].map((notice) => (options.removeHiddenText ? removeHiddenText(notice).text : notice));
+  for (const notice of notices) result.markdown += `\n\n_${notice}._`;
+  result.stats.markdownChars = result.markdown.length;
+  return result;
+}

package/src/metadata.js

@@ -1,78 +1,79 @@
-// Page metadata from the original document (read-only): meta tags, JSON-LD,
-// canonical link and language. The sanitized clone has no <meta> or scripts.
-
-function clean(value, limit = 500) {
-  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
-}
-
-function meta(doc, selectors) {
-  for (const selector of selectors) {
-    const value = doc.querySelector?.(selector)?.getAttribute?.("content");
-    if (value && value.trim()) return clean(value);
-  }
-  return "";
-}
-
-function absolute(value, base) {
-  const raw = clean(value, 2000);
-  if (!raw) return "";
-  try {
-    return base ? new URL(raw, base).href : raw;
-  } catch {
-    return raw;
-  }
-}
-
-function jsonLdNodes(doc) {
-  const nodes = [];
-  for (const script of Array.from(doc.querySelectorAll?.('script[type="application/ld+json"]') || [])) {
-    try {
-      const data = JSON.parse(script.textContent || "");
-      const queue = Array.isArray(data) ? [...data] : [data];
-      while (queue.length) {
-        const node = queue.shift();
-        if (!node || typeof node !== "object") continue;
-        nodes.push(node);
-        if (Array.isArray(node["@graph"])) queue.push(...node["@graph"]);
-      }
-    } catch {
-      // Malformed JSON-LD is common; ignore it.
-    }
-  }
-  return nodes;
-}
-
-function nameOf(value) {
-  if (!value) return "";
-  if (typeof value === "string") return clean(value);
-  if (Array.isArray(value)) return value.map(nameOf).filter(Boolean).join(", ");
-  return clean(value.name || "");
-}
-
-const ARTICLE_TYPES = /Article|BlogPosting|NewsArticle|Report|TechArticle|WebPage|ScholarlyArticle/i;
-
-/**
- * @param {Document} doc the original page document
- * @param {{ url?: string }} [options]
- */
-export function collectMetadata(doc, options = {}) {
-  const url = options.url || doc.URL || doc.location?.href || "";
-  const ld = jsonLdNodes(doc);
-  const article = ld.find((node) => ARTICLE_TYPES.test(String(node["@type"] || ""))) || {};
-  const canonicalHref = doc.querySelector?.("link[rel='canonical']")?.getAttribute?.("href") || "";
-  let hostname = "";
-  try { hostname = url ? new URL(url).hostname : ""; } catch { /* keep empty */ }
-
-  return {
-    url,
-    canonicalUrl: absolute(canonicalHref, url) || url,
-    documentTitle: clean(doc.title || doc.querySelector?.("title")?.textContent || ""),
-    title: meta(doc, ["meta[property='og:title']", "meta[name='twitter:title']"]) || clean(article.headline || "") || clean(doc.title || ""),
-    description: meta(doc, ["meta[name='description']", "meta[property='og:description']", "meta[name='twitter:description']"]),
-    author: meta(doc, ["meta[name='author']", "meta[property='article:author']", "meta[name='byl']"]) || nameOf(article.author),
-    published: meta(doc, ["meta[property='article:published_time']", "meta[name='date']", "meta[itemprop='datePublished']"]) || clean(article.datePublished || ""),
-    modified: meta(doc, ["meta[property='article:modified_time']", "meta[itemprop='dateModified']", "meta[name='last-modified']"]) || clean(article.dateModified || ""),
-    siteName: meta(doc, ["meta[property='og:site_name']", "meta[name='application-name']"]) || nameOf(article.publisher) || hostname,
-    lang: clean(doc.documentElement?.getAttribute?.("lang") || "", 40),
-  };
-}
+// Page metadata from the original document (read-only): meta tags, JSON-LD,
+// canonical link and language. The sanitized clone has no <meta> or scripts.
+
+function clean(value, limit = 500) {
+  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
+}
+
+function meta(doc, selectors) {
+  for (const selector of selectors) {
+    const value = doc.querySelector?.(selector)?.getAttribute?.("content");
+    if (value && value.trim()) return clean(value);
+  }
+  return "";
+}
+
+function absolute(value, base) {
+  const raw = clean(value, 2000);
+  if (!raw) return "";
+  try {
+    return base ? new URL(raw, base).href : raw;
+  } catch {
+    return raw;
+  }
+}
+
+function jsonLdNodes(doc) {
+  const nodes = [];
+  for (const script of Array.from(doc.querySelectorAll?.('script[type="application/ld+json"]') || [])) {
+    try {
+      const data = JSON.parse(script.textContent || "");
+      const queue = Array.isArray(data) ? [...data] : [data];
+      while (queue.length) {
+        const node = queue.shift();
+        if (!node || typeof node !== "object") continue;
+        nodes.push(node);
+        if (Array.isArray(node["@graph"])) queue.push(...node["@graph"]);
+      }
+    } catch {
+      // Malformed JSON-LD is common; ignore it.
+    }
+  }
+  return nodes;
+}
+
+/** @returns {string} */
+function nameOf(value) {
+  if (!value) return "";
+  if (typeof value === "string") return clean(value);
+  if (Array.isArray(value)) return value.map(nameOf).filter(Boolean).join(", ");
+  return clean(value.name || "");
+}
+
+const ARTICLE_TYPES = /Article|BlogPosting|NewsArticle|Report|TechArticle|WebPage|ScholarlyArticle/i;
+
+/**
+ * @param {Document} doc the original page document
+ * @param {{ url?: string }} [options]
+ */
+export function collectMetadata(doc, options = {}) {
+  const url = options.url || doc.URL || doc.location?.href || "";
+  const ld = jsonLdNodes(doc);
+  const article = ld.find((node) => ARTICLE_TYPES.test(String(node["@type"] || ""))) || {};
+  const canonicalHref = doc.querySelector?.("link[rel='canonical']")?.getAttribute?.("href") || "";
+  let hostname = "";
+  try { hostname = url ? new URL(url).hostname : ""; } catch { /* keep empty */ }
+
+  return {
+    url,
+    canonicalUrl: absolute(canonicalHref, url) || url,
+    documentTitle: clean(doc.title || doc.querySelector?.("title")?.textContent || ""),
+    title: meta(doc, ["meta[property='og:title']", "meta[name='twitter:title']"]) || clean(article.headline || "") || clean(doc.title || ""),
+    description: meta(doc, ["meta[name='description']", "meta[property='og:description']", "meta[name='twitter:description']"]),
+    author: meta(doc, ["meta[name='author']", "meta[property='article:author']", "meta[name='byl']"]) || nameOf(article.author),
+    published: meta(doc, ["meta[property='article:published_time']", "meta[name='date']", "meta[itemprop='datePublished']"]) || clean(article.datePublished || ""),
+    modified: meta(doc, ["meta[property='article:modified_time']", "meta[itemprop='dateModified']", "meta[name='last-modified']"]) || clean(article.dateModified || ""),
+    siteName: meta(doc, ["meta[property='og:site_name']", "meta[name='application-name']"]) || nameOf(article.publisher) || hostname,
+    lang: clean(doc.documentElement?.getAttribute?.("lang") || "", 40),
+  };
+}