page-markdown 0.1.3 → 0.1.5

package/src/metadata.js

@@ -1,78 +1,79 @@
-// Page metadata from the original document (read-only): meta tags, JSON-LD,
-// canonical link and language. The sanitized clone has no <meta> or scripts.
-
-function clean(value, limit = 500) {
-  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
-}
-
-function meta(doc, selectors) {
-  for (const selector of selectors) {
-    const value = doc.querySelector?.(selector)?.getAttribute?.("content");
-    if (value && value.trim()) return clean(value);
-  }
-  return "";
-}
-
-function absolute(value, base) {
-  const raw = clean(value, 2000);
-  if (!raw) return "";
-  try {
-    return base ? new URL(raw, base).href : raw;
-  } catch {
-    return raw;
-  }
-}
-
-function jsonLdNodes(doc) {
-  const nodes = [];
-  for (const script of Array.from(doc.querySelectorAll?.('script[type="application/ld+json"]') || [])) {
-    try {
-      const data = JSON.parse(script.textContent || "");
-      const queue = Array.isArray(data) ? [...data] : [data];
-      while (queue.length) {
-        const node = queue.shift();
-        if (!node || typeof node !== "object") continue;
-        nodes.push(node);
-        if (Array.isArray(node["@graph"])) queue.push(...node["@graph"]);
-      }
-    } catch {
-      // Malformed JSON-LD is common; ignore it.
-    }
-  }
-  return nodes;
-}
-
-function nameOf(value) {
-  if (!value) return "";
-  if (typeof value === "string") return clean(value);
-  if (Array.isArray(value)) return value.map(nameOf).filter(Boolean).join(", ");
-  return clean(value.name || "");
-}
-
-const ARTICLE_TYPES = /Article|BlogPosting|NewsArticle|Report|TechArticle|WebPage|ScholarlyArticle/i;
-
-/**
- * @param {Document} doc the original page document
- * @param {{ url?: string }} [options]
- */
-export function collectMetadata(doc, options = {}) {
-  const url = options.url || doc.URL || doc.location?.href || "";
-  const ld = jsonLdNodes(doc);
-  const article = ld.find((node) => ARTICLE_TYPES.test(String(node["@type"] || ""))) || {};
-  const canonicalHref = doc.querySelector?.("link[rel='canonical']")?.getAttribute?.("href") || "";
-  let hostname = "";
-  try { hostname = url ? new URL(url).hostname : ""; } catch { /* keep empty */ }
-
-  return {
-    url,
-    canonicalUrl: absolute(canonicalHref, url) || url,
-    documentTitle: clean(doc.title || doc.querySelector?.("title")?.textContent || ""),
-    title: meta(doc, ["meta[property='og:title']", "meta[name='twitter:title']"]) || clean(article.headline || "") || clean(doc.title || ""),
-    description: meta(doc, ["meta[name='description']", "meta[property='og:description']", "meta[name='twitter:description']"]),
-    author: meta(doc, ["meta[name='author']", "meta[property='article:author']", "meta[name='byl']"]) || nameOf(article.author),
-    published: meta(doc, ["meta[property='article:published_time']", "meta[name='date']", "meta[itemprop='datePublished']"]) || clean(article.datePublished || ""),
-    modified: meta(doc, ["meta[property='article:modified_time']", "meta[itemprop='dateModified']", "meta[name='last-modified']"]) || clean(article.dateModified || ""),
-    siteName: meta(doc, ["meta[property='og:site_name']", "meta[name='application-name']"]) || nameOf(article.publisher) || hostname,
-    lang: clean(doc.documentElement?.getAttribute?.("lang") || "", 40),
-  };
-}
+// Page metadata from the original document (read-only): meta tags, JSON-LD,
+// canonical link and language. The sanitized clone has no <meta> or scripts.
+
+function clean(value, limit = 500) {
+  return String(value ?? "").replace(/\s+/g, " ").trim().slice(0, limit);
+}
+
+function meta(doc, selectors) {
+  for (const selector of selectors) {
+    const value = doc.querySelector?.(selector)?.getAttribute?.("content");
+    if (value && value.trim()) return clean(value);
+  }
+  return "";
+}
+
+function absolute(value, base) {
+  const raw = clean(value, 2000);
+  if (!raw) return "";
+  try {
+    return base ? new URL(raw, base).href : raw;
+  } catch {
+    return raw;
+  }
+}
+
+function jsonLdNodes(doc) {
+  const nodes = [];
+  for (const script of Array.from(doc.querySelectorAll?.('script[type="application/ld+json"]') || [])) {
+    try {
+      const data = JSON.parse(script.textContent || "");
+      const queue = Array.isArray(data) ? [...data] : [data];
+      while (queue.length) {
+        const node = queue.shift();
+        if (!node || typeof node !== "object") continue;
+        nodes.push(node);
+        if (Array.isArray(node["@graph"])) queue.push(...node["@graph"]);
+      }
+    } catch {
+      // Malformed JSON-LD is common; ignore it.
+    }
+  }
+  return nodes;
+}
+
+/** @returns {string} */
+function nameOf(value) {
+  if (!value) return "";
+  if (typeof value === "string") return clean(value);
+  if (Array.isArray(value)) return value.map(nameOf).filter(Boolean).join(", ");
+  return clean(value.name || "");
+}
+
+const ARTICLE_TYPES = /Article|BlogPosting|NewsArticle|Report|TechArticle|WebPage|ScholarlyArticle/i;
+
+/**
+ * @param {Document} doc the original page document
+ * @param {{ url?: string }} [options]
+ */
+export function collectMetadata(doc, options = {}) {
+  const url = options.url || doc.URL || doc.location?.href || "";
+  const ld = jsonLdNodes(doc);
+  const article = ld.find((node) => ARTICLE_TYPES.test(String(node["@type"] || ""))) || {};
+  const canonicalHref = doc.querySelector?.("link[rel='canonical']")?.getAttribute?.("href") || "";
+  let hostname = "";
+  try { hostname = url ? new URL(url).hostname : ""; } catch { /* keep empty */ }
+
+  return {
+    url,
+    canonicalUrl: absolute(canonicalHref, url) || url,
+    documentTitle: clean(doc.title || doc.querySelector?.("title")?.textContent || ""),
+    title: meta(doc, ["meta[property='og:title']", "meta[name='twitter:title']"]) || clean(article.headline || "") || clean(doc.title || ""),
+    description: meta(doc, ["meta[name='description']", "meta[property='og:description']", "meta[name='twitter:description']"]),
+    author: meta(doc, ["meta[name='author']", "meta[property='article:author']", "meta[name='byl']"]) || nameOf(article.author),
+    published: meta(doc, ["meta[property='article:published_time']", "meta[name='date']", "meta[itemprop='datePublished']"]) || clean(article.datePublished || ""),
+    modified: meta(doc, ["meta[property='article:modified_time']", "meta[itemprop='dateModified']", "meta[name='last-modified']"]) || clean(article.dateModified || ""),
+    siteName: meta(doc, ["meta[property='og:site_name']", "meta[name='application-name']"]) || nameOf(article.publisher) || hostname,
+    lang: clean(doc.documentElement?.getAttribute?.("lang") || "", 40),
+  };
+}

package/src/refine.js

@@ -1,104 +1,109 @@
-// Corrections after extraction, using the standardized clone and the page's
-// metadata: restore a lead paragraph the extractor dropped, and choose a clean
-// title heading.
-
-const MIN_LEAD_CHARS = 40;
-const MATCH_CHARS = 60;
-const SEPARATORS = [" | ", " – ", " — ", " - ", " · ", " :: ", " » "];
-
-/** Lowercase, collapse whitespace, drop a trailing ellipsis. */
-function norm(text) {
-  return String(text || "").replace(/\s+/g, " ").replace(/(\.\.\.|…)\s*$/, "").trim().toLowerCase();
-}
-
-function textOf(html) {
-  return norm(String(html || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&"));
-}
-
-function startsAlike(a, b) {
-  const n = Math.min(MATCH_CHARS, a.length, b.length);
-  return n >= MIN_LEAD_CHARS && a.slice(0, n) === b.slice(0, n);
-}
-
-/**
- * Many sites state their summary as both the meta description and the first
- * paragraph. Extractors can drop that paragraph when it sits in a section
- * without a heading (MDN). If the description matches a paragraph in the page
- * that the extracted HTML lacks, put that paragraph first.
- *
- * @param {string} html extracted HTML
- * @param {Document} doc standardized clone
- * @param {string} description meta description
- * @returns {{ html: string, recovered: boolean }}
- */
-export function recoverLeadParagraph(html, doc, description) {
-  const lead = norm(description);
-  if (lead.length < MIN_LEAD_CHARS) return { html, recovered: false };
-  const extracted = textOf(html);
-  const paragraphs = Array.from(doc.querySelectorAll?.("p") || []).slice(0, 300);
-  for (const p of paragraphs) {
-    const text = norm(p.textContent);
-    if (text.length < MIN_LEAD_CHARS || !startsAlike(text, lead)) continue;
-    if (extracted.includes(text.slice(0, 80))) return { html, recovered: false };
-    return { html: `${p.outerHTML}\n${html}`, recovered: true };
-  }
-  return { html, recovered: false };
-}
-
-function cleanHeading(text) {
-  return String(text || "")
-    .replace(/\s+/g, " ")
-    .replace(/\s*[¶#§]\s*$/, "")
-    .trim();
-}
-
-function firstHeading(doc) {
-  const scoped = doc.querySelector?.("main h1, article h1, [role='main'] h1");
-  return cleanHeading((scoped || doc.querySelector?.("h1"))?.textContent);
-}
-
-// Drops site-name segments at either end: "Page | Site", "Site - Page".
-function stripSiteSuffix(title, siteName) {
-  let result = String(title || "").trim();
-  const site = norm(siteName);
-  if (!site) return result;
-  const isSite = (segment) => {
-    const value = norm(segment);
-    return Boolean(value) && (value.includes(site) || site.includes(value));
-  };
-  for (let changed = true; changed;) {
-    changed = false;
-    for (const separator of SEPARATORS) {
-      const last = result.lastIndexOf(separator);
-      if (last > 0 && isSite(result.slice(last + separator.length))) {
-        result = result.slice(0, last).trim();
-        changed = true;
-      }
-      const first = result.indexOf(separator);
-      if (first > 0 && isSite(result.slice(0, first))) {
-        result = result.slice(first + separator.length).trim();
-        changed = true;
-      }
-    }
-  }
-  return result;
-}
-
-/**
- * The heading for the capture: the page's own H1 when the document title or
- * og:title contains it (and it is not just the site name); otherwise og:title;
- * otherwise the title without a trailing site-name segment.
- *
- * @param {Document} doc standardized clone
- * @param {{ documentTitle?: string, title?: string, siteName?: string, extractedTitle?: string }} meta
- */
-export function chooseHeadline(doc, meta = {}) {
-  const h1 = firstHeading(doc);
-  const titles = [meta.documentTitle, meta.title, meta.extractedTitle].map(norm).filter(Boolean);
-  if (h1 && h1.length <= 200 && norm(h1) !== norm(meta.siteName) && titles.some((t) => t.includes(norm(h1)))) {
-    return h1;
-  }
-  const og = String(meta.title || "").trim();
-  if (og && og !== String(meta.documentTitle || "").trim()) return stripSiteSuffix(og, meta.siteName);
-  return stripSiteSuffix(meta.extractedTitle || meta.documentTitle || og, meta.siteName);
-}
+// Corrections after extraction, using the standardized clone and the page's
+// metadata: restore a lead paragraph the extractor dropped, and choose a clean
+// title heading.
+
+const MIN_LEAD_CHARS = 40;
+const MATCH_CHARS = 60;
+const SEPARATORS = [" | ", " – ", " — ", " - ", " · ", " :: ", " » "];
+
+/** Lowercase, collapse whitespace, drop a trailing ellipsis. */
+function norm(text) {
+  return String(text || "").replace(/\s+/g, " ").replace(/(\.\.\.|…)\s*$/, "").trim().toLowerCase();
+}
+
+/** The text of serialized HTML: tags to spaces, the entities serializers write decoded. */
+export function htmlText(html) {
+  return String(html || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ")
+    .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&amp;/g, "&");
+}
+
+// Tags became spaces in htmlText, so "<a>Foo</a>, the" reads "Foo , the".
+const withoutSpaces = (text) => text.replace(/\s+/g, "");
+
+function startsAlike(a, b) {
+  const n = Math.min(MATCH_CHARS, a.length, b.length);
+  return n >= MIN_LEAD_CHARS && a.slice(0, n) === b.slice(0, n);
+}
+
+/**
+ * Many sites state their summary as both the meta description and the first
+ * paragraph. Extractors can drop that paragraph when it sits in a section
+ * without a heading (MDN). If the description matches a paragraph in the page
+ * that the extracted HTML lacks, put that paragraph first.
+ *
+ * @param {string} html extracted HTML
+ * @param {Document} doc standardized clone
+ * @param {string} description meta description
+ * @returns {{ html: string, recovered: boolean }}
+ */
+export function recoverLeadParagraph(html, doc, description) {
+  const lead = norm(description);
+  if (lead.length < MIN_LEAD_CHARS) return { html, recovered: false };
+  const extracted = withoutSpaces(norm(htmlText(html)));
+  const paragraphs = Array.from(doc.querySelectorAll?.("p") || []).slice(0, 300);
+  for (const p of paragraphs) {
+    const text = norm(p.textContent);
+    if (text.length < MIN_LEAD_CHARS || !startsAlike(text, lead)) continue;
+    if (extracted.includes(withoutSpaces(text.slice(0, 80)))) return { html, recovered: false };
+    return { html: `${p.outerHTML}\n${html}`, recovered: true };
+  }
+  return { html, recovered: false };
+}
+
+function cleanHeading(text) {
+  return String(text || "")
+    .replace(/\s+/g, " ")
+    .replace(/\s*[¶#§]\s*$/, "")
+    .trim();
+}
+
+function firstHeading(doc) {
+  const scoped = doc.querySelector?.("main h1, article h1, [role='main'] h1");
+  return cleanHeading((scoped || doc.querySelector?.("h1"))?.textContent);
+}
+
+// Drops site-name segments at either end: "Page | Site", "Site - Page".
+function stripSiteSuffix(title, siteName) {
+  let result = String(title || "").trim();
+  const site = norm(siteName);
+  if (!site) return result;
+  const isSite = (segment) => {
+    const value = norm(segment);
+    return Boolean(value) && (value.includes(site) || site.includes(value));
+  };
+  for (let changed = true; changed;) {
+    changed = false;
+    for (const separator of SEPARATORS) {
+      const last = result.lastIndexOf(separator);
+      if (last > 0 && isSite(result.slice(last + separator.length))) {
+        result = result.slice(0, last).trim();
+        changed = true;
+      }
+      const first = result.indexOf(separator);
+      if (first > 0 && isSite(result.slice(0, first))) {
+        result = result.slice(first + separator.length).trim();
+        changed = true;
+      }
+    }
+  }
+  return result;
+}
+
+/**
+ * The heading for the capture: the page's own H1 when the document title or
+ * og:title contains it (and it is not just the site name); otherwise og:title;
+ * otherwise the title without a trailing site-name segment.
+ *
+ * @param {Document} doc standardized clone
+ * @param {{ documentTitle?: string, title?: string, siteName?: string, extractedTitle?: string }} [meta]
+ */
+export function chooseHeadline(doc, meta = {}) {
+  const h1 = firstHeading(doc);
+  const titles = [meta.documentTitle, meta.title, meta.extractedTitle].map(norm).filter(Boolean);
+  if (h1 && h1.length <= 200 && norm(h1) !== norm(meta.siteName) && titles.some((t) => t.includes(norm(h1)))) {
+    return h1;
+  }
+  const og = String(meta.title || "").trim();
+  if (og && og !== String(meta.documentTitle || "").trim()) return stripSiteSuffix(og, meta.siteName);
+  return stripSiteSuffix(meta.extractedTitle || meta.documentTitle || og, meta.siteName);
+}