page-markdown 0.1.3 → 0.1.5

package/src/extract.js

@@ -1,70 +1,76 @@
-// Stage 3: pick the readable content. Default: Mozilla Readability on a copy
-// of the standardized clone, falling back to the whole body when it finds
-// nothing or too little.
-// The class only: skips isProbablyReaderable, which the pipeline does not use.
-import Readability from "@mozilla/readability/Readability.js";
-
-export const DEFAULT_MIN_EXTRACT_CHARS = 100;
-
-function textLength(html = "") {
-  return String(html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().length;
-}
-
-function bodyHtml(doc) {
-  return (doc.body || doc.documentElement)?.innerHTML || "";
-}
-
-function copyDocument(doc) {
-  const copy = doc.cloneNode(true);
-  if (copy && copy.nodeType === 9 && copy.documentElement) return copy;
-  // Implementations whose Document.cloneNode is shallow: reparse.
-  const Parser = doc.defaultView?.DOMParser || globalThis.DOMParser;
-  return new Parser().parseFromString(doc.documentElement.outerHTML, "text/html");
-}
-
-/**
- * @typedef {{ html: string, method: string, fallbackReason?: string, title?: string,
- *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string }} Extraction
- */
-
-/**
- * @param {Document} doc standardized inert document (not mutated)
- * @param {{ extract?: "readable"|"full"|((doc: Document) => Extraction|null),
- *   minExtractChars?: number, readability?: object }} [options]
- * @returns {Extraction}
- */
-export function extractContent(doc, options = {}) {
-  const mode = options.extract ?? "readable";
-  const full = () => ({ html: bodyHtml(doc), method: "full_page" });
-
-  if (mode === "full") return full();
-
-  const minChars = options.minExtractChars ?? DEFAULT_MIN_EXTRACT_CHARS;
-  let result = null;
-  let failure = "";
-  try {
-    if (typeof mode === "function") {
-      result = mode(copyDocument(doc));
-    } else {
-      const article = new Readability(copyDocument(doc), { keepClasses: true, ...options.readability }).parse();
-      result = article && {
-        html: article.content || "",
-        method: "readability",
-        title: article.title || "",
-        byline: article.byline || "",
-        published: article.publishedTime || "",
-        siteName: article.siteName || "",
-        lang: article.lang || "",
-        excerpt: article.excerpt || "",
-      };
-    }
-  } catch (error) {
-    failure = `extractor_error: ${error?.message || error}`;
-  }
-
-  if (!result?.html) return { ...full(), fallbackReason: failure || "extractor_found_nothing" };
-  if (textLength(result.html) < minChars) {
-    return { ...full(), fallbackReason: "extracted_text_below_minimum", title: result.title };
-  }
-  return result;
-}
+// Stage 3: pick the readable content. Default: Mozilla Readability on a copy
+// of the standardized clone, falling back to the whole body when it finds
+// nothing or too little.
+// The class only: skips isProbablyReaderable, which the pipeline does not use.
+import Readability from "@mozilla/readability/Readability.js";
+
+export const DEFAULT_MIN_EXTRACT_CHARS = 100;
+
+function textLength(html = "") {
+  return String(html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().length;
+}
+
+function bodyHtml(doc) {
+  return (doc.body || doc.documentElement)?.innerHTML || "";
+}
+
+function copyDocument(doc) {
+  const copy = doc.cloneNode(true);
+  if (copy && copy.nodeType === 9 && copy.documentElement) return copy;
+  // Implementations whose Document.cloneNode is shallow: reparse.
+  const Parser = doc.defaultView?.DOMParser || globalThis.DOMParser;
+  return new Parser().parseFromString(doc.documentElement.outerHTML, "text/html");
+}
+
+/**
+ * @typedef {{ html: string, method: "readability"|"full_page"|(string & {}), fallbackReason?: string, title?: string,
+ *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string }} Extraction
+ */
+
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ * @typedef {(doc: Document) => Extraction|null|undefined} Extractor
+ */
+
+/**
+ * @param {Document} doc standardized inert document (not mutated)
+ * @param {{ extract?: "readable"|"full"|Extractor,
+ *   minExtractChars?: number, readability?: import("./types.js").ReadabilityOptions }} [options]
+ * @returns {Extraction}
+ */
+export function extractContent(doc, options = {}) {
+  const mode = options.extract ?? "readable";
+  const full = () => ({ html: bodyHtml(doc), method: "full_page" });
+
+  if (mode === "full") return full();
+
+  const minChars = options.minExtractChars ?? DEFAULT_MIN_EXTRACT_CHARS;
+  let result = null;
+  let failure = "";
+  try {
+    if (typeof mode === "function") {
+      result = mode(copyDocument(doc));
+    } else {
+      const article = new Readability(copyDocument(doc), { keepClasses: true, ...options.readability }).parse();
+      result = article && {
+        html: article.content || "",
+        method: "readability",
+        title: article.title || "",
+        byline: article.byline || "",
+        published: article.publishedTime || "",
+        siteName: article.siteName || "",
+        lang: article.lang || "",
+        excerpt: article.excerpt || "",
+      };
+    }
+  } catch (error) {
+    failure = `extractor_error: ${/** @type {any} */ (error)?.message || error}`;
+  }
+
+  if (!result?.html) return { ...full(), fallbackReason: failure || "extractor_found_nothing" };
+  if (textLength(result.html) < minChars) {
+    return { ...full(), fallbackReason: "extracted_text_below_minimum", title: result.title };
+  }
+  return result;
+}

package/src/facts.js

@@ -1,96 +1,99 @@
-// Short label/value facts the extractor left out: package sidebars (version,
-// license, downloads), price and spec boxes, definition lists, two-column
-// tables. Readability keeps the article and drops these asides, which is right
-// for reading and wrong when the question is "which version?". Opt-in.
-
-const MAX_FACTS = 24;
-const MAX_LABEL_CHARS = 40;
-const MAX_VALUE_CHARS = 100;
-const CHROME = "nav, footer, header, form, [role='navigation'], [role='contentinfo'], [role='banner'], [role='search']";
-
-function squash(text) {
-  return String(text || "").replace(/\s+/g, " ").trim();
-}
-
-// A sentence is prose, not a fact: "Returns a signal that aborts later."
-function isSentence(text) {
-  return /[.!?]$/.test(text) && text.split(" ").length >= 4;
-}
-
-function shortValue(el) {
-  if (!el || el.querySelector("h1, h2, h3, h4, h5, h6, table, pre")) return "";
-  // A list of links is navigation, not a fact.
-  const links = el.tagName === "A" ? [el] : el.querySelectorAll("a");
-  if (links.length > 1) return "";
-  const text = squash(el.textContent);
-  // So is a lone link, unless its text is data: a domain, path or number.
-  if (links.length === 1 && squash(links[0].textContent) === text && !/[\d./@]/.test(text)) return "";
-  return text.length <= MAX_VALUE_CHARS && !isSentence(text) ? text : "";
-}
-
-function* candidatePairs(doc) {
-  // Heading followed by a short value: <h3>Version</h3><p>1.2.3</p>.
-  for (const h of doc.querySelectorAll("h2, h3, h4, h5, h6")) {
-    yield [h.textContent, shortValue(h.nextElementSibling), h];
-  }
-  for (const dt of doc.querySelectorAll("dt")) {
-    const dd = dt.nextElementSibling;
-    if (dd?.tagName === "DD") yield [dt.textContent, shortValue(dd), dt];
-  }
-  // Two-cell rows: | Label | Value |.
-  for (const row of doc.querySelectorAll("tr")) {
-    const cells = Array.from(row.children).filter((c) => c.tagName === "TH" || c.tagName === "TD");
-    if (cells.length === 2) yield [cells[0].textContent, shortValue(cells[1]), row];
-  }
-}
-
-// "1.0.1 December 17, 2004 [5]" and "1.0.1December 17, 2004" compare equal.
-function compact(text) {
-  return text.replace(/\[\d+\]/g, "").replace(/\s+/g, "");
-}
-
-function escapeRegExp(text) {
-  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-/**
- * @param {Document} doc standardized clone
- * @param {string} articleHtml the extracted article
- * @returns {Array<{ label: string, value: string }>} facts absent from the article
- */
-export function collectPageFacts(doc, articleHtml) {
-  const article = squash(String(articleHtml || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ")
-    .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&amp;/g, "&"));
-  const articleCompact = compact(article);
-  const pairs = Array.from(candidatePairs(doc), ([label, value, el]) => [squash(label).replace(/\s*[:：¶#§]\s*$/, ""), value, el]);
-  // A "value" that is the next section's label is a heading run, not a fact.
-  const labels = new Set(pairs.map(([label]) => label));
-  const facts = [];
-  const seen = new Set();
-  for (const [label, value, el] of pairs) {
-    if (!label || !value || label.length > MAX_LABEL_CHARS || label === value || labels.has(value)) continue;
-    if (el.closest(CHROME)) continue;
-    const key = `${label}\u0000${value}`;
-    if (seen.has(key)) continue;
-    seen.add(key);
-    // Already in the article: short values as a whole word, longer ones
-    // ignoring spacing and citation marks.
-    const bare = compact(value);
-    if (bare.length >= 6 ? articleCompact.includes(bare)
-      : new RegExp(`(^|[^\\w])${escapeRegExp(value)}($|[^\\w])`).test(article)) continue;
-    facts.push({ label, value });
-    if (facts.length >= MAX_FACTS) break;
-  }
-  return facts;
-}
-
-function escapeMarkdown(text) {
-  return text.replace(/([\\`*_[\]<>|])/g, "\\$1");
-}
-
-/** "## Page facts" list, or "" when there are none. */
-export function pageFactsMarkdown(facts) {
-  if (!facts.length) return "";
-  const lines = facts.map(({ label, value }) => `- ${escapeMarkdown(label)}: ${escapeMarkdown(value)}`);
-  return `## Page facts\n\n${lines.join("\n")}`;
-}
+// Short label/value facts the extractor left out: package sidebars (version,
+// license, downloads), price and spec boxes, definition lists, two-column
+// tables. Readability keeps the article and drops these asides, which is right
+// for reading and wrong when the question is "which version?". Opt-in.
+import { htmlText } from "./refine.js";
+
+const MAX_FACTS = 24;
+const MAX_LABEL_CHARS = 40;
+const MAX_VALUE_CHARS = 100;
+const CHROME = "nav, footer, header, form, [role='navigation'], [role='contentinfo'], [role='banner'], [role='search']";
+
+function squash(text) {
+  return String(text || "").replace(/\s+/g, " ").trim();
+}
+
+// A sentence is prose, not a fact: "Returns a signal that aborts later."
+function isSentence(text) {
+  return /[.!?]$/.test(text) && text.split(" ").length >= 4;
+}
+
+function shortValue(el) {
+  if (!el || el.querySelector("h1, h2, h3, h4, h5, h6, table, pre")) return "";
+  // A list of links is navigation, not a fact.
+  const links = el.tagName === "A" ? [el] : el.querySelectorAll("a");
+  if (links.length > 1) return "";
+  const text = squash(el.textContent);
+  // So is a lone link, unless its text is data: a domain, path or number.
+  if (links.length === 1 && squash(links[0].textContent) === text && !/[\d./@]/.test(text)) return "";
+  return text.length <= MAX_VALUE_CHARS && !isSentence(text) ? text : "";
+}
+
+function* candidatePairs(doc) {
+  // Heading followed by a short value: <h3>Version</h3><p>1.2.3</p>.
+  for (const h of doc.querySelectorAll("h2, h3, h4, h5, h6")) {
+    yield [h.textContent, shortValue(h.nextElementSibling), h];
+  }
+  for (const dt of doc.querySelectorAll("dt")) {
+    const dd = dt.nextElementSibling;
+    if (dd?.tagName === "DD") yield [dt.textContent, shortValue(dd), dt];
+  }
+  // Two-cell rows: | Label | Value |.
+  for (const row of doc.querySelectorAll("tr")) {
+    const cells = Array.from(row.children).filter((c) => c.tagName === "TH" || c.tagName === "TD");
+    if (cells.length === 2) yield [cells[0].textContent, shortValue(cells[1]), row];
+  }
+}
+
+// "1.0.1 December 17, 2004 [5]" and "1.0.1December 17, 2004" compare equal.
+function compact(text) {
+  return text.replace(/\[\d+\]/g, "").replace(/\s+/g, "");
+}
+
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * @param {Document} doc standardized clone
+ * @param {string} articleHtml the extracted article
+ * @returns {Array<{ label: string, value: string }>} facts absent from the article
+ */
+export function collectPageFacts(doc, articleHtml) {
+  const article = squash(htmlText(articleHtml));
+  const articleCompact = compact(article);
+  const pairs = Array.from(candidatePairs(doc), ([label, value, el]) => [squash(label).replace(/\s*[:：¶#§]\s*$/, ""), value, el]);
+  // A "value" that is the next section's label is a heading run, not a fact.
+  const labels = new Set(pairs.map(([label]) => label));
+  const facts = [];
+  const seen = new Set();
+  for (const [label, value, el] of pairs) {
+    if (!label || !value || label.length > MAX_LABEL_CHARS || label === value || labels.has(value)) continue;
+    if (el.closest(CHROME)) continue;
+    const key = `${label}\u0000${value}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    // Already in the article: short values as a whole word, longer ones
+    // ignoring spacing and citation marks.
+    const bare = compact(value);
+    if (bare.length >= 6 ? articleCompact.includes(bare)
+      : new RegExp(`(^|[^\\w])${escapeRegExp(value)}($|[^\\w])`).test(article)) continue;
+    facts.push({ label, value });
+    if (facts.length >= MAX_FACTS) break;
+  }
+  return facts;
+}
+
+function escapeMarkdown(text) {
+  return text.replace(/([\\`*_[\]<>|])/g, "\\$1");
+}
+
+/**
+ * "## Page facts" list, or "" when there are none.
+ * @param {readonly import("./types.js").PageFact[]} facts
+ */
+export function pageFactsMarkdown(facts) {
+  if (!facts.length) return "";
+  const lines = facts.map(({ label, value }) => `- ${escapeMarkdown(label)}: ${escapeMarkdown(value)}`);
+  return `## Page facts\n\n${lines.join("\n")}`;
+}

package/src/hidden.js

@@ -1,44 +1,46 @@
-// Optional last stage: remove characters that hide text from people but not
-// from models (tag characters, variation-selector runs, invisible operators),
-// found by text-sanctifier's run finder. Legitimate sequences are kept: the
-// England, Scotland and Wales flags and a lone emoji or text presentation
-// selector. Unlike the clean stage, this also applies inside fenced code.
-import { findHiddenRuns } from "text-sanctifier/hidden";
-
-const NOTES = {
-  tag: "Hidden tag-character text removed",
-  variation_selector: "Variation selector runs removed",
-  invisible_operator: "Invisible operator runs removed",
-};
-
-/**
- * @param {string} text
- * @returns {{ text: string, runs: Array<{ kind: string, count: number }> }}
- */
-export function removeHiddenText(text) {
-  const source = String(text ?? "");
-  const { runs } = findHiddenRuns(source);
-  if (!runs.length) return { text: source, runs };
-  let out = "";
-  let at = 0;
-  for (const run of runs) {
-    out += source.slice(at, run.offset);
-    at = run.end;
-  }
-  return { text: out + source.slice(at), runs };
-}
-
-/**
- * One omission per kind of run: `count` runs, `chars` code points.
- * @param {Array<{ kind: string, count: number }>} runs
- */
-export function hiddenTextOmissions(runs) {
-  const byKind = new Map();
-  for (const run of runs) {
-    const entry = byKind.get(run.kind) || { kind: `hidden_${run.kind}`, count: 0, chars: 0, note: NOTES[run.kind] };
-    entry.count += 1;
-    entry.chars += run.count;
-    byKind.set(run.kind, entry);
-  }
-  return Array.from(byKind.values());
-}
+// Optional last stage: remove characters that hide text from people but not
+// from models (tag characters, variation-selector runs, invisible operators,
+// reserved tag-plane code points), found by text-sanctifier's run finder. Legitimate sequences are kept: the
+// England, Scotland and Wales flags and a lone emoji or text presentation
+// selector. Unlike the clean stage, this also applies inside fenced code.
+import { findHiddenRuns } from "text-sanctifier/hidden";
+
+const NOTES = {
+  tag: "Hidden tag-character text removed",
+  variation_selector: "Variation selector runs removed",
+  invisible_operator: "Invisible operator runs removed",
+  reserved_tag_plane: "Reserved tag-plane runs removed",
+};
+
+/**
+ * @param {string} text
+ * @returns {{ text: string, runs: import("./types.js").HiddenRun[] }}
+ */
+export function removeHiddenText(text) {
+  const source = String(text ?? "");
+  const { runs } = findHiddenRuns(source);
+  if (!runs.length) return { text: source, runs };
+  let out = "";
+  let at = 0;
+  for (const run of runs) {
+    out += source.slice(at, run.offset);
+    at = run.end;
+  }
+  return { text: out + source.slice(at), runs };
+}
+
+/**
+ * One omission per kind of run: `count` runs, `chars` code points.
+ * @param {ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind"|"count">>} runs
+ * @returns {import("./types.js").HiddenTextOmission[]}
+ */
+export function hiddenTextOmissions(runs) {
+  const byKind = new Map();
+  for (const run of runs) {
+    const entry = byKind.get(run.kind) || { kind: `hidden_${run.kind}`, count: 0, chars: 0, note: NOTES[run.kind] };
+    entry.count += 1;
+    entry.chars += run.count;
+    byKind.set(run.kind, entry);
+  }
+  return Array.from(byKind.values());
+}

package/src/index.js

@@ -10,7 +10,7 @@ import { recoverLeadParagraph, chooseHeadline } from "./refine.js";
 import { collectPageFacts, pageFactsMarkdown } from "./facts.js";
 import { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
 
-export const VERSION = "0.1.3";
+export const VERSION = "0.1.5";
 
 export { sanitizeToInertDocument, DEFAULT_SANITIZE_OPTIONS, isAdvertisementElement, createInertDocument } from "./sanitize.js";
 export { standardizeDocument } from "./standardize.js";
@@ -21,6 +21,7 @@ export { collectMetadata } from "./metadata.js";
 export { recoverLeadParagraph, chooseHeadline } from "./refine.js";
 export { collectPageFacts, pageFactsMarkdown } from "./facts.js";
 export { removeHiddenText, hiddenTextOmissions } from "./hidden.js";
+export * from "./types.js";
 
 function documentOf(input) {
   if (input?.nodeType === 9) return input;
@@ -34,6 +35,16 @@ function toMarkdown(html, mode) {
   return { markdown: applyMode(markdown, mode), removedJsonLines, convertedChars: converted.length };
 }
 
+// What relative links resolve against: the page's <base href>, else its URL.
+function baseUrl(doc, url) {
+  const href = doc.querySelector?.("base[href]")?.getAttribute("href");
+  try {
+    return href ? new URL(href, url || undefined).href : url;
+  } catch {
+    return url;
+  }
+}
+
 // A top-level heading in the prose; a "# comment" line in fenced code is not one.
 function hasTopHeading(markdown) {
   let found = false;
@@ -48,7 +59,7 @@ function hasTopHeading(markdown) {
  * @typedef {object} PageMarkdownOptions
  * @property {string} [url] page URL for absolute links (default: the document's URL)
  * @property {"full"|"no_links"|"text_only"} [mode="full"]
- * @property {"readable"|"full"|((doc: Document) => object|null)} [extract="readable"]
+ * @property {"readable"|"full"|import("./extract.js").Extractor} [extract="readable"]
  *   "readable": Readability with full-page fallback; "full": the whole body;
  *   or a custom extractor receiving a copy of the standardized document.
  * @property {number} [minExtractChars=100] below this, fall back to the full page
@@ -61,8 +72,8 @@ function hasTopHeading(markdown) {
  *   variation-selector runs and invisible operators from the Markdown (fenced
  *   code included), fullPageMarkdown and metadata; omissions count the runs
  *   removed from the Markdown
- * @property {object} [sanitize] see DEFAULT_SANITIZE_OPTIONS
- * @property {object} [readability] options passed to Readability
+ * @property {import("./types.js").SanitizeOptions} [sanitize] see DEFAULT_SANITIZE_OPTIONS
+ * @property {import("./types.js").ReadabilityOptions} [readability] options passed to Readability
  */
 
 /**
@@ -71,15 +82,17 @@ function hasTopHeading(markdown) {
  *
  * @param {Document|Element} input
  * @param {PageMarkdownOptions} [options]
+ * @returns {import("./types.js").PageMarkdownResult}
  */
 export function pageToMarkdown(input, options = {}) {
   const doc = documentOf(input);
   const url = options.url || doc.URL || doc.location?.href || "";
   const mode = options.mode || "full";
+  /** @type {string[]} */
   const warnings = [];
 
   const { document: inert, omissions } = sanitizeToInertDocument(input, options.sanitize);
-  const standardization = standardizeDocument(inert, { url });
+  const standardization = standardizeDocument(inert, { url: baseUrl(doc, url) });
   const extraction = extractContent(inert, {
     extract: options.extract,
     minExtractChars: options.minExtractChars,
@@ -106,6 +119,7 @@ export function pageToMarkdown(input, options = {}) {
     pageFacts = collectPageFacts(inert, extractedHtml);
     if (pageFacts.length) main.markdown = `${main.markdown}\n\n${pageFactsMarkdown(pageFacts)}`;
   }
+  /** @type {import("./types.js").PageMarkdownOmission[]} */
   const omissionList = [];
   if (omissions.shadowHosts) {
     omissionList.push({ kind: "shadow_content", count: omissions.shadowHosts, note: "Open shadow content omitted: capture limit" });
@@ -114,6 +128,7 @@ export function pageToMarkdown(input, options = {}) {
     omissionList.push({ kind: "json_line", count: main.removedJsonLines, note: "Large single-line JSON documents removed" });
   }
 
+  /** @type {import("./types.js").PageMarkdownResult} */
   const result = {
     version: VERSION,
     markdown: main.markdown,
@@ -158,5 +173,6 @@ export function pageToMarkdown(input, options = {}) {
   if (omissions.shadowHosts) {
     result.markdown += `\n\n_Open shadow content omitted: capture limit (${omissions.shadowHosts} host(s))._`;
   }
+  result.stats.markdownChars = result.markdown.length;
   return result;
 }