page-markdown 0.1.4 → 0.2.0

package/src/extract.js

@@ -1,70 +1,211 @@
-// Stage 3: pick the readable content. Default: Mozilla Readability on a copy
-// of the standardized clone, falling back to the whole body when it finds
-// nothing or too little.
-// The class only: skips isProbablyReaderable, which the pipeline does not use.
-import Readability from "@mozilla/readability/Readability.js";
-
-export const DEFAULT_MIN_EXTRACT_CHARS = 100;
-
-function textLength(html = "") {
-  return String(html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().length;
-}
-
-function bodyHtml(doc) {
-  return (doc.body || doc.documentElement)?.innerHTML || "";
-}
-
-function copyDocument(doc) {
-  const copy = doc.cloneNode(true);
-  if (copy && copy.nodeType === 9 && copy.documentElement) return copy;
-  // Implementations whose Document.cloneNode is shallow: reparse.
-  const Parser = doc.defaultView?.DOMParser || globalThis.DOMParser;
-  return new Parser().parseFromString(doc.documentElement.outerHTML, "text/html");
-}
-
-/**
- * @typedef {{ html: string, method: string, fallbackReason?: string, title?: string,
- *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string }} Extraction
- */
-
-/**
- * @param {Document} doc standardized inert document (not mutated)
- * @param {{ extract?: "readable"|"full"|((doc: Document) => Extraction|null),
- *   minExtractChars?: number, readability?: object }} [options]
- * @returns {Extraction}
- */
-export function extractContent(doc, options = {}) {
-  const mode = options.extract ?? "readable";
-  const full = () => ({ html: bodyHtml(doc), method: "full_page" });
-
-  if (mode === "full") return full();
-
-  const minChars = options.minExtractChars ?? DEFAULT_MIN_EXTRACT_CHARS;
-  let result = null;
-  let failure = "";
-  try {
-    if (typeof mode === "function") {
-      result = mode(copyDocument(doc));
-    } else {
-      const article = new Readability(copyDocument(doc), { keepClasses: true, ...options.readability }).parse();
-      result = article && {
-        html: article.content || "",
-        method: "readability",
-        title: article.title || "",
-        byline: article.byline || "",
-        published: article.publishedTime || "",
-        siteName: article.siteName || "",
-        lang: article.lang || "",
-        excerpt: article.excerpt || "",
-      };
-    }
-  } catch (error) {
-    failure = `extractor_error: ${error?.message || error}`;
-  }
-
-  if (!result?.html) return { ...full(), fallbackReason: failure || "extractor_found_nothing" };
-  if (textLength(result.html) < minChars) {
-    return { ...full(), fallbackReason: "extracted_text_below_minimum", title: result.title };
-  }
-  return result;
-}
+// Stage 3: pick the readable content. Default: Mozilla Readability on a copy
+// of the standardized clone, falling back to the whole body when it finds
+// nothing or too little, and to the page's main content when it keeps only a
+// small share of that (forum threads, product grids), brings the site's menu
+// along, or keeps too few of its sections (product pages).
+// The class only: skips isProbablyReaderable, which the pipeline does not use.
+import Readability from "@mozilla/readability/Readability.js";
+import { htmlText } from "./refine.js";
+
+export const DEFAULT_MIN_EXTRACT_CHARS = 100;
+export const DEFAULT_MIN_EXTRACT_SHARE = 0.4;
+// A small share of a short page may be all there is: don't fall back for less.
+const MIN_MISSED_CHARS = 500;
+export const PAGE_CHROME = "nav, footer, form, [role='navigation'], [role='banner'], [role='contentinfo'], [role='search']";
+const MAIN = "main, [role='main']";
+const COMMENTS = "#comments, #disqus_thread, .comments, .commentlist, .comment-list, .comment, [id^='comment-']";
+const HIDDEN_STYLE = /(?:^|;)\s*(?:display\s*:\s*none|visibility\s*:\s*hidden)/i;
+const TOGGLE = "button, summary, [aria-expanded]";
+// Shorter hidden text beside a button is a status message ("Added"), not an answer.
+const MIN_ANSWER_CHARS = 40;
+const MENU_SAMPLE_CHARS = 30;
+// Fewer h2s than this, and a page has no sections to lose.
+const MIN_SECTIONS = 3;
+const SECTION_SHARE_FACTOR = 1.5;
+
+// As far as a clone can tell, with no stylesheets. Not aria-hidden: shops put
+// the price people see in it, beside a longer one for screen readers.
+export function isHiddenElement(el) {
+  return el.hasAttribute("hidden") || HIDDEN_STYLE.test(el.getAttribute("style") || "");
+}
+
+function textLength(html = "") {
+  return String(html).replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim().length;
+}
+
+function bodyHtml(doc) {
+  return (doc.body || doc.documentElement)?.innerHTML || "";
+}
+
+// An accordion's answer, hidden until its button is pressed: FAQs on shop pages.
+function isCollapsedAnswer(el) {
+  return Boolean(el.previousElementSibling?.matches(TOGGLE)) && (el.textContent || "").trim().length >= MIN_ANSWER_CHARS;
+}
+
+// <main> or else the body, without navigation, other page chrome or anything
+// hidden, except code behind a "Show code" toggle and collapsed answers. A header
+// inside <main> usually belongs to the content; one in the body doesn't.
+function mainContentRoot(doc) {
+  const main = doc.querySelector(MAIN);
+  const root = (main || doc.body || doc.documentElement).cloneNode(true);
+  for (const el of Array.from(root.querySelectorAll(`${PAGE_CHROME}, aside, template, [role='complementary']${main ? "" : ", header"}`))) el.remove();
+  for (const el of Array.from(root.querySelectorAll("[hidden], [style]"))) {
+    if (isHiddenElement(el) && !el.querySelector("pre") && !isCollapsedAnswer(el)) el.remove();
+  }
+  for (const link of Array.from(root.querySelectorAll("a[href^='#']"))) {
+    if (/^\s*skip\b/i.test(link.textContent || "")) link.remove();
+  }
+  return root;
+}
+
+const withoutSpaces = (text) => text.replace(/\s+/g, "");
+
+// An article above its comments, where Readability kept the article: what's left
+// without them. null for a thread, where what Readability kept is itself a comment.
+function articleWithoutComments(root, articleHtml) {
+  const comments = Array.from(root.querySelectorAll(COMMENTS));
+  if (!comments.length) return null;
+  const kept = withoutSpaces(htmlText(articleHtml));
+  const sample = kept.slice(kept.length >> 1, (kept.length >> 1) + 60);
+  if (comments.some((el) => withoutSpaces(el.textContent || "").includes(sample))) return null;
+  for (const el of comments) el.remove();
+  return root;
+}
+
+// Readability chose a container above <main>, and the site's menu came with it.
+function keptSiteMenu(doc, kept) {
+  const main = doc.querySelector(MAIN);
+  if (!main) return false;
+  const text = withoutSpaces(main.textContent || "");
+  if (!kept.includes(text.slice(text.length >> 1, (text.length >> 1) + MENU_SAMPLE_CHARS))) return false;
+  return Array.from(doc.querySelectorAll("nav, [role='navigation']")).some((nav) => {
+    const menu = withoutSpaces(nav.textContent || "").slice(0, MENU_SAMPLE_CHARS);
+    return !main.contains(nav) && menu.length === MENU_SAMPLE_CHARS && kept.includes(menu);
+  });
+}
+
+// A heading that is one link names another page, not a section of this one.
+function isStoryCardTitle(h2) {
+  const text = withoutSpaces(h2.textContent || "");
+  return Boolean(h2.closest("a")) || Array.from(h2.querySelectorAll("a:not([href^='#'])")).some((a) => withoutSpaces(a.textContent || "") === text);
+}
+
+function lostMostSections(root, kept) {
+  const headings = Array.from(root.querySelectorAll("h2")).filter((h2) => !isStoryCardTitle(h2)).map((h2) => withoutSpaces(h2.textContent || "")).filter(Boolean);
+  return headings.length >= MIN_SECTIONS && headings.filter((heading) => !kept.includes(heading)).length * 2 > headings.length;
+}
+
+// Layout utilities, which say nothing about content: Tailwind's scroll-mt-2 and
+// overflow-hidden, md:hidden.
+const LAYOUT_CLASS = /^(?:\S+:)?(?:-?scroll-[mp][a-z]?-\S+|overflow(?:-[xy])?-(?:hidden|scroll)|(?<=:)hidden)$/;
+
+// Readability scores class and id names, and removes what looks like clutter:
+// Docusaurus's heading "anchorTargetHideOnScrollNavbar" matches "scroll", and so
+// does the scroll-mt-2 on every code block of Docker's docs.
+function withoutMisleadingNames(doc) {
+  for (const heading of Array.from(doc.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
+    heading.removeAttribute("class");
+    heading.removeAttribute("id");
+  }
+  for (const el of Array.from(doc.querySelectorAll("[class]"))) {
+    const names = String(el.getAttribute("class")).split(/\s+/);
+    const kept = names.filter((name) => !LAYOUT_CLASS.test(name));
+    if (kept.length < names.length) el.setAttribute("class", kept.join(" "));
+  }
+  // Readability removes what's aria-hidden, and icons in table cells usually are.
+  for (const icon of Array.from(doc.querySelectorAll("td svg[aria-hidden], th svg[aria-hidden]"))) icon.removeAttribute("aria-hidden");
+  return doc;
+}
+
+function copyDocument(doc) {
+  const copy = doc.cloneNode(true);
+  if (copy && copy.nodeType === 9 && copy.documentElement) return copy;
+  // Implementations whose Document.cloneNode is shallow: reparse.
+  const Parser = doc.defaultView?.DOMParser || globalThis.DOMParser;
+  return new Parser().parseFromString(doc.documentElement.outerHTML, "text/html");
+}
+
+/**
+ * @typedef {{ html: string, method: "readability"|"main_content"|"full_page"|(string & {}), fallbackReason?: string, title?: string,
+ *   byline?: string, published?: string, siteName?: string, lang?: string, excerpt?: string, commentsOmitted?: boolean }} Extraction
+ */
+
+/**
+ * Receives a copy of the standardized document. Returning nothing, empty HTML
+ * or too little text falls back to the full page; so does throwing.
+ * @typedef {(doc: Document) => Extraction|null|undefined} Extractor
+ */
+
+/**
+ * @param {Document} doc standardized inert document (not mutated)
+ * @param {{ extract?: "readable"|"full"|Extractor,
+ *   minExtractChars?: number, minExtractShare?: number, readability?: import("./types.js").ReadabilityOptions }} [options]
+ * @returns {Extraction}
+ */
+export function extractContent(doc, options = {}) {
+  const mode = options.extract ?? "readable";
+  const full = () => ({ html: bodyHtml(doc), method: "full_page" });
+
+  if (mode === "full") return full();
+
+  const minChars = options.minExtractChars ?? DEFAULT_MIN_EXTRACT_CHARS;
+  let result = null;
+  let failure = "";
+  try {
+    if (typeof mode === "function") {
+      result = mode(copyDocument(doc));
+    } else {
+      const article = new Readability(withoutMisleadingNames(copyDocument(doc)), { keepClasses: true, ...options.readability }).parse();
+      result = article && {
+        html: article.content || "",
+        method: "readability",
+        title: article.title || "",
+        byline: article.byline || "",
+        published: article.publishedTime || "",
+        siteName: article.siteName || "",
+        lang: article.lang || "",
+        excerpt: article.excerpt || "",
+      };
+    }
+  } catch (error) {
+    failure = `extractor_error: ${/** @type {any} */ (error)?.message || error}`;
+  }
+
+  // Too little from the default extractor: the page's own <main>, if it marks
+  // one and it holds enough, beats a full page that opens with site navigation.
+  const tooLittle = (fallbackReason, title) => {
+    if (typeof mode !== "function" && doc.querySelector(MAIN)) {
+      const html = mainContentRoot(doc).innerHTML || "";
+      if (textLength(html) >= minChars) return { html, method: "main_content", fallbackReason, title };
+    }
+    return { ...full(), fallbackReason, ...(title === undefined ? {} : { title }) };
+  };
+
+  if (!result?.html) return tooLittle(failure || "extractor_found_nothing");
+  if (textLength(result.html) < minChars) return tooLittle("extracted_text_below_minimum", result.title);
+  if (result.method === "readability") {
+    const share = options.minExtractShare ?? DEFAULT_MIN_EXTRACT_SHARE;
+    const kept = textLength(result.html);
+    const isSmall = (html) => {
+      const available = textLength(html);
+      return kept < available * share && available - kept >= MIN_MISSED_CHARS;
+    };
+    const keptText = withoutSpaces(htmlText(result.html));
+    // Section loss counts only where it takes more of the page than a related-links block.
+    const misses = (el) => {
+      const available = el.innerHTML || "";
+      return isSmall(available) || (kept < textLength(available) * share * SECTION_SHARE_FACTOR && lostMostSections(el, keptText));
+    };
+    const root = mainContentRoot(doc);
+    const html = root.innerHTML || "";
+    const reason = keptSiteMenu(doc, keptText) && textLength(html) >= minChars ? "extracted_site_menu"
+      : isSmall(html) ? "extracted_small_share_of_page"
+      : misses(root) ? "extracted_few_sections_of_page" : "";
+    if (reason) {
+      const article = reason === "extracted_site_menu" ? null : articleWithoutComments(root, result.html);
+      if (article && !misses(article)) return { ...result, commentsOmitted: true };
+      return { ...result, html, method: "main_content", fallbackReason: reason };
+    }
+  }
+  return result;
+}

package/src/facts.js

@@ -1,96 +1,102 @@
-// Short label/value facts the extractor left out: package sidebars (version,
-// license, downloads), price and spec boxes, definition lists, two-column
-// tables. Readability keeps the article and drops these asides, which is right
-// for reading and wrong when the question is "which version?". Opt-in.
-
-const MAX_FACTS = 24;
-const MAX_LABEL_CHARS = 40;
-const MAX_VALUE_CHARS = 100;
-const CHROME = "nav, footer, header, form, [role='navigation'], [role='contentinfo'], [role='banner'], [role='search']";
-
-function squash(text) {
-  return String(text || "").replace(/\s+/g, " ").trim();
-}
-
-// A sentence is prose, not a fact: "Returns a signal that aborts later."
-function isSentence(text) {
-  return /[.!?]$/.test(text) && text.split(" ").length >= 4;
-}
-
-function shortValue(el) {
-  if (!el || el.querySelector("h1, h2, h3, h4, h5, h6, table, pre")) return "";
-  // A list of links is navigation, not a fact.
-  const links = el.tagName === "A" ? [el] : el.querySelectorAll("a");
-  if (links.length > 1) return "";
-  const text = squash(el.textContent);
-  // So is a lone link, unless its text is data: a domain, path or number.
-  if (links.length === 1 && squash(links[0].textContent) === text && !/[\d./@]/.test(text)) return "";
-  return text.length <= MAX_VALUE_CHARS && !isSentence(text) ? text : "";
-}
-
-function* candidatePairs(doc) {
-  // Heading followed by a short value: <h3>Version</h3><p>1.2.3</p>.
-  for (const h of doc.querySelectorAll("h2, h3, h4, h5, h6")) {
-    yield [h.textContent, shortValue(h.nextElementSibling), h];
-  }
-  for (const dt of doc.querySelectorAll("dt")) {
-    const dd = dt.nextElementSibling;
-    if (dd?.tagName === "DD") yield [dt.textContent, shortValue(dd), dt];
-  }
-  // Two-cell rows: | Label | Value |.
-  for (const row of doc.querySelectorAll("tr")) {
-    const cells = Array.from(row.children).filter((c) => c.tagName === "TH" || c.tagName === "TD");
-    if (cells.length === 2) yield [cells[0].textContent, shortValue(cells[1]), row];
-  }
-}
-
-// "1.0.1 December 17, 2004 [5]" and "1.0.1December 17, 2004" compare equal.
-function compact(text) {
-  return text.replace(/\[\d+\]/g, "").replace(/\s+/g, "");
-}
-
-function escapeRegExp(text) {
-  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-/**
- * @param {Document} doc standardized clone
- * @param {string} articleHtml the extracted article
- * @returns {Array<{ label: string, value: string }>} facts absent from the article
- */
-export function collectPageFacts(doc, articleHtml) {
-  const article = squash(String(articleHtml || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ")
-    .replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&amp;/g, "&"));
-  const articleCompact = compact(article);
-  const pairs = Array.from(candidatePairs(doc), ([label, value, el]) => [squash(label).replace(/\s*[:：¶#§]\s*$/, ""), value, el]);
-  // A "value" that is the next section's label is a heading run, not a fact.
-  const labels = new Set(pairs.map(([label]) => label));
-  const facts = [];
-  const seen = new Set();
-  for (const [label, value, el] of pairs) {
-    if (!label || !value || label.length > MAX_LABEL_CHARS || label === value || labels.has(value)) continue;
-    if (el.closest(CHROME)) continue;
-    const key = `${label}\u0000${value}`;
-    if (seen.has(key)) continue;
-    seen.add(key);
-    // Already in the article: short values as a whole word, longer ones
-    // ignoring spacing and citation marks.
-    const bare = compact(value);
-    if (bare.length >= 6 ? articleCompact.includes(bare)
-      : new RegExp(`(^|[^\\w])${escapeRegExp(value)}($|[^\\w])`).test(article)) continue;
-    facts.push({ label, value });
-    if (facts.length >= MAX_FACTS) break;
-  }
-  return facts;
-}
-
-function escapeMarkdown(text) {
-  return text.replace(/([\\`*_[\]<>|])/g, "\\$1");
-}
-
-/** "## Page facts" list, or "" when there are none. */
-export function pageFactsMarkdown(facts) {
-  if (!facts.length) return "";
-  const lines = facts.map(({ label, value }) => `- ${escapeMarkdown(label)}: ${escapeMarkdown(value)}`);
-  return `## Page facts\n\n${lines.join("\n")}`;
-}
+// Short label/value facts the extractor left out: package sidebars (version,
+// license, downloads), price and spec boxes, definition lists, two-column
+// tables. Readability keeps the article and drops these asides, which is right
+// for reading and wrong when the question is "which version?". Opt-in.
+import { PAGE_CHROME } from "./extract.js";
+import { htmlText } from "./refine.js";
+
+const MAX_FACTS = 24;
+const MAX_LABEL_CHARS = 40;
+const MAX_VALUE_CHARS = 100;
+const CHROME = `${PAGE_CHROME}, header, [id*='cookie'], [class*='cookie'], [id*='consent'], [class*='consent'], [id*='onetrust'], [class*='onetrust']`;
+
+function squash(text) {
+  return String(text || "").replace(/\s+/g, " ").trim();
+}
+
+// A sentence is prose, not a fact: "Returns a signal that aborts later."
+function isSentence(text) {
+  return /[.!?]$/.test(text) && text.split(" ").length >= 4;
+}
+
+function shortValue(el) {
+  if (!el || el.querySelector("h1, h2, h3, h4, h5, h6, table, pre, form, button, input, select, textarea")) return "";
+  // A list of links is navigation, not a fact.
+  const links = el.tagName === "A" ? [el] : el.querySelectorAll("a");
+  if (links.length > 1) return "";
+  const text = squash(el.textContent);
+  // So is a lone link, unless its text is data: a domain, path or number.
+  if (links.length === 1 && squash(links[0].textContent) === text && !/[\d./@]/.test(text)) return "";
+  return text.length <= MAX_VALUE_CHARS && !isSentence(text) ? text : "";
+}
+
+function* candidatePairs(doc) {
+  // Heading followed by a short value: <h3>Version</h3><p>1.2.3</p>.
+  for (const h of doc.querySelectorAll("h2, h3, h4, h5, h6")) {
+    // A linked heading titles something else: another story's teaser, a card.
+    if (h.closest("a") || h.querySelector("a")) continue;
+    yield [h.textContent, shortValue(h.nextElementSibling), h];
+  }
+  for (const dt of doc.querySelectorAll("dt")) {
+    const dd = dt.nextElementSibling;
+    if (dd?.tagName === "DD") yield [dt.textContent, shortValue(dd), dt];
+  }
+  // Two-cell rows: | Label | Value |.
+  for (const row of doc.querySelectorAll("tr")) {
+    const cells = Array.from(row.children).filter((c) => c.tagName === "TH" || c.tagName === "TD");
+    if (cells.length === 2) yield [cells[0].textContent, shortValue(cells[1]), row];
+  }
+}
+
+// "1.0.1 December 17, 2004 [5]" and "1.0.1December 17, 2004" compare equal.
+function compact(text) {
+  return text.replace(/\[\d+\]/g, "").replace(/\s+/g, "");
+}
+
+function escapeRegExp(text) {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * @param {Document} doc standardized clone
+ * @param {string} articleHtml the extracted article
+ * @returns {Array<{ label: string, value: string }>} facts absent from the article
+ */
+export function collectPageFacts(doc, articleHtml) {
+  const article = squash(htmlText(articleHtml));
+  const articleCompact = compact(article);
+  const pairs = Array.from(candidatePairs(doc), ([label, value, el]) => [squash(label).replace(/\s*[:：¶#§]\s*$/, ""), value, el]);
+  // A "value" that is the next section's label is a heading run, not a fact.
+  const labels = new Set(pairs.map(([label]) => label));
+  const facts = [];
+  const seen = new Set();
+  for (const [label, value, el] of pairs) {
+    if (!label || !value || label.length > MAX_LABEL_CHARS || label === value || labels.has(value)) continue;
+    if (el.closest(CHROME)) continue;
+    const key = `${label}\u0000${value}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    // Already in the article: short values as a whole word, longer ones
+    // ignoring spacing and citation marks.
+    const bare = compact(value);
+    if (bare.length >= 6 ? articleCompact.includes(bare)
+      : new RegExp(`(^|[^\\w])${escapeRegExp(value)}($|[^\\w])`).test(article)) continue;
+    facts.push({ label, value });
+    if (facts.length >= MAX_FACTS) break;
+  }
+  return facts;
+}
+
+export function escapeMarkdown(text) {
+  return text.replace(/([\\`*_[\]<>|])/g, "\\$1");
+}
+
+/**
+ * "## Page facts" list, or "" when there are none.
+ * @param {readonly import("./types.js").PageFact[]} facts
+ */
+export function pageFactsMarkdown(facts) {
+  if (!facts.length) return "";
+  const lines = facts.map(({ label, value }) => `- ${escapeMarkdown(label)}: ${escapeMarkdown(value)}`);
+  return `## Page facts\n\n${lines.join("\n")}`;
+}

package/src/hidden.js

@@ -1,45 +1,46 @@
-// Optional last stage: remove characters that hide text from people but not
-// from models (tag characters, variation-selector runs, invisible operators,
-// reserved tag-plane code points), found by text-sanctifier's run finder. Legitimate sequences are kept: the
-// England, Scotland and Wales flags and a lone emoji or text presentation
-// selector. Unlike the clean stage, this also applies inside fenced code.
-import { findHiddenRuns } from "text-sanctifier/hidden";
-
-const NOTES = {
-  tag: "Hidden tag-character text removed",
-  variation_selector: "Variation selector runs removed",
-  invisible_operator: "Invisible operator runs removed",
-  reserved_tag_plane: "Reserved tag-plane runs removed",
-};
-
-/**
- * @param {string} text
- * @returns {{ text: string, runs: Array<{ kind: string, count: number }> }}
- */
-export function removeHiddenText(text) {
-  const source = String(text ?? "");
-  const { runs } = findHiddenRuns(source);
-  if (!runs.length) return { text: source, runs };
-  let out = "";
-  let at = 0;
-  for (const run of runs) {
-    out += source.slice(at, run.offset);
-    at = run.end;
-  }
-  return { text: out + source.slice(at), runs };
-}
-
-/**
- * One omission per kind of run: `count` runs, `chars` code points.
- * @param {Array<{ kind: string, count: number }>} runs
- */
-export function hiddenTextOmissions(runs) {
-  const byKind = new Map();
-  for (const run of runs) {
-    const entry = byKind.get(run.kind) || { kind: `hidden_${run.kind}`, count: 0, chars: 0, note: NOTES[run.kind] };
-    entry.count += 1;
-    entry.chars += run.count;
-    byKind.set(run.kind, entry);
-  }
-  return Array.from(byKind.values());
-}
+// Optional last stage: remove characters that hide text from people but not
+// from models (tag characters, variation-selector runs, invisible operators,
+// reserved tag-plane code points), found by text-sanctifier's run finder. Legitimate sequences are kept: the
+// England, Scotland and Wales flags and a lone emoji or text presentation
+// selector. Unlike the clean stage, this also applies inside fenced code.
+import { findHiddenRuns } from "text-sanctifier/hidden";
+
+const NOTES = {
+  tag: "Hidden tag-character text removed",
+  variation_selector: "Variation selector runs removed",
+  invisible_operator: "Invisible operator runs removed",
+  reserved_tag_plane: "Reserved tag-plane runs removed",
+};
+
+/**
+ * @param {string} text
+ * @returns {{ text: string, runs: import("./types.js").HiddenRun[] }}
+ */
+export function removeHiddenText(text) {
+  const source = String(text ?? "");
+  const { runs } = findHiddenRuns(source);
+  if (!runs.length) return { text: source, runs };
+  let out = "";
+  let at = 0;
+  for (const run of runs) {
+    out += source.slice(at, run.offset);
+    at = run.end;
+  }
+  return { text: out + source.slice(at), runs };
+}
+
+/**
+ * One omission per kind of run: `count` runs, `chars` code points.
+ * @param {ReadonlyArray<Pick<import("./types.js").HiddenRun, "kind"|"count">>} runs
+ * @returns {import("./types.js").HiddenTextOmission[]}
+ */
+export function hiddenTextOmissions(runs) {
+  const byKind = new Map();
+  for (const run of runs) {
+    const entry = byKind.get(run.kind) || { kind: `hidden_${run.kind}`, count: 0, chars: 0, note: NOTES[run.kind] };
+    entry.count += 1;
+    entry.chars += run.count;
+    byKind.set(run.kind, entry);
+  }
+  return Array.from(byKind.values());
+}