page-markdown 0.1.5 → 0.3.0

package/src/sanitize.js

@@ -1,5 +1,6 @@
 // Stage 1: clone the page into an inert document, inline open shadow roots
 // within limits, and strip non-content. The live page is never modified.
+import { CONCEALED_MARK, createConcealmentWalk, isText, isElement, DOM_LOCAL_NAME, DOM_GET_ROOT_NODE, DOM_CHILD_NODES, DOM_NODE_VALUE, DOM_SHADOW_ROOT, DOM_CONTENT, DOM_APPEND_CHILD, DOM_NODE_TYPE} from "./concealed.js";
 
 /** @type {Readonly<Required<import("./types.js").SanitizeOptions>>} */
 export const DEFAULT_SANITIZE_OPTIONS = Object.freeze({
@@ -9,6 +10,7 @@ export const DEFAULT_SANITIZE_OPTIONS = Object.freeze({
   stripMeta: true,
   stripLinkTags: true,
   stripAds: true,
+  detectConcealedText: false,
   // App-specific overlays or widgets to drop, e.g. "[data-app-overlay]".
   extraStripSelectors: [],
   shadowNodeLimit: 5000,
@@ -27,7 +29,7 @@ const STRIP_RULES = {
 // Advertising placements: explicit ad markup, plus custom elements named for ad
 // slots (e.g. MDN's <mdn-placement-sidebar>).
 const AD_SELECTOR = [
-  "ins.adsbygoogle", ".adsbygoogle", ".advertisement",
+  "ins.adsbygoogle", ".adsbygoogle", ".advertisement", ".ad-label",
   "[data-ad-slot]", "[data-ad-client]", "[data-ad-unit]",
   "[aria-label='Advertisement' i]", "[aria-label='Advertisements' i]", "[aria-label='Sponsored' i]",
   "iframe[src*='doubleclick.net']", "iframe[src*='googlesyndication.com']", "iframe[id^='google_ads_iframe']",
@@ -36,10 +38,10 @@ const AD_ELEMENT_NAME = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*-(?:placement|advert|adve
 
 /** @param {any} node */
 export function isAdvertisementElement(node) {
-  if (node?.nodeType !== 1) return false;
+  if (node?.[DOM_NODE_TYPE] !== 1) return false;
   // Sites put page-wide ad settings on <body> (heise.de's data-ad-unit).
-  if (node.localName === "html" || node.localName === "body") return false;
-  if (AD_ELEMENT_NAME.test(node.localName || "")) return true;
+  if (node[DOM_LOCAL_NAME] === "html" || node[DOM_LOCAL_NAME] === "body") return false;
+  if (AD_ELEMENT_NAME.test(node[DOM_LOCAL_NAME] || "")) return true;
   try {
     return Boolean(node.matches?.(AD_SELECTOR));
   } catch {
@@ -59,7 +61,7 @@ function stripSelectors(options) {
 }
 
 function isStripped(node, selectors, options) {
-  if (node.nodeType !== 1) return false;
+  if (!isElement(node)) return false;
   if (selectors.some((selector) => node.matches?.(selector))) return true;
   return Boolean(options.stripAds && isAdvertisementElement(node));
 }
@@ -79,8 +81,6 @@ export function createInertDocument(doc) {
   throw new Error("page-markdown: cannot create an inert document (no createHTMLDocument or DOMParser)");
 }
 
-const SHADOW_LIMIT = Symbol("shadow limit");
-
 function positiveLimit(value, name) {
   if (!Number.isSafeInteger(value) || value < 1) {
     throw new RangeError(`page-markdown: ${name} must be a positive integer`);
@@ -91,32 +91,38 @@ function positiveLimit(value, name) {
 /**
  * Clones `root` (a Document or Element) into a new inert document.
  * Open shadow roots replace their host's light children (native slots supply
- * assigned children). Limits apply only to shadow content; a host over the
- * limit is emptied and counted as an omission.
+ * assigned children). Limits apply only to shadow content, per host in the
+ * page's own tree, so slotted page content is not charged: docs sites put a
+ * whole article in one host's slot. A host over the limit keeps what fits and
+ * is counted as an omission.
  *
  * @param {Document|Element} root
  * @param {import("./types.js").SanitizeOptions} [options]
- * @returns {{ document: Document, omissions: { shadowHosts: number } }}
  */
 export function sanitizeToInertDocument(root, options = {}) {
   const opts = { ...DEFAULT_SANITIZE_OPTIONS, ...options };
-  const limits = {
-    nodes: positiveLimit(opts.shadowNodeLimit, "shadowNodeLimit"),
-    chars: positiveLimit(opts.shadowCharLimit, "shadowCharLimit"),
-    depth: positiveLimit(opts.shadowDepthLimit, "shadowDepthLimit"),
-  };
+  const limits = [
+    positiveLimit(opts.shadowNodeLimit, "shadowNodeLimit"),
+    positiveLimit(opts.shadowCharLimit, "shadowCharLimit"),
+    positiveLimit(opts.shadowDepthLimit, "shadowDepthLimit"),
+  ];
   const selectors = stripSelectors(opts);
-  const sourceDoc = /** @type {Document} */ (root.nodeType === 9 ? root : root.ownerDocument);
-  const source = root.nodeType === 9 ? /** @type {Document} */ (root).documentElement : root;
+  const sourceDoc = /** @type {Document} */ (root[DOM_NODE_TYPE] === 9 ? root : root.ownerDocument);
+  const source = root[DOM_NODE_TYPE] === 9 ? /** @type {Document} */ (root).documentElement : root;
   const inert = createInertDocument(sourceDoc);
+  const concealment = createConcealmentWalk(sourceDoc, opts.detectConcealedText);
 
   function shallow(node) {
-    if (node.nodeType !== 1) return inert.importNode(node, false);
-    const name = node.prefix ? `${node.prefix}:${node.localName}` : node.localName;
+    if (!isElement(node)) {
+      const copy = inert.importNode(node, false);
+      return copy;
+    }
+    const name = node.prefix ? `${node.prefix}:${node[DOM_LOCAL_NAME]}` : node[DOM_LOCAL_NAME];
     const copy = node.namespaceURI
       ? inert.createElementNS(node.namespaceURI, name)
       : inert.createElement(name);
     for (const attr of Array.from(node.attributes || [])) {
+      if (attr.name.toLowerCase() === CONCEALED_MARK) continue;
       // HTML allows colon-bearing names without a namespace (framework
       // directives); setAttributeNS rejects those.
       if (attr.namespaceURI) copy.setAttributeNS(attr.namespaceURI, attr.name, attr.value);
@@ -127,70 +133,113 @@ export function sanitizeToInertDocument(root, options = {}) {
 
   let nodes = 0;
   let chars = 0;
+  let overLimit = false;
   let shadowHosts = 0;
+  function carry(work, pair) {
+    for (let i = 2; i <= 8; i++) pair[i] = work[i];
+    return pair;
+  }
 
   function isAssigningSlot(node) {
-    return node.nodeType === 1 && node.localName === "slot"
-      && typeof node.assignedNodes === "function" && Boolean(node.getRootNode?.().host);
+    return isElement(node) && node[DOM_LOCAL_NAME] === "slot"
+      && typeof node.assignedNodes === "function" && !!node[DOM_GET_ROOT_NODE]?.().host;
   }
 
-  function shadowNode(node, depth) {
-    if (++nodes > limits.nodes || depth > limits.depth) throw SHADOW_LIMIT;
-    if (isStripped(node, selectors, opts)) return inert.createDocumentFragment();
-    const copy = shallow(node);
-    chars += node.nodeType === 1 ? (copy.outerHTML || "").length : (node.nodeValue || "").length * 6;
-    if (chars > limits.chars) throw SHADOW_LIMIT;
-    let children = node.childNodes;
+  function shadowNode(work, depth, lightHost) {
+    const [node, allocated] = work;
+    const inPage = lightHost || node[DOM_GET_ROOT_NODE]?.()[DOM_NODE_TYPE] === 9;
+    if (depth > limits[2] || (!inPage && ++nodes > limits[0])) {
+      overLimit = true;
+      return null;
+    }
+    if (!lightHost && isStripped(node, selectors, opts)) return inert.createDocumentFragment();
+    const copy = allocated || shallow(node);
+    work = concealment.inherit(work, node, copy);
+    const host = inPage && node[DOM_SHADOW_ROOT];
+    const outer = host && [nodes, chars];
+    let elementChars = 0;
+    if (host) nodes = chars = 0;
+    else {
+      if (!inPage) chars += isElement(node) ? (elementChars = (copy.outerHTML || "").length) : (node[DOM_NODE_VALUE] || "").length * 6;
+      if (chars > limits[1]) { overLimit = true; return null; }
+    }
+    let children = node[DOM_CHILD_NODES];
     let target = copy;
-    if (isAssigningSlot(node)) {
+    if (host) children = host[DOM_CHILD_NODES];
+    else if (isAssigningSlot(node)) {
       const assigned = node.assignedNodes({ flatten: true });
-      children = assigned.length ? assigned : node.childNodes;
+      children = assigned.length ? assigned : node[DOM_CHILD_NODES];
       target = inert.createDocumentFragment();
-    } else if (node.shadowRoot) {
-      children = node.shadowRoot.childNodes;
-    } else if (node.nodeType === 1 && node.localName === "template" && node.content) {
-      children = node.content.childNodes;
-      target = copy.content || copy;
+    } else if (node[DOM_SHADOW_ROOT]) {
+      children = node[DOM_SHADOW_ROOT][DOM_CHILD_NODES];
+    } else if (isElement(node) && node[DOM_LOCAL_NAME] === "template" && node[DOM_CONTENT]) {
+      children = node[DOM_CONTENT][DOM_CHILD_NODES];
+      target = copy[DOM_CONTENT] || copy;
+    }
+    const pairs = [];
+    let textAllocations = 0;
+    for (const child of Array.from(children || [])) {
+      // Stripped children still pass through the recursive node-budget gate.
+      // Only direct text must exist while this element's declaration is
+      // local. Element allocation stays behind the recursive budget gate.
+      const pageText = child[DOM_GET_ROOT_NODE]?.()[DOM_NODE_TYPE] === 9;
+      const allocateText = isText(child) && (pageText || ++textAllocations <= limits[0] - nodes);
+      pairs.push([child, allocateText ? shallow(child) : null]);
+    }
+    concealment.process(node, copy, work, host ? null : pairs);
+    if (!inPage && isElement(node)) {
+      // Detection may add the compatibility hidden attribute. Charge it before
+      // accepting children, just as when detection ran during shallow cloning.
+      chars += (copy.outerHTML || "").length - elementChars;
+      if (chars > limits[1]) { overLimit = true; return null; }
+    }
+    for (const pair of pairs) {
+      const childCopy = shadowNode(carry(work, pair), depth + 1);
+      if (childCopy) target[DOM_APPEND_CHILD](childCopy);
+      if (overLimit) break;
+    }
+    if (host) {
+      if (overLimit) shadowHosts++;
+      overLimit = false;
+      [nodes, chars] = outer;
+      return copy;
     }
-    for (const child of Array.from(children || [])) target.appendChild(shadowNode(child, depth + 1));
     return isAssigningSlot(node) ? target : copy;
   }
 
   const clone = shallow(source);
+  const initial = concealment.initial(source);
   // Iterative light-DOM walk: ordinary pages get no depth limit.
-  const pending = [[source, clone]];
+  const pending = [carry(initial, [source, clone])];
   while (pending.length) {
-    const [live, copy] = /** @type {any[]} */ (pending.pop());
-    if (live.shadowRoot) {
-      const fragment = inert.createDocumentFragment();
-      try {
-        for (const child of Array.from(live.shadowRoot.childNodes)) fragment.appendChild(shadowNode(child, 1));
-        while (copy.firstChild) copy.removeChild(copy.firstChild);
-        copy.appendChild(fragment);
-      } catch (error) {
-        if (error !== SHADOW_LIMIT) throw error;
-        while (copy.firstChild) copy.removeChild(copy.firstChild);
-        shadowHosts += 1;
-      }
+    const work = /** @type {any[]} */ (pending.pop());
+    const [live, copy] = work;
+    if (live[DOM_SHADOW_ROOT]) {
+      shadowNode(work, 0, true);
       continue;
     }
-    const children = live.content?.childNodes || live.childNodes;
-    const target = copy.content || copy;
+    const children = live[DOM_CONTENT]?.[DOM_CHILD_NODES] || live[DOM_CHILD_NODES];
+    const target = copy[DOM_CONTENT] || copy;
     const pairs = [];
     for (const child of Array.from(children || [])) {
       if (isStripped(child, selectors, opts)) continue;
       const childCopy = shallow(child);
-      target.appendChild(childCopy);
+      target[DOM_APPEND_CHILD](childCopy);
       pairs.push([child, childCopy]);
     }
-    for (let i = pairs.length - 1; i >= 0; i -= 1) pending.push(pairs[i]);
+    concealment.process(live, copy, work, pairs);
+    for (let i = pairs.length - 1; i >= 0; i -= 1) {
+      const pair = pairs[i];
+      if (!isElement(pair[0])) continue;
+      pending.push(carry(concealment.inherit(work, pair[0]), pair));
+    }
   }
 
-  if (clone.localName === "html") {
+  if (clone[DOM_LOCAL_NAME] === "html") {
     inert.removeChild(inert.documentElement);
-    inert.appendChild(clone);
+    inert[DOM_APPEND_CHILD](clone);
   } else {
-    inert.body.appendChild(clone);
+    inert.body[DOM_APPEND_CHILD](clone);
   }
-  return { document: inert, omissions: { shadowHosts } };
+  return { document: inert, omissions: { shadowHosts }, concealment };
 }

package/src/standardize.js

@@ -1,5 +1,6 @@
 // Stage 2: normalise markup on the inert clone so extraction and conversion
 // see consistent structure. Runs before extraction.
+import { isHiddenElement } from "./extract.js";
 
 const LAZY_SRC_ATTRS = ["data-src", "data-lazy-src", "data-original", "data-lazy", "data-url"];
 const LANGUAGE_CLASS = [
@@ -12,11 +13,22 @@ const LANGUAGE_CLASS = [
 const LANGUAGE_ALIASES = { python3: "python", py3: "python" };
 const NON_LANGUAGES = new Set(["default", "none", "text", "plain", "plaintext", "nohighlight", "notranslate"]);
 // Wrappers sites put around a heading next to "edit" or permalink widgets.
-const EDIT_WIDGETS = ".mw-editsection, .headerlink, .anchor, .heading-anchor, a.anchor, [aria-hidden='true'] a";
+// Removed whatever their text: "edit", "Section titled ...", a zero-width space.
+const ALWAYS_REMOVED_WIDGETS = ".mw-editsection, .hash-link, .sl-anchor-link";
+const EDIT_WIDGETS = `${ALWAYS_REMOVED_WIDGETS}, .headerlink, .anchor, .heading-anchor, a.anchor, [aria-hidden='true'] a`;
 // Content without text that a heading wrapper may still hold.
 const MEDIA = "img, picture, video, audio";
 // Highlighters mark up each code line with one of these.
 const CODE_LINE_ELEMENTS = new Set(["div", "p", "li", "tr", "section"]);
+const TAB_PANELS = "[role='tabpanel'], sp-tab-panel, sl-tab-panel";
+const TABS = "[role='tab'], sp-tab, sl-tab";
+// Longer than this, an inactive panel is a page of its own (an API or change log
+// tab), not a variant of the one shown (pnpm, Yarn).
+const MAX_INACTIVE_PANEL_CHARS = 5000;
+const MAX_TAB_LABEL_CHARS = 40;
+// Asides that are part of the text: notes and warnings. Readability drops every <aside>.
+const NOTE_ASIDE = /(?:^|[\s_-])(?:aside|admonition|callout|note|tip|caution|warning|danger)(?:$|[\s_-])/i;
+const OTHER_ASIDE = /related|sidebar|share|promo|newsletter|toc\b/i;
 // As URL parsers read a scheme: control characters and spaces don't count.
 const isUnsafeUrl = (value) => /^(?:javascript|vbscript|data):/i.test(String(value).replace(/[\x00-\x20]+/g, ""));
 
@@ -56,6 +68,21 @@ function hasMeaningfulContent(el) {
  * data: placeholder. Then drop images that are still inline data.
  */
 function standardizeImages(doc, base, stats) {
+  const squash = (text) => (text || "").replace(/\s+/g, " ").trim();
+  for (const element of doc.querySelectorAll("figcaption, .caption")) {
+    const caption = squash(element.textContent);
+    if (!caption || element.closest(".pswp-caption-content") || isHiddenElement(element)) continue;
+    const figure = element.closest("figure");
+    // PhotoSwipe's copy is hidden by CSS; galleries may have only that copy.
+    for (const copy of (figure || element.parentElement?.parentElement)?.querySelectorAll(".pswp-caption-content") || []) {
+      if (squash(copy.textContent) === caption) copy.remove();
+    }
+    for (const img of figure?.querySelectorAll("img") || []) {
+      for (const attr of ["alt", "title"]) {
+        if (squash(img.getAttribute(attr)) === caption) img.removeAttribute(attr);
+      }
+    }
+  }
   for (const img of Array.from(doc.querySelectorAll("img"))) {
     const src = img.getAttribute("src") || "";
     if (!src || src.startsWith("data:")) {
@@ -88,6 +115,19 @@ function standardizeLinks(doc, base, stats) {
       stats.emptyLinksRemoved += 1;
       continue;
     }
+    if (base && href) {
+      try {
+        const target = new URL(href, base);
+        // Some browser URL parsers accept percent-encoded spaces in a host.
+        // Special-scheme hosts cannot contain ASCII whitespace or controls.
+        if (/^(?:https?|ftp|wss?):$/.test(target.protocol)
+          && /[\x00-\x20\x7f]/.test(decodeURIComponent(target.hostname))) throw new TypeError("Invalid link host");
+      } catch {
+        a.replaceWith(...Array.from(a.childNodes));
+        stats.invalidLinksUnwrapped += 1;
+        continue;
+      }
+    }
     if (href) a.setAttribute("href", resolveUrl(href, base));
   }
 }
@@ -133,19 +173,43 @@ function standardizeCode(doc, stats) {
   }
 }
 
+/**
+ * A link to the heading's own id, inside it or right beside it, is a permalink
+ * whatever its class or wording ("Link to ...", "Section titled ...", "#"). A
+ * heading whose whole text is that link keeps the text. Left in place, the link
+ * makes a short heading's wrapper look like clutter to Readability.
+ */
+function removeOwnPermalinks(heading, stats) {
+  const id = heading.getAttribute("id");
+  if (!id) return;
+  const title = (heading.textContent || "").trim();
+  const inside = Array.from(heading.querySelectorAll("[href]"));
+  for (const link of [...inside, heading.previousElementSibling, heading.nextElementSibling]) {
+    if (link?.getAttribute("href") !== `#${id}`) continue;
+    if (inside.includes(link) && (link.textContent || "").trim() === title) link.replaceWith(...Array.from(link.childNodes));
+    else link.remove();
+    stats.headingWidgetsRemoved += 1;
+  }
+}
+
 /**
  * Headings wrapped with edit/permalink widgets (Wikipedia's div.mw-heading,
  * Sphinx headerlinks): drop the widgets, then replace a wrapper whose only
  * remaining element is the heading with the heading itself.
  */
 function standardizeHeadings(doc, stats) {
+  // An upper-deck kicker belongs with the title, not ordinary body prose.
+  for (const label of doc.querySelectorAll(".upper-deck__text")) {
+    label.innerHTML = `<strong>${label.innerHTML}</strong>`;
+  }
   for (const widget of Array.from(doc.querySelectorAll(EDIT_WIDGETS))) {
-    if (!(widget.textContent || "").trim() || widget.matches?.(".mw-editsection")) {
+    if (!(widget.textContent || "").trim() || widget.matches?.(ALWAYS_REMOVED_WIDGETS)) {
       widget.remove();
       stats.headingWidgetsRemoved += 1;
     }
   }
   for (const heading of Array.from(doc.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
+    removeOwnPermalinks(heading, stats);
     const parent = heading.parentElement;
     if (!parent || parent.localName !== "div") continue;
     const others = Array.from(parent.children).filter((el) => el !== heading);
@@ -157,9 +221,105 @@ function standardizeHeadings(doc, stats) {
   }
 }
 
+function isInactivePanel(panel) {
+  if (isHiddenElement(panel) || panel.getAttribute("aria-hidden") === "true" || panel.classList.contains("hidden")) return true;
+  // Custom elements that mark only the group: <sp-tabs selected="overview">.
+  const value = panel.getAttribute("value");
+  const selected = value && panel.parentElement?.closest("[selected]")?.getAttribute("selected");
+  return Boolean(selected) && selected !== value;
+}
+
+function tabOf(panel, tabs) {
+  const id = panel.getAttribute("id");
+  const labelledBy = panel.getAttribute("aria-labelledby");
+  const value = panel.getAttribute("value") || panel.getAttribute("name");
+  const named = tabs.find((tab) => (labelledBy && tab.getAttribute("id") === labelledBy)
+    || (id && (tab.getAttribute("aria-controls") === id || tab.getAttribute("href") === `#${id}`))
+    || (value && (tab.getAttribute("value") === value || tab.getAttribute("panel") === value)));
+  if (named) return named;
+  // Docusaurus ties them by position alone.
+  const parent = panel.parentElement;
+  const index = Array.from(parent?.children || []).filter((el) => el.matches?.(TAB_PANELS)).indexOf(panel);
+  for (let node = parent, depth = 0; node && depth < 4; node = node.parentElement, depth += 1) {
+    const list = node.querySelector("[role='tablist']");
+    if (list) return list.querySelectorAll("[role='tab']")[index];
+  }
+  return null;
+}
+
+/**
+ * Tabs read as one document: inactive panels are shown, each under its tab's
+ * name, in place of a tab strip that points nowhere. A long inactive panel is
+ * left out, and named in stats.omittedTabs.
+ */
+export function prepareConcealment(doc, concealment) {
+  // Docker's docs write aria-role, and call a panel a tab: only the strip's is right.
+  for (const list of Array.from(doc.querySelectorAll("[aria-role='tablist']"))) {
+    list.setAttribute("role", "tablist");
+    for (const tab of Array.from(list.children)) tab.setAttribute("role", "tab");
+    for (const panel of Array.from(list.nextElementSibling?.children || [])) panel.setAttribute("role", "tabpanel");
+  }
+  const panels = Array.from(doc.querySelectorAll(TAB_PANELS));
+  const hasProvenance = concealment && ["computed", "incomplete"].includes(concealment.check);
+  const inactive = new Map(panels.map((panel) => [panel, isInactivePanel(panel)]));
+  const panelLengths = new Map(panels.map((panel) => [panel, (panel.textContent || "").replace(/\s+/g, " ").trim().length]));
+  // Resolve short-panel exceptions before lifting, but copy labels only after
+  // lifting: concealed label text must never become a new visible heading.
+  for (const panel of panels) {
+    if (hasProvenance && inactive.get(panel) && panelLengths.get(panel) <= MAX_INACTIVE_PANEL_CHARS) {
+      concealment?.exempt(panel);
+    }
+  }
+  concealment?.lift(doc);
+  return { panels, hasProvenance, inactive, panelLengths };
+}
+
+function standardizeTabPanels(doc, stats, concealment) {
+  const { panels, hasProvenance, inactive, panelLengths } = prepareConcealment(doc, concealment);
+  if (!panels.length) return;
+  const tabs = Array.from(doc.querySelectorAll(TABS));
+  for (const panel of panels) {
+    const label = (tabOf(panel, tabs)?.textContent || "").replace(/\s+/g, " ").trim().slice(0, MAX_TAB_LABEL_CHARS);
+    const text = (panel.textContent || "").replace(/\s+/g, " ").trim();
+    if (hasProvenance ? inactive.get(panel) : isInactivePanel(panel)) {
+      if ((hasProvenance ? panelLengths.get(panel) : text.length) > MAX_INACTIVE_PANEL_CHARS) {
+        stats.omittedTabs.push(label);
+        panel.remove();
+        continue;
+      }
+      for (const attr of ["hidden", "aria-hidden", "style"]) panel.removeAttribute(attr);
+      panel.classList.remove("hidden");
+    }
+    if (label && panels.length > 1 && !text.toLowerCase().startsWith(label.toLowerCase())) {
+      const heading = doc.createElement("p");
+      heading.appendChild(doc.createElement("strong")).textContent = label;
+      panel.insertBefore(heading, panel.firstChild);
+    }
+  }
+  for (const el of Array.from(doc.querySelectorAll(`[role='tablist'], ${TABS}`))) el.remove();
+}
+
+function standardizeAsides(doc) {
+  for (const aside of Array.from(doc.querySelectorAll("main aside, article aside, [role='main'] aside"))) {
+    const name = `${aside.getAttribute("class") || ""} ${aside.getAttribute("aria-label") || ""} ${aside.getAttribute("role") || ""}`;
+    if (!NOTE_ASIDE.test(name) || OTHER_ASIDE.test(name)) continue;
+    const quote = doc.createElement("blockquote");
+    quote.append(...Array.from(aside.childNodes));
+    aside.replaceWith(quote);
+  }
+  // Readability treats "header" as clutter, even when it labels a warning.
+  for (const header of Array.from(doc.querySelectorAll("blockquote .admonition-header"))) {
+    const p = doc.createElement("p");
+    const strong = doc.createElement("strong");
+    strong.append(...Array.from(header.childNodes));
+    p.append(strong);
+    header.replaceWith(p);
+  }
+}
+
 /**
  * @param {Document} doc the inert clone (mutated in place)
- * @param {{ url?: string }} [options]
+ * @param {{ url?: string, concealment?: ReturnType<typeof import('./concealed.js').createConcealmentWalk> }} [options]
  */
 export function standardizeDocument(doc, options = {}) {
   const base = options.url || "";
@@ -167,11 +327,15 @@ export function standardizeDocument(doc, options = {}) {
     lazyImagesResolved: 0,
     inlineImagesDropped: 0,
     scriptLinksUnwrapped: 0,
+    invalidLinksUnwrapped: 0,
     emptyLinksRemoved: 0,
     preWrapped: 0,
     headingWidgetsRemoved: 0,
     headingsUnwrapped: 0,
+    omittedTabs: /** @type {string[]} */ ([]),
   };
+  standardizeTabPanels(doc, stats, options.concealment);
+  standardizeAsides(doc);
   standardizeHeadings(doc, stats);
   standardizeCode(doc, stats);
   standardizeImages(doc, base, stats);

package/src/types.js

@@ -14,6 +14,7 @@ export {};
  * @property {boolean} [stripMeta]
  * @property {boolean} [stripLinkTags]
  * @property {boolean} [stripAds]
+ * @property {boolean} [detectConcealedText] opt-in computed-style check, default false
  * @property {readonly string[]} [extraStripSelectors] app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
  * @property {number} [shadowNodeLimit]
  * @property {number} [shadowCharLimit]
@@ -40,7 +41,7 @@ export {};
 /**
  * @typedef {object} PageMarkdownExtraction
  * @property {Extraction["method"]} method
- * @property {string} [fallbackReason] why extraction fell back to the full page
+ * @property {string} [fallbackReason] why extraction fell back to the full page or the main content
  * @property {true} [leadRecovered] a dropped lead paragraph was restored
  * @property {number} [pageFacts] how many facts `pageFacts` appended
  */
@@ -69,9 +70,13 @@ export {};
  */
 
 /**
- * Check `kind` before reading `chars`: only hidden-text omissions have it.
+ * Check `kind` before reading `chars`: hidden Unicode omissions use code points;
+ * concealed_text uses UTF-16 units. Other kinds do not have it.
  * @typedef {{ kind: "shadow_content", count: number, note: string }
  *   | { kind: "json_line", count: number, note: string }
+ *   | { kind: "inactive_tabs", count: number, note: string }
+ *   | { kind: "comments", count: number, note: string }
+ *   | { kind: "concealed_text", count: number, chars: number, note: string }
  *   | HiddenTextOmission} PageMarkdownOmission
  */
 
@@ -83,12 +88,17 @@ export {};
  * @property {number} extractedHtmlChars
  * @property {number} convertedChars
  * @property {number} markdownChars
+ * @property {number} pageTextChars the page's text at capture time: a capture is a
+ *   snapshot of the DOM, so a caller can re-capture until this number settles
  * @property {StandardizationStats} standardization
  */
 
 /**
  * @typedef {object} PageMarkdownResult
  * @property {string} version the package version that made this result
+ * @property {ConcealedText[]} concealedText capped captured blocks, raw text in UTF-16 units
+ * @property {ConcealmentCheck} concealmentCheck source computed-style capability
+ * @property {boolean} concealedTextTruncated true if any block, text or entry cap omitted report text
  * @property {string} markdown
  * @property {PageMarkdownMetadata} metadata
  * @property {PageMarkdownExtraction} extraction
@@ -100,6 +110,17 @@ export {};
 
 /** @typedef {{ label: string, value: string }} PageFact */
 
+/**
+ * The one-time probe establishes that the document's view can return every
+ * required field and exposed optional field. Afterwards, incomplete means a
+ * CONSULTED value was empty or threw; unread irrelevant values are not checked.
+ * Computed means no such failure; static-only means no computed-style API;
+ * disabled means detection was turned off.
+ * @typedef {"computed"|"static-only"|"disabled"|"incomplete"} ConcealmentCheck
+ */
+/** @typedef {"display_none"|"visibility_hidden"|"opacity_zero"|"font_size_zero"|"clipped"|"transparent_text"|"same_color"} ConcealmentReason */
+/** @typedef {{ reason: ConcealmentReason, chars: number, text: string }} ConcealedText */
+
 /**
  * @typedef {string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean)} TurndownFilter
  */