page-markdown 0.2.0 → 0.3.0

package/src/sanitize.js

@@ -1,5 +1,6 @@
 // Stage 1: clone the page into an inert document, inline open shadow roots
 // within limits, and strip non-content. The live page is never modified.
+import { CONCEALED_MARK, createConcealmentWalk, isText, isElement, DOM_LOCAL_NAME, DOM_GET_ROOT_NODE, DOM_CHILD_NODES, DOM_NODE_VALUE, DOM_SHADOW_ROOT, DOM_CONTENT, DOM_APPEND_CHILD, DOM_NODE_TYPE} from "./concealed.js";
 
 /** @type {Readonly<Required<import("./types.js").SanitizeOptions>>} */
 export const DEFAULT_SANITIZE_OPTIONS = Object.freeze({
@@ -9,6 +10,7 @@ export const DEFAULT_SANITIZE_OPTIONS = Object.freeze({
   stripMeta: true,
   stripLinkTags: true,
   stripAds: true,
+  detectConcealedText: false,
   // App-specific overlays or widgets to drop, e.g. "[data-app-overlay]".
   extraStripSelectors: [],
   shadowNodeLimit: 5000,
@@ -36,10 +38,10 @@ const AD_ELEMENT_NAME = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*-(?:placement|advert|adve
 
 /** @param {any} node */
 export function isAdvertisementElement(node) {
-  if (node?.nodeType !== 1) return false;
+  if (node?.[DOM_NODE_TYPE] !== 1) return false;
   // Sites put page-wide ad settings on <body> (heise.de's data-ad-unit).
-  if (node.localName === "html" || node.localName === "body") return false;
-  if (AD_ELEMENT_NAME.test(node.localName || "")) return true;
+  if (node[DOM_LOCAL_NAME] === "html" || node[DOM_LOCAL_NAME] === "body") return false;
+  if (AD_ELEMENT_NAME.test(node[DOM_LOCAL_NAME] || "")) return true;
   try {
     return Boolean(node.matches?.(AD_SELECTOR));
   } catch {
@@ -59,7 +61,7 @@ function stripSelectors(options) {
 }
 
 function isStripped(node, selectors, options) {
-  if (node.nodeType !== 1) return false;
+  if (!isElement(node)) return false;
   if (selectors.some((selector) => node.matches?.(selector))) return true;
   return Boolean(options.stripAds && isAdvertisementElement(node));
 }
@@ -96,27 +98,31 @@ function positiveLimit(value, name) {
  *
  * @param {Document|Element} root
  * @param {import("./types.js").SanitizeOptions} [options]
- * @returns {{ document: Document, omissions: { shadowHosts: number } }}
  */
 export function sanitizeToInertDocument(root, options = {}) {
   const opts = { ...DEFAULT_SANITIZE_OPTIONS, ...options };
-  const limits = {
-    nodes: positiveLimit(opts.shadowNodeLimit, "shadowNodeLimit"),
-    chars: positiveLimit(opts.shadowCharLimit, "shadowCharLimit"),
-    depth: positiveLimit(opts.shadowDepthLimit, "shadowDepthLimit"),
-  };
+  const limits = [
+    positiveLimit(opts.shadowNodeLimit, "shadowNodeLimit"),
+    positiveLimit(opts.shadowCharLimit, "shadowCharLimit"),
+    positiveLimit(opts.shadowDepthLimit, "shadowDepthLimit"),
+  ];
   const selectors = stripSelectors(opts);
-  const sourceDoc = /** @type {Document} */ (root.nodeType === 9 ? root : root.ownerDocument);
-  const source = root.nodeType === 9 ? /** @type {Document} */ (root).documentElement : root;
+  const sourceDoc = /** @type {Document} */ (root[DOM_NODE_TYPE] === 9 ? root : root.ownerDocument);
+  const source = root[DOM_NODE_TYPE] === 9 ? /** @type {Document} */ (root).documentElement : root;
   const inert = createInertDocument(sourceDoc);
+  const concealment = createConcealmentWalk(sourceDoc, opts.detectConcealedText);
 
   function shallow(node) {
-    if (node.nodeType !== 1) return inert.importNode(node, false);
-    const name = node.prefix ? `${node.prefix}:${node.localName}` : node.localName;
+    if (!isElement(node)) {
+      const copy = inert.importNode(node, false);
+      return copy;
+    }
+    const name = node.prefix ? `${node.prefix}:${node[DOM_LOCAL_NAME]}` : node[DOM_LOCAL_NAME];
     const copy = node.namespaceURI
       ? inert.createElementNS(node.namespaceURI, name)
       : inert.createElement(name);
     for (const attr of Array.from(node.attributes || [])) {
+      if (attr.name.toLowerCase() === CONCEALED_MARK) continue;
       // HTML allows colon-bearing names without a namespace (framework
       // directives); setAttributeNS rejects those.
       if (attr.namespaceURI) copy.setAttributeNS(attr.namespaceURI, attr.name, attr.value);
@@ -129,89 +135,111 @@ export function sanitizeToInertDocument(root, options = {}) {
   let chars = 0;
   let overLimit = false;
   let shadowHosts = 0;
-
-  function isAssigningSlot(node) {
-    return node.nodeType === 1 && node.localName === "slot"
-      && typeof node.assignedNodes === "function" && Boolean(node.getRootNode?.().host);
+  function carry(work, pair) {
+    for (let i = 2; i <= 8; i++) pair[i] = work[i];
+    return pair;
   }
 
-  function copyShadowRoot(host, target, depth) {
-    const outer = [nodes, chars];
-    nodes = 0;
-    chars = 0;
-    for (const child of Array.from(host.shadowRoot.childNodes)) {
-      const childCopy = shadowNode(child, depth);
-      if (childCopy) target.appendChild(childCopy);
-      if (overLimit) break;
-    }
-    if (overLimit) shadowHosts += 1;
-    overLimit = false;
-    [nodes, chars] = outer;
+  function isAssigningSlot(node) {
+    return isElement(node) && node[DOM_LOCAL_NAME] === "slot"
+      && typeof node.assignedNodes === "function" && !!node[DOM_GET_ROOT_NODE]?.().host;
   }
 
-  function shadowNode(node, depth) {
-    const inPage = node.getRootNode?.().nodeType === 9;
-    if (depth > limits.depth || (!inPage && ++nodes > limits.nodes)) {
+  function shadowNode(work, depth, lightHost) {
+    const [node, allocated] = work;
+    const inPage = lightHost || node[DOM_GET_ROOT_NODE]?.()[DOM_NODE_TYPE] === 9;
+    if (depth > limits[2] || (!inPage && ++nodes > limits[0])) {
       overLimit = true;
       return null;
     }
-    if (isStripped(node, selectors, opts)) return inert.createDocumentFragment();
-    const copy = shallow(node);
-    if (inPage && node.shadowRoot) {
-      copyShadowRoot(node, copy, depth + 1);
-      return copy;
+    if (!lightHost && isStripped(node, selectors, opts)) return inert.createDocumentFragment();
+    const copy = allocated || shallow(node);
+    work = concealment.inherit(work, node, copy);
+    const host = inPage && node[DOM_SHADOW_ROOT];
+    const outer = host && [nodes, chars];
+    let elementChars = 0;
+    if (host) nodes = chars = 0;
+    else {
+      if (!inPage) chars += isElement(node) ? (elementChars = (copy.outerHTML || "").length) : (node[DOM_NODE_VALUE] || "").length * 6;
+      if (chars > limits[1]) { overLimit = true; return null; }
     }
-    if (!inPage) chars += node.nodeType === 1 ? (copy.outerHTML || "").length : (node.nodeValue || "").length * 6;
-    if (chars > limits.chars) {
-      overLimit = true;
-      return null;
-    }
-    let children = node.childNodes;
+    let children = node[DOM_CHILD_NODES];
     let target = copy;
-    if (isAssigningSlot(node)) {
+    if (host) children = host[DOM_CHILD_NODES];
+    else if (isAssigningSlot(node)) {
       const assigned = node.assignedNodes({ flatten: true });
-      children = assigned.length ? assigned : node.childNodes;
+      children = assigned.length ? assigned : node[DOM_CHILD_NODES];
       target = inert.createDocumentFragment();
-    } else if (node.shadowRoot) {
-      children = node.shadowRoot.childNodes;
-    } else if (node.nodeType === 1 && node.localName === "template" && node.content) {
-      children = node.content.childNodes;
-      target = copy.content || copy;
+    } else if (node[DOM_SHADOW_ROOT]) {
+      children = node[DOM_SHADOW_ROOT][DOM_CHILD_NODES];
+    } else if (isElement(node) && node[DOM_LOCAL_NAME] === "template" && node[DOM_CONTENT]) {
+      children = node[DOM_CONTENT][DOM_CHILD_NODES];
+      target = copy[DOM_CONTENT] || copy;
     }
+    const pairs = [];
+    let textAllocations = 0;
     for (const child of Array.from(children || [])) {
-      const childCopy = shadowNode(child, depth + 1);
-      if (childCopy) target.appendChild(childCopy);
+      // Stripped children still pass through the recursive node-budget gate.
+      // Only direct text must exist while this element's declaration is
+      // local. Element allocation stays behind the recursive budget gate.
+      const pageText = child[DOM_GET_ROOT_NODE]?.()[DOM_NODE_TYPE] === 9;
+      const allocateText = isText(child) && (pageText || ++textAllocations <= limits[0] - nodes);
+      pairs.push([child, allocateText ? shallow(child) : null]);
+    }
+    concealment.process(node, copy, work, host ? null : pairs);
+    if (!inPage && isElement(node)) {
+      // Detection may add the compatibility hidden attribute. Charge it before
+      // accepting children, just as when detection ran during shallow cloning.
+      chars += (copy.outerHTML || "").length - elementChars;
+      if (chars > limits[1]) { overLimit = true; return null; }
+    }
+    for (const pair of pairs) {
+      const childCopy = shadowNode(carry(work, pair), depth + 1);
+      if (childCopy) target[DOM_APPEND_CHILD](childCopy);
       if (overLimit) break;
     }
+    if (host) {
+      if (overLimit) shadowHosts++;
+      overLimit = false;
+      [nodes, chars] = outer;
+      return copy;
+    }
     return isAssigningSlot(node) ? target : copy;
   }
 
   const clone = shallow(source);
+  const initial = concealment.initial(source);
   // Iterative light-DOM walk: ordinary pages get no depth limit.
-  const pending = [[source, clone]];
+  const pending = [carry(initial, [source, clone])];
   while (pending.length) {
-    const [live, copy] = /** @type {any[]} */ (pending.pop());
-    if (live.shadowRoot) {
-      copyShadowRoot(live, copy, 1);
+    const work = /** @type {any[]} */ (pending.pop());
+    const [live, copy] = work;
+    if (live[DOM_SHADOW_ROOT]) {
+      shadowNode(work, 0, true);
       continue;
     }
-    const children = live.content?.childNodes || live.childNodes;
-    const target = copy.content || copy;
+    const children = live[DOM_CONTENT]?.[DOM_CHILD_NODES] || live[DOM_CHILD_NODES];
+    const target = copy[DOM_CONTENT] || copy;
     const pairs = [];
     for (const child of Array.from(children || [])) {
       if (isStripped(child, selectors, opts)) continue;
       const childCopy = shallow(child);
-      target.appendChild(childCopy);
+      target[DOM_APPEND_CHILD](childCopy);
       pairs.push([child, childCopy]);
     }
-    for (let i = pairs.length - 1; i >= 0; i -= 1) pending.push(pairs[i]);
+    concealment.process(live, copy, work, pairs);
+    for (let i = pairs.length - 1; i >= 0; i -= 1) {
+      const pair = pairs[i];
+      if (!isElement(pair[0])) continue;
+      pending.push(carry(concealment.inherit(work, pair[0]), pair));
+    }
   }
 
-  if (clone.localName === "html") {
+  if (clone[DOM_LOCAL_NAME] === "html") {
     inert.removeChild(inert.documentElement);
-    inert.appendChild(clone);
+    inert[DOM_APPEND_CHILD](clone);
   } else {
-    inert.body.appendChild(clone);
+    inert.body[DOM_APPEND_CHILD](clone);
   }
-  return { document: inert, omissions: { shadowHosts } };
+  return { document: inert, omissions: { shadowHosts }, concealment };
 }

package/src/standardize.js

@@ -68,6 +68,21 @@ function hasMeaningfulContent(el) {
  * data: placeholder. Then drop images that are still inline data.
  */
 function standardizeImages(doc, base, stats) {
+  const squash = (text) => (text || "").replace(/\s+/g, " ").trim();
+  for (const element of doc.querySelectorAll("figcaption, .caption")) {
+    const caption = squash(element.textContent);
+    if (!caption || element.closest(".pswp-caption-content") || isHiddenElement(element)) continue;
+    const figure = element.closest("figure");
+    // PhotoSwipe's copy is hidden by CSS; galleries may have only that copy.
+    for (const copy of (figure || element.parentElement?.parentElement)?.querySelectorAll(".pswp-caption-content") || []) {
+      if (squash(copy.textContent) === caption) copy.remove();
+    }
+    for (const img of figure?.querySelectorAll("img") || []) {
+      for (const attr of ["alt", "title"]) {
+        if (squash(img.getAttribute(attr)) === caption) img.removeAttribute(attr);
+      }
+    }
+  }
   for (const img of Array.from(doc.querySelectorAll("img"))) {
     const src = img.getAttribute("src") || "";
     if (!src || src.startsWith("data:")) {
@@ -100,6 +115,19 @@ function standardizeLinks(doc, base, stats) {
       stats.emptyLinksRemoved += 1;
       continue;
     }
+    if (base && href) {
+      try {
+        const target = new URL(href, base);
+        // Some browser URL parsers accept percent-encoded spaces in a host.
+        // Special-scheme hosts cannot contain ASCII whitespace or controls.
+        if (/^(?:https?|ftp|wss?):$/.test(target.protocol)
+          && /[\x00-\x20\x7f]/.test(decodeURIComponent(target.hostname))) throw new TypeError("Invalid link host");
+      } catch {
+        a.replaceWith(...Array.from(a.childNodes));
+        stats.invalidLinksUnwrapped += 1;
+        continue;
+      }
+    }
     if (href) a.setAttribute("href", resolveUrl(href, base));
   }
 }
@@ -170,6 +198,10 @@ function removeOwnPermalinks(heading, stats) {
  * remaining element is the heading with the heading itself.
  */
 function standardizeHeadings(doc, stats) {
+  // An upper-deck kicker belongs with the title, not ordinary body prose.
+  for (const label of doc.querySelectorAll(".upper-deck__text")) {
+    label.innerHTML = `<strong>${label.innerHTML}</strong>`;
+  }
   for (const widget of Array.from(doc.querySelectorAll(EDIT_WIDGETS))) {
     if (!(widget.textContent || "").trim() || widget.matches?.(ALWAYS_REMOVED_WIDGETS)) {
       widget.remove();
@@ -220,7 +252,7 @@ function tabOf(panel, tabs) {
  * name, in place of a tab strip that points nowhere. A long inactive panel is
  * left out, and named in stats.omittedTabs.
  */
-function standardizeTabPanels(doc, stats) {
+export function prepareConcealment(doc, concealment) {
   // Docker's docs write aria-role, and call a panel a tab: only the strip's is right.
   for (const list of Array.from(doc.querySelectorAll("[aria-role='tablist']"))) {
     list.setAttribute("role", "tablist");
@@ -228,13 +260,29 @@ function standardizeTabPanels(doc, stats) {
     for (const panel of Array.from(list.nextElementSibling?.children || [])) panel.setAttribute("role", "tabpanel");
   }
   const panels = Array.from(doc.querySelectorAll(TAB_PANELS));
+  const hasProvenance = concealment && ["computed", "incomplete"].includes(concealment.check);
+  const inactive = new Map(panels.map((panel) => [panel, isInactivePanel(panel)]));
+  const panelLengths = new Map(panels.map((panel) => [panel, (panel.textContent || "").replace(/\s+/g, " ").trim().length]));
+  // Resolve short-panel exceptions before lifting, but copy labels only after
+  // lifting: concealed label text must never become a new visible heading.
+  for (const panel of panels) {
+    if (hasProvenance && inactive.get(panel) && panelLengths.get(panel) <= MAX_INACTIVE_PANEL_CHARS) {
+      concealment?.exempt(panel);
+    }
+  }
+  concealment?.lift(doc);
+  return { panels, hasProvenance, inactive, panelLengths };
+}
+
+function standardizeTabPanels(doc, stats, concealment) {
+  const { panels, hasProvenance, inactive, panelLengths } = prepareConcealment(doc, concealment);
   if (!panels.length) return;
   const tabs = Array.from(doc.querySelectorAll(TABS));
   for (const panel of panels) {
     const label = (tabOf(panel, tabs)?.textContent || "").replace(/\s+/g, " ").trim().slice(0, MAX_TAB_LABEL_CHARS);
     const text = (panel.textContent || "").replace(/\s+/g, " ").trim();
-    if (isInactivePanel(panel)) {
-      if (text.length > MAX_INACTIVE_PANEL_CHARS) {
+    if (hasProvenance ? inactive.get(panel) : isInactivePanel(panel)) {
+      if ((hasProvenance ? panelLengths.get(panel) : text.length) > MAX_INACTIVE_PANEL_CHARS) {
         stats.omittedTabs.push(label);
         panel.remove();
         continue;
@@ -259,11 +307,19 @@ function standardizeAsides(doc) {
     quote.append(...Array.from(aside.childNodes));
     aside.replaceWith(quote);
   }
+  // Readability treats "header" as clutter, even when it labels a warning.
+  for (const header of Array.from(doc.querySelectorAll("blockquote .admonition-header"))) {
+    const p = doc.createElement("p");
+    const strong = doc.createElement("strong");
+    strong.append(...Array.from(header.childNodes));
+    p.append(strong);
+    header.replaceWith(p);
+  }
 }
 
 /**
  * @param {Document} doc the inert clone (mutated in place)
- * @param {{ url?: string }} [options]
+ * @param {{ url?: string, concealment?: ReturnType<typeof import('./concealed.js').createConcealmentWalk> }} [options]
  */
 export function standardizeDocument(doc, options = {}) {
   const base = options.url || "";
@@ -271,13 +327,14 @@ export function standardizeDocument(doc, options = {}) {
     lazyImagesResolved: 0,
     inlineImagesDropped: 0,
     scriptLinksUnwrapped: 0,
+    invalidLinksUnwrapped: 0,
     emptyLinksRemoved: 0,
     preWrapped: 0,
     headingWidgetsRemoved: 0,
     headingsUnwrapped: 0,
     omittedTabs: /** @type {string[]} */ ([]),
   };
-  standardizeTabPanels(doc, stats);
+  standardizeTabPanels(doc, stats, options.concealment);
   standardizeAsides(doc);
   standardizeHeadings(doc, stats);
   standardizeCode(doc, stats);

package/src/types.js

@@ -14,6 +14,7 @@ export {};
  * @property {boolean} [stripMeta]
  * @property {boolean} [stripLinkTags]
  * @property {boolean} [stripAds]
+ * @property {boolean} [detectConcealedText] opt-in computed-style check, default false
  * @property {readonly string[]} [extraStripSelectors] app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
  * @property {number} [shadowNodeLimit]
  * @property {number} [shadowCharLimit]
@@ -69,11 +70,13 @@ export {};
  */
 
 /**
- * Check `kind` before reading `chars`: only hidden-text omissions have it.
+ * Check `kind` before reading `chars`: hidden Unicode omissions use code points;
+ * concealed_text uses UTF-16 units. Other kinds do not have it.
  * @typedef {{ kind: "shadow_content", count: number, note: string }
  *   | { kind: "json_line", count: number, note: string }
  *   | { kind: "inactive_tabs", count: number, note: string }
  *   | { kind: "comments", count: number, note: string }
+ *   | { kind: "concealed_text", count: number, chars: number, note: string }
  *   | HiddenTextOmission} PageMarkdownOmission
  */
 
@@ -93,6 +96,9 @@ export {};
 /**
  * @typedef {object} PageMarkdownResult
  * @property {string} version the package version that made this result
+ * @property {ConcealedText[]} concealedText capped captured blocks, raw text in UTF-16 units
+ * @property {ConcealmentCheck} concealmentCheck source computed-style capability
+ * @property {boolean} concealedTextTruncated true if any block, text or entry cap omitted report text
  * @property {string} markdown
  * @property {PageMarkdownMetadata} metadata
  * @property {PageMarkdownExtraction} extraction
@@ -104,6 +110,17 @@ export {};
 
 /** @typedef {{ label: string, value: string }} PageFact */
 
+/**
+ * The one-time probe establishes that the document's view can return every
+ * required field and exposed optional field. Afterwards, incomplete means a
+ * CONSULTED value was empty or threw; unread irrelevant values are not checked.
+ * Computed means no such failure; static-only means no computed-style API;
+ * disabled means detection was turned off.
+ * @typedef {"computed"|"static-only"|"disabled"|"incomplete"} ConcealmentCheck
+ */
+/** @typedef {"display_none"|"visibility_hidden"|"opacity_zero"|"font_size_zero"|"clipped"|"transparent_text"|"same_color"} ConcealmentReason */
+/** @typedef {{ reason: ConcealmentReason, chars: number, text: string }} ConcealedText */
+
 /**
  * @typedef {string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean)} TurndownFilter
  */