page-markdown 0.1.4 → 0.2.0

package/src/standardize.js

@@ -1,5 +1,6 @@
 // Stage 2: normalise markup on the inert clone so extraction and conversion
 // see consistent structure. Runs before extraction.
+import { isHiddenElement } from "./extract.js";
 
 const LAZY_SRC_ATTRS = ["data-src", "data-lazy-src", "data-original", "data-lazy", "data-url"];
 const LANGUAGE_CLASS = [
@@ -12,9 +13,24 @@ const LANGUAGE_CLASS = [
 const LANGUAGE_ALIASES = { python3: "python", py3: "python" };
 const NON_LANGUAGES = new Set(["default", "none", "text", "plain", "plaintext", "nohighlight", "notranslate"]);
 // Wrappers sites put around a heading next to "edit" or permalink widgets.
-const EDIT_WIDGETS = ".mw-editsection, .headerlink, .anchor, .heading-anchor, a.anchor, [aria-hidden='true'] a";
+// Removed whatever their text: "edit", "Section titled ...", a zero-width space.
+const ALWAYS_REMOVED_WIDGETS = ".mw-editsection, .hash-link, .sl-anchor-link";
+const EDIT_WIDGETS = `${ALWAYS_REMOVED_WIDGETS}, .headerlink, .anchor, .heading-anchor, a.anchor, [aria-hidden='true'] a`;
 // Content without text that a heading wrapper may still hold.
 const MEDIA = "img, picture, video, audio";
+// Highlighters mark up each code line with one of these.
+const CODE_LINE_ELEMENTS = new Set(["div", "p", "li", "tr", "section"]);
+const TAB_PANELS = "[role='tabpanel'], sp-tab-panel, sl-tab-panel";
+const TABS = "[role='tab'], sp-tab, sl-tab";
+// Longer than this, an inactive panel is a page of its own (an API or change log
+// tab), not a variant of the one shown (pnpm, Yarn).
+const MAX_INACTIVE_PANEL_CHARS = 5000;
+const MAX_TAB_LABEL_CHARS = 40;
+// Asides that are part of the text: notes and warnings. Readability drops every <aside>.
+const NOTE_ASIDE = /(?:^|[\s_-])(?:aside|admonition|callout|note|tip|caution|warning|danger)(?:$|[\s_-])/i;
+const OTHER_ASIDE = /related|sidebar|share|promo|newsletter|toc\b/i;
+// As URL parsers read a scheme: control characters and spaces don't count.
+const isUnsafeUrl = (value) => /^(?:javascript|vbscript|data):/i.test(String(value).replace(/[\x00-\x20]+/g, ""));
 
 function resolveUrl(value, base) {
   const raw = String(value || "").trim();
@@ -44,7 +60,7 @@ function languageFrom(el, maxDepth = 3) {
 
 function hasMeaningfulContent(el) {
   if ((el.textContent || "").trim()) return true;
-  return Boolean(el.querySelector?.("img[alt]:not([alt='']), svg title"));
+  return Boolean(el.querySelector?.("img, svg title"));
 }
 
 /**
@@ -62,7 +78,7 @@ function standardizeImages(doc, base, stats) {
       }
     }
     const current = img.getAttribute("src") || "";
-    if (!current || current.startsWith("data:")) {
+    if (!current || isUnsafeUrl(current)) {
       img.remove();
       stats.inlineImagesDropped += 1;
       continue;
@@ -74,7 +90,7 @@ function standardizeImages(doc, base, stats) {
 function standardizeLinks(doc, base, stats) {
   for (const a of Array.from(doc.querySelectorAll("a"))) {
     const href = a.getAttribute("href") || "";
-    if (/^\s*javascript:/i.test(href)) {
+    if (isUnsafeUrl(href)) {
       a.replaceWith(...Array.from(a.childNodes));
       stats.scriptLinksUnwrapped += 1;
       continue;
@@ -88,26 +104,66 @@ function standardizeLinks(doc, base, stats) {
   }
 }
 
+/** Text of a code block, with <br> and block-level lines as line breaks. */
+function codeText(root) {
+  let text = "";
+  const walk = (node) => {
+    for (const child of Array.from(node.childNodes)) {
+      if (child.nodeType === 3) text += child.nodeValue;
+      else if (child.nodeType !== 1) continue;
+      else if (child.localName === "br") text += "\n";
+      else {
+        const block = CODE_LINE_ELEMENTS.has(child.localName);
+        if (block && text && !text.endsWith("\n")) text += "\n";
+        walk(child);
+        if (block && text && !text.endsWith("\n")) text += "\n";
+      }
+    }
+  };
+  walk(root);
+  return text;
+}
+
 /** <pre> without <code> (and code without a language) gets a language-tagged <code>. */
 function standardizeCode(doc, stats) {
   for (const pre of Array.from(doc.querySelectorAll("pre"))) {
-    const lang = languageFrom(pre);
-    let code = Array.from(pre.children || []).length === 1 && pre.firstElementChild?.localName === "code"
-      ? pre.firstElementChild
-      : null;
-    if (!code) {
-      code = doc.createElement("code");
-      code.textContent = pre.textContent || "";
-      while (pre.firstChild) pre.removeChild(pre.firstChild);
-      pre.appendChild(code);
-      stats.preWrapped += 1;
-    }
+    for (const button of Array.from(pre.querySelectorAll("button"))) button.remove();
+    const inner = pre.querySelector("code");
+    const lang = languageFrom(inner || pre, inner ? 4 : 3);
+    // One <code> and only whitespace around it: that <code> is the block.
+    const sole = pre.children.length === 1 && pre.firstElementChild === inner
+      && !(pre.textContent || "").replace(inner.textContent || "", "").trim();
+    const code = sole ? inner : doc.createElement("code");
+    if (!sole) stats.preWrapped += 1;
+    // Turndown reads textContent, so lines kept apart by markup become text.
+    code.textContent = codeText(sole ? inner : pre);
+    while (pre.firstChild) pre.removeChild(pre.firstChild);
+    pre.appendChild(code);
     if (lang && !/(?:^|\s)language-/.test(code.getAttribute("class") || "")) {
       code.setAttribute("class", `${code.getAttribute("class") || ""} language-${lang}`.trim());
     }
   }
 }
 
+/**
+ * A link to the heading's own id, inside it or right beside it, is a permalink
+ * whatever its class or wording ("Link to ...", "Section titled ...", "#"). A
+ * heading whose whole text is that link keeps the text. Left in place, the link
+ * makes a short heading's wrapper look like clutter to Readability.
+ */
+function removeOwnPermalinks(heading, stats) {
+  const id = heading.getAttribute("id");
+  if (!id) return;
+  const title = (heading.textContent || "").trim();
+  const inside = Array.from(heading.querySelectorAll("[href]"));
+  for (const link of [...inside, heading.previousElementSibling, heading.nextElementSibling]) {
+    if (link?.getAttribute("href") !== `#${id}`) continue;
+    if (inside.includes(link) && (link.textContent || "").trim() === title) link.replaceWith(...Array.from(link.childNodes));
+    else link.remove();
+    stats.headingWidgetsRemoved += 1;
+  }
+}
+
 /**
  * Headings wrapped with edit/permalink widgets (Wikipedia's div.mw-heading,
  * Sphinx headerlinks): drop the widgets, then replace a wrapper whose only
@@ -115,22 +171,96 @@ function standardizeCode(doc, stats) {
  */
 function standardizeHeadings(doc, stats) {
   for (const widget of Array.from(doc.querySelectorAll(EDIT_WIDGETS))) {
-    if (!(widget.textContent || "").trim() || widget.matches?.(".mw-editsection")) {
+    if (!(widget.textContent || "").trim() || widget.matches?.(ALWAYS_REMOVED_WIDGETS)) {
       widget.remove();
       stats.headingWidgetsRemoved += 1;
     }
   }
   for (const heading of Array.from(doc.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
+    removeOwnPermalinks(heading, stats);
     const parent = heading.parentElement;
     if (!parent || parent.localName !== "div") continue;
     const others = Array.from(parent.children).filter((el) => el !== heading);
-    if (others.every((el) => !(el.textContent || "").trim() && !el.matches?.(MEDIA) && !el.querySelector?.(MEDIA))) {
+    const hasBareText = Array.from(parent.childNodes).some((node) => node.nodeType === 3 && node.nodeValue.trim());
+    if (!hasBareText && others.every((el) => !(el.textContent || "").trim() && !el.matches?.(MEDIA) && !el.querySelector?.(MEDIA))) {
       parent.replaceWith(heading);
       stats.headingsUnwrapped += 1;
     }
   }
 }
 
+function isInactivePanel(panel) {
+  if (isHiddenElement(panel) || panel.getAttribute("aria-hidden") === "true" || panel.classList.contains("hidden")) return true;
+  // Custom elements that mark only the group: <sp-tabs selected="overview">.
+  const value = panel.getAttribute("value");
+  const selected = value && panel.parentElement?.closest("[selected]")?.getAttribute("selected");
+  return Boolean(selected) && selected !== value;
+}
+
+function tabOf(panel, tabs) {
+  const id = panel.getAttribute("id");
+  const labelledBy = panel.getAttribute("aria-labelledby");
+  const value = panel.getAttribute("value") || panel.getAttribute("name");
+  const named = tabs.find((tab) => (labelledBy && tab.getAttribute("id") === labelledBy)
+    || (id && (tab.getAttribute("aria-controls") === id || tab.getAttribute("href") === `#${id}`))
+    || (value && (tab.getAttribute("value") === value || tab.getAttribute("panel") === value)));
+  if (named) return named;
+  // Docusaurus ties them by position alone.
+  const parent = panel.parentElement;
+  const index = Array.from(parent?.children || []).filter((el) => el.matches?.(TAB_PANELS)).indexOf(panel);
+  for (let node = parent, depth = 0; node && depth < 4; node = node.parentElement, depth += 1) {
+    const list = node.querySelector("[role='tablist']");
+    if (list) return list.querySelectorAll("[role='tab']")[index];
+  }
+  return null;
+}
+
+/**
+ * Tabs read as one document: inactive panels are shown, each under its tab's
+ * name, in place of a tab strip that points nowhere. A long inactive panel is
+ * left out, and named in stats.omittedTabs.
+ */
+function standardizeTabPanels(doc, stats) {
+  // Docker's docs write aria-role, and call a panel a tab: only the strip's is right.
+  for (const list of Array.from(doc.querySelectorAll("[aria-role='tablist']"))) {
+    list.setAttribute("role", "tablist");
+    for (const tab of Array.from(list.children)) tab.setAttribute("role", "tab");
+    for (const panel of Array.from(list.nextElementSibling?.children || [])) panel.setAttribute("role", "tabpanel");
+  }
+  const panels = Array.from(doc.querySelectorAll(TAB_PANELS));
+  if (!panels.length) return;
+  const tabs = Array.from(doc.querySelectorAll(TABS));
+  for (const panel of panels) {
+    const label = (tabOf(panel, tabs)?.textContent || "").replace(/\s+/g, " ").trim().slice(0, MAX_TAB_LABEL_CHARS);
+    const text = (panel.textContent || "").replace(/\s+/g, " ").trim();
+    if (isInactivePanel(panel)) {
+      if (text.length > MAX_INACTIVE_PANEL_CHARS) {
+        stats.omittedTabs.push(label);
+        panel.remove();
+        continue;
+      }
+      for (const attr of ["hidden", "aria-hidden", "style"]) panel.removeAttribute(attr);
+      panel.classList.remove("hidden");
+    }
+    if (label && panels.length > 1 && !text.toLowerCase().startsWith(label.toLowerCase())) {
+      const heading = doc.createElement("p");
+      heading.appendChild(doc.createElement("strong")).textContent = label;
+      panel.insertBefore(heading, panel.firstChild);
+    }
+  }
+  for (const el of Array.from(doc.querySelectorAll(`[role='tablist'], ${TABS}`))) el.remove();
+}
+
+function standardizeAsides(doc) {
+  for (const aside of Array.from(doc.querySelectorAll("main aside, article aside, [role='main'] aside"))) {
+    const name = `${aside.getAttribute("class") || ""} ${aside.getAttribute("aria-label") || ""} ${aside.getAttribute("role") || ""}`;
+    if (!NOTE_ASIDE.test(name) || OTHER_ASIDE.test(name)) continue;
+    const quote = doc.createElement("blockquote");
+    quote.append(...Array.from(aside.childNodes));
+    aside.replaceWith(quote);
+  }
+}
+
 /**
  * @param {Document} doc the inert clone (mutated in place)
  * @param {{ url?: string }} [options]
@@ -145,7 +275,10 @@ export function standardizeDocument(doc, options = {}) {
     preWrapped: 0,
     headingWidgetsRemoved: 0,
     headingsUnwrapped: 0,
+    omittedTabs: /** @type {string[]} */ ([]),
   };
+  standardizeTabPanels(doc, stats);
+  standardizeAsides(doc);
   standardizeHeadings(doc, stats);
   standardizeCode(doc, stats);
   standardizeImages(doc, base, stats);

package/src/types.js

@@ -0,0 +1,122 @@
+/// <reference lib="dom" preserve="true" />
+// Types only: nothing here exists at runtime. `npm run build:types` turns these
+// and the JSDoc in the other modules into dist/types, which is what TypeScript
+// consumers see.
+export {};
+
+/** @typedef {"full"|"no_links"|"text_only"} OutputMode */
+
+/**
+ * @typedef {object} SanitizeOptions
+ * @property {boolean} [stripScripts]
+ * @property {boolean} [stripStyles]
+ * @property {boolean} [stripNoscript]
+ * @property {boolean} [stripMeta]
+ * @property {boolean} [stripLinkTags]
+ * @property {boolean} [stripAds]
+ * @property {readonly string[]} [extraStripSelectors] app-specific overlays or widgets to drop, e.g. "[data-app-overlay]"
+ * @property {number} [shadowNodeLimit]
+ * @property {number} [shadowCharLimit]
+ * @property {number} [shadowDepthLimit]
+ */
+
+/** @typedef {NonNullable<ConstructorParameters<typeof import("@mozilla/readability").Readability<string>>[1]>} ReadabilityOptions */
+
+/** @typedef {import("./extract.js").Extraction} Extraction */
+
+/** @typedef {import("./extract.js").Extractor} Extractor */
+
+/** @typedef {ReturnType<typeof import("./metadata.js").collectMetadata>} CollectedMetadata */
+
+/**
+ * @typedef {object} ExtractedMetadata
+ * @property {string} [headline] the heading chosen for the capture
+ * @property {string} [extractedTitle] the title the extractor reported
+ * @property {string} [byline] the byline the extractor reported
+ */
+
+/** @typedef {CollectedMetadata & ExtractedMetadata} PageMarkdownMetadata */
+
+/**
+ * @typedef {object} PageMarkdownExtraction
+ * @property {Extraction["method"]} method
+ * @property {string} [fallbackReason] why extraction fell back to the full page or the main content
+ * @property {true} [leadRecovered] a dropped lead paragraph was restored
+ * @property {number} [pageFacts] how many facts `pageFacts` appended
+ */
+
+/** @typedef {"tag"|"variation_selector"|"invisible_operator"|"reserved_tag_plane"} HiddenRunKind */
+
+/**
+ * A run as text-sanctifier's findHiddenRuns reports it.
+ * @typedef {object} HiddenRun
+ * @property {HiddenRunKind} kind
+ * @property {number} offset UTF-16 index of the first character
+ * @property {number} end UTF-16 index just past the last character
+ * @property {number} count code points in the run
+ * @property {number} line 1-based line of the first character
+ * @property {number} column 1-based column of the first character, in UTF-16 units
+ * @property {string} [decoded] tag runs only: the ASCII the tags spell
+ * @property {boolean} [decodedTruncated] tag runs only: true when `decoded` stops short of the run
+ */
+
+/**
+ * @typedef {object} HiddenTextOmission
+ * @property {`hidden_${HiddenRunKind}`} kind
+ * @property {number} count runs removed from `markdown`
+ * @property {number} chars code points in those runs
+ * @property {string} note
+ */
+
+/**
+ * Check `kind` before reading `chars`: only hidden-text omissions have it.
+ * @typedef {{ kind: "shadow_content", count: number, note: string }
+ *   | { kind: "json_line", count: number, note: string }
+ *   | { kind: "inactive_tabs", count: number, note: string }
+ *   | { kind: "comments", count: number, note: string }
+ *   | HiddenTextOmission} PageMarkdownOmission
+ */
+
+/** @typedef {ReturnType<typeof import("./standardize.js").standardizeDocument>} StandardizationStats */
+
+/**
+ * @typedef {object} PageMarkdownStats
+ * @property {OutputMode} mode
+ * @property {number} extractedHtmlChars
+ * @property {number} convertedChars
+ * @property {number} markdownChars
+ * @property {number} pageTextChars the page's text at capture time: a capture is a
+ *   snapshot of the DOM, so a caller can re-capture until this number settles
+ * @property {StandardizationStats} standardization
+ */
+
+/**
+ * @typedef {object} PageMarkdownResult
+ * @property {string} version the package version that made this result
+ * @property {string} markdown
+ * @property {PageMarkdownMetadata} metadata
+ * @property {PageMarkdownExtraction} extraction
+ * @property {PageMarkdownOmission[]} omissions
+ * @property {string[]} warnings
+ * @property {PageMarkdownStats} stats
+ * @property {string} [fullPageMarkdown] present only with `includeFullPage`
+ */
+
+/** @typedef {{ label: string, value: string }} PageFact */
+
+/**
+ * @typedef {string | string[] | ((node: HTMLElement, options: Record<string, unknown>) => boolean)} TurndownFilter
+ */
+
+/**
+ * Turndown's public methods; Turndown ships no types of its own.
+ * @typedef {object} TurndownInstance
+ * @property {(input: string | Node) => string} turndown
+ * @property {(plugin: TurndownPlugin | TurndownPlugin[]) => TurndownInstance} use
+ * @property {(key: string, rule: { filter: TurndownFilter, replacement: (content: string, node: Node, options: Record<string, unknown>) => string }) => TurndownInstance} addRule
+ * @property {(filter: TurndownFilter) => TurndownInstance} keep
+ * @property {(filter: TurndownFilter) => TurndownInstance} remove
+ * @property {(text: string) => string} escape
+ */
+
+/** @typedef {(service: TurndownInstance) => void} TurndownPlugin */