page-markdown 0.0.0-stage → 0.1.3

package/src/refine.js

@@ -0,0 +1,104 @@
+// Corrections after extraction, using the standardized clone and the page's
+// metadata: restore a lead paragraph the extractor dropped, and choose a clean
+// title heading.
+
+const MIN_LEAD_CHARS = 40;
+const MATCH_CHARS = 60;
+const SEPARATORS = [" | ", " – ", " — ", " - ", " · ", " :: ", " » "];
+
+/** Lowercase, collapse whitespace, drop a trailing ellipsis. */
+function norm(text) {
+  return String(text || "").replace(/\s+/g, " ").replace(/(\.\.\.|…)\s*$/, "").trim().toLowerCase();
+}
+
+function textOf(html) {
+  return norm(String(html || "").replace(/<[^>]*>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&"));
+}
+
+function startsAlike(a, b) {
+  const n = Math.min(MATCH_CHARS, a.length, b.length);
+  return n >= MIN_LEAD_CHARS && a.slice(0, n) === b.slice(0, n);
+}
+
+/**
+ * Many sites state their summary as both the meta description and the first
+ * paragraph. Extractors can drop that paragraph when it sits in a section
+ * without a heading (MDN). If the description matches a paragraph in the page
+ * that the extracted HTML lacks, put that paragraph first.
+ *
+ * @param {string} html extracted HTML
+ * @param {Document} doc standardized clone
+ * @param {string} description meta description
+ * @returns {{ html: string, recovered: boolean }}
+ */
+export function recoverLeadParagraph(html, doc, description) {
+  const lead = norm(description);
+  if (lead.length < MIN_LEAD_CHARS) return { html, recovered: false };
+  const extracted = textOf(html);
+  const paragraphs = Array.from(doc.querySelectorAll?.("p") || []).slice(0, 300);
+  for (const p of paragraphs) {
+    const text = norm(p.textContent);
+    if (text.length < MIN_LEAD_CHARS || !startsAlike(text, lead)) continue;
+    if (extracted.includes(text.slice(0, 80))) return { html, recovered: false };
+    return { html: `${p.outerHTML}\n${html}`, recovered: true };
+  }
+  return { html, recovered: false };
+}
+
+function cleanHeading(text) {
+  return String(text || "")
+    .replace(/\s+/g, " ")
+    .replace(/\s*[¶#§]\s*$/, "")
+    .trim();
+}
+
+function firstHeading(doc) {
+  const scoped = doc.querySelector?.("main h1, article h1, [role='main'] h1");
+  return cleanHeading((scoped || doc.querySelector?.("h1"))?.textContent);
+}
+
+// Drops site-name segments at either end: "Page | Site", "Site - Page".
+function stripSiteSuffix(title, siteName) {
+  let result = String(title || "").trim();
+  const site = norm(siteName);
+  if (!site) return result;
+  const isSite = (segment) => {
+    const value = norm(segment);
+    return Boolean(value) && (value.includes(site) || site.includes(value));
+  };
+  for (let changed = true; changed;) {
+    changed = false;
+    for (const separator of SEPARATORS) {
+      const last = result.lastIndexOf(separator);
+      if (last > 0 && isSite(result.slice(last + separator.length))) {
+        result = result.slice(0, last).trim();
+        changed = true;
+      }
+      const first = result.indexOf(separator);
+      if (first > 0 && isSite(result.slice(0, first))) {
+        result = result.slice(first + separator.length).trim();
+        changed = true;
+      }
+    }
+  }
+  return result;
+}
+
+/**
+ * The heading for the capture: the page's own H1 when the document title or
+ * og:title contains it (and it is not just the site name); otherwise og:title;
+ * otherwise the title without a trailing site-name segment.
+ *
+ * @param {Document} doc standardized clone
+ * @param {{ documentTitle?: string, title?: string, siteName?: string, extractedTitle?: string }} meta
+ */
+export function chooseHeadline(doc, meta = {}) {
+  const h1 = firstHeading(doc);
+  const titles = [meta.documentTitle, meta.title, meta.extractedTitle].map(norm).filter(Boolean);
+  if (h1 && h1.length <= 200 && norm(h1) !== norm(meta.siteName) && titles.some((t) => t.includes(norm(h1)))) {
+    return h1;
+  }
+  const og = String(meta.title || "").trim();
+  if (og && og !== String(meta.documentTitle || "").trim()) return stripSiteSuffix(og, meta.siteName);
+  return stripSiteSuffix(meta.extractedTitle || meta.documentTitle || og, meta.siteName);
+}

package/src/sanitize.js

@@ -0,0 +1,193 @@
+// Stage 1: clone the page into an inert document, inline open shadow roots
+// within limits, and strip non-content. The live page is never modified.
+
+export const DEFAULT_SANITIZE_OPTIONS = Object.freeze({
+  stripScripts: true,
+  stripStyles: true,
+  stripNoscript: true,
+  stripMeta: true,
+  stripLinkTags: true,
+  stripAds: true,
+  // App-specific overlays or widgets to drop, e.g. "[data-app-overlay]".
+  extraStripSelectors: [],
+  shadowNodeLimit: 5000,
+  shadowCharLimit: 500000,
+  shadowDepthLimit: 128,
+});
+
+const STRIP_RULES = {
+  stripScripts: "script",
+  stripStyles: "style",
+  stripNoscript: "noscript",
+  stripMeta: "meta",
+  stripLinkTags: "link",
+};
+
+// Advertising placements: explicit ad markup, plus custom elements named for ad
+// slots (e.g. MDN's <mdn-placement-sidebar>).
+const AD_SELECTOR = [
+  "ins.adsbygoogle", ".adsbygoogle", ".advertisement",
+  "[data-ad-slot]", "[data-ad-client]", "[data-ad-unit]",
+  "[aria-label='Advertisement' i]", "[aria-label='Advertisements' i]", "[aria-label='Sponsored' i]",
+  "iframe[src*='doubleclick.net']", "iframe[src*='googlesyndication.com']", "iframe[id^='google_ads_iframe']",
+].join(", ");
+const AD_ELEMENT_NAME = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*-(?:placement|advert|advertisement|ads?|sponsor|sponsored)(?:-[a-z0-9]+)*$/;
+
+/** @param {any} node */
+export function isAdvertisementElement(node) {
+  if (node?.nodeType !== 1) return false;
+  if (AD_ELEMENT_NAME.test(node.localName || "")) return true;
+  try {
+    return Boolean(node.matches?.(AD_SELECTOR));
+  } catch {
+    // DOM implementations without case-insensitive attribute selectors.
+    return Boolean(node.matches?.(AD_SELECTOR.replace(/ i\]/g, "]")));
+  }
+}
+
+function stripSelectors(options) {
+  const selectors = Object.entries(STRIP_RULES)
+    .filter(([flag]) => options[flag])
+    .map(([, selector]) => selector);
+  for (const extra of options.extraStripSelectors || []) {
+    if (typeof extra === "string" && extra.trim()) selectors.push(extra.trim());
+  }
+  return selectors;
+}
+
+function isStripped(node, selectors, options) {
+  if (node.nodeType !== 1) return false;
+  if (selectors.some((selector) => node.matches?.(selector))) return true;
+  return Boolean(options.stripAds && isAdvertisementElement(node));
+}
+
+/**
+ * Creates an empty HTML document unconnected to the page: no scripts run and
+ * no custom elements upgrade in it.
+ * @param {Document} doc
+ */
+export function createInertDocument(doc) {
+  const impl = doc?.implementation;
+  if (typeof impl?.createHTMLDocument === "function") return impl.createHTMLDocument("");
+  const Parser = doc?.defaultView?.DOMParser || globalThis.DOMParser;
+  if (typeof Parser === "function") {
+    return new Parser().parseFromString("<!doctype html><html><head></head><body></body></html>", "text/html");
+  }
+  throw new Error("page-markdown: cannot create an inert document (no createHTMLDocument or DOMParser)");
+}
+
+const SHADOW_LIMIT = Symbol("shadow limit");
+
+function positiveLimit(value, name) {
+  if (!Number.isSafeInteger(value) || value < 1) {
+    throw new RangeError(`page-markdown: ${name} must be a positive integer`);
+  }
+  return value;
+}
+
+/**
+ * Clones `root` (a Document or Element) into a new inert document.
+ * Open shadow roots replace their host's light children (native slots supply
+ * assigned children). Limits apply only to shadow content; a host over the
+ * limit is emptied and counted as an omission.
+ *
+ * @param {Document|Element} root
+ * @param {Partial<typeof DEFAULT_SANITIZE_OPTIONS>} [options]
+ * @returns {{ document: Document, omissions: { shadowHosts: number } }}
+ */
+export function sanitizeToInertDocument(root, options = {}) {
+  const opts = { ...DEFAULT_SANITIZE_OPTIONS, ...options };
+  const limits = {
+    nodes: positiveLimit(opts.shadowNodeLimit, "shadowNodeLimit"),
+    chars: positiveLimit(opts.shadowCharLimit, "shadowCharLimit"),
+    depth: positiveLimit(opts.shadowDepthLimit, "shadowDepthLimit"),
+  };
+  const selectors = stripSelectors(opts);
+  const sourceDoc = root.nodeType === 9 ? root : root.ownerDocument;
+  const source = root.nodeType === 9 ? root.documentElement : root;
+  const inert = createInertDocument(sourceDoc);
+
+  function shallow(node) {
+    if (node.nodeType !== 1) return inert.importNode(node, false);
+    const name = node.prefix ? `${node.prefix}:${node.localName}` : node.localName;
+    const copy = node.namespaceURI
+      ? inert.createElementNS(node.namespaceURI, name)
+      : inert.createElement(name);
+    for (const attr of Array.from(node.attributes || [])) {
+      // HTML allows colon-bearing names without a namespace (framework
+      // directives); setAttributeNS rejects those.
+      if (attr.namespaceURI) copy.setAttributeNS(attr.namespaceURI, attr.name, attr.value);
+      else copy.setAttribute(attr.name, attr.value);
+    }
+    return copy;
+  }
+
+  let nodes = 0;
+  let chars = 0;
+  let shadowHosts = 0;
+
+  function isAssigningSlot(node) {
+    return node.nodeType === 1 && node.localName === "slot"
+      && typeof node.assignedNodes === "function" && Boolean(node.getRootNode?.().host);
+  }
+
+  function shadowNode(node, depth) {
+    if (++nodes > limits.nodes || depth > limits.depth) throw SHADOW_LIMIT;
+    if (isStripped(node, selectors, opts)) return inert.createDocumentFragment();
+    const copy = shallow(node);
+    chars += node.nodeType === 1 ? (copy.outerHTML || "").length : (node.nodeValue || "").length * 6;
+    if (chars > limits.chars) throw SHADOW_LIMIT;
+    let children = node.childNodes;
+    let target = copy;
+    if (isAssigningSlot(node)) {
+      const assigned = node.assignedNodes({ flatten: true });
+      children = assigned.length ? assigned : node.childNodes;
+      target = inert.createDocumentFragment();
+    } else if (node.shadowRoot) {
+      children = node.shadowRoot.childNodes;
+    } else if (node.nodeType === 1 && node.localName === "template" && node.content) {
+      children = node.content.childNodes;
+      target = copy.content || copy;
+    }
+    for (const child of Array.from(children || [])) target.appendChild(shadowNode(child, depth + 1));
+    return isAssigningSlot(node) ? target : copy;
+  }
+
+  const clone = shallow(source);
+  // Iterative light-DOM walk: ordinary pages get no depth limit.
+  const pending = [[source, clone]];
+  while (pending.length) {
+    const [live, copy] = pending.pop();
+    if (live.shadowRoot) {
+      const fragment = inert.createDocumentFragment();
+      try {
+        for (const child of Array.from(live.shadowRoot.childNodes)) fragment.appendChild(shadowNode(child, 1));
+        while (copy.firstChild) copy.removeChild(copy.firstChild);
+        copy.appendChild(fragment);
+      } catch (error) {
+        if (error !== SHADOW_LIMIT) throw error;
+        while (copy.firstChild) copy.removeChild(copy.firstChild);
+        shadowHosts += 1;
+      }
+      continue;
+    }
+    const children = live.content?.childNodes || live.childNodes;
+    const target = copy.content || copy;
+    const pairs = [];
+    for (const child of Array.from(children || [])) {
+      if (isStripped(child, selectors, opts)) continue;
+      const childCopy = shallow(child);
+      target.appendChild(childCopy);
+      pairs.push([child, childCopy]);
+    }
+    for (let i = pairs.length - 1; i >= 0; i -= 1) pending.push(pairs[i]);
+  }
+
+  if (clone.localName === "html") {
+    inert.removeChild(inert.documentElement);
+    inert.appendChild(clone);
+  } else {
+    inert.body.appendChild(clone);
+  }
+  return { document: inert, omissions: { shadowHosts } };
+}

package/src/standardize.js

@@ -0,0 +1,154 @@
+// Stage 2: normalise markup on the inert clone so extraction and conversion
+// see consistent structure. Runs before extraction.
+
+const LAZY_SRC_ATTRS = ["data-src", "data-lazy-src", "data-original", "data-lazy", "data-url"];
+const LANGUAGE_CLASS = [
+  /(?:^|\s)(?:language|lang)-([\w+#.-]+)/i,
+  /(?:^|\s)highlight-(?:source-)?([\w+#.-]+)/i,
+  /(?:^|\s)mw-highlight-lang-([\w+#.-]+)/i,
+  /(?:^|\s)brush:\s*([\w+#.-]+)/i,
+];
+// Only normalise names that are clearly the same language; keep site names otherwise.
+const LANGUAGE_ALIASES = { python3: "python", py3: "python" };
+const NON_LANGUAGES = new Set(["default", "none", "text", "plain", "plaintext", "nohighlight", "notranslate"]);
+// Wrappers sites put around a heading next to "edit" or permalink widgets.
+const EDIT_WIDGETS = ".mw-editsection, .headerlink, .anchor, .heading-anchor, a.anchor, [aria-hidden='true'] a";
+// Content without text that a heading wrapper may still hold.
+const MEDIA = "img, picture, video, audio";
+
+function resolveUrl(value, base) {
+  const raw = String(value || "").trim();
+  if (!raw || raw.startsWith("#") || /^(mailto|tel|data):/i.test(raw)) return raw;
+  try {
+    return base ? new URL(raw, base).href : raw;
+  } catch {
+    return raw;
+  }
+}
+
+function languageFrom(el, maxDepth = 3) {
+  for (let node = el, depth = 0; node && node.nodeType === 1 && depth <= maxDepth; node = node.parentElement, depth += 1) {
+    const cls = node.getAttribute?.("class") || "";
+    for (const pattern of LANGUAGE_CLASS) {
+      const match = cls.match(pattern);
+      if (match) {
+        const lang = match[1].toLowerCase();
+        if (!NON_LANGUAGES.has(lang)) return LANGUAGE_ALIASES[lang] || lang;
+      }
+    }
+    const dataLang = node.getAttribute?.("data-lang") || node.getAttribute?.("data-language");
+    if (dataLang && !NON_LANGUAGES.has(dataLang.toLowerCase())) return dataLang.toLowerCase();
+  }
+  return "";
+}
+
+function hasMeaningfulContent(el) {
+  if ((el.textContent || "").trim()) return true;
+  return Boolean(el.querySelector?.("img[alt]:not([alt='']), svg title"));
+}
+
+/**
+ * Lazy images: copy a lazy-load attribute into src when src is missing or a
+ * data: placeholder. Then drop images that are still inline data.
+ */
+function standardizeImages(doc, base, stats) {
+  for (const img of Array.from(doc.querySelectorAll("img"))) {
+    const src = img.getAttribute("src") || "";
+    if (!src || src.startsWith("data:")) {
+      const lazy = LAZY_SRC_ATTRS.map((attr) => img.getAttribute(attr)).find((v) => v && !v.startsWith("data:"));
+      if (lazy) {
+        img.setAttribute("src", lazy);
+        stats.lazyImagesResolved += 1;
+      }
+    }
+    const current = img.getAttribute("src") || "";
+    if (!current || current.startsWith("data:")) {
+      img.remove();
+      stats.inlineImagesDropped += 1;
+      continue;
+    }
+    img.setAttribute("src", resolveUrl(current, base));
+  }
+}
+
+function standardizeLinks(doc, base, stats) {
+  for (const a of Array.from(doc.querySelectorAll("a"))) {
+    const href = a.getAttribute("href") || "";
+    if (/^\s*javascript:/i.test(href)) {
+      a.replaceWith(...Array.from(a.childNodes));
+      stats.scriptLinksUnwrapped += 1;
+      continue;
+    }
+    if (!hasMeaningfulContent(a)) {
+      a.remove();
+      stats.emptyLinksRemoved += 1;
+      continue;
+    }
+    if (href) a.setAttribute("href", resolveUrl(href, base));
+  }
+}
+
+/** <pre> without <code> (and code without a language) gets a language-tagged <code>. */
+function standardizeCode(doc, stats) {
+  for (const pre of Array.from(doc.querySelectorAll("pre"))) {
+    const lang = languageFrom(pre);
+    let code = Array.from(pre.children || []).length === 1 && pre.firstElementChild?.localName === "code"
+      ? pre.firstElementChild
+      : null;
+    if (!code) {
+      code = doc.createElement("code");
+      code.textContent = pre.textContent || "";
+      while (pre.firstChild) pre.removeChild(pre.firstChild);
+      pre.appendChild(code);
+      stats.preWrapped += 1;
+    }
+    if (lang && !/(?:^|\s)language-/.test(code.getAttribute("class") || "")) {
+      code.setAttribute("class", `${code.getAttribute("class") || ""} language-${lang}`.trim());
+    }
+  }
+}
+
+/**
+ * Headings wrapped with edit/permalink widgets (Wikipedia's div.mw-heading,
+ * Sphinx headerlinks): drop the widgets, then replace a wrapper whose only
+ * remaining element is the heading with the heading itself.
+ */
+function standardizeHeadings(doc, stats) {
+  for (const widget of Array.from(doc.querySelectorAll(EDIT_WIDGETS))) {
+    if (!(widget.textContent || "").trim() || widget.matches?.(".mw-editsection")) {
+      widget.remove();
+      stats.headingWidgetsRemoved += 1;
+    }
+  }
+  for (const heading of Array.from(doc.querySelectorAll("h1, h2, h3, h4, h5, h6"))) {
+    const parent = heading.parentElement;
+    if (!parent || parent.localName !== "div") continue;
+    const others = Array.from(parent.children).filter((el) => el !== heading);
+    if (others.every((el) => !(el.textContent || "").trim() && !el.matches?.(MEDIA) && !el.querySelector?.(MEDIA))) {
+      parent.replaceWith(heading);
+      stats.headingsUnwrapped += 1;
+    }
+  }
+}
+
+/**
+ * @param {Document} doc the inert clone (mutated in place)
+ * @param {{ url?: string }} [options]
+ */
+export function standardizeDocument(doc, options = {}) {
+  const base = options.url || "";
+  const stats = {
+    lazyImagesResolved: 0,
+    inlineImagesDropped: 0,
+    scriptLinksUnwrapped: 0,
+    emptyLinksRemoved: 0,
+    preWrapped: 0,
+    headingWidgetsRemoved: 0,
+    headingsUnwrapped: 0,
+  };
+  standardizeHeadings(doc, stats);
+  standardizeCode(doc, stats);
+  standardizeImages(doc, base, stats);
+  standardizeLinks(doc, base, stats);
+  return stats;
+}