guidelinescraper 1.0.0

package/README.md

@@ -0,0 +1,97 @@
+# Frontify Guideline Scraper
+
+Scrape a Frontify brand portal and save every guideline page as a PDF and clean semantic HTML.
+
+## How it works
+
+1. **Discover** — Queries Frontify's portal and document navigation APIs to build the full site tree (documents, pages, groups, headings, external links).
+2. **Crawl** — Visits every page with Playwright, expands accordions, forces lazy images to load, dismisses cookie/overlay dialogs, then saves a PDF and raw HTML snapshot.
+3. **Clean** — Strips the raw HTML down to semantic content (headings, text, images, tables) with no scripts, styles, or navigation chrome.
+
+## Setup
+
+```bash
+npm install
+npx playwright install chromium
+```
+
+## Usage
+
+```bash
+node crawl.mjs --url brand.uber.com
+```
+
+Or pass a full URL:
+
+```bash
+node crawl.mjs --url https://developer.frontify.com
+```
+
+### Options
+
+| Flag | Short | Description |
+|------|-------|-------------|
+| `--url <url>` | `-u` | Portal domain or full URL |
+| `--hub <id>` | `-h` | Hub ID (auto-detected if omitted) |
+| `--cookie <str>` | `-c` | Cookie header for authenticated portals |
+| `--help` | | Show help |
+
+These can also be set via environment variables or a `.env` file:
+
+```
+URL=brand.uber.com
+HUB_ID=25
+COOKIE=frontify-session-id=your-session-id
+```
+
+## Output
+
+```
+output/{domain}/
+  pdf/
+    Group Name/
+      Document Title.pdf
+      Document Title/
+        Page Title.pdf
+  html/
+    Group Name/
+      Document Title.html
+      ...
+```
+
+- **PDF** — Full-page A4 captures with background graphics, expanded accordions, and loaded lazy images.
+- **HTML** — Cleaned semantic HTML: headings, paragraphs, images, tables. No scripts, styles, classes, or navigation elements. Wrapped in minimal readable CSS.
+
+## Discover only
+
+Run the discovery step standalone to inspect or save the navigation tree:
+
+```bash
+node discover.mjs --url brand.uber.com --output brand.uber.com.json
+```
+
+This outputs a JSON tree of the portal's structure without crawling any pages.
+
+## Clean HTML only
+
+Re-clean previously scraped raw HTML:
+
+```bash
+node purge-html.mjs output/.raw/html output/clean
+```
+
+## Authenticated portals
+
+For portals that require login, grab your session cookie from browser dev tools and pass it:
+
+```bash
+node crawl.mjs --url brand.uber.com --cookie "frontify-session-id=your-session-id"
+```
+
+Or add it to `.env`:
+
+```
+COOKIE=frontify-session-id=your-session-id
+```
+
+See `.env.example` for reference.

package/crawl.mjs

@@ -0,0 +1,416 @@
+#!/usr/bin/env node
+import { PlaywrightCrawler } from "crawlee";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { parseArgs } from "node:util";
+import { discover } from "./discover.mjs";
+import { purge } from "./purge-html.mjs";
+
+process.env.PLAYWRIGHT_BROWSERS_PATH = path.join(
+  os.homedir(),
+  "Library",
+  "Caches",
+  "ms-playwright",
+);
+
+// ── CLI args ──
+const { values, positionals } = parseArgs({
+  options: {
+    url: { type: "string", short: "u" },
+    hub: { type: "string", short: "h" },
+    cookie: { type: "string", short: "c" },
+    help: { type: "boolean" },
+  },
+  allowPositionals: true,
+  strict: false,
+});
+
+if (values.help) {
+  console.log(`Usage: node crawl.mjs [options] [url]
+
+Options:
+  -u, --url <url>       Portal domain or URL
+  -h, --hub <id>        Hub ID (auto-detected if omitted)
+  -c, --cookie <str>    Cookie header for authenticated requests
+      --help            Show this help
+
+Environment variables (or .env file):
+  COOKIE                Cookie header value
+  URL                   Portal domain or URL
+  HUB_ID                Hub ID`);
+  process.exit(0);
+}
+
+const inputUrl = values.url || positionals[0] || process.env.URL;
+const hubId = values.hub || process.env.HUB_ID;
+const cookie = values.cookie || process.env.COOKIE || "";
+
+const OUTPUT_DIR = "output";
+
+// ── Phase 1: Discover navigation tree ──
+console.log("Discovering portal navigation…\n");
+const siteTree = await discover({ url: inputUrl, hubId, cookie });
+
+function sanitize(name) {
+  return name.replace(/[<>:"/\\|?*]+/g, "_").trim();
+}
+
+function collectPages(nodes, parentDir) {
+  const pages = [];
+
+  for (const node of nodes) {
+    if (node.type === "heading") continue;
+    if (node.type === "external_link") continue;
+
+    const name = sanitize(node.title);
+
+    if (node.type === "group" || node.type === "category") {
+      const groupDir = path.join(parentDir, name);
+      if (node.children) {
+        pages.push(...collectPages(node.children, groupDir));
+      }
+      continue;
+    }
+
+    if (node.url) {
+      const hasPageChildren =
+        node.children &&
+        node.children.some(
+          (c) =>
+            c.type === "page" ||
+            c.type === "document" ||
+            c.type === "group" ||
+            c.type === "category",
+        );
+
+      if (hasPageChildren) {
+        const dir = path.join(parentDir, name);
+        pages.push(...collectPages(node.children, dir));
+      } else {
+        pages.push({
+          url: node.url,
+          pdfPath: path.join(parentDir, `${name}.pdf`),
+        });
+      }
+    }
+  }
+
+  return pages;
+}
+
+const domainDir = path.join(OUTPUT_DIR, sanitize(siteTree.domain));
+const pages = collectPages(siteTree.children, path.join(domainDir, "pdf"));
+
+const urlToPdf = new Map(pages.map((p) => [p.url, p.pdfPath]));
+
+console.log(`Discovered ${pages.length} pages to crawl.\n`);
+for (const { url, pdfPath } of pages) {
+  console.log(`  ${pdfPath}  ←  ${url}`);
+}
+console.log();
+
+for (const { pdfPath } of pages) {
+  fs.mkdirSync(path.dirname(pdfPath), { recursive: true });
+  const htmlDir = path.dirname(pdfPath.replace(/\/pdf\//, "/html/"));
+  fs.mkdirSync(htmlDir, { recursive: true });
+}
+
+// ── Phase 2: Crawl pages ──
+const cookieConsentValue =
+  '{"isCookieConsentOpen":false,"preferencesScriptsEnabled":true,"statisticsScriptsEnabled":true}';
+
+const crawler = new PlaywrightCrawler({
+  headless: true,
+  launchContext: {
+    launchOptions: { args: ["--disable-dev-shm-usage"] },
+  },
+  maxConcurrency: 16,
+  maxRequestRetries: 2,
+  navigationTimeoutSecs: 120,
+
+  preNavigationHooks: [
+    async ({ page, request }) => {
+      const reqUrl = new URL(request.url);
+      const cookies = [
+        {
+          name: "cookieConsentUserData",
+          value: cookieConsentValue,
+          domain: reqUrl.hostname,
+          path: "/",
+        },
+      ];
+
+      if (cookie) {
+        for (const pair of cookie.split(";")) {
+          const eq = pair.indexOf("=");
+          if (eq === -1) continue;
+          cookies.push({
+            name: pair.slice(0, eq).trim(),
+            value: pair.slice(eq + 1).trim(),
+            domain: reqUrl.hostname,
+            path: "/",
+          });
+        }
+      }
+
+      await page.context().addCookies(cookies);
+    },
+  ],
+
+  async requestHandler({ page, request, log }) {
+    const pdfPath = urlToPdf.get(request.url);
+    if (!pdfPath) {
+      log.warning(`No PDF path mapped for ${request.url}, skipping.`);
+      return;
+    }
+
+    log.info(`Saving ${request.url} → ${pdfPath}`);
+
+    await page
+      .waitForLoadState("networkidle", { timeout: 30_000 })
+      .catch(() => {});
+
+    // Expand all collapsible / accordion content
+    await page.evaluate(async () => {
+      for (const el of document.querySelectorAll("details")) {
+        el.setAttribute("open", "");
+      }
+
+      const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+
+      const skipAncestors =
+        "nav, header, [role='navigation'], [role='search'], [class*='search'], [class*='nav'], [class*='header'], [class*='modal'], [class*='overlay']";
+
+      const triggerSelectors = [
+        '[data-state="closed"]',
+        ".accordion-trigger",
+        ".accordion-header",
+        ".accordion-toggle",
+        'button[class*="accordion"]',
+        'button[class*="collapse"]',
+        'button[class*="expand"]',
+        'div[class*="accordion"] > button',
+        'div[class*="accordion"] > [role="button"]',
+        'main [aria-expanded="false"]',
+        'article [aria-expanded="false"]',
+        '[role="main"] [aria-expanded="false"]',
+        '.content [aria-expanded="false"]',
+      ];
+
+      for (const selector of triggerSelectors) {
+        for (const trigger of document.querySelectorAll(selector)) {
+          if (trigger.closest(skipAncestors)) continue;
+          trigger.click();
+          await delay(100);
+        }
+      }
+
+      const panelSelectors = [
+        '[data-state="closed"]',
+        ".accordion-content",
+        ".collapse:not(.show)",
+        '[class*="accordion-panel"]',
+      ];
+      for (const el of document.querySelectorAll(panelSelectors.join(", "))) {
+        if (el.closest(skipAncestors)) continue;
+        el.style.setProperty("display", "block", "important");
+        el.style.setProperty("height", "auto", "important");
+        el.style.setProperty("max-height", "none", "important");
+        el.style.setProperty("overflow", "visible", "important");
+        el.style.setProperty("opacity", "1", "important");
+        el.style.setProperty("visibility", "visible", "important");
+        el.removeAttribute("aria-hidden");
+      }
+    });
+
+    await page.evaluate(() => {
+      for (const el of document.querySelectorAll(
+        '[class*="overlay"], [class*="modal"], [class*="search"][class*="open"], [role="dialog"], [class*="underlay"]',
+      )) {
+        el.style.setProperty("display", "none", "important");
+      }
+    });
+
+    await page.waitForTimeout(1000);
+
+    await page.evaluate(() => {
+      const lazyAttrs = ["data-src", "data-lazy-src", "data-original"];
+      const lazySrcsetAttrs = ["data-srcset", "data-lazy-srcset"];
+      const realSrcMap = new WeakMap();
+
+      for (const img of document.querySelectorAll("img")) {
+        img.removeAttribute("loading");
+        img.removeAttribute("decoding");
+
+        for (const attr of lazyAttrs) {
+          const val = img.getAttribute(attr);
+          if (val) {
+            img.src = val;
+            realSrcMap.set(img, val);
+            img.removeAttribute(attr);
+          }
+        }
+        for (const attr of lazySrcsetAttrs) {
+          const val = img.getAttribute(attr);
+          if (val) {
+            img.srcset = val;
+            img.removeAttribute(attr);
+          }
+        }
+      }
+
+      const observer = new MutationObserver((mutations) => {
+        for (const m of mutations) {
+          if (m.type === "attributes" && m.attributeName === "src") {
+            const img = m.target;
+            const saved = realSrcMap.get(img);
+            if (saved && img.src !== saved && img.src.startsWith("data:")) {
+              img.src = saved;
+            }
+          }
+        }
+      });
+      observer.observe(document.body, {
+        attributes: true,
+        attributeFilter: ["src"],
+        subtree: true,
+      });
+
+      for (const source of document.querySelectorAll("picture source")) {
+        for (const attr of [...lazyAttrs, ...lazySrcsetAttrs]) {
+          const val = source.getAttribute(attr);
+          if (val) {
+            if (attr.includes("srcset")) source.srcset = val;
+            source.removeAttribute(attr);
+          }
+        }
+      }
+
+      for (const el of document.querySelectorAll("*")) {
+        const bg = getComputedStyle(el).backgroundImage;
+        if (bg && bg !== "none" && bg.startsWith("url(")) {
+          const url = bg.slice(4, -1).replace(/["']/g, "");
+          if (/\.(jpe?g|png|gif|webp|svg|avif)/i.test(url)) {
+            const img = document.createElement("img");
+            img.src = url;
+            img.style.cssText = "width:100%;height:auto;display:block;";
+            el.style.backgroundImage = "none";
+            el.prepend(img);
+          }
+        }
+      }
+    });
+
+    await page
+      .waitForLoadState("networkidle", { timeout: 15_000 })
+      .catch(() => {});
+
+    for (let pass = 0; pass < 2; pass++) {
+      await page.evaluate(async () => {
+        const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+        const step = window.innerHeight;
+        const maxScroll = document.body.scrollHeight;
+        for (let y = 0; y <= maxScroll; y += step) {
+          window.scrollTo(0, y);
+          await delay(250);
+        }
+        window.scrollTo(0, 0);
+
+        for (const img of document.querySelectorAll("img[data-src]")) {
+          img.src = img.dataset.src;
+          img.removeAttribute("data-src");
+        }
+      });
+      await page
+        .waitForLoadState("networkidle", { timeout: 15_000 })
+        .catch(() => {});
+    }
+
+    await page
+      .waitForFunction(
+        () =>
+          [...document.querySelectorAll("img")].every((img) => {
+            if (img.src.startsWith("data:")) return true;
+            if (img.offsetWidth <= 1 && img.offsetHeight <= 1) return true;
+            return img.complete && img.naturalWidth > 0;
+          }),
+        { timeout: 30_000 },
+      )
+      .catch(() => {});
+
+    await page.evaluate(() => {
+      const style = document.createElement("style");
+      style.textContent = `
+        img {
+          opacity: 1 !important;
+          visibility: visible !important;
+          transition: none !important;
+          animation: none !important;
+        }
+      `;
+      document.head.appendChild(style);
+
+      for (const img of document.querySelectorAll("img")) {
+        img.style.setProperty("opacity", "1", "important");
+        img.style.setProperty("visibility", "visible", "important");
+        img.style.setProperty("transition", "none", "important");
+        img.style.setProperty("animation", "none", "important");
+      }
+    });
+
+    await page.waitForTimeout(1000);
+
+    await page.evaluate(() => {
+      const selectors = [
+        '[role="dialog"]',
+        '[class*="overlay"]',
+        '[class*="underlay"]',
+        '[class*="modal"]',
+        '[class*="cookie"]',
+        '[class*="consent"]',
+        '[class*="backdrop"]',
+      ];
+      for (const el of document.querySelectorAll(selectors.join(", "))) {
+        el.remove();
+      }
+
+      for (const el of document.querySelectorAll("*")) {
+        const s = getComputedStyle(el);
+        if (
+          s.position === "fixed" &&
+          el.offsetWidth >= window.innerWidth * 0.9 &&
+          el.offsetHeight >= window.innerHeight * 0.9 &&
+          !el.matches("header, nav, main, article, section, footer")
+        ) {
+          el.remove();
+        }
+      }
+    });
+
+    const pdfBuffer = await page.pdf({
+      format: "A4",
+      printBackground: true,
+      margin: { top: "20mm", bottom: "20mm", left: "15mm", right: "15mm" },
+    });
+
+    fs.writeFileSync(pdfPath, pdfBuffer);
+
+    const htmlPath = pdfPath
+      .replace(/\/pdf\//, "/html/")
+      .replace(/\.pdf$/, ".html");
+    fs.mkdirSync(path.dirname(htmlPath), { recursive: true });
+    const rawHtml = await page.content();
+    fs.writeFileSync(htmlPath, purge(rawHtml));
+
+    log.info(`Saved ${pdfPath} + ${htmlPath}`);
+  },
+
+  async failedRequestHandler({ request, log }) {
+    log.error(`Failed to crawl ${request.url} after retries.`);
+  },
+});
+
+await crawler.run(pages.map((p) => ({ url: p.url, uniqueKey: p.url })));
+
+console.log("\nDone. Output saved under:", domainDir);

package/discover.mjs

@@ -0,0 +1,222 @@
+import { writeFileSync } from "node:fs";
+import { parseArgs } from "node:util";
+
+/**
+ * Discover the full navigation tree for a Frontify portal.
+ * @param {{ url?: string, domain?: string, baseUrl?: string, hubId?: string, cookie?: string }} opts
+ * @returns {Promise<{ domain: string, hubId: number, baseUrl: string, children: object[] }>}
+ */
+export async function discover(opts = {}) {
+  const input = opts.url || opts.domain || "developer.frontify.com";
+  const isUrl = input.startsWith("https://") || input.startsWith("http://");
+  const baseUrl = opts.baseUrl || (isUrl ? new URL(input).origin : `https://${input}`);
+  const domain = opts.domain || (isUrl ? new URL(input).hostname : input);
+  let hubId = opts.hubId;
+  const cookie = opts.cookie || "";
+
+  const headers = {};
+  if (cookie) headers.cookie = cookie;
+
+  async function fetchJSON(url) {
+    const res = await fetch(url, { headers });
+    if (!res.ok) throw new Error(`${res.status} ${res.statusText} – ${url}`);
+    const json = await res.json();
+    if (!json.success) throw new Error(`API error – ${url}`);
+    return json.data;
+  }
+
+  async function detectHubId() {
+    console.error(`No hub ID provided, detecting from ${baseUrl} …`);
+    const res = await fetch(baseUrl, { headers });
+    if (!res.ok) throw new Error(`${res.status} fetching ${baseUrl}`);
+    const html = await res.text();
+    const match = html.match(/"hub_id"\s*:\s*(\d+)/);
+    if (!match) throw new Error("Could not find hub_id in page response");
+    console.error(`Detected hub_id: ${match[1]}`);
+    return match[1];
+  }
+
+  function pageUrl(documentId, categorySlug, pageSlug, headingSlug) {
+    let url = `${baseUrl}/document/${documentId}#`;
+    url += categorySlug ? `/${categorySlug}` : `/-`;
+    if (pageSlug) url += `/${pageSlug}`;
+    if (headingSlug) url += `/${headingSlug}`;
+    return url;
+  }
+
+  function buildHeadingNode(heading, documentId, categorySlug, pageSlug) {
+    const t = heading.translations.default;
+    return {
+      type: "heading",
+      id: heading.id,
+      title: t.title,
+      slug: t.slug,
+      url: pageUrl(documentId, categorySlug, pageSlug, t.slug),
+    };
+  }
+
+  function buildPageNode(page, documentId, categorySlug) {
+    const t = page.translations.default;
+    const node = {
+      type: "page",
+      id: page.id,
+      title: t.title,
+      slug: t.slug,
+      url: pageUrl(documentId, categorySlug, t.slug),
+      children: [],
+    };
+    if (page.children) {
+      for (const child of page.children) {
+        if (child.type === "page_heading") {
+          node.children.push(buildHeadingNode(child, documentId, categorySlug, t.slug));
+        }
+      }
+    }
+    return node;
+  }
+
+  function buildDocNavItems(items, documentId, categorySlug) {
+    const nodes = [];
+    for (const item of items) {
+      switch (item.type) {
+        case "page_category": {
+          const t = item.translations.default;
+          const cat = {
+            type: "category",
+            id: item.id,
+            title: t.title,
+            slug: t.slug,
+            children: [],
+          };
+          if (item.children) {
+            cat.children = buildDocNavItems(item.children, documentId, t.slug);
+          }
+          nodes.push(cat);
+          break;
+        }
+        case "page": {
+          nodes.push(buildPageNode(item, documentId, categorySlug));
+          break;
+        }
+        case "page_heading": {
+          nodes.push(buildHeadingNode(item, documentId, categorySlug, null));
+          break;
+        }
+      }
+    }
+    return nodes;
+  }
+
+  async function buildDocumentNode(doc) {
+    const t = doc.translations.default;
+    const node = {
+      type: "document",
+      id: doc.id,
+      title: t.title,
+      slug: t.slug,
+      url: `${baseUrl}/document/${doc.id}`,
+      children: [],
+    };
+    try {
+      const items = await fetchJSON(`${baseUrl}/api/document-navigation/${doc.id}`);
+      node.children = buildDocNavItems(items, doc.id, null);
+    } catch (err) {
+      node.error = err.message;
+    }
+    return node;
+  }
+
+  function buildLinkNode(doc) {
+    const t = doc.translations.default;
+    return {
+      type: "external_link",
+      id: doc.id,
+      title: t.title,
+      url: doc.url,
+      openInNewTab: doc.should_open_in_new_tab,
+    };
+  }
+
+  if (!hubId) hubId = await detectHubId();
+  console.error(`Crawling portal: ${baseUrl} (hub ${hubId})`);
+  const portalNav = await fetchJSON(`${baseUrl}/api/portal-navigation/${hubId}`);
+
+  const tree = { domain, hubId: Number(hubId), baseUrl, children: [] };
+
+  for (const item of portalNav) {
+    if (item.type === "document") {
+      tree.children.push(await buildDocumentNode(item));
+    } else if (item.type === "document_group") {
+      const group = {
+        type: "group",
+        id: item.id,
+        title: item.translations.default.name,
+        children: [],
+      };
+      const docPromises = item.children.map((child) => {
+        if (child.type === "link_document") return buildLinkNode(child);
+        return buildDocumentNode(child);
+      });
+      group.children = await Promise.all(docPromises);
+      tree.children.push(group);
+    } else if (item.type === "link_document") {
+      tree.children.push(buildLinkNode(item));
+    }
+  }
+
+  return tree;
+}
+
+function countType(node, type) {
+  let count = node.type === type ? 1 : 0;
+  if (node.children) {
+    for (const child of node.children) count += countType(child, type);
+  }
+  return count;
+}
+
+// ── CLI mode ──
+const isCLI = process.argv[1]?.endsWith("discover.mjs");
+if (isCLI) {
+  const { values, positionals } = parseArgs({
+    options: {
+      url:    { type: "string", short: "u" },
+      hub:    { type: "string", short: "h" },
+      output: { type: "string", short: "o" },
+      cookie: { type: "string", short: "c" },
+      help:   { type: "boolean" },
+    },
+    allowPositionals: true,
+    strict: false,
+  });
+
+  if (values.help) {
+    console.log(`Usage: node discover.mjs [options] [url] [hubId] [output]
+
+Options:
+  -u, --url <url>       Portal domain or URL
+  -h, --hub <id>        Hub ID (auto-detected if omitted)
+  -o, --output <file>   Output file (default: {domain}.json)
+  -c, --cookie <str>    Cookie header for authenticated requests
+      --help            Show this help
+
+Environment variables (or .env file):
+  COOKIE                Cookie header value
+  URL                   Portal domain or URL
+  HUB_ID                Hub ID`);
+    process.exit(0);
+  }
+
+  const input = values.url || positionals[0] || process.env.URL;
+  const hubId = values.hub || positionals[1] || process.env.HUB_ID;
+  const cookie = values.cookie || process.env.COOKIE || "";
+
+  const tree = await discover({ url: input, hubId, cookie });
+
+  const outputFile = values.output || positionals[2] || `${tree.domain}.json`;
+  console.error(
+    `Done – ${countType(tree, "document")} documents, ${countType(tree, "category")} categories, ${countType(tree, "page")} pages, ${countType(tree, "heading")} headings, ${countType(tree, "external_link")} external links`
+  );
+  writeFileSync(outputFile, JSON.stringify(tree, null, 2) + "\n");
+  console.error(`Written to ${outputFile}`);
+}

package/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "guidelinescraper",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Scrape a Frontify brand portal and save every page as PDF and clean HTML",
+  "bin": {
+    "guidelinescraper": "./crawl.mjs"
+  },
+  "scripts": {
+    "start": "node crawl.mjs"
+  },
+  "files": [
+    "crawl.mjs",
+    "discover.mjs",
+    "purge-html.mjs"
+  ],
+  "keywords": [
+    "frontify",
+    "scraper",
+    "pdf",
+    "brand-guidelines"
+  ],
+  "license": "ISC",
+  "dependencies": {
+    "crawlee": "^3.16.0",
+    "playwright": "^1.58.2"
+  }
+}

package/purge-html.mjs

@@ -0,0 +1,134 @@
+import fs from "node:fs";
+import path from "node:path";
+import { parseHTML } from "linkedom";
+
+const REMOVE_TAGS = new Set([
+  "script", "style", "link", "meta", "noscript", "iframe", "svg", "canvas",
+  "video", "audio", "source", "track", "object", "embed", "applet",
+  "form", "input", "textarea", "select", "button", "template",
+]);
+
+const REMOVE_SELECTORS = [
+  "nav", "header", "footer",
+  "[role='navigation']", "[role='banner']", "[role='search']", "[role='dialog']",
+  "[class*='overlay']", "[class*='underlay']", "[class*='modal']",
+  "[class*='cookie']", "[class*='consent']", "[class*='sidebar']",
+  "[class*='nav-']", "[class*='toolbar']", "[class*='skip-to-main']",
+  "[class*='search']", "[class*='tooltip']", "[class*='popover']",
+  "[class*='dropdown']", "[aria-hidden='true']",
+  "[data-test-id='skip-to-main-content-link']",
+];
+
+const KEEP_ATTRS = new Set([
+  "src", "href", "alt", "title", "id",
+  "colspan", "rowspan", "headers", "scope", "lang", "dir",
+]);
+
+const SEMANTIC_TAGS = new Set([
+  "html", "head", "body", "main", "article", "section", "aside",
+  "h1", "h2", "h3", "h4", "h5", "h6",
+  "p", "blockquote", "pre", "code",
+  "ul", "ol", "li", "dl", "dt", "dd",
+  "table", "thead", "tbody", "tfoot", "tr", "th", "td", "caption",
+  "a", "img", "figure", "figcaption", "picture",
+  "strong", "em", "b", "i", "u", "s", "mark", "small", "sub", "sup",
+  "br", "hr", "details", "summary", "time", "abbr", "cite", "q",
+]);
+
+function cleanNode(node) {
+  if (node.nodeType === 3) return;
+  if (node.nodeType !== 1) { node.remove(); return; }
+
+  const tag = node.tagName?.toLowerCase();
+  if (REMOVE_TAGS.has(tag)) { node.remove(); return; }
+
+  if (node.attributes) {
+    const toRemove = [];
+    for (const attr of node.attributes) {
+      if (!KEEP_ATTRS.has(attr.name)) toRemove.push(attr.name);
+    }
+    for (const name of toRemove) node.removeAttribute(name);
+  }
+
+  for (const child of [...node.childNodes]) cleanNode(child);
+
+  if (!SEMANTIC_TAGS.has(tag) && node.children?.length === 0 && !node.textContent?.trim()) {
+    node.remove();
+    return;
+  }
+
+  if (!SEMANTIC_TAGS.has(tag)) {
+    const parent = node.parentNode;
+    if (parent) {
+      while (node.firstChild) parent.insertBefore(node.firstChild, node);
+      node.remove();
+    }
+  }
+}
+
+export function purge(html) {
+  const { document } = parseHTML(html);
+
+  for (const selector of REMOVE_SELECTORS) {
+    try { for (const el of document.querySelectorAll(selector)) el.remove(); } catch {}
+  }
+
+  const main =
+    document.querySelector("main") ||
+    document.querySelector("[role='main']") ||
+    document.querySelector("#main") ||
+    document.querySelector(".main-content") ||
+    document.querySelector("#classic-theme") ||
+    document.body;
+
+  cleanNode(main);
+
+  const title = document.querySelector("title")?.textContent?.trim() || "";
+  const cleanHtml =
+    `<!DOCTYPE html>\n<html lang="${document.documentElement?.getAttribute("lang") || "en"}">\n<head>\n<meta charset="utf-8">\n<title>${title}</title>\n<style>body{max-width:72ch;margin:2rem auto;padding:0 1rem;font:1rem/1.6 monospace}img{max-width:100%;height:auto}</style>\n</head>\n<body>\n${main.innerHTML.trim()}\n</body>\n</html>`;
+
+  return cleanHtml.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+\n/g, "\n");
+}
+
+// CLI mode: run standalone on a directory
+const isCLI = process.argv[1]?.endsWith("purge-html.mjs");
+if (isCLI && process.argv[2]) {
+  const RAW_DIR = process.argv[2];
+  const CLEAN_DIR = process.argv[3] || RAW_DIR.replace(".raw", ".clean");
+
+  function collectFiles(dir) {
+    const files = [];
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, entry.name);
+      if (entry.isDirectory()) files.push(...collectFiles(full));
+      else if (entry.name.endsWith(".html")) files.push(full);
+    }
+    return files;
+  }
+
+  const files = collectFiles(RAW_DIR);
+  console.log(`Found ${files.length} HTML files in ${RAW_DIR}`);
+
+  let totalRawBytes = 0;
+  let totalCleanBytes = 0;
+
+  for (const file of files) {
+    const relPath = path.relative(RAW_DIR, file);
+    const outPath = path.join(CLEAN_DIR, relPath);
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+
+    const raw = fs.readFileSync(file, "utf-8");
+    const clean = purge(raw);
+    fs.writeFileSync(outPath, clean);
+
+    totalRawBytes += raw.length;
+    totalCleanBytes += clean.length;
+
+    const pct = ((1 - clean.length / raw.length) * 100).toFixed(0);
+    console.log(`  ${relPath}  ${(raw.length / 1024).toFixed(0)}K → ${(clean.length / 1024).toFixed(0)}K  (-${pct}%)`);
+  }
+
+  console.log(`\nDone. ${files.length} files cleaned.`);
+  console.log(`Total: ${(totalRawBytes / 1024).toFixed(0)}K → ${(totalCleanBytes / 1024).toFixed(0)}K  (-${((1 - totalCleanBytes / totalRawBytes) * 100).toFixed(0)}%)`);
+  console.log(`Output: ${CLEAN_DIR}`);
+}