crawldna 0.1.0

package/src/lib/browser.mjs

@@ -0,0 +1,242 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Lazy Playwright loader. Playwright is an optional dependency: nothing here
+// imports it until a crawl actually needs the browser as an actuator.
+
+let _pw = null;
+let _imported = null; // null = unknown
+let _browser = null;
+let _error = null; // clean, user-facing reason when unavailable
+
+async function importPlaywright() {
+  if (_imported !== null) return _imported;
+  try {
+    _pw = await import('playwright');
+    _imported = true;
+  } catch {
+    _imported = false;
+    _error = 'Playwright is not installed. Run: npm install playwright && npx playwright install chromium';
+  }
+  return _imported;
+}
+
+function cleanLaunchError(err) {
+  const msg = String((err && err.message) || err || '');
+  if (/Executable doesn'?t exist|playwright install|Looks like Playwright/i.test(msg)) {
+    return 'Chromium is not fully installed. Run: npx playwright install chromium';
+  }
+  return 'Browser launch failed: ' + msg.split('\n')[0];
+}
+
+/** A clean, user-facing reason the browser is unavailable, or null. */
+export function browserError() {
+  return _error;
+}
+
+/** Launch (once) and return a shared headless Chromium browser. Clean errors. */
+export async function getBrowser() {
+  if (_browser) return _browser;
+  if (!(await importPlaywright())) throw new Error(_error);
+  try {
+    _browser = await _pw.chromium.launch({ headless: true });
+  } catch (err) {
+    _error = cleanLaunchError(err);
+    throw new Error(_error);
+  }
+  return _browser;
+}
+
+/**
+ * True if the browser can actually launch — verified once by a real launch
+ * (the launched browser is cached for reuse), not just a module import. A
+ * missing/partial browser binary therefore degrades cleanly instead of throwing
+ * on every page mid-crawl. Never throws.
+ */
+export async function isBrowserAvailable() {
+  if (_browser) return true; // already launched and cached
+  if (!(await importPlaywright())) return false;
+  try {
+    await getBrowser();
+    return true;
+  } catch {
+    // Do NOT cache a failure: the browser may be installed between crawls (e.g.
+    // a long-running `serve`). Re-probe next time rather than staying stuck.
+    return false;
+  }
+}
+
+// Runs before any page script. Monkey-patches addEventListener so that ANY
+// element which registers a click-like listener — however it was built, even a
+// plain <div> with a JS handler — is tagged. This is how we spot non-obvious
+// interactivity universally, without per-site selectors.
+const SNIFFER = `(() => {
+  try {
+    const proto = EventTarget.prototype;
+    const orig = proto.addEventListener;
+    const INTERESTING = { click: 1, mousedown: 1, pointerdown: 1, keydown: 1, change: 1 };
+    proto.addEventListener = function (type, listener, opts) {
+      try {
+        if (INTERESTING[type] && this && this.nodeType === 1) {
+          this.setAttribute('data-crawldna-listener', '1');
+        }
+      } catch (e) {}
+      return orig.call(this, type, listener, opts);
+    };
+  } catch (e) {}
+})();`;
+
+const CONTEXT_OPTS = {
+  userAgent: 'crawldna/0.1 (+https://crawldna.com)',
+  viewport: { width: 1280, height: 900 },
+};
+
+// --- Browser-context pool (asset-cache reuse across pages) ------------------
+// A brand-new context per page throws away the browser's HTTP cache, so the site's
+// shared CSS/JS/fonts are re-downloaded on EVERY page — pure waste on a docs site with
+// hundreds of same-styled pages. Instead we keep a small pool of contexts and REUSE
+// them: within a context the HTTP cache is shared, so a page's static assets are fetched
+// once and served from cache thereafter. Parallelism is preserved — the pool holds up to
+// `_maxIdle` idle contexts (set to the crawl's concurrency), so N workers each keep their
+// own context alive between pages.
+//
+// COMPLETENESS IS NOT AT RISK (the project's first rule). Reuse only shares the ASSET
+// cache; it does not change what a page renders for us, because the engine (1) opens a
+// FRESH page and navigates fresh for every URL, and (2) exhaustively clicks EVERY reveal
+// control regardless of any remembered client state — so accumulated cookies/localStorage
+// can't hide content from extraction (a "remembered" tab is still clicked; a first-visit
+// tour is chrome, not content). Contexts are recycled after `_maxUses` pages as plain
+// hygiene against a pathological page corrupting one. Consent is still dismissed per page
+// by the reveal engine, so a carried-over consent cookie only ever helps.
+const _idleContexts = [];
+let _maxIdle = 8; // set from concurrency by configureContextPool
+const _maxUses = 100; // recycle a context after this many pages
+
+/** Size the idle pool to the crawl's concurrency so each worker keeps its own context. */
+export function configureContextPool(concurrency) {
+  _maxIdle = Math.max(1, Number(concurrency) || 1);
+}
+
+async function makeContext() {
+  const browser = await getBrowser();
+  const context = await browser.newContext(CONTEXT_OPTS);
+  await context.addInitScript(SNIFFER); // once per context, not per page
+  context.__uses = 0;
+  return context;
+}
+
+/** Return a context to the pool for reuse, or close it once it's served enough pages
+ *  (or the idle pool is already full). Never throws. */
+async function releaseContext(context) {
+  if (!context) return;
+  context.__uses = (context.__uses || 0) + 1;
+  if (context.__uses >= _maxUses || _idleContexts.length >= _maxIdle) {
+    await context.close().catch(() => {});
+    return;
+  }
+  _idleContexts.push(context);
+}
+
+/**
+ * A fresh page for one URL, drawn from a REUSED browser context (shared asset cache),
+ * plus a `release()` to return the context to the pool when the page is done.
+ * The sniffer is already installed on the context. Callers MUST call `release()` (not
+ * `context.close()`) to get the caching benefit — closing the context still works but
+ * forfeits reuse.
+ *
+ * @returns {Promise<{ page, context, release: () => Promise<void> }>}
+ */
+export async function newPage() {
+  // Reuse an idle context if one is available; a stale/broken one is dropped and a
+  // fresh context made, so a bad context can never wedge the crawl.
+  let context = _idleContexts.pop();
+  let page;
+  if (context) {
+    try {
+      page = await context.newPage();
+    } catch {
+      await context.close().catch(() => {});
+      context = null;
+    }
+  }
+  if (!context) {
+    context = await makeContext();
+    page = await context.newPage();
+  }
+
+  let released = false;
+  const release = async () => {
+    if (released) return; // idempotent: safe to call from both catch and finally
+    released = true;
+    try {
+      await page.close();
+    } catch {
+      /* ignore */
+    }
+    await releaseContext(context);
+  };
+  return { page, context, release };
+}
+
+/** Close the shared browser, if any, and drop the pooled contexts. Safe to call
+ *  multiple times (e.g. once per run — the browser relaunches lazily next run). */
+export async function closeBrowser() {
+  _idleContexts.length = 0; // the contexts are closed with the browser below
+  if (_browser) {
+    try {
+      await _browser.close();
+    } catch {
+      /* ignore */
+    }
+    _browser = null;
+  }
+}
+
+// --- run-scoped ownership of the shared browser ------------------------------
+// The browser is ONE process shared by every crawl in this Node process. A run
+// that closed it unconditionally when it finished would pull it out from under a
+// run still using it — exactly what happened when the UI stopped a crawl and
+// immediately started the next one (the old run's cleanup killed the new run's
+// pages). So runs RETAIN the browser for their lifetime and only the LAST release
+// actually closes it.
+let _retainers = 0;
+
+/** Mark a run as using the shared browser. Pair with releaseBrowser(). */
+export function retainBrowser() {
+  _retainers++;
+}
+
+/** Release a run's hold; closes the browser once no run holds it. */
+export async function releaseBrowser() {
+  _retainers = Math.max(0, _retainers - 1);
+  if (_retainers === 0) await closeBrowser();
+}
+
+let _installTried = false;
+
+/**
+ * Make sure the browser can launch, installing Chromium once if it is missing.
+ * Returns true when the browser is ready. Used at `serve` startup so the user
+ * never hits a mid-crawl "browser not installed" error.
+ */
+export async function ensureBrowser({ log = () => {} } = {}) {
+  if (await isBrowserAvailable()) return true;
+  if (_installTried) return false;
+  _installTried = true;
+
+  log('Chromium not found — installing it once (this is a one-time ~150MB download)…');
+  const { spawn } = await import('node:child_process');
+  const ok = await new Promise((resolve) => {
+    try {
+      const child = spawn('npx', ['playwright', 'install', 'chromium', 'chromium-headless-shell'], {
+        stdio: 'inherit',
+        shell: true,
+      });
+      child.on('exit', (code) => resolve(code === 0));
+      child.on('error', () => resolve(false));
+    } catch {
+      resolve(false);
+    }
+  });
+  if (!ok) return false;
+  return isBrowserAvailable();
+}

package/src/lib/challenge.mjs

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// #14 — anti-bot / challenge-page detection. ALWAYS ON: this is a PRECISION
+// guard, not a courtesy — when a site's defense serves a "checking your
+// browser" interstitial or a CAPTCHA wall (often with HTTP 200), keeping it
+// would put boilerplate in the output AS IF it were content, silently.
+//
+// Universality argument (same as the consent lexicon, #21a): we read the
+// ARTEFACT OF THE DEFENSE, not the site. Challenge pages are produced by a
+// handful of vendors (Cloudflare, hCaptcha, reCAPTCHA, DataDome, PerimeterX,
+// AWS WAF) and look the same everywhere, so their MECHANICAL markers — widget
+// script/iframe URLs, the cf-mitigated header, meta-refresh interstitials —
+// are universal signals. A page that merely TALKS about captchas (docs, blog
+// posts) has real text mass and no thin-page signal, so it never trips this.
+//
+// Policy on detection (enforced by the callers): NEVER kept as content, a loud
+// `anti-bot` warning with the URL, ONE retry with backoff (honouring
+// Retry-After), then a declared skip. NEVER bypassed — CAPTCHAs/challenges are
+// out of scope forever (ARCHITECTURE §14): we signal, we don't break through.
+
+// Vendor widget/iframe/script markers — mechanical evidence a challenge is
+// being SERVED (not merely mentioned; these are resource URLs and internals).
+const WIDGET_RE = new RegExp(
+  [
+    'challenges\\.cloudflare\\.com',
+    'cdn-cgi/challenge-platform',
+    '__cf_chl',
+    'cf_chl_',
+    'js\\.hcaptcha\\.com',
+    'hcaptcha\\.com/1/api\\.js',
+    'google\\.com/recaptcha/api',
+    'gstatic\\.com/recaptcha',
+    'recaptcha/api\\.js',
+    'captcha-delivery\\.com',
+    'geo\\.captcha-delivery\\.com',
+    'datadome',
+    'px-cdn\\.net',
+    'px-cloud\\.net',
+    '_pxhd',
+    'awswaf\\.com',
+    'amazonaws\\.com/captcha',
+  ].join('|'),
+  'i',
+);
+
+// Interstitial phrasing — meaningful only TOGETHER with a thin page AND a
+// mechanical corroborator (blocked status / meta-refresh / widget), never alone.
+const PHRASE_RE = new RegExp(
+  [
+    'checking your browser',
+    'verify(?:ing)? (?:that )?you are (?:a )?human',
+    'just a moment',
+    'attention required',
+    'enable javascript and cookies to continue',
+    'unusual traffic',
+    'are you a robot',
+    'ddos protection by',
+    'please stand by, while we are checking',
+  ].join('|'),
+  'i',
+);
+
+const META_REFRESH_RE = /<meta[^>]+http-equiv\s*=\s*["']?refresh/i;
+
+// "Thin": challenge pages carry a sentence or two (~200–400 chars of real
+// text); real content pages carry far more. The phrase/widget must ALSO match —
+// thinness alone never flags anything (an empty page is just an empty page).
+const THIN_CONTENT_CHARS = 800;
+
+/**
+ * Is this response a bot-defense challenge rather than content?
+ *
+ * @param {object} a
+ * @param {number} [a.status]      HTTP status (challenges use 200/403/429/503)
+ * @param {object} [a.headers]     response headers, lowercase keys
+ * @param {string} [a.html]        the page HTML (widget markers live in markup)
+ * @param {number} [a.contentLen]  chars of real (non-link) text on the page —
+ *                                 the same metric as extract.contentWordLen
+ * @returns {{ challenge: boolean, signal?: string }}
+ */
+export function detectChallenge({ status = 0, headers = {}, html = '', contentLen = 0 } = {}) {
+  const hdr = (name) => String(headers[name] || headers[String(name).toLowerCase()] || '');
+  // Cloudflare labels challenge responses explicitly — the strongest signal.
+  if (/challenge/i.test(hdr('cf-mitigated'))) {
+    return { challenge: true, signal: 'cf-mitigated: challenge response header' };
+  }
+  const h = String(html || '');
+  const thin = contentLen < THIN_CONTENT_CHARS;
+  const blocked = status === 403 || status === 429 || status === 503;
+  if (thin && WIDGET_RE.test(h)) {
+    return { challenge: true, signal: 'captcha/challenge widget on a near-empty page' + (blocked ? ` (HTTP ${status})` : '') };
+  }
+  if (thin && PHRASE_RE.test(h) && (blocked || META_REFRESH_RE.test(h))) {
+    return { challenge: true, signal: 'challenge interstitial marker' + (blocked ? ` (HTTP ${status})` : ' (meta-refresh)') };
+  }
+  return { challenge: false };
+}
+
+/** The backoff before the single retry: honour Retry-After (seconds or an
+ *  HTTP-date) when sane, else a fixed courtesy pause. Bounded. */
+export function challengeBackoffMs(headers = {}) {
+  const raw = String(headers['retry-after'] || headers['Retry-After'] || '').trim();
+  if (raw) {
+    const secs = Number(raw);
+    if (Number.isFinite(secs) && secs > 0) return Math.min(15000, secs * 1000);
+    const at = Date.parse(raw);
+    if (!Number.isNaN(at)) return Math.min(15000, Math.max(0, at - Date.now()));
+  }
+  return 2500;
+}

package/src/lib/faithful.mjs

@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Faithfulness verification for reshape output (#11). The reshape step is the ONLY
+// place the AI may reformat content — and therefore the only place it could ALTER or
+// INVENT a value. The prompt already demands "value-faithful", but a prompt is a
+// request, not a guarantee (observed live: asked for v-alert props past the context
+// budget, the model fabricated a plausible props table from its own memory — e.g. a
+// `'Close'` default where the crawled docs say `'$vuetify.close'` — with no warning).
+//
+// This module is the deterministic enforcement: extract the VALUE-LIKE atoms from a
+// produced file (numbers, URLs, inline code, quoted literals, code lines — the things
+// the reshape contract says must be copied exactly) and check that each one exists in
+// the crawled sources. Anything that doesn't is reported so the caller can FLAG it
+// instead of serving it silently. RAGAS-faithfulness, done dependency-free at home.
+//
+// Honest limits: matching is by normalised substring, deliberately GENEROUS — a value
+// that appears anywhere in the sources passes, so common words pass trivially and a
+// lucky guess can pass too. What it reliably catches is the dangerous case: specific
+// values (defaults, prices, URLs, code) that exist nowhere in the extraction.
+
+/** Lowercase + collapse whitespace: matching must not fail on wrapping/indentation. */
+const norm = (s) => String(s || '').toLowerCase().replace(/\s+/g, ' ').trim();
+
+const URL_RE = /https?:\/\/[^\s<>"'`)\]]+/g;
+const CODESPAN_RE = /`([^`\n]+)`/g;
+const FENCE_RE = /```[^\n]*\n([\s\S]*?)```/g;
+const SINGLE_QUOTED_RE = /'([^'\n]{2,80})'/g;
+const DOUBLE_QUOTED_RE = /"([^"\n]{2,80})"/g;
+// currency/percent/decimal numbers, or integers of 2+ digits (a lone "3" is noise)
+const NUM_RE = /[€$£]\s?\d[\d.,]*|\d[\d.,]*\s?%|\b\d+(?:[.,]\d+)+\b|\b\d{2,}\b/g;
+
+const wordsIn = (s) => String(s).trim().split(/\s+/).filter(Boolean).length;
+
+/**
+ * Extract the verifiable, value-like atoms of a Markdown document:
+ *   - `code-line` — each substantive line of a fenced code block (invented examples
+ *     are caught line by line);
+ *   - `code`      — inline code spans (prop names, defaults, commands);
+ *   - `url`       — absolute links;
+ *   - `string`    — short quoted literals ('$vuetify.close', "elevated");
+ *   - `number`    — prices/percentages/decimals/multi-digit integers.
+ * Prose is deliberately NOT extracted: the reshape contract allows rephrasing layout,
+ * only VALUES must survive verbatim.
+ *
+ * @returns {Array<{ value:string, kind:string }>} deduplicated by normalised value
+ */
+export function extractAtoms(markdown) {
+  let text = String(markdown || '');
+  const atoms = [];
+  const push = (value, kind) => atoms.push({ value: value.trim(), kind });
+
+  // fenced code first (then removed, so its content isn't re-matched as prose)
+  text = text.replace(FENCE_RE, (_m, body) => {
+    for (const line of String(body).split('\n')) {
+      const t = line.trim();
+      if (t.length >= 10 && /[a-z0-9]/i.test(t)) push(t, 'code-line');
+    }
+    return ' ';
+  });
+  text = text.replace(CODESPAN_RE, (_m, body) => {
+    if (body.trim().length >= 2 && body.length <= 120) push(body, 'code');
+    return ' ';
+  });
+  text = text.replace(URL_RE, (m) => {
+    push(m.replace(/[.,;:!?]+$/, ''), 'url');
+    return ' ';
+  });
+  for (const re of [SINGLE_QUOTED_RE, DOUBLE_QUOTED_RE]) {
+    text = text.replace(re, (_m, body) => {
+      if (wordsIn(body) <= 4) push(body, 'string');
+      return ' ';
+    });
+  }
+  // strip residual markdown structure so list markers/tables don't fabricate numbers
+  text = text.replace(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]/gm, ' ').replace(/^[ \t]*\|[\s:|-]*\|?[ \t]*$/gm, ' ');
+  for (const m of text.match(NUM_RE) || []) push(m, 'number');
+
+  const seen = new Set();
+  return atoms.filter((a) => {
+    const k = norm(a.value);
+    if (!k || seen.has(k)) return false;
+    seen.add(k);
+    return true;
+  });
+}
+
+/** A value that IS a number, however it was extracted — a `1,299` in a code span or
+ *  table cell deserves the same separator-insensitive matching as a bare number. */
+const isNumericish = (atom, n) => atom.kind === 'number' || /^[€$£]?\s?\d[\d.,]*\s?%?$/.test(n);
+
+/** Matching variants for an atom: normalised form; a Markdown-unescaped form (a model
+ *  writing a table escapes pipes — `string \| number` must match the source's
+ *  `string | number`); separator-insensitive digit forms for numbers ("1,299" must
+ *  match "1299" and vice versa). Generous by design. */
+function variantsOf(atom) {
+  const n = norm(atom.value);
+  const v = [n];
+  const unescaped = n.replace(/\\([\\`*_{}[\]()#+\-.!|<>~])/g, '$1');
+  if (unescaped !== n) v.push(unescaped);
+  if (isNumericish(atom, n)) {
+    const digits = n.replace(/[^\d]/g, '');
+    if (digits && digits !== n) v.push(digits);
+  }
+  return v;
+}
+
+/**
+ * Verify a produced document's value-like atoms against the crawled sources.
+ *
+ * @param {string} markdown          the produced file's content
+ * @param {string[]} sources         FULL source documents (not the model's context —
+ *                                   a value anywhere in the extraction is faithful)
+ * @param {{ allow?: string }} [o]   extra acceptable text: the user's own instruction
+ *                                   (values THEY typed — a date, a filename — are not
+ *                                   inventions even when absent from the sources)
+ * @returns {{ total:number, verified:number, unverified:string[], ratio:number }}
+ */
+export function verifyValues(markdown, sources, { allow = '' } = {}) {
+  const atoms = extractAtoms(markdown);
+  if (!atoms.length) return { total: 0, verified: 0, unverified: [], ratio: 1 };
+
+  const hay = norm((sources || []).join('\n'));
+  const hayDigits = hay.replace(/[^\da-z]/g, ''); // separator-insensitive number lane
+  const allowN = norm(allow);
+
+  const unverified = [];
+  let verified = 0;
+  for (const atom of atoms) {
+    const numeric = isNumericish(atom, norm(atom.value));
+    const ok = variantsOf(atom).some(
+      (v) => hay.includes(v) || (allowN && allowN.includes(v)) || (numeric && hayDigits.includes(v.replace(/[^\d]/g, ''))),
+    );
+    if (ok) verified++;
+    else unverified.push(atom.value);
+  }
+  return { total: atoms.length, verified, unverified, ratio: verified / atoms.length };
+}
+
+/**
+ * The warning block prepended to a produced file that failed verification — clearly
+ * tool-generated (blockquote, named), so it can't be mistaken for model output and
+ * can be stripped mechanically (see FIDELITY_BANNER_RE).
+ */
+export function fidelityBanner(f) {
+  const shown = f.unverified
+    .slice(0, 10)
+    .map((v) => '`' + String(v).replace(/`/g, '´') + '`')
+    .join(', ');
+  const more = f.unverified.length > 10 ? ` (+${f.unverified.length - 10} more)` : '';
+  const head =
+    f.total > 0 && f.verified / f.total < 0.5
+      ? `most of this file (${f.unverified.length} of ${f.total} checked values) could NOT be found in the crawled sources`
+      : `${f.unverified.length} of ${f.total} checked value(s) were not found in the crawled sources`;
+  return (
+    `> ⚠️ **Fidelity check (crawldna):** ${head} — the model may have invented them: ${shown}${more}.\n` +
+    `> Do not trust these values without checking the original extraction.`
+  );
+}
+
+/** Matches a fidelity banner at the start of a file (through its trailing blank line). */
+export const FIDELITY_BANNER_RE = /^> ⚠️ \*\*Fidelity check \(crawldna\):\*\*[^\n]*\n> [^\n]*\n\n/;
+
+/** Remove a leading fidelity banner, e.g. before re-feeding a produced file to the
+ *  model as context (the warning is for the USER, not content to iterate on). */
+export function stripFidelityBanner(text) {
+  return String(text || '').replace(FIDELITY_BANNER_RE, '');
+}

package/src/lib/fetcher.mjs

@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Network layer: plain fetch, with optional escalation to a real browser when
+// a page looks like a JS-rendered shell or the caller forces it.
+
+import { isBrowserAvailable, newPage } from './browser.mjs';
+import { settle } from './settle.mjs';
+
+const DEFAULT_HEADERS = {
+  'user-agent': 'crawldna/0.1 (+https://crawldna.com)',
+  accept:
+    'text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5',
+  'accept-language': 'en-US,en;q=0.9',
+};
+
+/** Plain `fetch` returning text. Never throws — failures come back as `ok:false`. */
+export async function fetchText(url, { headers, timeout = 30000, accept } = {}) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeout);
+  try {
+    const res = await fetch(url, {
+      redirect: 'follow',
+      signal: ctrl.signal,
+      headers: { ...DEFAULT_HEADERS, ...(accept ? { accept } : {}), ...headers },
+    });
+    const contentType = res.headers.get('content-type') || '';
+    // Plain object, lowercase keys — the challenge guard (#14) reads vendor
+    // headers like cf-mitigated / retry-after from here. (Named resHeaders: the
+    // `headers` PARAMETER above is the request's, and shadowing it here would
+    // TDZ the fetch call.)
+    const resHeaders = {};
+    for (const [k, v] of res.headers) resHeaders[k] = v;
+    const text = await res.text();
+    return { ok: res.ok, status: res.status, text, contentType, finalUrl: res.url || url, headers: resHeaders };
+  } catch (err) {
+    return { ok: false, status: 0, text: '', contentType: '', finalUrl: url, error: err, headers: {} };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * #6 — a conditional GET for incremental re-crawl. Sends If-None-Match /
+ * If-Modified-Since built from a page's stored validators; a 304 is the SERVER
+ * itself confirming the resource is byte-for-byte unchanged. Returns the fresh
+ * validators too (so they can be re-stored) and never throws. With no validators
+ * to send there is nothing to ask, so it reports "changed" (notModified:false).
+ * @param {string} url
+ * @param {{etag?:string,lastModified?:string,timeout?:number}} v
+ * @returns {Promise<{status:number,notModified:boolean,etag:string,lastModified:string}>}
+ */
+export async function conditionalGet(url, { etag, lastModified, timeout = 15000 } = {}) {
+  if (!etag && !lastModified) return { status: 0, notModified: false, etag: '', lastModified: '' };
+  const headers = {};
+  if (etag) headers['if-none-match'] = etag;
+  if (lastModified) headers['if-modified-since'] = lastModified;
+  const res = await fetchText(url, { headers, timeout });
+  return {
+    status: res.status,
+    notModified: res.status === 304,
+    etag: (res.headers && res.headers['etag']) || '',
+    lastModified: (res.headers && res.headers['last-modified']) || '',
+  };
+}
+
+/** Rough "did this HTML arrive with real content?" heuristic. */
+export function looksRendered(html) {
+  if (!html) return false;
+  const text = html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+  return text.length > 800;
+}
+
+/**
+ * Load a page's HTML, escalating to the browser when needed.
+ *
+ * `browserMode`:
+ *   - 'never'  : plain fetch only.
+ *   - 'auto'   : fetch first; render with the browser only if the page looks
+ *                like an empty SPA shell (and Playwright is available).
+ *   - 'always' : render with the browser (fall back to fetch if unavailable).
+ *
+ * Returns `{ html, finalUrl, status, rendered }`.
+ */
+export async function loadHtml(url, { browserMode = 'auto', ctx } = {}) {
+  let staticRes = null;
+
+  if (browserMode !== 'always') {
+    staticRes = await fetchText(url);
+    if (browserMode === 'never') {
+      return { html: staticRes.text, finalUrl: staticRes.finalUrl, status: staticRes.status, rendered: false, headers: staticRes.headers };
+    }
+    if (staticRes.ok && looksRendered(staticRes.text)) {
+      return { html: staticRes.text, finalUrl: staticRes.finalUrl, status: staticRes.status, rendered: false, headers: staticRes.headers };
+    }
+  }
+
+  // We want (or need) the browser.
+  if (await isBrowserAvailable()) {
+    let release;
+    try {
+      const p = await newPage();
+      release = p.release;
+      const { page } = p;
+      let status = 200;
+      let headers = {};
+      try {
+        // domcontentloaded + response-quiet (#15), NOT waitUntil:'networkidle':
+        // a site holding a websocket/long-poll open never reaches idle, which
+        // made this goto burn its FULL 45s timeout before falling through.
+        const resp = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 45000 });
+        status = resp ? resp.status() : 200;
+        if (resp) headers = resp.headers();
+        await settle(page, { maxMs: 8000 });
+      } catch {
+        // fall back to whatever rendered before the timeout
+      }
+      const html = await page.content();
+      const finalUrl = page.url();
+      return { html, finalUrl, status, rendered: true, headers };
+    } catch (err) {
+      if (staticRes) {
+        return { html: staticRes.text, finalUrl: staticRes.finalUrl, status: staticRes.status, rendered: false, error: err, headers: staticRes.headers };
+      }
+      return { html: '', finalUrl: url, status: 0, rendered: false, error: err, headers: {} };
+    } finally {
+      if (release) await release(); // return the context to the pool (reuse its asset cache)
+    }
+  }
+
+  // Browser wanted but unavailable.
+  if (browserMode === 'always' && ctx?.emit) {
+    ctx.emit({
+      type: 'warn',
+      url,
+      reason: 'browser-missing',
+      message:
+        'Browser mode "always" was requested but Playwright is not installed. ' +
+        'Run: npm install playwright && npx playwright install chromium. Using a plain fetch instead.',
+    });
+  }
+  if (staticRes) {
+    return { html: staticRes.text, finalUrl: staticRes.finalUrl, status: staticRes.status, rendered: false, headers: staticRes.headers };
+  }
+  const res = await fetchText(url);
+  return { html: res.text, finalUrl: res.finalUrl, status: res.status, rendered: false, headers: res.headers };
+}