crawldna 0.1.0

package/src/lib/semantic.mjs

@@ -0,0 +1,172 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// #22 — the semantic relevance tier: embeddings-backed task→link scoring.
+//
+// The lexical scorer (lib/relevance.mjs) is fast, free and deterministic, but
+// blind across languages and synonyms: an Italian task ("estrai i prezzi") on a
+// German site scores "Preise" 0. When the user configures an `embedModel`, this
+// tier embeds the task once per scan and every unique link ONCE (cached, batch
+// calls), and cosine similarity becomes the relevance score — multilingual by
+// nature, and hallucination-free (embeddings emit numbers, not text).
+//
+// Precision rules (the item's contract):
+//   - embeddings ORDER always, they CUT only through the explicit `minRelevance`
+//     opt-in — exactly like the lexical scores they replace;
+//   - the AI link gate stays the judge in targeted mode; this is the ranking
+//     signal under it, not a replacement;
+//   - a GENERIC task (no topic terms) never discriminates — everything scores 1,
+//     same as lexical, so a "get everything" crawl is never reordered into a
+//     preference it didn't express;
+//   - no `embedModel`, no-AI mode, or a failing backend → the lexical floor,
+//     with ONE loud warning (never a silent degrade);
+//   - rule #6: in no-AI mode the descriptor carries no embedModel and embed()
+//     refuses to run — zero model calls of ANY kind.
+
+import { embed, llmDisabled } from './llm.mjs';
+import { scoreLink, taskTerms } from './relevance.mjs';
+
+const BATCH = 64; // texts per embed call — bounded requests, few round-trips
+
+/** Cosine similarity clamped to [0, 1] (sentence embeddings sit ≥0 in practice;
+ *  the clamp just keeps the score a valid relevance value). */
+export function cosine(a, b) {
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  const n = Math.min(a.length, b.length);
+  for (let i = 0; i < n; i++) {
+    dot += a[i] * b[i];
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  if (!na || !nb) return 0;
+  return Math.max(0, Math.min(1, dot / Math.sqrt(na * nb)));
+}
+
+/** The text a link is embedded as: its human label + nearby heading + decoded
+ *  URL path — the same signals the lexical scorer reads, as one string. */
+function linkText(link) {
+  let path = '';
+  try {
+    path = decodeURIComponent(new URL(link.href).pathname).replace(/[/_-]+/g, ' ').trim();
+  } catch {
+    path = String(link.href || '');
+  }
+  return [link.label, link.context, path].filter(Boolean).join(' — ').slice(0, 300);
+}
+
+/**
+ * Per-scan relevance scorer. `scoreAll(candidates)` returns Map<href, score 0..1>,
+ * semantic when an embedModel is configured and answering, lexical otherwise.
+ * Create ONE per scan (the task is fixed there) and reuse it — the vector cache
+ * and the one-time failure warning live on the instance.
+ *
+ * @param {object} a
+ * @param {object} a.llm       the resolveLlm descriptor (reads .embedModel)
+ * @param {string} a.task      the scan's task (the query)
+ * @param {(msg:string)=>void} [a.onWarn]  called ONCE if the backend fails
+ */
+export function createScorer({ llm, task, onWarn } = {}) {
+  const terms = taskTerms(task);
+  // Semantic only when it can help AND is allowed: an embedModel is set, AI is
+  // not disabled, and the task actually names a topic (a generic task must not
+  // discriminate — same contract as the lexical scorer's all-1s).
+  let semantic = !!(llm && !llmDisabled(llm) && llm.embedModel && terms.length);
+  const cache = new Map(); // linkText -> vector, per scan
+  let taskVec = null;
+
+  const lexicalAll = (candidates) => {
+    const out = new Map();
+    for (const c of candidates) out.set(c.href, scoreLink(terms, c).score);
+    return out;
+  };
+
+  async function vectorsFor(texts) {
+    const missing = [...new Set(texts.filter((t) => !cache.has(t)))];
+    for (let i = 0; i < missing.length; i += BATCH) {
+      const batch = missing.slice(i, i + BATCH);
+      const vecs = await embed(llm, batch);
+      batch.forEach((t, j) => cache.set(t, vecs[j]));
+    }
+  }
+
+  return {
+    get semantic() {
+      return semantic;
+    },
+    /** @param {Array<{href:string,label?:string,context?:string}>} candidates
+     *  @returns {Promise<Map<string, number>>} href → relevance score in [0,1] */
+    async scoreAll(candidates) {
+      if (!semantic) return lexicalAll(candidates);
+      try {
+        if (!taskVec) {
+          await vectorsFor([task]);
+          taskVec = cache.get(task);
+        }
+        const texts = candidates.map(linkText);
+        await vectorsFor(texts);
+        const out = new Map();
+        candidates.forEach((c, i) => out.set(c.href, cosine(taskVec, cache.get(texts[i]))));
+        return out;
+      } catch (err) {
+        // ONE loud warning, then the lexical floor for the rest of the scan —
+        // a broken embedding backend must never break (or silently skew) a crawl.
+        semantic = false;
+        if (onWarn) {
+          try {
+            onWarn(
+              `Embedding model '${llm.embedModel}' failed (${String((err && err.message) || err).slice(0, 160)}) — ` +
+                'falling back to lexical relevance for this scan.',
+            );
+          } catch {
+            /* never break scoring over a warn */
+          }
+        }
+        return lexicalAll(candidates);
+      }
+    },
+  };
+}
+
+/**
+ * #22 for the reshape retrieval (lib/retrieve.mjs): score every H1–H3 section of
+ * every document against the instruction, semantically. Returns a lookup
+ * `(di, si) => score in [0,1]`, or null when the semantic tier is off/failed —
+ * the caller then keeps the lexical retrieval unchanged.
+ *
+ * Each section is embedded by its GIST (heading + first 300 chars), not its full
+ * body — that bounds the cost to ~sections×75 tokens while still capturing what
+ * the section is about (the standard head-window trick).
+ *
+ * @param {object} a
+ * @param {object} a.llm
+ * @param {string} a.instruction
+ * @param {Array<Array<{heading:string,text:string}>>} a.docSections  sections per document,
+ *        in the SAME order/indices the caller will use (sectionizeDoc output)
+ * @returns {Promise<null | ((di:number, si:number) => number)>}
+ */
+export async function semanticSectionScores({ llm, instruction, docSections }) {
+  if (!llm || llmDisabled(llm) || !llm.embedModel) return null;
+  try {
+    const gists = [];
+    const keys = [];
+    docSections.forEach((sections, di) => {
+      sections.forEach((s, si) => {
+        keys.push(`${di}:${si}`);
+        gists.push(`${s.heading}\n${s.text.slice(0, 300)}`);
+      });
+    });
+    if (!gists.length) return null;
+    const [qv, ...rest] = await (async () => {
+      const out = [];
+      const all = [instruction, ...gists];
+      for (let i = 0; i < all.length; i += BATCH) out.push(...(await embed(llm, all.slice(i, i + BATCH))));
+      return out;
+    })();
+    const scores = new Map();
+    keys.forEach((k, i) => scores.set(k, cosine(qv, rest[i])));
+    return (di, si) => scores.get(`${di}:${si}`) ?? 0;
+  } catch {
+    return null; // lexical retrieval takes over — never break a reshape over ranking
+  }
+}

package/src/lib/settle.mjs

@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// The render-wait signal (#15): "response-quiet + stable text", bounded.
+// Shared by the initial page render (engine/crawl-page.mjs), the reveal loop's
+// post-click wait and base restore (engine/actions.mjs, engine/reveal.mjs), and
+// the browser-escalation fetch (lib/fetcher.mjs). Pure JS over the Playwright
+// page interface — no Playwright import, so it unit-tests with a fake page.
+
+/**
+ * Wait for a page's async activity to FULLY land before the caller captures,
+ * WITHOUT over-waiting when nothing is happening.
+ *
+ * The hard case that shaped it (post-click): a widget whose FIRST interaction
+ * triggers a one-time cascade of lazy-loaded scripts (a booking calendar pulls
+ * in pikaday / jquery-ui / sweetalert / recaptcha the first time a day is
+ * picked) that delays the real content — the day's slot grid — by ~1s, during
+ * which the visible TEXT sits on a flat plateau. A DOM-stability poll is fooled
+ * by that plateau and snapshots the page without the slots; `networkidle` with
+ * a fixed timeout either under- or over-waits and, when it over-waits,
+ * desynchronises the click→reveal→restore rhythm so per-day panels are lost.
+ *
+ * The reliable signal is the RESPONSE STREAM: real content arrives on a network
+ * response, so wait until a response has come back AND the network has then
+ * been quiet for a short grace window, with the DOM text no longer changing.
+ * An action that fetches waits exactly as long as its cascade runs; one that
+ * fetches NOTHING sees no response and falls through after the same short
+ * grace — so no-ops stay cheap. Bounded by maxMs so recaptcha/ads heartbeats
+ * can't stall the crawl. Generic — no per-site assumptions.
+ *
+ * Crucially for the initial render (#15), quietness counts response EVENTS —
+ * not open connections like `networkidle` does — so a site holding a
+ * websocket / SSE / long-poll connection open (where the idle signal NEVER
+ * fires and a networkidle wait burned its full timeout on every page) exits
+ * after one grace window like any quiet page.
+ *
+ * @param {import('playwright').Page} page  anything with on/off('response'),
+ *   waitForTimeout(ms) and evaluate(fn) — duck-typed for testability
+ */
+export async function settle(page, { maxMs = 4500, graceMs = 650, intervalMs = 120 } = {}) {
+  const start = Date.now();
+  let sawResponse = false;
+  let lastResponse = start;
+  const onResponse = () => {
+    sawResponse = true;
+    lastResponse = Date.now();
+  };
+  page.on('response', onResponse);
+  try {
+    let prevLen = -1;
+    while (Date.now() - start < maxMs) {
+      await page.waitForTimeout(intervalMs);
+      let len;
+      try {
+        len = await page.evaluate(() => document.body.innerText.length);
+      } catch {
+        return;
+      }
+      // "Quiet" = the grace window has passed since the last response (or, if no
+      // response ever came, since the start). Settled once it is quiet AND the
+      // text has stopped changing — so a late render that trails the final
+      // response still gets one more poll before we conclude.
+      const since = sawResponse ? Date.now() - lastResponse : Date.now() - start;
+      if (since >= graceMs && len === prevLen) return;
+      prevLen = len;
+    }
+  } finally {
+    page.off('response', onResponse);
+  }
+}

package/src/lib/simhash.mjs

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// 64-bit SimHash (Charikar) — a near-duplicate fingerprint for text. Pure JS, no
+// dependencies. Two documents that differ only in boilerplate/template produce
+// fingerprints a small Hamming distance apart; genuinely different documents are far
+// apart. Used (opt-in) to collapse near-duplicate pages the exact-hash dedup misses.
+//
+// Why a fingerprint and not a full diff: it is O(1) to compare (one XOR + popcount over
+// 64 bits) and O(tokens) to build, so a whole crawl can be de-duped cheaply. Evidence:
+// Manku/Google "Detecting Near-Duplicates for Web Crawling"; Charikar SimHash.
+//
+// A fingerprint is `{ hi, lo }` — two unsigned 32-bit halves of the 64-bit value — so the
+// hot path (build + Hamming) stays on fast 32-bit integer math, never BigInt.
+
+/** MurmurHash3-style 32-bit hash over a string's UTF-16 code units. Good bit mixing (the
+ *  finalizer avalanches), deterministic, fast — ample for fingerprint features. */
+function murmur3_32(str, seed) {
+  let h = seed >>> 0;
+  for (let i = 0; i < str.length; i++) {
+    let k = Math.imul(str.charCodeAt(i), 0xcc9e2d51);
+    k = (k << 15) | (k >>> 17);
+    k = Math.imul(k, 0x1b873593);
+    h ^= k;
+    h = (h << 13) | (h >>> 19);
+    h = (Math.imul(h, 5) + 0xe6546b64) | 0;
+  }
+  h ^= str.length;
+  h ^= h >>> 16;
+  h = Math.imul(h, 0x85ebca6b);
+  h ^= h >>> 13;
+  h = Math.imul(h, 0xc2b2ae35);
+  h ^= h >>> 16;
+  return h >>> 0;
+}
+
+/** A 64-bit hash of a feature, as two independent 32-bit halves. */
+function hash64(str) {
+  return { hi: murmur3_32(str, 0x9747b28c), lo: murmur3_32(str, 0x01000193) };
+}
+
+/** Lowercased alphanumeric tokens (length ≥ 2). */
+function tokenize(text) {
+  const out = [];
+  for (const t of String(text || '').toLowerCase().split(/[^a-z0-9]+/)) {
+    if (t.length >= 2) out.push(t);
+  }
+  return out;
+}
+
+/**
+ * Weighted features for the fingerprint: word SHINGLES (n-grams) counted by frequency.
+ * Shingles (default bigrams) localise a change to a couple of features, so a small edit
+ * flips only a few fingerprint bits — the property near-dup detection relies on. Falls
+ * back to unigrams for very short texts.
+ */
+function features(text, shingle) {
+  const toks = tokenize(text);
+  const map = new Map();
+  const n = Math.max(1, shingle);
+  if (toks.length < n) {
+    for (const t of toks) map.set(t, (map.get(t) || 0) + 1);
+  } else {
+    for (let i = 0; i + n <= toks.length; i++) {
+      const key = toks.slice(i, i + n).join(' ');
+      map.set(key, (map.get(key) || 0) + 1);
+    }
+  }
+  return map;
+}
+
+/**
+ * Compute the 64-bit SimHash of `text`.
+ * @param {string} text
+ * @param {{ shingle?: number }} [opts]  n-gram size for features (default 2)
+ * @returns {{ hi: number, lo: number }}  the fingerprint (two unsigned 32-bit halves)
+ */
+export function simhash(text, { shingle = 2 } = {}) {
+  const feats = features(text, shingle);
+  const v = new Array(64).fill(0); // per-bit weighted vote
+  for (const [feat, w] of feats) {
+    const { hi, lo } = hash64(feat);
+    for (let i = 0; i < 32; i++) {
+      v[i] += (lo >>> i) & 1 ? w : -w;
+      v[i + 32] += (hi >>> i) & 1 ? w : -w;
+    }
+  }
+  let lo = 0;
+  let hi = 0;
+  for (let i = 0; i < 32; i++) {
+    if (v[i] > 0) lo |= 1 << i;
+    if (v[i + 32] > 0) hi |= 1 << i;
+  }
+  return { hi: hi >>> 0, lo: lo >>> 0 };
+}
+
+/** Count set bits in a 32-bit integer (SWAR popcount). */
+function popcount32(x) {
+  x = x - ((x >>> 1) & 0x55555555);
+  x = (x & 0x33333333) + ((x >>> 2) & 0x33333333);
+  x = (x + (x >>> 4)) & 0x0f0f0f0f;
+  return (Math.imul(x, 0x01010101) >>> 24);
+}
+
+/** Hamming distance (0..64) between two fingerprints. */
+export function hamming(a, b) {
+  return popcount32((a.hi ^ b.hi) >>> 0) + popcount32((a.lo ^ b.lo) >>> 0);
+}
+
+/** True when two fingerprints are within `maxHamming` bits — i.e. near-duplicates. */
+export function isNearDup(a, b, maxHamming) {
+  return hamming(a, b) <= maxHamming;
+}

package/src/lib/task.mjs

@@ -0,0 +1,65 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Task classification shared by the orchestrator and the engine.
+
+/** The valid values of the `mode` option (#20). */
+export const MODES = ['auto', 'complete', 'targeted'];
+
+/**
+ * #20 — resolve the EXPLICIT `mode` option into the three engine switches it
+ * controls. This is the single place where "what kind of crawl is this?" is
+ * decided, so the engine never re-derives it (and never sniffs the task prose
+ * outside of 'auto'):
+ *
+ *   - `docsShortcuts`: try the completeness shortcuts first (llms-full.txt /
+ *     sitemap seeding via the docs profile) instead of pure discovery crawling.
+ *   - `scopeSections`: run aiScopeContent per page (keep only task-relevant
+ *     sections, verbatim). Off = pages are always kept WHOLE.
+ *   - `linkGate`: send discovered links to the AI link gate (aiSelectLinks).
+ *     Off = every in-scope link is followed, ZERO gate calls — keep/drop has no
+ *     meaning when the user asked for everything, and the mirror/variant dedup
+ *     (default-on) is what keeps follow-everything contained.
+ *
+ * `complete` = "the whole site": shortcuts on, pages whole, no gate — AI (when
+ * enabled) still drives reveal + nav-plan, the jobs that find hidden content.
+ * `targeted` = "only what the task asks": gate + scoping on, regardless of how
+ * the task is phrased. Requires AI (refused with `noAi` — enforced upstream in
+ * crawlDocs, never silently).
+ * `auto` = the historical behaviour, kept ONLY for backward compatibility
+ * (library callers, saved runs, resume): the isDocsTask regex below decides.
+ * Anything unrecognised resolves to 'auto' here; crawlDocs validates first and
+ * rejects unknown values loudly.
+ */
+export function modeBehavior(mode, task) {
+  const m = mode === 'complete' || mode === 'targeted' ? mode : 'auto';
+  if (m === 'complete') return { mode: m, docsShortcuts: true, scopeSections: false, linkGate: false };
+  if (m === 'targeted') return { mode: m, docsShortcuts: false, scopeSections: true, linkGate: true };
+  const docs = isDocsTask(task);
+  return { mode: 'auto', docsShortcuts: docs, scopeSections: !docs, linkGate: true };
+}
+
+/**
+ * Does the task ask for a software/product DOCUMENTATION site (developer docs, API/SDK
+ * reference, guides) — the completeness-first docs path — versus a specific data task
+ * (a menu, prices, a list)? This is a deterministic backstop that reads the USER'S
+ * INSTRUCTION, never the website, so it stays universal (no per-site rules).
+ *
+ * #20: since the explicit `mode` option exists, this regex is consulted ONLY by
+ * `modeBehavior('auto', …)` — the backward-compatibility path. New callers (and
+ * the UI) pass mode 'complete' or 'targeted' and never reach it.
+ *
+ * It is MULTILINGUAL by design: it matches the documentation STEM rather than one
+ * language's spelling, so it works whatever language the task is written in (Latin
+ * script). `documenta` covers documentation / documentazione / documentación /
+ * documentação / documentatie; `dokumenta` covers German/Nordic (Dokumentation); plus
+ * `docs`, `api reference`, and `sdk`. The stem (not a bare "document") avoids false
+ * positives on data tasks like "extract the documents list".
+ *
+ * Why it matters: a true verdict (a) picks the docs profile (llms-full.txt / sitemap →
+ * complete, fast) and (b) keeps pages WHOLE (skips the per-section scoping meant for
+ * specific tasks). A non-English task wrongly read as "not docs" loses both — it was the
+ * bug where an Italian "documentazione" crawl explored blindly and trimmed sections.
+ */
+export function isDocsTask(task) {
+  return /\bdocs?\b|documenta|dokumenta|api[\s_-]*reference|\bsdk\b/i.test(task || '');
+}

package/src/lib/url.mjs

@@ -0,0 +1,155 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// URL helpers: normalise, dedupe, scope. Pure, no dependencies.
+
+/** Coerce a string|RegExp filter into a RegExp (or null). */
+export function toRegExp(pattern) {
+  if (!pattern) return null;
+  if (pattern instanceof RegExp) return pattern;
+  try {
+    return new RegExp(String(pattern));
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Normalise a URL for dedup/comparison:
+ * - resolve against `base` when relative
+ * - FRAGMENT policy: a plain id anchor (#install, #main-content, #step-3) just
+ *   points at a SECTION of the SAME page — keeping it makes the crawler fetch one
+ *   page many times over (the single biggest source of wasted work on docs sites,
+ *   e.g. firebase get-started#add-sdk / #kotlin / #next-steps as "separate pages").
+ *   A hash-ROUTE (#/contact, #!/features) IS a real separate page in a hash-routed
+ *   SPA. So we keep only route-like fragments (`#/…` or `#!…`) and drop plain
+ *   anchors, collapsing `page#a`, `page#b` and `page` to one. Content-dedup stays
+ *   as a second safety net.
+ * - lowercase the host
+ * - strip a trailing slash (except the root path)
+ * Returns null when the input cannot be parsed.
+ */
+// Query params that are tracking/analytics/locale noise — never select content.
+// (Content-selecting params like `api`, `tab`, `version` are deliberately kept.)
+const STRIP_PARAMS = new Set([
+  'utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'utm_id',
+  'gclid', 'fbclid', 'msclkid', 'mc_cid', 'mc_eid', '_ga', '_hsenc', '_hsmi',
+  'ref', 'ref_src', 'igshid', 'hl', 'lang', 'locale',
+]);
+// Some trackers append DYNAMIC param names (Google Analytics linker `_gl`, the
+// per-stream `_ga_XXXXXXX`, GA4 `_up`). Match these by PREFIX so a session id baked
+// into the name can't dodge the filter and spawn a duplicate URL of the same page.
+const STRIP_PARAM_PREFIXES = ['_ga', '_gl', '_up', 'utm_', 'mc_'];
+
+export function normalizeUrl(input, base) {
+  let u;
+  try {
+    u = new URL(input, base);
+  } catch {
+    return null;
+  }
+  if (u.protocol !== 'http:' && u.protocol !== 'https:') return null;
+  // A path that BEGINS with another absolute URL (`/https://other.site/…`) is the
+  // signature of a broken join — an absolute href glued onto a base — never a real
+  // route (seen live: https://0.vuetifyjs.com/https://v0play.vuetifyjs.com, a 404).
+  // Only the path PREFIX is rejected: nested URLs deeper in the path (Wayback-style
+  // /web/<ts>/https://…) or in the query (?redirect=https://…) are legitimate.
+  let decodedPath = u.pathname;
+  try {
+    decodedPath = decodeURIComponent(decodedPath);
+  } catch {
+    /* malformed %-escape: judge the raw path */
+  }
+  if (/^\/https?:\//i.test(decodedPath)) return null;
+  // Drop a plain in-page anchor; keep a hash ROUTE (#/… or #!…). See note above.
+  if (u.hash && !/^#[!/]/.test(u.hash)) u.hash = '';
+  u.hostname = u.hostname.toLowerCase();
+  if (u.pathname.length > 1) u.pathname = u.pathname.replace(/\/+$/, '');
+  for (const k of [...u.searchParams.keys()]) {
+    const lk = k.toLowerCase();
+    if (STRIP_PARAMS.has(lk) || STRIP_PARAM_PREFIXES.some((p) => lk.startsWith(p))) {
+      u.searchParams.delete(k);
+    }
+  }
+  return u.toString();
+}
+
+export function hostOf(u) {
+  try {
+    return new URL(u).hostname.toLowerCase();
+  } catch {
+    return '';
+  }
+}
+
+export function originOf(u) {
+  try {
+    return new URL(u).origin;
+  } catch {
+    return '';
+  }
+}
+
+export function pathOf(u) {
+  try {
+    return new URL(u).pathname;
+  } catch {
+    return '';
+  }
+}
+
+/** True when `url`'s host is the base host or a subdomain of it (not the parent). */
+export function sameSite(url, baseUrl) {
+  const a = hostOf(url);
+  const b = hostOf(baseUrl);
+  if (!a || !b) return false;
+  return a === b || a.endsWith('.' + b);
+}
+
+/**
+ * Decide whether `url` is in scope for a crawl rooted at `baseUrl`.
+ * - `exclude` wins if it matches.
+ * - if `include` is set, the URL must match it.
+ * - otherwise default scope is the same site (host or subdomain).
+ */
+export function inScope(url, baseUrl, { include, exclude } = {}) {
+  const exc = toRegExp(exclude);
+  if (exc && exc.test(url)) return false;
+  const inc = toRegExp(include);
+  if (inc) return inc.test(url);
+  return sameSite(url, baseUrl);
+}
+
+/** Resolve a possibly-relative href against a base URL; null if invalid. */
+export function resolveUrl(href, base) {
+  return normalizeUrl(href, base);
+}
+
+/**
+ * Key that groups URL-SIBLINGS: URLs whose path is the same once a leading
+ * locale-like segment (`/en/x`, `/pt-br/x` → `/x`) is stripped — so the same
+ * logical document reached via a mirror host (dev./staging./v2.), a UI-state
+ * query variant (`?panel=settings`), a hash-route twin, or a locale twin all
+ * share one key. Host and query are deliberately ignored: the frontier is
+ * already confined to one site by `inScope`, so a same-key page on another
+ * host is a same-site mirror, not a stranger. Sharing a key is only a HINT —
+ * callers must also check content (SimHash) before treating two pages as
+ * duplicates: e.g. `?version=1` vs `?version=2` share a key but genuinely
+ * differ, and measured Hamming distances keep them apart (see mirrorHamming).
+ */
+export function siblingKey(u) {
+  try {
+    const p = new URL(u).pathname.replace(/\/+$/, '') || '/';
+    return p.replace(/^\/[a-z]{2}(-[a-z0-9]{2,8})?(?=\/|$)/i, '') || '/';
+  } catch {
+    return '';
+  }
+}
+
+/** A filesystem-safe slug derived from arbitrary text. */
+export function slug(text) {
+  return String(text || '')
+    .toLowerCase()
+    .replace(/[^\w]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 80) || 'section';
+}

package/src/profiles/docs/llms.mjs

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Tier 1 of the documentation profile: /llms-full.txt.
+//
+// If a site publishes /llms-full.txt it already contains the entire docs set as
+// clean Markdown. We split it by sections and we are done — no browser, no
+// per-page fetching. /llms.txt (without -full) is only a curated index and is
+// never treated as the complete page list.
+
+import { fetchText } from '../../lib/fetcher.mjs';
+import { originOf, slug } from '../../lib/url.mjs';
+
+function looksLikeHtml(text) {
+  return /^\s*<(?:!doctype|html|\?xml)/i.test(text);
+}
+
+/**
+ * Find an *explicit* source URL declared for a section (a `Source:`/`URL:`
+ * marker, optionally inside a blockquote, optionally as a markdown link).
+ * We deliberately do NOT fall back to the first arbitrary link in the body —
+ * that picks up unrelated outbound links and mislabels the page.
+ */
+function findSource(md) {
+  // Only look near the top of the section.
+  const head = md.split(/\r?\n/).slice(0, 6).join('\n');
+  const m = head.match(/^\s*>?\s*(?:source|url|canonical)\s*[:=]\s*(?:\[[^\]]*\]\()?\s*<?(https?:\/\/[^\s)>]+)/im);
+  return m ? m[1] : null;
+}
+
+/** Split a big markdown document on heading lines matching `re`. */
+function splitOnHeading(text, re) {
+  const lines = text.split(/\r?\n/);
+  const sections = [];
+  let cur = null;
+  for (const line of lines) {
+    const m = re.exec(line);
+    if (m) {
+      if (cur) sections.push(cur);
+      cur = { title: m[1].trim(), lines: [line] };
+    } else {
+      if (!cur) cur = { title: '', lines: [] };
+      cur.lines.push(line);
+    }
+  }
+  if (cur) sections.push(cur);
+  return sections;
+}
+
+/**
+ * Fetch and split /llms-full.txt.
+ * @returns {Promise<null | { sourceUrl: string, pages: Array<{url,title,markdown}> }>}
+ */
+export async function tryLlmsFull(baseUrl) {
+  const origin = originOf(baseUrl);
+  if (!origin) return null;
+  const sourceUrl = origin + '/llms-full.txt';
+
+  const res = await fetchText(sourceUrl, { accept: 'text/plain, text/markdown, */*' });
+  if (!res.ok || !res.text || res.text.length < 500) return null;
+  if (looksLikeHtml(res.text)) return null; // a soft-404 served as HTML
+
+  // Prefer H1 boundaries; if the doc is one big H1-less blob, fall back to H2.
+  let sections = splitOnHeading(res.text, /^#\s+(.+)/);
+  if (sections.length < 2) sections = splitOnHeading(res.text, /^##\s+(.+)/);
+
+  const pages = sections
+    .map((s, i) => {
+      const markdown = s.lines.join('\n').trim();
+      const title = s.title || `Section ${i + 1}`;
+      const url = findSource(markdown) || `${origin}/#${slug(title)}`;
+      return { url, title, markdown };
+    })
+    .filter((p) => p.markdown.length > 0);
+
+  if (!pages.length) return null;
+  return { sourceUrl, pages };
+}