crawldna 0.1.0

package/src/lib/relevance.mjs

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Task-relevance scoring for discovered links — the universal, dependency-free core
+// of "information foraging" (focused crawling). Given the user's TASK and a link
+// (its URL + anchor label + nearby heading), it returns how related the link looks to
+// the task, using only TEXT the user wrote and text on the page — never any per-site or
+// per-framework URL-shape rule. The crawler uses this to:
+//   - follow the most on-task links FIRST (best-first frontier), and
+//   - OPTIONALLY prune clearly off-task links before the AI gate (focused mode, opt-in).
+// It NEVER decides to drop a link on its own in the default configuration: scoring only
+// reorders, and pruning happens only when the caller sets `minRelevance > 0`. The AI
+// link gate stays the primary judge; this is a cheap signal that helps it, not replaces
+// it. Aligned with: precision over speed (no drop by default), universal (no per-site
+// rules), task-driven (the task is the query).
+
+// Words that frame the REQUEST rather than name the TOPIC, plus URL noise. Kept
+// deliberately small and conservative — we stop "extract"/"di"/"www", never topic nouns
+// like "documentation", "menu", "prices", "auth". Multilingual (en + it) because tasks
+// here are often Italian.
+const STOP = new Set([
+  // english articles / prepositions / aux
+  'the', 'a', 'an', 'and', 'or', 'of', 'for', 'to', 'in', 'on', 'with', 'from', 'by',
+  'as', 'at', 'is', 'are', 'be', 'this', 'that', 'these', 'those', 'all', 'any', 'it',
+  'its', 'your', 'you', 'we', 'our', 'into',
+  // italian articles / prepositions
+  'di', 'la', 'il', 'lo', 'le', 'gli', 'un', 'una', 'uno', 'e', 'o', 'per', 'con', 'da',
+  'del', 'della', 'dei', 'degli', 'delle', 'al', 'alla', 'allo', 'ai', 'agli', 'alle',
+  'che', 'come', 'su', 'nel', 'nella', 'questo', 'questa', 'tutti', 'tutte', 'suo', 'sua',
+  // request-framing verbs (asking, not topic)
+  'extract', 'estrai', 'estrarre', 'get', 'find', 'fetch', 'scrape', 'use', 'usare',
+  'using', 'give', 'list', 'crawl', 'collect', 'gather', 'complete', 'completa',
+  'completo', 'completi', 'full', 'whole', 'entire', 'everything', 'tutto', 'tutta',
+  // url / locale noise
+  'www', 'http', 'https', 'com', 'org', 'net', 'html', 'htm', 'php', 'aspx', 'index',
+  'en', 'us',
+]);
+
+// CJK scripts (Han + kana + Hangul) carry no word boundaries, so word-splitting
+// alone yields one giant unusable token — those runs are emitted as character
+// BIGRAMS instead, the standard boundary-free trick, so two texts about the same
+// thing share tokens ("提取价格" and "价格表" meet on 价格).
+const CJK_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+const CJK_SPLIT_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+|[^\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu;
+
+/** Split arbitrary text into lowercased word tokens (camelCase-aware), minus stopwords,
+ *  pure numbers and 1-char fragments. Pure/deterministic.
+ *
+ *  #22 (the one lexical upgrade kept — the n-gram/IDF rework was evaluated and
+ *  REJECTED): Unicode-aware instead of ASCII-only. Accented words survive with
+ *  their diacritics FOLDED (NFKD, combining marks stripped) so "menù"/"menu" and
+ *  "documentación"/"documenta…" connect; Cyrillic/Greek tokenize as words; CJK
+ *  runs become character bigrams. The old ASCII splitter silently DESTROYED all
+ *  of these ("perché" → "perch", Cyrillic → nothing). Cross-language synonyms
+ *  remain the semantic tier's job (lib/semantic.mjs) — this only stops the
+ *  tokenizer from eating non-English text. */
+export function tokenize(text) {
+  const out = [];
+  const raw = String(text || '')
+    .replace(/([\p{Ll}\p{N}])(\p{Lu})/gu, '$1 $2') // camelCase → camel Case
+    .normalize('NFKD')
+    .replace(/\p{M}+/gu, ''); // fold diacritics; leaves Cyrillic/Greek/CJK intact
+  for (const tok of raw.toLowerCase().split(/[^\p{L}\p{N}]+/u)) {
+    if (!tok) continue;
+    if (CJK_RE.test(tok)) {
+      for (const run of tok.match(CJK_SPLIT_RE) || []) {
+        if (CJK_RE.test(run)) {
+          if (run.length === 1) out.push(run);
+          else for (let i = 0; i < run.length - 1; i++) out.push(run.slice(i, i + 2));
+        } else if (run.length >= 2 && !/^\d+$/.test(run) && !STOP.has(run)) {
+          out.push(run);
+        }
+      }
+      continue;
+    }
+    if (tok.length < 2) continue;
+    if (/^\d+$/.test(tok)) continue; // version/page numbers are noise here
+    if (STOP.has(tok)) continue;
+    out.push(tok);
+  }
+  return out;
+}
+
+/** Tokens mined from a URL: its path segments + query keys/values (host ignored — it's
+ *  the same site, so it carries no discriminating signal). */
+export function urlTokens(href) {
+  try {
+    const u = new URL(href);
+    const q = [...u.searchParams.entries()].map(([k, v]) => `${k} ${v}`).join(' ');
+    return tokenize(decodeURIComponent(u.pathname) + ' ' + q);
+  } catch {
+    return tokenize(href);
+  }
+}
+
+/** The distinct topic terms of a task (the "query"). Empty when the task is purely
+ *  generic ("extract everything") — callers treat that as "don't discriminate". */
+export function taskTerms(task) {
+  return [...new Set(tokenize(task))];
+}
+
+/** Does link-token `l` satisfy task-term `t`? Exact match, or a shared prefix for
+ *  longer words so "price"/"prices" and "document"/"documentazione" still connect — a
+ *  light, language-agnostic stemming with no per-language rules.
+ *  (Exported for reuse by the reshape context retrieval, lib/retrieve.mjs.) */
+export function termHit(t, l) {
+  if (l === t) return true;
+  return t.length >= 4 && l.length >= 4 && (l.startsWith(t) || t.startsWith(l));
+}
+
+/**
+ * Score one link's relevance to the task in [0, 1].
+ *   0   = shares no topic term with the task,
+ *   0.5 = shares one,
+ *   1   = shares two or more (saturates — one strong, on-topic match is enough).
+ * When the task has NO topic terms (fully generic), returns 1 for everything so scoring
+ * never disrupts a generic crawl.
+ *
+ * @param {string[]} terms  result of taskTerms(task) (precompute once per scan)
+ * @param {{href?:string, label?:string, context?:string}} link
+ * @returns {{ score:number, matched:number }}
+ */
+export function scoreLink(terms, link) {
+  if (!terms || terms.length === 0) return { score: 1, matched: 0 };
+  const toks = new Set([
+    ...urlTokens(link.href || ''),
+    ...tokenize(link.label || ''),
+    ...tokenize(link.context || ''),
+  ]);
+  let matched = 0;
+  for (const t of terms) {
+    for (const l of toks) {
+      if (termHit(t, l)) {
+        matched++;
+        break;
+      }
+    }
+  }
+  return { score: Math.min(1, matched / 2), matched };
+}

package/src/lib/retrieve.mjs

@@ -0,0 +1,165 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Context retrieval for reshape (#11, root cause): when the crawled sources exceed
+// the model budget, choose WHICH verbatim slices fill it — instead of blindly sending
+// the first N characters and letting the model "answer" the rest from its own memory
+// (observed live: a 2.7MB Vuetify extraction, "v-alert props" past the cap → the model
+// fabricated a plausible props table with wrong defaults, silently).
+//
+// Universal and deterministic: sections are scored against the USER'S INSTRUCTION with
+// the same task-tokenisation the crawler uses for link relevance (no per-site rules,
+// no embeddings, no dependencies). The AI stays the judge of the ANSWER; this only
+// decides what it gets to read. Content is never rewritten — sections are passed
+// verbatim, in document order, with omissions marked.
+
+import { tokenize, termHit } from './relevance.mjs';
+
+const norm = (s) => String(s || '').toLowerCase();
+
+/** Split Markdown into H1–H3 sections (fence-aware). Each section's `text` is the
+ *  verbatim slice including its heading line; content before the first heading becomes
+ *  an "(intro)" section. Mirrors the crawl's scoping granularity. */
+export function sectionizeDoc(markdown) {
+  const lines = String(markdown || '').split('\n');
+  const sections = [];
+  let cur = { heading: '(intro)', lines: [] };
+  let inFence = false;
+  for (const line of lines) {
+    if (/^\s*(```|~~~)/.test(line)) inFence = !inFence;
+    const h = !inFence && line.match(/^(#{1,3})\s+(.*)/);
+    if (h) {
+      if (cur.lines.length || sections.length === 0) sections.push(cur);
+      cur = { heading: h[2].trim().slice(0, 120), lines: [line] };
+    } else {
+      cur.lines.push(line);
+    }
+  }
+  sections.push(cur);
+  return sections
+    .map((s) => ({ heading: s.heading, text: s.lines.join('\n').trim() }))
+    .filter((s) => s.text);
+}
+
+/** How relevant one section is to the query terms: heading hits weigh most (a section
+ *  ABOUT the topic), body occurrences add up (capped, so a long page can't drown a
+ *  focused one). 0 = shares nothing with the request. */
+function scoreSection(section, terms, headTokens) {
+  let score = 0;
+  const bodyLC = norm(section.text);
+  for (const t of terms) {
+    if (headTokens.some((k) => termHit(t, k))) score += 3;
+    let count = 0;
+    let idx = 0;
+    while (count < 5 && (idx = bodyLC.indexOf(t, idx)) !== -1) {
+      count++;
+      idx += t.length;
+    }
+    score += count;
+  }
+  return score;
+}
+
+/**
+ * Choose the source content that fits a character budget, most-relevant first.
+ *
+ * @param {Array<{filename?:string, bytes?:number, content:string}>} documents
+ * @param {string} instruction  the user's request (the query)
+ * @param {number} budget       character budget for the combined content
+ * @param {(di:number, si:number, section:{heading:string,text:string}) => number} [sectionScore]
+ *        #22 — optional external scorer (the semantic tier: cosine 0..1 per
+ *        section). When given, it REPLACES the lexical term scoring — so a
+ *        cross-language request still ranks the right sections — while the
+ *        explicit document-reference boost and the packing stay identical.
+ * @returns {{
+ *   docs: Array<object>,   the documents to show (content possibly a verbatim subset;
+ *                          `partial: true` marks a doc whose sections were omitted)
+ *   truncated: boolean,    whether ANYTHING had to be left out
+ *   mode: 'full'|'retrieval'|'head',
+ *   omittedDocs: number    documents left out entirely (nothing relevant in them)
+ * }}
+ *
+ * Modes: 'full' = everything fits, untouched. 'retrieval' = the instruction's terms
+ * (or an explicit document reference by filename/byte size) selected the sections.
+ * 'head' = nothing to discriminate by — the caller keeps the legacy head-slice.
+ */
+export function selectRelevant(documents, instruction, budget, sectionScore = null) {
+  const docs = (documents || []).map((d) => ({ ...d }));
+  const total = docs.reduce((n, d) => n + String(d.content || '').length, 0);
+  if (total <= budget) return { docs, truncated: false, mode: 'full', omittedDocs: 0 };
+
+  const terms = tokenize(instruction);
+  // A document the user names explicitly — by filename or by its byte size (people do:
+  // "the original 2788831b") — is wanted regardless of term overlap: boost all of it.
+  const instrLC = norm(instruction);
+  const referenced = new Set();
+  docs.forEach((d, i) => {
+    if (d.filename && instrLC.includes(norm(d.filename))) referenced.add(i);
+    else if (d.bytes && instrLC.includes(String(d.bytes))) referenced.add(i);
+  });
+
+  // A semantic scorer can discriminate even when no lexical term overlaps
+  // (cross-language requests) — only bail to the head-slice when NOTHING can.
+  if (!terms.length && referenced.size === 0 && !sectionScore) {
+    return { docs, truncated: true, mode: 'head', omittedDocs: 0 };
+  }
+
+  // Score every section of every document against the instruction. The semantic
+  // score (cosine 0..1) is scaled ×10 so its ordering is meaningful next to the
+  // ×100 referenced-document boost, which must keep packing first.
+  const pool = [];
+  const perDocCount = new Array(docs.length).fill(0);
+  docs.forEach((d, di) => {
+    for (const [si, s] of sectionizeDoc(d.content).entries()) {
+      perDocCount[di]++;
+      const base = referenced.has(di) ? 100 : 0; // a named doc packs first, in order
+      const score =
+        base +
+        (sectionScore
+          ? Math.max(0, Number(sectionScore(di, si, s)) || 0) * 10
+          : terms.length
+            ? scoreSection(s, terms, tokenize(s.heading))
+            : 0);
+      pool.push({ di, si, score, text: s.text });
+    }
+  });
+  const scored = pool.filter((p) => p.score > 0);
+  if (!scored.length) return { docs, truncated: true, mode: 'head', omittedDocs: 0 };
+
+  // Greedy pack by score; ties resolve to document order so the result is stable.
+  scored.sort((a, b) => b.score - a.score || a.di - b.di || a.si - b.si);
+  let remaining = budget;
+  const chosen = [];
+  for (const p of scored) {
+    const size = p.text.length;
+    if (size <= remaining) {
+      chosen.push(p);
+      remaining -= size;
+    } else if (chosen.length === 0) {
+      // The single most relevant section alone exceeds the budget: take its head so
+      // the model always gets SOMETHING on-topic rather than nothing.
+      chosen.push({ ...p, text: p.text.slice(0, remaining) });
+      remaining = 0;
+    }
+    if (remaining < 200) break;
+  }
+
+  // Reassemble each document's chosen sections in DOCUMENT order, with omissions
+  // marked, so the model reads coherent (if abridged) documents.
+  const out = [];
+  docs.forEach((d, di) => {
+    const mine = chosen.filter((p) => p.di === di).sort((a, b) => a.si - b.si);
+    if (!mine.length) return;
+    const partial = mine.length < perDocCount[di];
+    out.push({
+      ...d,
+      content: mine.map((p) => p.text).join('\n\n[… sections not relevant to this request were omitted …]\n\n'),
+      partial,
+    });
+  });
+  return {
+    docs: out,
+    truncated: out.length < docs.length || out.some((d) => d.partial),
+    mode: 'retrieval',
+    omittedDocs: docs.length - out.length,
+  };
+}

package/src/lib/robots.mjs

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// #14 — politeness, OPT-IN (the tool stays user-directed, like wget):
+//   - a minimum per-HOST gap between requests (`delay`), reserved-slot style so
+//     concurrent workers queue behind each other instead of bursting;
+//   - robots.txt reading (`respectRobots`): Disallow/Allow with Google's
+//     longest-match semantics plus Crawl-delay. Disallowed URLs are SKIPPED WITH
+//     A WARNING — never silently (the warning is the contract).
+// Both are off by default: existing behaviour is byte-identical until the user
+// asks. Pure parsing lives here (offline-testable); fetching the file is the
+// caller's job (one request per origin, cached for the run).
+
+/**
+ * Parse robots.txt and return the rule group that applies to `ua`.
+ * Group selection follows the standard: the MOST SPECIFIC matching User-agent
+ * token wins (longest token contained in our UA); `*` is the fallback.
+ *
+ * @param {string} text  the robots.txt body
+ * @param {string} [ua]  our user-agent product token
+ * @returns {{ rules: Array<{type:'allow'|'disallow', path:string}>, crawlDelay: number|null }}
+ */
+export function parseRobots(text, ua = 'crawldna') {
+  const groups = [];
+  let cur = null;
+  let lastWasAgent = false;
+  for (const raw of String(text || '').split(/\r?\n/)) {
+    const line = raw.replace(/#.*$/, '').trim();
+    if (!line) continue;
+    const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
+    if (!m) continue;
+    const key = m[1].toLowerCase();
+    const value = m[2].trim();
+    if (key === 'user-agent') {
+      // consecutive User-agent lines share one group; a rule line closes it
+      if (!cur || !lastWasAgent) {
+        cur = { agents: [], rules: [], crawlDelay: null };
+        groups.push(cur);
+      }
+      cur.agents.push(value.toLowerCase());
+      lastWasAgent = true;
+      continue;
+    }
+    lastWasAgent = false;
+    if (!cur) continue; // rules before any User-agent line are invalid — ignored
+    if (key === 'disallow' || key === 'allow') cur.rules.push({ type: key, path: value });
+    else if (key === 'crawl-delay') {
+      const n = Number(value);
+      if (Number.isFinite(n) && n > 0) cur.crawlDelay = n;
+    }
+  }
+
+  const uaLC = String(ua || '').toLowerCase();
+  let best = null;
+  let bestLen = -1;
+  for (const g of groups) {
+    for (const a of g.agents) {
+      if (a === '*') {
+        if (bestLen < 0) best = g; // fallback only when nothing specific matched
+      } else if (uaLC.includes(a) && a.length > bestLen) {
+        best = g;
+        bestLen = a.length;
+      }
+    }
+  }
+  return best ? { rules: best.rules, crawlDelay: best.crawlDelay } : { rules: [], crawlDelay: null };
+}
+
+/** Does a single robots path rule (with `*` wildcards and a `$` end anchor)
+ *  match this URL path? An empty rule path matches nothing (per the spec,
+ *  "Disallow:" with no value allows everything). */
+function ruleMatches(rulePath, path) {
+  if (!rulePath) return false;
+  const anchored = rulePath.endsWith('$');
+  const body = anchored ? rulePath.slice(0, -1) : rulePath;
+  const re = body
+    .split('*')
+    .map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))
+    .join('[\\s\\S]*');
+  return new RegExp('^' + re + (anchored ? '$' : '')).test(path);
+}
+
+/**
+ * Is this URL path allowed by the parsed rules? Google semantics: the matching
+ * rule with the LONGEST path wins; on a tie, Allow beats Disallow; no matching
+ * rule = allowed.
+ *
+ * @param {Array<{type:string, path:string}>} rules  from parseRobots().rules
+ * @param {string} path  the URL's pathname + search
+ */
+export function isAllowed(rules, path) {
+  let best = null;
+  for (const r of rules || []) {
+    if (!r.path || !ruleMatches(r.path, path)) continue;
+    if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.type === 'allow')) {
+      best = r;
+    }
+  }
+  return !best || best.type === 'allow';
+}
+
+/**
+ * Per-host request pacer. `wait(url, delayMs)` resolves when this request may
+ * start, reserving the NEXT slot `delayMs` later — so N concurrent workers
+ * hitting one host space out at exactly one request per delay, while different
+ * hosts never wait on each other. delayMs ≤ 0 = no-op (the default-off path
+ * costs nothing).
+ */
+export function createHostGate() {
+  const nextAt = new Map(); // host → earliest ms timestamp the next request may start
+  return {
+    async wait(url, delayMs) {
+      if (!(delayMs > 0)) return;
+      let host;
+      try {
+        host = new URL(url).host;
+      } catch {
+        return;
+      }
+      const now = Date.now();
+      const at = Math.max(now, nextAt.get(host) || 0);
+      nextAt.set(host, at + delayMs);
+      if (at > now) await new Promise((r) => setTimeout(r, at - now));
+    },
+  };
+}