crawldna 0.1.0

package/src/eval/metrics.mjs

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// The measurement primitives for step #12 of TODO.md — the "measurement harness".
+//
+// These turn the project's promises ("nothing hidden is missed", "the output is
+// all-and-only what the task asked", "here is where the tokens go") into NUMBERS you
+// can put side by side BEFORE and AFTER a change, instead of trusting an estimate.
+//
+// Every function here is PURE and DEPENDENCY-FREE (only the project's own url helper),
+// so it is deterministic and verifiable offline — no model, no browser. The parts that
+// actually run a crawl live in the runner (scripts/eval.mjs); THIS file only scores an
+// output that already exists.
+//
+// The honest limits, kept front-of-mind (see TODO.md §"Previsioni oneste"):
+//   - Absolute completeness is NOT provable from a single crawl (academic result). What
+//     we CAN measure are PROXIES: did KNOWN hidden content survive (revealCoverage), and
+//     how much of the sitemap did we keep (sitemapCoverage).
+//   - "Task respect" is scored SWDE-style against a GOLDEN SET the user supplies: recall
+//     = the expected things are present, precision = the known-irrelevant things are not.
+//     It measures the crawl against a ground truth; it cannot invent one.
+
+import { normalizeUrl } from '../lib/url.mjs';
+
+/** Lowercase + collapse all whitespace to single spaces + trim — so a substring check
+ *  ignores incidental formatting (line wraps, indentation, doubled spaces). */
+export function normalizeText(s) {
+  return String(s || '').replace(/\s+/g, ' ').trim().toLowerCase();
+}
+
+/** Canonical form of a URL for set comparison; falls back to a trimmed lowercase string
+ *  when it cannot be parsed, so a comparison never silently drops an entry. */
+function canonUrl(u) {
+  return normalizeUrl(u) || String(u || '').trim().toLowerCase();
+}
+
+/** Round to `d` decimals for stable, readable ratios. */
+function round(n, d = 3) {
+  const f = 10 ** d;
+  return Math.round((Number(n) || 0) * f) / f;
+}
+
+/**
+ * (a)(iii) REVEAL RESIDUAL (#21d) — the closed loop's own completeness number, read from
+ * the crawl result itself (no golden set needed): for each kept page, how much text was
+ * STILL hidden in the main content when the reveal loop exited (`meta.revealResidualChars`,
+ * measured in-page — 0 = the page was drained). Complements revealCoverage: coverage
+ * checks KNOWN snippets survived, residual measures what provably did NOT come out.
+ *
+ * @param {Array<{url:string, meta?:{revealResidualChars?:number}}>} pages  kept pages
+ * @returns {{ pages:number, withResidual:number, chars:number, words:number,
+ *             worst:Array<{url:string,chars:number}> }}
+ */
+export function revealResidual(pages = []) {
+  const rows = [];
+  let chars = 0;
+  for (const p of pages || []) {
+    const c = (p && p.meta && p.meta.revealResidualChars) || 0;
+    if (c > 0) {
+      rows.push({ url: p.url, chars: c });
+      chars += c;
+    }
+  }
+  rows.sort((a, b) => b.chars - a.chars);
+  return {
+    pages: (pages || []).length,
+    withResidual: rows.length,
+    chars,
+    words: Math.round(chars / 6),
+    worst: rows.slice(0, 5),
+  };
+}
+
+/**
+ * (a)(i) REVEAL COMPLETENESS — did the interaction-hidden content survive into the
+ * output? Each `expected` string is a snippet that a human confirmed is present on the
+ * page ONLY after a click (a tab's body, an accordion's text, a "load more" item). If it
+ * appears in the crawl output, the reveal engine did its job for that snippet.
+ *
+ * @param {string} outputText  the crawl's Markdown (the whole scan, concatenated)
+ * @param {string[]} expected  snippets that must appear iff reveal worked
+ * @returns {{ total:number, found:number, missing:string[], ratio:number }}
+ */
+export function revealCoverage(outputText, expected = []) {
+  const hay = normalizeText(outputText);
+  const list = (expected || []).map((s) => String(s || '')).filter((s) => s.trim());
+  const missing = list.filter((s) => !hay.includes(normalizeText(s)));
+  const found = list.length - missing.length;
+  return { total: list.length, found, missing, ratio: list.length ? round(found / list.length) : 1 };
+}
+
+/**
+ * (a)(ii) PAGE COMPLETENESS — sitemap-coverage proxy. Of the URLs the site advertises in
+ * its sitemap, how many did the crawl keep? This is a PROXY (a crawl can legitimately
+ * skip off-task sitemap URLs, and can legitimately find pages NOT in the sitemap), so
+ * `missing` and `extra` are reported for inspection rather than as pass/fail.
+ *
+ * @param {string[]} keptUrls     URLs the crawl produced (scan.pages[].url)
+ * @param {string[]} sitemapUrls  URLs from the site's sitemap(s)
+ * @returns {{ total:number, covered:number, missing:string[], extra:string[], ratio:number }}
+ */
+export function sitemapCoverage(keptUrls = [], sitemapUrls = []) {
+  const kept = new Set((keptUrls || []).map(canonUrl));
+  const sm = [...new Set((sitemapUrls || []).map(canonUrl))];
+  const smSet = new Set(sm);
+  const missing = sm.filter((u) => !kept.has(u));
+  const covered = sm.length - missing.length;
+  const extra = [...kept].filter((u) => !smSet.has(u));
+  return { total: sm.length, covered, missing, extra, ratio: sm.length ? round(covered / sm.length) : 1 };
+}
+
+/**
+ * (b) TASK RESPECT — "all-and-only", scored SWDE-style against a golden set:
+ *   - RECALL    = fraction of `mustInclude` snippets present  → did we keep everything
+ *                 the task asked for? (a missing one = lost content).
+ *   - PRECISION = fraction of `mustExclude` snippets ABSENT   → did we drop the
+ *                 off-task/boilerplate the task did NOT ask for? (a present one = leaked
+ *                 chrome). Precision here is a PROXY: it can only see the known-bad
+ *                 markers the golden set lists, not every possible piece of junk.
+ * F1 is reported only when both lists are non-empty.
+ *
+ * @param {string} outputText
+ * @param {{ mustInclude?:string[], mustExclude?:string[] }} golden
+ */
+export function taskRespect(outputText, { mustInclude = [], mustExclude = [] } = {}) {
+  const hay = normalizeText(outputText);
+  const inc = (mustInclude || []).map((s) => String(s || '')).filter((s) => s.trim());
+  const exc = (mustExclude || []).map((s) => String(s || '')).filter((s) => s.trim());
+
+  const missing = inc.filter((s) => !hay.includes(normalizeText(s))); // asked-for but absent
+  const leaked = exc.filter((s) => hay.includes(normalizeText(s))); // off-task but present
+
+  const recall = inc.length ? round((inc.length - missing.length) / inc.length) : null;
+  const precision = exc.length ? round((exc.length - leaked.length) / exc.length) : null;
+  const f1 =
+    recall != null && precision != null && recall + precision > 0
+      ? round((2 * recall * precision) / (recall + precision))
+      : null;
+
+  return {
+    recall,
+    precision,
+    f1,
+    includeTotal: inc.length,
+    includeFound: inc.length - missing.length,
+    missing,
+    excludeTotal: exc.length,
+    leaked,
+  };
+}
+
+/**
+ * (a)(ii) RUN DIFF — compare two runs of the same site (e.g. before vs after a change,
+ * or re-crawl freshness). Pages are matched by canonical URL; a page present in both with
+ * a different byte size is "changed".
+ *
+ * @param {Array<{url:string, bytes?:number}>} a  the baseline run's pages
+ * @param {Array<{url:string, bytes?:number}>} b  the new run's pages
+ */
+export function diffRuns(a = [], b = []) {
+  const toMap = (pages) => {
+    const m = new Map();
+    for (const p of pages || []) {
+      const u = canonUrl(p && p.url);
+      if (!u) continue;
+      m.set(u, Number((p && p.bytes) || 0));
+    }
+    return m;
+  };
+  const A = toMap(a);
+  const B = toMap(b);
+
+  const added = [...B.keys()].filter((u) => !A.has(u));
+  const removed = [...A.keys()].filter((u) => !B.has(u));
+  const changed = [];
+  for (const [u, bytesA] of A) {
+    if (!B.has(u)) continue;
+    const bytesB = B.get(u);
+    if (bytesA !== bytesB) changed.push({ url: u, fromBytes: bytesA, toBytes: bytesB, delta: bytesB - bytesA });
+  }
+  const sum = (m) => [...m.values()].reduce((n, v) => n + v, 0);
+  return {
+    added,
+    removed,
+    changed,
+    pagesA: A.size,
+    pagesB: B.size,
+    bytesA: sum(A),
+    bytesB: sum(B),
+    bytesDelta: sum(B) - sum(A),
+  };
+}
+
+/**
+ * (c) TOKENS PER CALL TYPE — turn the metered `tokens.byKind` into a ranked table so you
+ * can see WHERE the tokens actually go (reveal vs scope vs links vs nav-plan), not just
+ * the grand total. Each row's `total` is input+output; `share` is its fraction of the
+ * grand total. Rows are sorted biggest-first.
+ *
+ * @param {{ calls?:number, inputTokens?:number, outputTokens?:number, byKind?:object }} tokens
+ */
+export function tokenBreakdown(tokens = {}) {
+  const byKind = (tokens && tokens.byKind) || {};
+  const rows = Object.entries(byKind).map(([kind, k]) => {
+    const input = Number(k.inputTokens || 0);
+    const output = Number(k.outputTokens || 0);
+    return {
+      kind,
+      calls: Number(k.calls || 0),
+      inputTokens: input,
+      outputTokens: output,
+      cachedInputTokens: Number(k.cachedInputTokens || 0),
+      total: input + output,
+    };
+  });
+  const grand = rows.reduce((n, r) => n + r.total, 0);
+  for (const r of rows) r.share = grand ? round(r.total / grand) : 0;
+  rows.sort((x, y) => y.total - x.total);
+
+  const totalInput = Number(tokens.inputTokens || 0);
+  const totalOutput = Number(tokens.outputTokens || 0);
+  const totalCached = Number(tokens.cachedInputTokens || 0);
+  return {
+    total: {
+      calls: Number(tokens.calls || 0),
+      inputTokens: totalInput,
+      outputTokens: totalOutput,
+      // #4: the slice of input served from the provider's prompt cache (~10× cheaper);
+      // cachedShare is its fraction of ALL input — the number that should GROW after
+      // the first calls of each kind when prefix caching is working.
+      cachedInputTokens: totalCached,
+      cachedShare: totalInput ? round(totalCached / totalInput) : 0,
+      total: totalInput + totalOutput,
+    },
+    rows,
+  };
+}

package/src/eval/report.mjs

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright (C) 2026 Bogdan Marian Vasaiu
+// Assemble the individual metrics (metrics.mjs) into ONE report for a crawl result,
+// and render it as a readable before/after table. Pure: it scores a result + a golden
+// spec that already exist; the runner (scripts/eval.mjs) is what produces them.
+//
+// A "golden spec" is the ground truth for one site, supplied by the user (see
+// eval/README.md):
+//   { name, url, task,
+//     expect: { revealContent?: string[], mustInclude?: string[],
+//               mustExclude?: string[], sitemapUrls?: string[] } }
+
+import { revealCoverage, revealResidual, sitemapCoverage, taskRespect, diffRuns, tokenBreakdown } from './metrics.mjs';
+
+/** Concatenate a scan's output into one text blob to score against. Prefers the
+ *  consolidated output files; falls back to the raw page Markdown. */
+function scanText(scan) {
+  const files = (scan.files || []).map((f) => f.markdown || '').filter(Boolean);
+  if (files.length) return files.join('\n\n');
+  return (scan.pages || []).map((p) => p.markdown || '').filter(Boolean).join('\n\n');
+}
+
+/** Every kept page across all scans, as { url, bytes, meta } — for coverage,
+ *  diff and the reveal-residual audit (#21d, reads meta.revealResidualChars). */
+function keptPages(result) {
+  const out = [];
+  for (const scan of result.scans || []) {
+    for (const p of scan.pages || []) {
+      out.push({ url: p.url, bytes: (p.meta && p.meta.bytes) || 0, meta: p.meta });
+    }
+  }
+  return out;
+}
+
+/**
+ * Score a finished crawl against a golden spec.
+ *
+ * @param {object} a
+ * @param {import('../index.mjs').Result|any} a.result  the crawlDocs result
+ * @param {object} a.spec                    the golden spec ({ name, url, task, expect })
+ * @param {string[]} [a.sitemapUrls]         sitemap URLs (from the spec or fetched live)
+ * @param {Array<{url,bytes}>} [a.baselinePages]  a previous run's pages, for a diff
+ * @returns {object} a structured report (feed to formatReport)
+ */
+export function evaluate({ result, spec = {}, sitemapUrls = null, baselinePages = null }) {
+  const expect = spec.expect || {};
+  const text = (result.scans || []).map(scanText).filter(Boolean).join('\n\n');
+  const pages = keptPages(result);
+  const sm = Array.isArray(sitemapUrls) ? sitemapUrls : Array.isArray(expect.sitemapUrls) ? expect.sitemapUrls : null;
+
+  const report = {
+    name: spec.name || spec.url || '(unnamed)',
+    url: spec.url || '',
+    task: spec.task || '',
+    pages: pages.length,
+    durationMs: (result.stats && result.stats.durationMs) || 0,
+    reveal: expect.revealContent && expect.revealContent.length ? revealCoverage(text, expect.revealContent) : null,
+    residual: revealResidual(pages),
+    task_respect:
+      (expect.mustInclude && expect.mustInclude.length) || (expect.mustExclude && expect.mustExclude.length)
+        ? taskRespect(text, expect)
+        : null,
+    sitemap: sm ? sitemapCoverage(pages.map((p) => p.url), sm) : null,
+    diff: baselinePages ? diffRuns(baselinePages, pages) : null,
+    tokens: tokenBreakdown((result.stats && result.stats.tokens) || {}),
+    warnings: (result.warnings || []).length,
+  };
+  return report;
+}
+
+const pct = (r) => (r == null ? ' n/a' : `${(r * 100).toFixed(1)}%`);
+const bar = (r) => {
+  if (r == null) return '';
+  const n = Math.round(Math.max(0, Math.min(1, r)) * 20);
+  return '[' + '#'.repeat(n) + '-'.repeat(20 - n) + ']';
+};
+
+/** Render a report (from {@link evaluate}) as a readable text block. */
+export function formatReport(report) {
+  const L = [];
+  L.push(`━━ ${report.name} ━━`);
+  L.push(`  task:   ${report.task}`);
+  L.push(`  url:    ${report.url}`);
+  L.push(`  pages:  ${report.pages} kept · ${report.durationMs} ms · ${report.warnings} warning(s)`);
+  L.push('');
+
+  // (a)(i) reveal completeness
+  if (report.reveal) {
+    const r = report.reveal;
+    L.push(`(a) reveal completeness   ${bar(r.ratio)} ${pct(r.ratio)}  (${r.found}/${r.total} hidden snippets present)`);
+    if (r.missing.length) for (const m of r.missing) L.push(`      MISSING: ${JSON.stringify(m.slice(0, 80))}`);
+  }
+
+  // (a)(iii) reveal residual — measured directly on the result, no golden set needed
+  if (report.residual && report.residual.pages) {
+    const r = report.residual;
+    const ratio = r.pages ? (r.pages - r.withResidual) / r.pages : 1;
+    L.push(`(a) reveal residual       ${bar(ratio)} ${pct(ratio)}  (${r.pages - r.withResidual}/${r.pages} pages fully drained · ~${r.words} words still hidden)`);
+    for (const w of r.worst) L.push(`      RESIDUAL: ${w.url} (~${Math.round(w.chars / 6)} words)`);
+  }
+
+  // (a)(ii) sitemap coverage
+  if (report.sitemap) {
+    const s = report.sitemap;
+    L.push(`(a) sitemap coverage      ${bar(s.ratio)} ${pct(s.ratio)}  (${s.covered}/${s.total} sitemap URLs kept · ${s.extra.length} beyond sitemap)`);
+    if (s.missing.length) L.push(`      ${s.missing.length} sitemap URL(s) not kept (proxy — some may be legitimately off-task)`);
+  }
+
+  // (b) task respect
+  if (report.task_respect) {
+    const t = report.task_respect;
+    if (t.recall != null) {
+      L.push(`(b) task recall           ${bar(t.recall)} ${pct(t.recall)}  (${t.includeFound}/${t.includeTotal} expected present)`);
+      for (const m of t.missing) L.push(`      MISSING: ${JSON.stringify(m.slice(0, 80))}`);
+    }
+    if (t.precision != null) {
+      L.push(`(b) task precision        ${bar(t.precision)} ${pct(t.precision)}  (${t.excludeTotal - t.leaked.length}/${t.excludeTotal} off-task absent)`);
+      for (const m of t.leaked) L.push(`      LEAKED:  ${JSON.stringify(m.slice(0, 80))}`);
+    }
+    if (t.f1 != null) L.push(`(b) task F1               ${bar(t.f1)} ${pct(t.f1)}`);
+  }
+
+  // run diff
+  if (report.diff) {
+    const d = report.diff;
+    L.push(
+      `    run diff vs baseline  +${d.added.length} added · -${d.removed.length} removed · ~${d.changed.length} changed · ` +
+        `${d.bytesDelta >= 0 ? '+' : ''}${d.bytesDelta} bytes`,
+    );
+  }
+
+  // (c) tokens per call type
+  L.push('');
+  const tk = report.tokens;
+  const cached = tk.total.cachedInputTokens
+    ? ` · ${tk.total.cachedInputTokens.toLocaleString()} in cached ${pct(tk.total.cachedShare)}`
+    : '';
+  L.push(`(c) tokens: ${tk.total.total.toLocaleString()} total (${tk.total.inputTokens.toLocaleString()} in · ${tk.total.outputTokens.toLocaleString()} out · ${tk.total.calls} calls${cached})`);
+  const pad = (s, n) => String(s).padEnd(n);
+  const padL = (s, n) => String(s).padStart(n);
+  L.push(`      ${pad('kind', 10)} ${padL('calls', 6)} ${padL('in', 9)} ${padL('out', 9)} ${padL('total', 10)} ${padL('share', 7)}`);
+  for (const r of tk.rows) {
+    L.push(
+      `      ${pad(r.kind, 10)} ${padL(r.calls, 6)} ${padL(r.inputTokens.toLocaleString(), 9)} ` +
+        `${padL(r.outputTokens.toLocaleString(), 9)} ${padL(r.total.toLocaleString(), 10)} ${padL(pct(r.share), 7)}`,
+    );
+  }
+  return L.join('\n');
+}