@stravica-ai/digital-operator-pa 0.1.0

package/dist/engineering-pa-cli/src/knowledge/eval.mjs

@@ -0,0 +1,446 @@
+// eval.mjs — `pa knowledge eval` retrieval-quality harness.
+//
+// Purpose (Deliverable B, w-2026-07-06-020): turn "the right doc at the
+// right time" from an unmeasured quantity into a scored, trend-lined,
+// scheduled signal. Every case runs through the SAME code path as
+// interactive search (post-Deliverable-A cascade included), so the
+// numbers describe what an operator or agent would actually see.
+//
+// Set:      operator/knowledge/_meta/retrieval-evals.json
+// Runs:     operator/knowledge/_meta/retrieval-eval-runs/<isoUtc>.json
+//
+// Case shape:
+//   { id, query, expectedDocs: string[], matchMode: 'any'|'all', notes? }
+//
+// Scoring:
+//   hit@1  — matchMode 'any': ≥1 expected doc is at position 1 in results
+//            matchMode 'all': every expected doc is at position 1 (only
+//            valid for single-expected-doc cases; multi-doc always false)
+//   hit@5  — same rule, top-5 window
+//   miss   — hit@5 false
+//
+// Determinism: the search cascade is deterministic (stable sort by score,
+// edge count, id). Same eval set + same graph = same scores.
+//
+// Orphan gauge runs in the same pass — total + per-domain, delta vs last
+// run, warnings when growth > 10 or any domain is 100% orphaned. The
+// gauge is a load-bearing hygiene signal, but never fails the run — it
+// tags the summary and boot-context renders the tag.
+//
+// Exit codes (tooling standard §6):
+//   0 ok
+//   1 usage error / eval set malformed / --strict-cases and any case missed
+//   2 no eval set present (informational — set is authored operator-side)
+//   3 IO failure (write run file)
+
+import { readFile, readdir, writeFile, mkdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+
+import { getRepoRoot, loadGraph, searchCascade, orphans, uniqueDomains } from './graph.mjs';
+
+const EVAL_SET_PATH = 'operator/knowledge/_meta/retrieval-evals.json';
+const RUNS_DIR = 'operator/knowledge/_meta/retrieval-eval-runs';
+const DEFAULT_ORPHAN_GROWTH_ALARM = 10;
+
+const USAGE = `pa knowledge eval — retrieval-quality harness
+
+Usage:
+  pa knowledge eval [--fixture <path>] [--out <dir>] [--full] [--format json|line|table]
+                    [--strict-cases] [--no-write] [--limit N]
+  pa knowledge eval --help
+
+Reads the eval set at operator/knowledge/_meta/retrieval-evals.json (or
+--fixture for tests), runs each case through the SAME search cascade as
+interactive callers, scores hit@1 / hit@5 / miss, computes the orphan
+hygiene gauge, and writes a run file under
+operator/knowledge/_meta/retrieval-eval-runs/.
+
+Flags:
+  --fixture <path>    Alternate eval-set path (used by tests)
+  --out <dir>         Alternate runs directory (used by tests)
+  --full              Run each search with --full (body scan)
+  --format json|line|table   Output shape (default: line — the workState summary)
+  --strict-cases      Exit 1 if any case misses hit@5
+  --no-write          Skip persisting the run file
+  --limit N           Only run the first N cases (dev convenience)
+`;
+
+export async function run(argv) {
+  const { flags, positional } = parseFlags(argv);
+  if (flags.help === true || flags.h === true) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+  if (positional.length > 0) {
+    process.stderr.write(`pa knowledge eval: unexpected arguments: ${positional.join(' ')}\n`);
+    process.stderr.write(USAGE);
+    return 1;
+  }
+  const format = resolveFormat(flags.format);
+
+  let repoRoot;
+  try {
+    repoRoot = await getRepoRoot();
+  } catch (err) {
+    process.stderr.write(`pa knowledge eval: ${err.message}\n`);
+    return err.exitCode ?? 2;
+  }
+
+  const evalSetPath = flags.fixture ? resolve(String(flags.fixture)) : join(repoRoot, EVAL_SET_PATH);
+  const runsDir = flags.out ? resolve(String(flags.out)) : join(repoRoot, RUNS_DIR);
+
+  const setLoad = await loadEvalSet(evalSetPath);
+  if (setLoad.status === 'missing') {
+    process.stderr.write(
+      `pa knowledge eval: no eval set at ${evalSetPath}\n` +
+      `\n` +
+      `Seed one by dropping a JSON file at that path with the shape:\n` +
+      `  { "version": "1.0",\n` +
+      `    "cases": [\n` +
+      `      { "id": "case-01", "query": "worktree isolation concurrent write workers",\n` +
+      `        "expectedDocs": ["operator/knowledge/…/worktree-isolation.md"],\n` +
+      `        "matchMode": "any", "notes": "audit case, 2026-07-06" }\n` +
+      `    ] }\n`,
+    );
+    return 2;
+  }
+  if (setLoad.status === 'malformed') {
+    process.stderr.write(`pa knowledge eval: eval set malformed at ${evalSetPath}: ${setLoad.error}\n`);
+    return 1;
+  }
+  const evalSet = setLoad.data;
+
+  let graph;
+  try {
+    graph = await loadGraph({ repoRoot });
+  } catch (err) {
+    process.stderr.write(`pa knowledge eval: ${err.message}\n`);
+    return err.exitCode ?? 3;
+  }
+
+  const nodeById = new Map(graph.nodes.map((n) => [n.id, n]));
+  const limit = Number.isInteger(Number(flags.limit)) ? Number(flags.limit) : evalSet.cases.length;
+  const full = flags.full === true;
+
+  const cases = [];
+  for (const c of evalSet.cases.slice(0, limit)) {
+    const validation = validateCase(c);
+    if (validation) {
+      process.stderr.write(`pa knowledge eval: invalid case: ${validation}\n`);
+      return 1;
+    }
+    const envelope = await searchCascade(graph, c.query, { full, repoRoot });
+    const topIds = envelope.results.map((r) => r.id);
+    const scored = scoreCase(c, topIds);
+    cases.push({
+      id: c.id,
+      query: c.query,
+      expectedDocs: c.expectedDocs.slice(),
+      matchMode: c.matchMode,
+      notes: c.notes ?? null,
+      mode: envelope.mode,
+      hit1: scored.hit1,
+      hit5: scored.hit5,
+      miss: scored.miss,
+      topResults: topIds.slice(0, 5),
+      expectedDocsPresent: expectedDocsPresent(c, topIds),
+      // Cases that reference a doc missing from the graph are the canonical
+      // regression signal (spec §Acceptance: "renamed doc caught on next
+      // run"). We flag it in each case so callers can distinguish a search
+      // miss from a stale eval reference.
+      expectedDocsMissingFromGraph: c.expectedDocs.filter((d) => !nodeById.has(d)),
+    });
+  }
+
+  const summary = summarise(cases);
+  const orphanGauge = computeOrphanGauge(graph);
+
+  const runId = new Date().toISOString().replace(/[:.]/g, '-').replace(/Z$/, '');
+  const generatedAt = new Date().toISOString();
+
+  let previousRun = null;
+  try {
+    previousRun = await readLatestRun(runsDir);
+  } catch { /* first-run friendly */ }
+
+  const delta = computeDelta(summary, orphanGauge, previousRun);
+  const alarms = buildAlarms(orphanGauge, delta);
+
+  const runFile = {
+    version: '1.0',
+    runId,
+    generatedAt,
+    graph: {
+      generatedAt: graph.generatedAt ?? null,
+      nodeCount: graph.nodes.length,
+      edgeCount: graph.edges.length,
+    },
+    cases,
+    summary,
+    orphans: orphanGauge,
+    delta,
+    alarms,
+    workStateLine: renderWorkStateLine(summary, orphanGauge, delta, alarms),
+  };
+
+  if (!(flags['no-write'] === true)) {
+    try {
+      await mkdir(runsDir, { recursive: true });
+      await writeFile(join(runsDir, runId + '.json'), JSON.stringify(runFile, null, 2) + '\n', 'utf8');
+    } catch (err) {
+      process.stderr.write(`pa knowledge eval: could not write run file: ${err.message}\n`);
+      return 3;
+    }
+  }
+
+  emit(runFile, format);
+
+  if (flags['strict-cases'] === true && summary.miss.count > 0) return 1;
+  return 0;
+}
+
+// --- eval-set loading -----------------------------------------------------
+
+async function loadEvalSet(path) {
+  if (!existsSync(path)) return { status: 'missing' };
+  let raw;
+  try {
+    raw = await readFile(path, 'utf8');
+  } catch (e) {
+    return { status: 'malformed', error: e.message };
+  }
+  let json;
+  try {
+    json = JSON.parse(raw);
+  } catch (e) {
+    return { status: 'malformed', error: 'invalid JSON: ' + e.message };
+  }
+  if (!json || !Array.isArray(json.cases)) {
+    return { status: 'malformed', error: 'expected { cases: [...] }' };
+  }
+  return { status: 'ok', data: json };
+}
+
+function validateCase(c) {
+  if (!c || typeof c !== 'object') return 'case is not an object';
+  if (!c.id || typeof c.id !== 'string') return 'case.id missing or not a string';
+  if (!c.query || typeof c.query !== 'string') return `case ${c.id}: query missing or not a string`;
+  if (!Array.isArray(c.expectedDocs) || c.expectedDocs.length === 0) {
+    return `case ${c.id}: expectedDocs must be a non-empty array`;
+  }
+  if (c.expectedDocs.some((d) => typeof d !== 'string')) {
+    return `case ${c.id}: expectedDocs must be strings`;
+  }
+  if (c.matchMode !== 'any' && c.matchMode !== 'all') {
+    return `case ${c.id}: matchMode must be 'any' or 'all'`;
+  }
+  return null;
+}
+
+// --- scoring --------------------------------------------------------------
+
+function scoreCase(c, topIds) {
+  const top1 = topIds.slice(0, 1);
+  const top5 = topIds.slice(0, 5);
+  const expected = new Set(c.expectedDocs);
+  if (c.matchMode === 'all') {
+    // All-mode is strict — every expected doc must sit inside the window.
+    const inTop1 = c.expectedDocs.every((d) => top1.includes(d));
+    const inTop5 = c.expectedDocs.every((d) => top5.includes(d));
+    return { hit1: inTop1, hit5: inTop5, miss: !inTop5 };
+  }
+  // matchMode 'any': at least one expected doc present.
+  const inTop1 = top1.some((d) => expected.has(d));
+  const inTop5 = top5.some((d) => expected.has(d));
+  return { hit1: inTop1, hit5: inTop5, miss: !inTop5 };
+}
+
+function expectedDocsPresent(c, topIds) {
+  const expected = new Set(c.expectedDocs);
+  return topIds.filter((d) => expected.has(d));
+}
+
+function summarise(cases) {
+  const total = cases.length;
+  const h1 = cases.filter((c) => c.hit1).length;
+  const h5 = cases.filter((c) => c.hit5).length;
+  const miss = cases.filter((c) => c.miss).length;
+  const byDomain = {};
+  for (const c of cases) {
+    const bucket = domainForCase(c);
+    if (!byDomain[bucket]) byDomain[bucket] = { total: 0, hit1: 0, hit5: 0, miss: 0 };
+    byDomain[bucket].total += 1;
+    if (c.hit1) byDomain[bucket].hit1 += 1;
+    if (c.hit5) byDomain[bucket].hit5 += 1;
+    if (c.miss) byDomain[bucket].miss += 1;
+  }
+  return {
+    total,
+    hit1: { count: h1, percentage: pct(h1, total) },
+    hit5: { count: h5, percentage: pct(h5, total) },
+    miss: { count: miss, percentage: pct(miss, total) },
+    byDomain,
+  };
+}
+
+function domainForCase(c) {
+  // Bucket a case by the domain of its FIRST expected doc's path — reads
+  // `operator/knowledge/<domain>/…`. Bare-slug expected docs bucket to
+  // `<none>`. This is a heuristic; it's fine because per-domain recall is
+  // an operator-facing eyeball metric, not a hard promotion gate.
+  const first = c.expectedDocs?.[0];
+  if (!first) return '<none>';
+  const m = /^operator\/knowledge\/([^/]+)/.exec(first);
+  return m ? m[1] : '<none>';
+}
+
+function pct(n, d) {
+  if (!d) return 0;
+  return Math.round((n / d) * 100);
+}
+
+// --- orphan gauge ---------------------------------------------------------
+
+function computeOrphanGauge(graph) {
+  const results = orphans(graph, { includeStubs: true });
+  const total = results.length;
+  const byDomain = {};
+  for (const r of results) {
+    const d = r.domain ?? '<none>';
+    byDomain[d] = (byDomain[d] ?? 0) + 1;
+  }
+  // Include totals-per-domain so we can flag domains that are 100% orphaned.
+  const domainTotals = {};
+  for (const n of graph.nodes) {
+    const d = n.domain ?? '<none>';
+    domainTotals[d] = (domainTotals[d] ?? 0) + 1;
+  }
+  const perDomain = {};
+  for (const d of uniqueDomains(graph)) {
+    perDomain[d] = { orphans: byDomain[d] ?? 0, total: domainTotals[d] ?? 0 };
+  }
+  if (byDomain['<none>'] != null) {
+    perDomain['<none>'] = { orphans: byDomain['<none>'], total: domainTotals['<none>'] ?? 0 };
+  }
+  return { total, byDomain: perDomain };
+}
+
+// --- delta + alarms -------------------------------------------------------
+
+async function readLatestRun(runsDir) {
+  if (!existsSync(runsDir)) return null;
+  const entries = await readdir(runsDir);
+  const files = entries.filter((n) => n.endsWith('.json')).sort();
+  if (!files.length) return null;
+  const raw = await readFile(join(runsDir, files[files.length - 1]), 'utf8');
+  try { return JSON.parse(raw); } catch { return null; }
+}
+
+function computeDelta(summary, orphanGauge, previous) {
+  if (!previous || !previous.summary || !previous.orphans) {
+    return { firstRun: true, hit5PercentagePointsDelta: 0, orphanCountDelta: 0 };
+  }
+  const prevH5 = previous.summary.hit5?.percentage ?? 0;
+  const prevOrphans = previous.orphans.total ?? 0;
+  return {
+    firstRun: false,
+    hit5PercentagePointsDelta: summary.hit5.percentage - prevH5,
+    orphanCountDelta: orphanGauge.total - prevOrphans,
+  };
+}
+
+function buildAlarms(orphanGauge, delta) {
+  const alarms = [];
+  if (!delta.firstRun && delta.orphanCountDelta > DEFAULT_ORPHAN_GROWTH_ALARM) {
+    alarms.push(`orphans grew by ${delta.orphanCountDelta} since last run (threshold ${DEFAULT_ORPHAN_GROWTH_ALARM})`);
+  }
+  for (const [domain, counts] of Object.entries(orphanGauge.byDomain)) {
+    if (counts.total > 0 && counts.orphans === counts.total) {
+      alarms.push(`${domain} domain is 100% orphaned (${counts.total} node${counts.total === 1 ? '' : 's'})`);
+    }
+  }
+  return alarms;
+}
+
+// --- render ---------------------------------------------------------------
+
+function renderWorkStateLine(summary, orphanGauge, delta, alarms) {
+  const hit5 = summary.hit5.percentage;
+  const deltaTag = delta.firstRun
+    ? '(first run)'
+    : `(${delta.hit5PercentagePointsDelta > 0 ? '+' : ''}${delta.hit5PercentagePointsDelta})`;
+  const orphanTag = delta.firstRun
+    ? ''
+    : ` (${delta.orphanCountDelta > 0 ? '+' : ''}${delta.orphanCountDelta})`;
+  const alarmTag = alarms.length > 0 ? ` ⚠ ${alarms.length} alarm${alarms.length === 1 ? '' : 's'}` : '';
+  return `retrievalEval: hit@5 ${hit5}% ${deltaTag} | orphans ${orphanGauge.total}${orphanTag}${alarmTag}`;
+}
+
+function emit(runFile, format) {
+  if (format === 'json') {
+    process.stdout.write(JSON.stringify(runFile, null, 2) + '\n');
+    return;
+  }
+  if (format === 'line') {
+    process.stdout.write(runFile.workStateLine + '\n');
+    return;
+  }
+  // table
+  const s = runFile.summary;
+  const lines = [];
+  lines.push(`Retrieval eval — ${runFile.cases.length} case${runFile.cases.length === 1 ? '' : 's'} — run ${runFile.runId}`);
+  lines.push('');
+  lines.push(`hit@1: ${s.hit1.count}/${s.total} (${s.hit1.percentage}%)`);
+  lines.push(`hit@5: ${s.hit5.count}/${s.total} (${s.hit5.percentage}%)`);
+  lines.push(`miss : ${s.miss.count}/${s.total} (${s.miss.percentage}%)`);
+  if (runFile.delta && !runFile.delta.firstRun) {
+    const sign = runFile.delta.hit5PercentagePointsDelta > 0 ? '+' : '';
+    lines.push(`Δhit@5: ${sign}${runFile.delta.hit5PercentagePointsDelta}pp vs last run`);
+  }
+  lines.push('');
+  lines.push(`Orphans: ${runFile.orphans.total}`);
+  if (runFile.delta && !runFile.delta.firstRun) {
+    const sign = runFile.delta.orphanCountDelta > 0 ? '+' : '';
+    lines.push(`Δorphans: ${sign}${runFile.delta.orphanCountDelta} vs last run`);
+  }
+  if (runFile.alarms.length > 0) {
+    lines.push('');
+    lines.push('Alarms:');
+    for (const a of runFile.alarms) lines.push(`  - ${a}`);
+  }
+  lines.push('');
+  lines.push(runFile.workStateLine);
+  process.stdout.write(lines.join('\n') + '\n');
+}
+
+// --- shared bits ----------------------------------------------------------
+
+function parseFlags(args) {
+  const flags = {};
+  const positional = [];
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === '-h' || a === '--help') { flags.help = true; continue; }
+    if (a.startsWith('--')) {
+      const key = a.slice(2);
+      const peek = args[i + 1];
+      if (peek == null || peek.startsWith('--')) { flags[key] = true; }
+      else { flags[key] = peek; i += 1; }
+    } else {
+      positional.push(a);
+    }
+  }
+  return { flags, positional };
+}
+
+function resolveFormat(raw) {
+  if (raw == null || raw === true) return 'line';
+  if (raw === 'json' || raw === 'line' || raw === 'table') return raw;
+  const err = new Error(`unknown --format: ${raw} (expected json | line | table)`);
+  err.exitCode = 1;
+  throw err;
+}
+
+// Test seams.
+export const _internals = { scoreCase, summarise, computeOrphanGauge, buildAlarms, renderWorkStateLine, computeDelta };