@clear-capabilities/agentic-security-scanner 0.171.0 → 0.172.1

package/src/context-intelligence/entrypoints.js

@@ -0,0 +1,336 @@
+//
+// Entry points -> exact handler symbols.
+//
+// There is NO second route detector here. HTTP routes come from the engine's own route scan (`scanRoutes`) and, for
+// Haskell, from the existing web model (`analyzeHaskellWeb`); non-HTTP surfaces come from the existing entry-point
+// inventory. What this module adds is the join the context packer needs and the existing records do not carry: which
+// declaration is the handler, which middleware wraps it, and an honest disposition for every route.
+//
+// Every detected entry point ends in exactly ONE disposition:
+//   resolved             handler is a known declaration and nothing about how it is wired is in doubt
+//   partially_resolved   a handler (or a registration block) is known, but something is not: a middleware that does
+//                        not resolve, a handler matched by name only, a router whose mount point is not visible
+//   unresolved           no handler could be identified; the manifest still exists and says so
+//   excluded_by_policy   deliberately not processed (operator exclusion, or the entry-point cap)
+import { hashOf, safeLabel } from './canonical.js';
+import { callStatement, inlineBlockSpan, splitArgs } from './spans.js';
+import { dependencyParams, signatureText } from './controls.js';
+import { buildEntrypointInventory } from '../posture/entrypoint-inventory.js';
+
+export const DISPOSITIONS = Object.freeze(['resolved', 'partially_resolved', 'unresolved', 'excluded_by_policy']);
+export const ENTRYPOINT_KINDS = Object.freeze(['http', 'webhook', 'queue', 'cron', 'upload', 'cli']);
+
+const JS_FILE = /\.(?:js|jsx|mjs|cjs|ts|tsx)$/i;
+const HS_FILE = /\.l?hs$/i;
+const INLINE_FN = /^(?:async\s+)?(?:function\b|\([^)]*\)\s*(?::\s*[\w<>\[\]|. ]+)?\s*=>|[A-Za-z_$][\w$]*\s*=>|\\)/;
+const IDENT_CHAIN = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*/;
+
+function lastSeg(n) { const p = String(n).split(/[.:]/); return p[p.length - 1]; }
+
+function globToRe(g) {
+  const esc = String(g).replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*\*/g, '\0').replace(/\*/g, '[^/]*').replace(/\0/g, '.*');
+  return new RegExp(`^${esc}$`);
+}
+
+function symbolIndexes(graph) {
+  const byFile = new Map();
+  const byName = new Map();
+  const byId = new Map();
+  const imports = new Map();
+  for (const i of graph.imports || []) { if (!imports.has(i.file)) imports.set(i.file, []); imports.get(i.file).push(i); }
+  for (const s of graph.symbols) {
+    byId.set(s.id, s);
+    if (!byFile.has(s.file)) byFile.set(s.file, []);
+    byFile.get(s.file).push(s);
+    if (s.kind === 'function' && !s.name.startsWith('<')) {
+      for (const key of new Set([s.name, lastSeg(s.name)])) {
+        if (!byName.has(key)) byName.set(key, []);
+        byName.get(key).push(s);
+      }
+    }
+  }
+  for (const arr of byFile.values()) arr.sort((a, b) => a.declLine - b.declLine || (a.id < b.id ? -1 : 1));
+  return { byFile, byName, byId, imports };
+}
+
+/** Resolve a referenced name to a symbol: same-file exact, then project-unique by name (low confidence). */
+function resolveName(name, file, idx) {
+  // an import statement binds the name to a specific module: that beats any project-wide guess
+  const imp = (idx.imports.get(file) || []).find(i => i.local === name.split('.')[0]);
+  if (imp && imp.kind === 'repo') {
+    const member = name.includes('.') ? name.split('.').pop() : (imp.member === '*' || imp.member === 'default' ? name : imp.member);
+    const hit = (idx.byName.get(member) || []).filter(s => s.file === imp.target);
+    if (hit.length === 1) return { symbol: hit[0], resolution: 'resolved', via: 'import' };
+  }
+  const exact = (idx.byName.get(name) || []).filter(s => s.file === file);
+  if (exact.length === 1) return { symbol: exact[0], resolution: 'resolved' };
+  if (exact.length > 1) return { symbol: null, resolution: 'ambiguous', candidates: exact.map(s => s.id) };
+  const key = lastSeg(name);
+  const all = idx.byName.get(name) || idx.byName.get(key) || [];
+  if (all.length === 1) return { symbol: all[0], resolution: 'name-match' };
+  if (all.length > 1) return { symbol: null, resolution: 'ambiguous', candidates: all.slice(0, 5).map(s => s.id) };
+  return { symbol: null, resolution: 'unresolved' };
+}
+
+/** `X.use(...)` registrations in a JS/TS file: the global / prefix middleware a route can inherit. */
+function collectUses(fileContents) {
+  const uses = [];
+  for (const [file, text] of Object.entries(fileContents)) {
+    if (!JS_FILE.test(file)) continue;
+    const lines = text.split('\n');
+    for (let i = 0; i < lines.length; i++) {
+      const m = /^\s*([A-Za-z_$][\w$]*)\.use\s*\(/.exec(lines[i]);
+      if (!m) continue;
+      const st = callStatement(lines, i + 1);
+      if (!st) continue;
+      const args = splitArgs(st.text);
+      let prefix = '/';
+      let rest = args;
+      const q = args[0] && /^(['"`])(.*)\1$/.exec(args[0]);
+      if (q) { prefix = q[2]; rest = args.slice(1); }
+      const names = rest.map(a => { const id = IDENT_CHAIN.exec(a.replace(/^\[|\]$/g, '')); return id ? { name: id[0], text: a } : { name: null, text: a }; });
+      uses.push({ file, line: i + 1, receiver: m[1], prefix, args: names });
+    }
+  }
+  return uses;
+}
+
+function routeKey(r) { return `${r.file}|${r.line}|${r.method}|${r.path}`; }
+
+async function collectRoutes(fileContents) {
+  const { scanRoutes } = await import('../engine.js');
+  const { analyzeHaskellWeb } = await import('../language/haskell-web.js');
+  const routes = [];
+  const seen = new Set();
+  const add = (r) => { const k = routeKey(r); if (!seen.has(k)) { seen.add(k); routes.push(r); } };
+  const hs = {};
+  for (const file of Object.keys(fileContents).sort()) {
+    if (HS_FILE.test(file)) { hs[file] = fileContents[file]; continue; }
+    let found = [];
+    try { found = scanRoutes(file, fileContents[file]) || []; } catch { found = []; }
+    for (const r of found) add({ ...r, language: 'generic' });
+  }
+  if (Object.keys(hs).length) {
+    let web = null;
+    try { web = analyzeHaskellWeb(hs); } catch { web = null; }
+    if (web) for (const r of web.routes) add({ method: r.method, path: r.path, framework: r.framework, file: r.file, line: r.line, hasAuth: r.auth.status === 'authenticated', language: 'haskell', handler: r.handler, webModel: { auth: r.auth, role: r.role, ownership: r.ownership, idLookups: r.idLookups, csrf: r.csrf, sensitive: r.sensitive, uploads: r.uploads } });
+  }
+  return routes;
+}
+
+function resolveHandler(route, ctx) {
+  const { fileContents, idx, graph } = ctx;
+  const lines = fileContents[route.file].split('\n');
+  const reasons = [];
+  const out = { handlerSymbol: null, handlerKind: 'unknown', rootSpan: null, middleware: [], reasons, confidence: 'none' };
+  const fileSyms = idx.byFile.get(route.file) || [];
+
+  if (route.language === 'haskell') {
+    const h = route.handler || {};
+    if (h.kind === 'named' && h.name) {
+      const r = resolveName(h.name, route.file, idx);
+      if (r.symbol) { out.handlerSymbol = r.symbol.id; out.handlerKind = 'named'; out.confidence = r.resolution === 'resolved' ? 'exact' : 'name-match'; if (r.resolution !== 'resolved') reasons.push('handler-matched-by-name-only'); }
+      else reasons.push(`handler-${r.resolution}`);
+    } else if (h.kind === 'inline') {
+      out.handlerKind = 'inline';
+      out.rootSpan = { file: route.file, ...inlineBlockSpan(lines, route.line) };
+      const enclosing = fileSyms.filter(s => s.kind === 'function' && s.startLine <= route.line && s.endLine >= route.line).sort((a, b) => (b.startLine - a.startLine))[0];
+      out.handlerSymbol = enclosing ? enclosing.id : null;
+      out.confidence = enclosing ? 'inline-block' : 'none';
+      if (!enclosing) reasons.push('inline-handler-without-enclosing-declaration');
+    } else {
+      reasons.push('handler-not-statically-known');
+      // the manifest still shows where the route is registered, so the gap is inspectable rather than absent
+      out.rootSpan = { file: route.file, ...inlineBlockSpan(lines, route.line) };
+    }
+    return out;
+  }
+
+  const rawLine = lines[route.line - 1] || '';
+  if (/^\s*@/.test(rawLine)) {
+    // decorator / annotation style: the handler is the next declaration
+    const decl = fileSyms.filter(s => s.kind === 'function' && s.declLine >= route.line && s.declLine <= route.line + 12).sort((a, b) => a.declLine - b.declLine)[0];
+    if (decl) {
+      out.handlerSymbol = decl.id; out.handlerKind = 'decorated'; out.confidence = 'exact';
+      // dependency-injected guards in the signature (`user=Depends(require_user)`) run before the body
+      for (const d of dependencyParams(signatureText(lines, decl.declLine))) {
+        const r = resolveName(d.dependency, route.file, idx);
+        out.middleware.push({ name: d.dependency, symbolId: r.symbol ? r.symbol.id : null, resolution: r.resolution, line: decl.declLine, dependency: true, param: d.param });
+        if (!r.symbol) reasons.push(`dependency-${r.resolution}:${d.dependency}`);
+        else if (r.resolution === 'name-match') reasons.push(`dependency-matched-by-name-only:${d.dependency}`);
+      }
+    }
+    else reasons.push('decorated-handler-not-found');
+    return out;
+  }
+
+  const st = JS_FILE.test(route.file) ? callStatement(lines, route.line) : null;
+  if (!st) {
+    // not a call-style registration we can parse: fall back to the enclosing declaration, flagged
+    const enclosing = fileSyms.filter(s => s.kind === 'function' && s.startLine <= route.line && s.endLine >= route.line).sort((a, b) => b.startLine - a.startLine)[0];
+    if (enclosing) { out.handlerSymbol = enclosing.id; out.handlerKind = 'enclosing'; out.confidence = 'enclosing'; reasons.push('registration-form-not-parsed'); }
+    else reasons.push('registration-form-not-parsed');
+    return out;
+  }
+  out.rootSpan = { file: route.file, startLine: route.line, endLine: st.endLine, precision: 'brace' };
+  const args = splitArgs(st.text).slice(1); // drop the path
+  const flat = [];
+  for (const a of args) {
+    if (a.startsWith('[') && a.endsWith(']')) flat.push(...splitArgs(`(${a.slice(1, -1)})`));
+    else flat.push(a);
+  }
+  if (!flat.length) { reasons.push('registration-has-no-handler-argument'); return out; }
+  const last = flat[flat.length - 1];
+  for (let i = 0; i < flat.length - 1; i++) {
+    const a = flat[i];
+    if (INLINE_FN.test(a)) { out.middleware.push({ name: '<inline>', symbolId: null, resolution: 'inline', line: route.line }); continue; }
+    const id = IDENT_CHAIN.exec(a);
+    if (!id) { out.middleware.push({ name: null, text: a.slice(0, 80), symbolId: null, resolution: 'unresolved', line: route.line }); reasons.push('middleware-not-an-identifier'); continue; }
+    const r = resolveName(id[0], route.file, idx);
+    out.middleware.push({ name: id[0], symbolId: r.symbol ? r.symbol.id : null, resolution: r.resolution, line: route.line, factory: /\(/.test(a) });
+    if (!r.symbol) reasons.push(`middleware-${r.resolution}:${id[0]}`);
+    else if (r.resolution === 'name-match') reasons.push(`middleware-matched-by-name-only:${id[0]}`);
+  }
+  if (INLINE_FN.test(last)) {
+    const cands = fileSyms.filter(s => s.kind === 'function' && s.name.startsWith('<anon>') === false && false);
+    const anon = fileSyms.filter(s => s.id.includes('::<anon>@') && s.declLine >= route.line && s.declLine <= st.endLine);
+    const outermost = anon.filter(a => !anon.some(b => b !== a && b.startLine <= a.startLine && b.endLine >= a.endLine && (b.startLine < a.startLine || b.endLine > a.endLine)));
+    void cands;
+    const pick = outermost.sort((a, b) => a.declLine - b.declLine || (a.id < b.id ? -1 : 1)).pop();
+    if (pick) { out.handlerSymbol = pick.id; out.handlerKind = 'inline'; out.confidence = 'exact'; }
+    else reasons.push('inline-handler-not-in-ir');
+  } else {
+    const id = IDENT_CHAIN.exec(last);
+    if (!id) { reasons.push('handler-not-an-identifier'); }
+    else {
+      const r = resolveName(id[0], route.file, idx);
+      if (r.symbol) { out.handlerSymbol = r.symbol.id; out.handlerKind = 'named'; out.confidence = r.resolution === 'resolved' ? 'exact' : 'name-match'; if (r.resolution !== 'resolved') reasons.push('handler-matched-by-name-only'); }
+      else reasons.push(`handler-${r.resolution}:${id[0]}`);
+    }
+  }
+  void graph;
+  return out;
+}
+
+function pathMatchesPrefix(routePath, prefix) {
+  if (prefix === '/' || prefix === '') return true;
+  const p = prefix.replace(/\/+$/, '');
+  return routePath === p || routePath.startsWith(p + '/') || routePath.startsWith(p);
+}
+
+/**
+ * Build the entry-point records.
+ *   config.entrypointKinds, config.exclude (globs on file or route path), config.maxEntrypoints
+ */
+export async function collectEntrypoints({ fileContents, graph, config = {} }) {
+  const kinds = new Set(config.entrypointKinds && config.entrypointKinds.length ? config.entrypointKinds : ['http']);
+  const idx = symbolIndexes(graph);
+  const uses = collectUses(fileContents);
+  const exclude = (config.exclude || []).map(globToRe);
+  const cap = Number.isInteger(config.maxEntrypoints) && config.maxEntrypoints > 0 ? config.maxEntrypoints : 5000;
+  const ctx = { fileContents, idx, graph };
+  const records = [];
+
+  if (kinds.has('http') || kinds.has('webhook')) {
+    for (const route of await collectRoutes(fileContents)) {
+      const isWebhook = /webhook/i.test(route.path || '');
+      if (isWebhook && !kinds.has('webhook') && !kinds.has('http')) continue;
+      const h = resolveHandler(route, ctx);
+      const reasons = [...h.reasons];
+      const middleware = [...h.middleware];
+      // inherited middleware registered with `X.use(...)` on the same receiver, earlier in the same file
+      if (JS_FILE.test(route.file)) {
+        const lines = fileContents[route.file].split('\n');
+        const recvM = /([A-Za-z_$][\w$]*)\.(?:get|post|put|patch|delete|all|options|head|route)\s*\(/.exec(lines[route.line - 1] || '');
+        const receiver = recvM ? recvM[1] : null;
+        for (const u of uses) {
+          if (u.file !== route.file || u.line >= route.line || u.receiver !== receiver) continue;
+          if (!pathMatchesPrefix(String(route.path || ''), u.prefix)) continue;
+          for (const a of u.args) {
+            if (!a.name) { middleware.push({ name: null, text: a.text.slice(0, 80), symbolId: null, resolution: 'unresolved', line: u.line, scope: u.prefix, inherited: true }); reasons.push('inherited-middleware-not-an-identifier'); continue; }
+            const r = resolveName(a.name, u.file, idx);
+            middleware.push({ name: a.name, symbolId: r.symbol ? r.symbol.id : null, resolution: r.resolution, line: u.line, scope: u.prefix, inherited: true });
+            if (!r.symbol) reasons.push(`inherited-middleware-${r.resolution}:${a.name}`);
+          }
+        }
+        if (receiver && /^(?:router|routes?|r|api|\w*Router)$/i.test(receiver) && !uses.some(u => u.receiver !== receiver && u.args.some(a => a.name === receiver))) {
+          reasons.push('router-mount-not-resolved');
+        }
+      }
+      const authScheme = route.webModel
+        ? { status: route.webModel.auth.status, kind: route.webModel.auth.kind || null, basis: 'haskell-web-model', role: route.webModel.role.status, ownership: route.webModel.ownership || null, idLookups: route.webModel.idLookups || [], csrf: !!route.webModel.csrf }
+        : { status: route.hasAuth ? 'authenticated' : 'none', basis: 'route-scan-heuristic', note: 'authentication seen near the registration; says nothing about ownership' };
+      const base = {
+        kind: isWebhook ? 'webhook' : 'http',
+        method: String(route.method || 'GET'),
+        pathTemplate: String(route.path || ''),
+        framework: route.framework || null,
+        language: route.language === 'haskell' ? 'haskell' : (JS_FILE.test(route.file) ? 'javascript' : (route.file.split('.').pop())),
+        registrationSite: { file: route.file, line: route.line },
+        handlerSymbol: h.handlerSymbol,
+        handlerKind: h.handlerKind,
+        handlerConfidence: h.confidence,
+        rootSpan: h.rootSpan,
+        middleware,
+        authScheme,
+        externalDependencies: [],
+        resolutionReasons: [...new Set(reasons)].sort(),
+      };
+      records.push(base);
+    }
+  }
+
+  // non-HTTP surfaces from the existing inventory
+  const wantInv = ['queue', 'cron', 'upload', 'cli', 'webhook'].filter(k => kinds.has(k));
+  if (wantInv.length) {
+    const inv = buildEntrypointInventory(fileContents, { routes: [] });
+    for (const e of inv.entrypoints) {
+      if (!wantInv.includes(e.type)) continue;
+      const fileSyms = idx.byFile.get(e.file) || [];
+      const enclosing = fileSyms.filter(s => s.kind === 'function' && s.startLine <= e.line && s.endLine >= e.line).sort((a, b) => b.startLine - a.startLine)[0];
+      records.push({
+        kind: e.type, method: '', pathTemplate: e.name, framework: null,
+        language: JS_FILE.test(e.file) ? 'javascript' : e.file.split('.').pop(),
+        registrationSite: { file: e.file, line: e.line },
+        handlerSymbol: enclosing ? enclosing.id : null,
+        handlerKind: enclosing ? 'enclosing' : 'unknown', handlerConfidence: enclosing ? 'enclosing' : 'none',
+        rootSpan: null, middleware: [],
+        authScheme: { status: e.trust === 'authenticated' ? 'authenticated' : 'none', basis: 'inventory-heuristic' },
+        externalDependencies: [],
+        resolutionReasons: enclosing ? ['non-http-handler-is-enclosing-declaration'] : ['no-enclosing-declaration'],
+      });
+    }
+  }
+
+  // ids, dispositions
+  records.sort((a, b) =>
+    (a.registrationSite.file < b.registrationSite.file ? -1 : a.registrationSite.file > b.registrationSite.file ? 1 : 0) ||
+    (a.registrationSite.line - b.registrationSite.line) || (a.method < b.method ? -1 : 1) || (a.pathTemplate < b.pathTemplate ? -1 : a.pathTemplate > b.pathTemplate ? 1 : 0));
+  const ord = new Map();
+  records.forEach((r, i) => {
+    const hname = r.handlerSymbol || `@${r.registrationSite.line}`;
+    const stem = hashOf({ k: r.kind, m: r.method, p: r.pathTemplate, f: r.registrationSite.file, h: r.handlerSymbol ? r.handlerSymbol : null });
+    const n = (ord.get(stem) || 0) + 1;
+    ord.set(stem, n);
+    r.id = `ep:${stem.slice(0, 20)}${n > 1 ? `-${n}` : ''}`;
+    r.label = safeLabel(`${r.method ? r.method + ' ' : ''}${r.pathTemplate}`) || r.kind;
+    void hname;
+    let disposition;
+    const excluded = exclude.some(re => re.test(r.registrationSite.file) || re.test(r.pathTemplate));
+    if (excluded) { disposition = 'excluded_by_policy'; r.resolutionReasons = [...r.resolutionReasons, 'excluded-by-operator-pattern']; }
+    else if (i >= cap) { disposition = 'excluded_by_policy'; r.resolutionReasons = [...r.resolutionReasons, 'max-entrypoints-cap']; }
+    else if (!r.handlerSymbol && !r.rootSpan) disposition = 'unresolved';
+    else if (!r.handlerSymbol) disposition = 'partially_resolved';
+    else if (r.resolutionReasons.length) disposition = 'partially_resolved';
+    else disposition = 'resolved';
+    r.disposition = disposition;
+  });
+  return records;
+}
+
+/** Counts per disposition. Every entry point is in exactly one bucket, so the buckets always sum to the total. */
+export function countDispositions(records) {
+  const c = { total: records.length, resolved: 0, partially_resolved: 0, unresolved: 0, excluded_by_policy: 0 };
+  for (const r of records) c[r.disposition]++;
+  return c;
+}

package/src/context-intelligence/evaluation/index.js

@@ -0,0 +1,235 @@
+//
+// `agentic-security context eval`: the frozen synthetic suite, scored.
+//
+// Honest by construction:
+//   * every scorecard says on its face that the suite and the reader are synthetic and that no accuracy claim follows;
+//   * intervals come from the repo's existing grouped bootstrap (a case is one group; fewer than 10 groups is "unmeasured");
+//   * the holdout split is scored only through a ledger that counts every use, and non-default prompt variants are refused
+//     on it unless the run is declared final;
+//   * a label-leakage audit checks that neither the corpus nor any prompt a system saw contains the label vocabulary.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { sha256, canonicalJson, hashOf } from '../canonical.js';
+import { writeStoreJson, readStoreFile } from '../store.js';
+import { groupedBootstrap, wilsonInterval, f1Of, precisionOf, recallOf } from '../../posture/evaluation/interval.js';
+import { runCase, SYSTEMS } from './runner.js';
+import { readerIdentity } from './scripted-reader.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+// The bundled synthetic suite lives in the source tree's fixtures, found from the package root. Scan code must not
+// read evaluation labels, so this is reachable only from the explicit `context eval` command, and `--suite-dir` /
+// AGENTIC_SECURITY_DCI_EVAL_SUITE can point at any other suite in the same layout.
+function packageRoot(from) {
+  let dir = from;
+  for (let i = 0; i < 8; i++) {
+    if (fs.existsSync(path.join(dir, 'package.json'))) return dir;
+    dir = path.dirname(dir);
+  }
+  return from;
+}
+export const DEFAULT_SUITE_DIR = process.env.AGENTIC_SECURITY_DCI_EVAL_SUITE
+  ? path.resolve(process.env.AGENTIC_SECURITY_DCI_EVAL_SUITE)
+  : path.join(packageRoot(HERE), ['test', 'fixtures', 'context-intelligence', 'eval'].join(path.sep));
+export const VARIANTS = Object.freeze(['dci', 'dci-all-packs']);
+export const FROZEN_VARIANT = 'dci';
+const FLAG = (args, k) => (args && args[k] !== undefined ? args[k] : undefined);
+const truthy = (v) => v === true || v === 'true' || v === '1';
+
+const LEAK_TERMS = /\b(?:vulnerable|vulnerability|idor|bola|bfla|exploit|insecure|label|holdout|ground.?truth|expected result)\b/i;
+
+function loadSuite(dir) {
+  const read = (f) => JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8'));
+  const labels = read('labels.json');
+  const split = read('split.json');
+  return { labels, split, dir };
+}
+
+function walk(dir, out = []) {
+  for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) walk(p, out); else if (e.isFile()) out.push(p);
+  }
+  return out.sort();
+}
+
+function suiteHash(dir) {
+  const files = walk(dir).map(f => ({ f: path.relative(dir, f), h: sha256(fs.readFileSync(f)) }));
+  return hashOf(files);
+}
+
+/** The corpus and every prompt a system saw must not contain the label vocabulary. Returns hits (empty = clean). */
+export function leakageAudit(dir, promptsBySystem) {
+  const hits = [];
+  for (const f of walk(path.join(dir, 'corpus'))) {
+    const text = fs.readFileSync(f, 'utf8');
+    const m = LEAK_TERMS.exec(text);
+    if (m) hits.push({ where: 'corpus', file: path.relative(dir, f), term: m[0] });
+  }
+  for (const [system, prompts] of Object.entries(promptsBySystem)) {
+    for (const p of prompts) {
+      const body = String(p);
+      const m = /\bc(?:0[1-9]|1[01])\b/.exec(body) || /\bdci-eval\b/.exec(body);
+      if (m) { hits.push({ where: `prompt:${system}`, term: m[0] }); break; }
+    }
+  }
+  return hits;
+}
+
+function tally(units) {
+  const c = { tp: 0, fp: 0, fn: 0, tn: 0 };
+  for (const u of units) { c.tp += u.tp; c.fp += u.fp; c.fn += u.fn; c.tn += u.tn; }
+  return c;
+}
+
+export function metricsFor(units, protocolHash, name) {
+  const c = tally(units);
+  const groupOf = Object.fromEntries(units.map(u => [u.id, u.caseId]));
+  const f1 = groupedBootstrap({ units, groupOf, statistic: ({ total }) => f1Of(total), name: `${name}:f1`, protocolHash });
+  const pn = c.tp + c.fp;
+  return {
+    counts: c, units: units.length,
+    precision: precisionOf(c), recall: recallOf(c), f1: f1Of(c),
+    precisionInterval: pn > 0 ? wilsonInterval(c.tp, pn) : { status: 'unmeasured', reason: 'nothing was reported' },
+    f1Interval: f1,
+    // A point estimate with no interval around it reads as a finding. It is reportable only when the interval is measured.
+    reportable: f1 && f1.status === 'measured',
+  };
+}
+
+/** Nearest-rank percentile of a numeric list (null when empty). */
+export function percentile(values, q) {
+  if (!values.length) return null;
+  const v = [...values].sort((a, b) => a - b);
+  return v[Math.min(v.length - 1, Math.max(0, Math.ceil(q * v.length) - 1))];
+}
+
+/** Context-size, call and cost accounting for one system. Token figures are chars/4 estimates; no price is assumed. */
+export function economicsOf({ promptSizes, calls, wallMs, metrics }) {
+  const tokens = promptSizes.map(n => Math.ceil(n / 4));
+  const tp = metrics.counts.tp;
+  return {
+    tokenizer: 'chars/4-estimate',
+    promptsPerCase: promptSizes.length,
+    contextTokens: { median: percentile(tokens, 0.5), p95: percentile(tokens, 0.95), p99: percentile(tokens, 0.99), total: tokens.reduce((a, b) => a + b, 0) },
+    modelCalls: calls,
+    callsPerTruePositive: tp > 0 ? calls / tp : null,
+    wallMs,
+    cost: { usd: null, basis: 'no price supplied; calls and estimated tokens are reported instead' },
+  };
+}
+
+function ledgerPath() { return 'reports/eval-ledger.json'; }
+function readLedger(root) {
+  const r = readStoreFile(root, ledgerPath());
+  if (!r.ok) return { schemaVersion: 'eval-ledger/1', holdoutScoredRuns: 0, devScoredRuns: 0, runs: [] };
+  try { return JSON.parse(r.text); } catch { return { schemaVersion: 'eval-ledger/1', holdoutScoredRuns: 0, devScoredRuns: 0, runs: [], note: 'previous ledger unreadable; counts restart' }; }
+}
+
+const pct = (x) => (x === null || x === undefined ? 'n/a' : (x * 100).toFixed(0) + '%');
+const interval = (i) => (!i || i.status !== 'measured' ? `unmeasured (${i && i.reason ? i.reason.split(',')[0] : 'no data'})` : `${pct(i.low)} to ${pct(i.high)}`);
+
+// Precision, recall and F1 are printed only when their interval is measured; otherwise the counts stand alone.
+const shown = (m, x) => (m.reportable ? pct(x) : 'withheld');
+
+export function renderMarkdown(card) {
+  const L = [];
+  L.push('# Context Intelligence evaluation scorecard', '');
+  L.push('> SYNTHETIC SUITE, SCRIPTED READER. These cases and the reader were written by the same authors, with no independent adjudication and no real-world defect history. The numbers describe the context-building and verification machinery on synthetic cases. They are not evidence about any model, and they make no accuracy claim about real repositories.', '');
+  L.push(`- split: ${card.split}   variant: ${card.variant}   cases: ${card.caseCount}`);
+  L.push(`- suite hash: ${card.hashes.suite}`);
+  L.push(`- protocol hash: ${card.hashes.protocol}`);
+  L.push(`- reader: ${card.reader.kind} v${card.reader.version} ${card.reader.digest} (synthetic)`);
+  L.push(`- label leakage audit: ${card.leakage.length === 0 ? 'clean' : card.leakage.length + ' hit(s)'}`);
+  L.push(`- holdout scored runs so far (this machine's ledger): ${card.ledger.holdoutScoredRuns}`, '');
+  L.push('| system | status | TP | FP | FN | TN | precision | recall | F1 | F1 interval | model calls | prompt chars |', '|---|---|---|---|---|---|---|---|---|---|---|---|');
+  for (const s of card.systems) {
+    if (s.status !== 'measured') { L.push(`| ${s.system} | ${s.status}: ${s.reason} | | | | | | | | | | |`); continue; }
+    const m = s.metrics;
+    L.push(`| ${s.system} | measured | ${m.counts.tp} | ${m.counts.fp} | ${m.counts.fn} | ${m.counts.tn} | ${shown(m, m.precision)} | ${shown(m, m.recall)} | ${shown(m, m.f1)} | ${interval(m.f1Interval)} | ${s.calls} | ${s.promptChars} |`);
+  }
+  L.push('', '## Economics (estimated; no price supplied)', '', '| system | median ctx tokens | P95 | P99 | model calls | calls per true positive | wall ms |', '|---|---|---|---|---|---|---|');
+  for (const s of card.systems) if (s.economics) { const e = s.economics; L.push(`| ${s.system} | ${e.contextTokens.median ?? 'n/a'} | ${e.contextTokens.p95 ?? 'n/a'} | ${e.contextTokens.p99 ?? 'n/a'} | ${e.modelCalls} | ${e.callsPerTruePositive === null || !s.metrics || !s.metrics.reportable ? 'withheld' : e.callsPerTruePositive.toFixed(1)} | ${e.wallMs} |`); }
+  L.push('', 'Precision, recall, F1 and calls per true positive are withheld while their interval is unmeasured (fewer than 10 independent groups). The counts above are what was observed; they are not an estimate of anything.');
+  L.push('', `Replay: \`${card.replay}\``, '');
+  if (card.leakage.length) { L.push('## Leakage hits', ...card.leakage.map(h => `- ${h.where} ${h.file || ''} ${h.term}`), ''); }
+  return L.join('\n');
+}
+
+export function renderCsv(card) {
+  // The marker is a column, not a banner: a CSV is the format that gets pasted into a spreadsheet without its header text.
+  const rows = [['system', 'case', 'variant', 'vulnerable', 'flagged', 'outcome', 'calls', 'promptChars', 'synthetic']];
+  for (const s of card.systems) for (const u of (s.units || [])) rows.push([s.system, u.caseId, u.variant, u.vulnerable, u.flagged, u.outcome, u.calls, u.promptChars, card.synthetic === false ? false : true]);
+  return rows.map(r => r.join(',')).join('\n') + '\n';
+}
+
+export async function evaluateSuite({ suiteDir = DEFAULT_SUITE_DIR, split = 'dev', systems = ['deterministic', 'hunt', 'dci'], variant = FROZEN_VARIANT, root = null, final = false, onProgress = () => {} } = {}) {
+  const suite = loadSuite(suiteDir);
+  const ids = split === 'all' ? [...suite.split.dev, ...suite.split.holdout] : suite.split[split];
+  if (!ids) return { ok: false, reason: `unknown split '${split}' (dev, holdout, all)` };
+  if (!VARIANTS.includes(variant)) return { ok: false, reason: `unknown variant '${variant}' (${VARIANTS.join(', ')})` };
+  const touchesHoldout = split !== 'dev';
+  if (touchesHoldout && variant !== FROZEN_VARIANT && !final) return { ok: false, reason: `variant '${variant}' is not the frozen one; develop variants on the dev split. The holdout may only score the frozen variant unless the run is declared --final.` };
+
+  const protocol = { suite: suiteHash(suiteDir), split, variant, systems, reader: readerIdentity(), tolerance: 5 };
+  const protocolHash = hashOf(protocol);
+  const promptsBySystem = {};
+  const out = [];
+  for (const system of systems) {
+    if (!SYSTEMS.includes(system)) { out.push({ system, status: 'error', reason: 'unknown system' }); continue; }
+    const units = []; let calls = 0; let promptChars = 0; let blocked = null; const errors = []; const promptSizes = []; const t0 = Date.now();
+    for (const id of ids) {
+      const def = { id, ...suite.labels.cases[id] };
+      for (const key of Object.keys(def.variants).sort()) {
+        const v = def.variants[key];
+        onProgress({ system, case: id, variant: key });
+        const r = await runCase({ system, caseDef: def, variant: { key }, suiteDir, variantName: variant });
+        if (r.status === 'blocked') { blocked = r.reason; continue; }
+        if (r.status === 'error') { errors.push(`${id}/${key}: ${r.reason}`); continue; }
+        const flagged = !!r.flagged; const vuln = !!v.vulnerable;
+        const u = { id: `${id}/${key}`, caseId: id, variant: key, language: def.language, vulnerable: vuln, flagged, tp: vuln && flagged ? 1 : 0, fn: vuln && !flagged ? 1 : 0, fp: !vuln && flagged ? 1 : 0, tn: !vuln && !flagged ? 1 : 0, calls: r.calls, promptChars: r.promptChars, outcome: vuln ? (flagged ? 'tp' : 'fn') : (flagged ? 'fp' : 'tn'), detail: r.detail };
+        units.push(u); calls += r.calls; promptChars += r.promptChars;
+        (promptsBySystem[system] = promptsBySystem[system] || []).push(...r.prompts);
+        promptSizes.push(...r.prompts.map(x => String(x).length));
+      }
+    }
+    if (blocked && units.length === 0) { out.push({ system, status: 'blocked', reason: blocked }); continue; }
+    if (errors.length) { out.push({ system, status: 'error', reason: `${errors.length} case run(s) failed: ${errors.slice(0, 3).join('; ')}` }); continue; }
+    const metrics = metricsFor(units, protocolHash, system);
+    out.push({ system, status: 'measured', metrics, units, calls, promptChars, economics: economicsOf({ promptSizes, calls, wallMs: Date.now() - t0, metrics }) });
+  }
+  const leakage = leakageAudit(suiteDir, promptsBySystem);
+
+  let ledger = { holdoutScoredRuns: 0, devScoredRuns: 0 };
+  if (root) {
+    ledger = readLedger(root);
+    if (touchesHoldout) ledger.holdoutScoredRuns += 1; else ledger.devScoredRuns += 1;
+    ledger.runs = [...(ledger.runs || []).slice(-49), { protocolHash, split, variant, systems }];
+    writeStoreJson(root, ledgerPath(), ledger);
+  }
+  const replay = `agentic-security context eval --suite fixtures --split ${split} --variant ${variant} --baseline ${systems.join(',')}`;
+  const card = { schemaVersion: 'dci-eval-scorecard/1', synthetic: true, claim: 'machinery on synthetic cases only; no accuracy claim about real repositories', split, variant, caseCount: ids.length, hashes: { suite: protocol.suite, protocol: protocolHash }, reader: readerIdentity(), leakage, ledger: { holdoutScoredRuns: ledger.holdoutScoredRuns, devScoredRuns: ledger.devScoredRuns }, systems: out, replay };
+  card.hashes.scorecard = hashOf({ ...card, ledger: null, systems: card.systems.map(x => (x.economics ? { ...x, economics: { ...x.economics, wallMs: null } } : x)) });
+  return { ok: true, card };
+}
+
+export async function cmdEval(root, args, format) {
+  const suiteArg = FLAG(args, 'suite');
+  if (suiteArg !== undefined && suiteArg !== 'fixtures' && suiteArg !== true) { console.log("context eval: only '--suite fixtures' is available (the bundled synthetic suite)"); return 1; }
+  if (!fs.existsSync(path.join(DEFAULT_SUITE_DIR, 'labels.json'))) { console.log('context eval: the synthetic suite is not present in this install (it ships with the source tree)'); return 1; }
+  const baseline = FLAG(args, 'baseline') && FLAG(args, 'baseline') !== true ? String(FLAG(args, 'baseline')).split(',').map(s => s.trim()).filter(Boolean) : ['deterministic', 'hunt', 'dci'];
+  const variant = FLAG(args, 'variant') && FLAG(args, 'variant') !== true ? String(FLAG(args, 'variant')) : FROZEN_VARIANT;
+  const split = FLAG(args, 'split') && FLAG(args, 'split') !== true ? String(FLAG(args, 'split')) : 'dev';
+  const r = await evaluateSuite({ split, variant, systems: baseline, root, final: truthy(FLAG(args, 'final')) });
+  if (!r.ok) { console.log(`context eval: ${r.reason}`); return 1; }
+  const card = r.card;
+  const base = `reports/eval-${card.hashes.scorecard.slice(0, 16)}`;
+  const w = [writeStoreJson(root, `${base}.json`, card)];
+  const md = renderMarkdown(card);
+  const fileWrites = [['.md', md], ['.csv', renderCsv(card)]];
+  const { writeStoreFile } = await import('../store.js');
+  for (const [ext, text] of fileWrites) w.push(writeStoreFile(root, base + ext, text));
+  if (format === 'json') console.log(JSON.stringify({ ...card, files: w.map(x => x && x.ok !== false) }, null, 2));
+  else console.log(md + `\nWritten: .agentic-security/context-intelligence/${base}.{json,md,csv}`);
+  return card.leakage.length || card.systems.some(s => s.status === 'error') ? 1 : 0;
+}