@clear-capabilities/agentic-security-scanner 0.175.0 → 0.177.0

package/src/posture/evaluation/lab/score.js

@@ -0,0 +1,135 @@
+// Scoring for the Eval Lab (PRD 6): detection, falsification, exploit proof, remediation, economics. Pure functions over
+// recorded attempts and the dataset's labels; no fs, no clock, no process.
+//
+// Principles carried over from the rest of the evaluation code:
+//   * a figure never appears without the counts behind it;
+//   * an attempt that timed out, errored or was blocked is a recorded NO-RESULT. For a known positive it is a miss; for a
+//     negative it is neither a true negative nor a false positive (it is `unscored`), so a model cannot look precise by crashing;
+//   * a quantity that could not be measured is `null` with a reason, never 0;
+//   * detection is scored on LOCATION. A finding of the right class in the right file but outside the labelled span is a
+//     file-level hit and a localization miss, and only a localized hit is a true positive.
+import { normalizeCwe } from './dataset.js';
+import { ratio, f1 } from './stats.js';
+
+const DEFAULT_LINE_TOLERANCE = 3;
+
+const resultOf = (a) => a && a.status === 'completed';
+
+/** @param {object[]} entries dataset entries @param {Map<string, object>} attempts entryId -> attempt @param {Map<string,string>} fileMaps `${entryId}\0${path}` -> staged path */
+export function scoreDetect({ entries, attempts, mapFile = (entryId, f) => f, tolerance = DEFAULT_LINE_TOLERANCE, targetCwes = null }) {
+  const cweSet = targetCwes ? new Set(targetCwes.map(normalizeCwe)) : null;
+  const perCwe = new Map();
+  const slot = (c) => { if (!perCwe.has(c)) perCwe.set(c, { tp: 0, fn: 0, fp: 0, fileHits: 0 }); return perCwe.get(c); };
+  const byCategory = {};
+  const c = { tp: 0, fn: 0, fp: 0, tn: 0, unscoredNegatives: 0, noResultPositives: 0, fileHits: 0, positives: 0, negatives: 0, attempted: 0, completed: 0 };
+  const misses = [];
+  for (const e of entries) {
+    const a = attempts.get(e.id);
+    if (!a) continue;
+    c.attempted++;
+    if (resultOf(a)) c.completed++;
+    const found = resultOf(a) ? (a.output?.findings ?? []) : [];
+    if (e.kind === 'vulnerable') {
+      c.positives++;
+      const cwe = normalizeCwe(e.cwe); const s = slot(cwe);
+      if (!resultOf(a)) { c.fn++; s.fn++; c.noResultPositives++; misses.push({ entryId: e.id, reason: `no-result:${a.status}` }); continue; }
+      const file = mapFile(e.id, e.expectedSpan.file);
+      const sameFile = found.filter((f) => f.file === file && normalizeCwe(f.cwe) === cwe);
+      const loc = sameFile.some((f) => Number.isInteger(f.line) && f.line >= e.expectedSpan.startLine - tolerance && f.line <= e.expectedSpan.endLine + tolerance);
+      if (sameFile.length) { c.fileHits++; s.fileHits++; }
+      if (loc) { c.tp++; s.tp++; } else { c.fn++; s.fn++; misses.push({ entryId: e.id, reason: sameFile.length ? 'wrong-location' : found.some((f) => f.file === file) ? 'wrong-class' : 'not-found' }); }
+    } else {
+      c.negatives++;
+      const cat = e.kind === 'hard-negative' ? e.category : 'fixed';
+      byCategory[cat] ??= { entries: 0, falsePositives: 0, unscored: 0 };
+      byCategory[cat].entries++;
+      if (!resultOf(a)) { c.unscoredNegatives++; byCategory[cat].unscored++; continue; }
+      const relevant = found.filter((f) => !cweSet || f.cwe == null || cweSet.has(normalizeCwe(f.cwe)));
+      if (relevant.length) {
+        c.fp++; byCategory[cat].falsePositives++;
+        for (const k of new Set(relevant.map((f) => normalizeCwe(f.cwe) ?? 'unclassified'))) slot(k).fp++;
+      } else c.tn++;
+    }
+  }
+  const precision = ratio(c.tp, c.tp + c.fp); const recall = ratio(c.tp, c.tp + c.fn);
+  const per = {};
+  for (const [k, v] of [...perCwe.entries()].sort()) { const p = ratio(v.tp, v.tp + v.fp); const r = ratio(v.tp, v.tp + v.fn); per[k] = { ...v, precision: p, recall: r, f1: v.tp + v.fn > 0 ? f1(p ?? (v.tp + v.fn > 0 ? 0 : null), r) : null }; }
+  const f1s = Object.values(per).filter((p) => p.tp + p.fn > 0 && p.f1 != null).map((p) => p.f1);
+  return {
+    counts: c, precision, recall, f1: c.positives > 0 && c.tp === 0 ? 0 : f1(precision, recall),
+    macroF1: f1s.length ? f1s.reduce((a, b) => a + b, 0) / f1s.length : null, perCwe: per,
+    fileLevelRecall: ratio(c.fileHits, c.positives), localizationAccuracy: ratio(c.tp, c.fileHits),
+    hardNegativeFalsePositiveRate: Object.fromEntries(Object.entries(byCategory).map(([k, v]) => [k, { ...v, rate: ratio(v.falsePositives, v.entries - v.unscored) }])),
+    completion: ratio(c.completed, c.attempted), misses, tolerance,
+  };
+}
+
+/** truth: Map candidateId -> 'vulnerable' | 'not-vulnerable'. verdicts: Map candidateId -> attempt-level verdict or null (no result). */
+export function scoreFalsify({ truth, verdicts }) {
+  const c = { vulnerable: 0, negative: 0, confirmedVuln: 0, refutedVuln: 0, abstainedVuln: 0, refutedNeg: 0, confirmedNeg: 0, abstainedNeg: 0, noResult: 0 };
+  for (const [id, t] of truth) {
+    const v = verdicts.get(id) ?? null;
+    if (t === 'vulnerable') c.vulnerable++; else c.negative++;
+    if (v == null) { c.noResult++; continue; }
+    if (t === 'vulnerable') { if (v === 'confirmed') c.confirmedVuln++; else if (v === 'refuted') c.refutedVuln++; else c.abstainedVuln++; }
+    else if (v === 'refuted') c.refutedNeg++; else if (v === 'confirmed') c.confirmedNeg++; else c.abstainedNeg++;
+  }
+  const tpr = ratio(c.confirmedVuln, c.vulnerable); const tnr = ratio(c.refutedNeg, c.negative);
+  return { counts: c, truePositiveRate: tpr, trueNegativeRate: tnr, balancedAccuracy: tpr == null || tnr == null ? null : (tpr + tnr) / 2, falseRefutationRate: ratio(c.refutedVuln, c.vulnerable), falseConfirmationRate: ratio(c.confirmedNeg, c.negative) };
+}
+
+/** proofs: array of {entryKind, reproducerValid, state, successes, runs, status}. */
+export function scoreExploit({ proofs }) {
+  const vuln = proofs.filter((p) => p.entryKind === 'vulnerable');
+  const neg = proofs.filter((p) => p.entryKind !== 'vulnerable');
+  const valid = proofs.filter((p) => p.reproducerValid);
+  const executed = valid.filter((p) => p.executed);
+  // reproduction is a property of the VULNERABLE code: a reproducer that (rightly) does nothing to fixed code is not a failure to reproduce
+  const executedVuln = executed.filter((p) => p.entryKind === 'vulnerable');
+  const reproduced = executedVuln.filter((p) => (p.successes ?? 0) > 0);
+  const confirmedVuln = vuln.filter((p) => p.state === 'EXPLOIT_CONFIRMED').length;
+  const confirmedNeg = neg.filter((p) => p.state === 'EXPLOIT_CONFIRMED').length;
+  return {
+    counts: { attempted: proofs.length, vulnerable: vuln.length, negative: neg.length, reproducerValid: valid.length, executed: executed.length, notExecuted: valid.length - executed.length, reproduced: reproduced.length, confirmedVuln, confirmedNeg },
+    pocGenerationSuccess: ratio(valid.length, proofs.length),
+    reproductionRate: ratio(reproduced.length, executedVuln.length),
+    confirmationRate: ratio(confirmedVuln, vuln.length),
+    falseConfirmationRate: ratio(confirmedNeg, neg.length),
+    unmeasured: valid.length > executed.length ? `${valid.length - executed.length} reproducer(s) could not be executed here (no execution environment); they are not counted as failures` : null,
+  };
+}
+
+/** patches: array of {applicable, patchProvided, exploitBlocked: true|false|null, testsPass: true|false|null, completeness, state}. */
+export function scorePatch({ patches }) {
+  const app = patches.filter((p) => p.applicable);
+  const provided = app.filter((p) => p.patchProvided);
+  const verified = provided.filter((p) => p.exploitBlocked !== null);
+  const blocked = verified.filter((p) => p.exploitBlocked === true);
+  const testable = provided.filter((p) => p.testsPass !== null);
+  const strict = provided.filter((p) => p.exploitBlocked === true && p.testsPass === true);
+  const tiers = { FULL: 0, DEGRADED: 0, INCOMPLETE: 0 };
+  for (const p of provided) if (tiers[p.completeness] !== undefined) tiers[p.completeness]++;
+  return {
+    counts: { applicable: app.length, patchProvided: provided.length, verifiable: verified.length, exploitBlocked: blocked.length, testsMeasured: testable.length, testsPassed: testable.filter((p) => p.testsPass).length, validStrict: strict.length },
+    patchRate: ratio(provided.length, app.length),
+    validPatchRate: ratio(strict.length, app.length),
+    exploitBlockedRate: ratio(blocked.length, app.length),
+    testPreservationRate: ratio(testable.filter((p) => p.testsPass).length, testable.length),
+    regressionRate: ratio(testable.filter((p) => !p.testsPass).length, testable.length),
+    completenessTier: tiers,
+    unmeasured: testable.length < provided.length ? `${provided.length - testable.length} patch(es) had no runnable test command, so test preservation is unmeasured for them` : null,
+  };
+}
+
+/** Sum what the attempts cost. Unknown usage is counted, never read as zero. */
+export function scoreEconomics(attempts) {
+  const known = attempts.filter((a) => a.usage?.usd != null);
+  const lat = attempts.map((a) => a.usage?.latencyMs).filter((x) => Number.isFinite(x)).sort((a, b) => a - b);
+  const sum = (k) => attempts.reduce((s, a) => s + (a.usage?.[k] ?? 0), 0);
+  return {
+    attempts: attempts.length, usdMeasured: known.reduce((s, a) => s + a.usage.usd, 0), attemptsWithUnknownCost: attempts.length - known.length,
+    inputTokens: sum('inputTokens'), outputTokens: sum('outputTokens'), tokensUnknownAttempts: attempts.filter((a) => a.usage?.inputTokens == null || a.usage?.outputTokens == null).length,
+    latencyMs: { median: lat.length ? (lat.length % 2 ? lat[lat.length >> 1] : (lat[(lat.length >> 1) - 1] + lat[lat.length >> 1]) / 2) : null, p95: lat.length ? lat[Math.min(lat.length - 1, Math.ceil(lat.length * 0.95) - 1)] : null, total: lat.reduce((a, b) => a + b, 0) },
+    timeouts: attempts.filter((a) => a.status === 'timeout').length, failures: attempts.filter((a) => ['error', 'blocked', 'unavailable'].includes(a.status)).length,
+  };
+}

package/src/posture/evaluation/lab/scoreboard.js

@@ -0,0 +1,48 @@
+// From measured runs to the table the router reads (PRD 6, requirement 7). One entry per (model, task, dataset): the task's
+// primary quality with its spread and run count, the MEDIAN cost and latency of a single task call, where the evidence came from,
+// and whether the model is local. Only model arms appear; the deterministic baseline is a comparison point, not a routing choice.
+// Pure.
+import { summarize, median } from './stats.js';
+import { PRIMARY } from './report.js';
+
+const SCOREBOARD_SCHEMA = 'agentic-security/model-scoreboard';
+export const ROUTER_TASK_OF = Object.freeze({ hunt: 'detect', falsification: 'falsify', 'exploit-generation': 'exploit', 'patch-generation': 'patch' });
+const EVAL_TASK_OF_ROUTER = Object.fromEntries(Object.entries(ROUTER_TASK_OF).map(([k, v]) => [v, k]));
+
+const primaryValue = (score, evalTask) => {
+  const s = score?.[PRIMARY[evalTask]];
+  return typeof s === 'number' && Number.isFinite(s) ? s : null;
+};
+
+export function scoreboardFromRun(run, armDefs = []) {
+  const entries = [];
+  for (const arm of run.arms.filter((a) => a.model)) {
+    const def = armDefs.find((d) => d.id === arm.id);
+    const rs = run.results.filter((r) => r.armId === arm.id);
+    for (const evalTask of ['detect', 'falsify', 'exploit', 'patch']) {
+      if (!rs.some((r) => r.scores[evalTask])) continue;
+      const q = summarize(rs.map((r) => primaryValue(r.scores[evalTask], evalTask)));
+      const calls = run.attempts.filter((a) => a.armId === arm.id && a.task === evalTask && a.status !== 'skipped-budget');
+      const costs = calls.map((a) => a.usage?.usd);
+      const costKnown = costs.length > 0 && costs.every((x) => x != null);
+      entries.push({
+        model: `${arm.model.provider}/${arm.model.id}`, armId: arm.id, local: arm.model.local, endpoint: def?.model?.endpoint ?? null,
+        routerTask: EVAL_TASK_OF_ROUTER[evalTask], evalTask, datasetHash: run.dataset.hash, datasetId: run.dataset.id,
+        quality: { mean: q.mean, stdev: q.stdev, ci95: q.ci95, n: q.n, representative: q.representative, metric: PRIMARY[evalTask] },
+        medianCostUsd: costKnown ? median(costs) : null, costKnown,
+        medianLatencyMs: median(calls.map((a) => a.usage?.latencyMs)),
+        failures: calls.filter((a) => a.status !== 'completed').length, calls: calls.length,
+        evidence: { runId: run.runId, runDigest: run.digest, measuredAt: run.finishedAt },
+      });
+    }
+  }
+  return entries;
+}
+
+/** Merge a run's entries into a scoreboard document; a newer entry for the same (model, task, dataset) replaces the older. */
+export function mergeScoreboard(existing, entries) {
+  const key = (e) => `${e.model}|${e.routerTask}|${e.datasetHash}`;
+  const map = new Map((existing?.entries ?? []).map((e) => [key(e), e]));
+  for (const e of entries) map.set(key(e), e);
+  return { schema: SCOREBOARD_SCHEMA, version: 1, updatedAt: new Date().toISOString(), entries: [...map.values()].sort((a, b) => key(a).localeCompare(key(b))) };
+}

package/src/posture/evaluation/lab/stats.js

@@ -0,0 +1,41 @@
+// Small, dependency-free statistics for repeated runs (PRD 6, requirement 6: never publish one lucky run as representative).
+// Pure: no fs, no clock. A summary of fewer than MIN_REPRESENTATIVE_RUNS values carries `representative: false` and says why,
+// so no consumer can print its mean as a headline without also printing that.
+
+export const MIN_REPRESENTATIVE_RUNS = 3;
+
+// two-sided 95% Student-t critical values for 1..30 degrees of freedom; 1.96 beyond
+const T95 = [12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.160, 2.145, 2.131, 2.120, 2.110, 2.101, 2.093, 2.086, 2.080, 2.074, 2.069, 2.064, 2.060, 2.056, 2.052, 2.048, 2.045, 2.042];
+
+const finite = (xs) => xs.filter((x) => typeof x === 'number' && Number.isFinite(x));
+
+export function median(xs) {
+  const v = finite(xs).sort((a, b) => a - b);
+  if (!v.length) return null;
+  const m = v.length >> 1;
+  return v.length % 2 ? v[m] : (v[m - 1] + v[m]) / 2;
+}
+
+/** mean, sample standard deviation, range and a 95% interval of the mean. Unmeasured (null) values are excluded and counted. */
+export function summarize(values) {
+  const all = Array.isArray(values) ? values : [];
+  const v = finite(all);
+  const unmeasured = all.length - v.length;
+  if (!v.length) return { n: 0, unmeasured, mean: null, stdev: null, min: null, max: null, ci95: null, representative: false, note: 'no measured value' };
+  const n = v.length;
+  const mean = v.reduce((a, b) => a + b, 0) / n;
+  const stdev = n > 1 ? Math.sqrt(v.reduce((a, b) => a + (b - mean) ** 2, 0) / (n - 1)) : null;
+  const t = n > 1 ? (T95[n - 2] ?? 1.96) : null;
+  const half = stdev == null ? null : t * stdev / Math.sqrt(n);
+  const representative = n >= MIN_REPRESENTATIVE_RUNS;
+  return {
+    n, unmeasured, mean, stdev, min: Math.min(...v), max: Math.max(...v),
+    ci95: half == null ? null : { low: mean - half, high: mean + half },
+    representative,
+    ...(representative ? {} : { note: `${n} run${n === 1 ? '' : 's'}: fewer than ${MIN_REPRESENTATIVE_RUNS}, not representative` }),
+  };
+}
+
+export const ratio = (num, den) => (den > 0 ? num / den : null);
+export const f1 = (p, r) => (p == null || r == null ? null : p + r === 0 ? 0 : (2 * p * r) / (p + r));
+export const pct = (x, digits = 1) => (x == null ? 'n/a' : `${(x * 100).toFixed(digits)}%`);

package/src/posture/evaluation/lab/store.js

@@ -0,0 +1,53 @@
+// Persistence for the Eval Lab: `.agentic-security/eval-lab/runs/<runId>.json` and `.agentic-security/eval-lab/scoreboard.json`.
+// Same machinery as the proof and detection stores (safe state writes, the encryption policy, retention as evidence, a secret
+// scrub before anything touches disk). A run or scoreboard whose content no longer matches its own digest is returned flagged
+// and never used for routing: a hand-edited scoreboard must not be able to steer which model gets the work.
+import fs from 'node:fs';
+import path from 'node:path';
+import { statePath, ensureStateDir, safeWriteState } from '../../state-dir.js';
+import { maybeEncryptForWrite, maybeDecryptForRead } from '../../encryption-provider.js';
+import { digestOf } from '../../assurance/identity.js';
+import { scrubDeep } from '../../proof-fabric/store.js';
+
+const DIR = 'eval-lab';
+const MAX_BYTES = 64 * 1024 * 1024;
+const safeId = (id) => /^[A-Za-z0-9._-]{1,80}$/.test(id);
+
+function write(root, parts, doc) {
+  const safe = scrubDeep(doc);
+  safe.digest = digestOf({ ...safe, digest: undefined });
+  const text = JSON.stringify(safe);
+  if (Buffer.byteLength(text) > MAX_BYTES) return { ok: false, reason: 'record exceeds the size limit' };
+  if (!ensureStateDir(root)) return { ok: false, reason: 'no safe state directory for this project' };
+  const enc = maybeEncryptForWrite(root, DIR, text);
+  if (!enc.ok) return { ok: false, reason: enc.reason };
+  const file = statePath(root, DIR, ...parts);
+  if (!safeWriteState(file, enc.content)) return { ok: false, reason: 'state write refused' };
+  return { ok: true, file, digest: safe.digest, encrypted: enc.encrypted };
+}
+function read(file) {
+  try {
+    const doc = JSON.parse(maybeDecryptForRead(fs.readFileSync(file, 'utf8')));
+    return { doc, tampered: doc.digest !== digestOf({ ...doc, digest: undefined }) };
+  } catch { return { doc: null, tampered: true }; }
+}
+
+export const saveRun = (root, run) => (safeId(run.runId) ? write(root, ['runs', `${run.runId}.json`], run) : { ok: false, reason: 'unsafe run id' });
+export function loadRun(root, runId) {
+  if (!safeId(String(runId))) return { doc: null, tampered: false, missing: true };
+  const file = statePath(root, DIR, 'runs', `${runId}.json`);
+  if (!fs.existsSync(file)) return { doc: null, tampered: false, missing: true };
+  return read(file);
+}
+export function listRuns(root) {
+  const dir = statePath(root, DIR, 'runs');
+  let names = [];
+  try { names = fs.readdirSync(dir).filter((n) => n.endsWith('.json')); } catch { return []; }
+  return names.map((n) => { const { doc, tampered } = read(path.join(dir, n)); return { runId: n.replace(/\.json$/, ''), tampered, startedAt: doc?.startedAt ?? null, dataset: doc?.dataset?.id ?? null, arms: doc?.arms?.map((a) => a.id) ?? [] }; }).sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt)));
+}
+export const saveScoreboard = (root, sb) => write(root, ['scoreboard.json'], sb);
+export function loadScoreboard(root) {
+  const file = statePath(root, DIR, 'scoreboard.json');
+  if (!fs.existsSync(file)) return { doc: null, tampered: false, missing: true };
+  return read(file);
+}