@clear-capabilities/agentic-security-scanner 0.175.0 → 0.177.0

package/src/posture/evaluation/lab/neutralize.js

@@ -0,0 +1,144 @@
+// Workspace neutralization and the leakage gate (PRD 6, benchmark integrity).
+//
+// A model must be scored on whether it can find the defect, not on whether it can read the answer. Before any arm sees a
+// workspace this module (1) copies it, (2) strips comments while keeping every line number (so a span still points at the same
+// code), (3) renames files whose NAME carries the weakness class, and (4) runs the existing custody audit
+// (`custody.auditWorkspace`: answer-key file names, symlinks, protected terms, advisory ids) plus three checks of its own:
+// a weakness id anywhere in the content, a vulnerability-describing comment that was left in, and a weakness-bearing file name
+// that the policy did not rename. Any hit quarantines the workspace: the entry is recorded as quarantined and is NOT scanned.
+//
+// What this is not: a proof that no paraphrase gets through. Comment stripping is best effort (a regular expression literal
+// containing `//` can be mis-read), and an identifier that happens to describe the bug stays. The audit result is recorded on
+// every run so a reader can see exactly what was checked.
+import fs from 'node:fs';
+import path from 'node:path';
+import { auditWorkspace } from '../custody.js';
+import { STATE_DIR_NAME } from '../../state-dir.js';
+
+const C_LIKE = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.java', '.c', '.h', '.cc', '.cpp', '.hpp', '.cs', '.go', '.rs', '.kt', '.kts', '.swift', '.scala', '.php']);
+const HASH_LIKE = new Set(['.py', '.rb', '.sh', '.bash', '.pl', '.yaml', '.yml', '.tf', '.toml']);
+const HASH_AND_C = new Set(['.php']);
+// languages where a single quote opens a string of any length; elsewhere it is a short character literal (or a Rust lifetime)
+const SQ_STRING = new Set(['.js', '.mjs', '.cjs', '.ts', '.tsx', '.jsx', '.php']);
+const SKIP = new Set(['.git', 'node_modules', STATE_DIR_NAME]);
+const CWE_NAME = /cwe[-_]?\d{1,5}/i;
+const CWE_CONTENT = /\bcwe[-_ ]?\d{2,5}\b/i;
+const DESCRIPTIVE_COMMENT = /\b(vulnerab\w*|insecure|exploit\w*|potential flaw|flaw:|fixed by|should be (?:sanitized|escaped|validated)|\bBAD\b|\bGOOD\b|sink|taint\w*)\b/i;
+
+/** Remove comments, keeping newlines. String-aware for ' " and ` quotes; best effort, see the file header. */
+export function stripComments(text, ext) {
+  const cLike = C_LIKE.has(ext);
+  const hash = HASH_LIKE.has(ext) || HASH_AND_C.has(ext);
+  if (!cLike && !hash) return text;
+  const s = String(text);
+  let out = '';
+  let i = 0;
+  const n = s.length;
+  while (i < n) {
+    const c = s[i];
+    const two = s.substr(i, 2);
+    if (cLike && two === '//') { while (i < n && s[i] !== '\n') i++; continue; }
+    if (cLike && two === '/*') { i += 2; while (i < n && s.substr(i, 2) !== '*/') { if (s[i] === '\n') out += '\n'; i++; } i += 2; continue; }
+    if (hash && c === '#' && !(ext === '.php' && s[i + 1] === '[')) { while (i < n && s[i] !== '\n') i++; continue; }
+    if (c === '"' || c === '`' || (c === "'" && (hash || SQ_STRING.has(ext) || /^'(?:\\.|[^'\\\n]){1,12}'/.test(s.slice(i, i + 16))))) {
+      const triple = (hash && ext === '.py' && s.substr(i, 3) === c.repeat(3)) ? c.repeat(3) : null;
+      const q = triple || c;
+      out += q; i += q.length;
+      while (i < n) {
+        if (s[i] === '\\') { out += s.substr(i, 2); i += 2; continue; }
+        if (s.substr(i, q.length) === q) { out += q; i += q.length; break; }
+        if (!triple && q !== '`' && s[i] === '\n') break; // an unterminated string ends at the line
+        out += s[i]; i++;
+      }
+      continue;
+    }
+    out += c; i++;
+  }
+  return out;
+}
+
+/** `*` matches within a path segment, `**` across segments; everything else is literal. */
+export function globToRegExp(glob) {
+  let re = '';
+  for (let i = 0; i < glob.length; i++) {
+    const c = glob[i];
+    if (c === '*' && glob[i + 1] === '*') { re += '.*'; i++; if (glob[i + 1] === '/') i++; } else if (c === '*') re += '[^/]*'; else re += c.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
+  }
+  return new RegExp(`^${re}$`, 'i');
+}
+
+const isText = (buf) => !buf.subarray(0, 4096).includes(0);
+
+function copyTree(src, dest, files = []) {
+  fs.mkdirSync(dest, { recursive: true });
+  for (const e of fs.readdirSync(src, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    if (SKIP.has(e.name)) continue;
+    const s = path.join(src, e.name); const d = path.join(dest, e.name);
+    if (e.isSymbolicLink()) { files.push({ rel: path.relative(dest, d), symlink: true }); fs.symlinkSync(fs.readlinkSync(s), d); continue; }
+    if (e.isDirectory()) { copyTree(s, d, files); continue; }
+    if (e.isFile()) fs.copyFileSync(s, d);
+  }
+  return files;
+}
+
+function listFiles(root, rel = '', out = []) {
+  for (const e of fs.readdirSync(path.join(root, rel), { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+    const r = rel ? `${rel}/${e.name}` : e.name;
+    if (e.isDirectory()) listFiles(root, r, out); else if (e.isFile()) out.push(r);
+  }
+  return out;
+}
+
+/**
+ * @returns {{ok: boolean, quarantined: boolean, destDir: string|null, fileMap: Record<string,string>, leaks: object[], audit: object}}
+ * `fileMap` maps original relative paths to staged ones, so a label's span can be located without telling the arm.
+ */
+export function stageNeutralized({ srcDir, destDir, policy = {}, terms = [] }) {
+  fs.rmSync(destDir, { recursive: true, force: true });
+  copyTree(srcDir, destDir);
+  const leaks = [];
+  const fileMap = {};
+  const renamedStems = [];
+  let n = 0;
+  // 0. remove files the policy names (change logs, advisories and the like) before anything is judged
+  if (Array.isArray(policy.removePaths) && policy.removePaths.length) {
+    const res = policy.removePaths.map(globToRegExp);
+    for (const rel of listFiles(destDir)) if (res.some((re) => re.test(rel))) fs.rmSync(path.join(destDir, rel), { force: true });
+  }
+  // 1. rename weakness-bearing file names (and note the stem so class/module references can follow)
+  for (const rel of listFiles(destDir)) {
+    const base = path.basename(rel);
+    if (!CWE_NAME.test(rel)) { fileMap[rel] = rel; continue; }
+    if (!policy.renameFiles) { fileMap[rel] = rel; leaks.push({ kind: 'weakness-in-file-name', file: rel, detail: 'the file name states the weakness class and the policy does not rename files' }); continue; }
+    const ext = path.extname(base);
+    const stem = base.slice(0, base.length - ext.length);
+    const next = `case${String(++n).padStart(4, '0')}${ext}`;
+    const relNext = path.posix.join(path.posix.dirname(rel) === '.' ? '' : path.posix.dirname(rel).replace(CWE_NAME, 'dir'), next);
+    fs.mkdirSync(path.dirname(path.join(destDir, relNext)), { recursive: true });
+    fs.renameSync(path.join(destDir, rel), path.join(destDir, relNext));
+    fileMap[rel] = relNext;
+    renamedStems.push([stem, next.slice(0, next.length - ext.length)]);
+  }
+  for (const d of fs.readdirSync(destDir, { withFileTypes: true })) if (d.isDirectory()) { try { fs.rmdirSync(path.join(destDir, d.name)); } catch { /* not empty */ } }
+  // 2. strip comments, follow renamed stems in text
+  for (const rel of listFiles(destDir)) {
+    const p = path.join(destDir, rel);
+    let buf; try { buf = fs.readFileSync(p); } catch { continue; }
+    if (!isText(buf)) continue;
+    let text = buf.toString('utf8');
+    const original = text;
+    if (policy.stripComments) text = stripComments(text, path.extname(rel).toLowerCase());
+    else {
+      const cm = text.split('\n').findIndex((l) => /(?:\/\/|\/\*|\*|#)/.test(l) && DESCRIPTIVE_COMMENT.test(l));
+      if (cm !== -1) leaks.push({ kind: 'descriptive-comment', file: rel, line: cm + 1, detail: 'a comment describes the weakness; set leakagePolicy.stripComments' });
+    }
+    for (const [from, to] of renamedStems) text = text.split(from).join(to);
+    if (CWE_CONTENT.test(text)) leaks.push({ kind: 'weakness-id-in-content', file: rel, detail: 'the content names a CWE id' });
+    if (text !== original) fs.writeFileSync(p, text);
+  }
+  // 3. the existing custody audit (answer-key names, symlinks, protected terms, advisory ids)
+  const audit = auditWorkspace(destDir, terms);
+  for (const l of audit.leaks) leaks.push(l);
+  if (leaks.length) { fs.rmSync(destDir, { recursive: true, force: true }); return { ok: false, quarantined: true, destDir: null, fileMap, leaks, audit: { scannedFiles: audit.scannedFiles, leaks: leaks.length } }; }
+  return { ok: true, quarantined: false, destDir, fileMap, leaks: [], audit: { scannedFiles: audit.scannedFiles, leaks: 0 } };
+}

package/src/posture/evaluation/lab/report.js

@@ -0,0 +1,194 @@
+// Eval Lab reports (PRD 6): quality + proof + remediation + cost + latency + repeatability for each arm, and the metric the
+// product is for, intelligence per dollar. Pure: no fs, no clock.
+//
+// Rules that keep the numbers honest:
+//   * a mean over fewer than three runs is printed with "not representative"; one lucky run is never a headline;
+//   * every spread (stdev, range, 95% interval) sits beside the mean it qualifies;
+//   * a dollar figure divides by MEASURED spend only, and when any attempt's cost is unknown the figure says so instead of
+//     treating unknown as free; a deterministic arm has no model cost, so a per-dollar figure for it is `n/a`, not infinite;
+//   * a quantity that was not measured is `n/a` (null), never 0;
+//   * the leakage audit and any incompleteness appear in the header of every format.
+import { summarize, pct } from './stats.js';
+
+export const REPORT_SCHEMA = 'agentic-security/eval-lab-report';
+export const REPORT_FORMATS = Object.freeze(['json', 'csv', 'md', 'html', 'chart']);
+
+const pick = (rs, task, path) => rs.map((r) => path.reduce((o, k) => (o == null ? null : o[k]), r.scores[task] ?? null));
+
+const METRICS = {
+  detect: [['precision', ['precision']], ['recall', ['recall']], ['f1', ['f1']], ['macroF1', ['macroF1']], ['localizationAccuracy', ['localizationAccuracy']], ['completion', ['completion']]],
+  falsify: [['balancedAccuracy', ['balancedAccuracy']], ['falseRefutationRate', ['falseRefutationRate']], ['falseConfirmationRate', ['falseConfirmationRate']]],
+  exploit: [['pocGenerationSuccess', ['pocGenerationSuccess']], ['reproductionRate', ['reproductionRate']], ['confirmationRate', ['confirmationRate']], ['falseConfirmationRate', ['falseConfirmationRate']]],
+  patch: [['validPatchRate', ['validPatchRate']], ['exploitBlockedRate', ['exploitBlockedRate']], ['testPreservationRate', ['testPreservationRate']], ['regressionRate', ['regressionRate']]],
+};
+
+/** The one number a task is judged by when ranking or routing. Defined once, here. */
+export const PRIMARY = Object.freeze({ detect: 'f1', falsify: 'balancedAccuracy', exploit: 'confirmationRate', patch: 'validPatchRate' });
+
+function recallAtBudget(run, armId, repeat, budgetUsd) {
+  const mine = run.attempts.filter((a) => a.armId === armId && a.repeat === repeat && a.task === 'detect');
+  if (!mine.length) return null;
+  if (mine.some((a) => a.usage?.usd == null)) return { value: null, reason: 'some attempts have unknown cost' };
+  const positives = mine.filter((a) => a.entryKind === 'vulnerable');
+  if (!positives.length) return null;
+  const res = run.results.find((r) => r.armId === armId && r.repeat === repeat);
+  const hit = new Set((res.scores.detect.misses ?? []).map((m) => m.entryId));
+  let spent = 0; let found = 0;
+  for (const a of mine) { // attempts are in dataset order within a repeat
+    spent += a.usage.usd;
+    if (spent > budgetUsd) break;
+    if (a.entryKind === 'vulnerable' && a.status === 'completed' && !hit.has(a.entryId)) found++;
+  }
+  return { value: found / positives.length, spentUsd: spent, budgetUsd };
+}
+
+export function summarizeRun(run, { budgetUsd = 10 } = {}) {
+  const arms = run.arms.map((arm) => {
+    const rs = run.results.filter((r) => r.armId === arm.id);
+    const tasks = {};
+    for (const [task, metrics] of Object.entries(METRICS)) {
+      if (!rs.some((r) => r.scores[task])) continue;
+      tasks[task] = Object.fromEntries(metrics.map(([name, path]) => [name, summarize(pick(rs, task, path))]));
+    }
+    const usdPerRepeat = rs.map((r) => (r.economics.attemptsWithUnknownCost > 0 ? null : r.economics.usdMeasured));
+    const econ = {
+      usd: summarize(usdPerRepeat), latencyMedianMs: summarize(rs.map((r) => r.economics.latencyMs.median)), latencyP95Ms: summarize(rs.map((r) => r.economics.latencyMs.p95)),
+      inputTokens: summarize(rs.map((r) => (r.economics.tokensUnknownAttempts > 0 ? null : r.economics.inputTokens))), outputTokens: summarize(rs.map((r) => (r.economics.tokensUnknownAttempts > 0 ? null : r.economics.outputTokens))),
+      timeouts: rs.reduce((s, r) => s + r.economics.timeouts, 0), failures: rs.reduce((s, r) => s + r.economics.failures, 0), unknownCostAttempts: rs.reduce((s, r) => s + r.economics.attemptsWithUnknownCost, 0),
+    };
+    const usdTotal = usdPerRepeat.every((x) => x != null) && usdPerRepeat.length ? usdPerRepeat.reduce((a, b) => a + b, 0) : null;
+    const modelCost = arm.kind === 'model';
+    const per = (count) => (!modelCost ? { value: null, reason: 'no model cost' } : usdTotal == null ? { value: null, reason: 'cost unknown for some attempts' } : usdTotal === 0 ? { value: null, reason: 'zero measured spend' } : { value: count / usdTotal * 100, perRun: count / rs.length });
+    const sum = (task, key) => rs.reduce((s, r) => s + (r.scores[task]?.counts?.[key] ?? 0), 0);
+    const ipd = {
+      confirmedVulnerabilitiesPer100Usd: tasks.exploit ? per(sum('exploit', 'confirmedVuln')) : { value: null, reason: 'exploit task not run' },
+      verifiedFixesPer100Usd: tasks.patch ? per(sum('patch', 'validStrict')) : { value: null, reason: 'patch task not run' },
+      truePositivesPer100Usd: tasks.detect ? per(sum('detect', 'tp')) : { value: null, reason: 'detect task not run' },
+      recallAtBudget: tasks.detect ? { budgetUsd, perRepeat: rs.map((r) => recallAtBudget(run, arm.id, r.repeat, budgetUsd)), summary: summarize(rs.map((r) => recallAtBudget(run, arm.id, r.repeat, budgetUsd)?.value ?? null)) } : null,
+    };
+    const dollarsPer = {
+      truePositive: tasks.detect && modelCost && usdTotal != null && sum('detect', 'tp') > 0 ? usdTotal / sum('detect', 'tp') : null,
+      confirmedExploit: tasks.exploit && modelCost && usdTotal != null && sum('exploit', 'confirmedVuln') > 0 ? usdTotal / sum('exploit', 'confirmedVuln') : null,
+      verifiedFix: tasks.patch && modelCost && usdTotal != null && sum('patch', 'validStrict') > 0 ? usdTotal / sum('patch', 'validStrict') : null,
+    };
+    const primary = Object.fromEntries(Object.entries(tasks).map(([t, m]) => [t, m[PRIMARY[t]]]));
+    return { id: arm.id, kind: arm.kind, model: arm.model, runs: rs.length, tasks, economics: econ, intelligencePerDollar: ipd, dollarsPer, primary };
+  });
+  // PRD 6 reliability: provider variance. The SAME model id reached through different providers is not necessarily the same behaviour
+  // (quantization, serving stack, sampling defaults), so when a run has such arms the spread between them is reported on its own.
+  const byModel = new Map();
+  for (const a of arms) if (a.kind === 'model' && a.model?.id) byModel.set(a.model.id, [...(byModel.get(a.model.id) ?? []), a]);
+  const providerVariance = [];
+  for (const [modelId, group] of byModel) {
+    if (new Set(group.map((g) => g.model.provider)).size < 2) continue;
+    for (const task of new Set(group.flatMap((g) => Object.keys(g.primary)))) {
+      const vals = group.map((g) => ({ arm: g.id, provider: g.model.provider, mean: g.primary[task]?.mean ?? null })).filter((v) => v.mean != null);
+      if (vals.length < 2) continue;
+      const means = vals.map((v) => v.mean);
+      providerVariance.push({ model: modelId, task, spread: Math.max(...means) - Math.min(...means), arms: vals, representative: group.every((g) => g.primary[task]?.representative) });
+    }
+  }
+  return {
+    schema: REPORT_SCHEMA, version: 1, runId: run.runId, runDigest: run.digest, providerVariance,
+    header: { scannerCommit: run.scanner?.commit ?? null, scannerVersion: run.scanner?.version ?? null, datasetId: run.dataset.id, datasetVersion: run.dataset.version, datasetHash: run.dataset.hash, split: run.dataset.split, models: run.arms.filter((a) => a.model).map((a) => `${a.model.provider}/${a.model.id}${a.model.local ? ' (local)' : ''}`), config: run.config, machine: run.machine, startedAt: run.startedAt, finishedAt: run.finishedAt },
+    leakageAudit: run.leakageAudit, complete: run.complete, incompleteReasons: run.incompleteReasons, arms,
+    notes: arms.some((a) => Object.values(a.primary).some((s) => s && !s.representative)) ? ['At least one figure rests on fewer than three runs and is marked not representative.'] : [],
+  };
+}
+
+const fmt = (s, kind = 'pct') => {
+  if (!s || s.mean == null) return 'n/a';
+  const base = kind === 'pct' ? pct(s.mean) : kind === 'usd' ? `$${s.mean.toFixed(4)}` : kind === 'ms' ? `${Math.round(s.mean)}ms` : String(s.mean);
+  const spread = s.stdev == null ? '' : ` ±${kind === 'pct' ? pct(s.stdev) : kind === 'usd' ? `$${s.stdev.toFixed(4)}` : kind === 'ms' ? `${Math.round(s.stdev)}ms` : s.stdev.toFixed(2)}`;
+  return `${base}${spread}${s.representative ? '' : ' (not representative)'}`;
+};
+const money = (x) => (x == null ? 'n/a' : x === 0 ? '$0.00' : x < 0.01 ? `$${x.toFixed(4)}` : `$${x.toFixed(2)}`);
+
+export function toMarkdown(rep) {
+  const h = rep.header;
+  const L = [`# Eval Lab report ${rep.runId}`, '',
+    `* Scanner: ${h.scannerVersion ?? 'unknown'} @ ${h.scannerCommit ? h.scannerCommit.slice(0, 12) : 'unknown commit'}`,
+    `* Dataset: ${h.datasetId} ${h.datasetVersion} (${h.split}), hash ${String(h.datasetHash).slice(-16)}`,
+    `* Models: ${h.models.length ? h.models.join(', ') : 'none (deterministic only)'}`,
+    `* Config: tasks ${h.config.tasks.join(', ')}; repeats ${h.config.repeats}; timeout ${h.config.timeoutMs}ms; budget ${h.config.budgetUsd ?? 'none'}`,
+    `* Machine: ${h.machine.platform}/${h.machine.arch}, ${h.machine.cpus} cpus, ${h.machine.memoryGb} GB, node ${h.machine.node}`,
+    `* Run: ${h.startedAt} to ${h.finishedAt}`,
+    `* Leakage audit: ${rep.leakageAudit.status} (${rep.leakageAudit.usable}/${rep.leakageAudit.entries} entries usable; comments ${rep.leakageAudit.stripComments ? 'stripped' : 'kept'})`,
+    ...(rep.complete ? [] : [`* **INCOMPLETE**: ${rep.incompleteReasons.join('; ')}`]), ''];
+  for (const task of Object.keys(METRICS)) {
+    const present = rep.arms.filter((a) => a.tasks[task]);
+    if (!present.length) continue;
+    const cols = METRICS[task].map(([n]) => n);
+    L.push(`## ${task}`, '', `| arm | runs | ${cols.join(' | ')} |`, `|---|---|${cols.map(() => '---').join('|')}|`);
+    for (const a of present) L.push(`| ${a.id} | ${a.runs} | ${cols.map((c) => fmt(a.tasks[task][c])).join(' | ')} |`);
+    L.push('');
+  }
+  L.push('## Cost, speed and reliability', '', '| arm | spend per run | median latency | p95 latency | timeouts | failures | unknown-cost attempts |', '|---|---|---|---|---|---|---|');
+  for (const a of rep.arms) L.push(`| ${a.id} | ${a.kind === 'deterministic' ? 'no model cost' : fmt(a.economics.usd, 'usd')} | ${fmt(a.economics.latencyMedianMs, 'ms')} | ${fmt(a.economics.latencyP95Ms, 'ms')} | ${a.economics.timeouts} | ${a.economics.failures} | ${a.economics.unknownCostAttempts} |`);
+  if (rep.providerVariance?.length) {
+    L.push('', '## Provider variance', '', 'The same model id reached through different providers:', '', '| model | task | spread | arms |', '|---|---|---|---|');
+    for (const v of rep.providerVariance) L.push(`| ${v.model} | ${v.task} | ${pct(v.spread)}${v.representative ? '' : ' (not representative)'} | ${v.arms.map((a) => `${a.arm} (${a.provider}) ${pct(a.mean)}`).join('; ')} |`);
+  }
+  L.push('', '## Intelligence per dollar', '', '| arm | confirmed vulnerabilities / $100 | verified fixes / $100 | true positives / $100 | recall at fixed budget |', '|---|---|---|---|---|');
+  for (const a of rep.arms) {
+    const i = a.intelligencePerDollar; const cell = (x) => (x?.value == null ? `n/a (${x?.reason ?? 'not measured'})` : x.value.toFixed(1));
+    const rb = i.recallAtBudget; L.push(`| ${a.id} | ${cell(i.confirmedVulnerabilitiesPer100Usd)} | ${cell(i.verifiedFixesPer100Usd)} | ${cell(i.truePositivesPer100Usd)} | ${rb ? `${fmt(rb.summary)} at $${rb.budgetUsd}` : 'n/a'} |`);
+  }
+  L.push('', '## Dollars per outcome', '', '| arm | per true positive | per confirmed exploit | per verified fix |', '|---|---|---|---|');
+  for (const a of rep.arms) L.push(`| ${a.id} | ${money(a.dollarsPer.truePositive)} | ${money(a.dollarsPer.confirmedExploit)} | ${money(a.dollarsPer.verifiedFix)} |`);
+  if (rep.notes.length) L.push('', ...rep.notes.map((n) => `> ${n}`));
+  return `${L.join('\n')}\n`;
+}
+
+function toCsv(rep) {
+  const q = (v) => { const s = v == null ? '' : String(v); return /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; };
+  const rows = [['arm', 'kind', 'task', 'metric', 'n', 'mean', 'stdev', 'min', 'max', 'ci95_low', 'ci95_high', 'representative']];
+  for (const a of rep.arms) {
+    for (const [task, ms] of Object.entries(a.tasks)) for (const [name, s] of Object.entries(ms)) rows.push([a.id, a.kind, task, name, s.n, s.mean, s.stdev, s.min, s.max, s.ci95?.low, s.ci95?.high, s.representative]);
+    for (const [name, s] of [['usd', a.economics.usd], ['latencyMedianMs', a.economics.latencyMedianMs], ['latencyP95Ms', a.economics.latencyP95Ms]]) rows.push([a.id, a.kind, 'economics', name, s.n, s.mean, s.stdev, s.min, s.max, s.ci95?.low, s.ci95?.high, s.representative]);
+  }
+  return `${rows.map((r) => r.map(q).join(',')).join('\n')}\n`;
+}
+
+const esc = (s) => String(s ?? '').replace(/[&<>"']/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
+export function toHtml(rep) {
+  const md = toMarkdown(rep).split('\n');
+  const out = ['<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Eval Lab report</title><style>body{font:14px/1.5 system-ui,sans-serif;max-width:60rem;margin:2rem auto;padding:0 1rem}table{border-collapse:collapse;margin:.5rem 0 1.5rem}td,th{border:1px solid #8884;padding:.25rem .6rem;text-align:left}h1,h2{margin-top:1.5rem}blockquote{border-left:3px solid #c80;margin:1rem 0;padding:.1rem 1rem}</style></head><body>'];
+  let table = null; let list = false;
+  const closeList = () => { if (list) { out.push('</ul>'); list = false; } };
+  for (const line of md) {
+    if (line.startsWith('|')) {
+      closeList();
+      const cells = line.split('|').slice(1, -1).map((c) => c.trim());
+      if (cells.every((c) => /^-+$/.test(c))) continue;
+      if (!table) { table = true; out.push(`<table><thead><tr>${cells.map((c) => `<th>${esc(c)}</th>`).join('')}</tr></thead><tbody>`); } else out.push(`<tr>${cells.map((c) => `<td>${esc(c)}</td>`).join('')}</tr>`);
+      continue;
+    }
+    if (table) { out.push('</tbody></table>'); table = null; }
+    if (line.startsWith('# ')) { closeList(); out.push(`<h1>${esc(line.slice(2))}</h1>`); } else if (line.startsWith('## ')) { closeList(); out.push(`<h2>${esc(line.slice(3))}</h2>`); } else if (line.startsWith('* ')) { if (!list) { out.push('<ul>'); list = true; } out.push(`<li>${esc(line.slice(2).replace(/\*\*/g, ''))}</li>`); } else if (line.startsWith('> ')) { closeList(); out.push(`<blockquote>${esc(line.slice(2))}</blockquote>`); } else if (line.trim()) { closeList(); out.push(`<p>${esc(line)}</p>`); }
+  }
+  closeList(); if (table) out.push('</tbody></table>');
+  out.push('</body></html>');
+  return `${out.join('\n')}\n`;
+}
+
+/** Chart-ready series: one point per arm and metric, with the error bar the repeats support. */
+export function toChartData(rep) {
+  const series = [];
+  for (const a of rep.arms) for (const [task, ms] of Object.entries(a.tasks)) for (const [metric, s] of Object.entries(ms)) {
+    if (s.mean == null) continue;
+    series.push({ arm: a.id, task, metric, value: s.mean, low: s.ci95?.low ?? null, high: s.ci95?.high ?? null, n: s.n, representative: s.representative });
+  }
+  const scatter = rep.arms.filter((a) => a.economics.usd.mean != null).flatMap((a) => Object.entries(a.primary).filter(([, s]) => s?.mean != null).map(([task, s]) => ({ arm: a.id, task, costUsd: a.economics.usd.mean, quality: s.mean, representative: s.representative })));
+  return { schema: 'agentic-security/eval-lab-chart-data', version: 1, runId: rep.runId, series, qualityVsCost: scatter };
+}
+
+export function renderReport(rep, format) {
+  if (format === 'json') return `${JSON.stringify(rep, null, 2)}\n`;
+  if (format === 'csv') return toCsv(rep);
+  if (format === 'md') return toMarkdown(rep);
+  if (format === 'html') return toHtml(rep);
+  if (format === 'chart') return `${JSON.stringify(toChartData(rep), null, 2)}\n`;
+  throw new Error(`unknown report format: ${format}`);
+}
+

package/src/posture/evaluation/lab/run.js

@@ -0,0 +1,225 @@
+// The Eval Lab run (PRD 6): arms x repeats x tasks x dataset entries, with the leakage controls in front and the scores behind.
+//
+//   prepare   materialize each entry's workspace, neutralize it, audit it (custody) -> staged dir, or quarantined and excluded
+//   execute   one attempt per (arm, repeat, task, entry), bounded concurrency, per-attempt timeout, cancellable
+//   score     detection, falsification, exploit proof and remediation, each recomputed from what the lab itself observed
+//
+// Mechanical evidence decides (the shared engineering contract): an arm's claim that its PoC works or its patch holds is never
+// taken. The lab executes the arm's reproducer in the proof fabric and replays the original exploit against the arm's patch.
+// An attempt that does not complete is recorded as such and never dropped, so a model cannot improve its score by failing.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { digestOf } from '../../assurance/identity.js';
+import { protectedTermsFrom } from '../custody.js';
+import { superviseSpawn } from '../../../sandbox/supervise.js';
+import { scrubEnv } from '../../../sandbox/trust-domains.js';
+import { proveFinding } from '../../proof-fabric/orchestrator.js';
+import { verifyFixWithProof } from '../../proof-fabric/fix-proof.js';
+import { validateReproducer } from '../../proof-fabric/reproducer.js';
+import { materializeEntry } from './materialize.js';
+import { stageNeutralized } from './neutralize.js';
+import { validateArm, runArmTask, TASKS } from './arms.js';
+import { resolveInside, normalizeCwe } from './dataset.js';
+import { scoreDetect, scoreFalsify, scoreExploit, scorePatch, scoreEconomics } from './score.js';
+
+export const RUN_SCHEMA = 'agentic-security/eval-lab-run';
+const RUN_VERSION = 1;
+export const DEFAULT_REPEATS = 3;
+const require = createRequire(import.meta.url);
+
+const hasTarget = (e) => !!(e.kind === 'vulnerable' ? e.expectedSpan : e.lookalikeSpan);
+const targetOf = (e) => (e.kind === 'vulnerable' ? { span: e.expectedSpan, cwe: normalizeCwe(e.cwe) } : { span: e.lookalikeSpan, cwe: normalizeCwe(e.resemblesCwe ?? e.cwe) });
+
+function machineProfile() {
+  const cpus = os.cpus();
+  return { platform: os.platform(), arch: os.arch(), cpus: cpus.length, cpuModel: cpus[0]?.model ?? null, memoryGb: Math.round(os.totalmem() / 2 ** 30), node: process.version };
+}
+function scannerIdentity() {
+  let version = null; let commit = null;
+  try { version = require('../../../../package.json').version; } catch { /* bundled: unknown */ }
+  try {
+    const r = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: path.dirname(new URL(import.meta.url).pathname), encoding: 'utf8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'] });
+    if (r.status === 0) commit = r.stdout.trim() || null;
+  } catch { /* not a checkout */ }
+  return { version, commit };
+}
+
+async function pool(jobs, limit, run, signal) {
+  let next = 0;
+  const workers = Array.from({ length: Math.max(1, Math.min(limit, jobs.length || 1)) }, async () => {
+    while (next < jobs.length && !signal?.aborted) { const j = jobs[next++]; await run(j); }
+  });
+  await Promise.all(workers);
+}
+
+/** Stage every entry's workspace. Returns the prepared map and the leakage-audit record for the run. */
+export function prepareWorkspaces({ manifest, baseDir, workRoot }) {
+  const policy = manifest.dataset.leakagePolicy;
+  const terms = protectedTermsFrom({ advisoryText: policy.protectedTerms ?? [] });
+  const prepared = new Map(); const quarantined = []; const failed = [];
+  let filesAudited = 0;
+  for (const e of manifest.entries) {
+    const raw = path.join(workRoot, `raw-${prepared.size + quarantined.length + failed.length}`);
+    const m = materializeEntry(e, { baseDir, destDir: raw });
+    if (!m.ok) { failed.push({ entryId: e.id, reason: m.reason }); fs.rmSync(raw, { recursive: true, force: true }); continue; }
+    const dest = path.join(workRoot, `w${prepared.size + quarantined.length + failed.length}`);
+    const s = stageNeutralized({ srcDir: raw, destDir: dest, policy, terms });
+    fs.rmSync(raw, { recursive: true, force: true });
+    filesAudited += s.audit.scannedFiles;
+    if (!s.ok) { quarantined.push({ entryId: e.id, leaks: s.leaks.slice(0, 5).map((l) => ({ kind: l.kind, file: l.file ?? null })) }); continue; }
+    prepared.set(e.id, { dir: dest, fileMap: s.fileMap });
+  }
+  const status = failed.length || quarantined.length ? (prepared.size ? 'entries-excluded' : 'no-entries-usable') : 'clean';
+  return { prepared, terms, audit: { status, mode: 'enforce', stripComments: !!policy.stripComments, renameFiles: !!policy.renameFiles, entries: manifest.entries.length, usable: prepared.size, quarantined, unavailable: failed, filesAudited, termCount: terms.length, scope: 'workspace contents and every request the lab sends an arm; an arm\'s own internal prompts are not visible to the lab' } };
+}
+
+async function testsPass({ argv, dir, files, timeoutMs, signal }) {
+  if (!argv) return null;
+  const copy = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-test-'));
+  try {
+    fs.cpSync(dir, copy, { recursive: true, dereference: false });
+    for (const [rel, content] of Object.entries(files)) {
+      const t = path.resolve(copy, rel);
+      if (!t.startsWith(copy + path.sep)) return null;
+      fs.mkdirSync(path.dirname(t), { recursive: true }); fs.writeFileSync(t, content);
+    }
+    const r = await superviseSpawn(argv[0], argv.slice(1), { cwd: copy, env: scrubEnv(process.env), timeoutMs, signal });
+    if (r.spawnError || r.timedOut || r.cancelled) return null;
+    return r.exitCode === 0;
+  } finally { fs.rmSync(copy, { recursive: true, force: true }); }
+}
+
+/**
+ * @param {object} o
+ * @param {{manifest: object, baseDir: string, hash: string}} o.dataset
+ * @param {object[]} o.arms
+ * @param {string[]} [o.tasks]
+ * @param {number} [o.repeats]
+ */
+export async function runEval({ dataset, arms, tasks = ['detect'], repeats = DEFAULT_REPEATS, concurrency = 2, timeoutMs = 120_000, budgetUsd = null, scanRoot = process.cwd(), signal, onProgress = () => {}, deps = {} } = {}) {
+  const now = deps.now ?? Date.now;
+  const startedAt = new Date(now()).toISOString();
+  const errors = [];
+  if (!Array.isArray(arms) || !arms.length) errors.push('at least one arm is required');
+  const ids = new Set();
+  for (const a of arms ?? []) { const v = validateArm(a); if (!v.ok) errors.push(`arm ${a?.id ?? '?'}: ${v.errors.join('; ')}`); if (ids.has(a?.id)) errors.push(`arm id ${a.id} is duplicated`); ids.add(a?.id); }
+  if (!tasks.length || !tasks.every((t) => TASKS.includes(t))) errors.push(`tasks must be some of ${TASKS.join(', ')}`);
+  if (!(Number.isInteger(repeats) && repeats >= 1 && repeats <= 50)) errors.push('repeats must be an integer from 1 to 50');
+  if (errors.length) return { ok: false, errors };
+
+  const { manifest, baseDir } = dataset;
+  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-lab-'));
+  try {
+    const prep = prepareWorkspaces({ manifest, baseDir, workRoot });
+    const entriesById = new Map(manifest.entries.map((e) => [e.id, e]));
+    const usable = manifest.entries.filter((e) => prep.prepared.has(e.id));
+    const attempts = [];
+    const spend = { usd: 0 };
+    const armPlans = arms.map((a) => ({ arm: a, repeats: a.kind === 'deterministic' ? 1 : repeats, tasks: tasks.filter((t) => a.tasks.includes(t)) }));
+    const jobs = [];
+    for (const plan of armPlans) for (let r = 0; r < plan.repeats; r++) for (const t of plan.tasks) {
+      for (const e of usable) {
+        if (t !== 'detect' && !hasTarget(e)) continue;
+        if (t === 'patch' && !(e.kind === 'vulnerable' && e.reproducer)) continue;
+        jobs.push({ arm: plan.arm, repeat: r, task: t, entry: e });
+      }
+    }
+    let completedJobs = 0;
+    const armLimit = new Map();
+    const take = async (arm) => { const cap = arm.maxConcurrency ?? (arm.kind === 'model' ? 1 : concurrency); const st = armLimit.get(arm.id) ?? { n: 0, q: [] }; armLimit.set(arm.id, st); if (st.n >= cap) await new Promise((res) => st.q.push(res)); st.n++; };
+    const give = (arm) => { const st = armLimit.get(arm.id); st.n--; st.q.shift()?.(); };
+
+    await pool(jobs, concurrency, async (job) => {
+      const { arm, task, entry } = job;
+      const rec = { armId: arm.id, repeat: job.repeat, task, entryId: entry.id, entryKind: entry.kind };
+      if (budgetUsd != null && spend.usd >= budgetUsd) { attempts.push({ ...rec, status: 'skipped-budget', reason: `the ${budgetUsd} USD budget was reached`, usage: { usd: null, latencyMs: 0 } }); return; }
+      await take(arm);
+      try {
+        const ws = prep.prepared.get(entry.id);
+        const t = targetOf(entry);
+        const mappedFile = t.span ? (ws.fileMap[t.span.file] ?? t.span.file) : null;
+        const base = { language: manifest.dataset.language, workspace: ws.dir };
+        let request = { schema: 'agentic-security/eval-request', task, ...base };
+        if (task === 'falsify') request = { ...request, candidates: [{ id: 'c1', file: mappedFile, line: t.span.startLine, cwe: t.cwe }] };
+        if (task === 'exploit' || task === 'patch') request = { ...request, target: { file: mappedFile, line: t.span.startLine, endLine: t.span.endLine, cwe: t.cwe } };
+        const res = await (deps.runArmTask ?? runArmTask)({ arm, task, request, workspaceDir: ws.dir, terms: prep.terms, scanRoot, signal, timeoutMs, egress: deps.egress, scanFn: deps.scanFn });
+        if (res.usage?.usd != null) spend.usd += res.usage.usd;
+        const out = { ...rec, status: res.status, reason: res.reason ?? null, usage: res.usage };
+        if (res.status === 'completed') {
+          if (task === 'detect') out.output = { findings: res.output.findings.slice(0, 200) };
+          if (task === 'falsify') out.output = { verdict: res.output.verdicts?.c1 ?? 'uncertain' };
+          if (task === 'exploit') {
+            const rp = res.output.reproducer;
+            const valid = rp && validateReproducer(rp).length === 0;
+            out.proof = { reproducerProvided: !!rp, reproducerValid: !!valid, executed: false, state: null, successes: null, runs: null };
+            if (valid) {
+              const proved = await (deps.proveFn ?? proveFinding)({ finding: { id: `eval-${entry.id}`, stableId: `eval-${entry.id}`, parser: 'IR-TAINT' }, reproducer: { ...rp, findingId: `eval-${entry.id}` }, sourceRoot: ws.dir, policy: { timeoutMs: Math.min(timeoutMs, 60_000) }, signal });
+              const ran = proved.attempts && proved.attempts.run > 0 && proved.environment?.mode !== 'unavailable';
+              out.proof = { ...out.proof, executed: !!ran, state: proved.state, successes: proved.attempts?.successes ?? null, runs: proved.attempts?.run ?? null, degraded: (proved.degradation ?? []).slice(0, 5) };
+            }
+          }
+          if (task === 'patch') {
+            const files = res.output.files;
+            const rpFile = resolveInside(baseDir, entry.reproducer);
+            out.patch = { applicable: true, patchProvided: !!files && Object.keys(files).length > 0, exploitBlocked: null, testsPass: null, completeness: null, state: null, bytes: files ? Object.values(files).reduce((s, v) => s + v.length, 0) : 0, digest: files ? digestOf(files) : null };
+            if (out.patch.patchProvided && rpFile) {
+              let rp = JSON.parse(fs.readFileSync(rpFile, 'utf8'));
+              if (rp.sources) rp = { ...rp, sources: Object.fromEntries(Object.entries(rp.sources).map(([d, s]) => [d, ws.fileMap[s] ?? s])) };
+              const fp = await (deps.verifyPatchFn ?? verifyFixWithProof)({ finding: { id: `eval-${entry.id}`, stableId: `eval-${entry.id}` }, reproducer: { ...rp, findingId: `eval-${entry.id}` }, files, scanRoot: ws.dir, policy: { timeoutMs: Math.min(timeoutMs, 60_000) }, staticVerification: null, signal });
+              const pre = fp.legs?.prePatch?.state === 'EXPLOIT_CONFIRMED';
+              const post = fp.legs?.postPatch;
+              out.patch.state = fp.state; out.patch.completeness = fp.completeness ?? null;
+              out.patch.exploitBlocked = pre && post && post.attempts > 0 && post.completed === post.attempts ? post.exploitSucceeded === 0 : null;
+              out.patch.testsPass = await testsPass({ argv: entry.test ?? manifest.dataset.test ?? null, dir: ws.dir, files, timeoutMs: Math.min(timeoutMs, 120_000), signal });
+            }
+          }
+        }
+        attempts.push(out);
+      } finally { give(arm); completedJobs++; onProgress({ done: completedJobs, total: jobs.length }); }
+    }, signal);
+
+    // ---- scoring: per arm, per repeat, per task
+    const mapFileFor = (entryId, f) => prep.prepared.get(entryId)?.fileMap[f] ?? f;
+    const results = [];
+    for (const plan of armPlans) {
+      for (let r = 0; r < plan.repeats; r++) {
+        const mine = attempts.filter((a) => a.armId === plan.arm.id && a.repeat === r);
+        const scores = {};
+        for (const t of plan.tasks) {
+          const ta = mine.filter((a) => a.task === t);
+          if (t === 'detect') scores.detect = scoreDetect({ entries: usable, attempts: new Map(ta.map((a) => [a.entryId, a])), mapFile: mapFileFor, targetCwes: manifest.dataset.cwe });
+          if (t === 'falsify') {
+            const truth = new Map(); const verdicts = new Map();
+            for (const a of ta) { const e = entriesById.get(a.entryId); truth.set(a.entryId, e.kind === 'vulnerable' ? 'vulnerable' : 'not-vulnerable'); verdicts.set(a.entryId, a.status === 'completed' ? a.output?.verdict ?? 'uncertain' : null); }
+            scores.falsify = scoreFalsify({ truth, verdicts });
+          }
+          if (t === 'exploit') scores.exploit = scoreExploit({ proofs: ta.map((a) => ({ entryKind: a.entryKind, reproducerValid: !!a.proof?.reproducerValid, executed: !!a.proof?.executed, state: a.proof?.state ?? null, successes: a.proof?.successes ?? null, runs: a.proof?.runs ?? null })) });
+          if (t === 'patch') scores.patch = scorePatch({ patches: ta.map((a) => a.patch ?? { applicable: true, patchProvided: false, exploitBlocked: null, testsPass: null, completeness: null }) });
+        }
+        results.push({ armId: plan.arm.id, repeat: r, scores, economics: scoreEconomics(mine) });
+      }
+    }
+    const finishedAt = new Date(now()).toISOString();
+    const config = { tasks, repeats, concurrency, timeoutMs, budgetUsd, lineTolerance: 3 };
+    const armsRecord = arms.map((a) => ({ id: a.id, kind: a.kind, tasks: a.tasks, model: a.model ? { id: a.model.id, provider: a.model.provider, local: a.model.local, version: a.model.version ?? null } : null, adapter: a.adapter.type, repeats: armPlans.find((p) => p.arm.id === a.id).repeats, configDigest: digestOf({ ...a, adapter: { ...a.adapter } }) }));
+    const run = {
+      schema: RUN_SCHEMA, version: RUN_VERSION,
+      runId: `elab-${digestOf({ d: dataset.hash, a: armsRecord, config, startedAt }).slice(-14)}`,
+      startedAt, finishedAt, scanner: scannerIdentity(), machine: machineProfile(),
+      dataset: { id: manifest.dataset.id, version: manifest.dataset.version, hash: dataset.hash, split: manifest.dataset.split, language: manifest.dataset.language, license: manifest.dataset.license, entries: manifest.entries.length },
+      config, arms: armsRecord, leakageAudit: prep.audit, attempts, results,
+      complete: jobs.length === attempts.length && !attempts.some((a) => a.status === 'skipped-budget') && prep.audit.status === 'clean',
+      incompleteReasons: [
+        ...(prep.audit.status !== 'clean' ? [`leakage audit: ${prep.audit.status} (${prep.audit.quarantined.length} quarantined, ${prep.audit.unavailable.length} unavailable)`] : []),
+        ...(attempts.some((a) => a.status === 'skipped-budget') ? ['budget reached; later attempts were skipped'] : []),
+        ...(signal?.aborted ? ['cancelled'] : []),
+      ],
+      spendUsdMeasured: spend.usd,
+    };
+    run.digest = digestOf({ ...run, digest: undefined });
+    return { ok: true, run };
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+}