@clear-capabilities/agentic-security-scanner 0.174.0 → 0.176.0

package/src/posture/evaluation/lab/arm-ollama.js

@@ -0,0 +1,88 @@
+// A built-in model arm for a LOCAL Ollama server (PRD 6: "model adapters"). It reuses the repository's own Ollama provider
+// (`llm-validator/ollama-provider.js`): loopback is enforced there, a failure is a typed error, and nothing here ever falls back to
+// another provider. The lab adds what an evaluation needs on top: the request is built from the already neutralised, already audited
+// workspace, passed through the leakage guard, and the model's JSON answer goes through the same validation as a command arm's.
+//
+// Only loopback hosts are accepted by this adapter. A remote or cloud model is reached through the `command` adapter, where the
+// egress policy decides before anything leaves the machine.
+//
+// What this proves and does not: against a stand-in Ollama server (test/fixtures/eval-lab/fake-ollama.mjs) it proves the request shape,
+// the leakage guard, the timeout and error mapping, the answer validation and the usage accounting. It says nothing about how any
+// real model performs; that is what running the lab with a real model measures.
+import fs from 'node:fs';
+import path from 'node:path';
+import { callOllamaChat, DEFAULT_OLLAMA_HOST } from '../../../llm-validator/ollama-provider.js';
+import { isLoopbackUrl } from '../../../llm-validator/local-endpoint.js';
+import { guardPrompt } from '../custody.js';
+
+const MAX_FILES = 12;
+const MAX_FILE_BYTES = 12 * 1024;
+const MAX_PROMPT_BYTES = 96 * 1024;
+const CODE_EXT = /\.(?:js|mjs|cjs|jsx|ts|tsx|py|java|go|rb|php|c|h|cc|cpp|cs|rs|kt|swift|sh)$/i;
+
+function listSources(dir, limit = 400) {
+  const out = []; const stack = [''];
+  while (stack.length && out.length < limit) {
+    const rel = stack.pop();
+    let ents; try { ents = fs.readdirSync(path.join(dir, rel), { withFileTypes: true }); } catch { continue; }
+    for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+      if (e.isSymbolicLink() || e.name.startsWith('.') || e.name === 'node_modules') continue;
+      const r = rel ? `${rel}/${e.name}` : e.name;
+      if (e.isDirectory()) stack.push(r); else if (e.isFile() && CODE_EXT.test(e.name)) out.push(r);
+    }
+  }
+  return out.sort();
+}
+
+/** Numbered source for the files a task needs: the target file whole (bounded) for single-target tasks, else the first few files. */
+function sourceBlock(workspaceDir, wanted) {
+  const files = wanted?.length ? wanted : listSources(workspaceDir).slice(0, MAX_FILES);
+  const parts = [];
+  for (const f of files.slice(0, MAX_FILES)) {
+    const abs = path.resolve(workspaceDir, f);
+    if (!abs.startsWith(path.resolve(workspaceDir) + path.sep)) continue;
+    let text; try { text = fs.readFileSync(abs, 'utf8'); } catch { continue; }
+    const clipped = text.length > MAX_FILE_BYTES ? `${text.slice(0, MAX_FILE_BYTES)}\n/* truncated */` : text;
+    parts.push(`### ${f}\n${clipped.split('\n').map((l, i) => `${String(i + 1).padStart(4)}| ${l}`).join('\n')}`);
+  }
+  return parts.join('\n\n');
+}
+
+const SCHEMAS = {
+  detect: { type: 'object', required: ['findings'], properties: { findings: { type: 'array', items: { type: 'object', required: ['file', 'line', 'cwe'], properties: { file: { type: 'string' }, line: { type: 'integer' }, cwe: { type: 'string' }, confidence: { type: 'number' } } } } } },
+  falsify: { type: 'object', required: ['verdicts'], properties: { verdicts: { type: 'object', additionalProperties: { type: 'string', enum: ['confirmed', 'refuted', 'uncertain'] } } } },
+  exploit: { type: 'object', required: ['reproducer'], properties: { reproducer: { type: 'object' } } },
+  patch: { type: 'object', required: ['files'], properties: { files: { type: 'object', additionalProperties: { type: 'string' } } } },
+};
+
+function promptFor(task, request, workspaceDir) {
+  const t = request.target;
+  const code = sourceBlock(workspaceDir, t?.file ? [t.file] : request.candidates?.map((c) => c.file).filter(Boolean));
+  const head = 'You are a security analyst. Answer with ONE JSON object and nothing else. Do not explain.';
+  if (task === 'detect') return `${head}\nFind security vulnerabilities in the code below. Report each as {"file","line","cwe","confidence"} (cwe like "CWE-89"; confidence 0 to 1). Report only what you can point to a line for. If there are none, return {"findings":[]}.\n\n${code}`;
+  if (task === 'falsify') return `${head}\nFor each candidate, decide whether it is a real vulnerability: "confirmed", "refuted" or "uncertain". Return {"verdicts":{"<id>":"<verdict>"}}.\nCandidates: ${JSON.stringify(request.candidates)}\n\n${code}`;
+  if (task === 'exploit') return `${head}\nWrite a minimal reproducer for the ${t.cwe ?? 'vulnerability'} at ${t.file}:${t.line}. Return {"reproducer":{"schema":"agentic-security/reproducer","version":1,"findingId":"eval","lang":"javascript","files":{"<name>":"<source>"},"run":["{{NODE}}","<name>","{{CANARY:command}}"],"oracle":{"type":"command_execution","params":{"canary":"command"}}}}. The program must perform the dangerous action only if the vulnerability is real.\n\n${code}`;
+  return `${head}\nFix the vulnerability at ${t.file}:${t.line} (${t.cwe ?? 'unspecified'}). Return {"files":{"<path>":"<the complete new file contents>"}} containing only the files you change.\n\n${code}`;
+}
+
+/**
+ * @returns {Promise<{status: 'completed'|'timeout'|'error'|'blocked'|'unavailable', reason?: string, body?: object, reported?: object}>}
+ */
+export async function runOllamaArm({ arm, task, request, workspaceDir, terms = [], timeoutMs = 120_000, signal, callChat = callOllamaChat }) {
+  const host = String(arm.adapter.host ?? arm.model.endpoint ?? DEFAULT_OLLAMA_HOST).replace(/\/+$/, '');
+  if (!isLoopbackUrl(host)) return { status: 'blocked', reason: 'the ollama adapter only talks to a loopback server; reach a remote model through the command adapter so the egress policy decides' };
+  const prompt = promptFor(task, request, workspaceDir);
+  if (Buffer.byteLength(prompt) > MAX_PROMPT_BYTES) return { status: 'error', reason: 'the task needs more source than the adapter sends in one prompt' };
+  const guard = guardPrompt(prompt, terms);
+  if (!guard.ok) return { status: 'blocked', reason: `leakage guard: the prompt contained a protected term (${guard.leaks[0]?.kind})` };
+  if (signal?.aborted) return { status: 'error', reason: 'cancelled' };
+  const r = await Promise.race([
+    callChat({ host, model: arm.model.id, messages: [{ role: 'user', content: prompt }], maxTokens: arm.adapter.maxTokens ?? 2048, schema: SCHEMAS[task], timeouts: { connectTimeoutMs: 3000, requestTimeoutMs: arm.adapter.timeoutMs ?? timeoutMs } }),
+    new Promise((resolve) => signal?.addEventListener('abort', () => resolve({ ok: false, code: 'cancelled', reason: 'cancelled' }), { once: true })),
+  ]);
+  if (!r.ok) return { status: r.code === 'ollama-timeout' ? 'timeout' : r.code === 'ollama-unreachable' || r.code === 'ollama-model-not-installed' ? 'unavailable' : 'error', reason: `${r.code}: ${String(r.reason).slice(0, 160)}` };
+  let body;
+  try { body = JSON.parse(r.result.text); } catch { return { status: 'error', reason: 'invalid-response: the model did not return JSON', reported: r.result.usage }; }
+  if (!body || typeof body !== 'object' || Array.isArray(body)) return { status: 'error', reason: 'invalid-response: expected a JSON object', reported: r.result.usage };
+  return { status: 'ok', body: { ...body, usage: r.result.usage ?? undefined } };
+}

package/src/posture/evaluation/lab/arms.js

@@ -0,0 +1,142 @@
+// Arms: the things the Eval Lab measures (PRD 6). An arm is the deterministic scanner, or a model reached through a small
+// operator-supplied command, or a LOCAL Ollama server through the repository's own provider. There is deliberately no built-in client
+// for a remote model: provider calls in this repository go through the feature gate, the egress policy, redaction and the audit chain,
+// and the lab must not become a second path around them.
+// An arm whose model is not local is therefore checked against the egress policy BEFORE it runs, and a denial is recorded as
+// `blocked`, not as a score.
+//
+// Honesty rules enforced here:
+//   * `local` is declared, never inferred, and a "local" arm that names a non-loopback endpoint is refused as a lie;
+//   * there is no fallback from one arm to another, so a local model can never silently become a cloud call;
+//   * the command receives its request in a FILE and answers on stdout; it never reads stdin, and it runs with a scrubbed
+//     environment, a wall-clock timeout and cleanup of its whole process tree (`superviseSpawn`);
+//   * everything the lab sends is passed through the leakage guard first, and a hit blocks the call;
+//   * token and dollar usage are whatever the arm reports and are labelled `reported`; absent usage is `unknown`, never zero.
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import { superviseSpawn } from '../../../sandbox/supervise.js';
+import { scrubEnv } from '../../../sandbox/trust-domains.js';
+import { evaluateEgress } from '../../../egress/policy.js';
+import { guardPrompt } from '../custody.js';
+import { defaultScanFn } from '../runner.js';
+import { LAYER_ENV } from '../scan-child.js';
+import { runOllamaArm } from './arm-ollama.js';
+
+export const TASKS = Object.freeze(['detect', 'falsify', 'exploit', 'patch']);
+const ARM_KINDS = Object.freeze(['deterministic', 'model']);
+const ID_RE = /^[a-z0-9][a-z0-9._-]{0,63}$/;
+const MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
+const LOOPBACK = /^(?:https?:\/\/)?(?:localhost|127(?:\.\d{1,3}){3}|\[::1\])(?::\d+)?(?:\/|$)/i;
+
+export function validateArm(a) {
+  const errors = [];
+  const bad = (m) => errors.push(m);
+  if (!a || typeof a !== 'object') return { ok: false, errors: ['arm must be an object'] };
+  if (typeof a.id !== 'string' || !ID_RE.test(a.id)) bad('arm.id must be a lower-case identifier');
+  if (!ARM_KINDS.includes(a.kind)) bad(`arm.kind must be one of ${ARM_KINDS.join(', ')}`);
+  if (!Array.isArray(a.tasks) || !a.tasks.length || !a.tasks.every((t) => TASKS.includes(t))) bad(`arm.tasks must list some of ${TASKS.join(', ')}`);
+  const ad = a.adapter;
+  if (!ad || typeof ad !== 'object') bad('arm.adapter is required');
+  else if (ad.type === 'builtin-scanner') {
+    if (a.kind !== 'deterministic') bad('the builtin-scanner adapter is the deterministic arm');
+    if (a.tasks?.some((t) => t !== 'detect')) bad('the deterministic scanner only performs the detect task');
+    if (ad.layer !== undefined && !['deterministic-only', 'deep-taint'].includes(ad.layer)) bad('adapter.layer must be deterministic-only or deep-taint (a model layer is a model arm)');
+  } else if (ad.type === 'command') {
+    if (!Array.isArray(ad.argv) || !ad.argv.length || !ad.argv.every((x) => typeof x === 'string' && x.length < 2000 && !x.includes('\0'))) bad('adapter.argv must be a non-empty array of strings (no shell string)');
+    if (ad.timeoutMs !== undefined && !(Number.isInteger(ad.timeoutMs) && ad.timeoutMs >= 1000 && ad.timeoutMs <= 3_600_000)) bad('adapter.timeoutMs must be between 1000 and 3600000');
+  } else if (ad.type === 'ollama') {
+    if (a.kind !== 'model') bad('the ollama adapter is a model arm');
+    if (a.model?.local !== true) bad('the ollama adapter talks to a local server, so model.local must be true');
+    if (ad.host !== undefined && !LOOPBACK.test(String(ad.host))) bad('adapter.host must be a loopback address; use the command adapter for a remote model');
+    if (ad.timeoutMs !== undefined && !(Number.isInteger(ad.timeoutMs) && ad.timeoutMs >= 1000 && ad.timeoutMs <= 3_600_000)) bad('adapter.timeoutMs must be between 1000 and 3600000');
+  } else bad('adapter.type must be builtin-scanner, command or ollama');
+  if (a.kind === 'model') {
+    const m = a.model;
+    if (!m || typeof m.id !== 'string' || !m.id || typeof m.provider !== 'string' || !m.provider) bad('a model arm needs model.id and model.provider');
+    else {
+      if (typeof m.local !== 'boolean') bad('model.local must be true or false; it is declared, never inferred');
+      if (m.local === true && m.endpoint && !LOOPBACK.test(String(m.endpoint))) bad('model.local is true but model.endpoint is not a loopback address');
+      if (m.local === false && (typeof m.endpoint !== 'string' || !m.endpoint)) bad('a non-local model arm must name its endpoint so the egress policy can judge it');
+    }
+    if (a.kind === 'model' && ad?.type === 'builtin-scanner') bad('a model arm needs a command or ollama adapter');
+  }
+  if (a.price !== undefined) {
+    const p = a.price;
+    if (!p || !(Number.isFinite(p.inputPerMTok) && p.inputPerMTok >= 0 && Number.isFinite(p.outputPerMTok) && p.outputPerMTok >= 0)) bad('arm.price needs non-negative inputPerMTok and outputPerMTok');
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/** Dollars from reported tokens and a declared price, or the arm's own reported figure. `unknown` is null, never zero. */
+export function usageOf(arm, reported, latencyMs) {
+  const u = reported && typeof reported === 'object' ? reported : {};
+  const num = (x) => (Number.isFinite(x) && x >= 0 ? x : null);
+  const inputTokens = num(u.inputTokens); const outputTokens = num(u.outputTokens);
+  let usd = num(u.usd); let source = usd != null ? 'reported' : 'unknown';
+  if (usd == null && arm.price && inputTokens != null && outputTokens != null) {
+    usd = (inputTokens * arm.price.inputPerMTok + outputTokens * arm.price.outputPerMTok) / 1e6; source = 'computed-from-reported-tokens';
+  }
+  if (arm.kind === 'deterministic') { usd = 0; source = 'no-model'; }
+  return { inputTokens, outputTokens, usd, usdSource: source, latencyMs };
+}
+
+const normalizeFinding = (f) => ({ file: String(f.file ?? ''), line: Number.isInteger(f.line) ? f.line : null, cwe: f.cwe ?? null, confidence: typeof f.confidence === 'number' ? f.confidence : null });
+
+/** Turn an arm's parsed JSON answer into the task's output, or an `error` result. Shared by every adapter, so a model gets no softer validation than a command. */
+function shapeBody(task, body, done) {
+  const out = {};
+  if (task === 'detect') { if (!Array.isArray(body.findings)) return done('error', { reason: 'invalid-response: detect needs "findings"', reported: body.usage }); out.findings = body.findings.slice(0, 5000).filter((f) => f && typeof f === 'object').map(normalizeFinding); }
+  else if (task === 'falsify') { if (!body.verdicts || typeof body.verdicts !== 'object') return done('error', { reason: 'invalid-response: falsify needs "verdicts"', reported: body.usage }); out.verdicts = Object.fromEntries(Object.entries(body.verdicts).slice(0, 5000).map(([k, v]) => [k, ['confirmed', 'refuted', 'uncertain'].includes(v) ? v : 'uncertain'])); }
+  else if (task === 'exploit') { if (!body.reproducer || typeof body.reproducer !== 'object') return done('completed', { output: { reproducer: null }, reported: body.usage }); out.reproducer = body.reproducer; }
+  else if (task === 'patch') { if (!body.files || typeof body.files !== 'object' || Array.isArray(body.files)) return done('completed', { output: { files: null }, reported: body.usage }); out.files = Object.fromEntries(Object.entries(body.files).slice(0, 64).filter(([k, v]) => typeof k === 'string' && typeof v === 'string' && v.length < 1_000_000)); }
+  return done('completed', { output: out, reported: body.usage });
+}
+
+/**
+ * Run one task for one arm. Never throws, never returns a score: it returns what happened.
+ * @returns {{status: 'completed'|'timeout'|'error'|'blocked'|'unavailable', reason?: string, output?: object, usage: object}}
+ */
+export async function runArmTask({ arm, task, request, workspaceDir, terms = [], scanRoot, signal, timeoutMs = 120_000, egress = evaluateEgress, scanFn = defaultScanFn }) {
+  const t0 = Date.now();
+  const done = (status, extra = {}) => ({ status, usage: usageOf(arm, extra.reported, Date.now() - t0), ...extra, reported: undefined });
+  if (!arm.tasks.includes(task)) return done('unavailable', { reason: `arm does not perform ${task}` });
+  if (arm.adapter.type === 'builtin-scanner') {
+    const layer = arm.adapter.layer ?? 'deep-taint';
+    try {
+      const res = await scanFn(workspaceDir, { layer, timeoutMs, env: { ...scrubEnv(process.env), ...LAYER_ENV[layer] } });
+      return done('completed', { output: { findings: (res.findings || []).map(normalizeFinding) } });
+    } catch (e) {
+      return done(e.code === 'TIMEOUT' ? 'timeout' : e.code === 'UNAVAILABLE' ? 'unavailable' : 'error', { reason: String(e.message).slice(0, 200) });
+    }
+  }
+  // model arm through the command or ollama adapter
+  if (arm.model.local === false) {
+    const d = egress({ scanRoot, purpose: 'eval-lab', endpoint: arm.model.endpoint, model: arm.model.id, provider: arm.model.provider, role: `eval-${task}`, dataClass: 'source-code' });
+    if (!d.allowed) return done('blocked', { reason: `egress policy: ${d.reason}` });
+  }
+  if (arm.adapter.type === 'ollama') {
+    const o = await runOllamaArm({ arm, task, request, workspaceDir, terms, timeoutMs, signal });
+    if (o.status !== 'ok') return done(o.status, { reason: o.reason, reported: o.reported });
+    return shapeBody(task, o.body, done);
+  }
+  const guard = guardPrompt(JSON.stringify(request), terms);
+  if (!guard.ok) return done('blocked', { reason: `leakage guard: the request contained a protected term (${guard.leaks[0]?.kind})` });
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'eval-arm-'));
+  try {
+    const reqFile = path.join(dir, 'request.json');
+    fs.writeFileSync(reqFile, JSON.stringify(request));
+    const argv = arm.adapter.argv.map((x) => x.split('{{REQUEST}}').join(reqFile));
+    const env = { ...scrubEnv(process.env), AGENTIC_EVAL_REQUEST: reqFile, AGENTIC_EVAL_TASK: task };
+    const r = await superviseSpawn(argv[0], argv.slice(1), { cwd: dir, env, timeoutMs: arm.adapter.timeoutMs ?? timeoutMs, signal, maxOutputBytes: MAX_RESPONSE_BYTES });
+    if (r.spawnError) return done('error', { reason: `could not start the arm: ${r.spawnError}` });
+    if (r.timedOut) return done('timeout', { reason: 'the arm did not answer within its time limit' });
+    if (r.cancelled) return done('error', { reason: 'cancelled' });
+    if (r.outputCapped) return done('error', { reason: 'the arm answered with more output than allowed' });
+    if (r.exitCode !== 0) return done('error', { reason: `the arm exited ${r.exitCode}${r.exitSignal ? ` (${r.exitSignal})` : ''}` });
+    let body;
+    try { body = JSON.parse(r.stdout); } catch { return done('error', { reason: 'invalid-response: the arm did not print JSON' }); }
+    if (!body || typeof body !== 'object' || Array.isArray(body)) return done('error', { reason: 'invalid-response: expected a JSON object' });
+    return shapeBody(task, body, done);
+  } finally { fs.rmSync(dir, { recursive: true, force: true }); }
+}

package/src/posture/evaluation/lab/cli.js

@@ -0,0 +1,135 @@
+// CLI for the Eval Lab and the model router: `eval run|compare|report|list`, `models benchmark`, `models route explain`.
+// Non-interactive (nothing reads stdin), bounded (every attempt has a timeout, the run is cancellable with SIGINT), and local:
+// the lab itself makes no network call. A model arm that is not local needs `--allow-external`, and even then the egress policy
+// is consulted for it before it runs.
+//
+// Exit codes: 0 done, 1 done but incomplete or no recommendation, 4 usage or precondition.
+import fs from 'node:fs';
+import path from 'node:path';
+import { load as loadYaml } from '../../../util/yaml.js';
+import { loadDatasetFile } from './dataset.js';
+import { runEval, DEFAULT_REPEATS } from './run.js';
+import { validateArm, TASKS } from './arms.js';
+import { summarizeRun, renderReport, REPORT_FORMATS, PRIMARY } from './report.js';
+import { saveRun, loadRun, listRuns, saveScoreboard, loadScoreboard } from './store.js';
+import { scoreboardFromRun, mergeScoreboard } from './scoreboard.js';
+import { routeTask, explainRoute, ROUTER_TASKS, ROUTER_POLICIES } from '../../routing/policy-router.js';
+
+const MAX_ARMS_BYTES = 1024 * 1024;
+
+function readArms(file) {
+  let doc;
+  try {
+    const st = fs.statSync(file);
+    if (!st.isFile() || st.size > MAX_ARMS_BYTES) return { ok: false, errors: [`${file} is not a regular file under ${MAX_ARMS_BYTES} bytes`] };
+    const text = fs.readFileSync(file, 'utf8');
+    doc = /\.json$/i.test(file) ? JSON.parse(text) : loadYaml(text);
+  } catch (e) { return { ok: false, errors: [`cannot read ${file}: ${String(e.message).split('\n')[0]}`] }; }
+  const arms = Array.isArray(doc) ? doc : doc?.arms;
+  if (!Array.isArray(arms) || !arms.length) return { ok: false, errors: ['the arms file needs a list of arms (or { "arms": [...] })'] };
+  const errors = arms.flatMap((a, i) => { const v = validateArm(a); return v.ok ? [] : v.errors.map((e) => `arms[${i}]: ${e}`); });
+  return errors.length ? { ok: false, errors } : { ok: true, arms };
+}
+
+const usageRun = () => { console.error('usage: eval run --dataset <file> --arms <file> [--tasks detect,falsify,exploit,patch] [--repeats N] [--concurrency N] [--timeout S] [--budget USD] [--allow-external]\n       eval report <run-id> [--format md|json|csv|html|chart] [--out <file>]\n       eval compare <run-id> [<other-run-id>]\n       eval list'); return 4; };
+
+async function doRun(args, root, { updateScoreboard }) {
+  const f = args.flags;
+  if (!f.dataset || !f.arms) return usageRun();
+  const ds = loadDatasetFile(path.resolve(String(f.dataset)));
+  if (!ds.ok) { console.error(`Invalid dataset:\n  ${ds.errors.join('\n  ')}`); return 4; }
+  const ar = readArms(path.resolve(String(f.arms)));
+  if (!ar.ok) { console.error(`Invalid arms:\n  ${ar.errors.join('\n  ')}`); return 4; }
+  const external = ar.arms.filter((a) => a.kind === 'model' && a.model.local === false);
+  if (external.length && !f['allow-external']) { console.error(`These arms send source code to a non-local model: ${external.map((a) => a.id).join(', ')}. Pass --allow-external to run them; the egress policy is still consulted for each.`); return 4; }
+  const tasks = f.tasks ? String(f.tasks).split(',').map((s) => s.trim()).filter(Boolean) : ['detect'];
+  if (!tasks.every((t) => TASKS.includes(t))) { console.error(`--tasks must be some of ${TASKS.join(', ')}`); return 4; }
+  const num = (v, name, lo, hi) => { if (v === undefined) return undefined; const n = Number(v); if (!Number.isFinite(n) || n < lo || n > hi) { console.error(`--${name} must be a number from ${lo} to ${hi}`); return null; } return n; };
+  const repeats = num(f.repeats, 'repeats', 1, 50); const concurrency = num(f.concurrency, 'concurrency', 1, 32); const timeoutS = num(f.timeout, 'timeout', 1, 3600); const budget = num(f.budget, 'budget', 0, 1e6);
+  if (repeats === null || concurrency === null || timeoutS === null || budget === null) return 4;
+  const ac = new AbortController(); const onSig = () => ac.abort(); process.once('SIGINT', onSig);
+  let res;
+  try {
+    res = await runEval({ dataset: ds, arms: ar.arms, tasks, repeats: repeats ?? DEFAULT_REPEATS, concurrency: concurrency ?? 2, timeoutMs: (timeoutS ?? 120) * 1000, budgetUsd: budget ?? null, scanRoot: root, signal: ac.signal });
+  } finally { process.off('SIGINT', onSig); }
+  if (!res.ok) { console.error(`Cannot run:\n  ${res.errors.join('\n  ')}`); return 4; }
+  const run = res.run;
+  const saved = saveRun(root, run);
+  let sbNote = '';
+  if (updateScoreboard) {
+    const cur = loadScoreboard(root);
+    if (cur.tampered) sbNote = 'The existing scoreboard failed its integrity check and was not updated; remove it to start again.';
+    else { const w = saveScoreboard(root, mergeScoreboard(cur.doc, scoreboardFromRun(run, ar.arms))); sbNote = w.ok ? 'Scoreboard updated.' : `Scoreboard not stored: ${w.reason}`; }
+  }
+  const rep = summarizeRun(run);
+  if (f.json) console.log(JSON.stringify({ runId: run.runId, stored: saved.ok ? saved.file : null, complete: run.complete, report: rep }, null, 2));
+  else { console.log(renderReport(rep, 'md')); console.log(saved.ok ? `Run stored: ${run.runId}` : `(run not stored: ${saved.reason})`); if (sbNote) console.log(sbNote); }
+  return run.complete ? 0 : 1;
+}
+
+export async function cmdEval(args, root) {
+  const sub = args._[1];
+  if (sub === 'run') return doRun(args, root, { updateScoreboard: false });
+  if (sub === 'list') {
+    const runs = listRuns(root);
+    if (args.flags.json) { console.log(JSON.stringify(runs, null, 2)); return 0; }
+    if (!runs.length) console.log('No eval runs yet. Start one with: eval run --dataset <file> --arms <file>');
+    for (const r of runs) console.log(`${r.runId}  ${r.startedAt}  ${r.dataset}  arms: ${r.arms.join(', ')}${r.tampered ? '  (FAILED INTEGRITY CHECK)' : ''}`);
+    return 0;
+  }
+  if (sub === 'report' || sub === 'compare') {
+    const id = String(args._[2] ?? '');
+    if (!id) return usageRun();
+    const a = loadRun(root, id);
+    if (a.missing) { console.error(`No run ${id}. List runs with: eval list`); return 4; }
+    if (a.tampered) { console.error(`Run ${id} failed its integrity check and is not used.`); return 1; }
+    if (sub === 'report') {
+      const format = String(args.flags.format ?? 'md');
+      if (!REPORT_FORMATS.includes(format)) { console.error(`--format must be one of ${REPORT_FORMATS.join(', ')}`); return 4; }
+      const text = renderReport(summarizeRun(a.doc), format);
+      if (args.flags.out) { fs.writeFileSync(path.resolve(String(args.flags.out)), text); console.log(`Wrote ${format} report to ${path.resolve(String(args.flags.out))}`); } else process.stdout.write(text);
+      return 0;
+    }
+    const other = args._[3] ? loadRun(root, String(args._[3])) : null;
+    if (other && (other.missing || other.tampered)) { console.error(`Run ${args._[3]} is ${other.missing ? 'missing' : 'not intact'}.`); return other.missing ? 4 : 1; }
+    return compare(a.doc, other?.doc ?? null, !!args.flags['allow-different-datasets'], !!args.flags.json);
+  }
+  return usageRun();
+}
+
+function compare(a, b, allowDiff, json) {
+  const ra = summarizeRun(a);
+  if (!b) {
+    const rows = ra.arms.map((arm) => ({ arm: arm.id, kind: arm.kind, runs: arm.runs, ...Object.fromEntries(Object.entries(arm.primary).map(([t, s]) => [`${t}:${PRIMARY[t]}`, s?.mean ?? null])), usdPerRun: arm.economics.usd.mean, medianLatencyMs: arm.economics.latencyMedianMs.mean, representative: Object.values(arm.primary).every((s) => s?.representative) }));
+    if (json) console.log(JSON.stringify({ runId: a.runId, rows }, null, 2)); else console.log(renderReport(ra, 'md'));
+    return 0;
+  }
+  if (a.dataset.hash !== b.dataset.hash && !allowDiff) { console.error(`These runs used different datasets (${a.dataset.id} ${String(a.dataset.hash).slice(-8)} vs ${b.dataset.id} ${String(b.dataset.hash).slice(-8)}); a comparison would not mean anything. Pass --allow-different-datasets to see the numbers anyway.`); return 1; }
+  const rb = summarizeRun(b);
+  const out = [];
+  for (const arm of ra.arms) {
+    const other = rb.arms.find((x) => x.id === arm.id);
+    if (!other) continue;
+    for (const [t, s] of Object.entries(arm.primary)) { const o = other.primary[t]; if (s?.mean == null || o?.mean == null) continue; out.push({ arm: arm.id, task: t, metric: PRIMARY[t], a: s.mean, b: o.mean, delta: o.mean - s.mean, representative: !!(s.representative && o.representative) }); }
+  }
+  if (json) console.log(JSON.stringify({ a: a.runId, b: b.runId, sameDataset: a.dataset.hash === b.dataset.hash, rows: out }, null, 2));
+  else { console.log(`${a.runId} -> ${b.runId}${a.dataset.hash === b.dataset.hash ? '' : '  (DIFFERENT DATASETS)'}`); for (const r of out) console.log(`  ${r.arm} ${r.task}/${r.metric}: ${(r.a * 100).toFixed(1)}% -> ${(r.b * 100).toFixed(1)}% (${r.delta >= 0 ? '+' : ''}${(r.delta * 100).toFixed(1)} points)${r.representative ? '' : '  not representative'}`); if (!out.length) console.log('  no arm appears in both runs'); }
+  return 0;
+}
+
+export async function cmdModelsLab(args, root) {
+  const sub = args._[1];
+  if (sub === 'benchmark') return doRun(args, root, { updateScoreboard: true });
+  if (sub === 'route') {
+    if (args._[2] !== 'explain') { console.error(`usage: models route explain <${Object.keys(ROUTER_TASKS).join('|')}> [--policy ${ROUTER_POLICIES.join('|')}] [--threshold 0.8] [--tolerance 0.02] [--dataset-hash <hash>]`); return 4; }
+    const task = String(args._[3] ?? '');
+    if (!task) { console.error(`models route explain needs a task: ${Object.keys(ROUTER_TASKS).join(', ')}`); return 4; }
+    const sb = loadScoreboard(root);
+    if (sb.tampered) { console.error('The scoreboard failed its integrity check and is not used for routing. Remove .agentic-security/eval-lab/scoreboard.json and re-run: models benchmark'); return 1; }
+    const n = (v) => (v === undefined ? undefined : Number(v));
+    const r = routeTask({ task, policy: args.flags.policy ? String(args.flags.policy) : 'balanced', scoreboard: sb.doc, qualityThreshold: n(args.flags.threshold) ?? null, ...(args.flags.tolerance !== undefined ? { tolerance: n(args.flags.tolerance) } : {}), ...(args.flags['min-runs'] !== undefined ? { minRuns: n(args.flags['min-runs']) } : {}), datasetHash: args.flags['dataset-hash'] ? String(args.flags['dataset-hash']) : null, scanRoot: root });
+    if (args.flags.json) console.log(JSON.stringify(r, null, 2)); else console.log(explainRoute(r));
+    return r.ok ? 0 : (r.blocked === 'unknown-task' || r.blocked === 'invalid-policy' ? 4 : 1);
+  }
+  return null;
+}

package/src/posture/evaluation/lab/dataset.js

@@ -0,0 +1,139 @@
+// Dataset manifest for the Eval Lab (PRD 6, requirements 1, 3 and 4).
+//
+// A dataset is the thing a model is measured on, so everything that could make a score meaningless is a validation error here,
+// not a default:
+//   * `leakagePolicy.mode` must be `enforce`. There is no `off`; a dataset cannot opt out of the leakage controls.
+//   * a real-code dataset (`git-pairs`) must name a license that permits use. An unknown license refuses the dataset.
+//   * every vulnerable entry carries the exact span the defect occupies (`expectedSpan`), so detection is scored on LOCATION, not
+//     on "a finding of the right class exists somewhere in the file".
+//   * every hard negative says what it resembles and which kind of safe code it is (sanitization, authorization, ...), because a
+//     negative that does not look like a vulnerability measures nothing.
+//
+// Labels (kind, cwe, span, category, reproducer) live in the manifest and are NEVER written into a workspace an arm can read.
+// A workspace reference is a directory inside the manifest's own folder, or a revision of a local git repository.
+//
+// Pure except `loadDatasetFile`, which reads one bounded file.
+import fs from 'node:fs';
+import path from 'node:path';
+import { load as loadYaml } from '../../../util/yaml.js';
+import { digestOf } from '../../assurance/identity.js';
+
+const DATASET_SCHEMA = 'agentic-security/eval-dataset';
+export const SPLITS = Object.freeze(['dev', 'validation', 'sealed']);
+const ENTRY_KINDS = Object.freeze(['vulnerable', 'fixed', 'hard-negative']);
+export const HARD_NEGATIVE_CATEGORIES = Object.freeze(['sanitization', 'authorization', 'parameterization', 'bounds-check', 'correct-crypto', 'safe-wrapper']);
+const SOURCE_KINDS = Object.freeze(['directory', 'git-pairs']);
+const PERMITTED_LICENSES = Object.freeze(['MIT', 'Apache-2.0', 'BSD-2-Clause', 'BSD-3-Clause', 'ISC', 'MPL-2.0', '0BSD', 'Unlicense', 'CC0-1.0', 'GPL-2.0-only', 'GPL-2.0-or-later', 'GPL-3.0-only', 'GPL-3.0-or-later', 'LGPL-2.1-only', 'LGPL-2.1-or-later', 'LGPL-3.0-only', 'LGPL-3.0-or-later', 'AGPL-3.0-only', 'AGPL-3.0-or-later']);
+const MAX_MANIFEST_BYTES = 5 * 1024 * 1024;
+const MAX_ENTRIES = 20000;
+const ID_RE = /^[a-z0-9][a-z0-9._-]{0,79}$/;
+const CWE_RE = /^CWE-\d{1,5}$/;
+const REV_RE = /^[0-9a-f]{7,64}$/i;
+
+export const normalizeCwe = (c) => { const m = /(\d{1,5})/.exec(String(c ?? '')); return m ? `CWE-${Number(m[1])}` : null; };
+
+function spanErrors(span, where) {
+  const errs = [];
+  if (!span || typeof span !== 'object') return [`${where} is required`];
+  if (typeof span.file !== 'string' || !span.file || span.file.includes('\0') || span.file.startsWith('/') || span.file.split(/[\\/]/).includes('..')) errs.push(`${where}.file must be a relative path inside the workspace`);
+  if (!Number.isInteger(span.startLine) || span.startLine < 1) errs.push(`${where}.startLine must be a positive integer`);
+  if (!Number.isInteger(span.endLine) || span.endLine < span.startLine) errs.push(`${where}.endLine must be an integer at or after startLine`);
+  return errs;
+}
+
+const argvOk = (a) => Array.isArray(a) && a.length > 0 && a.length <= 64 && a.every((x) => typeof x === 'string' && x.length < 2000 && !x.includes('\0'));
+
+/** @returns {{ok: boolean, errors: string[]}} */
+export function validateDatasetManifest(m) {
+  const errors = [];
+  const bad = (msg) => errors.push(msg);
+  if (!m || typeof m !== 'object' || Array.isArray(m)) return { ok: false, errors: ['manifest must be an object'] };
+  const d = m.dataset;
+  if (!d || typeof d !== 'object') return { ok: false, errors: ['manifest needs a "dataset" object'] };
+  if (typeof d.id !== 'string' || !ID_RE.test(d.id)) bad('dataset.id must be a lower-case identifier');
+  if (typeof d.version !== 'string' || !/^[0-9A-Za-z][0-9A-Za-z._+-]{0,39}$/.test(d.version)) bad('dataset.version is required');
+  const langs = Array.isArray(d.language) ? d.language : [d.language];
+  if (!langs.length || !langs.every((l) => typeof l === 'string' && /^[a-z][a-z0-9+#-]{0,19}$/.test(l))) bad('dataset.language must name one or more languages');
+  if (!d.source || typeof d.source !== 'object' || !SOURCE_KINDS.includes(d.source.kind)) bad(`dataset.source.kind must be one of ${SOURCE_KINDS.join(', ')}`);
+  if (!SPLITS.includes(d.split)) bad(`dataset.split must be one of ${SPLITS.join(', ')}`);
+  if (typeof d.license !== 'string' || !d.license.trim()) bad('dataset.license is required');
+  else if (d.source?.kind === 'git-pairs' && !PERMITTED_LICENSES.includes(d.license)) bad(`a real-code dataset needs a license that permits use (one of ${PERMITTED_LICENSES.slice(0, 6).join(', ')}, ...); "${d.license}" is not accepted`);
+  if (!Array.isArray(d.cwe) || !d.cwe.length || !d.cwe.every((c) => CWE_RE.test(String(c)))) bad('dataset.cwe must list CWE ids such as CWE-89');
+  const lp = d.leakagePolicy;
+  if (!lp || typeof lp !== 'object') bad('dataset.leakagePolicy is required');
+  else {
+    if (lp.mode !== 'enforce') bad('dataset.leakagePolicy.mode must be "enforce"; a dataset cannot turn the leakage controls off');
+    for (const k of ['stripComments', 'renameFiles']) if (lp[k] !== undefined && typeof lp[k] !== 'boolean') bad(`dataset.leakagePolicy.${k} must be a boolean`);
+    if (lp.removePaths !== undefined && !(Array.isArray(lp.removePaths) && lp.removePaths.every((g) => typeof g === 'string' && g.length >= 1 && g.length <= 120 && !g.includes('\0')))) bad('dataset.leakagePolicy.removePaths must be a list of simple globs');
+    if (lp.protectedTerms !== undefined && !(Array.isArray(lp.protectedTerms) && lp.protectedTerms.every((t) => typeof t === 'string' && t.length >= 4 && t.length <= 200))) bad('dataset.leakagePolicy.protectedTerms must be a list of strings of at least 4 characters');
+  }
+  if (d.build !== undefined && !argvOk(d.build)) bad('dataset.build must be an argv array');
+  if (d.test !== undefined && !argvOk(d.test)) bad('dataset.test must be an argv array');
+
+  const entries = m.entries;
+  if (!Array.isArray(entries) || !entries.length) bad('entries must be a non-empty list');
+  else if (entries.length > MAX_ENTRIES) bad(`entries is limited to ${MAX_ENTRIES}`);
+  else {
+    const seen = new Set();
+    entries.forEach((e, i) => {
+      const w = `entries[${i}]`;
+      if (!e || typeof e !== 'object') { bad(`${w} must be an object`); return; }
+      if (typeof e.id !== 'string' || !ID_RE.test(e.id)) bad(`${w}.id must be a lower-case identifier`);
+      else if (seen.has(e.id)) bad(`${w}.id "${e.id}" is duplicated`);
+      else seen.add(e.id);
+      if (!ENTRY_KINDS.includes(e.kind)) bad(`${w}.kind must be one of ${ENTRY_KINDS.join(', ')}`);
+      if (!e.workspace || typeof e.workspace !== 'object') bad(`${w}.workspace is required`);
+      else if (typeof e.workspace.dir === 'string') {
+        if (e.workspace.dir.startsWith('/') || e.workspace.dir.includes('\0') || e.workspace.dir.split(/[\\/]/).includes('..')) bad(`${w}.workspace.dir must stay inside the manifest folder`);
+      } else if (e.workspace.git && typeof e.workspace.git === 'object') {
+        if (typeof e.workspace.git.repo !== 'string' || !e.workspace.git.repo) bad(`${w}.workspace.git.repo is required`);
+        if (typeof e.workspace.git.rev !== 'string' || !REV_RE.test(e.workspace.git.rev)) bad(`${w}.workspace.git.rev must be a commit id (a moving ref such as a branch is not reproducible)`);
+      } else bad(`${w}.workspace needs "dir" or "git"`);
+      if (e.kind === 'vulnerable') {
+        if (!CWE_RE.test(String(e.cwe))) bad(`${w}.cwe is required for a vulnerable entry`);
+        errors.push(...spanErrors(e.expectedSpan, `${w}.expectedSpan`));
+        if (e.category !== undefined || e.lookalikeSpan !== undefined) bad(`${w}: category and lookalikeSpan belong to negative entries`);
+      } else if (e.kind === 'hard-negative') {
+        if (!HARD_NEGATIVE_CATEGORIES.includes(e.category)) bad(`${w}.category must be one of ${HARD_NEGATIVE_CATEGORIES.join(', ')}`);
+        if (!CWE_RE.test(String(e.resemblesCwe))) bad(`${w}.resemblesCwe must name the weakness it looks like`);
+        errors.push(...spanErrors(e.lookalikeSpan, `${w}.lookalikeSpan`));
+        if (e.expectedSpan !== undefined) bad(`${w}: a hard negative has no expectedSpan; it is not vulnerable`);
+      } else if (e.kind === 'fixed') {
+        if (!CWE_RE.test(String(e.resemblesCwe ?? e.cwe))) bad(`${w}.resemblesCwe must name the weakness that was fixed`);
+        if (e.lookalikeSpan !== undefined) errors.push(...spanErrors(e.lookalikeSpan, `${w}.lookalikeSpan`));
+      }
+      if (e.reproducer !== undefined && (typeof e.reproducer !== 'string' || e.reproducer.startsWith('/') || e.reproducer.split(/[\\/]/).includes('..'))) bad(`${w}.reproducer must be a relative path inside the manifest folder`);
+      if (e.test !== undefined && !argvOk(e.test)) bad(`${w}.test must be an argv array`);
+    });
+    if (!entries.some((e) => e?.kind === 'vulnerable')) bad('a dataset needs at least one vulnerable entry');
+    if (!entries.some((e) => e?.kind === 'hard-negative' || e?.kind === 'fixed')) bad('a dataset needs at least one negative entry (fixed or hard-negative); without one, precision cannot be measured');
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/** Content hash of everything that defines what is measured. Key order does not matter. */
+export const datasetHash = (m) => digestOf({ dataset: m.dataset, entries: m.entries });
+
+/** Read and validate a manifest (JSON or YAML), bounded. Entry paths resolve against the manifest's own folder. */
+export function loadDatasetFile(file) {
+  let st;
+  try { st = fs.statSync(file); } catch { return { ok: false, errors: [`cannot read ${file}`] }; }
+  if (!st.isFile() || st.size > MAX_MANIFEST_BYTES) return { ok: false, errors: [`${file} is not a regular file under ${MAX_MANIFEST_BYTES} bytes`] };
+  let doc;
+  try {
+    const text = fs.readFileSync(file, 'utf8');
+    doc = /\.json$/i.test(file) ? JSON.parse(text) : loadYaml(text);
+  } catch (e) { return { ok: false, errors: [`cannot parse ${path.basename(file)}: ${String(e.message).split('\n')[0]}`] }; }
+  const v = validateDatasetManifest(doc);
+  if (!v.ok) return { ok: false, errors: v.errors };
+  return { ok: true, manifest: doc, baseDir: path.dirname(path.resolve(file)), hash: datasetHash(doc) };
+}
+
+/** Resolve a manifest-relative path and confirm it did not escape (symlinks included). Returns null when it did. */
+export function resolveInside(baseDir, rel) {
+  const target = path.resolve(baseDir, rel);
+  let real;
+  try { real = fs.realpathSync(target); } catch { return null; }
+  const root = fs.realpathSync(baseDir);
+  return real === root || real.startsWith(root + path.sep) ? real : null;
+}

package/src/posture/evaluation/lab/materialize.js

@@ -0,0 +1,44 @@
+// Turn a dataset entry's workspace reference into a directory of files (PRD 6, requirement 3: vulnerable commit -> fixing commit).
+// Two sources: a folder inside the manifest's own directory, or a revision of a LOCAL git repository read with `git archive`
+// (so no checkout, no hooks and no working tree are involved) and unpacked through the hardened tar reader. Nothing is fetched
+// from the network, a moving ref is refused at validation time, and every limit is a typed failure rather than a truncation.
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { readTarEntries, TarSafetyError } from '../../../artifact/safe-tar.js';
+import { resolveInside } from './dataset.js';
+
+const MATERIALIZE_LIMITS = Object.freeze({ gitTimeoutMs: 120_000, maxArchiveBytes: 512 * 1024 * 1024, maxFiles: 50_000, maxFileBytes: 16 * 1024 * 1024 });
+
+export function materializeEntry(entry, { baseDir, destDir, limits = MATERIALIZE_LIMITS } = {}) {
+  fs.rmSync(destDir, { recursive: true, force: true });
+  fs.mkdirSync(destDir, { recursive: true });
+  const ws = entry.workspace;
+  if (ws.dir !== undefined) {
+    const src = resolveInside(baseDir, ws.dir);
+    if (!src || !fs.statSync(src).isDirectory()) return { ok: false, reason: 'workspace directory is missing or outside the manifest folder' };
+    fs.cpSync(src, destDir, { recursive: true, dereference: false, filter: (s) => !['.git', 'node_modules'].includes(path.basename(s)) });
+    return { ok: true, source: 'directory' };
+  }
+  const repo = resolveInside(baseDir, ws.git.repo) ?? (path.isAbsolute(ws.git.repo) && fs.existsSync(ws.git.repo) ? fs.realpathSync(ws.git.repo) : null);
+  if (!repo) return { ok: false, reason: 'git repository not found' };
+  const r = spawnSync('git', ['-C', repo, 'archive', '--format=tar', ws.git.rev], { timeout: limits.gitTimeoutMs, killSignal: 'SIGKILL', maxBuffer: limits.maxArchiveBytes, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, GIT_CONFIG_NOSYSTEM: '1', GIT_TERMINAL_PROMPT: '0' } });
+  if (r.error) return { ok: false, reason: r.error.code === 'ETIMEDOUT' ? 'git archive timed out' : r.error.code === 'ENOBUFS' ? 'archive exceeds the size limit' : `git archive failed: ${r.error.code || r.error.message}` };
+  if (r.status !== 0) return { ok: false, reason: `git archive failed: ${String(r.stderr).split('\n')[0].slice(0, 160)}` };
+  let files = 0;
+  try {
+    for (const e of readTarEntries(r.stdout)) {
+      if (e.unsafe) return { ok: false, reason: `unsafe path in the archive: ${String(e.name).slice(0, 80)}` };
+      if (e.typeflag !== '0' && e.typeflag !== '\0') continue; // directories are implied; links and devices are never created
+      if (++files > limits.maxFiles) return { ok: false, reason: `more than ${limits.maxFiles} files` };
+      if (e.size > limits.maxFileBytes) return { ok: false, reason: `a file exceeds ${limits.maxFileBytes} bytes` };
+      const target = path.resolve(destDir, e.name);
+      if (target !== destDir && !target.startsWith(destDir + path.sep)) return { ok: false, reason: 'archive path escapes the workspace' };
+      fs.mkdirSync(path.dirname(target), { recursive: true });
+      fs.writeFileSync(target, e.data);
+    }
+  } catch (err) {
+    return { ok: false, reason: err instanceof TarSafetyError ? `archive refused: ${err.message}` : `archive unreadable: ${err.message}` };
+  }
+  return { ok: true, source: 'git', files };
+}