@clear-capabilities/agentic-security-scanner 0.74.0

package/src/posture/fix-verify.js

@@ -0,0 +1,130 @@
+// Closed-loop /fix verification (Sentinel-parity FR-L4-4, FR-L4-5).
+//
+// Given a candidate patch (the new file content + the finding stableId being
+// fixed), verify it:
+//
+//   1. The original finding's stableId no longer fires on the patched file.
+//   2. No new findings at severity ≥ medium were introduced by the patch.
+//   3. The project's existing linter (when present) passes on the patched file.
+//
+// If any of those fail, the caller is expected to NOT apply the patch and
+// instead surface a "fix plan" — a numbered list of steps the engineer can
+// follow — rather than dump a broken patch on the user.
+
+import { spawnSync } from 'node:child_process';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { runFullScan } from '../engine.js';
+
+const SEVERITY_RANK = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
+
+// Run a focused re-scan over just the patched file(s) using the in-memory
+// engine. No filesystem write needed — we hand the new content in via the
+// fileContents map.
+export async function verifyPatch({
+  scanRoot,
+  originalFindingStableId,
+  files,   // { [relPath]: newContent }
+  depFileContents = {},
+} = {}) {
+  if (!files || typeof files !== 'object') return { ok: false, reason: 'no-files-provided' };
+  const fileContents = { ...files };
+  let scan;
+  try {
+    scan = await runFullScan({ fileContents, depFileContents, scanRoot }, () => {});
+  } catch (e) {
+    return { ok: false, reason: 'rescan-failed', error: e.message };
+  }
+  const findings = (scan && scan.findings) || [];
+  const stillHasOriginal = !!originalFindingStableId &&
+    findings.some(f => f.stableId === originalFindingStableId);
+  if (stillHasOriginal) {
+    return { ok: false, reason: 'original-finding-still-present', stableId: originalFindingStableId };
+  }
+  const introducedHighOrAbove = findings.filter(f =>
+    (SEVERITY_RANK[f.severity] ?? 9) <= SEVERITY_RANK.medium);
+  // Don't count findings on lines outside the patched files — but our
+  // fileContents map IS the patched files, so every finding is in-scope.
+  return {
+    ok: introducedHighOrAbove.length === 0,
+    reason: introducedHighOrAbove.length === 0 ? 'verified' : 'introduced-new-findings',
+    introduced: introducedHighOrAbove.map(f => ({
+      vuln: f.vuln, file: f.file, line: f.line, severity: f.severity,
+      stableId: f.stableId,
+    })),
+  };
+}
+
+// Detect which linter the project uses and run it on the patched files.
+// Returns { ok, runner, output } or { ok: true, runner: 'none' } when no
+// linter is configured (silent pass).
+export function runProjectLinter(scanRoot, filePaths) {
+  if (!scanRoot || !Array.isArray(filePaths) || filePaths.length === 0) {
+    return { ok: true, runner: 'none' };
+  }
+  const has = (p) => { try { return fs.existsSync(path.join(scanRoot, p)); } catch { return false; } };
+  // Pick the linter by config file present in the repo root.
+  const jsFiles = filePaths.filter(f => /\.(?:js|jsx|ts|tsx|mjs|cjs)$/i.test(f));
+  const pyFiles = filePaths.filter(f => /\.py$/i.test(f));
+  const goFiles = filePaths.filter(f => /\.go$/i.test(f));
+  const javaFiles = filePaths.filter(f => /\.java$/i.test(f));
+
+  if (jsFiles.length && (has('.eslintrc') || has('.eslintrc.json') || has('.eslintrc.js') || has('eslint.config.js') || has('eslint.config.mjs'))) {
+    return runLinter(scanRoot, 'eslint', ['--no-error-on-unmatched-pattern', ...jsFiles]);
+  }
+  if (pyFiles.length && (has('pyproject.toml') || has('ruff.toml') || has('.ruff.toml'))) {
+    return runLinter(scanRoot, 'ruff', ['check', ...pyFiles]);
+  }
+  if (pyFiles.length && has('.flake8')) {
+    return runLinter(scanRoot, 'flake8', pyFiles);
+  }
+  if (goFiles.length && (has('.golangci.yml') || has('.golangci.yaml'))) {
+    return runLinter(scanRoot, 'golangci-lint', ['run', ...goFiles]);
+  }
+  if (javaFiles.length && has('checkstyle.xml')) {
+    return runLinter(scanRoot, 'checkstyle', ['-c', 'checkstyle.xml', ...javaFiles]);
+  }
+  return { ok: true, runner: 'none' };
+}
+
+function runLinter(cwd, cmd, args) {
+  let r;
+  try {
+    r = spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: 60_000 });
+  } catch (e) {
+    return { ok: true, runner: cmd, skipped: true, reason: 'binary-missing', error: e.message };
+  }
+  if (r.error && r.error.code === 'ENOENT') {
+    return { ok: true, runner: cmd, skipped: true, reason: 'binary-missing' };
+  }
+  if (r.status === null) {
+    return { ok: false, runner: cmd, reason: 'timed-out', output: (r.stderr || r.stdout || '').slice(-2000) };
+  }
+  return {
+    ok: r.status === 0,
+    runner: cmd,
+    exitCode: r.status,
+    output: ((r.stderr || '') + (r.stdout || '')).slice(-2000),
+  };
+}
+
+// Top-level verify: re-scan + lint. Returns the combined verdict + a
+// human-readable summary string suitable for surfacing to the user.
+export async function verifyFix({
+  scanRoot,
+  originalFindingStableId,
+  files,
+  depFileContents,
+} = {}) {
+  const rescan = await verifyPatch({ scanRoot, originalFindingStableId, files, depFileContents });
+  const lint = runProjectLinter(scanRoot, Object.keys(files || {}));
+  const ok = rescan.ok && (lint.ok || lint.skipped);
+  const summary = [
+    `re-scan: ${rescan.ok ? 'PASS' : 'FAIL — ' + rescan.reason}`,
+    `linter:  ${lint.runner === 'none' ? 'skipped (no linter config)'
+              : lint.skipped ? `${lint.runner} not installed`
+              : lint.ok ? `${lint.runner} PASS`
+              : `${lint.runner} FAIL (exit ${lint.exitCode})`}`,
+  ].join('\n');
+  return { ok, rescan, lint, summary };
+}

package/src/posture/flow-narration.js

@@ -0,0 +1,105 @@
+// LLM-driven flow narration (FR-LOGIC-6).
+//
+// For each high-severity finding, produce a one-paragraph narrative of:
+//   - how the attacker gets to this code path
+//   - what they get if it works
+//   - what it costs the business
+//
+// Two modes:
+//   1. LLM mode (AGENTIC_SECURITY_LLM_ENDPOINT set): post the finding to
+//      the configured LLM endpoint, get back a sanitized narrative.
+//   2. Template mode (default): emit a deterministic template based on the
+//      finding's family + cost-framing data from blast-radius.
+//
+// Fail-closed: any LLM error → template fallback, never a missing field.
+
+const TEMPLATES = {
+  'sql-injection': (f) =>
+    `An unauthenticated attacker sends a crafted request to ${_routeOf(f)} containing UNION-style SQL syntax in the ${f.source?.variable || 'tainted'} field. The server's database driver executes the injected query verbatim, returning rows from any table the connection has read access to. Typical impact: full table dump of users (emails, password hashes), bypass of authentication via boolean-blind exfiltration. If the DB role has write privileges, the attacker can also INSERT/UPDATE arbitrary rows. Recovery cost: incident response, customer notification, password reset, regulatory reporting if PII leaked.`,
+  'command-injection': (f) =>
+    `The handler at ${f.file}:${f.line} passes user-controlled input to a shell-spawning function. An attacker can append shell metacharacters (";", "$(...)", backticks) to execute arbitrary commands as the application's UID. Typical impact: read of /etc/passwd, /proc/self/environ (env vars including secrets), outbound connections to attacker-controlled hosts (data exfil). On unprivileged containers the blast radius is limited to that container; on privileged or root-owned processes, the attacker can pivot to the host.`,
+  'xss': (f) =>
+    `An attacker injects HTML/JS markup into user-controllable input. The server reflects (or stores) it without encoding, so when a victim browser renders the page, the attacker's script executes in the victim's session origin. Typical impact: session cookie theft, CSRF-bypass on internal endpoints, account takeover via API calls executed under the victim's auth. Cost: incident response, customer notification, potential data egress depending on what the victim's session can access.`,
+  'ssrf': (f) =>
+    `The handler fetches a URL constructed from user input. An attacker supplies a URL pointing at cloud-metadata endpoints (169.254.169.254 on AWS, metadata.google.internal on GCP) or internal services not exposed externally. Typical impact: theft of IAM credentials attached to the instance, fingerprinting / exploitation of internal services, port-scanning the VPC. Cost: full AWS account compromise in the worst case (IAM credential rotation, audit, blast-radius review of every action taken under the leaked credentials).`,
+  'path-traversal': (f) =>
+    `The handler opens a file at a path derived from user input without confining the resolved path to an intended directory. An attacker submits "../../etc/passwd" (or %-encoded variants) to read arbitrary files the application has access to. Typical impact: leakage of config files, secrets, source code, /etc/passwd. Cost: depends on what files are readable — usually low-to-medium unless secrets land in the readable set.`,
+  'code-injection': (f) =>
+    `User input is fed into a code-evaluation function (eval, new Function, exec). An attacker supplies arbitrary code that executes in the application's runtime context, with full access to the application's data, env, and outbound network. Typical impact: equivalent to remote code execution; same recovery cost as command-injection.`,
+  'csrf': (f) =>
+    `The state-changing endpoint at ${f.file}:${f.line} doesn't validate that the request originated from your own application. An attacker hosts a page that issues a same-shape request from a logged-in victim's browser. Typical impact: state changes performed under the victim's identity — password change, money movement, role escalation. Cost: depends on what state can change; for billing endpoints, this is fraud-level.`,
+  'open-redirect': (f) =>
+    `The endpoint redirects to a URL the attacker controls. Used as part of phishing chains: victim clicks a legitimate-looking link to your domain, gets redirected to attacker.example, enters credentials thinking they're still on your site. Typical impact: phishing-amplified credential theft; reputational damage if your domain ends up on a phish-tracking list.`,
+  'insecure-deserialization': (f) =>
+    `The handler deserializes attacker-controlled bytes via pickle/yaml-load/Marshal. The deserialization callback invokes arbitrary code from class constructors / __reduce__ / __wakeup__. Typical impact: equivalent to remote code execution. Cost: full incident response, including investigating whether the attacker established persistence.`,
+  'xxe': (f) =>
+    `The XML parser at ${f.file}:${f.line} resolves external entities. An attacker submits XML referencing file:///etc/passwd or http://internal/. Typical impact: file disclosure, SSRF, blind out-of-band exfiltration of secrets. Cost: similar to SSRF + path-traversal combined.`,
+};
+
+function _routeOf(f) {
+  if (!f) return '<endpoint>';
+  return `${f.file || '?'}:${f.line || '?'}`;
+}
+
+function _templateFor(f) {
+  const fam = f.family;
+  if (TEMPLATES[fam]) return TEMPLATES[fam](f);
+  return `A finding of type "${f.vuln || fam || 'unknown'}" at ${_routeOf(f)}. Severity: ${f.severity || 'unknown'}. Review the remediation field for class-specific guidance.`;
+}
+
+// Render the narration without an LLM. Always available; used as the fallback
+// when no LLM endpoint is configured.
+function _renderTemplate(f) {
+  return _templateFor(f);
+}
+
+// Optional LLM call. Disabled by default; opt-in via env. Falls back to the
+// template on any error.
+async function _renderLlm(f) {
+  const endpoint = process.env.AGENTIC_SECURITY_LLM_ENDPOINT;
+  if (!endpoint) return null;
+  const apiKey = process.env.AGENTIC_SECURITY_LLM_API_KEY;
+  const headers = { 'Content-Type': 'application/json' };
+  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
+  const prompt = `You are explaining a security finding to a developer in one paragraph.
+Vuln: ${f.vuln}
+CWE: ${f.cwe}
+Severity: ${f.severity}
+Location: ${f.file}:${f.line}
+Snippet: ${(f.snippet || '').slice(0, 200)}
+
+Write ONE paragraph (5-7 sentences) covering: (1) how an attacker reaches this code, (2) what they get if exploited, (3) typical recovery cost. Plain English, no marketing language, no emoji.`;
+  try {
+    const r = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify({ prompt }) });
+    if (!r.ok) return null;
+    const j = await r.json().catch(() => null);
+    const text = j && (j.response || j.text || j.content || j.output ||
+                       j.choices?.[0]?.message?.content || j.message?.content);
+    if (typeof text !== 'string' || text.length < 30) return null;
+    // Sanitize: strip control chars, markdown fences, HTML metachars.
+    return text.replace(/[\x00-\x1f\x7f]/g, ' ').replace(/[<>&]/g, ' ')
+               .replace(/```/g, '').replace(/\s+/g, ' ').trim().slice(0, 1500);
+  } catch { return null; }
+}
+
+/**
+ * Annotate findings with f.narration. Default mode is template; opt-in to
+ * LLM via AGENTIC_SECURITY_LLM_ENDPOINT.
+ */
+export async function annotateNarration(findings, opts = {}) {
+  if (!Array.isArray(findings)) return;
+  const useLlm = !!opts.useLlm || process.env.AGENTIC_SECURITY_FLOW_NARRATION_LLM === '1';
+  for (const f of findings) {
+    if (!f || typeof f !== 'object') continue;
+    // Only narrate severity ≥ high to keep output tight on noisy projects.
+    if (!/critical|high/i.test(f.severity || '')) {
+      f.narration = null;
+      continue;
+    }
+    let text = useLlm ? await _renderLlm(f) : null;
+    if (!text) text = _renderTemplate(f);
+    f.narration = text;
+  }
+}
+
+export const _internals = { TEMPLATES, _renderTemplate, _templateFor };

package/src/posture/grader-calibration.js

@@ -0,0 +1,156 @@
+// Human ⇆ LLM grader calibration (eval-post recommendation #5).
+//
+// The Anthropic eval-post quote we're implementing:
+//   "LLM-based rubrics should be frequently calibrated against expert
+//    human judgment to grade these agents effectively."
+//
+// We have two grader streams in this codebase already:
+//   - HUMAN: `/triage` writes per-finding verdicts (tp/fp/wontfix) to
+//     `.agentic-security/triage-feedback.json`, keyed by stableId.
+//   - LLM:   `llm-validator` writes per-finding verdicts (accept/reject/
+//     escalate) to its cache under `.agentic-security/llm-cache/*.json`,
+//     plus `validator_verdict` on each finding in `last-scan.json`.
+//
+// This module joins them on stableId and reports inter-rater agreement
+// (Cohen's κ) over the overlap. κ < 0.6 means the LLM rubric is drifting
+// from human judgment — operator should re-tune the prompt or escalate
+// human review.
+//
+// Verdict mapping (TP/FP are the only ones κ measures):
+//   HUMAN tp        ↔ LLM accept   ("real finding")
+//   HUMAN fp        ↔ LLM reject   ("false positive")
+//   HUMAN wontfix   → excluded from κ (not a quality signal)
+//   LLM escalate    → excluded from κ (deliberate "I don't know")
+//   LLM unvalidated → excluded from κ (validator didn't run)
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+const TRIAGE_FILE = '.agentic-security/triage-feedback.json';
+const SCAN_FILE   = '.agentic-security/last-scan.json';
+
+function _loadTriageFeedback(scanRoot) {
+  const fp = path.join(scanRoot, TRIAGE_FILE);
+  if (!fs.existsSync(fp)) return [];
+  try { return JSON.parse(fs.readFileSync(fp, 'utf8')).entries || []; }
+  catch { return []; }
+}
+
+function _loadScanVerdicts(scanRoot) {
+  const fp = path.join(scanRoot, SCAN_FILE);
+  if (!fs.existsSync(fp)) return [];
+  try {
+    const scan = JSON.parse(fs.readFileSync(fp, 'utf8'));
+    return (scan.findings || []).map(f => ({
+      stableId: f.stableId || null,
+      verdict: f.validator_verdict || null,
+      confidence: typeof f.llm_confidence === 'number' ? f.llm_confidence : null,
+    })).filter(e => e.stableId && e.verdict);
+  } catch { return []; }
+}
+
+// Cohen's κ for two binary raters:
+//   p_o = observed agreement = (concordant) / n
+//   p_e = expected by chance = sum over classes c of (p_human_c * p_llm_c)
+//   κ   = (p_o - p_e) / (1 - p_e)
+// κ = 1: perfect agreement. κ = 0: chance. κ < 0: worse than chance.
+// Common threshold: κ >= 0.6 is "substantial agreement" (Landis & Koch).
+export function cohensKappa(pairs) {
+  if (!Array.isArray(pairs) || pairs.length === 0) return { kappa: null, reason: 'no-pairs' };
+  let agree = 0, hPos = 0, lPos = 0, n = pairs.length;
+  for (const { human, llm } of pairs) {
+    if (human === llm) agree++;
+    if (human === 'positive') hPos++;
+    if (llm === 'positive') lPos++;
+  }
+  const pO = agree / n;
+  const pHumanPos = hPos / n, pLlmPos = lPos / n;
+  const pE = pHumanPos * pLlmPos + (1 - pHumanPos) * (1 - pLlmPos);
+  if (pE >= 0.9999) {
+    // All raters agree on the same class — κ is undefined (1-pE → 0). Report
+    // perfect or majority agreement honestly without dividing by ~0.
+    return { kappa: pO === 1 ? 1 : null, reason: pO === 1 ? 'perfect-agreement' : 'pE-saturated', pO, pE, n };
+  }
+  const kappa = (pO - pE) / (1 - pE);
+  return { kappa, pO, pE, n };
+}
+
+// Join human triage with LLM verdicts on stableId. Returns:
+//   { pairs: [{ stableId, human, llm }], ... }
+// where `human` and `llm` are mapped into the {positive, negative} binary used
+// for κ. Findings that fall into the excluded buckets (wontfix, escalate,
+// unvalidated) are stripped from `pairs` but counted under `excluded`.
+export function joinHumanLlm(triageEntries, validatorEntries) {
+  // Most-recent triage entry wins per stableId.
+  const latestHuman = new Map();
+  for (const e of triageEntries) {
+    if (!e.stableId) continue;
+    const prev = latestHuman.get(e.stableId);
+    if (!prev || String(e.at || '').localeCompare(String(prev.at || '')) > 0) {
+      latestHuman.set(e.stableId, e);
+    }
+  }
+  const llmById = new Map();
+  for (const v of validatorEntries) llmById.set(v.stableId, v);
+
+  const pairs = [];
+  const excluded = { human_wontfix: 0, llm_escalate: 0, llm_unvalidated: 0, llm_not_applicable: 0, no_llm_for_this_stableid: 0 };
+  for (const [stableId, hum] of latestHuman) {
+    const llm = llmById.get(stableId);
+    if (!llm) { excluded.no_llm_for_this_stableid++; continue; }
+    if (hum.verdict === 'wontfix') { excluded.human_wontfix++; continue; }
+    if (llm.verdict === 'escalate') { excluded.llm_escalate++; continue; }
+    if (llm.verdict === 'unvalidated') { excluded.llm_unvalidated++; continue; }
+    if (llm.verdict === 'not-applicable') { excluded.llm_not_applicable++; continue; }
+    // Map to binary.
+    const humanBin = hum.verdict === 'tp' ? 'positive' : hum.verdict === 'fp' ? 'negative' : null;
+    const llmBin   = llm.verdict === 'accept' ? 'positive' : llm.verdict === 'reject' ? 'negative' : null;
+    if (!humanBin || !llmBin) continue;
+    pairs.push({ stableId, human: humanBin, llm: llmBin, llmConfidence: llm.confidence });
+  }
+  return { pairs, excluded, totalTriaged: latestHuman.size, totalValidated: llmById.size };
+}
+
+// Full calibration report for a scanRoot.
+//
+// alarmAt: the κ threshold below which the operator should re-tune. Default
+// 0.6 matches the "substantial agreement" cutoff in Landis & Koch (1977).
+// MIN_N_FOR_ALARM: don't alarm on n<10; the CI on a small sample swamps κ.
+const MIN_N_FOR_ALARM = 10;
+
+export function calibrateGraders(scanRoot, { alarmAt = 0.6 } = {}) {
+  const triage = _loadTriageFeedback(scanRoot);
+  const llm = _loadScanVerdicts(scanRoot);
+  const join = joinHumanLlm(triage, llm);
+  const kapp = cohensKappa(join.pairs);
+  const alarm = kapp.kappa !== null && kapp.n >= MIN_N_FOR_ALARM && kapp.kappa < alarmAt;
+  return {
+    when: new Date().toISOString(),
+    triageEntries: triage.length,
+    validatorEntries: llm.length,
+    overlap: join.pairs.length,
+    excluded: join.excluded,
+    kappa: kapp.kappa,
+    pObserved: kapp.pO,
+    pExpected: kapp.pE,
+    kappaInterpretation: _interpretKappa(kapp.kappa, kapp.n),
+    alarm,
+    alarmThreshold: alarmAt,
+    note: alarm
+      ? `LLM verdicts diverging from human triage (κ=${kapp.kappa?.toFixed(3)} < ${alarmAt}). Re-tune the validator prompt or escalate human review.`
+      : kapp.kappa === null
+        ? `Insufficient overlap (n=${kapp.n ?? 0}); skip calibration until more triage feedback accumulates.`
+        : `Validator and human triage substantially agree (κ=${kapp.kappa.toFixed(3)}, n=${kapp.n}).`,
+  };
+}
+
+function _interpretKappa(k, n) {
+  if (k === null) return 'undefined';
+  if (n < MIN_N_FOR_ALARM) return 'insufficient-sample';
+  if (k < 0)    return 'worse-than-chance';
+  if (k < 0.2)  return 'slight';
+  if (k < 0.4)  return 'fair';
+  if (k < 0.6)  return 'moderate';
+  if (k < 0.8)  return 'substantial';
+  return 'almost-perfect';
+}

package/src/posture/harness-discovery.js

@@ -0,0 +1,113 @@
+// Multi-harness configuration discovery.
+//
+// Finds every agent-harness configuration directory the user has, both at
+// the project root AND under ~/. The discovered files feed the
+// claude-settings / claude-md-prompt-injection / claude-hook-injection
+// detectors so we audit Claude / Cursor / Codex / Gemini / Kiro / OpenCode /
+// Trae / Qwen / Zed / Continue / Aider with one sweep.
+//
+// Used by the `/scan --harness` mode.
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as os from 'node:os';
+
+export const HARNESS_DIRS = [
+  '.claude', '.cursor', '.codex', '.gemini', '.kiro',
+  '.opencode', '.trae', '.qwen', '.zed', '.continue', '.aider',
+  '.codebuddy', '.copilot',
+];
+
+const HARNESS_FILES = [
+  // settings + permissions
+  'settings.json', 'settings.local.json', 'config.json',
+  // instruction files (lifted into context every session)
+  'CLAUDE.md', 'AGENTS.md', 'GEMINI.md', 'CURSOR.md', 'CODEX.md',
+  'KIRO.md', 'QWEN.md', 'TRAE.md', 'OPENCODE.md', 'SYSTEM_PROMPT.md',
+  // mcp
+  'mcp.json', '.mcp.json', 'mcp_servers.json', 'claude_desktop_config.json',
+  // hooks
+  'hooks.json', 'hooks.yml', 'hooks.yaml',
+];
+
+const HARNESS_SUBDIRS = ['agents', 'skills', 'commands', 'hooks', 'rules'];
+
+const MAX_FILE_SIZE = 1_000_000;
+
+async function _readSafe(fp) {
+  try {
+    const stat = await fs.stat(fp);
+    if (stat.size > MAX_FILE_SIZE) return null;
+    return await fs.readFile(fp, 'utf8');
+  } catch { return null; }
+}
+
+async function _walkHarnessDir(harnessRoot, harnessName, out) {
+  // Top-level config files.
+  for (const fn of HARNESS_FILES) {
+    const fp = path.join(harnessRoot, fn);
+    const content = await _readSafe(fp);
+    if (content !== null) out[fp] = content;
+  }
+  // Subdirs holding instruction-style files.
+  for (const sub of HARNESS_SUBDIRS) {
+    const dp = path.join(harnessRoot, sub);
+    try {
+      const entries = await fs.readdir(dp, { withFileTypes: true });
+      for (const e of entries) {
+        if (!e.isFile()) continue;
+        if (!/\.(?:md|json|yaml|yml)$/i.test(e.name)) continue;
+        const fp = path.join(dp, e.name);
+        const content = await _readSafe(fp);
+        if (content !== null) out[fp] = content;
+      }
+    } catch { /* dir does not exist — fine */ }
+  }
+  // Project-root CLAUDE.md / AGENTS.md (some users put them outside .claude/).
+  // Only walked when the harness is .claude.
+  void harnessName;
+}
+
+// Discover harness configs at one of:
+//   1. The project root (e.g. /path/to/repo/.claude, /path/to/repo/.cursor)
+//   2. Home directory (e.g. ~/.claude, ~/.cursor) — opt-in via includeHome=true
+export async function discoverHarnessConfigs(projectRoot, opts = {}) {
+  const includeHome = !!opts.includeHome;
+  const out = {};
+
+  // Project-rooted instruction files commonly placed at repo root.
+  for (const fn of ['CLAUDE.md', 'AGENTS.md', 'GEMINI.md', 'CURSOR.md', 'CODEX.md', 'KIRO.md', 'QWEN.md', 'TRAE.md', 'OPENCODE.md']) {
+    const fp = path.join(projectRoot, fn);
+    const content = await _readSafe(fp);
+    if (content !== null) out[fp] = content;
+  }
+
+  for (const dir of HARNESS_DIRS) {
+    const harnessRoot = path.join(projectRoot, dir);
+    try { await fs.access(harnessRoot); } catch { continue; }
+    await _walkHarnessDir(harnessRoot, dir, out);
+  }
+
+  if (includeHome) {
+    const home = os.homedir();
+    if (home) {
+      for (const dir of HARNESS_DIRS) {
+        const harnessRoot = path.join(home, dir);
+        try { await fs.access(harnessRoot); } catch { continue; }
+        await _walkHarnessDir(harnessRoot, dir, out);
+      }
+    }
+  }
+
+  return out;
+}
+
+// Inventory of which harnesses are present (for grade / summary).
+export function summarizeHarnessPresence(fileContents) {
+  const present = new Set();
+  for (const fp of Object.keys(fileContents || {})) {
+    const m = /\.(claude|cursor|codex|gemini|kiro|opencode|trae|qwen|zed|continue|aider|codebuddy|copilot)[\\/]/.exec(fp);
+    if (m) present.add(m[1]);
+  }
+  return [...present].sort();
+}

package/src/posture/holdout-eval.js

@@ -0,0 +1,144 @@
+// Held-out evaluator (premortem #16).
+//
+// Takes a labeled JSONL file of (predicted_confidence, actual_label) records
+// and computes the honest measurements an auditor will ask for: Brier score,
+// expected calibration error (ECE), per-family precision/recall.
+//
+// Replaces the tautological computeBrierFromHistory removed in premortem #9.
+//
+// Input file shape — one JSON object per line:
+//
+//   {"family": "sql-injection", "stableId": "...", "predicted": 0.78,
+//    "actual": 1, "note": "TP — exploited in pen-test"}
+//   {"family": "xss", "predicted": 0.65, "actual": 0,
+//    "note": "FP — sanitizer present but flow-analysis missed it"}
+//
+// `actual` must be 0 (false-positive / clean) or 1 (true-positive). `predicted`
+// is the calibrated_confidence we'd ship to the customer.
+//
+// Output shape — see `evaluateHeldOut` return value.
+
+import * as fs from 'node:fs';
+import { brierScore, computeBrierOnHeldOut, wilsonInterval } from './calibration.js';
+
+const ECE_BINS_DEFAULT = 10;
+
+export function parseLabeledJsonl(text) {
+  if (typeof text !== 'string' || !text.length) return [];
+  const out = [];
+  for (const line of text.split('\n')) {
+    const t = line.trim();
+    if (!t) continue;
+    try {
+      const o = JSON.parse(t);
+      if (!o || typeof o !== 'object') continue;
+      const p = typeof o.predicted === 'number' ? o.predicted : null;
+      const a = (o.actual === 1 || o.actual === true) ? 1
+              : (o.actual === 0 || o.actual === false) ? 0
+              : null;
+      if (p === null || a === null) continue;
+      out.push({
+        family: typeof o.family === 'string' ? o.family : 'unknown',
+        stableId: typeof o.stableId === 'string' ? o.stableId : null,
+        predicted: Math.max(0, Math.min(1, p)),
+        actual: a,
+        note: typeof o.note === 'string' ? o.note : '',
+      });
+    } catch { /* skip malformed lines */ }
+  }
+  return out;
+}
+
+export function loadLabeledJsonl(filepath) {
+  if (!filepath || !fs.existsSync(filepath)) return [];
+  return parseLabeledJsonl(fs.readFileSync(filepath, 'utf8'));
+}
+
+// Expected calibration error: bucket predictions into `nBins` equal-width
+// bins, compare bucket-mean prediction vs bucket-mean actual.
+// ECE = sum over bins of (|bin|/N) * |mean_pred - mean_actual|.
+// ECE = 0 is perfect; common-good calibration ≤ 0.05.
+export function expectedCalibrationError(samples, nBins = ECE_BINS_DEFAULT) {
+  if (!Array.isArray(samples) || samples.length === 0) return null;
+  const n = samples.length;
+  const bins = Array.from({ length: nBins }, () => ({ preds: [], actuals: [] }));
+  for (const s of samples) {
+    const p = Math.max(0, Math.min(1, s.predicted));
+    // Clamp only for the bin index — `1.0` lands in the last bin without
+    // distorting the bin's mean prediction.
+    const idx = Math.min(nBins - 1, Math.floor(Math.min(0.99999, p) * nBins));
+    bins[idx].preds.push(p);
+    bins[idx].actuals.push(s.actual);
+  }
+  let ece = 0;
+  const per = [];
+  for (let i = 0; i < nBins; i++) {
+    const b = bins[i];
+    if (b.preds.length === 0) { per.push({ bin: i, n: 0 }); continue; }
+    const mp = b.preds.reduce((a, c) => a + c, 0) / b.preds.length;
+    const ma = b.actuals.reduce((a, c) => a + c, 0) / b.actuals.length;
+    ece += (b.preds.length / n) * Math.abs(mp - ma);
+    per.push({ bin: i, n: b.preds.length, mean_pred: mp, mean_actual: ma, gap: mp - ma });
+  }
+  return { ece, perBin: per, nBins, total: n };
+}
+
+export function perFamily(samples) {
+  const fams = {};
+  for (const s of samples) {
+    const f = s.family || 'unknown';
+    if (!fams[f]) fams[f] = { tp: 0, fp: 0, fn: 0, tn: 0, n: 0 };
+    fams[f].n++;
+    // We don't have a separate threshold here; "TP" = positive label;
+    // "FP" = negative label. Precision is the engine's positive predictive
+    // value at the operating point its calibration assigned.
+    if (s.actual === 1) fams[f].tp++;
+    else if (s.actual === 0) fams[f].fp++;
+  }
+  return fams;
+}
+
+// One-shot evaluation: Brier + ECE + per-family TP/FP + overall precision.
+// Returns null only when there's truly no data; never returns a tautological
+// zero.
+export function evaluateHeldOut(samples) {
+  if (!Array.isArray(samples) || samples.length === 0) {
+    return { ok: false, reason: 'no-samples' };
+  }
+  const brierR = computeBrierOnHeldOut(samples.map(s => ({
+    predicted: s.predicted, actual: s.actual,
+  })));
+  const ece = expectedCalibrationError(samples);
+  const fams = perFamily(samples);
+  const totalTP = Object.values(fams).reduce((a, f) => a + f.tp, 0);
+  const totalFP = Object.values(fams).reduce((a, f) => a + f.fp, 0);
+  const precision = (totalTP + totalFP) > 0 ? totalTP / (totalTP + totalFP) : 0;
+  // Wilson CI on the overall positive-rate as a calibration sanity check.
+  const ci = wilsonInterval(totalTP, totalTP + totalFP);
+  return {
+    ok: true,
+    n: samples.length,
+    brier: brierR.brier,
+    ece: ece?.ece ?? null,
+    eceDetail: ece,
+    precision,
+    precisionCi95: ci,
+    perFamily: fams,
+    notes: [
+      ...(samples.length < 100 ? ['n<100: Brier and ECE have wide confidence; treat as directional, not decision-grade.'] : []),
+      ...(brierR.brier !== null && brierR.brier > 0.10 ? [`brier=${brierR.brier.toFixed(3)} exceeds PRD target 0.10`] : []),
+      ...(ece && ece.ece > 0.05 ? [`ece=${ece.ece.toFixed(3)} exceeds 0.05 calibration target`] : []),
+    ],
+  };
+}
+
+// CLI-friendly summary line.
+export function summarize(result) {
+  if (!result || !result.ok) return `held-out: ${result?.reason || 'unknown error'}`;
+  return [
+    `n=${result.n}`,
+    `brier=${result.brier != null ? result.brier.toFixed(3) : 'null'}`,
+    `ece=${result.ece != null ? result.ece.toFixed(3) : 'null'}`,
+    `precision=${result.precision.toFixed(3)} CI95=[${result.precisionCi95[0].toFixed(3)},${result.precisionCi95[1].toFixed(3)}]`,
+  ].join(' · ');
+}