@clear-capabilities/agentic-security-scanner 0.133.0 → 0.136.2

package/src/posture/auditor-walkthrough.js

@@ -8,6 +8,8 @@
 //
 //   nist-csf-2          NIST Cybersecurity Framework 2.0
 //   nist-ai-600-1       NIST AI Risk Management Framework, GenAI profile
+//   nist-privacy-1-1    NIST Privacy Framework 1.1 (see posture/privacy-framework.js
+//                       for the assessment + remediation layer over it)
 //   owasp-asvs-5        OWASP Application Security Verification Standard 5.0
 //   owasp-llm-top-10    OWASP Top 10 for LLM Applications 2025
 //   eu-ai-act           EU AI Act (Regulation 2024/1689)
@@ -32,9 +34,8 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { statePath, stateWritesEnabled } from './state-dir.js';
 const BUNDLED_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), 'compliance-frameworks');
-const STATE = '.agentic-security';
-
 function _readJson(fp) {
   try { return JSON.parse(fs.readFileSync(fp, 'utf8')); } catch { return null; }
 }
@@ -52,7 +53,7 @@ export function listFrameworks(scanRoot) {
     }
   } catch {}
   if (scanRoot) {
-    const projDir = path.join(scanRoot, STATE, 'compliance');
+    const projDir = statePath(scanRoot, 'compliance');
     if (fs.existsSync(projDir)) {
       try {
         for (const sub of fs.readdirSync(projDir)) {
@@ -73,7 +74,7 @@ export function listFrameworks(scanRoot) {
  */
 export function loadFramework(scanRoot, id) {
   if (scanRoot) {
-    const projFp = path.join(scanRoot, STATE, 'compliance', id, 'controls.json');
+    const projFp = statePath(scanRoot, 'compliance', id, 'controls.json');
     if (fs.existsSync(projFp)) return _readJson(projFp);
   }
   for (const fn of fs.readdirSync(BUNDLED_DIR)) {
@@ -164,11 +165,11 @@ export function evaluateFramework(scanRoot, fw, scan) {
         // A '.../' sentinel marks a source-relative artifact (project source,
         // e.g. a hook or agent file) — resolve it against the scan root itself.
         // Everything else is a runtime artifact under the STATE dir. Without
-        // this, `path.join(scanRoot, STATE, '.../x')` never resolves and the
+        // this, `statePath(scanRoot, '.../x')` never resolves and the
         // control falsely reads "not present" for every project.
         const resolved = !target ? null
           : target.startsWith('.../') ? path.join(scanRoot, target.slice(4))
-          : path.join(scanRoot, STATE, target);
+          : statePath(scanRoot, target);
         const label = target ? target.replace(/^\.\.\.\//, '') : '(unmapped)';
         if (resolved && fs.existsSync(resolved)) {
           obs.push(`✓ ${mod}: ${label} present.`);
@@ -251,7 +252,8 @@ export function renderWalkthrough(fw, evaluation, opts = {}) {
  * Persist the walkthrough at .agentic-security/auditor-walkthroughs/<id>.md
  */
 export function persistWalkthrough(scanRoot, fw, body) {
-  const dir = path.join(scanRoot, STATE, 'auditor-walkthroughs');
+  const dir = statePath(scanRoot, 'auditor-walkthroughs');
+  if (!stateWritesEnabled()) return null;
   try { fs.mkdirSync(dir, { recursive: true }); } catch {}
   const fp = path.join(dir, `${fw.id}.md`);
   try { fs.writeFileSync(fp, body); } catch {}

package/src/posture/auth-posture-import.js

@@ -27,6 +27,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { statePath } from './state-dir.js';
 const KNOWN_GOOD = new Set([
   'session+csrf', 'session+admin', 'session+claim', 'session',
   'jwt+verify', 'oauth+pkce', 'mtls',
@@ -41,15 +42,15 @@ const FAMILIES_GATEABLE_BY_AUTH = new Set([
 ]);
 
 const CANDIDATE_PATHS = [
-  '.agentic-security/auth-posture.json',
-  '.agentic-security/auth-posture.yml',
-  '.agentic-security/auth-posture.yaml',
+  'auth-posture.json',
+  'auth-posture.yml',
+  'auth-posture.yaml',
 ];
 
 export function loadAuthPosture(scanRoot) {
   const root = scanRoot || process.cwd();
   for (const rel of CANDIDATE_PATHS) {
-    const fp = path.join(root, rel);
+    const fp = statePath(root, rel);
     if (!fs.existsSync(fp)) continue;
     try {
       const text = fs.readFileSync(fp, 'utf8');

package/src/posture/autopilot.js

@@ -0,0 +1,225 @@
+// PRD Epic 4 — the autonomous loop.
+//
+// Every stage already existed as a separate command. What did not exist was the
+// chain: scan → prove → validate → fix → RE-VERIFY, with state between stages
+// and a gate before anything is written. This module is that chain and nothing
+// else — it owns no analysis, and each stage is injected, so the orchestration
+// can be tested without running an engine or calling a model.
+//
+// THE RULE THAT MAKES IT SAFE TO AUTOMATE: a fix is applied only if the PoC
+// that proved the bug no longer fires AND the test suite still passes. That is
+// `VERIFIED_FIXED`. Anything else is `NEEDS_REVIEW` and is NOT written. A loop
+// that applies patches on the strength of "the scanner stopped complaining"
+// automates the failure mode where a cosmetic edit silences a detector while
+// the vulnerability remains — the re-scan proves the DETECTOR went quiet, and
+// only re-running the exploit proves the hole is shut.
+//
+// GATES ARE ON BY DEFAULT. `apply` requires an explicit opt-in. Autonomy plus
+// write access is the combination that turns a bad patch into a bad commit, and
+// the default must be the one that cannot.
+//
+// RESUMABLE, AND HONEST ABOUT WHAT IT SKIPPED. Each stage records its outcome
+// per finding; a resumed run replays completed stages from state rather than
+// re-running them. A stage that was skipped is reported as skipped — never
+// folded into "nothing to do", which is how an interrupted run reads as a clean
+// one.
+
+import fs from 'node:fs';
+import path from 'node:path';
+
+const SCHEMA = 'agentic-security/autopilot@1';
+
+export const STAGES = Object.freeze(['scan', 'prove', 'validate', 'fix', 'reverify']);
+
+// Outcomes a finding can end a run in. Closed set: a new outcome must be
+// declared here, because the report groups on it and an unknown value would
+// silently vanish from every count.
+export const OUTCOMES = Object.freeze([
+  'VERIFIED_FIXED',   // patch applied (or ready): PoC no longer fires, tests pass
+  'NEEDS_REVIEW',     // a fix exists but did not re-verify — NOT applied
+  'NO_FIX',           // nothing was synthesised
+  'UNPROVEN',         // no PoC fired, so no fix was attempted at this tier
+]);
+
+export function loadAutopilotState(stateFile) {
+  try {
+    const j = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
+    if (j && j.schema === SCHEMA) return j;
+  } catch { /* absent -> clean */ }
+  return { schema: SCHEMA, stages: {}, findings: {} };
+}
+
+function _save(stateFile, state) {
+  if (!stateFile) return;
+  try {
+    fs.mkdirSync(path.dirname(stateFile), { recursive: true });
+    fs.writeFileSync(stateFile, JSON.stringify(state, null, 2));
+  } catch { /* state is an optimisation; losing it must not fail a run */ }
+}
+
+/**
+ * Run the loop.
+ *
+ * Every stage is injected. `apply` defaults to false: the loop produces
+ * VERIFIED_FIXED patches and does not write them unless told to.
+ *
+ * @param {object} stages
+ *   scan()                      -> { findings: [] }
+ *   prove(finding)              -> { proofTier, proofEvidence, poc }
+ *   validate(finding)           -> { verdict: 'upheld'|'refuted'|'undecided' }
+ *   synthesizeFix(finding)      -> { patch: {file: content} } | null
+ *   verifyFix(finding, patch)   -> { ok, pocStillFires, testsPass, reason }
+ *   applyFix(finding, patch)    -> void   (only called when apply === true)
+ */
+export async function runAutopilot({
+  stages = {}, stateFile = null, resume = true, apply = false,
+  onStage = () => {}, severities = ['critical', 'high'], maxFindings = Infinity,
+} = {}) {
+  const required = ['scan'];
+  for (const r of required) {
+    if (typeof stages[r] !== 'function') return { ok: false, reason: `no ${r} stage supplied` };
+  }
+  const state = resume && stateFile ? loadAutopilotState(stateFile) : { schema: SCHEMA, stages: {}, findings: {} };
+  const skipped = [];
+
+  // ── scan ────────────────────────────────────────────────────────────────
+  let findings;
+  if (resume && state.stages.scan) {
+    findings = state.stages.scan.findings || [];
+    skipped.push('scan');
+  } else {
+    const r = await stages.scan();
+    findings = r?.findings || [];
+    state.stages.scan = { findings };
+    _save(stateFile, state);
+  }
+  onStage({ stage: 'scan', count: findings.length });
+
+  // Only the severities asked for reach the expensive stages. Reported, so a
+  // reader can see what was in scope rather than assuming everything was.
+  const all = findings.filter(f => severities.includes(String(f.severity || '').toLowerCase()));
+  const outOfScope = findings.length - all.length;
+  // Every stage costs a sandboxed process and possibly a test-suite run, so the
+  // count is bounded. Reported, never silent: findings past the cap were NOT
+  // examined, and a run that quietly stopped at N would read as a run that
+  // found only N.
+  const inScope = all.slice(0, Number.isFinite(maxFindings) ? Math.max(0, maxFindings) : all.length);
+  const capped = all.length - inScope.length;
+
+  const results = [];
+  for (const f of inScope) {
+    const key = f.stableId || `${f.file}:${f.line}:${f.vuln}`;
+    const prior = resume ? state.findings[key] : null;
+    if (prior?.outcome) { results.push(prior); continue; }
+
+    const rec = { key, file: f.file, line: f.line, vuln: f.vuln, severity: f.severity };
+
+    // ── prove ─────────────────────────────────────────────────────────────
+    let proved = null;
+    if (typeof stages.prove === 'function') {
+      proved = await stages.prove(f).catch(e => ({ error: String(e?.message || e) }));
+      rec.proofTier = proved?.proofTier || null;
+    }
+    const isProven = rec.proofTier === 'execution-proven';
+
+    // ── validate ──────────────────────────────────────────────────────────
+    if (typeof stages.validate === 'function') {
+      const v = await stages.validate(f).catch(() => null);
+      rec.validation = v?.verdict || null;
+      // A refuted finding is NOT dropped — same recall-preserving rule the rest
+      // of the engine follows. It is recorded and skipped for fixing.
+      if (rec.validation === 'refuted') {
+        rec.outcome = 'NEEDS_REVIEW';
+        rec.reason = 'an independent verifier refuted this finding; not fixed automatically';
+        results.push(rec); state.findings[key] = rec; _save(stateFile, state);
+        onStage({ stage: 'validate', key, verdict: 'refuted' });
+        continue;
+      }
+    }
+
+    // Fixing is gated on proof at this tier. An unproven finding may still be
+    // real — it is reported UNPROVEN, not dismissed.
+    if (!isProven) {
+      rec.outcome = 'UNPROVEN';
+      rec.reason = proved?.proofEvidence?.reason || 'no proof-of-concept demonstrated this finding';
+      results.push(rec); state.findings[key] = rec; _save(stateFile, state);
+      continue;
+    }
+
+    // ── fix ───────────────────────────────────────────────────────────────
+    let patch = null;
+    if (typeof stages.synthesizeFix === 'function') {
+      patch = await stages.synthesizeFix(f).catch(() => null);
+    }
+    if (!patch || !patch.patch) {
+      rec.outcome = 'NO_FIX';
+      rec.reason = 'no patch was synthesised';
+      results.push(rec); state.findings[key] = rec; _save(stateFile, state);
+      continue;
+    }
+
+    // ── re-verify ─────────────────────────────────────────────────────────
+    // The gate. Without a verifier we cannot claim VERIFIED_FIXED, so we do
+    // not — an unverifiable patch is NEEDS_REVIEW even if it looks right.
+    if (typeof stages.verifyFix !== 'function') {
+      rec.outcome = 'NEEDS_REVIEW';
+      rec.reason = 'no verifyFix stage supplied — a patch that cannot be re-verified is never applied';
+      results.push(rec); state.findings[key] = rec; _save(stateFile, state);
+      continue;
+    }
+    const v = await stages.verifyFix(f, patch).catch(e => ({ ok: false, reason: String(e?.message || e) }));
+    rec.pocStillFires = v?.pocStillFires === true;
+    rec.testsPass = v?.testsPass !== false;
+
+    if (v?.ok && !rec.pocStillFires && rec.testsPass) {
+      rec.outcome = 'VERIFIED_FIXED';
+      if (apply && typeof stages.applyFix === 'function') {
+        try { await stages.applyFix(f, patch); rec.applied = true; }
+        catch (e) { rec.applied = false; rec.outcome = 'NEEDS_REVIEW'; rec.reason = `apply failed: ${e.message}`; }
+      } else {
+        rec.applied = false;
+        rec.reason = apply ? 'no applyFix stage supplied' : 'gates on: patch is ready but was not written';
+      }
+    } else {
+      rec.outcome = 'NEEDS_REVIEW';
+      rec.reason = rec.pocStillFires
+        ? 'the proof-of-concept still fires against the patch — the vulnerability is not fixed'
+        : (!rec.testsPass ? 'the project test suite fails with this patch'
+          : (v?.reason || 'the patch did not re-verify'));
+    }
+    results.push(rec);
+    state.findings[key] = rec;
+    _save(stateFile, state);
+    onStage({ stage: 'reverify', key, outcome: rec.outcome });
+  }
+
+  return { ok: true, results, skipped, outOfScope, capped, summary: summarizeAutopilot(results, outOfScope, capped) };
+}
+
+export function summarizeAutopilot(results, outOfScope = 0, capped = 0) {
+  const byOutcome = Object.fromEntries(OUTCOMES.map(o => [o, 0]));
+  for (const r of results) if (r.outcome in byOutcome) byOutcome[r.outcome]++;
+  return {
+    considered: results.length,
+    outOfScope,
+    capped,
+    byOutcome,
+    applied: results.filter(r => r.applied).length,
+  };
+}
+
+/** One line. Leads with what was NOT fixed, because that is the actionable part. */
+export function renderAutopilotSummary(s) {
+  if (!s) return null;
+  const b = s.byOutcome;
+  const bits = [`${s.considered} finding(s) in scope`];
+  if (b.VERIFIED_FIXED) bits.push(`${b.VERIFIED_FIXED} VERIFIED_FIXED (${s.applied} applied)`);
+  if (b.NEEDS_REVIEW) bits.push(`${b.NEEDS_REVIEW} NEEDS_REVIEW — not applied`);
+  if (b.NO_FIX) bits.push(`${b.NO_FIX} with no patch`);
+  if (b.UNPROVEN) bits.push(`${b.UNPROVEN} unproven (not dismissed — no PoC fired)`);
+  if (s.outOfScope) bits.push(`${s.outOfScope} below the severity floor and not considered`);
+  if (s.capped) bits.push(`${s.capped} in scope but NOT examined (per-run cap) — unexamined, not clean`);
+  return bits.join('; ') + '.';
+}
+
+export const _internals = { SCHEMA };

package/src/posture/comparison.js

@@ -0,0 +1,181 @@
+// PRD Epic 7.2 — head-to-head comparison scoring.
+//
+// WHY THIS SHIPS WITHOUT A SINGLE PARTICIPANT NAME IN IT. A benchmark whose
+// competitors are hard-coded by the vendor being measured is marketing with a
+// methodology section. This repository publishes the HARNESS and the answer
+// key; the operator supplies the participants. Nothing here — no constant, no
+// default config, no example — names any tool, and the report renders whatever
+// labels the operator chose. That is not a limitation working around a rule; a
+// comparison anyone can re-run against tools of their own choosing is the only
+// kind worth publishing, and the only kind a reader has reason to believe.
+//
+// THE ONE FAILURE MODE THIS MODULE EXISTS TO PREVENT. Two tools scored over
+// different subsets of a corpus are not comparable, and the difference is
+// invisible in the output: a tool that crashed on the 40 hardest entries and
+// was scored over the remaining 170 looks like it beat one that completed all
+// 210. So every rate here is computed over the INTERSECTION of entries every
+// participant completed, that intersection is reported alongside each
+// participant's own completion count, and a participant that completed nothing
+// in common with the others is refused rather than shown with an empty score.
+//
+// MATCHING IS CWE-ONLY, ON PURPOSE. Our own corpus entries carry a `vuln_match`
+// phrase in this engine's wording; scoring an external tool against our
+// phrasing would score it on vocabulary. CWE is the one identifier every
+// participant can be expected to emit, so it is the only key used, and it is
+// applied identically to every participant including this engine. A participant
+// that reports no CWE at all is scored as reporting nothing — stated in the
+// output rather than silently counted as a miss.
+
+/** Verdict for one participant on one corpus entry. */
+export const OUTCOMES = Object.freeze(['tp', 'fn', 'fp', 'tn']);
+
+function _cweSet(findings) {
+  const s = new Set();
+  for (const f of findings || []) {
+    const raw = f && (f.cwe ?? f.CWE ?? f.ruleId ?? '');
+    for (const m of String(raw).matchAll(/CWE[-_ ]?(\d+)/gi)) s.add(`CWE-${m[1]}`);
+  }
+  return s;
+}
+
+/**
+ * Score one participant over the entries it completed.
+ *
+ * @param {object[]} entries  [{id, cwe}]
+ * @param {object}   results  entryId -> {pre: findings[], post: findings[]} | {error}
+ */
+export function scoreParticipant(entries, results) {
+  const per = new Map();
+  let noCwe = 0;
+  for (const e of entries) {
+    const r = results?.[e.id];
+    if (!r || r.error || !Array.isArray(r.pre) || !Array.isArray(r.post)) continue;
+
+    const want = String(e.cwe || '').toUpperCase();
+    const pre = _cweSet(r.pre);
+    const post = _cweSet(r.post);
+    if (!pre.size && (r.pre || []).length) noCwe++;
+
+    // pre/ is the vulnerable tree: reporting the CWE is a true positive.
+    // post/ is the fixed tree: reporting it again is a false positive.
+    per.set(e.id, {
+      detected: pre.has(want),
+      falsePositive: post.has(want),
+    });
+  }
+  return { per, completed: per.size, noCwe };
+}
+
+function _rates(tp, fn, fp, tn) {
+  const precision = tp + fp > 0 ? tp / (tp + fp) : null;
+  const recall = tp + fn > 0 ? tp / (tp + fn) : null;
+  const f1 = precision !== null && recall !== null && precision + recall > 0
+    ? (2 * precision * recall) / (precision + recall) : null;
+  return { tp, fn, fp, tn, precision, recall, f1 };
+}
+
+/**
+ * Compare every participant over the entries ALL of them completed.
+ *
+ * @param {object[]} entries       [{id, cwe}]
+ * @param {object[]} participants  [{id, results}]
+ * @returns {object} {ok, reason?, intersection, scores[], skippedEntries[]}
+ */
+export function compareParticipants(entries, participants) {
+  if (!Array.isArray(entries) || !entries.length) return { ok: false, reason: 'no corpus entries' };
+  if (!Array.isArray(participants) || participants.length < 2) {
+    return { ok: false, reason: 'a comparison needs at least two participants' };
+  }
+
+  const scored = participants.map((p) => ({ ...p, ...scoreParticipant(entries, p.results) }));
+
+  // The intersection. This is the whole point: rates over anything else are
+  // rates over different exams.
+  let common = null;
+  for (const s of scored) {
+    const ids = new Set(s.per.keys());
+    common = common === null ? ids : new Set([...common].filter((id) => ids.has(id)));
+  }
+  if (!common || common.size === 0) {
+    return {
+      ok: false,
+      reason: 'no corpus entry was completed by every participant — there is nothing they can be compared on',
+      completion: Object.fromEntries(scored.map((s) => [s.id, s.completed])),
+    };
+  }
+
+  const scores = scored.map((s) => {
+    let tp = 0, fn = 0, fp = 0, tn = 0;
+    for (const id of common) {
+      const v = s.per.get(id);
+      if (v.detected) tp++; else fn++;
+      if (v.falsePositive) fp++; else tn++;
+    }
+    return {
+      id: s.id,
+      ...(_rates(tp, fn, fp, tn)),
+      completed: s.completed,
+      notCompleted: entries.length - s.completed,
+      noCwe: s.noCwe,
+    };
+  });
+
+  return {
+    ok: true,
+    corpusSize: entries.length,
+    intersection: common.size,
+    // Named so a reader can check the exam rather than trust the grade.
+    scoredEntryIds: [...common].sort(),
+    scores: scores.sort((a, b) => (b.f1 ?? -1) - (a.f1 ?? -1)),
+  };
+}
+
+const pct = (v) => (v === null || v === undefined ? 'n/a' : `${(v * 100).toFixed(1)}%`);
+
+/** Markdown. Discloses the exam before the grades, never after. */
+export function renderComparison(cmp) {
+  if (!cmp || !cmp.ok) {
+    return `# Comparison\n\nNOT SCORED: ${cmp?.reason || 'unknown reason'}\n`;
+  }
+  const out = [];
+  out.push('# Head-to-head comparison');
+  out.push('');
+  out.push(`Scored over the **${cmp.intersection} of ${cmp.corpusSize}** corpus entries that *every*`);
+  out.push('participant completed. Entries any participant failed to complete are excluded from');
+  out.push('every score, including this engine\'s — a rate computed over a different subset is a');
+  out.push('rate for a different exam.');
+  out.push('');
+  out.push('Matching is by CWE only. Participants report findings in their own vocabulary, so');
+  out.push('scoring against any one tool\'s phrasing would measure vocabulary rather than');
+  out.push('detection. The same rule is applied to every participant.');
+  out.push('');
+  out.push('| Participant | F1 | Precision | Recall | TP | FN | FP | Corpus completed |');
+  out.push('|---|---|---|---|---|---|---|---|');
+  for (const s of cmp.scores) {
+    out.push(`| ${s.id} | ${pct(s.f1)} | ${pct(s.precision)} | ${pct(s.recall)} | ${s.tp} | ${s.fn} | ${s.fp} | ${s.completed}/${cmp.corpusSize} |`);
+  }
+  out.push('');
+  const incomplete = cmp.scores.filter((s) => s.notCompleted > 0);
+  if (incomplete.length) {
+    out.push('## Entries not completed');
+    out.push('');
+    out.push('A participant that could not run on an entry is UNSCORED there, never scored as a');
+    out.push('miss. Counting a crash as a false negative would penalise a tool for a harness');
+    out.push('problem; counting it as a pass would reward it for one.');
+    out.push('');
+    for (const s of incomplete) out.push(`- **${s.id}** — ${s.notCompleted} entr(y/ies) not completed`);
+    out.push('');
+  }
+  const noCwe = cmp.scores.filter((s) => s.noCwe > 0);
+  if (noCwe.length) {
+    out.push('## Findings carrying no CWE');
+    out.push('');
+    for (const s of noCwe) {
+      out.push(`- **${s.id}** — ${s.noCwe} entr(y/ies) where findings were reported but none carried a CWE,`);
+      out.push('  so they could not be matched. This depresses that participant\'s recall for a');
+      out.push('  reporting-format reason rather than a detection one.');
+    }
+    out.push('');
+  }
+  return out.join('\n') + '\n';
+}