@clear-capabilities/agentic-security-scanner 0.134.0 → 0.136.2

package/src/posture/accuracy-scorecard.js

@@ -121,7 +121,14 @@ export function buildScorecard(inputs) {
       notGeneralRecall: true,
       notGeneralFalsePositiveRate: true,
       f1Emitted: false,
-      f1OmissionReason: 'no labelled real-world population is available to measure precision over, so any F1 would combine denominators that do not describe the same population',
+      // Was: "no labelled real-world population is available". That stopped
+      // being true when bench/independent/ was built, and a stale justification
+      // is worse than none — it argues against a measurement that now exists.
+      // No F1 is emitted FOR THE CURATED CORPUS, and the reason is narrower:
+      // its `post/` fixtures are authored here to be silent, so its precision
+      // denominator describes fixture design rather than engine behaviour.
+      // F1 over the independent population IS reported, in its own section.
+      f1OmissionReason: 'the curated corpus authors both its vulnerable and its fixed fixtures, so an F1 over it would measure fixture design, not accuracy; F1 over the labelled third-party population is reported separately from bench/independent/RESULT.json',
     },
     corpus: {
       measuredThisRun: true,
@@ -138,6 +145,17 @@ export function buildScorecard(inputs) {
       corpusBaseline: committed.corpusBaseline
         ? { source: 'bench/cve-replay/corpus-baseline.json', generatedAt: committed.corpusBaseline.generatedAt, total: committed.corpusBaseline.total, passing: committed.corpusBaseline.passing }
         : null,
+      independent: committed.independent
+        ? {
+          source: 'bench/independent/RESULT.json',
+          measuredAt: committed.independent.measuredAt || null,
+          engineVersion: committed.independent.engineVersion || null,
+          population: committed.independent.population || null,
+          overall: committed.independent.overall || null,
+          wide: committed.independent.wide || null,
+          byLanguage: committed.independent.byLanguage || null,
+        }
+        : null,
       proofCorpus: proof
         ? {
           source: 'bench/proof-corpus/results/summary.json',
@@ -307,6 +325,46 @@ export function renderScorecardMarkdown(m) {
   }
   L.push('Per-file counts are in `docs/scorecard.json`.');
   L.push('');
+  const ind = m.committedInputs.independent;
+  if (ind && ind.overall) {
+    L.push('## Independent evaluation population — the number that matters');
+    L.push('');
+    L.push('Everything above is a **regression net**: its fixtures and its labels are both');
+    L.push('written here, which is why its detection rate sits at the ceiling by');
+    L.push('construction. `bench/independent/` is the other instrument — real upstream code');
+    L.push('at the commit where a vulnerability really existed, with the CWE assigned by a');
+    L.push('public advisory database rather than by this project.');
+    L.push('');
+    L.push(`**Measured ${ind.measuredAt} on engine ${ind.engineVersion}, ` +
+      `n=${ind.population?.scoredEntries}, ${ind.population?.unscored} unscored** ` +
+      '(*committed artifact*, `' + ind.source + '` — read, not re-run: scoring takes ~32 minutes).');
+    L.push('');
+    L.push('| | Advisory-local (**the claim**) | Wide (diagnostic) |');
+    L.push('| --- | --- | --- |');
+    L.push(`| Precision | **${formatRate(ind.overall.precision?.n, ind.overall.precision?.d)}** | ${formatRate(ind.wide?.precision?.n, ind.wide?.precision?.d)} |`);
+    L.push(`| Recall | **${formatRate(ind.overall.recall?.n, ind.overall.recall?.d)}** | ${formatRate(ind.wide?.recall?.n, ind.wide?.recall?.d)} |`);
+    L.push(`| F1 | **${ind.overall.f1 === null || ind.overall.f1 === undefined ? 'n/a' : ind.overall.f1.toFixed(3)}** | ${ind.wide?.f1 === null || ind.wide?.f1 === undefined ? 'n/a' : ind.wide.f1.toFixed(3)} |`);
+    L.push('');
+    if (ind.byLanguage) {
+      L.push('| Language | n | Recall | Precision |');
+      L.push('| --- | --- | --- | --- |');
+      for (const [k, v] of Object.entries(ind.byLanguage)) {
+        L.push(`| ${k} | ${v.entries} | ${formatRate(v.recall?.n, v.recall?.d)} | ${formatRate(v.precision?.n, v.precision?.d)} |`);
+      }
+      L.push('');
+    }
+    L.push('**Quote the advisory-local column.** "Wide" scores the same scans without');
+    L.push('restricting findings to the files the advisory\'s fix commit touched — it asks');
+    L.push('only whether the CWE appeared *anywhere* in the package. Over scopes holding up');
+    L.push('to 1740 findings that is close to asking whether the codebase contains the bug');
+    L.push('class at all, a question with a much easier yes. It is kept because it is the');
+    L.push('only way to tell whether a change moved the engine or moved the benchmark.');
+    L.push('');
+    L.push('Against ~100% on the curated corpus above. **That gap is the most useful number');
+    L.push('in this document**, and publishing it is the point of the exercise. The figure');
+    L.push('went DOWN when the benchmark was corrected, and is published that way.');
+    L.push('');
+  }
   L.push('## Committed artifacts referenced (not re-run by this command)');
   L.push('');
   const cb = m.committedInputs.corpusBaseline;

package/src/posture/agents-memory.js

@@ -29,6 +29,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { statePath, stateWritesEnabled } from './state-dir.js';
 const MEMORY_FILE = '.agentic-security/AGENTS.md';
 const ARCHIVE_FILE = '.agentic-security/AGENTS.md.archive';
 const MAX_BYTES = 20 * 1024;
@@ -36,8 +37,8 @@ const MAX_ENTRY_BYTES = 2 * 1024;
 const ARCHIVE_MAX_BYTES = 200 * 1024;
 const HEADER = '# AGENTS.md\n\nAgent-authored continual-learning notes. Each entry: timestamp + agent name + one short paragraph. New entries appended at the bottom; oldest entries rotate to AGENTS.md.archive when this file exceeds 20 KB.\n\n';
 
-function _resolve(scanRoot) { return path.join(scanRoot, MEMORY_FILE); }
-function _archivePath(scanRoot) { return path.join(scanRoot, ARCHIVE_FILE); }
+function _resolve(scanRoot) { return statePath(scanRoot, 'AGENTS.md'); }
+function _archivePath(scanRoot) { return statePath(scanRoot, 'AGENTS.md.archive'); }
 
 export function readAgentsMemory(scanRoot) {
   const fp = _resolve(scanRoot);
@@ -65,7 +66,8 @@ export function appendAgentsMemory(scanRoot, { agent, body }) {
   const entry = `\n## ${ts}  agent: ${agent}\n\n${snippet}\n`;
   try {
     const fp = _resolve(scanRoot);
-    fs.mkdirSync(path.dirname(fp), { recursive: true });
+    if (!stateWritesEnabled()) return;
+  fs.mkdirSync(path.dirname(fp), { recursive: true });
     if (!fs.existsSync(fp)) fs.writeFileSync(fp, HEADER);
     fs.appendFileSync(fp, entry);
     _maybeRotate(scanRoot);

package/src/posture/auditor-walkthrough.js

@@ -8,6 +8,8 @@
 //
 //   nist-csf-2          NIST Cybersecurity Framework 2.0
 //   nist-ai-600-1       NIST AI Risk Management Framework, GenAI profile
+//   nist-privacy-1-1    NIST Privacy Framework 1.1 (see posture/privacy-framework.js
+//                       for the assessment + remediation layer over it)
 //   owasp-asvs-5        OWASP Application Security Verification Standard 5.0
 //   owasp-llm-top-10    OWASP Top 10 for LLM Applications 2025
 //   eu-ai-act           EU AI Act (Regulation 2024/1689)
@@ -32,9 +34,8 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { statePath, stateWritesEnabled } from './state-dir.js';
 const BUNDLED_DIR = path.join(path.dirname(new URL(import.meta.url).pathname), 'compliance-frameworks');
-const STATE = '.agentic-security';
-
 function _readJson(fp) {
   try { return JSON.parse(fs.readFileSync(fp, 'utf8')); } catch { return null; }
 }
@@ -52,7 +53,7 @@ export function listFrameworks(scanRoot) {
     }
   } catch {}
   if (scanRoot) {
-    const projDir = path.join(scanRoot, STATE, 'compliance');
+    const projDir = statePath(scanRoot, 'compliance');
     if (fs.existsSync(projDir)) {
       try {
         for (const sub of fs.readdirSync(projDir)) {
@@ -73,7 +74,7 @@ export function listFrameworks(scanRoot) {
  */
 export function loadFramework(scanRoot, id) {
   if (scanRoot) {
-    const projFp = path.join(scanRoot, STATE, 'compliance', id, 'controls.json');
+    const projFp = statePath(scanRoot, 'compliance', id, 'controls.json');
     if (fs.existsSync(projFp)) return _readJson(projFp);
   }
   for (const fn of fs.readdirSync(BUNDLED_DIR)) {
@@ -164,11 +165,11 @@ export function evaluateFramework(scanRoot, fw, scan) {
         // A '.../' sentinel marks a source-relative artifact (project source,
         // e.g. a hook or agent file) — resolve it against the scan root itself.
         // Everything else is a runtime artifact under the STATE dir. Without
-        // this, `path.join(scanRoot, STATE, '.../x')` never resolves and the
+        // this, `statePath(scanRoot, '.../x')` never resolves and the
         // control falsely reads "not present" for every project.
         const resolved = !target ? null
           : target.startsWith('.../') ? path.join(scanRoot, target.slice(4))
-          : path.join(scanRoot, STATE, target);
+          : statePath(scanRoot, target);
         const label = target ? target.replace(/^\.\.\.\//, '') : '(unmapped)';
         if (resolved && fs.existsSync(resolved)) {
           obs.push(`✓ ${mod}: ${label} present.`);
@@ -251,7 +252,8 @@ export function renderWalkthrough(fw, evaluation, opts = {}) {
  * Persist the walkthrough at .agentic-security/auditor-walkthroughs/<id>.md
  */
 export function persistWalkthrough(scanRoot, fw, body) {
-  const dir = path.join(scanRoot, STATE, 'auditor-walkthroughs');
+  const dir = statePath(scanRoot, 'auditor-walkthroughs');
+  if (!stateWritesEnabled()) return null;
   try { fs.mkdirSync(dir, { recursive: true }); } catch {}
   const fp = path.join(dir, `${fw.id}.md`);
   try { fs.writeFileSync(fp, body); } catch {}

package/src/posture/auth-posture-import.js

@@ -27,6 +27,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 
+import { statePath } from './state-dir.js';
 const KNOWN_GOOD = new Set([
   'session+csrf', 'session+admin', 'session+claim', 'session',
   'jwt+verify', 'oauth+pkce', 'mtls',
@@ -41,15 +42,15 @@ const FAMILIES_GATEABLE_BY_AUTH = new Set([
 ]);
 
 const CANDIDATE_PATHS = [
-  '.agentic-security/auth-posture.json',
-  '.agentic-security/auth-posture.yml',
-  '.agentic-security/auth-posture.yaml',
+  'auth-posture.json',
+  'auth-posture.yml',
+  'auth-posture.yaml',
 ];
 
 export function loadAuthPosture(scanRoot) {
   const root = scanRoot || process.cwd();
   for (const rel of CANDIDATE_PATHS) {
-    const fp = path.join(root, rel);
+    const fp = statePath(root, rel);
     if (!fs.existsSync(fp)) continue;
     try {
       const text = fs.readFileSync(fp, 'utf8');