@clear-capabilities/agentic-security-scanner 0.195.26 → 0.195.27

package/src/posture/rule-compiler/regression.js

@@ -15,18 +15,22 @@ import { statePath } from '../state-dir.js';
 import { buildCorpus, scoreRule } from './corpus.js';
 import { evaluateRule } from './promotion.js';
 import { runRule, normalizeCustomRule } from '../custom-rules.js';
+import { createHash } from 'node:crypto';
 import { compileToCustomRule } from './ir.js';
 import { loadIR, loadCorpus, loadProvenance } from './store.js';
 import { stateOf } from './lifecycle.js';
+import { SHIPPED_NEGATIVE_CORPUS } from './negative-corpus.js';
 
 const FP_DIR = 'rule-compiler/fp-corpus';
 const MAX_FP_FILE = 256 * 1024;
 const MAX_FP_FILES = 500;
 const EXT_OF = { '.js': 'javascript', '.mjs': 'javascript', '.cjs': 'javascript', '.ts': 'typescript', '.py': 'python', '.go': 'go', '.rb': 'ruby', '.java': 'java', '.rs': 'rust', '.cs': 'csharp', '.php': 'php' };
 
-/** Operator-supplied false positives (real code a rule should never flag). Plain files; symlinks and oversized files are ignored. */
-function operatorFalsePositives(root) {
-  const dir = statePath(root, ...FP_DIR.split('/'));
+// GA-07 §3 (0.195.27): one or more directories of safe code shared ACROSS repositories (a fleet's common false-positive corpus),
+// named by path-separator-joined entries in this variable. Same rules as the project directory: plain files, size and count caps.
+const SHARED_FP_ENV = 'AGENTIC_SECURITY_RULES_FP_CORPUS';
+
+function falsePositiveFiles(dir, prefix) {
   const out = [];
   let names = [];
   try { names = fs.readdirSync(dir).sort(); } catch { return out; }
@@ -35,19 +39,32 @@ function operatorFalsePositives(root) {
     try {
       const st = fs.lstatSync(f);
       if (!st.isFile() || st.isSymbolicLink() || st.size > MAX_FP_FILE) continue;
-      out.push({ label: `operator-fp:${n}`, file: n, language: EXT_OF[path.extname(n).toLowerCase()] ?? 'any', code: fs.readFileSync(f, 'utf8') });
+      out.push({ label: `${prefix}:${n}`, file: n, language: EXT_OF[path.extname(n).toLowerCase()] ?? 'any', code: fs.readFileSync(f, 'utf8') });
     } catch { /* unreadable: skipped */ }
   }
   return out;
 }
 
+/** Operator-supplied false positives (real code a rule should never flag): the project's own directory, then every shared directory. */
+function operatorFalsePositives(root, env = process.env) {
+  const out = falsePositiveFiles(statePath(root, ...FP_DIR.split('/')), 'operator-fp');
+  for (const dir of String(env[SHARED_FP_ENV] || '').split(path.delimiter).map((s) => s.trim()).filter(Boolean)) {
+    out.push(...falsePositiveFiles(path.resolve(dir), `shared-fp:${path.basename(dir)}`));
+  }
+  return out;
+}
+
 export function promotedRuleIds(ledger) {
   return [...new Set(ledger.entries.map((e) => e.ruleId))].filter((id) => stateOf(ledger, id) === 'PROMOTED');
 }
 
-/** The shared false-positive corpus as seen by `exceptId`: the safe code of every OTHER promoted rule, plus the operator's files. */
-export function sharedFalsePositiveCorpus(root, ledger, exceptId = null) {
-  const items = [];
+/**
+ * The shared false-positive corpus as seen by `exceptId`: the safe code shipped with the scanner (`negative-corpus.js`, the same on
+ * every machine and in every repository), the safe code of every OTHER promoted rule, the operator's project files, and any
+ * directories shared across repositories through AGENTIC_SECURITY_RULES_FP_CORPUS.
+ */
+export function sharedFalsePositiveCorpus(root, ledger, exceptId = null, env = process.env) {
+  const items = SHIPPED_NEGATIVE_CORPUS.map((s) => ({ label: `shipped-fp:${s.name}`, language: s.language, code: s.code }));
   for (const id of promotedRuleIds(ledger)) {
     if (id === exceptId) continue;
     const ir = loadIR(root, id); const spec = loadCorpus(root, id);
@@ -55,7 +72,12 @@ export function sharedFalsePositiveCorpus(root, ledger, exceptId = null) {
     let corpus; try { corpus = buildCorpus(ir, spec); } catch { continue; }
     for (const n of corpus.negatives) items.push({ label: `rule:${id}:${n.label}`, language: ir.languages[0] ?? 'any', code: n.code });
   }
-  return [...items, ...operatorFalsePositives(root)];
+  return [...items, ...operatorFalsePositives(root, env)];
+}
+
+/** The digest of a corpus half, as `rules evaluate` records it in the provenance record and `rules regress` re-derives it. */
+export function corpusHash(list) {
+  return createHash('sha256').update(JSON.stringify(list.map((x) => [x.label, x.code]).sort())).digest('hex');
 }
 
 const FILE_FOR_LANG = { javascript: 'x.js', typescript: 'x.ts', python: 'x.py', go: 'x.go', ruby: 'x.rb', java: 'x.java', rust: 'x.rs', csharp: 'x.cs', php: 'x.php', any: 'x.txt' };
@@ -85,8 +107,16 @@ export function regressRules(root, ledger, { only = null, thresholds } = {}) {
     let ev; let corpus;
     try { corpus = buildCorpus(ir, spec); ev = evaluateRule(ir, corpus, thresholds); } catch (e) { results.push({ id, status: 'broken', reasons: [`cannot be scored: ${String(e.message).slice(0, 120)}`] }); continue; }
     const fp = falsePositiveHits(ir, sharedFalsePositiveCorpus(root, ledger, id));
-    const baseline = loadProvenance(root, id)?.benchmarkResults ?? null;
+    const prov = loadProvenance(root, id);
+    const baseline = prov?.benchmarkResults ?? null;
     const reasons = [...ev.reasons];
+    // GA-07 §3 (0.195.27): the promotion was a review of ONE rule against ONE corpus, both digested into the promotion event. A
+    // corpus edited after the review (an independent example dropped, a protected variant softened) can still clear the gate, so
+    // the gate alone would not notice; the digests do. Such a rule is `review-stale`: not regressed, but no longer the reviewed one.
+    const reviewed = prov?.promotionEvent?.evidence ?? null;
+    if (reviewed) {
+      if (reviewed.positiveCorpusHash !== corpusHash(corpus.positives) || reviewed.negativeCorpusHash !== corpusHash(corpus.negatives)) reasons.push('the corpus changed since the reviewed promotion (its digests no longer match the promotion event); re-evaluate and promote again');
+    }
     if (fp.length) reasons.push(`fires on ${fp.length} item(s) of the shared false-positive corpus: ${fp.slice(0, 3).map((h) => h.label).join(', ')}${fp.length > 3 ? ', ...' : ''}`);
     if (baseline && ev.score.recall != null && baseline.recall != null && ev.score.recall < baseline.recall) reasons.push(`recall fell from ${(baseline.recall * 100).toFixed(1)}% to ${(ev.score.recall * 100).toFixed(1)}% since promotion`);
     results.push({ id, status: reasons.length ? 'regressed' : 'ok', reasons, precision: ev.score.precision, recall: ev.score.recall, protectedFailures: ev.score.protectedFailures, fpCorpusHits: fp.length });

package/src/posture/rule-compiler/store.js

@@ -5,6 +5,7 @@
 // using the SAME filename convention `runRuleTests` already reads (`vulnerable`/`clean` in the path).
 import fs from 'node:fs';
 import path from 'node:path';
+import crypto from 'node:crypto';
 import { statePath, ensureStateDir, safeWriteState } from '../state-dir.js';
 import { newLedger, verifyLedger } from './lifecycle.js';
 import { signLastScan, verifyLastScan } from '../integrity.js';
@@ -35,20 +36,85 @@ export function loadLedger(scanRoot, env = process.env) {
   const v = verifyLedger(ledger);
   if (!v.ok) return { ...ledger, tampered: true, tamperReason: v.reason };
   const sig = verifyLastScan(text, `${f}.sig`);
-  if (sig === false) return { ...ledger, tampered: true, tamperReason: 'the ledger signature does not verify under this install key' };
+  if (sig === true) return { ...ledger, verifiedBy: 'install-key' };
+  // GA-07 §3 (0.195.27): the install key never leaves the machine that promoted the rule, so on a CI runner (or any other
+  // clone) the HMAC cannot verify. A ledger that `promote --sign-key` also signed with the operator's Ed25519 key is accepted
+  // when that signature verifies against a key the environment trusts: AGENTIC_SECURITY_RULES_PUBLIC_KEY (set outside the
+  // repository, a CI variable) or the trusted-keys store the rule packs already use. The chain check above still applies, and
+  // the signature covers the whole file, so a dropped last entry fails it like any other edit. A project-local key file alone
+  // is not trust (an author who can write .agentic-security/ could add one), which is why the env variable is the CI route.
+  const author = verifyAuthorSignature(text, `${f}${AUTHOR_SIG_SUFFIX}`, authorKeysFrom(scanRoot, env));
+  if (author.ok) return { ...ledger, verifiedBy: 'author-key', authorKeyId: author.keyId };
+  if (sig === false) return { ...ledger, tampered: true, tamperReason: `the ledger signature does not verify under this install key${author.present ? ` and its author signature does not verify against any trusted key (${author.reason})` : ' and it carries no author signature (promote with --sign-key to make it portable)'}` };
   if (sig === null && ledger.entries.length && env.AGENTIC_SECURITY_RULES_UNSIGNED !== '1') {
     return { ...ledger, tampered: true, tamperReason: 'the ledger is unsigned (written before 0.195.23 or by another tool); set AGENTIC_SECURITY_RULES_UNSIGNED=1 once to accept and re-sign it' };
   }
-  return ledger;
+  return { ...ledger, verifiedBy: sig === null ? 'unsigned-accepted' : 'install-key' };
 }
 
-export function saveLedger(scanRoot, ledger) {
+const AUTHOR_SIG_SUFFIX = '.author.sig';
+const PUBLIC_KEY_ENV = 'AGENTIC_SECURITY_RULES_PUBLIC_KEY';
+
+/** The Ed25519 public keys a ledger's author signature may verify against: the environment variable (one or more base64 raw keys,
+ * comma separated) first, then the rule-pack trust store (bundled keys, and the project file only when the operator honours it). */
+function authorKeysFrom(scanRoot, env) {
+  const keys = [];
+  for (const k of String(env[PUBLIC_KEY_ENV] || '').split(',').map((s) => s.trim()).filter(Boolean)) keys.push({ id: 'env', publicKey: k, alg: 'ed25519' });
+  const origErr = console.error; console.error = () => {};
+  try { keys.push(...loadTrustedKeys(scanRoot)); } catch { /* no trust store */ } finally { console.error = origErr; }
+  return keys;
+}
+
+function verifyAuthorSignature(text, sigPath, keys) {
+  let sig;
+  try { sig = fs.readFileSync(sigPath); } catch { return { ok: false, present: false, reason: 'no author signature' }; }
+  if (!keys.length) return { ok: false, present: true, reason: `no trusted key (set ${PUBLIC_KEY_ENV})` };
+  const body = Buffer.from(text, 'utf8');
+  for (const k of keys) {
+    try {
+      const keyBytes = Buffer.from(k.publicKey, 'base64');
+      if (keyBytes.length !== 32) continue;
+      const keyObj = crypto.createPublicKey({ key: { kty: 'OKP', crv: 'Ed25519', x: keyBytes.toString('base64url') }, format: 'jwk' });
+      if (crypto.verify(null, body, keyObj, sig)) return { ok: true, present: true, keyId: k.id || '(unnamed)' };
+    } catch { /* next key */ }
+  }
+  return { ok: false, present: true, reason: 'bad signature' };
+}
+
+function signWithAuthorKey(text, privateKeyB64) {
+  const privBytes = Buffer.from(privateKeyB64, 'base64');
+  if (privBytes.length !== 32) throw new Error('private key must be 32 bytes (raw ed25519)');
+  const keyObj = crypto.createPrivateKey({ key: Buffer.concat([Buffer.from('302e020100300506032b657004220420', 'hex'), privBytes]), format: 'der', type: 'pkcs8' });
+  return crypto.sign(null, Buffer.from(text, 'utf8'), keyObj);
+}
+
+/** The public key (base64 raw) that pairs with a raw Ed25519 private key; printed by `promote` so the operator can set it in CI. */
+export function publicKeyOf(privateKeyB64) {
+  const privBytes = Buffer.from(privateKeyB64, 'base64');
+  if (privBytes.length !== 32) return null;
+  const keyObj = crypto.createPrivateKey({ key: Buffer.concat([Buffer.from('302e020100300506032b657004220420', 'hex'), privBytes]), format: 'der', type: 'pkcs8' });
+  return Buffer.from(crypto.createPublicKey(keyObj).export({ format: 'jwk' }).x, 'base64url').toString('base64');
+}
+
+/**
+ * Write the ledger with its install-key signature and, when the operator's Ed25519 key is given, an author signature beside it
+ * (`ledger.json.author.sig`) that another machine can verify (see loadLedger). Without a key an existing author signature is
+ * removed rather than left stale: a signature that no longer matches the file would read as tampering on the next clone.
+ */
+export function saveLedger(scanRoot, ledger, { authorKey = null } = {}) {
   if (ledger?.tampered) return { ok: false, reason: `refusing to overwrite a ledger that failed its integrity check (${ledger.tamperReason})` };
   if (!ensureStateDir(scanRoot)) return { ok: false, reason: 'no safe state directory for this project' };
   const f = statePath(scanRoot, DIR, 'ledger.json');
-  const text = JSON.stringify(ledger, null, 2);
-  const ok = safeWriteState(f, text) && safeWriteState(`${f}.sig`, signLastScan(text));
-  return { ok };
+  const { verifiedBy, authorKeyId, ...body } = ledger;
+  const text = JSON.stringify(body, null, 2);
+  let ok = safeWriteState(f, text) && safeWriteState(`${f}.sig`, signLastScan(text));
+  let authorSigned = false;
+  if (ok && authorKey) {
+    try { ok = safeWriteState(`${f}${AUTHOR_SIG_SUFFIX}`, signWithAuthorKey(text, authorKey)); authorSigned = ok; } catch (e) { return { ok: false, reason: `cannot author-sign the ledger: ${String(e.message).slice(0, 120)}` }; }
+  } else if (ok) {
+    try { fs.unlinkSync(`${f}${AUTHOR_SIG_SUFFIX}`); } catch { /* none to remove */ }
+  }
+  return { ok, authorSigned };
 }
 
 export function saveIR(scanRoot, ir) {