@clear-capabilities/agentic-security-scanner 0.143.0 → 0.145.0

package/src/posture/calibration-feedback.js

@@ -0,0 +1,201 @@
+// FR-806 (assurance-hardening PRD): "Validate model calibration against
+// accepted and realized incidents where customers opt in | Calibration
+// reports are aggregated and privacy-preserving."
+//
+// SCOPE, stated explicitly because the PRD's own wording is terse: this
+// codebase has no SaaS control plane and takes no runtime cloud calls
+// (root CLAUDE.md's own "No runtime cloud calls" rule, and E10's own goal
+// of "consistent policy... without requiring a SaaS control plane") — so
+// "aggregated" here means aggregated WITHIN one installation, across every
+// feedback event an operator has recorded over time, never aggregated
+// ACROSS installations on some central server. An operator who wants a
+// cross-organization rollup can feed this module's own report output into
+// their own aggregation, the same way `fleet.js` composes many single-repo
+// scans without a hosted backend.
+//
+// TWO OUTCOMES, matching the PRD's own two named cases:
+//   'accepted-risk'    — an operator/customer reviewed a finding, accepted
+//                        the risk, and (later, of their own accord) reports
+//                        that no incident occurred. A well-calibrated model
+//                        should have predicted LOW confidence/risk for
+//                        these.
+//   'realized-incident' — an operator reports that a finding's
+//                        vulnerability WAS actually exploited or otherwise
+//                        caused a real incident. A well-calibrated model
+//                        should have predicted HIGH confidence/risk for
+//                        these — a realized incident on a LOW-predicted
+//                        finding is exactly the miscalibration this
+//                        requirement exists to surface.
+//
+// PRIVACY-PRESERVING AT THE SOURCE, not just at the report layer: a
+// feedback record snapshots only the model's OWN prediction signals
+// (confidence, severity, riskDollars.ev) plus the operator's outcome and
+// optional free-text note — never file path, line, vuln title, or code
+// snippet. This mirrors this codebase's existing privacy modules' own
+// discipline (dataflow/privacy-*.js) of never persisting more than a
+// report needs to answer its one question.
+//
+// OPT-IN, genuinely: nothing here is ever auto-populated by a scan. A
+// record exists only when an operator explicitly calls
+// `recordCalibrationFeedback` (via the CLI's `calibration-feedback record`
+// command) — the file simply does not exist for every project that never
+// opts in, and every read degrades to "no data" rather than throwing.
+
+import * as fs from 'node:fs';
+import * as crypto from 'node:crypto';
+import { statePath, stateDir, isSafeStateDir, stateWritesEnabled } from './state-dir.js';
+
+export const CALIBRATION_FEEDBACK_FILE = 'calibration-feedback.jsonl';
+export const OUTCOMES = Object.freeze(['accepted-risk', 'realized-incident']);
+
+// Below this many samples, a rate is an artifact of the sample, not a
+// property of the model — same precedent as fix-metrics.js's RELIABLE_N.
+const RELIABLE_N = 10;
+
+function _findFinding(scanRoot, findingId) {
+  try {
+    const raw = JSON.parse(fs.readFileSync(statePath(scanRoot, 'last-scan.json'), 'utf8'));
+    const findings = Array.isArray(raw.findings) ? raw.findings : [];
+    return findings.find(f => f && (f.id === findingId || f.stableId === findingId)) || null;
+  } catch { return null; }
+}
+
+// FR-806 privacy fix: the caller-supplied findingId is routinely a finding's
+// plain `.id` (e.g. "client-side:DANGEROUS_INNERHTML:src/billing/secret.js:142"),
+// which embeds the exact file path and line this module's own docstring
+// promises never to persist. Never write the caller's raw string to disk:
+// prefer the matched finding's own privacy-safe `.stableId` (a hash, by
+// construction elsewhere in this codebase), and otherwise hash the input
+// ourselves so a stale/removed finding's id can never leak path/line either.
+function _privacySafeFindingId(findingId, finding) {
+  if (finding && typeof finding.stableId === 'string' && finding.stableId) return finding.stableId;
+  return crypto.createHash('sha256').update(findingId).digest('hex').slice(0, 16);
+}
+
+/**
+ * Record one opt-in calibration-feedback event. Snapshots ONLY the
+ * model's own prediction signals for the named finding (if it is still
+ * present in the last scan — a finding fixed/removed since is still
+ * recordable, just without a fresh snapshot) plus the outcome and an
+ * optional note. Never throws; returns {ok, record} or {ok:false, reason}.
+ */
+export function recordCalibrationFeedback(scanRoot, { findingId, outcome, note } = {}) {
+  if (!findingId || typeof findingId !== 'string') return { ok: false, reason: '--finding-id is required' };
+  if (!OUTCOMES.includes(outcome)) return { ok: false, reason: `--outcome must be one of: ${OUTCOMES.join(', ')}` };
+  const finding = _findFinding(scanRoot, findingId);
+  const record = {
+    at: new Date().toISOString(),
+    findingId: _privacySafeFindingId(findingId, finding),
+    outcome,
+    predictedConfidence: finding && typeof finding.confidence === 'number' ? finding.confidence : null,
+    predictedConfidenceTier: finding?.confidenceTier || null,
+    predictedSeverity: finding?.severity || null,
+    predictedRiskEv: finding?.riskDollars && typeof finding.riskDollars.ev === 'number' ? finding.riskDollars.ev : null,
+    note: note ? String(note).slice(0, 280) : null,
+  };
+  // Append-only, same primitives as fix-metrics.js/triage-memory.js's own
+  // JSONL writers — a single fs.appendFileSync, never a read-modify-write
+  // of the whole file (which would also reintroduce a TOCTOU between an
+  // existence check and the write, the exact anti-pattern this codebase's
+  // own conventions forbid).
+  const dir = stateDir(scanRoot);
+  if (!isSafeStateDir(dir)) return { ok: false, reason: 'no safe state directory' };
+  if (!stateWritesEnabled()) return { ok: false, reason: 'state writes are disabled (--no-state)' };
+  try {
+    fs.mkdirSync(dir, { recursive: true });
+    fs.appendFileSync(statePath(scanRoot, CALIBRATION_FEEDBACK_FILE), JSON.stringify(record) + '\n', 'utf8');
+  } catch (e) { return { ok: false, reason: e.message }; }
+  return { ok: true, record };
+}
+
+/**
+ * Every well-formed feedback event ever recorded. A line that fails to
+ * parse or lacks a valid outcome is skipped, not fatal.
+ */
+export function loadCalibrationFeedback(scanRoot) {
+  let fp;
+  try { fp = statePath(scanRoot, CALIBRATION_FEEDBACK_FILE); } catch { return []; }
+  let raw;
+  try { raw = fs.readFileSync(fp, 'utf8'); } catch { return []; }
+  const out = [];
+  for (const line of raw.split('\n')) {
+    if (!line.trim()) continue;
+    try {
+      const rec = JSON.parse(line);
+      if (rec && OUTCOMES.includes(rec.outcome)) out.push(rec);
+    } catch { /* torn or hand-edited line — drop it, keep the rest */ }
+  }
+  return out;
+}
+
+function _avg(nums) {
+  const v = nums.filter(n => typeof n === 'number' && Number.isFinite(n));
+  return v.length ? v.reduce((a, b) => a + b, 0) / v.length : null;
+}
+
+function _summarizeOutcome(records) {
+  const withConfidence = records.map(r => r.predictedConfidence).filter(c => typeof c === 'number');
+  return {
+    n: records.length,
+    reliable: records.length >= RELIABLE_N,
+    avgPredictedConfidence: _avg(withConfidence),
+    withoutPrediction: records.length - withConfidence.length,
+  };
+}
+
+/**
+ * Aggregate ALL recorded feedback (this installation only — see the
+ * module header for why cross-installation aggregation is out of scope)
+ * into a privacy-preserving report: rates and averages only, never a
+ * per-finding breakdown, never file/line/vuln text (none of that was ever
+ * stored in the first place).
+ */
+export function buildCalibrationReport(scanRoot) {
+  const records = loadCalibrationFeedback(scanRoot);
+  const acceptedRisk = records.filter(r => r.outcome === 'accepted-risk');
+  const realizedIncident = records.filter(r => r.outcome === 'realized-incident');
+  return {
+    schema: 'agentic-security/calibration-report@1',
+    generatedAt: new Date().toISOString(),
+    totalEvents: records.length,
+    acceptedRisk: _summarizeOutcome(acceptedRisk),
+    realizedIncident: _summarizeOutcome(realizedIncident),
+    // The calibration question itself: accepted-risk events SHOULD skew
+    // toward low predicted confidence; realized-incident events SHOULD
+    // skew toward high. This flag is a coarse, disclosed-uncertainty
+    // signal, not a verdict — it only ever fires when BOTH buckets have
+    // enough samples to say anything at all (RELIABLE_N each).
+    possibleMiscalibration: (() => {
+      const a = _summarizeOutcome(acceptedRisk);
+      const r = _summarizeOutcome(realizedIncident);
+      if (!a.reliable || !r.reliable || a.avgPredictedConfidence == null || r.avgPredictedConfidence == null) return null;
+      return r.avgPredictedConfidence <= a.avgPredictedConfidence;
+    })(),
+  };
+}
+
+/**
+ * One block of human-readable summary, or null when nothing has ever been
+ * recorded — genuinely opt-in, so "nothing recorded" is the expected
+ * default state for almost every project, not an error.
+ */
+export function renderCalibrationReportSummary(report) {
+  if (!report || report.totalEvents === 0) return null;
+  const lines = [
+    'Calibration feedback (this installation, opt-in):',
+    `  accepted-risk:     n=${report.acceptedRisk.n}${report.acceptedRisk.reliable ? '' : ' (below reliable sample size)'}` +
+      (report.acceptedRisk.avgPredictedConfidence != null ? `  avg predicted confidence=${report.acceptedRisk.avgPredictedConfidence.toFixed(2)}` : ''),
+    `  realized-incident: n=${report.realizedIncident.n}${report.realizedIncident.reliable ? '' : ' (below reliable sample size)'}` +
+      (report.realizedIncident.avgPredictedConfidence != null ? `  avg predicted confidence=${report.realizedIncident.avgPredictedConfidence.toFixed(2)}` : ''),
+  ];
+  if (report.possibleMiscalibration === true) {
+    lines.push('  ⚠ realized incidents were NOT predicted with higher confidence than accepted risks — possible miscalibration.');
+  } else if (report.possibleMiscalibration === false) {
+    lines.push('  ✓ realized incidents were predicted with higher confidence than accepted risks, as expected.');
+  } else {
+    lines.push('  (not enough samples in both buckets yet to assess calibration — this is not a pass or fail.)');
+  }
+  return lines.join('\n');
+}
+
+export const _internals = { RELIABLE_N, _findFinding };

package/src/posture/calibration-seed.json

@@ -1,6 +1,8 @@
 {
   "_doc": "Seed calibration corpus for P1.3 / FR-UX-1. Each entry is (family, tp, fp) from running the engine against labeled benchmarks. The runtime merges this with the customer's .agentic-security/validator-metrics.json. Customer counts override when their n is higher.",
   "_source": "OWASP Benchmark v1.2 (Java) + Juliet Java + Juliet C/C++ + synthetic-bench fixtures + curated NodeGoat — counts as-of v0.48.0. ~30 samples per family minimum for calibrated emit.",
+  "_generatedAt": "2026-05-18T19:15:00-07:00",
+  "_generatedAtNote": "FR-207: the actual git commit timestamp of this file (git log -1 --format=%aI), recorded here as a machine-readable field so posture/calibration.js's calibrationFreshness() can compute a real age instead of inferring one from the '_source' version string. Update this value only when the family counts below actually change.",
   "_caveat": "This is a SEED corpus, not a held-out test set. The PRD G1 target (Brier ≤ 0.10) requires a separate held-out labeled set; that work is queued for Phase 5 finalization. Calibrated values shipped now are honest empirical TP rates from this seed, with their Wilson 95% CI and N visible so consumers can judge.",
   "families": {
     "sql-injection": { "tp": 41, "fp": 3 },

package/src/posture/calibration.js

@@ -102,6 +102,31 @@ function _readJsonMaybe(fp) {
 // Load history from .agentic-security/validator-metrics.json + the bundled
 // seed file. The bundled seed ships with this release; the customer file
 // overrides per-family when N is higher there.
+// FR-207: freshness of the seed calibration table itself. This is
+// maintainer-authored data (not a per-customer opt-in like
+// compliance-policy.js's `_staleness`), so the check is always-on against
+// one fixed threshold rather than an opt-in interval — there is no
+// owner/reviewer workflow for it to key off. `_generatedAt` missing (an
+// old seed file predating this field) is treated the same as this
+// codebase's other "never dated == already stale" cases, not a free pass.
+const CALIBRATION_MAX_AGE_MS = 180 * 24 * 60 * 60 * 1000;
+let _calibrationFreshnessCache = null;
+export function calibrationFreshness() {
+  if (_calibrationFreshnessCache) return _calibrationFreshnessCache;
+  const seedPath = new URL('./calibration-seed.json', import.meta.url);
+  let generatedAt = null;
+  try { generatedAt = JSON.parse(fs.readFileSync(seedPath, 'utf8'))._generatedAt || null; } catch { /* unreadable seed -> unknown, not fabricated */ }
+  const ts = generatedAt ? Date.parse(generatedAt) : NaN;
+  const baseline = Number.isFinite(ts) ? ts : 0;
+  const ageMs = Date.now() - baseline;
+  _calibrationFreshnessCache = {
+    generatedAt,
+    ageDays: Math.floor(ageMs / 86400000),
+    stale: ageMs > CALIBRATION_MAX_AGE_MS,
+  };
+  return _calibrationFreshnessCache;
+}
+
 export function loadCalibrationHistory(scanRoot) {
   const customer = _readJsonMaybe(statePath(scanRoot, 'validator-metrics.json')) || {};
   const seedPath = new URL('./calibration-seed.json', import.meta.url);

package/src/posture/compliance-evidence-signing.js

@@ -0,0 +1,131 @@
+// Sign compliance evidence manifests (assurance-hardening PRD FR-505).
+//
+// "Sign evidence manifests when signing is configured | Signature
+// verification detects altered findings, scope, policy, or evidence
+// references."
+//
+// SAME TRUST DOMAIN AS evidence-bundle.js, DELIBERATELY REUSING ITS KEY
+// --------------------------------------------------------------------------
+// A compliance evidence manifest and a finding evidence bundle make the
+// SAME kind of claim: "this artifact is exactly what MY install produced,
+// unaltered since." Both are self-attestation by the running install, not
+// a claim authored by a third party and distributed for others to verify
+// (that is policy-bundle.js's job, and why IT needed a separate key — see
+// D-0026). Reusing evidence-bundle.js's `ensureKeyPair()` directly means an
+// operator who already generated a key for finding attestations does not
+// need a second one for compliance evidence; the two artifact types are
+// simply two things the SAME key can honestly attest to.
+//
+// EA-03 DISCIPLINE, APPLIED FROM THE FIRST WRITE (see D-0026)
+// --------------------------------------------------------------------------
+// canonicalComplianceEvidenceBytes signs an explicit ALLOWLIST — every key
+// emitEvidenceJsonLd() can actually produce, no more, no less — and
+// verifyComplianceEvidence REJECTS any top-level key outside
+// {that allowlist, signature}. A field stapled on after signing (a status
+// silently edited, a control removed, a narrative_evidence line added) must
+// fail verification, not verify successfully with the addition unnoticed.
+//
+// WHY THE FULL DOCUMENT, NOT JUST evidenceDigest (FR-504)
+// --------------------------------------------------------------------------
+// computeEvidenceDigest (FR-504) binds `controls` down to {id, status} only
+// — enough to prove "the CONCLUSION did not change," but not enough to
+// catch an edited check reason or narrative evidence bullet without
+// touching the top-line status. FR-505's acceptance criterion is broader
+// ("altered ... evidence references"), so the signature here covers the
+// full per-control shape (checks[], narrative_evidence[]) that
+// emitEvidenceJsonLd actually renders, not just the FR-504 digest.
+
+import * as fs from 'node:fs';
+import * as crypto from 'node:crypto';
+import { keyPaths } from './evidence-bundle.js';
+
+export const COMPLIANCE_EVIDENCE_SCHEMA = 'agentic-security/compliance-evidence@1';
+
+/**
+ * "Sign evidence manifests WHEN SIGNING IS CONFIGURED" — this is the opt-in
+ * check. Unlike `agentic-security attest` (an explicit command an operator
+ * runs specifically to sign something, where generating a key on first use
+ * is the right default), compliance evidence is emitted automatically on
+ * every ordinary scan — auto-generating a signing key as a side effect of a
+ * routine scan would be surprising, unrequested behavior. "Configured"
+ * therefore means an operator has ALREADY set up the shared key (by running
+ * `attest` once, or by any other means) — this function only ever READS,
+ * never generates. Read-first-in-try/catch (D-0012) — no
+ * existsSync-then-readFileSync.
+ *
+ * @returns {{privateKeyPem: string, publicKeyPem: string}|null}
+ */
+export function loadSigningKeyIfConfigured() {
+  const p = keyPaths();
+  let privateKeyPem, publicKeyPem;
+  try { privateKeyPem = fs.readFileSync(p.privateKey, 'utf8'); } catch { return null; }
+  try { publicKeyPem = fs.readFileSync(p.publicKey, 'utf8'); } catch { return null; }
+  return { privateKeyPem, publicKeyPem };
+}
+
+// The complete set of top-level keys a real emitEvidenceJsonLd() output can
+// carry, BEFORE signing. Must be kept in sync with that function — the
+// signed allowlist and the "known keys" allowlist for unsigned-field
+// rejection are the SAME set, exactly as evidence-bundle.js/policy-bundle.js
+// both already establish.
+const EVIDENCE_FIELDS = [
+  '@context', '@type', 'schemaVersion', 'statusSemantics', 'policySource',
+  'framework', 'version', 'generatedAt',
+  'disclaimer', 'provenance', 'evidenceDigest', 'summary', 'controls',
+];
+const EVIDENCE_TOP_LEVEL_KEYS = new Set([...EVIDENCE_FIELDS, 'signature']);
+
+// Deterministic JSON, keys sorted at every level — same algorithm as
+// evidence-bundle.js's canonicalJson and policy-bundle.js's canonicalJson
+// (each duplicated locally rather than imported, per those modules' own
+// stated reasoning: a pure, three-line function not worth coupling three
+// signing modules' formats together for).
+function _canonicalJson(value) {
+  if (value === null || typeof value !== 'object') return JSON.stringify(value ?? null);
+  if (Array.isArray(value)) return `[${value.map(_canonicalJson).join(',')}]`;
+  const keys = Object.keys(value).sort();
+  return `{${keys.map(k => `${JSON.stringify(k)}:${_canonicalJson(value[k])}`).join(',')}}`;
+}
+
+export function canonicalComplianceEvidenceBytes(jsonld) {
+  const signed = {};
+  for (const k of EVIDENCE_FIELDS) signed[k] = jsonld[k] ?? null;
+  return Buffer.from(_canonicalJson(signed), 'utf8');
+}
+
+/** Sign a compliance evidence document. Returns a new object; the input is not mutated. */
+export function signComplianceEvidence(jsonld, privateKeyPem) {
+  const sig = crypto.sign(null, canonicalComplianceEvidenceBytes(jsonld), privateKeyPem);
+  return {
+    ...jsonld,
+    signature: { algorithm: 'ed25519', canonicalisation: COMPLIANCE_EVIDENCE_SCHEMA, value: sig.toString('base64') },
+  };
+}
+
+/**
+ * Verify a signed compliance evidence document with a PUBLIC key. Never
+ * throws — a malformed or tampered document from an untrusted source (an
+ * auditor or GRC tool checking what they were handed) is an expected input.
+ *
+ * @returns {{ok: boolean, reason: string|null}}
+ */
+export function verifyComplianceEvidence(jsonld, publicKeyPem) {
+  if (!jsonld || typeof jsonld !== 'object') return { ok: false, reason: 'document is not an object' };
+  const unknownKeys = Object.keys(jsonld).filter(k => !EVIDENCE_TOP_LEVEL_KEYS.has(k));
+  if (unknownKeys.length) {
+    return { ok: false, reason: `unrecognised top-level key(s) not covered by the signature: ${unknownKeys.join(', ')}` };
+  }
+  const sig = jsonld.signature;
+  if (!sig?.value) return { ok: false, reason: 'document is unsigned' };
+  if (sig.algorithm !== 'ed25519') return { ok: false, reason: `unsupported algorithm: ${sig.algorithm}` };
+  if (!publicKeyPem) return { ok: false, reason: 'no public key supplied' };
+  let sigOk = false;
+  try {
+    sigOk = crypto.verify(null, canonicalComplianceEvidenceBytes(jsonld), publicKeyPem, Buffer.from(sig.value, 'base64'));
+  } catch (e) {
+    return { ok: false, reason: `verification error: ${e.message}` };
+  }
+  return sigOk
+    ? { ok: true, reason: null }
+    : { ok: false, reason: 'signature does not match the document contents — it was modified after signing' };
+}