@clear-capabilities/agentic-security-scanner 0.74.0

package/src/llm-validator/.agentic-security/last-scan.json.sig

@@ -0,0 +1 @@
+4c7773244a54b5c390324ea1d7302921511d2ac2526cebc27cf16deff479564f

package/src/llm-validator/.agentic-security/scan-history.json

@@ -0,0 +1,168 @@
+[
+  {
+    "timestamp": "2026-05-18T17:45:05.442Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:30:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:80:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:81:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:87:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T21:57:42.021Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T22:28:47.440Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T22:29:03.014Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T22:29:18.932Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T23:16:38.135Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-18T23:16:47.744Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:55:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:114:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:115:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:121:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-19T00:12:15.461Z",
+    "label": "scan",
+    "total": 4,
+    "critical": 0,
+    "high": 1,
+    "medium": 3,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:57:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:index.js:116:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:117:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:123:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)"
+    ]
+  },
+  {
+    "timestamp": "2026-05-19T22:20:17.395Z",
+    "label": "scan",
+    "total": 7,
+    "critical": 0,
+    "high": 1,
+    "medium": 6,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:57:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:consistency.js:66:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:consistency.js:67:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:116:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:117:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:123:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "toctou-fs:index.js:116"
+    ]
+  },
+  {
+    "timestamp": "2026-05-20T12:29:43.775Z",
+    "label": "scan",
+    "total": 7,
+    "critical": 0,
+    "high": 1,
+    "medium": 6,
+    "low": 0,
+    "kev": 0,
+    "ids": [
+      "prompt-tpl:index.js:57:Prompt_Template__user_input_interpolated_into_prompt_string_",
+      "struct:consistency.js:66:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:consistency.js:67:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:116:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:117:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "struct:index.js:123:Synchronous_Blocking_I/O_(DoS_Risk_in_Server_Context)",
+      "toctou-fs:index.js:116"
+    ]
+  }
+]

package/src/llm-validator/.agentic-security/streak.json

@@ -0,0 +1,20 @@
+{
+  "firstScanDate": "2026-05-18T17:45:05.477Z",
+  "lastScanDate": "2026-05-20T12:29:43.794Z",
+  "totalScans": 10,
+  "daysCleanCritical": 3,
+  "lastCleanDate": "2026-05-20",
+  "lastCriticalDate": null,
+  "hasEverHadCritical": false,
+  "bestDaysCleanCritical": 3,
+  "totalFindingsAtFirstScan": 6,
+  "totalFindingsAtLastScan": 9,
+  "totalFixesInferred": 0,
+  "lastGrade": "A-",
+  "bestGrade": "A-",
+  "launchCheckPassedAt": null,
+  "achievements": [
+    "first-scan"
+  ],
+  "previousGrade": "A-"
+}

package/src/llm-validator/consistency.js

@@ -0,0 +1,141 @@
+// Pass^k consistency harness for the LLM validator.
+//
+// The validator is the only LLM in the production code path. Its verdicts
+// (accept / reject / escalate) feed back into which findings ship. If it
+// returns different verdicts on the same finding across runs, the customer
+// sees inconsistent reports — and we'd never know unless we measured.
+//
+// This module runs the validator N times against a fixed set of findings,
+// records each trial's verdict per finding, and reports:
+//   - pass@k:   probability that at least one of k trials gave verdict X
+//   - pass^k:   probability that ALL k trials gave the SAME verdict
+//   - per-finding consistency rate
+//   - cache-key consistency: do cached runs match initial runs?
+//
+// "If your agent has a 75% per-trial success rate and you run 3 trials, the
+//  probability of passing all three is (0.75)³ ≈ 42%."
+//   — Anthropic on demystifying evals for AI agents
+//
+// The validator's caching is invalidated for each trial by varying the
+// challenge nonce; this gives an honest cold-cache pass^k. To measure the
+// warm-cache equivalent, set `useCache: true`.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { validateOne } from './index.js';
+
+// Build a deterministic test finding from the project's last-scan.json,
+// or accept a hand-crafted one. Returns a clone safe to mutate per trial.
+export function makeTrialFinding(template) {
+  return JSON.parse(JSON.stringify(template));
+}
+
+// Run `trials` independent passes of `validateOne` on each finding. Returns
+// a structured report. Caller is responsible for AGENTIC_SECURITY_LLM_VALIDATE
+// / AGENTIC_SECURITY_LLM_ENDPOINT being set; if they aren't, the result will
+// show every trial verdict as "unvalidated" — which is still a valid signal
+// (the harness ran cleanly; the validator is simply off).
+export async function measureConsistency({
+  findings,
+  fileContents = {},
+  scanRoot,
+  trials = 5,
+  useCache = false,
+} = {}) {
+  if (!Array.isArray(findings) || findings.length === 0) {
+    return { ok: false, reason: 'no-findings-supplied' };
+  }
+  if (!Number.isInteger(trials) || trials < 2 || trials > 50) {
+    return { ok: false, reason: 'trials-out-of-range' };
+  }
+  // For cold-cache mode, delete each finding's cache entry between trials.
+  // We do this by NOT setting a cache dir per trial — easier: each trial
+  // gets a fresh tmp scanRoot for the cache.
+  const perFinding = new Map();   // findingId → { verdicts: [], confidences: [] }
+  for (const f of findings) perFinding.set(f.id || f.stableId || 'unknown', { verdicts: [], confidences: [], reasons: [] });
+
+  for (let t = 0; t < trials; t++) {
+    for (const finding of findings) {
+      const trial = makeTrialFinding(finding);
+      // If useCache is false, we want each trial to bypass the cache; we
+      // simulate by mutating the finding's file in a way the cache key
+      // hashes over. Simpler: clear the per-scanRoot cache before each trial.
+      if (!useCache && scanRoot) {
+        try {
+          const cacheDir = path.join(scanRoot, '.agentic-security', 'llm-cache');
+          if (fs.existsSync(cacheDir)) {
+            for (const e of fs.readdirSync(cacheDir)) fs.unlinkSync(path.join(cacheDir, e));
+          }
+        } catch { /* best-effort */ }
+      }
+      try {
+        await validateOne(trial, fileContents, scanRoot);
+      } catch (e) {
+        trial.validator_verdict = 'error';
+        trial._validatorError = String((e && e.message) || e);
+      }
+      const key = finding.id || finding.stableId || 'unknown';
+      const slot = perFinding.get(key);
+      slot.verdicts.push(trial.validator_verdict || 'unset');
+      slot.confidences.push(typeof trial.llm_confidence === 'number' ? trial.llm_confidence : null);
+      slot.reasons.push(trial._validatorError || null);
+    }
+  }
+
+  // Score per-finding consistency.
+  const findingReports = [];
+  let stableCount = 0;
+  for (const [id, slot] of perFinding) {
+    const counts = {};
+    for (const v of slot.verdicts) counts[v] = (counts[v] || 0) + 1;
+    const dominant = Object.entries(counts).sort((a, b) => b[1] - a[1])[0];
+    const stable = dominant[1] === trials;
+    if (stable) stableCount++;
+    findingReports.push({
+      id,
+      verdicts: slot.verdicts,
+      dominantVerdict: dominant[0],
+      dominantRate: dominant[1] / trials,
+      stable,
+      confidenceMean: _mean(slot.confidences),
+      confidenceStdev: _stdev(slot.confidences),
+    });
+  }
+  const passK = stableCount / perFinding.size;
+
+  return {
+    ok: true,
+    trials,
+    findingCount: perFinding.size,
+    passK_unanimous: passK,    // pass^k where the bar is "all k agree"
+    findings: findingReports,
+    when: new Date().toISOString(),
+  };
+}
+
+function _mean(arr) {
+  const v = arr.filter(x => typeof x === 'number');
+  if (!v.length) return null;
+  return v.reduce((a, c) => a + c, 0) / v.length;
+}
+function _stdev(arr) {
+  const v = arr.filter(x => typeof x === 'number');
+  if (v.length < 2) return null;
+  const m = _mean(v);
+  const sq = v.reduce((a, c) => a + (c - m) * (c - m), 0) / (v.length - 1);
+  return Math.sqrt(sq);
+}
+
+// Render a one-screen summary for CLI.
+export function summarize(report) {
+  if (!report || !report.ok) return `consistency: ${report?.reason || 'unknown error'}`;
+  const lines = [];
+  lines.push(`llm-validator consistency — trials=${report.trials}, findings=${report.findingCount}`);
+  lines.push(`  pass^${report.trials} (unanimous): ${(report.passK_unanimous * 100).toFixed(1)}%`);
+  lines.push('');
+  for (const f of report.findings) {
+    lines.push(`  · ${String(f.id).slice(0, 40).padEnd(40)} ${f.stable ? 'STABLE' : 'FLAPS'}  ${f.dominantVerdict} (${(f.dominantRate*100).toFixed(0)}%)  conf=${f.confidenceMean?.toFixed(2) ?? 'n/a'}±${f.confidenceStdev?.toFixed(2) ?? 'n/a'}`);
+    if (!f.stable) lines.push(`      verdicts: [${f.verdicts.join(', ')}]`);
+  }
+  return lines.join('\n');
+}