@clear-capabilities/agentic-security-scanner 0.133.0 → 0.134.0

package/src/posture/comparison.js

@@ -0,0 +1,181 @@
+// PRD Epic 7.2 — head-to-head comparison scoring.
+//
+// WHY THIS SHIPS WITHOUT A SINGLE PARTICIPANT NAME IN IT. A benchmark whose
+// competitors are hard-coded by the vendor being measured is marketing with a
+// methodology section. This repository publishes the HARNESS and the answer
+// key; the operator supplies the participants. Nothing here — no constant, no
+// default config, no example — names any tool, and the report renders whatever
+// labels the operator chose. That is not a limitation working around a rule; a
+// comparison anyone can re-run against tools of their own choosing is the only
+// kind worth publishing, and the only kind a reader has reason to believe.
+//
+// THE ONE FAILURE MODE THIS MODULE EXISTS TO PREVENT. Two tools scored over
+// different subsets of a corpus are not comparable, and the difference is
+// invisible in the output: a tool that crashed on the 40 hardest entries and
+// was scored over the remaining 170 looks like it beat one that completed all
+// 210. So every rate here is computed over the INTERSECTION of entries every
+// participant completed, that intersection is reported alongside each
+// participant's own completion count, and a participant that completed nothing
+// in common with the others is refused rather than shown with an empty score.
+//
+// MATCHING IS CWE-ONLY, ON PURPOSE. Our own corpus entries carry a `vuln_match`
+// phrase in this engine's wording; scoring an external tool against our
+// phrasing would score it on vocabulary. CWE is the one identifier every
+// participant can be expected to emit, so it is the only key used, and it is
+// applied identically to every participant including this engine. A participant
+// that reports no CWE at all is scored as reporting nothing — stated in the
+// output rather than silently counted as a miss.
+
+/** Verdict for one participant on one corpus entry. */
+export const OUTCOMES = Object.freeze(['tp', 'fn', 'fp', 'tn']);
+
+function _cweSet(findings) {
+  const s = new Set();
+  for (const f of findings || []) {
+    const raw = f && (f.cwe ?? f.CWE ?? f.ruleId ?? '');
+    for (const m of String(raw).matchAll(/CWE[-_ ]?(\d+)/gi)) s.add(`CWE-${m[1]}`);
+  }
+  return s;
+}
+
+/**
+ * Score one participant over the entries it completed.
+ *
+ * @param {object[]} entries  [{id, cwe}]
+ * @param {object}   results  entryId -> {pre: findings[], post: findings[]} | {error}
+ */
+export function scoreParticipant(entries, results) {
+  const per = new Map();
+  let noCwe = 0;
+  for (const e of entries) {
+    const r = results?.[e.id];
+    if (!r || r.error || !Array.isArray(r.pre) || !Array.isArray(r.post)) continue;
+
+    const want = String(e.cwe || '').toUpperCase();
+    const pre = _cweSet(r.pre);
+    const post = _cweSet(r.post);
+    if (!pre.size && (r.pre || []).length) noCwe++;
+
+    // pre/ is the vulnerable tree: reporting the CWE is a true positive.
+    // post/ is the fixed tree: reporting it again is a false positive.
+    per.set(e.id, {
+      detected: pre.has(want),
+      falsePositive: post.has(want),
+    });
+  }
+  return { per, completed: per.size, noCwe };
+}
+
+function _rates(tp, fn, fp, tn) {
+  const precision = tp + fp > 0 ? tp / (tp + fp) : null;
+  const recall = tp + fn > 0 ? tp / (tp + fn) : null;
+  const f1 = precision !== null && recall !== null && precision + recall > 0
+    ? (2 * precision * recall) / (precision + recall) : null;
+  return { tp, fn, fp, tn, precision, recall, f1 };
+}
+
+/**
+ * Compare every participant over the entries ALL of them completed.
+ *
+ * @param {object[]} entries       [{id, cwe}]
+ * @param {object[]} participants  [{id, results}]
+ * @returns {object} {ok, reason?, intersection, scores[], skippedEntries[]}
+ */
+export function compareParticipants(entries, participants) {
+  if (!Array.isArray(entries) || !entries.length) return { ok: false, reason: 'no corpus entries' };
+  if (!Array.isArray(participants) || participants.length < 2) {
+    return { ok: false, reason: 'a comparison needs at least two participants' };
+  }
+
+  const scored = participants.map((p) => ({ ...p, ...scoreParticipant(entries, p.results) }));
+
+  // The intersection. This is the whole point: rates over anything else are
+  // rates over different exams.
+  let common = null;
+  for (const s of scored) {
+    const ids = new Set(s.per.keys());
+    common = common === null ? ids : new Set([...common].filter((id) => ids.has(id)));
+  }
+  if (!common || common.size === 0) {
+    return {
+      ok: false,
+      reason: 'no corpus entry was completed by every participant — there is nothing they can be compared on',
+      completion: Object.fromEntries(scored.map((s) => [s.id, s.completed])),
+    };
+  }
+
+  const scores = scored.map((s) => {
+    let tp = 0, fn = 0, fp = 0, tn = 0;
+    for (const id of common) {
+      const v = s.per.get(id);
+      if (v.detected) tp++; else fn++;
+      if (v.falsePositive) fp++; else tn++;
+    }
+    return {
+      id: s.id,
+      ...(_rates(tp, fn, fp, tn)),
+      completed: s.completed,
+      notCompleted: entries.length - s.completed,
+      noCwe: s.noCwe,
+    };
+  });
+
+  return {
+    ok: true,
+    corpusSize: entries.length,
+    intersection: common.size,
+    // Named so a reader can check the exam rather than trust the grade.
+    scoredEntryIds: [...common].sort(),
+    scores: scores.sort((a, b) => (b.f1 ?? -1) - (a.f1 ?? -1)),
+  };
+}
+
+const pct = (v) => (v === null || v === undefined ? 'n/a' : `${(v * 100).toFixed(1)}%`);
+
+/** Markdown. Discloses the exam before the grades, never after. */
+export function renderComparison(cmp) {
+  if (!cmp || !cmp.ok) {
+    return `# Comparison\n\nNOT SCORED: ${cmp?.reason || 'unknown reason'}\n`;
+  }
+  const out = [];
+  out.push('# Head-to-head comparison');
+  out.push('');
+  out.push(`Scored over the **${cmp.intersection} of ${cmp.corpusSize}** corpus entries that *every*`);
+  out.push('participant completed. Entries any participant failed to complete are excluded from');
+  out.push('every score, including this engine\'s — a rate computed over a different subset is a');
+  out.push('rate for a different exam.');
+  out.push('');
+  out.push('Matching is by CWE only. Participants report findings in their own vocabulary, so');
+  out.push('scoring against any one tool\'s phrasing would measure vocabulary rather than');
+  out.push('detection. The same rule is applied to every participant.');
+  out.push('');
+  out.push('| Participant | F1 | Precision | Recall | TP | FN | FP | Corpus completed |');
+  out.push('|---|---|---|---|---|---|---|---|');
+  for (const s of cmp.scores) {
+    out.push(`| ${s.id} | ${pct(s.f1)} | ${pct(s.precision)} | ${pct(s.recall)} | ${s.tp} | ${s.fn} | ${s.fp} | ${s.completed}/${cmp.corpusSize} |`);
+  }
+  out.push('');
+  const incomplete = cmp.scores.filter((s) => s.notCompleted > 0);
+  if (incomplete.length) {
+    out.push('## Entries not completed');
+    out.push('');
+    out.push('A participant that could not run on an entry is UNSCORED there, never scored as a');
+    out.push('miss. Counting a crash as a false negative would penalise a tool for a harness');
+    out.push('problem; counting it as a pass would reward it for one.');
+    out.push('');
+    for (const s of incomplete) out.push(`- **${s.id}** — ${s.notCompleted} entr(y/ies) not completed`);
+    out.push('');
+  }
+  const noCwe = cmp.scores.filter((s) => s.noCwe > 0);
+  if (noCwe.length) {
+    out.push('## Findings carrying no CWE');
+    out.push('');
+    for (const s of noCwe) {
+      out.push(`- **${s.id}** — ${s.noCwe} entr(y/ies) where findings were reported but none carried a CWE,`);
+      out.push('  so they could not be matched. This depresses that participant\'s recall for a');
+      out.push('  reporting-format reason rather than a detection one.');
+    }
+    out.push('');
+  }
+  return out.join('\n') + '\n';
+}

package/src/posture/execution-proof.js

@@ -51,7 +51,31 @@ function _materialise(root, files) {
  *   patch: pass the patched contents and a still-`execution-proven` verdict
  *   means the fix did not close the hole.
  */
-export async function proveFinding(finding, { timeoutMs = 10000, force, files } = {}) {
+// How long a proof-of-concept gets to write its marker.
+//
+// WHY THIS IS GENEROUS, AND WHY THAT IS NEARLY FREE. The budget is only ever
+// spent when a PoC is stuck: a working one writes its marker and exits in about
+// a second, so raising the ceiling costs nothing in the common path. What a
+// tight ceiling DOES cost is correctness — the budget covers spawning a
+// confined process and starting a Node runtime inside it, and on a loaded
+// machine that alone can eat several seconds. At 10s this timed out under
+// ordinary parallel test load and reported "re-verification did not execute",
+// which the release gate then surfaced as a failure. The measurement has to be
+// of the proof-of-concept, not of how busy the machine happened to be.
+//
+// A timeout is still never evidence about the finding: `proven` is decided by
+// the marker file, and a timed-out run falls back to the finding's static tier
+// rather than claiming `proof-failed`. This ceiling only decides how long we
+// wait before giving up, not what we conclude.
+//
+// Override on a slow or heavily-loaded runner, matching the convention used by
+// AGENTIC_SECURITY_PY_PROBE_TIMEOUT_MS and AGENTIC_SECURITY_DEEP_TIMEOUT_MS.
+export const DEFAULT_PROOF_TIMEOUT_MS =
+  Number(process.env.AGENTIC_SECURITY_PROOF_TIMEOUT_MS) > 0
+    ? Number(process.env.AGENTIC_SECURITY_PROOF_TIMEOUT_MS)
+    : 45000;
+
+export async function proveFinding(finding, { timeoutMs = DEFAULT_PROOF_TIMEOUT_MS, force, files } = {}) {
   const poc = finding?.poc;
   if (!poc?.code) {
     return attachProofTier(finding, _evidence({ tier: proofTierOf(finding), reason: 'no proof-of-concept attached' }));

package/src/posture/logic-claims.js

@@ -0,0 +1,266 @@
+// PRD Epic 6 — the business-logic tier's missing half.
+//
+// The deterministic side of business logic already exists and is wired:
+// `sast/logic.js` carries the canonical anti-patterns, `posture/business-logic.js`
+// builds the per-route authZ matrix, extracts state machines and finds
+// negative-test gaps. What did NOT exist was any handling of the OTHER
+// producer — the reviewing agent, which is the only party that can read intent
+// and is therefore the only one that can find the flaws patterns cannot.
+//
+// THE PROBLEM WITH THAT PRODUCER. Everything else in this engine can be
+// checked: a taint finding has a path, an execution-proven finding has a marker
+// file, an SCA finding has a version range. A logic claim is prose. It arrives
+// asserting that a handler lets one user act on another's resource, and there
+// is nothing in the finding that a second party could disagree with. An
+// unrefutable claim is the weakest thing this engine emits, and it was the only
+// tier with no way to be wrong.
+//
+// WHAT THIS MODULE DOES. It takes claims from a reviewing agent and puts them
+// through deterministic lenses that can REFUTE them — cheaply, offline, and
+// without asking a model to grade its own homework:
+//
+//   citation   — the file exists and the cited line is inside it. A claim about
+//                `routes/orders.js:214` in a 90-line file is refuted on the
+//                spot; that is the signature of a fabricated location.
+//   quotation  — the snippet the claim quotes actually appears at the cited
+//                line (± a small window). A claim that misquotes the code it is
+//                about was not written from the code.
+//   corroboration — for the claim kinds that MAKE a checkable assertion about
+//                the source ("this route has no authentication"), check it.
+//                A route that plainly does authenticate refutes it.
+//
+// RECALL-PRESERVING, same precedent as `falsification.js` and `proof-gate.js`.
+// A refuted claim is marked and kept, never deleted and never severity-touched.
+// Refutation here means "no second party could corroborate this", which is a
+// triage signal, not proof the reviewer was wrong.
+//
+// SEPARATION IS ENFORCED, NOT ASSUMED. The agent is stamped as producer and
+// these lenses record under their own verifier ids, so `assertSeparation`
+// refuses if anything ever tries to verify its own claim. That is why the
+// lenses live here in deterministic code rather than in the agent's prompt: a
+// reviewer asked to double-check itself is the same party voting twice.
+
+import { recordProducer, recordVerdict, consensusOf } from './verification-separation.js';
+
+export const PRODUCER = 'agent:logic-reviewer';
+
+export const VERIFIER_CITATION = 'verifier:citation';
+const VERIFIER_QUOTATION = 'verifier:quotation';
+export const VERIFIER_CORROBORATION = 'verifier:logic-corroboration';
+
+// Claim kinds that assert something checkable about the source. Anything else
+// is accepted as unverifiable-but-recorded rather than silently upheld.
+const CLAIM_KINDS = Object.freeze([
+  'missing-authentication',
+  'missing-authorization',
+  'missing-ownership-check',
+  'state-transition-bypass',
+  'race-condition',
+  'other',
+]);
+
+// Reused deliberately from the same vocabulary the authZ matrix uses, so a
+// corroboration verdict and a matrix finding cannot disagree about what
+// "authenticated" means in this codebase.
+const AUTH_HINTS = [
+  /\breq\.user\b/, /\breq\.auth\b/, /\brequest\.user\b/,
+  /requireAuth|isAuthenticated|@login_required|@requires_auth|@jwt_required/,
+  /authorize|authMiddleware|verifyJWT|jwt\.verify\b/, /\bpassport\b/,
+  /\bgetSession\b|\bcurrentUser\b/,
+];
+const OWNERSHIP_HINTS = [
+  /\bowner(?:Id)?\b/i, /\buser_?id\s*[=:]/i,
+  /\.userId\s*===\s*req\.user/, /\.owner\s*===\s*req\.user/,
+  /where\s*:\s*\{[^}]*user/i,
+];
+
+// How far from the cited line a quoted snippet may appear before the citation
+// is treated as not corroborated. Small on purpose: an agent reading the file
+// is off by a line or two, not by twenty.
+const QUOTE_WINDOW = 3;
+
+function _lines(content) { return String(content).split('\n'); }
+
+function _normalize(s) {
+  return String(s).replace(/\s+/g, ' ').trim().toLowerCase();
+}
+
+/**
+ * The enclosing handler body around a line, bounded by blank-line-separated
+ * top-level blocks. Deliberately crude: a corroboration lens that guessed at
+ * scope precisely would be a parser, and a wrong guess here REFUTES a real
+ * finding. So the window is generous — it errs toward finding the auth check
+ * and therefore toward refusing to refute.
+ */
+function _enclosingBlock(content, line) {
+  const ls = _lines(content);
+  const idx = Math.max(0, Math.min(ls.length - 1, line - 1));
+  let start = idx, end = idx;
+  while (start > 0 && !/^\s*$/.test(ls[start - 1])) start--;
+  while (end < ls.length - 1 && !/^\s*$/.test(ls[end + 1])) end++;
+  // Widen by a few lines either side: middleware often sits on the route line
+  // above the block the flaw is in.
+  start = Math.max(0, start - 5);
+  end = Math.min(ls.length - 1, end + 5);
+  return ls.slice(start, end + 1).join('\n');
+}
+
+/**
+ * Put one claim through the deterministic lenses.
+ *
+ * @param {object} claim  {file, line, vuln, kind, description, snippet?, severity?}
+ * @param {object|Map} fileContents  file -> source
+ * @returns {object} the claim as a finding, carrying `verification`
+ */
+export function verifyLogicClaim(claim, fileContents) {
+  const finding = {
+    ...claim,
+    parser: 'LOGIC-AGENT',
+    family: claim.family || 'business-logic',
+    kind: CLAIM_KINDS.includes(claim.kind) ? claim.kind : 'other',
+  };
+  recordProducer(finding, claim.producer || PRODUCER);
+
+  const read = (f) => {
+    if (!fileContents) return null;
+    if (typeof fileContents.get === 'function') return fileContents.get(f) ?? null;
+    return fileContents[f] ?? null;
+  };
+  const content = claim.file ? read(claim.file) : null;
+
+  // ── citation ──────────────────────────────────────────────────────────────
+  if (content === null) {
+    recordVerdict(finding, {
+      verifierId: VERIFIER_CITATION, lens: 'citation', verdict: 'refuted',
+      reason: `no file '${claim.file}' was scanned, so the cited location does not exist`,
+    });
+    finding.consensus = consensusOf(finding);
+    finding.quarantined = true;
+    return finding;
+  }
+  const total = _lines(content).length;
+  const line = Number(claim.line);
+  if (!Number.isInteger(line) || line < 1 || line > total) {
+    recordVerdict(finding, {
+      verifierId: VERIFIER_CITATION, lens: 'citation', verdict: 'refuted',
+      reason: `cited line ${claim.line} is outside ${claim.file} (${total} lines)`,
+    });
+  } else {
+    recordVerdict(finding, {
+      verifierId: VERIFIER_CITATION, lens: 'citation', verdict: 'upheld',
+      reason: `${claim.file}:${line} exists`,
+    });
+  }
+
+  // ── quotation ─────────────────────────────────────────────────────────────
+  // Only a lens when the claim actually quotes something. A claim with no
+  // snippet is UNDECIDED here, not upheld — silence is not corroboration.
+  if (!claim.snippet || !String(claim.snippet).trim()) {
+    recordVerdict(finding, {
+      verifierId: VERIFIER_QUOTATION, lens: 'quotation', verdict: 'undecided',
+      reason: 'the claim quotes no source, so there is nothing to check it against',
+    });
+  } else {
+    const want = _normalize(claim.snippet);
+    const ls = _lines(content);
+    const lo = Math.max(0, (line || 1) - 1 - QUOTE_WINDOW);
+    const hi = Math.min(ls.length, (line || 1) + QUOTE_WINDOW);
+    const window = _normalize(ls.slice(lo, hi).join(' '));
+    const anywhere = _normalize(content);
+    if (window.includes(want)) {
+      recordVerdict(finding, {
+        verifierId: VERIFIER_QUOTATION, lens: 'quotation', verdict: 'upheld',
+        reason: 'the quoted source appears at the cited line',
+      });
+    } else if (anywhere.includes(want)) {
+      // Right file, wrong line. Not a fabrication, but the location is not
+      // usable as-is, so it is not corroboration either.
+      recordVerdict(finding, {
+        verifierId: VERIFIER_QUOTATION, lens: 'quotation', verdict: 'undecided',
+        reason: 'the quoted source is in the file but not at the cited line',
+      });
+    } else {
+      recordVerdict(finding, {
+        verifierId: VERIFIER_QUOTATION, lens: 'quotation', verdict: 'refuted',
+        reason: 'the quoted source does not appear in the cited file',
+      });
+    }
+  }
+
+  // ── corroboration ─────────────────────────────────────────────────────────
+  const block = _enclosingBlock(content, line || 1);
+  if (finding.kind === 'missing-authentication') {
+    const hit = AUTH_HINTS.find((re) => re.test(block));
+    recordVerdict(finding, hit
+      ? { verifierId: VERIFIER_CORROBORATION, lens: 'authz', verdict: 'refuted',
+        reason: `the handler around this line does authenticate (${hit.source})` }
+      : { verifierId: VERIFIER_CORROBORATION, lens: 'authz', verdict: 'upheld',
+        reason: 'no authentication marker anywhere in the enclosing handler' });
+  } else if (finding.kind === 'missing-authorization' || finding.kind === 'missing-ownership-check') {
+    const hit = OWNERSHIP_HINTS.find((re) => re.test(block));
+    recordVerdict(finding, hit
+      ? { verifierId: VERIFIER_CORROBORATION, lens: 'authz', verdict: 'refuted',
+        reason: `the handler around this line does scope the record to a user (${hit.source})` }
+      : { verifierId: VERIFIER_CORROBORATION, lens: 'authz', verdict: 'upheld',
+        reason: 'no ownership scoping in the enclosing handler' });
+  } else {
+    // No deterministic lens exists for this kind. Said out loud rather than
+    // counted as agreement — an unchecked claim and a corroborated one must
+    // not read the same in the consensus.
+    recordVerdict(finding, {
+      verifierId: VERIFIER_CORROBORATION, lens: 'authz', verdict: 'undecided',
+      reason: `no deterministic lens covers claim kind '${finding.kind}'`,
+    });
+  }
+
+  finding.consensus = consensusOf(finding);
+  // Quarantine, not deletion — the same contract falsification uses.
+  finding.quarantined = finding.consensus.verdict === 'refuted';
+  return finding;
+}
+
+/**
+ * Verify a batch. Nothing is dropped: the returned list is the same length as
+ * the input, in the same order.
+ */
+export function ingestLogicClaims(claims, { fileContents = null } = {}) {
+  const list = Array.isArray(claims) ? claims : [];
+  const out = list.map((c) => {
+    try { return verifyLogicClaim(c, fileContents); }
+    catch (e) {
+      // A lens that throws must not swallow the claim.
+      const f = { ...c, parser: 'LOGIC-AGENT', family: 'business-logic', quarantined: false };
+      f.consensus = { verdict: 'undecided', upheld: 0, refuted: 0, undecided: 0, lenses: [] };
+      f.verificationError = String(e?.message || e);
+      return f;
+    }
+  });
+  return { claims: out, summary: summarizeLogicClaims(out) };
+}
+
+function summarizeLogicClaims(claims) {
+  const s = { total: claims.length, corroborated: 0, refuted: 0, unverifiable: 0 };
+  for (const c of claims) {
+    const v = c.consensus?.verdict;
+    if (v === 'upheld') s.corroborated++;
+    else if (v === 'refuted') s.refuted++;
+    else s.unverifiable++;
+  }
+  return s;
+}
+
+/** One line. Leads with what could not be corroborated. */
+export function renderLogicClaimSummary(s) {
+  if (!s || !s.total) return null;
+  const bits = [`${s.total} business-logic claim(s)`];
+  if (s.refuted) bits.push(`${s.refuted} REFUTED by a deterministic lens (quarantined, not deleted)`);
+  if (s.unverifiable) bits.push(`${s.unverifiable} unverifiable — no lens could agree or disagree`);
+  if (s.corroborated) bits.push(`${s.corroborated} corroborated`);
+  return bits.join('; ') + '.';
+}
+
+// Not exported: the quotation verifier id, the claim-kind vocabulary and the
+// batch summariser have no consumer outside this module. Kept internal rather
+// than exported-and-unused — an export with no call site is how dead code gets
+// shipped and then trusted.
+export const _internals = { AUTH_HINTS, OWNERSHIP_HINTS, _enclosingBlock, QUOTE_WINDOW, CLAIM_KINDS, VERIFIER_QUOTATION, summarizeLogicClaims };