@clear-capabilities/agentic-security-scanner 0.127.0 → 0.130.0

package/src/posture/root-cause-sweep.js

@@ -0,0 +1,262 @@
+// Addition #3 — Root-cause sweep with total-count accounting.
+//
+// A detector fires on the instance it can prove. But the same root cause is
+// usually copy-pasted across the codebase, and most of those siblings never
+// trip a rule (different variable names, an assignment wrapper, a file the
+// scanner didn't reach with taint). This module takes CONFIRMED findings and
+// sweeps every source line for structural siblings of the same sink, then
+// accounts for every match honestly:
+//
+//     found === candidates + mitigated        (per sweep, always)
+//
+// where `found` is every structural match across the repo EXCLUDING the
+// finding's own origin site, `mitigated` is the subset a detector already
+// covered (a finding exists at that file:line), and `candidates` is the
+// remainder — new instances nobody has looked at yet. Nothing is dropped.
+//
+// Matching reuses semantic-clone's normalized token-shape hashing (`shapeHash`)
+// so that `db.query(a)` and `db.query(b)` collapse to one shape. Pure shape is
+// too loose on its own (`db.query(x)` and `console.log(x)` both normalize to
+// `ID.ID(ID)`), so we anchor on the LITERAL callee (`db.query`) and use the
+// shape only to confirm the argument arity/structure. Anchor + shape = precise.
+//
+// Like semantic-clone this is a coarse structural approximation, not a proof of
+// semantic equivalence. It catches the common "same call, cloned around" case.
+
+import { shapeHash } from './semantic-clone.js';
+
+// shapeHash defaults to minTokens:8 (tuned to avoid trivial clone collisions on
+// whole functions). A single call expression is short — `foo(a)` is 4 tokens,
+// `db.query(a)` is 6 — so we lower the floor for call-granular matching.
+const MIN_SHAPE_TOKENS = 3;
+
+// A callee whose final segment is one of these is control flow, not a sink call.
+const CONTROL_KEYWORDS = new Set([
+  'if', 'for', 'while', 'switch', 'catch', 'return', 'function', 'with', 'do', 'await',
+]);
+
+function escapeRegex(s) {
+  return String(s).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+// Accept both a Map and a plain { path: source } object.
+function toMap(fileContents) {
+  if (fileContents instanceof Map) return fileContents;
+  const m = new Map();
+  if (fileContents && typeof fileContents === 'object') {
+    for (const k of Object.keys(fileContents)) m.set(k, fileContents[k]);
+  }
+  return m;
+}
+
+// A finding qualifies for a sweep when it is confirmed. With confirmedOnly
+// disabled we sweep everything (the caller has opted out of the gate).
+function qualifies(finding, confirmedOnly) {
+  if (!confirmedOnly) return true;
+  return finding.confirmed === true || finding.confidenceTier === 'high';
+}
+
+// The origin site is the finding's own location; siblings must exclude it.
+function originSite(finding) {
+  if (finding.sink && finding.sink.file && finding.sink.line != null) {
+    return { file: finding.sink.file, line: finding.sink.line };
+  }
+  return { file: finding.file ?? null, line: finding.line ?? null };
+}
+
+// Every file:line that already carries a finding — used to classify a match as
+// 'mitigated-or-known' vs. a fresh 'candidate'.
+function buildKnownLocations(findings) {
+  const set = new Set();
+  for (const f of Array.isArray(findings) ? findings : []) {
+    if (!f || typeof f !== 'object') continue;
+    if (f.file != null && f.line != null) set.add(`${f.file}:${f.line}`);
+    if (f.sink && f.sink.file != null && f.sink.line != null) set.add(`${f.sink.file}:${f.sink.line}`);
+  }
+  return set;
+}
+
+// Pull the leading callee path out of a call snippet: `db.query(x)` → `db.query`.
+function extractCallee(snippet) {
+  if (!snippet || typeof snippet !== 'string') return null;
+  const m = snippet.match(/([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*)\s*\(/);
+  if (!m) return null;
+  const callee = m[1];
+  const last = callee.split('.').pop();
+  if (CONTROL_KEYWORDS.has(last)) return null;
+  return callee;
+}
+
+// Extract the full balanced call expression for `callee` from `text`:
+// `const r = db.query(f(1), g);` → `db.query(f(1), g)`. Null if absent/unbalanced.
+function extractCall(text, callee) {
+  if (!text || typeof text !== 'string') return null;
+  const re = new RegExp('(?:^|[^\\w$.])' + escapeRegex(callee) + '\\s*\\(');
+  const m = re.exec(text);
+  if (!m) return null;
+  const calleeStart = m.index + m[0].indexOf(callee); // callee offset within the match
+
+  const open = text.indexOf('(', calleeStart);
+  if (open < 0) return null;
+  let depth = 0;
+  for (let i = open; i < text.length; i++) {
+    const ch = text[i];
+    if (ch === '(') depth++;
+    else if (ch === ')') {
+      depth--;
+      if (depth === 0) return text.slice(calleeStart, i + 1);
+    }
+  }
+  return null; // unbalanced on this line
+}
+
+// Structural shape of a sink snippet: hash of the normalized call expression
+// (callee + args reduced to token kinds), reusing semantic-clone's hasher.
+function sinkShapeOf(snippet) {
+  if (!snippet || typeof snippet !== 'string') return null;
+  const callee = extractCallee(snippet);
+  const call = callee ? (extractCall(snippet, callee) || snippet) : snippet;
+  return shapeHash(call, { minTokens: MIN_SHAPE_TOKENS });
+}
+
+// Build a searchable pattern from a finding's sink (preferred) or fall back to
+// vuln/cwe keywords when no snippet is available.
+function deriveSinkPattern(finding) {
+  const snippet = finding?.sink?.snippet || finding?.snippet || '';
+  const callee = extractCallee(snippet);
+  if (callee) {
+    return {
+      kind: 'call',
+      callee,
+      shape: sinkShapeOf(snippet),
+      regex: new RegExp('(?:^|[^\\w$.])' + escapeRegex(callee) + '\\s*\\('),
+      display: `${callee}(…)`,
+    };
+  }
+  const kw = keywordFor(finding);
+  if (kw) {
+    return { kind: 'keyword', keyword: kw, shape: null, regex: new RegExp(escapeRegex(kw), 'i'), display: kw };
+  }
+  return null;
+}
+
+// Source pattern is reported for context; the sweep itself is sink-driven.
+function deriveSourcePattern(finding) {
+  const s = finding?.source?.snippet;
+  if (s && typeof s === 'string' && s.trim()) return { display: s.trim() };
+  const kw = keywordFor(finding);
+  if (kw) return { display: kw };
+  return null;
+}
+
+function keywordFor(finding) {
+  const v = (finding?.vuln ?? '').toString().trim();
+  if (v) return v;
+  const cwe = (finding?.cwe ?? '').toString().trim();
+  if (cwe) return cwe;
+  return null;
+}
+
+// Does a single source line structurally match the sink pattern?
+function matchLine(pattern, line) {
+  if (!pattern || typeof line !== 'string') return false;
+  if (pattern.kind === 'call') {
+    if (!pattern.regex.test(line)) return false;      // literal callee anchor
+    if (pattern.shape == null) return true;           // anchor-only (snippet too short to shape)
+    const call = extractCall(line, pattern.callee);
+    if (!call) return false;
+    return shapeHash(call, { minTokens: MIN_SHAPE_TOKENS }) === pattern.shape;
+  }
+  if (pattern.kind === 'keyword') {
+    return pattern.regex.test(line);
+  }
+  return false;
+}
+
+/**
+ * Sweep confirmed findings for sibling instances of the same root cause.
+ *
+ * @param {Array<object>} findings   scan findings (confirmed ones drive sweeps)
+ * @param {Map|object}    fileContents  { path: source } — Map or plain object
+ * @param {object}        opts       { confirmedOnly = true }
+ * @returns {{ sweeps: Array<object>, totals: {found,candidates,mitigated} }}
+ */
+export function sweepRootCauses(findings, fileContents, opts = {}) {
+  const confirmedOnly = opts?.confirmedOnly !== false;
+  const list = Array.isArray(findings) ? findings : [];
+  const files = toMap(fileContents);
+  const knownLocations = buildKnownLocations(list);
+
+  const sweeps = [];
+  const totals = { found: 0, candidates: 0, mitigated: 0 };
+
+  for (const finding of list) {
+    if (!finding || typeof finding !== 'object') continue;
+    if (!qualifies(finding, confirmedOnly)) continue;
+
+    const sinkPattern = deriveSinkPattern(finding);
+    if (!sinkPattern) continue; // nothing searchable — skip rather than fabricate
+    const sourcePattern = deriveSourcePattern(finding);
+    const origin = originSite(finding);
+
+    const instances = [];
+    for (const [path, source] of files) {
+      if (source == null) continue;
+      const lines = String(source).split(/\r?\n/);
+      for (let i = 0; i < lines.length; i++) {
+        const line = lines[i];
+        if (!matchLine(sinkPattern, line)) continue;
+        const lineNo = i + 1;
+        if (path === origin.file && lineNo === origin.line) continue; // exclude the finding's own site
+        const status = knownLocations.has(`${path}:${lineNo}`) ? 'mitigated-or-known' : 'candidate';
+        instances.push({ file: path, line: lineNo, snippet: line.trim(), status });
+      }
+    }
+
+    const candidates = instances.filter((x) => x.status === 'candidate').length;
+    const mitigated = instances.filter((x) => x.status === 'mitigated-or-known').length;
+    const found = instances.length; // every match is exactly one status → invariant holds by construction
+
+    sweeps.push({
+      fromFindingId: finding.id ?? finding.stableId ?? null,
+      sourcePattern: sourcePattern ? sourcePattern.display : null,
+      sinkPattern: sinkPattern.display,
+      found,
+      candidates,
+      mitigated,
+      remaining: candidates, // unaccounted instances that still need triage
+      instances,
+    });
+
+    totals.found += found;
+    totals.candidates += candidates;
+    totals.mitigated += mitigated;
+  }
+
+  return { sweeps, totals };
+}
+
+/**
+ * One short human line per sweep, e.g.:
+ *   "root-cause sweep: 20 found, 3 candidate, 17 mitigated"
+ */
+export function formatSweepLedger(result) {
+  if (!result || !Array.isArray(result.sweeps)) return '';
+  return result.sweeps
+    .map((s) => `root-cause sweep: ${s.found} found, ${s.candidates} candidate, ${s.mitigated} mitigated`)
+    .join('\n');
+}
+
+export const _internals = {
+  MIN_SHAPE_TOKENS,
+  sinkShapeOf,
+  deriveSinkPattern,
+  deriveSourcePattern,
+  extractCallee,
+  extractCall,
+  matchLine,
+  qualifies,
+  originSite,
+  buildKnownLocations,
+  toMap,
+};

package/src/posture/rule-overrides.js

@@ -9,7 +9,7 @@
 
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as yaml from 'js-yaml';
+import * as yaml from '../util/yaml.js';
 import { verifyLastScan } from './integrity.js';
 import { statePath } from './state-dir.js';
 

package/src/posture/sca-policy.js

@@ -38,7 +38,7 @@
 
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as yaml from 'js-yaml';
+import * as yaml from '../util/yaml.js';
 
 const DEFAULT_POLICY = {
   acceptRisk: [],

package/src/posture/scan-checkpoint.js

@@ -0,0 +1,277 @@
+// Scan checkpointing / resume (roadmap R8).
+//
+// Long scans currently restart from zero if interrupted, which is what caps the
+// repository size this engine can usefully handle. This module lets the per-file
+// loop in `engine.js#runFullScan` durably record what it has already analysed so
+// a second invocation replays that work instead of redoing it.
+//
+// THE PROPERTY THAT MATTERS: a resumed scan must produce the same finding set as
+// an uninterrupted one. A checkpoint that silently drops findings converts a slow
+// scan into a quietly incomplete one, which is strictly worse than no checkpoint
+// at all. Three design decisions follow from that and should not be relaxed:
+//
+//   1. We persist the *complete* per-file contribution, not just findings —
+//      routes, taint sources/sinks/sanitizers, logic vulns, secrets, ciphers,
+//      the per-file result the cross-file taint pass reads, and the suppression
+//      log delta. Anything the per-file loop appends to must round-trip, or the
+//      post-loop cross-file passes would see a different world on resume.
+//   2. Only the per-file loop is checkpointed. Every cross-file pass and the
+//      whole annotation pipeline re-runs from scratch on resume, so nothing that
+//      depends on the global picture can be stale by construction.
+//   3. Invalidation is conservative to the point of being blunt. The run key
+//      covers the engine version, the ruleset version, the bundle SHA, a content
+//      hash of every file in the scan (which subsumes mtime), and the scanner's
+//      own environment switches. If any of it moved, the checkpoint is discarded
+//      and the scan starts clean. Redoing work is merely slow; resuming stale
+//      work is a correctness bug.
+//
+// CRASH SAFETY: append-and-fsync. The file is a JSONL log — one header line
+// pinning the run key, then one self-describing record per completed file,
+// each carrying a SHA-256 of its own payload. Every record is written with a
+// single `writeSync` and immediately `fsyncSync`'d before the next file is
+// analysed, so a process killed at any instant leaves either a complete record
+// or a torn tail. On recovery we read forward while records verify and truncate
+// the file at the last byte offset that did, so a torn tail is discarded rather
+// than resumed into. Nothing is ever rewritten in place, so there is no window
+// in which the file is neither the old state nor the new one.
+//
+// Everything here follows the posture convention of never throwing: a failure to
+// open, read or append degrades to "no checkpoint", which just means a full scan.
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+
+const STATE_DIR = '.agentic-security';
+const FILE_NAME = 'scan-checkpoint.jsonl';
+const FORMAT = 'agentic-security-scan-checkpoint/1';
+
+// Env switches that change what the engine emits are part of the run identity.
+// These three are deliberately excluded: they change how the run is driven, not
+// what it would find.
+const RUN_KEY_ENV_EXCLUDE = new Set([
+  'AGENTIC_SECURITY_RESUME',
+  'AGENTIC_SECURITY_CHECKPOINT_ABORT_AFTER',
+  'AGENTIC_SECURITY_HMAC_KEY',
+]);
+
+export function checkpointPath(scanRoot) {
+  return path.join(scanRoot || '.', STATE_DIR, FILE_NAME);
+}
+
+function _sha(s) {
+  return crypto.createHash('sha256').update(s, 'utf8').digest('hex');
+}
+
+// SHA-256 of the running bundle, taken from the sidecar next to it. Returns
+// 'unavailable' when running from source — same convention as the attestation
+// path, and deliberately not a guess at some other bundle's hash.
+export function bundleShaForRunKey() {
+  try {
+    const here = path.dirname(fileURLToPath(import.meta.url));
+    // src/posture/ -> src/ -> scanner/
+    const sidecar = path.resolve(here, '..', '..', 'dist', 'agentic-security.mjs.sha256');
+    const raw = fs.readFileSync(sidecar, 'utf8').trim();
+    const m = /^([0-9a-f]{64})\b/.exec(raw);
+    if (m) return m[1];
+  } catch { /* not running from a built tree */ }
+  return 'unavailable';
+}
+
+/**
+ * Everything that would invalidate previously-completed per-file work, reduced
+ * to one hex digest. Content hashes rather than mtimes: strictly stronger, and
+ * immune to filesystems with coarse or non-monotonic timestamps.
+ */
+export function computeRunKey({
+  engineVersion, rulesetVersion, bundleSha,
+  fileContents = {}, depFileContents = {}, env = process.env,
+} = {}) {
+  const h = crypto.createHash('sha256');
+  h.update(FORMAT); h.update('\n');
+  h.update(String(engineVersion ?? '')); h.update('\n');
+  h.update(String(rulesetVersion ?? '')); h.update('\n');
+  h.update(String(bundleSha ?? 'unavailable')); h.update('\n');
+  for (const [label, map] of [['f', fileContents], ['d', depFileContents]]) {
+    const names = Object.keys(map || {}).sort();
+    h.update(label); h.update(String(names.length)); h.update('\n');
+    for (const n of names) {
+      h.update(n); h.update('\0');
+      h.update(_sha(String(map[n] ?? '')));
+      h.update('\n');
+    }
+  }
+  const envKeys = Object.keys(env || {})
+    .filter(k => k.startsWith('AGENTIC_SECURITY_') && !RUN_KEY_ENV_EXCLUDE.has(k))
+    .sort();
+  h.update('e'); h.update(String(envKeys.length)); h.update('\n');
+  for (const k of envKeys) { h.update(k); h.update('='); h.update(String(env[k])); h.update('\n'); }
+  return h.digest('hex');
+}
+
+// A value is safe to checkpoint only if JSON can carry it back unchanged. Dates,
+// regexes, Maps, Sets, functions and BigInts all survive `JSON.stringify` in a
+// lossy or throwing way; recording one would mean the resumed run sees different
+// data than the uninterrupted run did. We refuse the record instead, and the
+// file just gets rescanned.
+function _jsonSafe(v, depth = 0, seen = new Set()) {
+  if (depth > 24) return false;
+  if (v === null || v === undefined) return true;
+  const t = typeof v;
+  if (t === 'string' || t === 'boolean') return true;
+  if (t === 'number') return Number.isFinite(v);
+  if (t === 'function' || t === 'symbol' || t === 'bigint') return false;
+  if (t !== 'object') return false;
+  if (seen.has(v)) return false;
+  seen.add(v);
+  try {
+    if (Array.isArray(v)) {
+      for (const x of v) if (!_jsonSafe(x, depth + 1, seen)) return false;
+      return true;
+    }
+    const proto = Object.getPrototypeOf(v);
+    if (proto !== Object.prototype && proto !== null) return false;
+    for (const k of Object.keys(v)) if (!_jsonSafe(v[k], depth + 1, seen)) return false;
+    return true;
+  } finally {
+    seen.delete(v);
+  }
+}
+
+function _emptyHandle(reason) {
+  return {
+    enabled: false, file: null, fd: null, runKey: null,
+    recovered: new Map(), order: [], written: new Set(),
+    discarded: false, reason,
+  };
+}
+
+function _headerLine(runKey) {
+  return JSON.stringify({ v: FORMAT, runKey }) + '\n';
+}
+
+// Read forward from a byte offset, keeping records while they verify. Returns
+// the offset of the first byte that did NOT verify, so the caller can truncate.
+function _recover(handle, file, runKey) {
+  let buf;
+  try { buf = fs.readFileSync(file); }
+  catch { return -1; } // no file yet
+  const text = buf.toString('utf8');
+  const nl = text.indexOf('\n');
+  if (nl < 0) return 0;
+  let header = null;
+  try { header = JSON.parse(text.slice(0, nl)); } catch { return 0; }
+  if (!header || header.v !== FORMAT || header.runKey !== runKey) return 0;
+
+  let offset = Buffer.byteLength(text.slice(0, nl + 1), 'utf8');
+  let cursor = nl + 1;
+  for (;;) {
+    const end = text.indexOf('\n', cursor);
+    if (end < 0) break;                       // torn tail: no terminating newline
+    const line = text.slice(cursor, end);
+    cursor = end + 1;
+    if (!line) { offset = Buffer.byteLength(text.slice(0, cursor), 'utf8'); continue; }
+    let rec;
+    try { rec = JSON.parse(line); } catch { break; }
+    if (!rec || typeof rec.f !== 'string' || typeof rec.d !== 'string') break;
+    if (rec.c !== _sha(rec.d)) break;         // tampered or torn-then-patched
+    let payload;
+    try { payload = JSON.parse(rec.d); } catch { break; }
+    if (!handle.recovered.has(rec.f)) handle.order.push(rec.f);
+    handle.recovered.set(rec.f, payload);
+    offset = Buffer.byteLength(text.slice(0, cursor), 'utf8');
+  }
+  return offset;
+}
+
+/**
+ * Open (or start) the checkpoint for `scanRoot` under `runKey`. Never throws.
+ * A handle whose `enabled` is false silently no-ops through the rest of the API.
+ */
+export function openCheckpoint(scanRoot, { runKey } = {}) {
+  if (!scanRoot || !runKey) return _emptyHandle('no-run-key');
+  const handle = _emptyHandle(null);
+  try {
+    const dir = path.join(scanRoot, STATE_DIR);
+    fs.mkdirSync(dir, { recursive: true });
+    const file = checkpointPath(scanRoot);
+    handle.file = file;
+    handle.runKey = runKey;
+
+    const keepBytes = _recover(handle, file, runKey);
+    if (keepBytes <= 0) {
+      // Absent, foreign, or unreadable — start clean. Conservative by design.
+      handle.recovered.clear();
+      handle.order.length = 0;
+      handle.discarded = keepBytes === 0;
+      fs.writeFileSync(file, _headerLine(runKey));
+    } else {
+      // Drop any torn tail so appends land after the last verified record.
+      try {
+        const size = fs.statSync(file).size;
+        if (size !== keepBytes) fs.truncateSync(file, keepBytes);
+      } catch { /* best-effort */ }
+    }
+
+    handle.fd = fs.openSync(file, 'a');
+    handle.enabled = true;
+  } catch (e) {
+    try { if (handle.fd !== null) fs.closeSync(handle.fd); } catch { /* ignore */ }
+    return _emptyHandle(String((e && e.message) || e));
+  }
+  return handle;
+}
+
+/**
+ * Durably record that `relPath` is fully analysed, along with everything that
+ * analysis produced. `findings` is the per-file payload object (see the engine
+ * call site); it must be plain JSON data. Returns true only if the record is on
+ * disk and fsync'd.
+ */
+export function recordFileDone(handle, relPath, findings) {
+  if (!handle || !handle.enabled || handle.fd === null || typeof relPath !== 'string') return false;
+  try {
+    if (!_jsonSafe(findings)) return false;
+    const d = JSON.stringify(findings === undefined ? null : findings);
+    if (typeof d !== 'string') return false;
+    const line = JSON.stringify({ f: relPath, c: _sha(d), d }) + '\n';
+    fs.writeSync(handle.fd, line);
+    fs.fsyncSync(handle.fd);
+    handle.written.add(relPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Files already analysed — recovered from a prior run plus written by this one. */
+export function completedFiles(handle) {
+  const out = new Set();
+  if (!handle) return out;
+  for (const f of handle.recovered ? handle.recovered.keys() : []) out.add(f);
+  for (const f of handle.written || []) out.add(f);
+  return out;
+}
+
+/** Recovered per-file payloads, in the order they were originally recorded. */
+export function resumeFindings(handle) {
+  if (!handle || !handle.recovered) return [];
+  return (handle.order || []).map(file => ({ file, findings: handle.recovered.get(file) }));
+}
+
+/**
+ * Close the handle. `complete: true` means the scan finished — the checkpoint is
+ * removed so the next run cannot resume state that has already been consumed.
+ */
+export function closeCheckpoint(handle, { complete = false } = {}) {
+  if (!handle || !handle.enabled) return false;
+  let ok = true;
+  try { if (handle.fd !== null) fs.closeSync(handle.fd); } catch { ok = false; }
+  handle.fd = null;
+  handle.enabled = false;
+  if (complete && handle.file) {
+    try { fs.rmSync(handle.file, { force: true }); } catch { ok = false; }
+  }
+  return ok;
+}

package/src/posture/suppressions.js

@@ -7,7 +7,7 @@
 
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import * as yaml from 'js-yaml';
+import * as yaml from '../util/yaml.js';
 import { statePath, safeWriteState } from './state-dir.js';
 
 const MS_PER_DAY = 86400000;