@clear-capabilities/agentic-security-scanner 0.172.1 → 0.173.0

package/src/posture/provenance/git-evidence.js

@@ -277,6 +277,24 @@ export function candidateCommitsForLine(scanRoot, file, line, { since } = {}) {
   return [...new Set(shas)];
 }
 
+/**
+ * The names a file had before it was last renamed or moved, nearest first (E7-02). `git log --follow` reports each rename as `R<score> old new`; chaining
+ * them back gives every earlier name, so a replay at an old commit can look for the file where it used to be. Empty for a file that was never renamed or
+ * where the history cannot be read. Not recorded: copies, which git reports only with extra flags and which do not rename a file.
+ */
+export function renameChain(scanRoot, file) {
+  const rel = _relPath(scanRoot, file);
+  if (!rel) return [];
+  const r = _run(scanRoot, ['log', '--follow', '-M', '--name-status', '--format=commit %H', '--no-textconv', '--', rel]);
+  if (!r.ok) return [];
+  const names = []; let current = rel;
+  for (const ln of r.stdout.split('\n')) {
+    const m = /^R\d*\t([^\t]+)\t([^\t]+)$/.exec(ln);
+    if (m && m[2] === current) { names.push(m[1]); current = m[1]; }
+  }
+  return names;
+}
+
 export function candidateCommitsForFile(scanRoot, file, { since } = {}) {
   const rel = _relPath(scanRoot, file);
   if (!rel) return [];
@@ -311,9 +329,75 @@ export function getRootCommits(scanRoot) {
   return r.stdout.split('\n').map((s) => s.trim()).filter(Boolean);
 }
 
+// ---- one blame per file, not one per line (E7-02) ---------------------------------------------------------------------------------
+//
+// Resolving a finding blames the finding's line, and again for every evidence line on its path. A file with a dozen findings therefore started a dozen or
+// more `git blame` processes, and on a large repository that is what used up the provenance time budget before the later findings were reached
+// (`budget_exhausted`). Attribution of a line does not depend on which other lines were asked about, so the second time a file is asked about it is blamed
+// whole, once, and every later line is answered from memory. The first question about a file still uses the cheap single-line form, so a file with one
+// finding costs what it always did. The answer is keyed by the file's size and modification time, so a file that changed mid-scan is blamed again.
+const _fileBlame = new Map();
+const _seenOnce = new Set();
+const WHOLE_FILE_MAX_LINES = 20000;
+const GIT_WHOLE_FILE_TIMEOUT_MS = 20000;
+
+function parseBlamePorcelain(out) {
+  const lines = new Map(); const metaBySha = new Map();
+  const rows = out.split('\n');
+  let cur = null;
+  for (let i = 0; i < rows.length; i++) {
+    const r = rows[i];
+    if (cur === null) {
+      const m = /^([0-9a-f]{40,64}) (\d+) (\d+)(?: (\d+))?$/.exec(r);
+      if (m) { cur = { sha: m[1], final: Number(m[3]) }; if (!metaBySha.has(cur.sha)) metaBySha.set(cur.sha, { commit: cur.sha }); }
+      continue;
+    }
+    if (r.startsWith('\t')) {
+      const meta = metaBySha.get(cur.sha);
+      lines.set(cur.final, /^0+$/.test(cur.sha) ? { commit: null, uncommitted: true } : { ...meta });
+      cur = null; continue;
+    }
+    const meta = metaBySha.get(cur.sha);
+    if (r.startsWith('author ')) meta.authorName = r.slice(7);
+    else if (r.startsWith('author-mail ')) meta.authorEmail = r.slice(12).replace(/[<>]/g, '');
+    else if (r.startsWith('author-time ')) meta.authorDate = new Date(parseInt(r.slice(12), 10) * 1000).toISOString();
+    else if (r.startsWith('summary ')) meta.summary = r.slice(8);
+  }
+  return lines;
+}
+
+function blameWholeFile(scanRoot, rel) {
+  const abs = path.join(scanRoot, rel);
+  let st; try { st = fs.statSync(abs); } catch { return null; }
+  const key = `${st.size}:${st.mtimeMs}`;
+  const cacheKey = `${scanRoot}\0${rel}`;
+  const hit = _fileBlame.get(cacheKey);
+  if (hit && hit.key === key) return hit.lines;
+  if (st.size > 2 * 1024 * 1024) return null;
+  let r;
+  try {
+    const stdout = cp.execFileSync('git', hardenGitArgs(['blame', '--porcelain', '--no-textconv', '--', rel]), { cwd: scanRoot, encoding: 'utf8', timeout: GIT_WHOLE_FILE_TIMEOUT_MS, stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 64 * 1024 * 1024, env: hardenGitEnv() });
+    r = { ok: true, stdout };
+  } catch { r = { ok: false }; }
+  if (!r.ok) return null;
+  const lines = parseBlamePorcelain(r.stdout);
+  if (lines.size === 0 || lines.size > WHOLE_FILE_MAX_LINES) return null;
+  _fileBlame.set(cacheKey, { key, lines });
+  return lines;
+}
+
+/** Forget remembered blames (tests, and a long-lived process that wants the memory back). */
+export function clearBlameCache() { _fileBlame.clear(); _seenOnce.clear(); }
+
 export function blameLine(scanRoot, file, line) {
   const rel = _relPath(scanRoot, file);
   if (!rel || !line || line < 1) return null;
+  const once = `${scanRoot}\0${rel}`;
+  if (process.env.AGENTIC_SECURITY_NO_BLAME_CACHE !== '1' && _seenOnce.has(once)) {
+    const whole = blameWholeFile(scanRoot, rel);
+    const m = whole && whole.get(line);
+    if (m) return { ...m };
+  } else _seenOnce.add(once);
   // `--no-textconv`: VERIFIED exploitable without it — `git blame` applies
   // a hostile `.gitattributes` textconv driver by default in current git.
   const r = _run(scanRoot, ['blame', '-L', `${line},${line}`, '--porcelain', '--no-textconv', '--', rel]);

package/src/posture/provenance/origin-resolver.js

@@ -30,6 +30,12 @@ import { checkAbsentInSomeParent, detectRevert, detectCherryPick } from './dag-w
 import { loadRepoLineage } from './repo-lineage.js';
 import { resolveAIAuthorship } from './ai-authorship.js';
 
+// Findings produced by the deep (IR taint) layer exist only when that layer runs. A replay that skips it can never see such a finding at ANY commit, so origin
+// resolution would end "never confirmed" for every one of them (78 of 559 findings in this repository's own scan did). The same reasoning is applied to
+// Haskell and Nix in predicate-replay.js.
+const DEEP_PARSERS = new Set(['IR-TAINT', 'IR-TAINT-IFDS', 'IR-PRIVACY-TAINT']);
+const needsDeepReplay = (finding) => !!finding && DEEP_PARSERS.has(finding.parser);
+
 function relevantFiles(finding) {
   const files = new Set();
   if (finding.file) files.add(finding.file);
@@ -224,7 +230,7 @@ export async function resolveOrigin(scanRoot, finding, { since, deadlineAt, repo
   const replayCache = new Map();
   const replay = (sha) => {
     if (replayCache.has(sha)) return replayCache.get(sha);
-    const p = replayAt(scanRoot, sha, files, stableId);
+    const p = replayAt(scanRoot, sha, files, stableId, { deep: needsDeepReplay(finding) });
     replayCache.set(sha, p);
     return p;
   };

package/src/posture/provenance/predicate-replay.js

@@ -11,7 +11,7 @@
 
 import { runFullScan, _snapshotSuppressionLog, _restoreSuppressionLog } from '../../engine.js';
 import { computeStableId } from '../stable-id.js';
-import { getBlobAtCommit, listFilesAtCommit } from './git-evidence.js';
+import { getBlobAtCommit, listFilesAtCommit, renameChain } from './git-evidence.js';
 
 // Task 11 concurrency fix: `coordinator.js` resolves several findings' origins
 // CONCURRENTLY (its own comment: "the scheduler runs these four at a time"),
@@ -57,7 +57,11 @@ export function _runExclusive(fn) {
   return run;
 }
 
-export async function replayAt(scanRoot, sha, files, targetStableId) {
+/**
+ * @param {object} [opts] `deep` replays with the deep layer on. `renames` follows a file back through renames and moves: at a commit where the current path
+ *   does not exist, the file's earlier names are tried, and the content is read under the CURRENT path so the replayed finding keeps the identity it has now.
+ */
+export async function replayAt(scanRoot, sha, files, targetStableId, opts = {}) {
   const fileContents = {};
   // A NixOS option is decided by every module the configuration imports, including ones deleted later, so a Nix
   // finding is replayed against ALL Nix sources and the lock file as they existed at that commit.
@@ -65,8 +69,14 @@ export async function replayAt(scanRoot, sha, files, targetStableId) {
     const all = listFilesAtCommit(scanRoot, sha, /(?:\.nix|(?:^|\/)flake\.lock)$/i);
     files = [...new Set([...files, ...all])];
   }
+  let chains = null;
   for (const f of files) {
-    const content = getBlobAtCommit(scanRoot, sha, f);
+    let content = getBlobAtCommit(scanRoot, sha, f);
+    if (content == null) {
+      chains = chains || new Map();
+      if (!chains.has(f)) chains.set(f, renameChain(scanRoot, f));
+      for (const old of chains.get(f)) { content = getBlobAtCommit(scanRoot, sha, old); if (content != null) break; }
+    }
     if (content != null) fileContents[f] = content;
   }
   if (Object.keys(fileContents).length === 0) {
@@ -108,7 +118,7 @@ export async function replayAt(scanRoot, sha, files, targetStableId) {
         // message for methodology.
         // Haskell/Nix findings come from the IR-taint layer, which only runs in deep mode: a replay that skips it
         // can never see the finding at ANY commit, so origin resolution would end "never confirmed" for every one.
-        const _langDeep = Object.keys(fileContents).some((f) => /\.(?:l?hs|nix)$/i.test(f)) ? { deep: true } : {};
+        const _langDeep = (opts.deep || Object.keys(fileContents).some((f) => /\.(?:l?hs|nix)$/i.test(f))) ? { deep: true } : {};
         return await runFullScan({ fileContents, scanRoot, provenance: false, skipAnnotators: true, ..._langDeep }, () => {});
       } finally {
         _restoreSuppressionLog(_suppSnapshot);

package/src/posture/runtime-correlation.js

@@ -36,6 +36,8 @@ import * as readline from 'node:readline';
 import { createReadStream } from 'node:fs';
 
 import { statePath } from './state-dir.js';
+import { loadTraceMeta } from './runtime-trace-ingest.js';
+import { linkObservationToSnapshot, normalizeIdentity } from '../lineage/deployment-identity.js';
 const DEFAULT_TRACE_NAMES = ['runtime-trace.jsonl', 'runtime.jsonl', 'ebpf-trace.jsonl'];
 const DEFAULT_OBSERVATION_WINDOW_DAYS = 30;
 
@@ -55,7 +57,12 @@ export async function loadTrace(scanRoot, opts = {}) {
     fileLinesObserved: new Map(), // file → Set<line>
     syscallsObserved: new Set(),
     recordCount: 0,
+    // E5-02: with a snapshot identity, each record is compared with it. A record from a different build is dropped and counted; one that carries no
+    // comparable identity is kept as unattested; one that matches is counted as matched.
+    recordsMatched: 0, recordsUnattested: 0, recordsMismatched: 0,
   };
+  trace.meta = loadTraceMeta(chosen);
+  const snap = opts.snapshotIdentity ? normalizeIdentity(opts.snapshotIdentity) : null;
   const window = (opts.windowDays || DEFAULT_OBSERVATION_WINDOW_DAYS) * 86400_000;
   const now = Date.now();
   const stream = createReadStream(chosen, { encoding: 'utf8' });
@@ -68,6 +75,11 @@ export async function loadTrace(scanRoot, opts = {}) {
       const tsMs = Date.parse(r.ts);
       if (Number.isFinite(tsMs) && now - tsMs > window) continue;
     }
+    if (snap) {
+      const link = linkObservationToSnapshot({ commit: r.commit, buildHash: r.buildHash, imageDigest: r.imageDigest, sbomRef: r.sbomRef, environment: r.environment }, snap);
+      if (link.state === 'mismatched') { trace.recordsMismatched++; continue; }
+      if (link.state === 'matched') trace.recordsMatched++; else trace.recordsUnattested++;
+    } else trace.recordsUnattested++;
     trace.recordCount++;
     switch (r.kind) {
       case 'function-call':
@@ -138,38 +150,72 @@ export function findingObservedInRuntime(finding, trace) {
 }
 
 /**
- * Annotate all findings with runtimeObserved + demote unobserved findings.
- * Demotion: critical → high, high → medium, medium → low.
+ * Annotate every finding with what the runtime trace says about it, and say what that does NOT mean.
  *
- * Findings classified as unknown are left alone (no demotion). This is
- * the principled position — absence of observation in a partial trace
- * is not evidence of dead code.
+ * `runtimeObserved` keeps its three values (true, false, 'unknown'); `runtimeEvidence` states them in words:
+ *   observed               a function, line or route on the finding's path appears in the trace
+ *   not_observed_in_window nothing on its path appears in the window. This is a statement about the window and the instrumentation, NOT a statement
+ *                          that the code is unreachable or safe: low traffic, a rare path, an uninstrumented file and an incomplete window all look
+ *                          the same from here
+ *   unknown                no trace, or the file appears with no matching line
+ *
+ * SEVERITY IS NOT CHANGED BY DEFAULT. This module used to lower the severity of every unobserved finding one tier. Absence of observation cannot
+ * justify that (and the lineage observation path never did), so demotion is now an explicit, opt-in policy: `opts.demote === true` or
+ * AGENTIC_SECURITY_RUNTIME_DEMOTE=1. Even then it needs cited evidence: the trace must hold at least `minRecords` records, the finding must be rooted in a
+ * route, and that route must be absent from a trace that does hold route hits (traffic reached the service and did not reach this route). A finding that
+ * fails any condition is left as it is.
+ * A demoted finding records the evidence under `_runtimeDemotionEvidence`; a finding is never removed.
  */
 export async function annotateRuntimeCorrelation(scanRoot, findings, opts = {}) {
-  if (!Array.isArray(findings)) return { observed: 0, dead: 0, unknown: 0 };
+  if (!Array.isArray(findings)) return { observed: 0, dead: 0, unknown: 0, demoted: 0 };
   const trace = await loadTrace(scanRoot, opts);
-  if (!trace) {
-    for (const f of findings) f.runtimeObserved = 'unknown';
-    return { observed: 0, dead: 0, unknown: findings.length, trace: null };
+  const windowDays = opts.windowDays || DEFAULT_OBSERVATION_WINDOW_DAYS;
+  if (!trace || trace.recordCount === 0) {
+    // no file, or nothing usable in it (empty, outside the window, or every record from another build): there is nothing to compare a finding with
+    const basis = !trace ? 'no runtime trace was available' : (trace.recordsMismatched > 0 ? `all ${trace.recordsMismatched} usable record(s) came from a different build` : 'the trace holds no records inside the window');
+    for (const f of findings) { f.runtimeObserved = 'unknown'; f.runtimeEvidence = { state: 'unknown', basis, windowDays, recordCount: 0, ...(trace ? { recordsMismatched: trace.recordsMismatched } : {}) }; }
+    return { observed: 0, dead: 0, unknown: findings.length, demoted: 0, trace: trace ? { recordCount: 0, path: trace.path, recordsMismatched: trace.recordsMismatched } : null };
   }
-  let observed = 0, dead = 0, unknown = 0;
+  const demote = opts.demote === true || process.env.AGENTIC_SECURITY_RUNTIME_DEMOTE === '1';
+  const minRecords = Number.isInteger(opts.minRecords) ? opts.minRecords : 100;
+  let observed = 0, dead = 0, unknown = 0, demoted = 0;
   const demoteLadder = { critical: 'high', high: 'medium', medium: 'low', low: 'info' };
   for (const f of findings) {
-    const v = findingObservedInRuntime(f, trace);
+    let v = findingObservedInRuntime(f, trace);
+    // E5-03: a trace ingested with sampling below 1, or with no declared sampling, cannot show that something did NOT run. Absence becomes unknown.
+    if (v === false && trace.meta && trace.meta.sampleRate !== 1) v = 'unknown';
     f.runtimeObserved = v;
+    const file = f.file || (f.sink && f.sink.file);
+    f.runtimeEvidence = v === true ? { state: 'observed', basis: 'a function, line or route on the finding\'s path is in the trace', windowDays, recordCount: trace.recordCount, corroboration: (opts.snapshotIdentity && trace.recordsMatched > 0 && trace.recordsUnattested === 0) ? 'corroborated' : 'unattested', recordsMismatched: trace.recordsMismatched }
+      : v === false ? { state: 'not_observed_in_window', basis: 'nothing on the finding\'s path is in the trace', windowDays, recordCount: trace.recordCount, meaning: 'absence of observation is not evidence that the code is unreachable or safe' }
+      : { state: 'unknown', basis: 'the trace cannot say (the file appears but no line on the path matches, or there is no trace)', windowDays, recordCount: trace.recordCount };
+    f.runtimeEvidence.sampling = samplingOf(trace);
     if (v === true) observed++;
     else if (v === false) {
       dead++;
-      // Demote one tier — the finding is real but unobserved in 30 days
-      // of production traffic, so it's not P0.
       const next = demoteLadder[f.severity];
-      if (next) {
+      // The cited evidence is TRAFFIC ON THE FINDING'S OWN ROUTE'S NEIGHBOURS: the finding is rooted in a route, the trace holds route hits, and this
+      // route is not among them. A finding with no route, or a trace with no route hits, has nothing to cite and is left alone.
+      const routePath = f._inRoute && f._inRoute.path;
+      const routeEvidence = !!(routePath && trace.routesObserved.size > 0 && !trace.routesObserved.has(routePath));
+      if (demote && next && routeEvidence && trace.recordCount >= minRecords) {
         f._runtimeDemoted = f.severity;
+        f._runtimeDemotionEvidence = { policy: 'opt-in runtime demotion', basis: 'route-rooted finding whose route was not hit while other routes were', route: routePath, routesObserved: trace.routesObserved.size, recordCount: trace.recordCount, minRecords, windowDays, tracePath: trace.path };
         f.severity = next;
+        demoted++;
       }
     } else unknown++;
   }
-  return { observed, dead, unknown, trace: { recordCount: trace.recordCount, path: trace.path } };
+  return { observed, dead, unknown, demoted, trace: { recordCount: trace.recordCount, path: trace.path } };
+}
+
+/** How completely the trace sees the service: 'complete', 'sampled' (with the rate), or 'undisclosed'. */
+export function samplingOf(trace) {
+  const m = trace && trace.meta;
+  if (!m) return { state: 'undisclosed', basis: 'the trace came with no ingestion record, so how much traffic it samples is not known' };
+  if (m.sampleRate === 1) return { state: 'complete', rate: 1 };
+  if (typeof m.sampleRate === 'number') return { state: 'sampled', rate: m.sampleRate, basis: 'not observed cannot be concluded from a sampled trace' };
+  return { state: 'undisclosed', basis: 'the collector did not declare a sampling rate' };
 }
 
 export const _internals = { DEFAULT_TRACE_NAMES, DEFAULT_OBSERVATION_WINDOW_DAYS };

package/src/posture/runtime-explanation.js

@@ -0,0 +1,116 @@
+// Side-by-side static / runtime explanation of one finding (E5-07).
+//
+// One object, built once from one snapshot, feeds every surface: the CLI JSON, the SARIF result property, the explorer's two panes, and the export.
+// Because they all read the same object, they cannot disagree, and because the object carries a digest of its own content, a pane built from a
+// different snapshot is detected (`panesInSync`) instead of being shown beside the other.
+//
+//   static    what the code says: the finding's static reachability level and the edges of its path
+//   runtime   what a running build showed: per-edge state, build-match counts, sampling, and the observations that are MISSING, each named
+//   compare   the path's corroboration (pathCorroboration): corroborated only when every edge is `both`
+//
+// DETERMINISTIC. No clock, no randomness, keys in a fixed order, arrays sorted: the same inputs give byte-identical output and the same digest.
+// OFFLINE. Nothing here reads the network; the inputs are values already computed.
+// PRIVATE. The explanation is built field by field from normalized values (routes with ids generalised, symbols, counts, states). No raw runtime
+// record is embedded, and `exportExplanation` scans every string it is about to emit with the same canary patterns the ingestion step uses and
+// REFUSES to export if one matches.
+import { createHash } from 'node:crypto';
+import { canaryHit } from './runtime-trace-ingest.js';
+import { pathCorroboration } from './graph-reconciliation.js';
+import { buildEvidenceLevels } from './exploitability-evidence.js';
+
+const EXPLANATION_SCHEMA = 'agentic-security/runtime-explanation@1';
+const SKIP_CANARY = new Set(['commit', 'buildHash', 'imageDigest', 'digest', 'snapshotDigest']);
+
+const canon = (v) => (Array.isArray(v) ? v.map(canon) : v && typeof v === 'object' ? Object.fromEntries(Object.keys(v).sort().map((k) => [k, canon(v[k])])) : v);
+const digestOf = (o) => createHash('sha256').update(JSON.stringify(canon(o))).digest('hex');
+
+function missingFor(edge) {
+  if (!edge) return 'the path edge is not in the reconciled graph: no runtime statement is possible';
+  if (edge.state === 'both') return null;
+  if (edge.state === 'conflicted') return `runtime evidence contradicts the static graph for ${edge.key}`;
+  if (edge.state === 'unknown') return edge.reason || `runtime records for ${edge.key} cannot be tied to this snapshot`;
+  if (edge.mismatchedRecords > 0) return `only a different build was observed using ${edge.key}`;
+  return `no runtime record observed for ${edge.key}`;
+}
+
+/**
+ * @param {object} finding
+ * @param {{reconciled?:object, pathKeys?:string[], sampling?:object, snapshotIdentity?:object, windowDays?:number}} ctx
+ */
+export function buildRuntimeExplanation(finding, ctx = {}) {
+  const { levels } = buildEvidenceLevels(finding);
+  const keys = [...(ctx.pathKeys || [])];
+  const byKey = new Map(((ctx.reconciled && ctx.reconciled.edges) || []).map((e) => [e.key, e]));
+  const edges = keys.map((k) => { const e = byKey.get(k); return { key: k, state: e ? e.state : 'statically_inferred', confidence: e ? e.confidence : null, matchedRecords: e ? e.matchedRecords : 0, mismatchedRecords: e ? e.mismatchedRecords : 0, unattestedRecords: e ? e.unattestedRecords : 0 }; });
+  const missing = [];
+  if (!ctx.reconciled) missing.push('no runtime trace was reconciled with the static graph');
+  keys.forEach((k) => { const m = ctx.reconciled ? missingFor(byKey.get(k)) : null; if (m) missing.push(m); });
+  const sampling = ctx.sampling && ctx.sampling.state ? { state: ctx.sampling.state, ...(typeof ctx.sampling.rate === 'number' ? { rate: ctx.sampling.rate } : {}) } : { state: 'undisclosed' };
+  if (sampling.state !== 'complete') missing.push(sampling.state === 'sampled' ? `the trace samples ${sampling.rate} of traffic: absence of an observation proves nothing` : 'the trace does not say how much traffic it samples: absence of an observation proves nothing');
+  const s = ctx.snapshotIdentity || {};
+  const snapshot = { commit: s.commit ?? null, buildHash: s.buildHash ?? null, imageDigest: s.imageDigest ?? null, environment: s.environment ?? null, dirty: s.dirty === true };
+  const body = {
+    schema: EXPLANATION_SCHEMA,
+    findingId: finding && finding.id != null ? String(finding.id) : null,
+    // the refactor-stable id other evidence is keyed by; `findingId` alone is the scan-local id and changes when code moves
+    findingStableId: finding && finding.stableId != null ? String(finding.stableId) : null,
+    snapshot,
+    static: { staticallyReachable: levels.staticallyReachable, externallyReachable: levels.externallyReachable, pathEdges: keys },
+    runtime: { observed: levels.runtimeObserved, sampling, windowDays: ctx.windowDays ?? null, edges, missingObservations: [...new Set(missing)].sort() },
+    compare: ctx.reconciled ? pathCorroboration(ctx.reconciled, keys) : { state: 'uncorroborated', reason: 'no runtime evidence was supplied' },
+    offline: true,
+  };
+  return { ...body, digest: digestOf(body) };
+}
+
+/** True when the explanation still hashes to its own digest, i.e. nothing was altered after it was built. */
+export function explanationIntact(e) {
+  if (!e || typeof e !== 'object' || typeof e.digest !== 'string') return false;
+  const { digest, ...body } = e;
+  return digestOf(body) === digest;
+}
+
+/** The properties a SARIF result or CLI JSON entry carries. A compact view of the same object, with the digest that ties it back. */
+export function explanationSummary(e) {
+  return { schema: e.schema, digest: e.digest, compare: e.compare.state, runtimeObserved: e.runtime.observed.value, sampling: e.runtime.sampling.state, missingObservations: e.runtime.missingObservations.length, snapshotCommit: e.snapshot.commit };
+}
+
+/**
+ * The explorer's two panes, both derived from one explanation. Selecting an edge returns BOTH panes for that edge, stamped with the snapshot digest.
+ */
+export function buildViewModel(e) {
+  const rows = new Map(e.runtime.edges.map((r) => [r.key, r]));
+  return {
+    snapshotDigest: e.digest,
+    keys: [...rows.keys()],
+    select(key) {
+      const r = rows.get(key);
+      if (!r) return null;
+      return {
+        key, snapshotDigest: e.digest,
+        staticPane: { key, kind: key.split('|')[0], inStaticGraph: true, reachability: e.static.staticallyReachable.value },
+        runtimePane: { key, state: r.state, confidence: r.confidence, matched: r.matchedRecords, mismatched: r.mismatchedRecords, unattested: r.unattestedRecords, missing: r.state === 'both' ? null : missingFor({ key, ...r }) },
+      };
+    },
+  };
+}
+
+/** Two panes may be shown together only when they are about the same key and were built from the same snapshot. */
+export function panesInSync(sel) {
+  return !!sel && sel.staticPane.key === sel.runtimePane.key && sel.staticPane.key === sel.key && typeof sel.snapshotDigest === 'string';
+}
+export function selectionsInSync(a, b) { return !!a && !!b && a.snapshotDigest === b.snapshotDigest && a.key === b.key; }
+
+/**
+ * JSON for sharing. Refuses (throws) when any string in it matches a secret or personal-data pattern.
+ */
+export function exportExplanation(e) {
+  if (!explanationIntact(e)) throw new Error('refusing to export: the explanation was altered after it was built');
+  const walk = (v, k) => {
+    if (typeof v === 'string') { if (!SKIP_CANARY.has(k)) { const hit = canaryHit(v); if (hit) throw new Error(`refusing to export: a value matches the ${hit} pattern`); } }
+    else if (Array.isArray(v)) v.forEach((x) => walk(x, k));
+    else if (v && typeof v === 'object') for (const [kk, x] of Object.entries(v)) walk(x, kk);
+  };
+  walk(e, null);
+  return JSON.stringify(canon(e), null, 2) + '\n';
+}

package/src/posture/runtime-trace-ingest.js

@@ -0,0 +1,194 @@
+// Opt-in, privacy-preserving ingestion of runtime observations (E5-03).
+//
+// A runtime collector can see far more than this scanner needs: request bodies, headers, tokens, user identifiers. The scanner needs only WHICH
+// route, function or file was exercised, and in which build. This module is the one door between a collector's output and the trace file that
+// runtime-correlation.js reads, and it lets through only that.
+//
+// RULES (each is a test)
+//   1. Off by default. Nothing is ingested unless the project's runtime policy says `collect: true` (or the caller passes that policy in).
+//   2. An allowlist, not a blocklist. A record is rebuilt field by field from the fields below; every other field is dropped, counted by NAME and
+//      never persisted, whatever it holds. A new field a collector invents is dropped until this list names it.
+//   3. Values must look like what the field is: a route is a path, a symbol is a dotted identifier, a file is a relative path. A value that does not
+//      fit rejects the whole record.
+//   4. A canary scan runs over every value that IS allowed through (a route like /users/jane@example.com/orders, a symbol that is really a token):
+//      an email, bearer token, JWT, cloud key, private key, card number (Luhn), SSN, or long opaque string rejects the whole record.
+//      Query strings and fragments never survive: the route is the path only, and id-like segments (numbers, UUIDs, long hex) become `:id`.
+//   5. Retention: records older than `retentionDays` are removed from the file when it is next written, and by `applyRetention`.
+//   6. Disclosure: a sidecar `runtime-trace.meta.json` records the policy, the declared sampling rate, what was dropped and rejected (counts by
+//      reason, never contents), and when. A trace with sampling below 1, or no declared sampling, can never support "not observed": see
+//      runtime-correlation.js. Everything works offline from local files.
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { statePath } from './state-dir.js';
+import { safeWriteState } from './state-dir.js';
+
+export const POLICY_FILE = 'runtime-policy.json';
+const TRACE_FILE = 'runtime-trace.jsonl';
+const META_FILE = 'runtime-trace.meta.json';
+const DEFAULT_RETENTION_DAYS = 30;
+const ALLOWED_KINDS = Object.freeze(['function-call', 'route-hit', 'syscall', 'file-touch', 'service-call']);
+export const ALLOWED_FIELDS = Object.freeze(['ts', 'kind', 'qid', 'route', 'fileRel', 'line', 'count', 'name', 'caller', 'service', 'peer', 'commit', 'buildHash', 'imageDigest', 'sbomRef', 'environment']);
+
+const SYMBOL = /^[A-Za-z_$][\w$]*(?:[.:#/][A-Za-z_$][\w$]*)*(?:\(\))?$/;
+const TOKEN = /^[A-Za-z0-9._:-]{1,128}$/;
+const REL_PATH = /^(?!.*\.\.)(?!\/)[\w@.+/-]{1,300}$/;
+const SYSCALL = /^[a-z_][a-z0-9_]{0,40}$/;
+
+// Canary patterns: what a secret or personal datum looks like. A hit rejects the record. Deliberately broad: a falsely rejected record costs one
+// observation; a falsely accepted one persists a secret.
+const CANARIES = [
+  ['email', /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/],
+  ['bearer-token', /\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/i],
+  ['jwt', /\beyJ[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]{6,}\.[A-Za-z0-9_-]*/],
+  ['cloud-key', /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b|\bAIza[0-9A-Za-z_-]{30,}\b|\b(?:sk|pk|rk)_(?:live|test)_[0-9A-Za-z]{10,}\b|\bgh[pousr]_[A-Za-z0-9]{20,}\b|\bxox[abprs]-[A-Za-z0-9-]{10,}\b/],
+  ['private-key', /-----BEGIN [A-Z ]*PRIVATE KEY-----/],
+  ['ssn', /\b\d{3}-\d{2}-\d{4}\b/],
+  ['secret-assignment', /\b(?:password|passwd|pwd|secret|token|api[_-]?key|authorization)\s*[=:]\s*\S{3,}/i],
+];
+
+function luhn(digits) {
+  let sum = 0; let alt = false;
+  for (let i = digits.length - 1; i >= 0; i--) { let d = digits.charCodeAt(i) - 48; if (alt) { d *= 2; if (d > 9) d -= 9; } sum += d; alt = !alt; }
+  return sum % 10 === 0;
+}
+
+/** Name of the first canary the string trips, or null. */
+export function canaryHit(s) {
+  if (typeof s !== 'string') return null;
+  for (const [name, re] of CANARIES) if (re.test(s)) return name;
+  for (const m of s.matchAll(/\b\d(?:[ -]?\d){12,18}\b/g)) { const d = m[0].replace(/[ -]/g, ''); if (d.length >= 13 && d.length <= 19 && luhn(d)) return 'payment-card'; }
+  if (/[A-Za-z0-9+/_-]{40,}={0,2}/.test(s)) return 'opaque-string';
+  return null;
+}
+
+const SECRET_CANARY_NAMES = new Set(['bearer-token', 'jwt', 'cloud-key', 'private-key', 'secret-assignment']);
+/** Name of the first SECRET-shaped pattern in a text (keys, tokens, passwords, card numbers), or null. Narrower than canaryHit: personal data and opaque strings are not matched. */
+export function secretCanaryHit(s) {
+  if (typeof s !== 'string') return null;
+  for (const [name, re] of CANARIES) if (SECRET_CANARY_NAMES.has(name) && re.test(s)) return name;
+  for (const m of s.matchAll(/\b\d(?:[ -]?\d){12,18}\b/g)) { const d = m[0].replace(/[ -]/g, ''); if (d.length >= 13 && d.length <= 19 && luhn(d)) return 'payment-card'; }
+  return null;
+}
+
+/** The path of a route with the query and fragment removed and id-like segments generalised. Null when it is not a path. */
+export function normalizeRoute(raw) {
+  if (typeof raw !== 'string' || raw.length > 300) return null;
+  let p = raw.trim();
+  const cut = p.search(/[?#]/); if (cut >= 0) p = p.slice(0, cut);
+  const m = /^[a-z][a-z0-9+.-]*:\/\/[^/]+(\/.*)?$/i.exec(p); if (m) p = m[1] || '/';
+  if (!p.startsWith('/') || /[\s<>"'\\]/.test(p)) return null;
+  return p.split('/').map((seg) => (/^\d+$/.test(seg) || /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(seg) || /^[0-9a-f]{16,}$/i.test(seg) ? ':id' : seg)).join('/');
+}
+
+/**
+ * Rebuild one record from the allowlist.
+ * @returns {{record:object}|{rejected:string, dropped:string[]}}  `dropped` lists the NAMES of fields not persisted (never their values)
+ */
+export function normalizeRecord(raw) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { rejected: 'not-an-object', dropped: [] };
+  const dropped = Object.keys(raw).filter((k) => !ALLOWED_FIELDS.includes(k));
+  if (!ALLOWED_KINDS.includes(raw.kind)) return { rejected: 'unknown-kind', dropped };
+  const out = { kind: raw.kind };
+  const reject = (reason) => ({ rejected: reason, dropped });
+  if (raw.ts !== undefined) { if (typeof raw.ts !== 'string' || !Number.isFinite(Date.parse(raw.ts))) return reject('bad-timestamp'); out.ts = new Date(raw.ts).toISOString(); }
+  if (raw.qid !== undefined) { if (typeof raw.qid !== 'string' || raw.qid.length > 300 || !SYMBOL.test(raw.qid)) return reject('bad-symbol'); out.qid = raw.qid; }
+  if (raw.caller !== undefined) { if (typeof raw.caller !== 'string' || raw.caller.length > 300 || !SYMBOL.test(raw.caller)) return reject('bad-caller'); out.caller = raw.caller; }
+  for (const k of ['service', 'peer']) if (raw[k] !== undefined) { if (typeof raw[k] !== 'string' || !TOKEN.test(raw[k])) return reject(`bad-${k}`); out[k] = raw[k]; }
+  if (raw.route !== undefined) { const r = normalizeRoute(raw.route); if (!r) return reject('bad-route'); out.route = r; }
+  if (raw.fileRel !== undefined) { if (typeof raw.fileRel !== 'string' || !REL_PATH.test(raw.fileRel)) return reject('bad-file'); out.fileRel = raw.fileRel; }
+  if (raw.name !== undefined) { if (typeof raw.name !== 'string' || !SYSCALL.test(raw.name)) return reject('bad-name'); out.name = raw.name; }
+  for (const k of ['line', 'count']) if (raw[k] !== undefined) { if (!Number.isInteger(raw[k]) || raw[k] < 0 || raw[k] > 1e9) return reject(`bad-${k}`); out[k] = raw[k]; }
+  for (const k of ['commit', 'buildHash', 'imageDigest', 'sbomRef', 'environment']) if (raw[k] !== undefined) { if (typeof raw[k] !== 'string' || !TOKEN.test(raw[k])) return reject(`bad-${k}`); out[k] = raw[k]; }
+  // canary scan over everything that is about to be persisted, except the identity fields whose shape (hex, digest) is itself long and opaque
+  for (const [k, v] of Object.entries(out)) {
+    if (typeof v !== 'string' || ['kind', 'ts', 'commit', 'buildHash', 'imageDigest', 'sbomRef'].includes(k)) continue;
+    const hit = canaryHit(v);
+    if (hit) return reject(`canary:${hit}`);
+  }
+  if (raw.kind === 'service-call' && (!out.service || !out.peer)) return reject('service-call-without-endpoints');
+  if (raw.kind === 'route-hit' && !out.route) return reject('route-hit-without-route');
+  if (raw.kind === 'function-call' && !out.qid && !out.fileRel) return reject('function-call-without-symbol');
+  return { record: out, dropped };
+}
+
+/** Read the project's runtime policy; null when absent or unreadable. */
+function loadRuntimePolicy(scanRoot) {
+  try {
+    const p = JSON.parse(fs.readFileSync(statePath(scanRoot, POLICY_FILE), 'utf8'));
+    return p && typeof p === 'object' ? p : null;
+  } catch { return null; }
+}
+
+const bump = (o, k) => { o[k] = (o[k] || 0) + 1; };
+
+/** Remove records older than the retention window from the trace file. Returns how many were removed. */
+export function applyRetention(scanRoot, { retentionDays = DEFAULT_RETENTION_DAYS, now = Date.now() } = {}) {
+  const file = statePath(scanRoot, TRACE_FILE);
+  let text; try { text = fs.readFileSync(file, 'utf8'); } catch { return { removed: 0, kept: 0 }; }
+  const keep = []; let removed = 0;
+  for (const line of text.split('\n')) {
+    if (!line.trim()) continue;
+    let r; try { r = JSON.parse(line); } catch { removed++; continue; }
+    const t = r && r.ts ? Date.parse(r.ts) : NaN;
+    if (Number.isFinite(t) && now - t > retentionDays * 86400_000) { removed++; continue; }
+    keep.push(line);
+  }
+  if (removed > 0 && !safeWriteState(file, keep.length ? keep.join('\n') + '\n' : '')) return { removed: 0, kept: keep.length + removed, error: 'could not rewrite the trace file' };
+  return { removed, kept: keep.length };
+}
+
+/**
+ * Ingest collector records.
+ * @param {string} scanRoot
+ * @param {object[]} records
+ * @param {object} opts policy (overrides the policy file), sampleRate (declared by the collector, 0..1), now
+ * @returns {{ok:boolean, reason?:string, accepted:number, rejected:Object<string,number>, droppedFields:Object<string,number>, meta?:object}}
+ */
+export function ingestRuntimeRecords(scanRoot, records, opts = {}) {
+  const policy = opts.policy || loadRuntimePolicy(scanRoot);
+  if (!policy || policy.collect !== true) return { ok: false, reason: 'runtime collection is not enabled: set {"collect": true} in .agentic-security/runtime-policy.json', accepted: 0, rejected: {}, droppedFields: {} };
+  if (!Array.isArray(records)) return { ok: false, reason: 'records must be an array', accepted: 0, rejected: {}, droppedFields: {} };
+  const allowedKinds = Array.isArray(policy.kinds) ? policy.kinds.filter((k) => ALLOWED_KINDS.includes(k)) : ALLOWED_KINDS;
+  const retentionDays = Number.isInteger(policy.retentionDays) && policy.retentionDays > 0 ? policy.retentionDays : DEFAULT_RETENTION_DAYS;
+  const declared = opts.sampleRate !== undefined ? opts.sampleRate : policy.sampleRate;
+  const sampleRate = typeof declared === 'number' && declared > 0 && declared <= 1 ? declared : null;
+  const now = opts.now ?? Date.now();
+  const rejected = {}; const droppedFields = {}; const lines = [];
+  for (const raw of records) {
+    const n = normalizeRecord(raw);
+    for (const d of n.dropped || []) bump(droppedFields, d);
+    if (n.rejected) { bump(rejected, n.rejected.startsWith('canary:') ? n.rejected : n.rejected); continue; }
+    if (!allowedKinds.includes(n.record.kind)) { bump(rejected, 'kind-not-allowed-by-policy'); continue; }
+    if (!n.record.ts) n.record.ts = new Date(now).toISOString();
+    lines.push(JSON.stringify(n.record));
+  }
+  const file = statePath(scanRoot, TRACE_FILE);
+  const ret = applyRetention(scanRoot, { retentionDays, now });
+  let existing = ''; try { existing = fs.readFileSync(file, 'utf8'); } catch { /* none yet */ }
+  if (lines.length && !safeWriteState(file, existing + lines.join('\n') + '\n')) return { ok: false, reason: 'could not write the trace file (no project marker or state writes disabled)', accepted: 0, rejected, droppedFields };
+  const prior = (() => { try { return JSON.parse(fs.readFileSync(statePath(scanRoot, META_FILE), 'utf8')); } catch { return null; } })();
+  const meta = {
+    schema: 'agentic-security/runtime-trace-meta@1',
+    updatedAt: new Date(now).toISOString(),
+    retentionDays,
+    // null means the collector did not say. Both null and < 1 mean "absence of an observation proves nothing"
+    sampleRate,
+    samplingDisclosed: sampleRate !== null,
+    totals: {
+      accepted: ((prior && prior.totals && prior.totals.accepted) || 0) + lines.length,
+      rejected: sumInto((prior && prior.totals && prior.totals.rejected) || {}, rejected),
+      droppedFields: sumInto((prior && prior.totals && prior.totals.droppedFields) || {}, droppedFields),
+      expiredByRetention: ((prior && prior.totals && prior.totals.expiredByRetention) || 0) + ret.removed,
+    },
+    note: 'Counts only. No dropped or rejected value is stored.',
+  };
+  safeWriteState(statePath(scanRoot, META_FILE), JSON.stringify(meta, null, 2) + '\n');
+  return { ok: true, accepted: lines.length, rejected, droppedFields, expired: ret.removed, meta };
+}
+
+function sumInto(base, add) { const o = { ...base }; for (const [k, v] of Object.entries(add)) o[k] = (o[k] || 0) + v; return o; }
+
+/** The sidecar next to a trace file, or null. */
+export function loadTraceMeta(traceFile) {
+  try { return JSON.parse(fs.readFileSync(path.join(path.dirname(traceFile), META_FILE), 'utf8')); } catch { return null; }
+}