@clear-capabilities/agentic-security-scanner 0.167.0 → 0.169.0

package/src/lineage/runtime-observation.js

@@ -135,10 +135,16 @@ export const RUNTIME_OBSERVATION_FIELDS = Object.freeze([
   'firstObservedAt', 'lastObservedAt', 'matchMethod', 'matchConfidence', 'retention', 'importedAt',
 ]);
 
-// Only one adapter is actually implemented today — an unimplemented
-// adapter name must be rejected, not accepted on faith that it will
-// someday exist (RO/3b).
-export const RUNTIME_OBSERVATION_ADAPTERS = Object.freeze(['native-jsonl']);
+// An unimplemented adapter name must be rejected, not accepted on faith
+// that it will someday exist (RO/3b). The four PRD "Runtime-Corroborated
+// Security Twin" adapters (`observation-adapters.js`) were added
+// alongside `native-jsonl` as the real P0 telemetry-input gap that PRD
+// closes — OTLP's own richer, evolving attribute surface stays deferred
+// (`'otel-json'` here is this codebase's own minimal OTLP/JSON reader,
+// never the full semantic-conventions vocabulary).
+export const RUNTIME_OBSERVATION_ADAPTERS = Object.freeze([
+  'native-jsonl', 'otel-json', 'envoy-access-log', 'k8s-audit-log', 'http-access-log',
+]);
 
 // FR-505's own four named metadata families: service/workload identity,
 // endpoint/destination identity, protocol/TLS metadata, and schema/
@@ -160,8 +166,23 @@ export const RUNTIME_ATTRIBUTE_KEYS = Object.freeze([
   'network.protocol', 'network.transport', 'tls.version', 'tls.cipher', 'tls.verified',
   // schema/attribute NAMES already approved for telemetry
   'schema.name', 'schema.attributeNames',
+  // the one security-DECISION field (PRD "Runtime-Corroborated Security
+  // Twin" §P0-1's "security decision" family) — added for the K8s audit
+  // adapter, which is defined by the exact allow/deny fact this key
+  // records. A closed two-value enum, never a free-form identifier: see
+  // RUNTIME_SECURITY_DECISION_VALUES below and its own dedicated check in
+  // validateObservationAttributes.
+  'network.securityDecision',
 ]);
 
+// The closed enum for 'network.securityDecision' — deliberately narrower
+// than the generic identifier-value check every other scalar attribute
+// gets, since this field has exactly two legitimate values and nothing
+// else should ever be accepted under it (not 'allow'/'deny', not 'true'/
+// 'false', not a raw HTTP status code — an adapter normalizes to one of
+// these two strings itself).
+export const RUNTIME_SECURITY_DECISION_VALUES = Object.freeze(['allowed', 'denied']);
+
 // The one attribute key whose value is an array rather than a scalar —
 // itself must be a member of RUNTIME_ATTRIBUTE_KEYS.
 export const RUNTIME_ARRAY_ATTRIBUTE_KEYS = Object.freeze(['schema.attributeNames']);
@@ -272,6 +293,18 @@ export function validateObservationAttributes(attributes) {
       }
       continue;
     }
+    if (key === 'network.securityDecision') {
+      // A closed two-value enum, deliberately narrower than the generic
+      // identifier-value check below — this field has exactly two
+      // legitimate values, and accepting anything identifier-shaped
+      // ('unknown', 'maybe', a raw status code string) would let an
+      // adapter smuggle an unreviewed vocabulary through a field whose
+      // whole purpose is a binary allow/deny fact.
+      if (!RUNTIME_SECURITY_DECISION_VALUES.includes(value)) {
+        err(`$.attributes["${key}"]`, `must be one of ${RUNTIME_SECURITY_DECISION_VALUES.join('|')} — this field records a closed allow/deny fact, never a free-form identifier`);
+      }
+      continue;
+    }
     const t = typeof value;
     if (t === 'string') {
       if (!_isIdentifierValue(value)) {

package/src/posture/artifact-registry.js

@@ -121,6 +121,9 @@ export const ARTIFACT_REGISTRY = [
   { name: 'shadow-findings.json', kind: 'file', classification: 'generated', retentionClass: 'scan' },
   { name: 'reproducers', kind: 'dir', classification: 'operator-config', source: 'src/posture/proof-fabric/cli.js (loadReproducer)', note: 'Operator-supplied serialized reproducers read by `triage exploit --prove`. Input, never written by the scanner, so reset leaves it alone.' },
   { name: 'proofs', kind: 'dir', classification: 'generated', retentionClass: 'evidence', confidential: true, source: 'src/posture/proof-fabric/store.js (saveProof)', note: 'Universal Exploit Proof Fabric evidence: hashed records of exploit reproductions. Never holds plaintext credentials, per-run nonces or raw secrets (the runner redacts before persisting). Encrypted per the encryption policy via maybeEncryptForWrite, and purged by retention like any evidence-class artifact; a legal hold on `proofs` blocks the purge.' },
+  { name: 'fleet-state.json', kind: 'file', classification: 'generated', retentionClass: 'scan', source: 'src/posture/fleet/cli.js, src/posture/fleet.js (loadFleetState/_saveFleetState)', note: 'Per-repository resume checkpoint for `fleet scan` — which fleet repositories are already complete, so a resumed run skips them. Scanner-managed; a plain reset may delete it.' },
+  { name: 'fleet-rollup.json', kind: 'file', classification: 'generated', retentionClass: 'scan', source: 'src/posture/fleet/cli.js', note: 'The last fleet rollup (rollupFleet output), kept only so the NEXT `fleet scan` can compute a delta against it and so `fleet status` has something to read without re-scanning.' },
+  { name: 'fleet-graph.json', kind: 'file', classification: 'generated', retentionClass: 'scan', source: 'src/posture/fleet/cli.js, src/posture/fleet/fleet-graph.js', note: 'The cross-repository dependency graph built from each fleet repository\'s own manifest, consumed by `fleet query --depends-on`. Rebuilt on every `fleet scan`.' },
   { name: 'mcp-audit.log', kind: 'file', classification: 'generated', retentionClass: 'evidence' },
   { name: 'egress-audit.log', kind: 'file', classification: 'generated', retentionClass: 'evidence', note: "FR-604 per-call egress audit log — hash-chained NDJSON written by egress/audit.js's recordEgressCall, never read as config" },
   { name: 'hook-throttle.json', kind: 'file', classification: 'generated', retentionClass: 'cache' },

package/src/posture/fleet/cli.js

@@ -0,0 +1,103 @@
+// `agentic-security fleet init|scan|status|resume|query`.
+//
+// Reuses the existing posture/fleet.js (runFleet: bounded concurrency, resumable checkpoint, per-repo isolation,
+// honest rollup) for the scanning half — this module adds only the manifest-driven entry point and the
+// cross-repository dependency graph (fleet-graph.js) that runFleet has no notion of.
+import fs from 'node:fs';
+import path from 'node:path';
+import { loadFleetManifest, starterManifestYaml } from './fleet-manifest.js';
+import { buildFleetDependencyGraph, blastRadius } from './fleet-graph.js';
+import { runFleet, rollupFleet, renderFleetSummary } from '../fleet.js';
+import { statePath, ensureStateDir, safeWriteState } from '../state-dir.js';
+
+const GRAPH_FILE = 'fleet-graph.json';
+const STATE_FILE = 'fleet-state.json';
+
+function graphPath(root) { return statePath(root, GRAPH_FILE); }
+function statePathFor(root) { return statePath(root, STATE_FILE); }
+
+export async function cmdFleet(args, root) {
+  const sub = args._[1];
+  if (sub === 'init') return cmdInit(args, root);
+  if (sub === 'scan') return cmdScan(args, root);
+  if (sub === 'status') return cmdStatus(args, root);
+  if (sub === 'resume') return cmdScan(args, root, { resumeOnly: true });
+  if (sub === 'query') return cmdQuery(args, root);
+  console.error('usage: fleet init | scan <manifest.yaml> | status <manifest.yaml> | resume <manifest.yaml> | query --depends-on <repo-id> [<manifest.yaml>]');
+  return 4;
+}
+
+function manifestArg(args) {
+  const m = args._[2];
+  return m && !String(m).startsWith('--') ? m : 'fleet.yaml';
+}
+
+async function cmdInit(args, root) {
+  const target = path.resolve(root, manifestArg(args) === 'fleet.yaml' ? 'fleet.yaml' : manifestArg(args));
+  if (fs.existsSync(target) && !args.flags.force) { console.error(`${target} already exists. Pass --force to overwrite.`); return 4; }
+  fs.writeFileSync(target, starterManifestYaml(args.flags.organization || 'example'));
+  console.log(`Wrote ${target}. Edit it to list your repositories, then run: agentic-security fleet scan ${path.basename(target)}`);
+  return 0;
+}
+
+async function cmdScan(args, root, { resumeOnly = false } = {}) {
+  const manifestFile = path.resolve(root, manifestArg(args));
+  const loaded = loadFleetManifest(manifestFile);
+  if (!loaded.ok) { console.error(`Invalid fleet manifest: ${loaded.errors.join('; ')}`); return 4; }
+  const { manifest } = loaded;
+
+  if (!ensureStateDir(root)) { console.error('No safe state directory for this project.'); return 4; }
+  const prevRollupFile = statePath(root, 'fleet-rollup.json');
+  let previous = null;
+  try { previous = JSON.parse(fs.readFileSync(prevRollupFile, 'utf8')); } catch { /* first run */ }
+
+  const { runScan: runScanFn } = await import('../../runScan.js');
+  // runScanFn resolves to { scan, meta } already — runFleet's worker expects exactly that shape from runScan, so
+  // this must NOT wrap `scan` a second time (an earlier draft did, and every repo silently reported 0 findings:
+  // `_summarise` reads `scan.findings` defensively rather than throwing on the wrong shape).
+  const runScan = (repoPath) => runScanFn(repoPath, { provenance: false });
+
+  console.error(`[fleet] scanning ${manifest.repositories.length} repository(ies) for ${manifest.organization} (${manifest.workers} worker(s))...`);
+  const result = await runFleet({
+    repos: manifest.repositories.map((r) => r.id), concurrency: manifest.workers,
+    stateFile: statePathFor(root), resume: true, previous,
+    runScan: async (repoId) => runScan(manifest.repositories.find((r) => r.id === repoId).path),
+    onProgress: (entry) => { console.error(`[fleet] ${entry.ok ? 'done' : 'FAILED'}: ${entry.repo}${entry.ok ? ` (${entry.total} finding(s))` : ` — ${entry.error}`}`); },
+  });
+  if (resumeOnly && result.skipped.length === manifest.repositories.length) console.error('[fleet] nothing to resume — every repository was already complete.');
+
+  safeWriteState(prevRollupFile, JSON.stringify(result.rollup, null, 2));
+  const graph = buildFleetDependencyGraph(manifest.repositories);
+  safeWriteState(graphPath(root), JSON.stringify({ organization: manifest.organization, builtAt: new Date().toISOString(), ...graph }, null, 2));
+
+  console.log(renderFleetSummary(result.rollup));
+  console.log(`\nRepositories requested: ${manifest.repositories.length}`);
+  console.log(`Complete: ${result.results.filter((r) => r.ok).length + result.skipped.length}`);
+  console.log(`Failed: ${result.results.filter((r) => !r.ok).length}`);
+  const anyFailed = result.results.some((r) => !r.ok);
+  return anyFailed ? 1 : 0;
+}
+
+async function cmdStatus(args, root) {
+  const f = statePath(root, 'fleet-rollup.json');
+  if (!fs.existsSync(f)) { console.error('No fleet scan has been run yet. Run `agentic-security fleet scan <manifest.yaml>` first.'); return 4; }
+  const rollup = JSON.parse(fs.readFileSync(f, 'utf8'));
+  if (args.flags.json) console.log(JSON.stringify(rollup, null, 2));
+  else console.log(renderFleetSummary(rollup));
+  return 0;
+}
+
+async function cmdQuery(args, root) {
+  const target = args.flags['depends-on'];
+  if (!target) { console.error('usage: fleet query --depends-on <repo-id>'); return 4; }
+  const f = graphPath(root);
+  if (!fs.existsSync(f)) { console.error('No fleet dependency graph yet. Run `agentic-security fleet scan <manifest.yaml>` first.'); return 4; }
+  const graph = JSON.parse(fs.readFileSync(f, 'utf8'));
+  if (!graph.nodes.some((n) => n.id === target)) { console.error(`${target} is not a repository in the last fleet scan.`); return 4; }
+  const radius = blastRadius(graph, target);
+  if (args.flags.json) { console.log(JSON.stringify({ target, blastRadius: radius }, null, 2)); return 0; }
+  if (!radius.length) { console.log(`No fleet repository depends on ${target} (by declared package name).`); return 0; }
+  console.log(`Repositories depending on ${target} (directly or transitively):`);
+  for (const r of radius) console.log(`  ${r.id}  (depth ${r.depth}, via ${r.path.slice(1).join(' -> ')})`);
+  return 0;
+}

package/src/posture/fleet/fleet-graph.js

@@ -0,0 +1,142 @@
+// The cross-repository dependency graph for the fleet feature (PRD 4).
+//
+// Reads each repo's own manifest to learn (a) the package name it publishes under, if any, and (b) the package
+// names it depends on. An edge A -> B means "A declares a dependency that resolves to B's own published name" —
+// i.e. B is upstream of A, and a defect in B can affect A. This is a real, lightweight cross-repo dependency graph
+// built purely from manifests already on disk; it does not need deployment config (Kubernetes/compose/Terraform)
+// the way the existing deployment-boundary graph does, so a fleet of plain application repos works out of the box.
+import fs from 'node:fs';
+import path from 'node:path';
+
+const MANIFESTS = Object.freeze([
+  { file: 'package.json', ecosystem: 'npm' },
+  { file: 'go.mod', ecosystem: 'go' },
+  { file: 'Cargo.toml', ecosystem: 'cargo' },
+  { file: 'requirements.txt', ecosystem: 'pypi' },
+  { file: 'pyproject.toml', ecosystem: 'pypi' },
+]);
+
+function readBounded(p, maxBytes = 1024 * 1024) {
+  try {
+    const st = fs.statSync(p);
+    if (!st.isFile() || st.size > maxBytes) return null;
+    return fs.readFileSync(p, 'utf8');
+  } catch { return null; }
+}
+
+function parsePackageJson(text) {
+  let j;
+  try { j = JSON.parse(text); } catch { return { name: null, deps: new Set() }; }
+  const deps = new Set([...Object.keys(j.dependencies || {}), ...Object.keys(j.devDependencies || {}), ...Object.keys(j.peerDependencies || {})]);
+  return { name: typeof j.name === 'string' ? j.name : null, deps };
+}
+
+function parseGoMod(text) {
+  const m = /^module\s+(\S+)/m.exec(text);
+  const deps = new Set();
+  const block = /require\s*\(([\s\S]*?)\)/.exec(text);
+  const lines = block ? block[1].split('\n') : text.split('\n').filter((l) => /^require\s/.test(l.trim()));
+  for (const raw of lines) {
+    const l = raw.replace(/^require\s+/, '').trim();
+    const dm = /^(\S+)\s+v?\S+/.exec(l);
+    if (dm) deps.add(dm[1]);
+  }
+  return { name: m ? m[1] : null, deps };
+}
+
+function parseCargoToml(text) {
+  const nm = /\[package\][\s\S]*?\bname\s*=\s*"([^"]+)"/.exec(text);
+  const deps = new Set();
+  const section = /\[dependencies\]([\s\S]*?)(\n\[|$)/.exec(text);
+  if (section) {
+    for (const line of section[1].split('\n')) {
+      const dm = /^\s*([A-Za-z0-9_-]+)\s*=/.exec(line);
+      if (dm) deps.add(dm[1]);
+    }
+  }
+  return { name: nm ? nm[1] : null, deps };
+}
+
+function parseRequirementsTxt(text) {
+  const deps = new Set();
+  for (const raw of text.split('\n')) {
+    const l = raw.trim();
+    if (!l || l.startsWith('#') || l.startsWith('-')) continue;
+    const dm = /^([A-Za-z0-9._-]+)/.exec(l);
+    if (dm) deps.add(dm[1].toLowerCase());
+  }
+  return { name: null, deps };
+}
+
+function parsePyprojectToml(text) {
+  const nm = /\[project\][\s\S]*?\bname\s*=\s*"([^"]+)"/.exec(text);
+  const deps = new Set();
+  const section = /dependencies\s*=\s*\[([\s\S]*?)\]/.exec(text);
+  if (section) {
+    for (const m of section[1].matchAll(/"([A-Za-z0-9._-]+)/g)) deps.add(m[1].toLowerCase());
+  }
+  return { name: nm ? nm[1] : null, deps };
+}
+
+const PARSERS = { 'package.json': parsePackageJson, 'go.mod': parseGoMod, 'Cargo.toml': parseCargoToml, 'requirements.txt': parseRequirementsTxt, 'pyproject.toml': parsePyprojectToml };
+
+/** One repo's declared identity and dependency names, merged across every manifest file it has. */
+export function readRepoManifest(repoPath) {
+  let name = null;
+  const deps = new Set();
+  const ecosystems = new Set();
+  for (const { file, ecosystem } of MANIFESTS) {
+    const text = readBounded(path.join(repoPath, file));
+    if (text == null) continue;
+    const parsed = PARSERS[file](text);
+    if (parsed.name && !name) name = parsed.name;
+    for (const d of parsed.deps) deps.add(d);
+    ecosystems.add(ecosystem);
+  }
+  return { name, deps, ecosystems: [...ecosystems] };
+}
+
+/**
+ * Build the fleet dependency graph. `repos`: [{id, path}]. An edge {from, to} means `from` depends on `to` (so a
+ * defect in `to` can affect `from`). Matching is by declared package NAME only — two repos that happen to share a
+ * name under different ecosystems are not disambiguated, a disclosed limit, not a guess.
+ */
+export function buildFleetDependencyGraph(repos) {
+  const manifests = new Map();
+  for (const r of repos) manifests.set(r.id, { ...r, manifest: readRepoManifest(r.path) });
+
+  const byName = new Map();
+  for (const [id, r] of manifests) if (r.manifest.name) {
+    if (!byName.has(r.manifest.name)) byName.set(r.manifest.name, []);
+    byName.get(r.manifest.name).push(id);
+  }
+
+  const edges = [];
+  for (const [id, r] of manifests) {
+    for (const dep of r.manifest.deps) {
+      const targets = byName.get(dep);
+      if (!targets) continue;
+      for (const to of targets) if (to !== id) edges.push({ from: id, to, via: dep });
+    }
+  }
+  const nodes = [...manifests.values()].map((r) => ({ id: r.id, path: r.path, declaredName: r.manifest.name, ecosystems: r.manifest.ecosystems }));
+  return { nodes, edges };
+}
+
+/** Transitive downstream closure: which fleet repos depend on `repoId`, directly or transitively, with the shortest path to each. */
+export function blastRadius(graph, repoId) {
+  const incoming = new Map();
+  for (const e of graph.edges) { if (!incoming.has(e.to)) incoming.set(e.to, []); incoming.get(e.to).push(e.from); }
+  const best = new Map();
+  const queue = [{ id: repoId, path: [repoId] }];
+  while (queue.length) {
+    const { id, path: p } = queue.shift();
+    for (const from of incoming.get(id) || []) {
+      if (best.has(from)) continue;
+      const newPath = [...p, from];
+      best.set(from, { depth: newPath.length - 1, path: newPath });
+      queue.push({ id: from, path: newPath });
+    }
+  }
+  return [...best.entries()].map(([id, v]) => ({ id, ...v })).sort((a, b) => a.depth - b.depth || (a.id < b.id ? -1 : 1));
+}

package/src/posture/fleet/fleet-manifest.js

@@ -0,0 +1,77 @@
+// The fleet manifest (PRD 4 "Fleet Manifest"). `fleet.yaml`:
+//
+//   organization: example
+//   repositories:
+//     - id: frontend
+//       path: ../frontend
+//   workers: 4
+//   perRepoTimeout: 20m
+//
+// Parsing reuses util/yaml.js (the one YAML entry point in this codebase). Validation is closed-world: an unknown
+// top-level key, a duplicate repository id, or a path that does not resolve to a real directory all refuse the
+// whole manifest rather than silently skipping one repo.
+import fs from 'node:fs';
+import path from 'node:path';
+import { load as loadYaml } from '../../util/yaml.js';
+
+const FLEET_MANIFEST_SCHEMA_KEYS = Object.freeze(['organization', 'repositories', 'workers', 'perRepoTimeout']);
+const DURATION_RE = /^(\d+)(s|m|h)$/;
+
+export function parseDuration(s, fallbackMs) {
+  if (typeof s === 'number' && s > 0) return s * 1000;
+  const m = DURATION_RE.exec(String(s ?? ''));
+  if (!m) return fallbackMs;
+  const n = Number(m[1]);
+  return m[2] === 's' ? n * 1000 : m[2] === 'm' ? n * 60000 : n * 3600000;
+}
+
+/** Validate a parsed manifest object against the manifest's own base directory (for path resolution). */
+export function validateFleetManifest(doc, { baseDir = '.' } = {}) {
+  const errors = [];
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc)) return { ok: false, errors: ['manifest must be a YAML mapping'] };
+  for (const k of Object.keys(doc)) if (!FLEET_MANIFEST_SCHEMA_KEYS.includes(k)) errors.push(`unknown top-level key: ${k}`);
+  if (typeof doc.organization !== 'string' || !doc.organization) errors.push('organization is required and must be a string');
+  if (!Array.isArray(doc.repositories) || doc.repositories.length === 0) errors.push('repositories must be a non-empty array');
+  const seen = new Set();
+  const repos = [];
+  for (const [i, r] of (Array.isArray(doc.repositories) ? doc.repositories : []).entries()) {
+    if (!r || typeof r !== 'object') { errors.push(`repositories[${i}] must be a mapping`); continue; }
+    if (typeof r.id !== 'string' || !/^[a-z][a-z0-9-]{0,63}$/.test(r.id)) { errors.push(`repositories[${i}].id must be a lowercase kebab-case string`); continue; }
+    if (seen.has(r.id)) { errors.push(`duplicate repository id: ${r.id}`); continue; }
+    seen.add(r.id);
+    if (typeof r.path !== 'string' || !r.path) { errors.push(`repositories[${i}].path is required`); continue; }
+    const abs = path.resolve(baseDir, r.path);
+    let isDir = false;
+    try { isDir = fs.statSync(abs).isDirectory(); } catch { /* reported below */ }
+    if (!isDir) errors.push(`repositories[${i}].path does not resolve to a directory: ${r.path}`);
+    repos.push({ id: r.id, path: abs });
+  }
+  if (doc.workers != null && !(Number.isInteger(doc.workers) && doc.workers >= 1 && doc.workers <= 32)) errors.push('workers must be an integer in 1..32');
+  if (doc.perRepoTimeout != null && parseDuration(doc.perRepoTimeout, null) == null) errors.push(`perRepoTimeout is not a valid duration: ${doc.perRepoTimeout}`);
+  if (errors.length) return { ok: false, errors };
+  return {
+    ok: true, errors: [],
+    manifest: {
+      organization: doc.organization, repositories: repos,
+      workers: doc.workers ?? 4, perRepoTimeoutMs: parseDuration(doc.perRepoTimeout, 20 * 60000),
+    },
+  };
+}
+
+export function loadFleetManifest(filePath) {
+  let text;
+  try { text = fs.readFileSync(filePath, 'utf8'); } catch (e) { return { ok: false, errors: [`cannot read ${filePath}: ${e.message}`] }; }
+  let doc;
+  try { doc = loadYaml(text); } catch (e) { return { ok: false, errors: [`invalid YAML: ${e.message}`] }; }
+  return validateFleetManifest(doc, { baseDir: path.dirname(path.resolve(filePath)) });
+}
+
+export function starterManifestYaml(organization = 'example') {
+  return `organization: ${organization}
+repositories:
+  - id: example-repo
+    path: .
+workers: 4
+perRepoTimeout: 20m
+`;
+}