@clear-capabilities/agentic-security-scanner 0.128.1 → 0.132.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +223 -0
  2. package/bin/agentic-security.js +52 -2
  3. package/dist/11.index.js +2 -2
  4. package/dist/113.index.js +498 -7
  5. package/dist/178.index.js +1 -1
  6. package/dist/207.index.js +220 -0
  7. package/dist/238.index.js +218 -0
  8. package/dist/259.index.js +975 -0
  9. package/dist/384.index.js +1 -1
  10. package/dist/415.index.js +1 -1
  11. package/dist/435.index.js +4 -4
  12. package/dist/526.index.js +844 -0
  13. package/dist/637.index.js +1 -1
  14. package/dist/830.index.js +1 -1
  15. package/dist/agentic-security.mjs +106 -194
  16. package/dist/agentic-security.mjs.sha256 +1 -1
  17. package/package.json +33 -17
  18. package/src/dataflow/CLAUDE.md +4 -1
  19. package/src/dataflow/async-sequencing.js +8 -3
  20. package/src/dataflow/catalog.js +278 -11
  21. package/src/dataflow/cross-repo.js +1 -1
  22. package/src/dataflow/cross-service-taint.js +1 -1
  23. package/src/dataflow/engine.js +182 -61
  24. package/src/dataflow/ifds.js +10 -5
  25. package/src/dataflow/index.js +15 -3
  26. package/src/dataflow/points-to.js +8 -2
  27. package/src/dataflow/proof-gate.js +7 -0
  28. package/src/dataflow/sanitizer-gate.js +89 -0
  29. package/src/dataflow/tabulation.js +14 -3
  30. package/src/engine.js +170 -7
  31. package/src/integrations/index.js +1 -1
  32. package/src/ir/CLAUDE.md +49 -4
  33. package/src/ir/call-sites.js +66 -0
  34. package/src/ir/callgraph.js +174 -7
  35. package/src/ir/class-hierarchy.js +22 -2
  36. package/src/ir/index.js +138 -51
  37. package/src/ir/ir-stats.js +126 -0
  38. package/src/ir/parser-cpp.js +829 -0
  39. package/src/ir/parser-cs.js +4 -1
  40. package/src/ir/parser-go.js +4 -1
  41. package/src/ir/parser-js.js +13 -1
  42. package/src/ir/parser-kt.js +4 -1
  43. package/src/ir/parser-php.js +10 -3
  44. package/src/ir/parser-py-cst.js +62 -10
  45. package/src/ir/tree-sitter-loader.js +13 -1
  46. package/src/llm-validator/index.js +9 -2
  47. package/src/llm-validator/redact.js +157 -0
  48. package/src/mcp/tools.js +2 -2
  49. package/src/posture/CLAUDE.md +193 -1
  50. package/src/posture/accuracy-scorecard.js +317 -0
  51. package/src/posture/api-contract.js +1 -1
  52. package/src/posture/attestation.js +202 -0
  53. package/src/posture/auditor-walkthrough.js +12 -3
  54. package/src/posture/compliance-policy.js +1 -1
  55. package/src/posture/corpus-enroll.js +303 -0
  56. package/src/posture/corpus-match.js +52 -0
  57. package/src/posture/cross-lang-openapi.js +1 -1
  58. package/src/posture/custom-rules.js +3 -3
  59. package/src/posture/execution-proof.js +92 -0
  60. package/src/posture/exploitability-probability.js +1 -1
  61. package/src/posture/falsification.js +45 -1
  62. package/src/posture/fix-metrics.js +197 -0
  63. package/src/posture/fix-verify.js +129 -2
  64. package/src/posture/license-policy.js +1 -1
  65. package/src/posture/profile.js +1 -1
  66. package/src/posture/proof-tier.js +33 -0
  67. package/src/posture/relevance.js +379 -0
  68. package/src/posture/root-cause-sweep.js +0 -0
  69. package/src/posture/rule-overrides.js +1 -1
  70. package/src/posture/sca-policy.js +1 -1
  71. package/src/posture/scan-checkpoint.js +277 -0
  72. package/src/posture/suppressions.js +1 -1
  73. package/src/posture/test-runner.js +147 -0
  74. package/src/posture/verification-separation.js +131 -0
  75. package/src/report/index.js +11 -0
  76. package/src/runScan.js +5 -7
  77. package/src/sandbox/CLAUDE.md +340 -0
  78. package/src/sandbox/backend-disabled.js +14 -0
  79. package/src/sandbox/backend-namespace.js +335 -0
  80. package/src/sandbox/backend-userspace.js +83 -0
  81. package/src/sandbox/capabilities.js +181 -0
  82. package/src/sandbox/index.js +30 -0
  83. package/src/sandbox/limits.js +63 -0
  84. package/src/sandbox/result.js +104 -0
  85. package/src/sca/dep-confusion.js +1 -1
  86. package/src/util/glob.js +173 -0
  87. package/src/util/yaml.js +24 -0
@@ -0,0 +1,379 @@
1
+ // R6 (threat-model-first scoping) + R9 (attack-surface-forward analysis).
2
+ //
3
+ // Every other precision mechanism in this engine kills a false positive by
4
+ // PATTERN — a sanitizer on the path, a proof that didn't reproduce, a
5
+ // confidence model fit to labelled families. This module kills one by
6
+ // RELEVANCE: a finding sitting on a path an attacker can actually reach,
7
+ // inside something the threat model says is worth attacking, matters more
8
+ // than one that is neither. It is a different axis, and it composes with
9
+ // (never replaces) the sink-driven taint engine.
10
+ //
11
+ // R9 — start at the attack surface and reason FORWARD. `entrypoint-
12
+ // inventory.js` already enumerates every attacker-reachable entry
13
+ // point (HTTP/queue/cron/CLI/env/upload/webhook). Here that inventory
14
+ // stops being a report and starts being an input: we walk the module
15
+ // import graph out from every entry-point file and record which files
16
+ // are reachable from the attack surface at all.
17
+ // R6 — `threat-model.js` already derives assets, trust boundaries and a
18
+ // STRIDE classification. Here that model re-ranks: a finding on a
19
+ // modelled asset, or one classified into a modelled STRIDE bucket,
20
+ // outranks an identical finding that is in neither.
21
+ //
22
+ // ── Recall-preserving contract (non-negotiable) ────────────────────────────
23
+ // Same precedent as `falsification.js` and `dataflow/proof-gate.js`:
24
+ // • Never removes a finding. The array in is the array out, same length,
25
+ // same order, same objects.
26
+ // • Never touches `severity`. Ever. Severity is the customer's triage
27
+ // contract; relevance is an ordinal re-rank underneath it.
28
+ // • Never asserts `unreachable` without POSITIVE evidence. "I could not
29
+ // determine it" is `entrypointReachable: null` / `relevanceTier:
30
+ // 'unknown'` — a distinct state from "I determined it is not reachable".
31
+ // A negative verdict additionally requires the intra-repo import graph to
32
+ // be provably COMPLETE: one unresolved relative import anywhere means the
33
+ // graph has a hole an attacker's path could be hiding in, and every
34
+ // would-be `unreachable` degrades to `unknown`.
35
+ // • Demotion has a floor. An unreachable finding's exploitability is scaled,
36
+ // never zeroed — a wrong reachability call must cost rank, not visibility.
37
+ //
38
+ // Fields set on each finding:
39
+ // entrypointReachable : true | false | null (null ≠ false)
40
+ // relevance : number 0..1
41
+ // relevanceTier : 'direct' | 'indirect' | 'unreachable' | 'unknown'
42
+ // relevanceFactors : string[] (human-readable, like exploitabilityFactors)
43
+
44
+ // ── Tunables ───────────────────────────────────────────────────────────────
45
+ const BASE_SCORE = {
46
+ direct: 0.75,
47
+ indirect: 0.50,
48
+ unreachable: 0.15,
49
+ unknown: 0.40,
50
+ };
51
+ // Cap so that no accumulation of R6 bonuses can push an evidenced-unreachable
52
+ // finding into the same band as a reachable one.
53
+ const UNREACHABLE_CAP = 0.30;
54
+
55
+ // Exploitability re-rank multipliers (R6's "re-rank exploitability").
56
+ const EXPLOIT_MULT = { direct: 1.15, indirect: 1.0, unreachable: 0.6, unknown: 1.0 };
57
+ const EXPLOIT_FLOOR = 0.05; // demotion never zeroes a finding out
58
+
59
+ const IMPORT_EXTS = ['', '.js', '.mjs', '.cjs', '.jsx', '.ts', '.tsx', '/index.js', '/index.ts', '/index.mjs'];
60
+ const MAX_FILES = 5000; // graph-size guard: bail to 'unknown' beyond this
61
+
62
+ // ── Source-level import extraction ─────────────────────────────────────────
63
+ // Static, literal specifiers only. Anything non-literal is recorded as a hole.
64
+ const RE_IMPORT_FROM = /\bimport\s[^;'"`]*?from\s*['"]([^'"]+)['"]/g;
65
+ const RE_IMPORT_BARE = /\bimport\s*['"]([^'"]+)['"]/g;
66
+ const RE_EXPORT_FROM = /\bexport\s[^;'"`]*?from\s*['"]([^'"]+)['"]/g;
67
+ const RE_REQUIRE = /\brequire\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
68
+ const RE_DYN_IMPORT = /\bimport\s*\(\s*['"]([^'"]+)['"]\s*\)/g;
69
+ const RE_PY_FROM = /^\s*from\s+([.\w]+)\s+import\s/gm;
70
+ const RE_PY_IMPORT = /^\s*import\s+([.\w]+)/gm;
71
+ const RE_JAVA_IMPORT = /^\s*import\s+(?:static\s+)?([\w.]+);/gm;
72
+ // Non-literal module loads: the graph cannot see through these.
73
+ const RE_DYNAMIC_HOLE = /\brequire\s*\(\s*[^'")\s]|\bimport\s*\(\s*[^'")\s]/;
74
+
75
+ function _entries(fileContents) {
76
+ if (!fileContents) return [];
77
+ if (fileContents instanceof Map) {
78
+ return [...fileContents.entries()].filter(([k, v]) => typeof k === 'string' && typeof v === 'string');
79
+ }
80
+ if (typeof fileContents === 'object') {
81
+ return Object.entries(fileContents).filter(([k, v]) => typeof k === 'string' && typeof v === 'string');
82
+ }
83
+ return [];
84
+ }
85
+
86
+ function _norm(p) {
87
+ const parts = String(p).replace(/\\/g, '/').split('/');
88
+ const out = [];
89
+ for (const seg of parts) {
90
+ if (seg === '' || seg === '.') continue;
91
+ if (seg === '..') { out.pop(); continue; }
92
+ out.push(seg);
93
+ }
94
+ return out.join('/');
95
+ }
96
+
97
+ function _dirOf(file) {
98
+ const i = String(file).replace(/\\/g, '/').lastIndexOf('/');
99
+ return i < 0 ? '' : String(file).slice(0, i);
100
+ }
101
+
102
+ // Resolve one specifier to a key in `known`. Returns the key, or null.
103
+ function _resolve(spec, fromFile, known, suffixIndex) {
104
+ if (!spec) return null;
105
+ const relative = spec.startsWith('.');
106
+ const base = relative ? _norm(`${_dirOf(fromFile)}/${spec}`) : null;
107
+
108
+ if (relative) {
109
+ for (const ext of IMPORT_EXTS) {
110
+ const cand = base + ext;
111
+ if (known.has(cand)) return cand;
112
+ }
113
+ // Also try the raw (already-extensioned) form as written.
114
+ if (known.has(base)) return base;
115
+ return null;
116
+ }
117
+
118
+ // Non-relative: python dotted module, java FQCN, or a bare package name.
119
+ const dotted = spec.replace(/\./g, '/');
120
+ for (const ext of ['.py', '.java', '.js', '.ts', '/__init__.py']) {
121
+ const hit = suffixIndex.get(dotted + ext);
122
+ if (hit) return hit;
123
+ }
124
+ return null;
125
+ }
126
+
127
+ /**
128
+ * Build the attack-surface reachability view.
129
+ *
130
+ * @returns {{
131
+ * entryFiles: Set<string>, unauthEntryFiles: Set<string>,
132
+ * reachableFiles: Set<string>, knownFiles: Set<string>,
133
+ * graphComplete: boolean, holes: number,
134
+ * }}
135
+ */
136
+ function buildReachabilityGraph(fileContents, opts = {}) {
137
+ const knownFiles = new Set();
138
+ const entryFiles = new Set();
139
+ const unauthEntryFiles = new Set();
140
+ const reachableFiles = new Set();
141
+ let graphComplete = false;
142
+ let holes = 0;
143
+
144
+ try {
145
+ const files = _entries(fileContents);
146
+ if (files.length === 0 || files.length > MAX_FILES) {
147
+ return { entryFiles, unauthEntryFiles, reachableFiles, knownFiles, graphComplete: false, holes: 1 };
148
+ }
149
+ for (const [k] of files) knownFiles.add(k);
150
+
151
+ // Index by path suffix so a python/java module name can find its file.
152
+ const suffixIndex = new Map();
153
+ for (const k of knownFiles) {
154
+ const norm = k.replace(/\\/g, '/');
155
+ const segs = norm.split('/');
156
+ for (let i = 0; i < segs.length; i++) {
157
+ const suf = segs.slice(i).join('/');
158
+ if (!suffixIndex.has(suf)) suffixIndex.set(suf, k);
159
+ }
160
+ }
161
+
162
+ // 1) Entry-point files, from the inventory and/or the raw route list.
163
+ const inv = opts.entrypointInventory;
164
+ const invEntries = inv && Array.isArray(inv.entrypoints) ? inv.entrypoints
165
+ : Array.isArray(opts.entrypoints) ? opts.entrypoints : [];
166
+ for (const e of invEntries) {
167
+ if (!e || typeof e.file !== 'string' || !e.file) continue;
168
+ entryFiles.add(e.file);
169
+ if (e.trust !== 'authenticated') unauthEntryFiles.add(e.file);
170
+ }
171
+ for (const r of (Array.isArray(opts.routes) ? opts.routes : [])) {
172
+ if (!r || typeof r.file !== 'string' || !r.file) continue;
173
+ entryFiles.add(r.file);
174
+ if (r.hasAuth !== true) unauthEntryFiles.add(r.file);
175
+ }
176
+
177
+ // 2) Import edges. A hole is an edge the graph cannot see: an unresolved
178
+ // intra-repo (relative) specifier, or a non-literal module load. Holes
179
+ // are recorded PER FILE because only holes in files that turn out to be
180
+ // REACHABLE can hide a path into somewhere we'd otherwise call
181
+ // unreachable — a hidden edge always originates in the importing file,
182
+ // so a hole inside an already-unreachable file cannot make anything
183
+ // reachable. Anything else would make a negative verdict impossible in
184
+ // any real repository (one `require(varName)` anywhere would veto all).
185
+ const edges = new Map();
186
+ const holeFiles = new Set();
187
+ for (const [file, src] of files) {
188
+ const out = new Set();
189
+ if (RE_DYNAMIC_HOLE.test(src)) { holes++; holeFiles.add(file); }
190
+ for (const re of [RE_IMPORT_FROM, RE_EXPORT_FROM, RE_IMPORT_BARE, RE_REQUIRE, RE_DYN_IMPORT,
191
+ RE_PY_FROM, RE_PY_IMPORT, RE_JAVA_IMPORT]) {
192
+ re.lastIndex = 0;
193
+ let m;
194
+ while ((m = re.exec(src))) {
195
+ const spec = m[1];
196
+ if (!spec) continue;
197
+ const target = _resolve(spec, file, knownFiles, suffixIndex);
198
+ if (target && target !== file) out.add(target);
199
+ // Only an unresolved INTRA-repo (relative) specifier is a hole:
200
+ // a bare package name points at a third-party module that was
201
+ // never part of `fileContents` in the first place.
202
+ else if (!target && spec.startsWith('.')) { holes++; holeFiles.add(file); }
203
+ }
204
+ }
205
+ edges.set(file, out);
206
+ }
207
+
208
+ // 3) Forward BFS from the attack surface.
209
+ const queue = [...entryFiles].filter(f => knownFiles.has(f));
210
+ for (const f of queue) reachableFiles.add(f);
211
+ while (queue.length) {
212
+ const cur = queue.shift();
213
+ for (const next of (edges.get(cur) || [])) {
214
+ if (reachableFiles.has(next)) continue;
215
+ reachableFiles.add(next);
216
+ queue.push(next);
217
+ }
218
+ }
219
+
220
+ // A negative verdict is admissible only when we actually found an attack
221
+ // surface AND no file reachable from it has an invisible edge.
222
+ let reachableHole = false;
223
+ for (const f of reachableFiles) if (holeFiles.has(f)) { reachableHole = true; break; }
224
+ graphComplete = !reachableHole && entryFiles.size > 0;
225
+ } catch (_) {
226
+ return { entryFiles, unauthEntryFiles, reachableFiles, knownFiles, graphComplete: false, holes: holes || 1 };
227
+ }
228
+ return { entryFiles, unauthEntryFiles, reachableFiles, knownFiles, graphComplete, holes };
229
+ }
230
+
231
+ // ── R6: threat-model bonuses ───────────────────────────────────────────────
232
+ function _threatBonus(f, threatModel, factors) {
233
+ let bonus = 0;
234
+ if (!threatModel || typeof threatModel !== 'object') return bonus;
235
+
236
+ const assets = Array.isArray(threatModel.assets) ? threatModel.assets : [];
237
+ for (const a of assets) {
238
+ if (!a || a.file !== f.file) continue;
239
+ const exposed = a.exposure === 'public-api' || a.exposure === 'external-api';
240
+ bonus += exposed ? 0.10 : 0.07;
241
+ factors.push(`modelled asset in file: ${a.category || a.name || 'asset'}${exposed ? ` (${a.exposure})` : ''}`);
242
+ break;
243
+ }
244
+
245
+ const boundaries = Array.isArray(threatModel.trustBoundaries) ? threatModel.trustBoundaries : [];
246
+ if (boundaries.some(b => b && b.file === f.file)) {
247
+ bonus += 0.05;
248
+ factors.push('trust boundary crossed in this file');
249
+ }
250
+
251
+ const stride = threatModel.stride && typeof threatModel.stride === 'object' ? threatModel.stride : null;
252
+ if (stride) {
253
+ for (const [cat, items] of Object.entries(stride)) {
254
+ if (!Array.isArray(items)) continue;
255
+ if (items.some(it => it && it.file === f.file && (it.line === f.line || it.vuln === f.vuln))) {
256
+ bonus += 0.08;
257
+ factors.push(`modelled STRIDE threat: ${cat}`);
258
+ break;
259
+ }
260
+ }
261
+ }
262
+ return bonus;
263
+ }
264
+
265
+ /**
266
+ * Pure scorer. Does NOT mutate the finding.
267
+ *
268
+ * @param graph the object returned by buildReachabilityGraph (or a subset
269
+ * with entryFiles / reachableFiles / knownFiles / graphComplete).
270
+ * @returns {{ tier, score, reachable, factors }}
271
+ */
272
+ function scoreRelevance(f, graph, threatModel) {
273
+ const factors = [];
274
+ const g = graph || {};
275
+ const entryFiles = g.entryFiles instanceof Set ? g.entryFiles : new Set();
276
+ const reachableFiles = g.reachableFiles instanceof Set ? g.reachableFiles : new Set();
277
+ const knownFiles = g.knownFiles instanceof Set ? g.knownFiles : new Set();
278
+ const unauthEntryFiles = g.unauthEntryFiles instanceof Set ? g.unauthEntryFiles : new Set();
279
+ const file = f && typeof f.file === 'string' ? f.file : null;
280
+
281
+ let tier;
282
+ let reachable;
283
+ if (!file || entryFiles.size === 0) {
284
+ tier = 'unknown';
285
+ reachable = null;
286
+ factors.push(entryFiles.size === 0
287
+ ? 'no attack surface enumerated — reachability not determinable'
288
+ : 'finding has no file — reachability not determinable');
289
+ } else if (entryFiles.has(file)) {
290
+ tier = 'direct';
291
+ reachable = true;
292
+ factors.push('finding sits in an entry-point file (direct attack surface)');
293
+ if (unauthEntryFiles.has(file)) factors.push('entry point is unauthenticated');
294
+ } else if (reachableFiles.has(file)) {
295
+ tier = 'indirect';
296
+ reachable = true;
297
+ factors.push('reachable from an entry point via the module import graph');
298
+ } else if (g.graphComplete === true && knownFiles.has(file)) {
299
+ tier = 'unreachable';
300
+ reachable = false;
301
+ factors.push('no import path from any enumerated entry point (import graph complete)');
302
+ } else {
303
+ tier = 'unknown';
304
+ reachable = null;
305
+ factors.push(knownFiles.has(file)
306
+ ? 'import graph incomplete — no reachability verdict admissible'
307
+ : 'file not present in the scanned set — reachability not determinable');
308
+ }
309
+
310
+ let score = BASE_SCORE[tier];
311
+ score += _threatBonus(f || {}, threatModel, factors);
312
+ if (tier === 'direct' && unauthEntryFiles.has(file)) score += 0.10;
313
+ if (tier === 'unreachable') score = Math.min(score, UNREACHABLE_CAP);
314
+ score = Math.max(0, Math.min(1, score));
315
+
316
+ return { tier, score: Math.round(score * 1000) / 1000, reachable, factors };
317
+ }
318
+
319
+ /**
320
+ * Default-on annotator. Recall-preserving: never removes a finding, never
321
+ * touches severity, never asserts 'unreachable' without positive evidence.
322
+ *
323
+ * @param ctx.fileContents Map|object of scanned sources
324
+ * @param ctx.entrypointInventory output of buildEntrypointInventory()
325
+ * @param ctx.routes route list (fallback attack surface)
326
+ * @param ctx.threatModel output of buildThreatModel()
327
+ */
328
+ export function annotateRelevance(findings, ctx = {}) {
329
+ if (!Array.isArray(findings)) return findings;
330
+ let graph;
331
+ try {
332
+ const c = ctx && typeof ctx === 'object' ? ctx : {};
333
+ graph = buildReachabilityGraph(c.fileContents, {
334
+ entrypointInventory: c.entrypointInventory,
335
+ entrypoints: c.entrypoints,
336
+ routes: c.routes,
337
+ });
338
+ } catch (_) {
339
+ graph = { entryFiles: new Set(), unauthEntryFiles: new Set(), reachableFiles: new Set(), knownFiles: new Set(), graphComplete: false, holes: 1 };
340
+ }
341
+ const threatModel = ctx && typeof ctx === 'object' ? ctx.threatModel : null;
342
+
343
+ for (const f of findings) {
344
+ if (!f || typeof f !== 'object') continue;
345
+ try {
346
+ const r = scoreRelevance(f, graph, threatModel);
347
+ f.entrypointReachable = r.reachable;
348
+ f.relevance = r.score;
349
+ f.relevanceTier = r.tier;
350
+ f.relevanceFactors = r.factors;
351
+
352
+ // R6 re-rank: exploitability is an ordinal priority, so scaling it by
353
+ // relevance is exactly the intended re-ranking. Severity is untouched,
354
+ // and demotion has a floor — a wrong call costs rank, not visibility.
355
+ const mult = EXPLOIT_MULT[r.tier];
356
+ if (typeof f.exploitability === 'number' && Number.isFinite(f.exploitability) && mult !== 1) {
357
+ const adjusted = Math.max(EXPLOIT_FLOOR, Math.min(1, f.exploitability * mult));
358
+ f.exploitability = Math.round(adjusted * 100) / 100;
359
+ if (typeof f.priorityScore === 'number') f.priorityScore = f.exploitability;
360
+ if (Array.isArray(f.exploitabilityFactors)) f.exploitabilityFactors.push(`relevance:${r.tier}`);
361
+ // Keep the tier label consistent with the re-ranked score. Same
362
+ // thresholds as annotateExploitability; severity is NOT derived here.
363
+ if (f.exploitability >= 0.80) f.exploitabilityTier = 'critical';
364
+ else if (f.exploitability >= 0.60) f.exploitabilityTier = 'high';
365
+ else if (f.exploitability >= 0.35) f.exploitabilityTier = 'medium';
366
+ else f.exploitabilityTier = 'low';
367
+ }
368
+ } catch (_) {
369
+ f.entrypointReachable = null;
370
+ f.relevance = BASE_SCORE.unknown;
371
+ f.relevanceTier = 'unknown';
372
+ f.relevanceFactors = ['relevance scoring failed — no verdict'];
373
+ }
374
+ }
375
+ return findings;
376
+ }
377
+
378
+ // Test-only surface (underscore-prefixed: not part of the public API).
379
+ export const _internals = { buildReachabilityGraph, scoreRelevance, BASE_SCORE, EXPLOIT_MULT };
Binary file
@@ -9,7 +9,7 @@
9
9
 
10
10
  import * as fs from 'node:fs';
11
11
  import * as path from 'node:path';
12
- import * as yaml from 'js-yaml';
12
+ import * as yaml from '../util/yaml.js';
13
13
  import { verifyLastScan } from './integrity.js';
14
14
  import { statePath } from './state-dir.js';
15
15
 
@@ -38,7 +38,7 @@
38
38
 
39
39
  import * as fs from 'node:fs';
40
40
  import * as path from 'node:path';
41
- import * as yaml from 'js-yaml';
41
+ import * as yaml from '../util/yaml.js';
42
42
 
43
43
  const DEFAULT_POLICY = {
44
44
  acceptRisk: [],
@@ -0,0 +1,277 @@
1
+ // Scan checkpointing / resume (roadmap R8).
2
+ //
3
+ // Long scans currently restart from zero if interrupted, which is what caps the
4
+ // repository size this engine can usefully handle. This module lets the per-file
5
+ // loop in `engine.js#runFullScan` durably record what it has already analysed so
6
+ // a second invocation replays that work instead of redoing it.
7
+ //
8
+ // THE PROPERTY THAT MATTERS: a resumed scan must produce the same finding set as
9
+ // an uninterrupted one. A checkpoint that silently drops findings converts a slow
10
+ // scan into a quietly incomplete one, which is strictly worse than no checkpoint
11
+ // at all. Three design decisions follow from that and should not be relaxed:
12
+ //
13
+ // 1. We persist the *complete* per-file contribution, not just findings —
14
+ // routes, taint sources/sinks/sanitizers, logic vulns, secrets, ciphers,
15
+ // the per-file result the cross-file taint pass reads, and the suppression
16
+ // log delta. Anything the per-file loop appends to must round-trip, or the
17
+ // post-loop cross-file passes would see a different world on resume.
18
+ // 2. Only the per-file loop is checkpointed. Every cross-file pass and the
19
+ // whole annotation pipeline re-runs from scratch on resume, so nothing that
20
+ // depends on the global picture can be stale by construction.
21
+ // 3. Invalidation is conservative to the point of being blunt. The run key
22
+ // covers the engine version, the ruleset version, the bundle SHA, a content
23
+ // hash of every file in the scan (which subsumes mtime), and the scanner's
24
+ // own environment switches. If any of it moved, the checkpoint is discarded
25
+ // and the scan starts clean. Redoing work is merely slow; resuming stale
26
+ // work is a correctness bug.
27
+ //
28
+ // CRASH SAFETY: append-and-fsync. The file is a JSONL log — one header line
29
+ // pinning the run key, then one self-describing record per completed file,
30
+ // each carrying a SHA-256 of its own payload. Every record is written with a
31
+ // single `writeSync` and immediately `fsyncSync`'d before the next file is
32
+ // analysed, so a process killed at any instant leaves either a complete record
33
+ // or a torn tail. On recovery we read forward while records verify and truncate
34
+ // the file at the last byte offset that did, so a torn tail is discarded rather
35
+ // than resumed into. Nothing is ever rewritten in place, so there is no window
36
+ // in which the file is neither the old state nor the new one.
37
+ //
38
+ // Everything here follows the posture convention of never throwing: a failure to
39
+ // open, read or append degrades to "no checkpoint", which just means a full scan.
40
+
41
+ import * as fs from 'node:fs';
42
+ import * as path from 'node:path';
43
+ import * as crypto from 'node:crypto';
44
+ import { fileURLToPath } from 'node:url';
45
+
46
+ const STATE_DIR = '.agentic-security';
47
+ const FILE_NAME = 'scan-checkpoint.jsonl';
48
+ const FORMAT = 'agentic-security-scan-checkpoint/1';
49
+
50
+ // Env switches that change what the engine emits are part of the run identity.
51
+ // These three are deliberately excluded: they change how the run is driven, not
52
+ // what it would find.
53
+ const RUN_KEY_ENV_EXCLUDE = new Set([
54
+ 'AGENTIC_SECURITY_RESUME',
55
+ 'AGENTIC_SECURITY_CHECKPOINT_ABORT_AFTER',
56
+ 'AGENTIC_SECURITY_HMAC_KEY',
57
+ ]);
58
+
59
+ export function checkpointPath(scanRoot) {
60
+ return path.join(scanRoot || '.', STATE_DIR, FILE_NAME);
61
+ }
62
+
63
+ function _sha(s) {
64
+ return crypto.createHash('sha256').update(s, 'utf8').digest('hex');
65
+ }
66
+
67
+ // SHA-256 of the running bundle, taken from the sidecar next to it. Returns
68
+ // 'unavailable' when running from source — same convention as the attestation
69
+ // path, and deliberately not a guess at some other bundle's hash.
70
+ export function bundleShaForRunKey() {
71
+ try {
72
+ const here = path.dirname(fileURLToPath(import.meta.url));
73
+ // src/posture/ -> src/ -> scanner/
74
+ const sidecar = path.resolve(here, '..', '..', 'dist', 'agentic-security.mjs.sha256');
75
+ const raw = fs.readFileSync(sidecar, 'utf8').trim();
76
+ const m = /^([0-9a-f]{64})\b/.exec(raw);
77
+ if (m) return m[1];
78
+ } catch { /* not running from a built tree */ }
79
+ return 'unavailable';
80
+ }
81
+
82
+ /**
83
+ * Everything that would invalidate previously-completed per-file work, reduced
84
+ * to one hex digest. Content hashes rather than mtimes: strictly stronger, and
85
+ * immune to filesystems with coarse or non-monotonic timestamps.
86
+ */
87
+ export function computeRunKey({
88
+ engineVersion, rulesetVersion, bundleSha,
89
+ fileContents = {}, depFileContents = {}, env = process.env,
90
+ } = {}) {
91
+ const h = crypto.createHash('sha256');
92
+ h.update(FORMAT); h.update('\n');
93
+ h.update(String(engineVersion ?? '')); h.update('\n');
94
+ h.update(String(rulesetVersion ?? '')); h.update('\n');
95
+ h.update(String(bundleSha ?? 'unavailable')); h.update('\n');
96
+ for (const [label, map] of [['f', fileContents], ['d', depFileContents]]) {
97
+ const names = Object.keys(map || {}).sort();
98
+ h.update(label); h.update(String(names.length)); h.update('\n');
99
+ for (const n of names) {
100
+ h.update(n); h.update('\0');
101
+ h.update(_sha(String(map[n] ?? '')));
102
+ h.update('\n');
103
+ }
104
+ }
105
+ const envKeys = Object.keys(env || {})
106
+ .filter(k => k.startsWith('AGENTIC_SECURITY_') && !RUN_KEY_ENV_EXCLUDE.has(k))
107
+ .sort();
108
+ h.update('e'); h.update(String(envKeys.length)); h.update('\n');
109
+ for (const k of envKeys) { h.update(k); h.update('='); h.update(String(env[k])); h.update('\n'); }
110
+ return h.digest('hex');
111
+ }
112
+
113
+ // A value is safe to checkpoint only if JSON can carry it back unchanged. Dates,
114
+ // regexes, Maps, Sets, functions and BigInts all survive `JSON.stringify` in a
115
+ // lossy or throwing way; recording one would mean the resumed run sees different
116
+ // data than the uninterrupted run did. We refuse the record instead, and the
117
+ // file just gets rescanned.
118
+ function _jsonSafe(v, depth = 0, seen = new Set()) {
119
+ if (depth > 24) return false;
120
+ if (v === null || v === undefined) return true;
121
+ const t = typeof v;
122
+ if (t === 'string' || t === 'boolean') return true;
123
+ if (t === 'number') return Number.isFinite(v);
124
+ if (t === 'function' || t === 'symbol' || t === 'bigint') return false;
125
+ if (t !== 'object') return false;
126
+ if (seen.has(v)) return false;
127
+ seen.add(v);
128
+ try {
129
+ if (Array.isArray(v)) {
130
+ for (const x of v) if (!_jsonSafe(x, depth + 1, seen)) return false;
131
+ return true;
132
+ }
133
+ const proto = Object.getPrototypeOf(v);
134
+ if (proto !== Object.prototype && proto !== null) return false;
135
+ for (const k of Object.keys(v)) if (!_jsonSafe(v[k], depth + 1, seen)) return false;
136
+ return true;
137
+ } finally {
138
+ seen.delete(v);
139
+ }
140
+ }
141
+
142
+ function _emptyHandle(reason) {
143
+ return {
144
+ enabled: false, file: null, fd: null, runKey: null,
145
+ recovered: new Map(), order: [], written: new Set(),
146
+ discarded: false, reason,
147
+ };
148
+ }
149
+
150
+ function _headerLine(runKey) {
151
+ return JSON.stringify({ v: FORMAT, runKey }) + '\n';
152
+ }
153
+
154
+ // Read forward from a byte offset, keeping records while they verify. Returns
155
+ // the offset of the first byte that did NOT verify, so the caller can truncate.
156
+ function _recover(handle, file, runKey) {
157
+ let buf;
158
+ try { buf = fs.readFileSync(file); }
159
+ catch { return -1; } // no file yet
160
+ const text = buf.toString('utf8');
161
+ const nl = text.indexOf('\n');
162
+ if (nl < 0) return 0;
163
+ let header = null;
164
+ try { header = JSON.parse(text.slice(0, nl)); } catch { return 0; }
165
+ if (!header || header.v !== FORMAT || header.runKey !== runKey) return 0;
166
+
167
+ let offset = Buffer.byteLength(text.slice(0, nl + 1), 'utf8');
168
+ let cursor = nl + 1;
169
+ for (;;) {
170
+ const end = text.indexOf('\n', cursor);
171
+ if (end < 0) break; // torn tail: no terminating newline
172
+ const line = text.slice(cursor, end);
173
+ cursor = end + 1;
174
+ if (!line) { offset = Buffer.byteLength(text.slice(0, cursor), 'utf8'); continue; }
175
+ let rec;
176
+ try { rec = JSON.parse(line); } catch { break; }
177
+ if (!rec || typeof rec.f !== 'string' || typeof rec.d !== 'string') break;
178
+ if (rec.c !== _sha(rec.d)) break; // tampered or torn-then-patched
179
+ let payload;
180
+ try { payload = JSON.parse(rec.d); } catch { break; }
181
+ if (!handle.recovered.has(rec.f)) handle.order.push(rec.f);
182
+ handle.recovered.set(rec.f, payload);
183
+ offset = Buffer.byteLength(text.slice(0, cursor), 'utf8');
184
+ }
185
+ return offset;
186
+ }
187
+
188
+ /**
189
+ * Open (or start) the checkpoint for `scanRoot` under `runKey`. Never throws.
190
+ * A handle whose `enabled` is false silently no-ops through the rest of the API.
191
+ */
192
+ export function openCheckpoint(scanRoot, { runKey } = {}) {
193
+ if (!scanRoot || !runKey) return _emptyHandle('no-run-key');
194
+ const handle = _emptyHandle(null);
195
+ try {
196
+ const dir = path.join(scanRoot, STATE_DIR);
197
+ fs.mkdirSync(dir, { recursive: true });
198
+ const file = checkpointPath(scanRoot);
199
+ handle.file = file;
200
+ handle.runKey = runKey;
201
+
202
+ const keepBytes = _recover(handle, file, runKey);
203
+ if (keepBytes <= 0) {
204
+ // Absent, foreign, or unreadable — start clean. Conservative by design.
205
+ handle.recovered.clear();
206
+ handle.order.length = 0;
207
+ handle.discarded = keepBytes === 0;
208
+ fs.writeFileSync(file, _headerLine(runKey));
209
+ } else {
210
+ // Drop any torn tail so appends land after the last verified record.
211
+ try {
212
+ const size = fs.statSync(file).size;
213
+ if (size !== keepBytes) fs.truncateSync(file, keepBytes);
214
+ } catch { /* best-effort */ }
215
+ }
216
+
217
+ handle.fd = fs.openSync(file, 'a');
218
+ handle.enabled = true;
219
+ } catch (e) {
220
+ try { if (handle.fd !== null) fs.closeSync(handle.fd); } catch { /* ignore */ }
221
+ return _emptyHandle(String((e && e.message) || e));
222
+ }
223
+ return handle;
224
+ }
225
+
226
+ /**
227
+ * Durably record that `relPath` is fully analysed, along with everything that
228
+ * analysis produced. `findings` is the per-file payload object (see the engine
229
+ * call site); it must be plain JSON data. Returns true only if the record is on
230
+ * disk and fsync'd.
231
+ */
232
+ export function recordFileDone(handle, relPath, findings) {
233
+ if (!handle || !handle.enabled || handle.fd === null || typeof relPath !== 'string') return false;
234
+ try {
235
+ if (!_jsonSafe(findings)) return false;
236
+ const d = JSON.stringify(findings === undefined ? null : findings);
237
+ if (typeof d !== 'string') return false;
238
+ const line = JSON.stringify({ f: relPath, c: _sha(d), d }) + '\n';
239
+ fs.writeSync(handle.fd, line);
240
+ fs.fsyncSync(handle.fd);
241
+ handle.written.add(relPath);
242
+ return true;
243
+ } catch {
244
+ return false;
245
+ }
246
+ }
247
+
248
+ /** Files already analysed — recovered from a prior run plus written by this one. */
249
+ export function completedFiles(handle) {
250
+ const out = new Set();
251
+ if (!handle) return out;
252
+ for (const f of handle.recovered ? handle.recovered.keys() : []) out.add(f);
253
+ for (const f of handle.written || []) out.add(f);
254
+ return out;
255
+ }
256
+
257
+ /** Recovered per-file payloads, in the order they were originally recorded. */
258
+ export function resumeFindings(handle) {
259
+ if (!handle || !handle.recovered) return [];
260
+ return (handle.order || []).map(file => ({ file, findings: handle.recovered.get(file) }));
261
+ }
262
+
263
+ /**
264
+ * Close the handle. `complete: true` means the scan finished — the checkpoint is
265
+ * removed so the next run cannot resume state that has already been consumed.
266
+ */
267
+ export function closeCheckpoint(handle, { complete = false } = {}) {
268
+ if (!handle || !handle.enabled) return false;
269
+ let ok = true;
270
+ try { if (handle.fd !== null) fs.closeSync(handle.fd); } catch { ok = false; }
271
+ handle.fd = null;
272
+ handle.enabled = false;
273
+ if (complete && handle.file) {
274
+ try { fs.rmSync(handle.file, { force: true }); } catch { ok = false; }
275
+ }
276
+ return ok;
277
+ }