@clear-capabilities/agentic-security-scanner 0.142.0 → 0.144.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/CHANGELOG.md +399 -0
  2. package/bin/agentic-security.js +530 -54
  3. package/dist/1.index.js +223 -0
  4. package/dist/113.index.js +108 -17
  5. package/dist/144.index.js +163 -0
  6. package/dist/178.index.js +1 -1
  7. package/dist/238.index.js +3 -2
  8. package/dist/265.index.js +191 -0
  9. package/dist/384.index.js +1 -1
  10. package/dist/435.index.js +165 -52
  11. package/dist/526.index.js +108 -17
  12. package/dist/552.index.js +97 -0
  13. package/dist/637.index.js +1 -1
  14. package/dist/730.index.js +311 -0
  15. package/dist/736.index.js +301 -0
  16. package/dist/824.index.js +7 -0
  17. package/dist/905.index.js +88 -22
  18. package/dist/920.index.js +491 -0
  19. package/dist/970.index.js +109 -0
  20. package/dist/agentic-security.mjs +13 -13
  21. package/dist/agentic-security.mjs.sha256 +1 -1
  22. package/dist/calibration-seed.json +2 -0
  23. package/package.json +19 -11
  24. package/src/dataflow/index.js +18 -0
  25. package/src/dataflow/privacy-catalog.js +290 -0
  26. package/src/dataflow/privacy-deep-walker.js +515 -0
  27. package/src/dataflow/privacy-governance.js +126 -0
  28. package/src/dataflow/privacy-inventory.js +154 -0
  29. package/src/dataflow/privacy-sink-policy.js +125 -0
  30. package/src/dataflow/privacy-taint.js +115 -54
  31. package/src/dataflow/privacy-taxonomy.js +233 -0
  32. package/src/discovery/disprove.js +7 -3
  33. package/src/discovery/hunter.js +9 -5
  34. package/src/discovery/index.js +2 -2
  35. package/src/discovery/llm-invoke.js +69 -13
  36. package/src/egress/audit.js +147 -0
  37. package/src/egress/policy.js +313 -0
  38. package/src/egress/redact.js +180 -0
  39. package/src/engine.js +575 -288
  40. package/src/fix/apply-fix-service.js +403 -0
  41. package/src/fix/approver-registry.js +157 -0
  42. package/src/llm-validator/index.js +86 -9
  43. package/src/llm-validator/model-status.js +66 -0
  44. package/src/mcp/tools.js +157 -50
  45. package/src/pipeline/analyzer-supervisor.js +93 -0
  46. package/src/pipeline/analyzer-worker.js +26 -0
  47. package/src/pipeline/annotator-runner.js +33 -0
  48. package/src/pipeline/assurance-mode.js +91 -0
  49. package/src/pipeline/cascade-worker-pool.js +172 -0
  50. package/src/pipeline/cascade-worker.js +43 -0
  51. package/src/pipeline/coverage-ledger.js +0 -0
  52. package/src/pipeline/detector-runner.js +51 -0
  53. package/src/pipeline/enrichment-completion.js +58 -0
  54. package/src/pipeline/evidence-provenance.js +91 -0
  55. package/src/pipeline/finding-schema.js +101 -0
  56. package/src/pipeline/legacy-compat.js +101 -0
  57. package/src/pipeline/producer-collector.js +48 -0
  58. package/src/pipeline/producer-registry.js +112 -0
  59. package/src/pipeline/scan-health.js +144 -0
  60. package/src/posture/CLAUDE.md +2 -0
  61. package/src/posture/accuracy-scorecard.js +96 -1
  62. package/src/posture/adversary-agent.js +15 -3
  63. package/src/posture/artifact-registry.js +217 -0
  64. package/src/posture/auditor-walkthrough.js +70 -8
  65. package/src/posture/calibration-feedback.js +201 -0
  66. package/src/posture/calibration-seed.json +2 -0
  67. package/src/posture/calibration.js +25 -0
  68. package/src/posture/compliance-evidence-signing.js +131 -0
  69. package/src/posture/compliance-policy.js +314 -17
  70. package/src/posture/custom-rules.js +36 -0
  71. package/src/posture/deterministic.js +8 -1
  72. package/src/posture/encryption-provider.js +205 -0
  73. package/src/posture/evidence-grade-wording.js +71 -0
  74. package/src/posture/fix-history.js +113 -19
  75. package/src/posture/fix-honesty-gate.js +47 -6
  76. package/src/posture/fix-verify.js +56 -7
  77. package/src/posture/fleet.js +0 -0
  78. package/src/posture/flow-narration.js +7 -2
  79. package/src/posture/legal-hold.js +140 -0
  80. package/src/posture/llm-redteam.js +10 -1
  81. package/src/posture/material-change.js +90 -0
  82. package/src/posture/policy-bundle.js +274 -0
  83. package/src/posture/privacy-framework.js +33 -6
  84. package/src/posture/production-feedback.js +179 -0
  85. package/src/posture/retention-policy.js +132 -0
  86. package/src/posture/risk-dollars.js +216 -26
  87. package/src/posture/scan-checkpoint.js +176 -31
  88. package/src/posture/state-dir.js +36 -1
  89. package/src/posture/state-lifecycle-report.js +77 -0
  90. package/src/posture/suppressions.js +59 -3
  91. package/src/privacy/ir-adapter.js +380 -0
  92. package/src/report/index.js +83 -18
  93. package/src/report/oscal.js +635 -0
  94. package/src/sast/cpp.js +3 -14
  95. package/src/sca/llm-function-extract.js +6 -0
@@ -9,7 +9,7 @@
9
9
  // Discount reachability tier (route-reachable > function-reachable
10
10
  // > unknown > unreachable)
11
11
  //
12
- // EV per finding = P × Impact × Discount × ConfidenceFloor
12
+ // EV per finding = P × Impact × Discount × ConfidenceWeight
13
13
  //
14
14
  // Industry breach-cost figures used here are sourced from publicly
15
15
  // reported aggregates (Ponemon Cost of a Data Breach Report — IBM/Verizon
@@ -19,6 +19,14 @@
19
19
  //
20
20
  // Disclaimer: this is an order-of-magnitude estimate for prioritization.
21
21
  // It is NOT an actuarial or insurance assessment.
22
+ //
23
+ // FR-803/FR-805 (assurance-hardening PRD): the single `ev` number above is
24
+ // only ever the BASE scenario. Every finding also gets `range` (low/base/
25
+ // high spanning conservative/base/severe), `scenarios` (all three, named),
26
+ // `assumptions` (which table entry and which tier produced each factor),
27
+ // and `modelVersion` — so a reader can trace any dollar figure back to
28
+ // exactly the inputs and methodology version that produced it, and compare
29
+ // scenarios without any of this ever touching `finding.severity`.
22
30
 
23
31
  import * as fs from 'node:fs';
24
32
  import * as path from 'node:path';
@@ -78,36 +86,126 @@ const REACH_DISCOUNT = {
78
86
  'function-reachable-but-not-route':0.4,
79
87
  };
80
88
 
89
+ // Tiny YAML — parses the two flat "section:\n key: number" blocks
90
+ // (impactUSD, familyBaseProb) plus four top-level flat scalar keys (FR-802's
91
+ // organization-specific inputs, added below). Deliberately line-based rather
92
+ // than one multi-line regex: a single regex here
93
+ // (`(?:\s+\w+\s*:\s*\d+\s*\n?)+`) looked correct but had a real, silent bug
94
+ // — `\s*` before the optional trailing `\n?` greedily consumed the NEXT
95
+ // line's leading indentation too, so the repeated group's `+` quantifier
96
+ // only ever matched ONE entry before its required leading `\s+` failed to
97
+ // find any whitespace left to consume. A risk-config.yml with more than one
98
+ // impactUSD entry silently kept only the first — found while adding
99
+ // familyBaseProb support (which inherited the identical bug) and testing
100
+ // it with more than one entry, per this session's standing rule of testing
101
+ // multi-entry cases, not just the single-entry case a hand-rolled parser's
102
+ // own author tends to reach for first.
103
+ //
104
+ // FR-802: `organizationScale`, `industry`, `recordCount`, `controlStrength`
105
+ // are free-form top-level scalars (not sections) — an operator states them
106
+ // directly, e.g. `industry: healthcare`. `recordCount` is parsed as an
107
+ // integer; a non-numeric value is treated as not configured (never silently
108
+ // coerced to NaN, which `!= null` would otherwise count as "present").
109
+ const FLAT_SCALAR_KEYS = ['organizationScale', 'industry', 'recordCount', 'controlStrength'];
110
+
81
111
  function _loadConfig(scanRoot) {
82
112
  const fp = statePath(scanRoot, 'risk-config.yml');
83
113
  if (!fs.existsSync(fp)) return null;
84
114
  try {
85
115
  const body = fs.readFileSync(fp, 'utf8');
86
- // Tiny YAML — look for impactUSD / familyBaseProb overrides
87
116
  const cfg = {};
88
- const impactMatch = body.match(/^impactUSD\s*:\s*\n((?:\s+\w+\s*:\s*\d+\s*\n?)+)/m);
89
- if (impactMatch) {
90
- cfg.impactUSD = {};
91
- for (const m of impactMatch[1].matchAll(/(\w+)\s*:\s*(\d+)/g)) cfg.impactUSD[m[1]] = parseInt(m[2], 10);
117
+ let section = null;
118
+ for (const line of body.split(/\r?\n/)) {
119
+ const sectionMatch = line.match(/^(impactUSD|familyBaseProb)\s*:\s*$/);
120
+ if (sectionMatch) { section = sectionMatch[1]; continue; }
121
+ const flatMatch = !section && line.match(/^([A-Za-z][\w-]*)\s*:\s*(\S.*?)\s*$/);
122
+ if (flatMatch && FLAT_SCALAR_KEYS.includes(flatMatch[1])) {
123
+ if (flatMatch[1] === 'recordCount') {
124
+ const n = parseInt(flatMatch[2], 10);
125
+ if (Number.isFinite(n)) cfg.recordCount = n;
126
+ } else {
127
+ cfg[flatMatch[1]] = flatMatch[2];
128
+ }
129
+ continue;
130
+ }
131
+ if (!section) continue;
132
+ const entryMatch = line.match(/^[ \t]+([\w-]+)\s*:\s*([\d.]+)\s*$/);
133
+ if (entryMatch) {
134
+ if (!cfg[section]) cfg[section] = {};
135
+ cfg[section][entryMatch[1]] = section === 'impactUSD' ? parseInt(entryMatch[2], 10) : parseFloat(entryMatch[2]);
136
+ } else if (line.trim() !== '') {
137
+ section = null; // a non-indented, non-blank line ends the current section
138
+ }
92
139
  }
93
140
  return cfg;
94
141
  } catch { return null; }
95
142
  }
96
143
 
97
- function _baseProb(family) {
144
+ // FR-801/FR-802: every dollar estimate is a SCENARIO by default — generic,
145
+ // industry-wide probability/impact tables, not this organization's actual
146
+ // exposure. `ORG_SPECIFIC_DIMENSIONS` is the full set of tunable knobs this
147
+ // module recognizes (includes `familyBaseProb`, a bonus calibration input
148
+ // that isn't one of FR-802's five named inputs). The stronger claim FR-802
149
+ // gates — "likely organizational loss" — requires ALL FIVE of its literally
150
+ // named inputs (scale, industry, record count, control strength, impact),
151
+ // tracked separately as `REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS` so that
152
+ // configuring `familyBaseProb` alone (or any subset of the five) can never
153
+ // unlock that label — only 'scenario_partially_configured', same as before.
154
+ const ORG_SPECIFIC_DIMENSIONS = ['impactUSD', 'familyBaseProb', 'organizationScale', 'industry', 'recordCount', 'controlStrength'];
155
+ const REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS = ['impactUSD', 'organizationScale', 'industry', 'recordCount', 'controlStrength'];
156
+
157
+ function _scenarioDisclosure(cfg) {
158
+ const configured = ORG_SPECIFIC_DIMENSIONS.filter(d => cfg && cfg[d] != null);
159
+ const unconfigured = ORG_SPECIFIC_DIMENSIONS.filter(d => !configured.includes(d));
160
+ const requiredMissing = REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS.filter(d => !(cfg && cfg[d] != null));
161
+
162
+ let status, message;
163
+ if (requiredMissing.length === 0) {
164
+ status = 'scenario_organization_specific';
165
+ message = `All organization-specific inputs are configured (${REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS.join(', ')}) — this estimate reflects a likely organizational loss for your organization, not a generic industry scenario.`;
166
+ } else if (configured.length > 0) {
167
+ status = 'scenario_partially_configured';
168
+ message = `Uses organization-configured values for: ${configured.join(', ')}. Still missing for a likely-organizational-loss estimate: ${requiredMissing.join(', ')}. This is NOT a likely-organizational-loss estimate.`;
169
+ } else {
170
+ status = 'scenario_default';
171
+ message = 'Uses generic industry-wide scenario defaults. No organization-specific inputs are configured (see .agentic-security/risk-config.yml) — this is NOT a likely-organizational-loss estimate.';
172
+ }
173
+
174
+ return {
175
+ status,
176
+ configuredInputs: configured,
177
+ unconfiguredInputs: unconfigured,
178
+ requiredForOrganizationSpecificLoss: REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS,
179
+ missingRequiredInputs: requiredMissing,
180
+ message,
181
+ };
182
+ }
183
+
184
+ function _baseProb(family, cfg) {
98
185
  if (!family) return 0.05;
186
+ const overrides = cfg && cfg.familyBaseProb;
187
+ if (overrides) {
188
+ const hit = overrides[family] ?? overrides[String(family).toLowerCase()];
189
+ if (typeof hit === 'number') return hit;
190
+ }
99
191
  return FAMILY_BASE_PROB[family] || FAMILY_BASE_PROB[String(family).toLowerCase()] || 0.05;
100
192
  }
101
193
 
194
+ // FR-803: split out from _impactFor so an assumption can name WHICH tier
195
+ // produced the number, not just the number itself.
196
+ function _impactTierOf(finding) {
197
+ const dc = Array.isArray(finding.dataClasses) ? finding.dataClasses : [];
198
+ if (dc.includes('PHI')) return 'PHI';
199
+ if (dc.includes('PCI')) return 'PCI';
200
+ if (dc.includes('PII')) return 'PII';
201
+ if (dc.includes('Confidential')) return 'Confidential';
202
+ if (finding.threatModel?.crownJewel) return 'crown-jewel';
203
+ return 'default';
204
+ }
205
+
102
206
  function _impactFor(finding, cfg) {
103
207
  const table = cfg && cfg.impactUSD ? { ...IMPACT_USD, ...cfg.impactUSD } : IMPACT_USD;
104
- const dc = Array.isArray(finding.dataClasses) ? finding.dataClasses : [];
105
- if (dc.includes('PHI')) return table.PHI;
106
- if (dc.includes('PCI')) return table.PCI;
107
- if (dc.includes('PII')) return table.PII;
108
- if (dc.includes('Confidential')) return table.Confidential;
109
- if (finding.threatModel?.crownJewel) return table['crown-jewel'];
110
- return table.default;
208
+ return table[_impactTierOf(finding)];
111
209
  }
112
210
 
113
211
  // SCA entries carry reachabilityTier/routeReachable (engine.js's SCA
@@ -124,12 +222,16 @@ function _relevanceTierToReachTier(relevanceTier) {
124
222
  }
125
223
  }
126
224
 
127
- function _reachDiscount(finding) {
128
- const tier = finding.reachabilityTier
225
+ // FR-803: split out from _reachDiscount so an assumption can name the tier.
226
+ function _reachTierOf(finding) {
227
+ return finding.reachabilityTier
129
228
  || (finding.routeReachable && 'route-reachable')
130
229
  || _relevanceTierToReachTier(finding.relevanceTier)
131
230
  || 'unknown';
132
- return REACH_DISCOUNT[tier] || 0.3;
231
+ }
232
+
233
+ function _reachDiscount(finding) {
234
+ return REACH_DISCOUNT[_reachTierOf(finding)] || 0.3;
133
235
  }
134
236
 
135
237
  function _epssProb(finding) {
@@ -138,28 +240,111 @@ function _epssProb(finding) {
138
240
  return null;
139
241
  }
140
242
 
243
+ // FR-803: names WHERE the probability number came from, for the assumptions
244
+ // list — not just the value.
245
+ function _probSource(finding, cfg) {
246
+ if (_epssProb(finding) != null) return 'EPSS score (finding-specific)';
247
+ const overrides = cfg && cfg.familyBaseProb;
248
+ if (overrides && (overrides[finding.family] != null || overrides[String(finding.family).toLowerCase()] != null)) {
249
+ return 'operator-configured familyBaseProb override';
250
+ }
251
+ return 'built-in industry base-rate table';
252
+ }
253
+
254
+ // Bump ONLY when the EV formula or a table's underlying MEANING changes
255
+ // (e.g. re-deriving FAMILY_BASE_PROB from a new source) — never for an
256
+ // additive field on the output shape, which stays backward compatible.
257
+ export const RISK_MODEL_VERSION = '1.0.0';
258
+
259
+ // FR-805: scenario multipliers apply only to the two GENERIC assumption
260
+ // inputs — probability-of-exploit and impact-per-incident — that FR-801
261
+ // already discloses as industry-wide defaults. Reachability discount and
262
+ // confidence weight are both measured facts about THIS finding in THIS
263
+ // scan, not assumptions with real-world spread, so they stay fixed across
264
+ // scenarios rather than being perturbed for effect.
265
+ const SCENARIO_MULTIPLIERS = {
266
+ conservative: { prob: 0.5, impact: 0.6 },
267
+ base: { prob: 1.0, impact: 1.0 },
268
+ severe: { prob: 1.75, impact: 1.5 },
269
+ };
270
+
141
271
  /**
142
272
  * Compute EV per finding. Mutates the finding in place: adds
143
- * .riskDollars = { ev, prob, impact, discount }.
273
+ * .riskDollars = { ev, prob, impact, discount, confidenceWeight,
274
+ * scenarioStatus, range: {low, base, high}, scenarios: {conservative,
275
+ * base, severe}, assumptions: [...], modelVersion, confidence }.
144
276
  */
145
277
  export function annotateRiskDollars(scanRoot, findings) {
146
- if (!Array.isArray(findings) || findings.length === 0) return { total: 0, sumEv: 0 };
278
+ const disclosure = _scenarioDisclosure(_loadConfig(scanRoot));
279
+ if (!Array.isArray(findings) || findings.length === 0) return { total: 0, sumEv: 0, scenario: disclosure };
147
280
  const cfg = _loadConfig(scanRoot);
148
281
  let sumEv = 0;
149
282
  let critEv = 0, highEv = 0;
150
283
  for (const f of findings) {
151
284
  const epss = _epssProb(f);
152
- const prob = epss != null ? epss : _baseProb(f.family);
285
+ const prob = epss != null ? epss : _baseProb(f.family, cfg);
286
+ const impactTier = _impactTierOf(f);
153
287
  const impact = _impactFor(f, cfg);
154
- const discount = _reachDiscount(f);
155
- const confidenceFloor = Math.max(0.4, f.confidence || 0.8);
156
- const ev = Math.round(prob * impact * discount * confidenceFloor);
157
- f.riskDollars = { ev, prob: Number(prob.toFixed(3)), impact, discount, confidenceFloor: Number(confidenceFloor.toFixed(2)) };
288
+ const reachTier = _reachTierOf(f);
289
+ const discount = REACH_DISCOUNT[reachTier] || 0.3;
290
+ // FR-804 (assurance-hardening PRD): this used to be
291
+ // `Math.max(0.4, f.confidence || 0.8)` TWO stacked bugs. First, `||`
292
+ // treats a real confidence of 0 (a legitimate, very-low value) as falsy
293
+ // and silently substitutes 0.8, the HIGH default — a genuinely
294
+ // near-zero-confidence finding got the same dollar weight as a
295
+ // near-certain one. Second, the unconditional `Math.max(0.4, ...)`
296
+ // floor inflated every OTHER low-confidence finding (0.05, 0.1, ...) up
297
+ // to 0.4 regardless, which is exactly "an artificially high confidence
298
+ // without an explicit reason" FR-804 names. Fixed: use the finding's
299
+ // real confidence whenever it is genuinely present (including 0), with
300
+ // no floor; 0.8 remains the default ONLY when confidence is absent
301
+ // (not a number at all), which is a "we don't know" default, not an
302
+ // inflation of a known low value.
303
+ const confidenceWeight = typeof f.confidence === 'number' ? f.confidence : 0.8;
304
+
305
+ // FR-805: three named scenarios, sharing the SAME discount/confidence —
306
+ // only the two generic inputs (prob, impact) are perturbed. `ev` (base
307
+ // scenario) is unchanged from before this cycle, since base's
308
+ // multipliers are 1.0/1.0.
309
+ const scenarios = {};
310
+ for (const [name, mult] of Object.entries(SCENARIO_MULTIPLIERS)) {
311
+ const scenarioProb = Math.min(1, prob * mult.prob);
312
+ const scenarioImpact = Math.round(impact * mult.impact);
313
+ scenarios[name] = Math.round(scenarioProb * scenarioImpact * discount * confidenceWeight);
314
+ }
315
+ const ev = scenarios.base;
316
+
317
+ // FR-803: every dollar value traces to its own assumptions and a model
318
+ // version — not just a bare number.
319
+ const assumptions = [
320
+ `probability of exploit: ${Number(prob.toFixed(3))} (source: ${_probSource(f, cfg)})`,
321
+ `impact estimate: ${fmtUsd(impact)} (tier: ${impactTier}, ${cfg?.impactUSD?.[impactTier] != null ? 'operator-configured' : 'built-in default'})`,
322
+ `reachability discount: ${discount} (tier: ${reachTier})`,
323
+ `confidence weight: ${Number(confidenceWeight.toFixed(2))} (${typeof f.confidence === 'number' ? 'measured from this scan' : 'unknown — defaulted'})`,
324
+ ];
325
+ // FR-803: a distinct confidence indicator on the estimate itself — not
326
+ // the per-finding confidenceWeight multiplier, but a coarse tier
327
+ // reflecting how much of this number is organization-specific vs.
328
+ // generic-default, folded with the finding's own detection confidence.
329
+ const confidence = disclosure.status === 'scenario_organization_specific' && confidenceWeight >= 0.7
330
+ ? 'high'
331
+ : (disclosure.status === 'scenario_default' || confidenceWeight < 0.4 ? 'low' : 'medium');
332
+
333
+ f.riskDollars = {
334
+ ev, prob: Number(prob.toFixed(3)), impact, discount,
335
+ confidenceWeight: Number(confidenceWeight.toFixed(2)),
336
+ scenarioStatus: disclosure.status,
337
+ range: { low: scenarios.conservative, base: scenarios.base, high: scenarios.severe },
338
+ scenarios,
339
+ assumptions,
340
+ modelVersion: RISK_MODEL_VERSION,
341
+ confidence,
342
+ };
158
343
  sumEv += ev;
159
344
  if (f.severity === 'critical') critEv += ev;
160
345
  else if (f.severity === 'high') highEv += ev;
161
346
  }
162
- return { total: findings.length, sumEv, critEv, highEv };
347
+ return { total: findings.length, sumEv, critEv, highEv, scenario: disclosure };
163
348
  }
164
349
 
165
350
  /**
@@ -172,4 +357,9 @@ export function fmtUsd(n) {
172
357
  return `$${n}`;
173
358
  }
174
359
 
175
- export const _internals = { FAMILY_BASE_PROB, IMPACT_USD, REACH_DISCOUNT, _baseProb, _impactFor, _reachDiscount };
360
+ export const _internals = {
361
+ FAMILY_BASE_PROB, IMPACT_USD, REACH_DISCOUNT, ORG_SPECIFIC_DIMENSIONS,
362
+ REQUIRED_FOR_ORGANIZATION_SPECIFIC_LOSS, SCENARIO_MULTIPLIERS,
363
+ _baseProb, _impactFor, _impactTierOf, _reachDiscount, _reachTierOf,
364
+ _probSource, _loadConfig, _scenarioDisclosure,
365
+ };
@@ -18,22 +18,40 @@
18
18
  // 2. Only the per-file loop is checkpointed. Every cross-file pass and the
19
19
  // whole annotation pipeline re-runs from scratch on resume, so nothing that
20
20
  // depends on the global picture can be stale by construction.
21
- // 3. Invalidation is conservative to the point of being blunt. The run key
22
- // covers the engine version, the ruleset version, the bundle SHA, a content
23
- // hash of every file in the scan (which subsumes mtime), and the scanner's
24
- // own environment switches. If any of it moved, the checkpoint is discarded
25
- // and the scan starts clean. Redoing work is merely slow; resuming stale
26
- // work is a correctness bug.
21
+ // 3. Invalidation is split into a GLOBAL identity and a PER-FILE identity
22
+ // (assurance-hardening PRD FR-208 "changed inputs invalidate only
23
+ // affected checkpoints and record the invalidation reason"). The engine
24
+ // version, ruleset version, bundle SHA, dependency-manifest contents, and
25
+ // the scanner's own environment switches form the GLOBAL key: any of them
26
+ // changing affects how EVERY file would be analysed (decision #2 above —
27
+ // cross-file/dependency-derived results aren't checkpointed per-file at
28
+ // all), so a global-key mismatch still discards the whole checkpoint,
29
+ // exactly as before. A single scanned file's own content is NOT part of
30
+ // the global key: only THAT file's record is invalidated when its content
31
+ // changes, because a single file's own per-file analysis result depends
32
+ // only on that file's content plus the (unchanged) global identity — the
33
+ // cross-file passes that COULD make it depend on other files always
34
+ // re-run from scratch per decision #2. This was a real, deliberate
35
+ // widening from the original "any change discards everything" design,
36
+ // not a relaxation of the correctness property: it only reuses a
37
+ // per-file result when nothing that result could possibly depend on has
38
+ // moved. Redoing work is merely slow; resuming stale work is a
39
+ // correctness bug, and that bar has not moved.
27
40
  //
28
41
  // CRASH SAFETY: append-and-fsync. The file is a JSONL log — one header line
29
- // pinning the run key, then one self-describing record per completed file,
30
- // each carrying a SHA-256 of its own payload. Every record is written with a
42
+ // pinning the global key (plus a plaintext meta summary used only to explain a
43
+ // mismatch, never to decide one), then one self-describing record per
44
+ // completed file, each carrying a SHA-256 of its own payload AND a SHA-256 of
45
+ // the source content it was computed from. Every record is written with a
31
46
  // single `writeSync` and immediately `fsyncSync`'d before the next file is
32
47
  // analysed, so a process killed at any instant leaves either a complete record
33
48
  // or a torn tail. On recovery we read forward while records verify and truncate
34
49
  // the file at the last byte offset that did, so a torn tail is discarded rather
35
50
  // than resumed into. Nothing is ever rewritten in place, so there is no window
36
- // in which the file is neither the old state nor the new one.
51
+ // in which the file is neither the old state nor the new one. A record whose
52
+ // OWN content hash no longer matches the current file is a separate, later
53
+ // check from tamper/tear detection — it is structurally intact, so it is not
54
+ // truncated away, just excluded from what gets replayed (see `_recover`).
37
55
  //
38
56
  // Everything here follows the posture convention of never throwing: a failure to
39
57
  // open, read or append degrades to "no checkpoint", which just means a full scan.
@@ -54,7 +72,12 @@ import { stateWritesEnabled } from './state-dir.js';
54
72
  // exists to prevent. The read-only switch above is still honoured.
55
73
  const STATE_DIR = '.agentic-security';
56
74
  const FILE_NAME = 'scan-checkpoint.jsonl';
57
- const FORMAT = 'agentic-security-scan-checkpoint/1';
75
+ // Bumped from /1: the header now stores a GLOBAL-only key (FR-208) instead of
76
+ // one that folds in every scanned file's content, plus a plaintext meta
77
+ // summary. A /1 checkpoint on disk must never be half-interpreted under /2
78
+ // semantics, so the format bump alone is enough to force a clean discard of
79
+ // anything written before this change — the safest possible migration.
80
+ const FORMAT = 'agentic-security-scan-checkpoint/2';
58
81
 
59
82
  // Env switches that change what the engine emits are part of the run identity.
60
83
  // These three are deliberately excluded: they change how the run is driven, not
@@ -92,6 +115,14 @@ export function bundleShaForRunKey() {
92
115
  * Everything that would invalidate previously-completed per-file work, reduced
93
116
  * to one hex digest. Content hashes rather than mtimes: strictly stronger, and
94
117
  * immune to filesystems with coarse or non-monotonic timestamps.
118
+ *
119
+ * Retained as its own function (still folding in `fileContents`) because it is
120
+ * a useful, independently-tested "did ANYTHING about this scan change"
121
+ * fingerprint, and because `computeGlobalKey` below is defined in terms of it
122
+ * (fileContents forced empty) — changing this function's hash would silently
123
+ * change that one too. Production checkpointing itself uses `computeGlobalKey`,
124
+ * not this function directly; see FR-208's note in the module header for why
125
+ * file content is no longer part of what invalidates the WHOLE checkpoint.
95
126
  */
96
127
  export function computeRunKey({
97
128
  engineVersion, rulesetVersion, bundleSha,
@@ -119,6 +150,50 @@ export function computeRunKey({
119
150
  return h.digest('hex');
120
151
  }
121
152
 
153
+ /**
154
+ * FR-208: the GLOBAL portion of run identity — everything that would affect
155
+ * how EVERY scanned file is analysed. Deliberately excludes `fileContents`
156
+ * (a single file's own content is checked per-record in `_recover` instead)
157
+ * but keeps `depFileContents`: dependency-manifest-derived findings are not
158
+ * part of any per-file checkpoint record (they're recomputed fresh every run,
159
+ * same as cross-file taint — decision #2 in the module header), so there is
160
+ * no per-file granularity to offer there; any manifest change invalidates
161
+ * everything, same as before this change.
162
+ */
163
+ export function computeGlobalKey({ engineVersion, rulesetVersion, bundleSha, depFileContents = {}, env = process.env } = {}) {
164
+ return computeRunKey({ engineVersion, rulesetVersion, bundleSha, fileContents: {}, depFileContents, env });
165
+ }
166
+
167
+ /**
168
+ * A plaintext (non-hashed) summary of the same inputs `computeGlobalKey` folds
169
+ * in, stored alongside the key in the checkpoint header. Used ONLY to explain
170
+ * a global-key mismatch after the fact (`_explainGlobalMismatch`) — never to
171
+ * decide whether one occurred; the key comparison remains the sole source of
172
+ * truth for that, so a caller cannot bypass invalidation by supplying
173
+ * mismatched meta.
174
+ */
175
+ export function globalKeyMeta({ engineVersion, rulesetVersion, bundleSha, depFileContents = {}, env = process.env } = {}) {
176
+ return {
177
+ engineVersion: String(engineVersion ?? ''),
178
+ rulesetVersion: String(rulesetVersion ?? ''),
179
+ bundleSha: String(bundleSha ?? 'unavailable'),
180
+ depFingerprint: computeRunKey({ engineVersion: '', rulesetVersion: '', bundleSha: '', fileContents: {}, depFileContents, env: {} }),
181
+ envFingerprint: computeRunKey({ engineVersion: '', rulesetVersion: '', bundleSha: '', fileContents: {}, depFileContents: {}, env }),
182
+ };
183
+ }
184
+
185
+ function _explainGlobalMismatch(oldMeta, newMeta) {
186
+ oldMeta = oldMeta || {};
187
+ newMeta = newMeta || {};
188
+ const parts = [];
189
+ if (oldMeta.engineVersion !== newMeta.engineVersion) parts.push(`engine version changed (${oldMeta.engineVersion || 'unknown'} -> ${newMeta.engineVersion || 'unknown'})`);
190
+ if (oldMeta.rulesetVersion !== newMeta.rulesetVersion) parts.push(`ruleset version changed (${oldMeta.rulesetVersion || 'unknown'} -> ${newMeta.rulesetVersion || 'unknown'})`);
191
+ if (oldMeta.bundleSha !== newMeta.bundleSha) parts.push('the running bundle changed');
192
+ if (oldMeta.depFingerprint !== newMeta.depFingerprint) parts.push('a dependency manifest file changed');
193
+ if (oldMeta.envFingerprint !== newMeta.envFingerprint) parts.push('an AGENTIC_SECURITY_* environment switch changed');
194
+ return parts.length ? `global scan identity changed: ${parts.join(', ')}` : 'global scan identity changed';
195
+ }
196
+
122
197
  // A value is safe to checkpoint only if JSON can carry it back unchanged. Dates,
123
198
  // regexes, Maps, Sets, functions and BigInts all survive `JSON.stringify` in a
124
199
  // lossy or throwing way; recording one would mean the resumed run sees different
@@ -150,28 +225,39 @@ function _jsonSafe(v, depth = 0, seen = new Set()) {
150
225
 
151
226
  function _emptyHandle(reason) {
152
227
  return {
153
- enabled: false, file: null, fd: null, runKey: null,
228
+ enabled: false, file: null, fd: null, globalKey: null,
154
229
  recovered: new Map(), order: [], written: new Set(),
155
- discarded: false, reason,
230
+ discarded: false, reason, invalidatedFiles: [], fileContents: {},
156
231
  };
157
232
  }
158
233
 
159
- function _headerLine(runKey) {
160
- return JSON.stringify({ v: FORMAT, runKey }) + '\n';
234
+ function _headerLine(globalKey, meta) {
235
+ return JSON.stringify({ v: FORMAT, gk: globalKey, meta: meta || {} }) + '\n';
161
236
  }
162
237
 
163
238
  // Read forward from a byte offset, keeping records while they verify. Returns
164
- // the offset of the first byte that did NOT verify, so the caller can truncate.
165
- function _recover(handle, file, runKey) {
239
+ // {offset, discardReason}: offset is the byte position the caller should keep
240
+ // (or truncate to); discardReason is set only when offset===0 (the whole
241
+ // checkpoint is being discarded) and explains why, per FR-208.
242
+ //
243
+ // A record whose payload hash (`c`) doesn't verify is torn/tampered and ends
244
+ // recovery right there, same as before FR-208. A record that verifies
245
+ // structurally but whose OWN content hash (`h`) no longer matches the
246
+ // CURRENT content of that file is a different, later check — it is not
247
+ // corrupt, so it does not truncate the file; it is simply excluded from
248
+ // `handle.recovered` and named in `handle.invalidatedFiles`, so only that
249
+ // file gets re-analysed this run.
250
+ function _recover(handle, file, globalKey, meta, fileContents) {
166
251
  let buf;
167
252
  try { buf = fs.readFileSync(file); }
168
- catch { return -1; } // no file yet
253
+ catch { return { offset: -1, discardReason: null }; } // no file yet
169
254
  const text = buf.toString('utf8');
170
255
  const nl = text.indexOf('\n');
171
- if (nl < 0) return 0;
256
+ if (nl < 0) return { offset: 0, discardReason: 'empty or corrupt checkpoint file' };
172
257
  let header = null;
173
- try { header = JSON.parse(text.slice(0, nl)); } catch { return 0; }
174
- if (!header || header.v !== FORMAT || header.runKey !== runKey) return 0;
258
+ try { header = JSON.parse(text.slice(0, nl)); } catch { return { offset: 0, discardReason: 'corrupt checkpoint header' }; }
259
+ if (!header || header.v !== FORMAT) return { offset: 0, discardReason: 'checkpoint format changed' };
260
+ if (header.gk !== globalKey) return { offset: 0, discardReason: _explainGlobalMismatch(header.meta, meta) };
175
261
 
176
262
  let offset = Buffer.byteLength(text.slice(0, nl + 1), 'utf8');
177
263
  let cursor = nl + 1;
@@ -187,19 +273,50 @@ function _recover(handle, file, runKey) {
187
273
  if (rec.c !== _sha(rec.d)) break; // tampered or torn-then-patched
188
274
  let payload;
189
275
  try { payload = JSON.parse(rec.d); } catch { break; }
276
+ // Structurally intact and parseable — advance past it regardless of what
277
+ // the per-file content check below decides; a stale-but-intact record is
278
+ // not a torn or tampered one and must not be truncated away.
279
+ offset = Buffer.byteLength(text.slice(0, cursor), 'utf8');
280
+ // FR-208: per-file content check. A record written with no tracked
281
+ // source hash (`h` absent — e.g. a caller that never supplied
282
+ // fileContents) has nothing to compare against, so it is trusted exactly
283
+ // as it was before this per-file check existed.
284
+ if (rec.h != null) {
285
+ const current = fileContents ? fileContents[rec.f] : undefined;
286
+ if (typeof current !== 'string') {
287
+ handle.invalidatedFiles.push({ file: rec.f, reason: 'no longer part of this scan' });
288
+ continue;
289
+ }
290
+ if (rec.h !== _sha(current)) {
291
+ handle.invalidatedFiles.push({ file: rec.f, reason: 'content changed since it was checkpointed' });
292
+ continue;
293
+ }
294
+ }
190
295
  if (!handle.recovered.has(rec.f)) handle.order.push(rec.f);
191
296
  handle.recovered.set(rec.f, payload);
192
- offset = Buffer.byteLength(text.slice(0, cursor), 'utf8');
193
297
  }
194
- return offset;
298
+ return { offset, discardReason: null };
195
299
  }
196
300
 
197
301
  /**
198
- * Open (or start) the checkpoint for `scanRoot` under `runKey`. Never throws.
199
- * A handle whose `enabled` is false silently no-ops through the rest of the API.
302
+ * Open (or start) the checkpoint for `scanRoot` under `globalKey`. Never
303
+ * throws. A handle whose `enabled` is false silently no-ops through the rest
304
+ * of the API.
305
+ *
306
+ * `fileContents` (FR-208): the CURRENT content of every file this scan would
307
+ * analyse, keyed by the same relative path `recordFileDone` is called with.
308
+ * Passed through to per-file content comparison on recovery, and to
309
+ * `recordFileDone` for hashing each new record's own content. Omitting it
310
+ * (or passing `{}`) degrades every record to trust-on-structural-validity
311
+ * only — the pre-FR-208 behaviour — which is exactly what a caller testing
312
+ * the checkpoint PROTOCOL itself (not per-file invalidation) wants, and
313
+ * exactly what the low-level tests in this module's test file rely on.
314
+ *
315
+ * `meta`: a plaintext summary of the inputs behind `globalKey` (see
316
+ * `globalKeyMeta`), stored in the header purely to explain a mismatch later.
200
317
  */
201
- export function openCheckpoint(scanRoot, { runKey } = {}) {
202
- if (!scanRoot || !runKey) return _emptyHandle('no-run-key');
318
+ export function openCheckpoint(scanRoot, { globalKey, meta = {}, fileContents = {} } = {}) {
319
+ if (!scanRoot || !globalKey) return _emptyHandle('no-global-key');
203
320
  // A read-only scan cannot checkpoint, and must not try. Resume is purely an
204
321
  // optimisation — without it the scan recomputes, which is slower and
205
322
  // identical — so `--no-state` wins over `AGENTIC_SECURITY_RESUME=1` rather
@@ -207,20 +324,25 @@ export function openCheckpoint(scanRoot, { runKey } = {}) {
207
324
  // the API, so no caller needs a new branch. (PRD M1)
208
325
  if (!stateWritesEnabled()) return _emptyHandle('state-writes-disabled');
209
326
  const handle = _emptyHandle(null);
327
+ handle.fileContents = fileContents || {};
210
328
  try {
211
329
  const dir = path.join(scanRoot, STATE_DIR);
212
330
  fs.mkdirSync(dir, { recursive: true });
213
331
  const file = checkpointPath(scanRoot);
214
332
  handle.file = file;
215
- handle.runKey = runKey;
333
+ handle.globalKey = globalKey;
216
334
 
217
- const keepBytes = _recover(handle, file, runKey);
335
+ const { offset: keepBytes, discardReason } = _recover(handle, file, globalKey, meta, handle.fileContents);
218
336
  if (keepBytes <= 0) {
219
- // Absent, foreign, or unreadable — start clean. Conservative by design.
337
+ // Absent, foreign, or globally invalidated — start clean. A byte offset
338
+ // of -1 (no file yet) is a fresh checkpoint, not a discard, so it gets
339
+ // no reason; 0 (format/global-key mismatch, or corrupt header) does.
220
340
  handle.recovered.clear();
221
341
  handle.order.length = 0;
342
+ handle.invalidatedFiles = [];
222
343
  handle.discarded = keepBytes === 0;
223
- fs.writeFileSync(file, _headerLine(runKey));
344
+ handle.reason = keepBytes === 0 ? discardReason : null;
345
+ fs.writeFileSync(file, _headerLine(globalKey, meta));
224
346
  } else {
225
347
  // Drop any torn tail so appends land after the last verified record.
226
348
  try {
@@ -243,6 +365,13 @@ export function openCheckpoint(scanRoot, { runKey } = {}) {
243
365
  * analysis produced. `findings` is the per-file payload object (see the engine
244
366
  * call site); it must be plain JSON data. Returns true only if the record is on
245
367
  * disk and fsync'd.
368
+ *
369
+ * Also records a SHA-256 of `relPath`'s current content (from the
370
+ * `fileContents` the handle was opened with), so a later run's `_recover` can
371
+ * tell whether this specific file has changed since — the FR-208 per-file
372
+ * half of invalidation. A handle opened without `fileContents` (or one that
373
+ * doesn't include this path) records no hash, which `_recover` treats as
374
+ * "nothing to compare," matching this module's pre-FR-208 behaviour exactly.
246
375
  */
247
376
  export function recordFileDone(handle, relPath, findings) {
248
377
  if (!handle || !handle.enabled || handle.fd === null || typeof relPath !== 'string') return false;
@@ -250,7 +379,9 @@ export function recordFileDone(handle, relPath, findings) {
250
379
  if (!_jsonSafe(findings)) return false;
251
380
  const d = JSON.stringify(findings === undefined ? null : findings);
252
381
  if (typeof d !== 'string') return false;
253
- const line = JSON.stringify({ f: relPath, c: _sha(d), d }) + '\n';
382
+ const src = handle.fileContents ? handle.fileContents[relPath] : undefined;
383
+ const h = typeof src === 'string' ? _sha(src) : null;
384
+ const line = JSON.stringify({ f: relPath, h, c: _sha(d), d }) + '\n';
254
385
  fs.writeSync(handle.fd, line);
255
386
  fs.fsyncSync(handle.fd);
256
387
  handle.written.add(relPath);
@@ -260,6 +391,20 @@ export function recordFileDone(handle, relPath, findings) {
260
391
  }
261
392
  }
262
393
 
394
+ /**
395
+ * FR-208: individually-invalidated files from a checkpoint whose GLOBAL
396
+ * identity still matched — each entry names the file and why (content
397
+ * changed, or the file is no longer part of this scan). Deliberately does
398
+ * NOT enumerate every file when the whole checkpoint was discarded instead
399
+ * (`handle.discarded`/`handle.reason` already say why, once, for all of
400
+ * them — reading the rest of a large journal just to repeat the same reason
401
+ * per file would cost real I/O for no new information). Always `[]` on a
402
+ * disabled handle, a brand-new checkpoint, or a global discard.
403
+ */
404
+ export function invalidatedFiles(handle) {
405
+ return (handle && Array.isArray(handle.invalidatedFiles)) ? handle.invalidatedFiles : [];
406
+ }
407
+
263
408
  /** Files already analysed — recovered from a prior run plus written by this one. */
264
409
  export function completedFiles(handle) {
265
410
  const out = new Set();