@clear-capabilities/agentic-security-scanner 0.161.0 → 0.163.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/CHANGELOG.md +46 -0
  2. package/bin/agentic-security.js +98 -2
  3. package/dist/1340.index.js +1871 -0
  4. package/dist/3477.index.js +727 -0
  5. package/dist/3499.index.js +23 -3
  6. package/dist/4113.index.js +59 -8
  7. package/dist/5014.index.js +833 -0
  8. package/dist/5756.index.js +2 -2
  9. package/dist/6257.index.js +1 -1
  10. package/dist/6907.index.js +248 -0
  11. package/dist/6994.index.js +1 -1
  12. package/dist/7033.index.js +33 -5
  13. package/dist/7439.index.js +1822 -0
  14. package/dist/7838.index.js +38 -2
  15. package/dist/8798.index.js +1 -1
  16. package/dist/9207.index.js +5 -0
  17. package/dist/9560.index.js +2336 -27
  18. package/dist/957.index.js +1 -1
  19. package/dist/{1905.index.js → 9999.index.js} +306 -13
  20. package/dist/agentic-security.mjs +12 -12
  21. package/dist/agentic-security.mjs.sha256 +1 -1
  22. package/package.json +39 -7
  23. package/src/capabilities/CLAUDE.md +114 -0
  24. package/src/capabilities/address.js +73 -0
  25. package/src/capabilities/attack-coverage.js +93 -0
  26. package/src/capabilities/decide.js +256 -0
  27. package/src/capabilities/delegate.js +69 -0
  28. package/src/capabilities/egress.js +20 -0
  29. package/src/capabilities/hook-advice.js +50 -0
  30. package/src/capabilities/index.js +32 -0
  31. package/src/capabilities/manifest.js +344 -0
  32. package/src/capabilities/outbound.js +189 -0
  33. package/src/capabilities/paths.js +57 -0
  34. package/src/capabilities/probes.js +211 -0
  35. package/src/capabilities/proxy.js +182 -0
  36. package/src/capabilities/reasons.js +67 -0
  37. package/src/capabilities/receipts.js +356 -0
  38. package/src/capabilities/records.js +58 -0
  39. package/src/capabilities/recovery.js +367 -0
  40. package/src/capabilities/report.js +81 -0
  41. package/src/capabilities/runner.js +236 -0
  42. package/src/capabilities/secrets.js +26 -0
  43. package/src/capabilities/tool-gate.js +119 -0
  44. package/src/capabilities/tool-registry.js +60 -0
  45. package/src/dataflow/CLAUDE.md +29 -0
  46. package/src/dataflow/catalog.js +21 -0
  47. package/src/dataflow/engine.js +20 -0
  48. package/src/dataflow/guard-dominance.js +352 -0
  49. package/src/discovery/CLAUDE.md +18 -0
  50. package/src/discovery/index.js +43 -1
  51. package/src/discovery/lenses.js +18 -0
  52. package/src/discovery/memory.js +7 -5
  53. package/src/engine.js +80 -15
  54. package/src/fix/apply-fix-service.js +33 -3
  55. package/src/integrations/index.js +5 -2
  56. package/src/ir/CLAUDE.md +14 -0
  57. package/src/ir/implicit-handler-params.js +74 -0
  58. package/src/ir/parser-cs.js +20 -0
  59. package/src/ir/parser-go.js +30 -13
  60. package/src/ir/parser-java.js +11 -0
  61. package/src/language/nix-eval-isolation.js +33 -5
  62. package/src/lineage/CLAUDE.md +31 -0
  63. package/src/lineage/deployment/adapter-cloud.js +202 -0
  64. package/src/lineage/deployment/adapter-compose.js +90 -0
  65. package/src/lineage/deployment/adapter-kit.js +142 -0
  66. package/src/lineage/deployment/adapter-kubernetes.js +247 -0
  67. package/src/lineage/deployment/attack-paths.js +443 -0
  68. package/src/lineage/deployment/boundaries-run.js +223 -0
  69. package/src/lineage/deployment/boundary-graph.js +567 -0
  70. package/src/lineage/deployment/cross-repo.js +253 -0
  71. package/src/lineage/deployment/drift.js +383 -0
  72. package/src/lineage/deployment/ingest.js +221 -0
  73. package/src/lineage/deployment/projection.js +244 -0
  74. package/src/lineage/deployment/trace-correlation.js +541 -0
  75. package/src/lineage/source-registry.js +19 -0
  76. package/src/mcp/CLAUDE.md +14 -3
  77. package/src/mcp/invariant-tools.js +64 -0
  78. package/src/mcp/portfolio-tools.js +79 -0
  79. package/src/mcp/redact.js +15 -0
  80. package/src/mcp/server.js +24 -3
  81. package/src/mcp/tools.js +30 -5
  82. package/src/posture/CLAUDE.md +505 -0
  83. package/src/posture/accuracy-scorecard.js +95 -4
  84. package/src/posture/assurance/baseline-inventory.js +218 -0
  85. package/src/posture/assurance/baseline.js +322 -0
  86. package/src/posture/assurance/bounded-io.js +145 -0
  87. package/src/posture/assurance/config.js +299 -0
  88. package/src/posture/assurance/contracts.js +315 -0
  89. package/src/posture/assurance/identity.js +62 -0
  90. package/src/posture/assurance/migrations.js +259 -0
  91. package/src/posture/assurance/rollback.js +214 -0
  92. package/src/posture/assurance/rollout.js +200 -0
  93. package/src/posture/assurance/schema-kit.js +111 -0
  94. package/src/posture/assurance/verification-record.js +258 -0
  95. package/src/posture/attestation.js +5 -0
  96. package/src/posture/autopilot.js +28 -1
  97. package/src/posture/business-logic.js +26 -4
  98. package/src/posture/evaluation/bakeoff.js +148 -0
  99. package/src/posture/evaluation/custody.js +200 -0
  100. package/src/posture/evaluation/deployment-ablation.js +416 -0
  101. package/src/posture/evaluation/dev-cases.js +65 -0
  102. package/src/posture/evaluation/economics.js +85 -0
  103. package/src/posture/evaluation/gates.js +161 -0
  104. package/src/posture/evaluation/grouping.js +101 -0
  105. package/src/posture/evaluation/interval.js +132 -0
  106. package/src/posture/evaluation/invariant-ablation.js +325 -0
  107. package/src/posture/evaluation/labels.js +200 -0
  108. package/src/posture/evaluation/protocol.js +308 -0
  109. package/src/posture/evaluation/release-gate.js +209 -0
  110. package/src/posture/evaluation/report.js +158 -0
  111. package/src/posture/evaluation/runner.js +273 -0
  112. package/src/posture/evaluation/scan-child.js +80 -0
  113. package/src/posture/evaluation/score.js +174 -0
  114. package/src/posture/evaluation/synthetic.js +90 -0
  115. package/src/posture/evaluation/why-missed.js +219 -0
  116. package/src/posture/evidence-bundle.js +93 -7
  117. package/src/posture/execution-proof.js +11 -1
  118. package/src/posture/fix-verify-loop.js +20 -6
  119. package/src/posture/fix-verify.js +37 -2
  120. package/src/posture/fleet.js +11 -2
  121. package/src/posture/invariants/cli.js +95 -0
  122. package/src/posture/invariants/coverage.js +164 -0
  123. package/src/posture/invariants/export.js +178 -0
  124. package/src/posture/invariants/expressions.js +46 -0
  125. package/src/posture/invariants/lifecycle.js +289 -0
  126. package/src/posture/invariants/project-input.js +55 -0
  127. package/src/posture/invariants/repair.js +340 -0
  128. package/src/posture/invariants/run.js +45 -0
  129. package/src/posture/invariants/scenarios.js +311 -0
  130. package/src/posture/invariants/schema.js +167 -0
  131. package/src/posture/invariants/shrink.js +407 -0
  132. package/src/posture/invariants/state-assertions.js +165 -0
  133. package/src/posture/logic-claims.js +12 -0
  134. package/src/posture/model-routing.js +40 -0
  135. package/src/posture/oracles/adapters.js +531 -0
  136. package/src/posture/oracles/conformance.js +276 -0
  137. package/src/posture/oracles/oracle.js +453 -0
  138. package/src/posture/oracles/registry.js +76 -0
  139. package/src/posture/oracles/scenario-classes.js +223 -0
  140. package/src/posture/portfolio/backend.js +279 -0
  141. package/src/posture/portfolio/bundle.js +314 -0
  142. package/src/posture/portfolio/cli.js +142 -0
  143. package/src/posture/portfolio/manifest.js +263 -0
  144. package/src/posture/portfolio/progress.js +267 -0
  145. package/src/posture/portfolio/resume.js +130 -0
  146. package/src/posture/portfolio/retention.js +207 -0
  147. package/src/posture/portfolio/scheduler.js +488 -0
  148. package/src/posture/portfolio/signing.js +155 -0
  149. package/src/posture/portfolio/wording.js +59 -0
  150. package/src/posture/portfolio/work-units.js +405 -0
  151. package/src/posture/replay/replay.js +250 -0
  152. package/src/posture/routing/adapter.js +140 -0
  153. package/src/posture/routing/calibration.js +278 -0
  154. package/src/posture/routing/canary.js +141 -0
  155. package/src/posture/routing/control.js +35 -0
  156. package/src/posture/routing/decide.js +211 -0
  157. package/src/posture/routing/drift.js +137 -0
  158. package/src/posture/routing/economics.js +185 -0
  159. package/src/posture/routing/feedback.js +207 -0
  160. package/src/posture/routing/index.js +31 -0
  161. package/src/posture/routing/outcomes.js +281 -0
  162. package/src/posture/routing/promotion.js +291 -0
  163. package/src/posture/routing/receipts.js +59 -0
  164. package/src/posture/routing/report.js +142 -0
  165. package/src/posture/routing/shadow.js +53 -0
  166. package/src/posture/specification-mining.js +26 -0
  167. package/src/posture/verification/advisory-state.js +71 -0
  168. package/src/posture/verification/emit.js +257 -0
  169. package/src/posture/verification/hypothesis-promotion.js +211 -0
  170. package/src/posture/verification/patch-negative.js +226 -0
  171. package/src/posture/verification/patch-promotion.js +113 -0
  172. package/src/posture/verification/projection.js +183 -0
  173. package/src/posture/verification/repair-records.js +124 -0
  174. package/src/posture/verifier.js +16 -0
  175. package/src/report/index.js +100 -1
  176. package/src/sandbox/CLAUDE.md +51 -0
  177. package/src/sandbox/backend-namespace.js +7 -0
  178. package/src/sandbox/backend-userspace.js +151 -34
  179. package/src/sandbox/control-probes.js +215 -0
  180. package/src/sandbox/supervise.js +201 -0
  181. package/src/sandbox/trust-boundary.js +113 -0
  182. package/src/sandbox/trust-domains.js +171 -0
@@ -26,6 +26,8 @@
26
26
 
27
27
  import fs from 'node:fs';
28
28
  import path from 'node:path';
29
+ import { emitVerification } from './verification/emit.js';
30
+ import { verificationFields } from './verification/projection.js';
29
31
 
30
32
  const SCHEMA = 'agentic-security/autopilot@1';
31
33
 
@@ -73,8 +75,15 @@ function _save(stateFile, state) {
73
75
  */
74
76
  export async function runAutopilot({
75
77
  stages = {}, stateFile = null, resume = true, apply = false,
76
- onStage = () => {}, severities = ['critical', 'high'], maxFindings = Infinity,
78
+ onStage = () => {}, severities = ['critical', 'high'], maxFindings = Infinity, commit = null,
77
79
  } = {}) {
80
+ // X-201: every settled finding carries the one version-1 verification record, ADDITIVELY (`outcome`, `reason` and
81
+ // every other field are unchanged). The chain's own proof and re-verify stages run under the older sandbox path, so
82
+ // the record never says `confirmed` or `refuted` here: it records what was observed and what remains unproven.
83
+ const seal = (rec, finding, patch) => {
84
+ const e = emitVerification('autopilot', rec, { finding, patch: patch?.patch, commit });
85
+ if (e.ok) rec.verificationRecord = e.record;
86
+ };
78
87
  const required = ['scan'];
79
88
  for (const r of required) {
80
89
  if (typeof stages[r] !== 'function') return { ok: false, reason: `no ${r} stage supplied` };
@@ -138,6 +147,7 @@ export async function runAutopilot({
138
147
  if (rec.validation === 'refuted') {
139
148
  rec.outcome = 'NEEDS_REVIEW';
140
149
  rec.reason = 'an independent verifier refuted this finding; not fixed automatically';
150
+ seal(rec, f, null);
141
151
  results.push(rec); state.findings[key] = rec; _save(stateFile, state);
142
152
  onStage({ stage: 'validate', key, verdict: 'refuted' });
143
153
  continue;
@@ -149,6 +159,7 @@ export async function runAutopilot({
149
159
  if (!isProven) {
150
160
  rec.outcome = 'UNPROVEN';
151
161
  rec.reason = proved?.proofEvidence?.reason || 'no proof-of-concept demonstrated this finding';
162
+ seal(rec, f, null);
152
163
  results.push(rec); state.findings[key] = rec; _save(stateFile, state);
153
164
  continue;
154
165
  }
@@ -161,6 +172,7 @@ export async function runAutopilot({
161
172
  if (!patch || !patch.patch) {
162
173
  rec.outcome = 'NO_FIX';
163
174
  rec.reason = 'no patch was synthesised';
175
+ seal(rec, f, null);
164
176
  results.push(rec); state.findings[key] = rec; _save(stateFile, state);
165
177
  continue;
166
178
  }
@@ -171,6 +183,7 @@ export async function runAutopilot({
171
183
  if (typeof stages.verifyFix !== 'function') {
172
184
  rec.outcome = 'NEEDS_REVIEW';
173
185
  rec.reason = 'no verifyFix stage supplied — a patch that cannot be re-verified is never applied';
186
+ seal(rec, f, patch);
174
187
  results.push(rec); state.findings[key] = rec; _save(stateFile, state);
175
188
  continue;
176
189
  }
@@ -194,6 +207,7 @@ export async function runAutopilot({
194
207
  : (!rec.testsPass ? 'the project test suite fails with this patch'
195
208
  : (v?.reason || 'the patch did not re-verify'));
196
209
  }
210
+ seal(rec, f, patch);
197
211
  results.push(rec);
198
212
  state.findings[key] = rec;
199
213
  _save(stateFile, state);
@@ -203,6 +217,19 @@ export async function runAutopilot({
203
217
  return { ok: true, results, skipped, outOfScope, capped, summary: summarizeAutopilot(results, outOfScope, capped) };
204
218
  }
205
219
 
220
+ /**
221
+ * X-206: the autopilot response as it is serialized for a caller. Every field of every result is kept (the native `outcome`
222
+ * vocabulary and its meaning are unchanged); a result that carries a verification record additionally gets `verificationView`, the
223
+ * shared projection every other interface shows for the same finding. Results without a record are returned as they were.
224
+ */
225
+ export function serializeAutopilotResult(res) {
226
+ if (!res || !Array.isArray(res.results)) return res;
227
+ return {
228
+ ...res,
229
+ results: res.results.map((r) => (r && r.verificationRecord ? { ...r, ...verificationFields(r.verificationRecord, { replay: r.verificationReplay }) } : r)),
230
+ };
231
+ }
232
+
206
233
  export function summarizeAutopilot(results, outOfScope = 0, capped = 0) {
207
234
  const byOutcome = Object.fromEntries(OUTCOMES.map(o => [o, 0]));
208
235
  for (const r of results) if (r.outcome in byOutcome) byOutcome[r.outcome]++;
@@ -17,6 +17,8 @@
17
17
  // case. Heuristic: check the test-files corpus for any test asserting
18
18
  // a 401 / 403 status against the route's path.
19
19
 
20
+ import { inferredInvariant } from './invariants/lifecycle.js';
21
+
20
22
  const JS_TS_RE = /\.(?:js|jsx|ts|tsx|mjs|cjs)$/i;
21
23
  const PY_RE = /\.py$/i;
22
24
 
@@ -135,20 +137,25 @@ function emitAuthZMatrixFindings(matrix) {
135
137
 
136
138
  // ─── FR-LOGIC-2 State machine extraction ──────────────────────────────────
137
139
 
138
- function extractStateMachine(fileContents) {
140
+ function collectStateSets(fileContents) {
139
141
  // Find sites where a literal set of statuses appears (e.g.
140
142
  // `STATUSES = ['pending', 'approved', 'rejected']` or an enum-like in TS).
141
- // Then look for direct writes like `.status = "<not in set>"` and flag.
142
- const stateSets = []; // [{file, line, name, values: Set<string>}]
143
+ const stateSets = []; // [{file, line, name, values: Set<string>, ordered: string[]}]
143
144
  for (const [fp, c] of Object.entries(fileContents || {})) {
144
145
  if (typeof c !== 'string' || !JS_TS_RE.test(fp)) continue;
145
146
  const re = /\b(STATUSES|STATES|PHASES|ALLOWED_STATUSES|[A-Z_]+_STATUSES)\s*=\s*\[\s*((?:['"][^'"]+['"]\s*,?\s*){2,})\]/g;
146
147
  let m;
147
148
  while ((m = re.exec(c))) {
148
149
  const values = [...m[2].matchAll(/['"]([^'"]+)['"]/g)].map(x => x[1]);
149
- stateSets.push({ file: fp, line: _lineOf(c, m.index), name: m[1], values: new Set(values) });
150
+ stateSets.push({ file: fp, line: _lineOf(c, m.index), name: m[1], values: new Set(values), ordered: values });
150
151
  }
151
152
  }
153
+ return stateSets;
154
+ }
155
+
156
+ function extractStateMachine(fileContents) {
157
+ // Then look for direct writes like `.status = "<not in set>"` and flag.
158
+ const stateSets = collectStateSets(fileContents);
152
159
  if (!stateSets.length) return [];
153
160
  const findings = [];
154
161
  for (const [fp, c] of Object.entries(fileContents || {})) {
@@ -236,4 +243,19 @@ export function scanBusinessLogic(fileContents) {
236
243
  }
237
244
 
238
245
  // For tests + the no-dead-modules check.
246
+ // X-402: a declared status list read as an inferred workflow contract. The declared order is the only evidence for the allowed
247
+ // transitions (each value to the next), which is a guess a reviewer must confirm, so the proposal carries a high uncertainty and is
248
+ // advisory until approved in the signed ledger.
249
+ export function mineWorkflowInvariants(fileContents) {
250
+ const out = [];
251
+ for (const set of collectStateSets(fileContents)) {
252
+ const states = set.ordered.filter((v) => /^[A-Za-z0-9_.:/-]{1,64}$/.test(v));
253
+ if (states.length < 2) continue;
254
+ const allowed = states.slice(0, -1).map((from, i) => ({ from, to: states[i + 1] }));
255
+ const inv = inferredInvariant({ class: 'workflow-order', source: 'state-machine', evidence: { file: set.file, line: set.line, set: set.name }, uncertainty: 0.7, action: 'advance', allowed: allowed.slice(0, 16) });
256
+ if (inv) out.push(inv);
257
+ }
258
+ return out;
259
+ }
260
+
239
261
  export const _internals = { extractAuthZMatrix, extractStateMachine, findNegativeTestGaps };
@@ -0,0 +1,148 @@
1
+ // Bake-off recipe: the manifest, the adapter contract and the claim guard (QA-008.AC02, QA-008.AC03).
2
+ //
3
+ // A bake-off compares this engine with other tools on one workload. The failure modes that make such a comparison worthless are
4
+ // all about what is silently different between the runs, so the recipe pins them in a manifest, and this module refuses to
5
+ // produce a comparative claim when any of them is not identical:
6
+ //
7
+ // workload the same target set, by digest, for every participant
8
+ // scope the same languages and families
9
+ // provider and model named per participant (an engine with no model says so)
10
+ // verification one standard for what counts as a confirmed defect
11
+ // limits the same time and spend ceilings
12
+ // failures a timeout or crash is a recorded failure, scored as a miss, never dropped
13
+ //
14
+ // Comparators are GENERIC SLOTS (`comparator-a`, `comparator-b`, ...). This repository names no other tool: the operator maps a
15
+ // slot to whatever they run. An adapter can declare itself `unavailable` (not installed, no licence, no provider access), and that
16
+ // participant is `not-evaluated`: it is not a zero, not a loss and not excluded quietly. It appears in the report with its
17
+ // reason. A comparative accuracy or cost claim exists only between participants that were actually evaluated on the identical
18
+ // workload under the identical standard, and only for a quantity both measured; otherwise the claim is withheld with the reasons.
19
+ //
20
+ // Scoring is delegated to comparison.js (CWE-only matching, intersection of completed entries), unchanged.
21
+ //
22
+ // Pure: no fs, no clock, no process execution. An adapter's `run` is supplied by the caller.
23
+
24
+ import { digestOf } from '../assurance/identity.js';
25
+ import { compareParticipants, scoreParticipant } from '../comparison.js';
26
+
27
+ const BAKEOFF_SCHEMA = 'agentic-security/bakeoff-manifest';
28
+ const ENGINE_SLOT = 'this-engine';
29
+ const SLOT_RE = /^(?:this-engine|comparator-[a-z])$/;
30
+ export const UNAVAILABLE_REASONS = Object.freeze(['not-installed', 'no-licence', 'no-provider-access', 'not-configured', 'platform-unsupported', 'operator-declined']);
31
+
32
+ const isObj = (x) => x && typeof x === 'object' && !Array.isArray(x);
33
+ const err = (path, message) => ({ code: 'BAD_MANIFEST', path, message });
34
+
35
+ /** @returns {{ok: boolean, errors: object[]}} */
36
+ export function validateBakeoffManifest(m) {
37
+ const errors = [];
38
+ if (!isObj(m)) return { ok: false, errors: [err('', 'manifest must be an object')] };
39
+ if (m.schema !== BAKEOFF_SCHEMA) errors.push(err('schema', `schema must be ${BAKEOFF_SCHEMA}`));
40
+ const w = m.workload;
41
+ if (!isObj(w) || !Array.isArray(w.targetIds) || !w.targetIds.length) errors.push(err('workload.targetIds', 'name the target ids the workload consists of'));
42
+ else {
43
+ if (!/^sha256:[0-9a-f]{64}$/.test(w.workloadDigest || '')) errors.push(err('workload.workloadDigest', 'the workload is pinned by digest'));
44
+ else if (w.workloadDigest !== workloadDigestOf(w)) errors.push(err('workload.workloadDigest', 'the digest does not match the targets and tree digests listed'));
45
+ }
46
+ if (!isObj(m.scope) || !Array.isArray(m.scope.languages) || !Array.isArray(m.scope.families)) errors.push(err('scope', 'declare the languages and families in scope'));
47
+ if (!isObj(m.verification) || !m.verification.standard) errors.push(err('verification.standard', 'one verification standard must be named (what counts as a confirmed defect)'));
48
+ const lim = m.limits;
49
+ if (!isObj(lim) || !(lim.perTargetTimeoutMs > 0) || !(lim.wallClockMs > 0) || typeof lim.spendCeilingUsd !== 'number') errors.push(err('limits', 'perTargetTimeoutMs, wallClockMs and spendCeilingUsd are all required'));
50
+ if (m.failurePolicy !== 'count-as-misses') errors.push(err('failurePolicy', 'failures are scored as misses; the only accepted policy is count-as-misses'));
51
+ const ps = m.participants;
52
+ if (!Array.isArray(ps) || ps.length < 2) errors.push(err('participants', 'a bake-off needs this engine and at least one comparator slot'));
53
+ else {
54
+ const seen = new Set();
55
+ ps.forEach((p, i) => {
56
+ if (!isObj(p) || !SLOT_RE.test(p.slot || '')) { errors.push(err(`participants[${i}].slot`, 'a slot is "this-engine" or "comparator-a".."comparator-z"; no product name belongs in a recipe')); return; }
57
+ if (seen.has(p.slot)) errors.push(err(`participants[${i}].slot`, `duplicate slot ${p.slot}`));
58
+ seen.add(p.slot);
59
+ if (!isObj(p.provider) || !('model' in p.provider)) errors.push(err(`participants[${i}].provider`, 'name the provider and model, or {"model": null} for a tool that uses none'));
60
+ });
61
+ if (!seen.has(ENGINE_SLOT)) errors.push(err('participants', 'this-engine must be one of the participants'));
62
+ }
63
+ return { ok: errors.length === 0, errors };
64
+ }
65
+
66
+ /** The digest a workload is pinned by: its targets and their tree digests, nothing else. */
67
+ export function workloadDigestOf(workload) {
68
+ const targets = Object.fromEntries((workload.targetIds || []).slice().sort().map((id) => [id, workload.treeDigests?.[id] ?? null]));
69
+ return digestOf({ kind: 'bakeoff-workload', targets });
70
+ }
71
+
72
+ /**
73
+ * Decide, per participant, whether it can be evaluated. `adapters` maps a slot to `{ available: boolean, reason?, run? }`.
74
+ * A slot with no adapter, or an adapter that says it is unavailable, is `not-evaluated` with a typed reason.
75
+ */
76
+ export function resolveParticipants(manifest, adapters = {}) {
77
+ return manifest.participants.map((p) => {
78
+ const a = adapters[p.slot];
79
+ if (!a) return { slot: p.slot, provider: p.provider, state: 'not-evaluated', reason: 'not-configured', detail: 'no adapter was supplied for this slot' };
80
+ if (a.available === false) {
81
+ const reason = UNAVAILABLE_REASONS.includes(a.reason) ? a.reason : 'not-configured';
82
+ return { slot: p.slot, provider: p.provider, state: 'not-evaluated', reason, detail: a.detail || null };
83
+ }
84
+ return { slot: p.slot, provider: p.provider, state: 'evaluated', reason: null, detail: null };
85
+ });
86
+ }
87
+
88
+ /**
89
+ * Judge a finished bake-off.
90
+ * @param {object} o
91
+ * @param {object} o.manifest a valid manifest
92
+ * @param {object[]} o.entries [{id, cwe}] the workload's answer key
93
+ * @param {Object<string, {workloadDigest: string, verificationStandard: string, results: object, costUsd?: number|null, completed?: number}>} o.runs by slot, for evaluated participants
94
+ * @param {Object<string, object>} [o.adapters]
95
+ */
96
+ export function judgeBakeoff({ manifest, entries, runs = {}, adapters = {} }) {
97
+ const v = validateBakeoffManifest(manifest);
98
+ if (!v.ok) return { ok: false, errors: v.errors };
99
+ const resolved = resolveParticipants(manifest, adapters);
100
+ const participants = resolved.map((r) => {
101
+ if (r.state !== 'evaluated') return r;
102
+ const run = runs[r.slot];
103
+ if (!run) return { ...r, state: 'not-evaluated', reason: 'no-run-recorded', detail: 'the adapter was available but no result was recorded' };
104
+ return { ...r, run: { workloadDigest: run.workloadDigest, verificationStandard: run.verificationStandard, costUsd: run.costUsd ?? null } };
105
+ });
106
+ const evaluated = participants.filter((p) => p.state === 'evaluated');
107
+ const withheld = [];
108
+ for (const p of participants.filter((x) => x.state === 'not-evaluated')) withheld.push({ slot: p.slot, reason: `${p.slot} was not evaluated (${p.reason}); no comparison with it is made` });
109
+
110
+ // Comparability: every evaluated participant must have run the manifest's workload under the manifest's standard.
111
+ const mismatched = evaluated.filter((p) => p.run.workloadDigest !== manifest.workload.workloadDigest || p.run.verificationStandard !== manifest.verification.standard);
112
+ for (const p of mismatched) withheld.push({ slot: p.slot, reason: `${p.slot} ran a different workload or verification standard than the manifest pins; its figures are not comparable` });
113
+ const comparable = evaluated.filter((p) => !mismatched.includes(p));
114
+
115
+ let comparison = null;
116
+ const claims = [];
117
+ if (comparable.length >= 2) {
118
+ const parts = comparable.map((p) => ({ id: p.slot, results: runs[p.slot].results }));
119
+ comparison = compareParticipants(entries, parts);
120
+ if (!comparison.ok) { withheld.push({ slot: null, reason: `scoring refused: ${comparison.reason}` }); comparison = null; }
121
+ } else {
122
+ withheld.push({ slot: null, reason: 'fewer than two participants were evaluated on the identical workload; there is nothing to compare' });
123
+ }
124
+ // End-to-end view per participant: an entry it could not complete is a MISS here (the comparison above is over the shared
125
+ // intersection and is labelled as such). Both views are published; neither replaces the other.
126
+ const endToEnd = {};
127
+ for (const p of comparable) {
128
+ const sc = scoreParticipant(entries, runs[p.slot].results);
129
+ let detected = 0; for (const v of sc.per.values()) if (v.detected) detected++;
130
+ endToEnd[p.slot] = { entries: entries.length, completed: sc.completed, failedOrMissing: entries.length - sc.completed, detected, recallEndToEnd: entries.length ? detected / entries.length : null, rule: 'failures and missing results count as misses' };
131
+ }
132
+ if (comparison) {
133
+ // An accuracy claim is a statement of the measured figures over the shared intersection; it never says "better" for a tie or an
134
+ // interval overlap, and it needs every participant to have completed the shared entries.
135
+ claims.push({ kind: 'accuracy', basis: 'shared-intersection', participants: comparable.map((p) => p.slot), statement: 'figures over the entries every compared participant completed; see the comparison table' });
136
+ const costs = comparable.map((p) => ({ slot: p.slot, usd: runs[p.slot].costUsd ?? null }));
137
+ if (costs.every((c) => typeof c.usd === 'number')) claims.push({ kind: 'cost', participants: costs.map((c) => c.slot), costsUsd: Object.fromEntries(costs.map((c) => [c.slot, c.usd])), statement: 'measured spend per participant under the same limits' });
138
+ else withheld.push({ slot: null, reason: `no cost claim: ${costs.filter((c) => typeof c.usd !== 'number').map((c) => c.slot).join(', ')} did not report a measured cost` });
139
+ }
140
+ const report = {
141
+ schema: 'agentic-security/bakeoff-report', ok: true, errors: [], manifestDigest: digestOf(manifest),
142
+ workloadDigest: manifest.workload.workloadDigest, participants, comparison, endToEnd, claims, withheld,
143
+ superiority: 'none claimed',
144
+ note: 'a participant that was not evaluated is reported as such; it is never a zero, a loss or a silent omission',
145
+ };
146
+ report.reportDigest = digestOf({ ...report, reportDigest: undefined });
147
+ return report;
148
+ }
@@ -0,0 +1,200 @@
1
+ // Label custody and leakage controls (QA-002.AC03).
2
+ //
3
+ // Two trust domains meet here. The CUSTODIAN (verifier domain) holds adjudicated
4
+ // labels. The engine under test, and any worker or model prompt, is on the other
5
+ // side and must never see an answer key. This module provides the custodian's
6
+ // store, the workspace staging that builds what the engine is allowed to read,
7
+ // and the audits that fail a run when anything label-shaped crosses over.
8
+ //
9
+ // Enforcement, honestly stated:
10
+ // - role checks use the existing trust-domain policy (`mayDo`): workers and
11
+ // targets are denied sealed-label read AND write;
12
+ // - staging and prompt audits are PATTERN checks over known protected terms and
13
+ // file names. They catch planted and accidental leaks of the kinds listed in
14
+ // `leakFixtures`; they are not a proof that no paraphrase gets through;
15
+ // - process-level read denial of the label directory is the sandbox trust
16
+ // boundary's job (`runInBoundary({ labelDirs })`), which this module only
17
+ // feeds. Where no backend proves that control the boundary reports `blocked`.
18
+
19
+ import * as fs from 'node:fs';
20
+ import * as path from 'node:path';
21
+ import { DOMAINS, RESOURCES, mayDo, isUntrusted, scrubEnv } from '../../sandbox/trust-domains.js';
22
+ import { digestOf } from '../assurance/identity.js';
23
+ import { STATE_DIR_NAME } from '../state-dir.js';
24
+ import { validateDefectLabel, validateNegativeLabel } from './labels.js';
25
+
26
+ export const LABEL_FILE = 'adjudicated-labels.json';
27
+
28
+ // File names that look like an answer key. Regexes, not literals, so this file
29
+ // is not itself a reference to any particular key file.
30
+ const LEAK_FILENAME_PATTERNS = Object.freeze([
31
+ /(^|\/)(labels?|answers?|expected|ground[-_]?truth|answer[-_]?key|adjudicat\w*|oracle)[^/]*\.(json|jsonl|ya?ml|csv|txt)$/i,
32
+ /(^|\/)result\.json$/i,
33
+ /(^|\/)corpus-baseline[^/]*$/i,
34
+ /(^|\/)(advisory|advisories|ghsa|osv)[^/]*\.(json|md|txt|ya?ml)$/i,
35
+ ]);
36
+
37
+ const BENCHMARK_NAMES = Object.freeze(['owasp benchmark', 'juliet', 'sard test', 'cve-replay', 'corpus-baseline', 'answer key', 'ground truth']);
38
+ const ADVISORY_ID = /\b(CVE-\d{4}-\d{4,}|GHSA-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4})\b/i;
39
+ const MIN_TERM = 8;
40
+
41
+ /**
42
+ * Terms that must never reach the engine or a prompt: reviewer evidence refs and
43
+ * rationale, advisory ids and prose, root-cause ids, and fixed-source hints.
44
+ */
45
+ export function protectedTermsFrom({ defects = [], negatives = [], advisoryText = [], fixedSourceHints = [], benchmarkNames = BENCHMARK_NAMES } = {}) {
46
+ const terms = new Set();
47
+ const add = (s) => { if (typeof s === 'string' && s.trim().length >= MIN_TERM) terms.add(s.trim().toLowerCase()); };
48
+ for (const d of defects) {
49
+ add(d.rootCauseId);
50
+ for (const e of d.evidence || []) add(e.ref);
51
+ for (const a of d.advisoryRefs || []) add(a);
52
+ }
53
+ for (const n of negatives) add(n.rationale);
54
+ for (const s of [...advisoryText, ...fixedSourceHints, ...benchmarkNames]) add(s);
55
+ return [...terms].sort();
56
+ }
57
+
58
+ /** Leaks in a piece of text (a prompt, a file body). */
59
+ export function auditText(text, terms, { strictAdvisoryIds = true } = {}) {
60
+ const hay = String(text ?? '');
61
+ const lower = hay.toLowerCase();
62
+ const leaks = [];
63
+ for (const t of terms || []) if (lower.includes(t)) leaks.push({ kind: 'protected-term', detail: t.length > 60 ? `${t.slice(0, 57)}...` : t });
64
+ if (strictAdvisoryIds) {
65
+ const m = ADVISORY_ID.exec(hay);
66
+ if (m) leaks.push({ kind: 'advisory-id', detail: m[1] });
67
+ }
68
+ return leaks;
69
+ }
70
+
71
+ /** Gate a model/scanner prompt. `ok:false` means the call must not be made. */
72
+ export function guardPrompt(prompt, terms, opts) {
73
+ const leaks = auditText(prompt, terms, opts);
74
+ return { ok: leaks.length === 0, leaks };
75
+ }
76
+
77
+ const SKIP_DIRS = new Set(['.git', STATE_DIR_NAME, 'node_modules']);
78
+
79
+ /**
80
+ * Audit a directory the engine may read: no answer-key file names, no symlinks
81
+ * (which could point outside it), no protected term or advisory id in content.
82
+ * Bounded so a hostile tree cannot make the audit itself the problem.
83
+ */
84
+ export function auditWorkspace(dir, terms, { strictAdvisoryIds = true, maxFiles = 20000, maxFileBytes = 2 * 1024 * 1024 } = {}) {
85
+ const leaks = [];
86
+ let scanned = 0; let skippedLarge = 0; let truncated = false;
87
+ const walk = (d, rel) => {
88
+ let entries = [];
89
+ try { entries = fs.readdirSync(d, { withFileTypes: true }); } catch { return; }
90
+ for (const e of entries.sort((a, b) => a.name.localeCompare(b.name))) {
91
+ if (truncated) return;
92
+ const r = rel ? `${rel}/${e.name}` : e.name;
93
+ if (e.isSymbolicLink()) { leaks.push({ kind: 'symlink', file: r, detail: 'symlinks are refused in an engine workspace' }); continue; }
94
+ if (e.isDirectory()) { if (!SKIP_DIRS.has(e.name)) walk(path.join(d, e.name), r); continue; }
95
+ if (!e.isFile()) continue;
96
+ if (++scanned > maxFiles) { truncated = true; leaks.push({ kind: 'audit-truncated', file: r, detail: `more than ${maxFiles} files; the audit cannot vouch for the rest` }); return; }
97
+ if (LEAK_FILENAME_PATTERNS.some((re) => re.test(r))) { leaks.push({ kind: 'answer-key-file', file: r, detail: 'file name looks like a label, answer key or advisory record' }); continue; }
98
+ let st; try { st = fs.statSync(path.join(d, e.name)); } catch { continue; }
99
+ if (st.size > maxFileBytes) { skippedLarge++; continue; }
100
+ let body; try { body = fs.readFileSync(path.join(d, e.name), 'utf8'); } catch { continue; }
101
+ for (const l of auditText(body, terms, { strictAdvisoryIds })) leaks.push({ ...l, file: r });
102
+ }
103
+ };
104
+ walk(dir, '');
105
+ return { ok: leaks.length === 0, leaks, scannedFiles: scanned, skippedLargeFiles: skippedLarge };
106
+ }
107
+
108
+ /**
109
+ * Copy a target tree into a fresh workspace for the engine, then audit it. On any
110
+ * leak the workspace is removed and the target is reported quarantined; the run
111
+ * must record that as a failure, never silently scan the tree anyway.
112
+ */
113
+ export function stageWorkspace({ srcDir, destDir, terms, strictAdvisoryIds = true }) {
114
+ fs.rmSync(destDir, { recursive: true, force: true });
115
+ fs.cpSync(srcDir, destDir, {
116
+ recursive: true, dereference: false,
117
+ filter: (s) => !SKIP_DIRS.has(path.basename(s)) || s === srcDir,
118
+ });
119
+ const audit = auditWorkspace(destDir, terms, { strictAdvisoryIds });
120
+ if (!audit.ok) {
121
+ fs.rmSync(destDir, { recursive: true, force: true });
122
+ return { ok: false, quarantined: true, leaks: audit.leaks, destDir: null };
123
+ }
124
+ return { ok: true, quarantined: false, leaks: [], destDir };
125
+ }
126
+
127
+ // ---------------------------------------------------------------- the custodian's store
128
+
129
+ /** Content hash binding a label set: key-order independent, order independent. */
130
+ export function sealLabels(defects, negatives) {
131
+ const sortById = (xs) => [...xs].sort((a, b) => String(a.id).localeCompare(String(b.id)));
132
+ return digestOf({ kind: 'evaluation-labels', defects: sortById(defects || []), negatives: sortById(negatives || []) });
133
+ }
134
+
135
+ const isCustodian = (actor) => actor?.domain === DOMAINS.VERIFIER && actor?.role === 'custodian' && !isUntrusted(actor.domain);
136
+
137
+ function denied(code, message) { return { ok: false, errors: [{ code, path: '', message }] }; }
138
+
139
+ /**
140
+ * Write the label set. Only the custodian may; a worker or the target is denied
141
+ * by the trust-domain policy. Every label is validated first: one invalid label
142
+ * rejects the whole write.
143
+ */
144
+ export function custodianWriteLabels(labelDir, actor, { defects = [], negatives = [] } = {}) {
145
+ if (!isCustodian(actor)) return denied('CUSTODY_DENIED', `domain '${actor?.domain}' role '${actor?.role}' may not write sealed labels`);
146
+ const errors = [];
147
+ for (const d of defects) { const v = validateDefectLabel(d); if (!v.ok) errors.push(...v.errors.map((e) => ({ ...e, path: `${d?.id}.${e.path}` }))); }
148
+ for (const n of negatives) { const v = validateNegativeLabel(n); if (!v.ok) errors.push(...v.errors.map((e) => ({ ...e, path: `${n?.id}.${e.path}` }))); }
149
+ if (errors.length) return { ok: false, errors };
150
+ fs.mkdirSync(labelDir, { recursive: true, mode: 0o700 });
151
+ try { fs.chmodSync(labelDir, 0o700); } catch { /* best effort */ }
152
+ const file = path.join(labelDir, LABEL_FILE);
153
+ fs.writeFileSync(file, JSON.stringify({ defects, negatives }), { mode: 0o600 });
154
+ return { ok: true, errors: [], labelsHash: sealLabels(defects, negatives), count: { defects: defects.length, negatives: negatives.length } };
155
+ }
156
+
157
+ /** Read the label set. Allowed only where the policy grants sealed-label read, and only if it still matches the sealed hash. */
158
+ export function custodianReadLabels(labelDir, actor, { expectedHash } = {}) {
159
+ if (!mayDo(actor?.domain, RESOURCES.SEALED_LABELS, 'read')) return denied('CUSTODY_DENIED', `domain '${actor?.domain}' may not read sealed labels`);
160
+ let data;
161
+ try { data = JSON.parse(fs.readFileSync(path.join(labelDir, LABEL_FILE), 'utf8')); } catch (e) { return denied('LABELS_UNREADABLE', `cannot read labels: ${e.code || e.message}`); }
162
+ const defects = Array.isArray(data?.defects) ? data.defects : []; const negatives = Array.isArray(data?.negatives) ? data.negatives : [];
163
+ const labelsHash = sealLabels(defects, negatives);
164
+ if (expectedHash && labelsHash !== expectedHash) return denied('LABELS_HASH_MISMATCH', 'labels changed after they were sealed');
165
+ return { ok: true, errors: [], defects, negatives, labelsHash };
166
+ }
167
+
168
+ // realpath of the nearest existing ancestor plus the not-yet-existing remainder, so a path that does not exist
169
+ // yet still compares correctly against one reached through a symlinked parent (macOS /var -> /private/var).
170
+ function real(p) {
171
+ let cur = path.resolve(p); const rest = [];
172
+ for (;;) {
173
+ try { return path.join(fs.realpathSync(cur), ...rest.reverse()); } catch { /* climb */ }
174
+ const parent = path.dirname(cur);
175
+ if (parent === cur) return path.resolve(p);
176
+ rest.push(path.basename(cur)); cur = parent;
177
+ }
178
+ }
179
+
180
+ /** A workspace that contains, or sits inside, the label directory would hand the engine the answer key by construction. */
181
+ export function assertCustodyIsolation(labelDir, workspaceDir) {
182
+ const a = real(labelDir); const b = real(workspaceDir);
183
+ if (a === b || a.startsWith(b + path.sep) || b.startsWith(a + path.sep)) {
184
+ return denied('CUSTODY_OVERLAP', `workspace ${workspaceDir} overlaps the label directory ${labelDir}`);
185
+ }
186
+ return { ok: true, errors: [] };
187
+ }
188
+
189
+ /** Environment for the engine under test: secret-looking variables, evaluation variables and anything naming the label directory are removed. */
190
+ export function engineEnvironment(base, { labelDir } = {}) {
191
+ const scrubbed = scrubEnv(base || {});
192
+ const out = {};
193
+ const labelReal = labelDir ? real(labelDir) : null;
194
+ for (const [k, v] of Object.entries(scrubbed)) {
195
+ if (/^AGENTIC_(SECURITY_)?EVAL/i.test(k)) continue;
196
+ if (labelDir && typeof v === 'string' && (v.includes(labelDir) || v.includes(labelReal))) continue;
197
+ out[k] = v;
198
+ }
199
+ return out;
200
+ }