@clear-capabilities/agentic-security-scanner 0.161.0 → 0.163.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/bin/agentic-security.js +98 -2
- package/dist/1340.index.js +1871 -0
- package/dist/3477.index.js +727 -0
- package/dist/3499.index.js +23 -3
- package/dist/4113.index.js +59 -8
- package/dist/5014.index.js +833 -0
- package/dist/5756.index.js +2 -2
- package/dist/6257.index.js +1 -1
- package/dist/6907.index.js +248 -0
- package/dist/6994.index.js +1 -1
- package/dist/7033.index.js +33 -5
- package/dist/7439.index.js +1822 -0
- package/dist/7838.index.js +38 -2
- package/dist/8798.index.js +1 -1
- package/dist/9207.index.js +5 -0
- package/dist/9560.index.js +2336 -27
- package/dist/957.index.js +1 -1
- package/dist/{1905.index.js → 9999.index.js} +306 -13
- package/dist/agentic-security.mjs +12 -12
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +39 -7
- package/src/capabilities/CLAUDE.md +114 -0
- package/src/capabilities/address.js +73 -0
- package/src/capabilities/attack-coverage.js +93 -0
- package/src/capabilities/decide.js +256 -0
- package/src/capabilities/delegate.js +69 -0
- package/src/capabilities/egress.js +20 -0
- package/src/capabilities/hook-advice.js +50 -0
- package/src/capabilities/index.js +32 -0
- package/src/capabilities/manifest.js +344 -0
- package/src/capabilities/outbound.js +189 -0
- package/src/capabilities/paths.js +57 -0
- package/src/capabilities/probes.js +211 -0
- package/src/capabilities/proxy.js +182 -0
- package/src/capabilities/reasons.js +67 -0
- package/src/capabilities/receipts.js +356 -0
- package/src/capabilities/records.js +58 -0
- package/src/capabilities/recovery.js +367 -0
- package/src/capabilities/report.js +81 -0
- package/src/capabilities/runner.js +236 -0
- package/src/capabilities/secrets.js +26 -0
- package/src/capabilities/tool-gate.js +119 -0
- package/src/capabilities/tool-registry.js +60 -0
- package/src/dataflow/CLAUDE.md +29 -0
- package/src/dataflow/catalog.js +21 -0
- package/src/dataflow/engine.js +20 -0
- package/src/dataflow/guard-dominance.js +352 -0
- package/src/discovery/CLAUDE.md +18 -0
- package/src/discovery/index.js +43 -1
- package/src/discovery/lenses.js +18 -0
- package/src/discovery/memory.js +7 -5
- package/src/engine.js +80 -15
- package/src/fix/apply-fix-service.js +33 -3
- package/src/integrations/index.js +5 -2
- package/src/ir/CLAUDE.md +14 -0
- package/src/ir/implicit-handler-params.js +74 -0
- package/src/ir/parser-cs.js +20 -0
- package/src/ir/parser-go.js +30 -13
- package/src/ir/parser-java.js +11 -0
- package/src/language/nix-eval-isolation.js +33 -5
- package/src/lineage/CLAUDE.md +31 -0
- package/src/lineage/deployment/adapter-cloud.js +202 -0
- package/src/lineage/deployment/adapter-compose.js +90 -0
- package/src/lineage/deployment/adapter-kit.js +142 -0
- package/src/lineage/deployment/adapter-kubernetes.js +247 -0
- package/src/lineage/deployment/attack-paths.js +443 -0
- package/src/lineage/deployment/boundaries-run.js +223 -0
- package/src/lineage/deployment/boundary-graph.js +567 -0
- package/src/lineage/deployment/cross-repo.js +253 -0
- package/src/lineage/deployment/drift.js +383 -0
- package/src/lineage/deployment/ingest.js +221 -0
- package/src/lineage/deployment/projection.js +244 -0
- package/src/lineage/deployment/trace-correlation.js +541 -0
- package/src/lineage/source-registry.js +19 -0
- package/src/mcp/CLAUDE.md +14 -3
- package/src/mcp/invariant-tools.js +64 -0
- package/src/mcp/portfolio-tools.js +79 -0
- package/src/mcp/redact.js +15 -0
- package/src/mcp/server.js +24 -3
- package/src/mcp/tools.js +30 -5
- package/src/posture/CLAUDE.md +505 -0
- package/src/posture/accuracy-scorecard.js +95 -4
- package/src/posture/assurance/baseline-inventory.js +218 -0
- package/src/posture/assurance/baseline.js +322 -0
- package/src/posture/assurance/bounded-io.js +145 -0
- package/src/posture/assurance/config.js +299 -0
- package/src/posture/assurance/contracts.js +315 -0
- package/src/posture/assurance/identity.js +62 -0
- package/src/posture/assurance/migrations.js +259 -0
- package/src/posture/assurance/rollback.js +214 -0
- package/src/posture/assurance/rollout.js +200 -0
- package/src/posture/assurance/schema-kit.js +111 -0
- package/src/posture/assurance/verification-record.js +258 -0
- package/src/posture/attestation.js +5 -0
- package/src/posture/autopilot.js +28 -1
- package/src/posture/business-logic.js +26 -4
- package/src/posture/evaluation/bakeoff.js +148 -0
- package/src/posture/evaluation/custody.js +200 -0
- package/src/posture/evaluation/deployment-ablation.js +416 -0
- package/src/posture/evaluation/dev-cases.js +65 -0
- package/src/posture/evaluation/economics.js +85 -0
- package/src/posture/evaluation/gates.js +161 -0
- package/src/posture/evaluation/grouping.js +101 -0
- package/src/posture/evaluation/interval.js +132 -0
- package/src/posture/evaluation/invariant-ablation.js +325 -0
- package/src/posture/evaluation/labels.js +200 -0
- package/src/posture/evaluation/protocol.js +308 -0
- package/src/posture/evaluation/release-gate.js +209 -0
- package/src/posture/evaluation/report.js +158 -0
- package/src/posture/evaluation/runner.js +273 -0
- package/src/posture/evaluation/scan-child.js +80 -0
- package/src/posture/evaluation/score.js +174 -0
- package/src/posture/evaluation/synthetic.js +90 -0
- package/src/posture/evaluation/why-missed.js +219 -0
- package/src/posture/evidence-bundle.js +93 -7
- package/src/posture/execution-proof.js +11 -1
- package/src/posture/fix-verify-loop.js +20 -6
- package/src/posture/fix-verify.js +37 -2
- package/src/posture/fleet.js +11 -2
- package/src/posture/invariants/cli.js +95 -0
- package/src/posture/invariants/coverage.js +164 -0
- package/src/posture/invariants/export.js +178 -0
- package/src/posture/invariants/expressions.js +46 -0
- package/src/posture/invariants/lifecycle.js +289 -0
- package/src/posture/invariants/project-input.js +55 -0
- package/src/posture/invariants/repair.js +340 -0
- package/src/posture/invariants/run.js +45 -0
- package/src/posture/invariants/scenarios.js +311 -0
- package/src/posture/invariants/schema.js +167 -0
- package/src/posture/invariants/shrink.js +407 -0
- package/src/posture/invariants/state-assertions.js +165 -0
- package/src/posture/logic-claims.js +12 -0
- package/src/posture/model-routing.js +40 -0
- package/src/posture/oracles/adapters.js +531 -0
- package/src/posture/oracles/conformance.js +276 -0
- package/src/posture/oracles/oracle.js +453 -0
- package/src/posture/oracles/registry.js +76 -0
- package/src/posture/oracles/scenario-classes.js +223 -0
- package/src/posture/portfolio/backend.js +279 -0
- package/src/posture/portfolio/bundle.js +314 -0
- package/src/posture/portfolio/cli.js +142 -0
- package/src/posture/portfolio/manifest.js +263 -0
- package/src/posture/portfolio/progress.js +267 -0
- package/src/posture/portfolio/resume.js +130 -0
- package/src/posture/portfolio/retention.js +207 -0
- package/src/posture/portfolio/scheduler.js +488 -0
- package/src/posture/portfolio/signing.js +155 -0
- package/src/posture/portfolio/wording.js +59 -0
- package/src/posture/portfolio/work-units.js +405 -0
- package/src/posture/replay/replay.js +250 -0
- package/src/posture/routing/adapter.js +140 -0
- package/src/posture/routing/calibration.js +278 -0
- package/src/posture/routing/canary.js +141 -0
- package/src/posture/routing/control.js +35 -0
- package/src/posture/routing/decide.js +211 -0
- package/src/posture/routing/drift.js +137 -0
- package/src/posture/routing/economics.js +185 -0
- package/src/posture/routing/feedback.js +207 -0
- package/src/posture/routing/index.js +31 -0
- package/src/posture/routing/outcomes.js +281 -0
- package/src/posture/routing/promotion.js +291 -0
- package/src/posture/routing/receipts.js +59 -0
- package/src/posture/routing/report.js +142 -0
- package/src/posture/routing/shadow.js +53 -0
- package/src/posture/specification-mining.js +26 -0
- package/src/posture/verification/advisory-state.js +71 -0
- package/src/posture/verification/emit.js +257 -0
- package/src/posture/verification/hypothesis-promotion.js +211 -0
- package/src/posture/verification/patch-negative.js +226 -0
- package/src/posture/verification/patch-promotion.js +113 -0
- package/src/posture/verification/projection.js +183 -0
- package/src/posture/verification/repair-records.js +124 -0
- package/src/posture/verifier.js +16 -0
- package/src/report/index.js +100 -1
- package/src/sandbox/CLAUDE.md +51 -0
- package/src/sandbox/backend-namespace.js +7 -0
- package/src/sandbox/backend-userspace.js +151 -34
- package/src/sandbox/control-probes.js +215 -0
- package/src/sandbox/supervise.js +201 -0
- package/src/sandbox/trust-boundary.js +113 -0
- package/src/sandbox/trust-domains.js +171 -0
|
@@ -23,6 +23,7 @@ import { spawnSync } from 'node:child_process';
|
|
|
23
23
|
import * as fs from 'node:fs';
|
|
24
24
|
import * as path from 'node:path';
|
|
25
25
|
import { verifyFix } from './fix-verify.js';
|
|
26
|
+
import { emitVerification } from './verification/emit.js';
|
|
26
27
|
|
|
27
28
|
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
28
29
|
|
|
@@ -131,20 +132,32 @@ export async function verifyFixWithTests({
|
|
|
131
132
|
runTests = true,
|
|
132
133
|
testRunnerOverride,
|
|
133
134
|
testTimeoutMs,
|
|
135
|
+
patchNegative,
|
|
136
|
+
assuranceConfig,
|
|
134
137
|
} = {}) {
|
|
135
|
-
const scanLint = await verifyFix({ scanRoot, originalFindingStableId, files, depFileContents });
|
|
138
|
+
const scanLint = await verifyFix({ scanRoot, originalFindingStableId, files, depFileContents, patchNegative, assuranceConfig });
|
|
136
139
|
const legs = {
|
|
137
140
|
scan: { ok: scanLint.rescan?.ok ?? scanLint.ok, detail: scanLint.rescan ?? scanLint },
|
|
138
141
|
lint: { ok: scanLint.lint?.ok ?? true, detail: scanLint.lint ?? null },
|
|
139
142
|
tests: { ok: true, detail: null, skipped: true, reason: 'not-run' },
|
|
140
143
|
};
|
|
141
|
-
|
|
142
|
-
|
|
144
|
+
// X-201: the closed loop's own record (additive). It runs the detector, linter and project tests, never an exploit
|
|
145
|
+
// oracle, so it can say `not-run` or `error` and nothing stronger.
|
|
146
|
+
const withRecord = (res) => {
|
|
147
|
+
const e = emitVerification('fix-verify-loop', res, { originalFindingStableId, files, scanRoot });
|
|
148
|
+
return { ...res, verificationRecord: e.ok ? e.record : null };
|
|
149
|
+
};
|
|
150
|
+
// X-204: the patch-negative leg is present only when it was requested and its feature is on. A patch the oracle did not verify
|
|
151
|
+
// is never `verified-clean`, whatever the other legs say.
|
|
152
|
+
const pn = scanLint.patchNegative && scanLint.patchNegative.status !== 'disabled' ? scanLint.patchNegative : null;
|
|
153
|
+
if (pn) legs.patchNegative = { ok: pn.verifiedFix === true, detail: { status: pn.status, incompleteStep: pn.incompleteStep, failureCode: pn.failureCode, reason: pn.reason } };
|
|
154
|
+
if (!legs.scan.ok || !legs.lint.ok || (pn && !legs.patchNegative.ok)) {
|
|
155
|
+
return withRecord({
|
|
143
156
|
ok: false,
|
|
144
157
|
verdict: 'verification-failed',
|
|
145
158
|
legs,
|
|
146
159
|
summary: _summarize(legs, 'verification-failed'),
|
|
147
|
-
};
|
|
160
|
+
});
|
|
148
161
|
}
|
|
149
162
|
if (runTests) {
|
|
150
163
|
const tests = runProjectTests(scanRoot, { runnerOverride: testRunnerOverride, timeoutMs: testTimeoutMs });
|
|
@@ -153,8 +166,8 @@ export async function verifyFixWithTests({
|
|
|
153
166
|
const allOk = legs.scan.ok && legs.lint.ok && legs.tests.ok;
|
|
154
167
|
const verdict = !allOk
|
|
155
168
|
? 'verification-failed'
|
|
156
|
-
: (legs.tests.skipped ? 'untested-but-passes' : 'verified-clean');
|
|
157
|
-
return { ok: allOk, verdict, legs, summary: _summarize(legs, verdict) };
|
|
169
|
+
: pn ? 'verified-fix' : (legs.tests.skipped ? 'untested-but-passes' : 'verified-clean');
|
|
170
|
+
return withRecord({ ok: allOk, verdict, legs, summary: _summarize(legs, verdict) });
|
|
158
171
|
}
|
|
159
172
|
|
|
160
173
|
function _summarize(legs, verdict) {
|
|
@@ -162,5 +175,6 @@ function _summarize(legs, verdict) {
|
|
|
162
175
|
bits.push(`scan: ${legs.scan.ok ? 'pass' : 'fail'}`);
|
|
163
176
|
bits.push(`lint: ${legs.lint.skipped ? 'skip' : legs.lint.ok ? 'pass' : 'fail'}`);
|
|
164
177
|
bits.push(`tests: ${legs.tests.skipped ? 'skip' : legs.tests.ok ? 'pass' : 'fail'}`);
|
|
178
|
+
if (legs.patchNegative) bits.push(`patch-negative: ${legs.patchNegative.ok ? 'pass' : `fail (${legs.patchNegative.detail.incompleteStep})`}`);
|
|
165
179
|
return `${verdict} (${bits.join(' · ')})`;
|
|
166
180
|
}
|
|
@@ -22,6 +22,7 @@ import { runFullScan } from '../engine.js';
|
|
|
22
22
|
import { gateFixOutput } from './fix-honesty-gate.js';
|
|
23
23
|
import { runProjectTests } from './test-runner.js';
|
|
24
24
|
import { recordFixAttempt } from './fix-metrics.js';
|
|
25
|
+
import { emitVerification, headCommit } from './verification/emit.js';
|
|
25
26
|
|
|
26
27
|
const SEVERITY_RANK = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
|
|
27
28
|
|
|
@@ -187,6 +188,8 @@ export async function verifyFix({
|
|
|
187
188
|
testTimeoutMs,
|
|
188
189
|
recordMetrics = true,
|
|
189
190
|
poc,
|
|
191
|
+
patchNegative,
|
|
192
|
+
assuranceConfig,
|
|
190
193
|
} = {}) {
|
|
191
194
|
// R5 (reporting half) — time each stage as it runs. Measured here rather
|
|
192
195
|
// than inside each stage because only this function knows the boundaries of
|
|
@@ -269,7 +272,26 @@ export async function verifyFix({
|
|
|
269
272
|
}
|
|
270
273
|
_lap('honesty');
|
|
271
274
|
|
|
272
|
-
|
|
275
|
+
// X-204: patch-negative verification, only when the caller asked for it AND the `patch-negative-verification` feature is
|
|
276
|
+
// on. `patchNegative` is the requester's exploit scenario and functional cases ({ original, functional, commit?, ... }); the
|
|
277
|
+
// candidate `files` are the proposed diff. With the flag off, or no request, nothing here runs and every field below is
|
|
278
|
+
// exactly what it was before.
|
|
279
|
+
let pn = null;
|
|
280
|
+
if (patchNegative && typeof patchNegative === 'object') {
|
|
281
|
+
try {
|
|
282
|
+
const { verifyPatchNegative } = await import('./verification/patch-negative.js');
|
|
283
|
+
pn = await verifyPatchNegative({
|
|
284
|
+
hypothesisId: originalFindingStableId, ...patchNegative, patch: { files },
|
|
285
|
+
commit: patchNegative.commit ?? headCommit(scanRoot),
|
|
286
|
+
}, { config: assuranceConfig });
|
|
287
|
+
} catch (e) {
|
|
288
|
+
pn = { status: 'error', verifiedFix: false, incompleteStep: 'patch-negative', failureCode: 'harness-error', reason: `patch-negative verification failed to run: ${String(e?.message || e).slice(0, 200)}`, summary: '' };
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
const pnActive = pn !== null && pn.status !== 'disabled';
|
|
292
|
+
_lap('patchNegative');
|
|
293
|
+
|
|
294
|
+
const ok = rescan.ok && (lint.ok || lint.skipped) && testsOk && pocOk && (honesty ? honesty.ok : true) && (pnActive ? pn.verifiedFix === true : true);
|
|
273
295
|
|
|
274
296
|
// FR-305 (assurance-hardening PRD): `ok` alone conflates "every leg
|
|
275
297
|
// genuinely ran and passed" with "passed, but a required leg was skipped
|
|
@@ -311,6 +333,7 @@ export async function verifyFix({
|
|
|
311
333
|
: pocLeg.status === 'no-longer-proven' ? 'poc: PASS (ran against the patch and no longer demonstrates the vulnerability)'
|
|
312
334
|
: `poc: inconclusive — not counted either way (${pocLeg.reason || 'no detail reported'})`,
|
|
313
335
|
// FR-305: never let a degraded pass read the same as a full one.
|
|
336
|
+
pnActive ? `patch-negative: ${pn.verifiedFix ? 'PASS (verified-fix)' : `FAIL - not verified-fix, incomplete step '${pn.incompleteStep}' (${pn.failureCode}): ${pn.reason}`}` : null,
|
|
314
337
|
ok && !verifiedFull ? `NOTE: PASSED, but NOT fully verified — ${degradedLegs.join('; ')}` : null,
|
|
315
338
|
].filter(Boolean).join('\n');
|
|
316
339
|
// Persist the attempt so the distribution can be reported from real runs.
|
|
@@ -337,5 +360,17 @@ export async function verifyFix({
|
|
|
337
360
|
});
|
|
338
361
|
}
|
|
339
362
|
|
|
340
|
-
|
|
363
|
+
// X-201: the one verification record, ADDITIVE. `ok` and every field above stay exactly as they were; the record
|
|
364
|
+
// says what was actually established (a static re-scan is not an exploit oracle, so without a PoC leg it is
|
|
365
|
+
// `not-run`, and nothing here can reach `confirmed`/`refuted`: that takes a trusted oracle adapter).
|
|
366
|
+
const emitted = emitVerification('fix-verify', { rescan, lint, tests, poc: pocLeg }, {
|
|
367
|
+
originalFindingStableId, files, scanRoot, finding: poc?.finding,
|
|
368
|
+
});
|
|
369
|
+
const verificationRecord = emitted.ok ? emitted.record : null;
|
|
370
|
+
|
|
371
|
+
return {
|
|
372
|
+
ok, verifiedFull, degradedLegs, rescan, lint, tests, testedPrePatch: _testedPrePatch, honesty, poc: pocLeg, durations, summary, verificationRecord,
|
|
373
|
+
// additive, present only when patch-negative verification was requested (X-204)
|
|
374
|
+
...(pn ? { patchNegative: pn, ...(pnActive ? { fixStatus: pn.verifiedFix ? 'verified-fix' : 'not-verified-fix' } : {}) } : {}),
|
|
375
|
+
};
|
|
341
376
|
}
|
package/src/posture/fleet.js
CHANGED
|
@@ -167,6 +167,10 @@ function _provenanceDebtForRepo(findings, now = Date.now()) {
|
|
|
167
167
|
export async function runFleet({
|
|
168
168
|
repos = [], concurrency = 4, stateFile, runScan,
|
|
169
169
|
resume = true, previous = null, onProgress = () => {},
|
|
170
|
+
// X-704: repositories whose every durable portfolio work unit is verified and current
|
|
171
|
+
// (portfolio/work-units.js `fullyVerifiedRepositories`). They are skipped like a resumed completed repo,
|
|
172
|
+
// and, as with a resumed repo, the skip is reported in `skipped` rather than counted as scanned.
|
|
173
|
+
portfolioVerified = [],
|
|
170
174
|
} = {}) {
|
|
171
175
|
if (typeof runScan !== 'function') {
|
|
172
176
|
return { ok: false, reason: 'no runScan supplied' };
|
|
@@ -176,7 +180,7 @@ export async function runFleet({
|
|
|
176
180
|
const skipped = [];
|
|
177
181
|
|
|
178
182
|
const queue = repos.filter((r) => {
|
|
179
|
-
if (resume && state.completed[r]) { skipped.push(r); return false; }
|
|
183
|
+
if ((resume && state.completed[r]) || portfolioVerified.includes(r)) { skipped.push(r); return false; }
|
|
180
184
|
return true;
|
|
181
185
|
});
|
|
182
186
|
|
|
@@ -408,8 +412,13 @@ export function renderFleetSummary(rollup) {
|
|
|
408
412
|
if (prov.reposWithNoProvenDebt.length) bits.push(`${prov.reposWithNoProvenDebt.length} repo(s) with no complete-status provenance`);
|
|
409
413
|
provClause = ` PROVENANCE: ${bits.join('; ')}.`;
|
|
410
414
|
}
|
|
415
|
+
// X-707: portfolio progress is its own clause, present only when the `portfolio-assurance` feature attached it
|
|
416
|
+
// (portfolio/progress.js `attachPortfolioProgress`), so flag-off output is byte-identical. It states verified units and
|
|
417
|
+
// repositories separately from the scan counts above: a finished controller is not a passing portfolio.
|
|
418
|
+
const pp = rollup.portfolioProgress;
|
|
419
|
+
const portfolioClause = pp ? ` PORTFOLIO: ${pp.units.verified}/${pp.units.total} unit(s) verified, ${pp.coverage.fullyVerified}/${pp.coverage.total} repo(s) fully verified, ${pp.review.pending} pending human review; ${pp.completion.statement}` : '';
|
|
411
420
|
return `${head}${rollup.scanned} repo(s) scanned, ${rollup.total} finding(s)`
|
|
412
|
-
+ `${sev ? ` (${sev})` : ''}, ${rollup.proven} execution-proven.${nf}${govClause}${provClause}`;
|
|
421
|
+
+ `${sev ? ` (${sev})` : ''}, ${rollup.proven} execution-proven.${nf}${govClause}${provClause}${portfolioClause}`;
|
|
413
422
|
}
|
|
414
423
|
|
|
415
424
|
/**
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
// `agentic-security invariants <export|coverage|regress>` (X-405 to X-407 CLI surface).
|
|
2
|
+
//
|
|
3
|
+
// invariants export --invariant <file> --fixture <dir> [--ledger <file>] [--commit <sha>] [--seed <n>] [--output <file>] [--json]
|
|
4
|
+
// The reproducible scenario package for one contract (see export.js). Builds nothing and runs nothing when the
|
|
5
|
+
// `invariant-scenarios` feature is off. Exit 0 ok, 1 refused (disabled, rejected, unsupported, blocked), 2 usage.
|
|
6
|
+
// invariants coverage --invariant <file> | --invariants-dir <dir> [--ledger <file>] [--results <file>] [--json]
|
|
7
|
+
// The bounded business-coverage report (see coverage.js). `--results` is a JSON array of `{ invariantId, result }` where
|
|
8
|
+
// `result` is what `verifyInvariant` returned; without it the report is the static inventory with every contract's
|
|
9
|
+
// transitions listed as untested, which is the honest answer when nothing was run. Exit 0, 1 refused, 2 usage.
|
|
10
|
+
// invariants regress <artifact.json>
|
|
11
|
+
// Re-execute a regression artifact from the file alone (see repair.js). Exit 0 every leg settled as recorded, 1 a leg did
|
|
12
|
+
// not (or the artifact is invalid), 3 not run (a prerequisite is unmet or the feature is off): not-run is not a pass.
|
|
13
|
+
//
|
|
14
|
+
// The logic lives here, not in bin/, so the same functions are exercised by tests without spawning the CLI; bin/ only
|
|
15
|
+
// dispatches. Paths are the operator's own and are resolved against the working directory (the MCP tool, which takes paths from
|
|
16
|
+
// an agent, confines them to its session root instead).
|
|
17
|
+
import * as fs from 'node:fs';
|
|
18
|
+
import * as path from 'node:path';
|
|
19
|
+
import { resolveAssuranceConfig } from '../assurance/config.js';
|
|
20
|
+
import { exportFromFiles } from './export.js';
|
|
21
|
+
import { businessCoverage } from './coverage.js';
|
|
22
|
+
import { runRegressionArtifact } from './repair.js';
|
|
23
|
+
import { readJsonFile } from './project-input.js';
|
|
24
|
+
|
|
25
|
+
const USAGE = [
|
|
26
|
+
'Usage: agentic-security invariants export --invariant <file> --fixture <dir> [--ledger <file>] [--commit <sha>] [--seed <n>] [--output <file>] [--json]',
|
|
27
|
+
' agentic-security invariants coverage --invariant <file> | --invariants-dir <dir> [--ledger <file>] [--results <file>] [--json]',
|
|
28
|
+
' agentic-security invariants regress <artifact.json>',
|
|
29
|
+
].join('\n');
|
|
30
|
+
|
|
31
|
+
const abs = (cwd, p) => path.resolve(cwd, String(p));
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {{ _: string[], flags: object }} args parsed arguments; `_[0]` is `invariants`, `_[1]` the sub-command
|
|
35
|
+
* @param {{ cwd?: string, out?: (s: string) => void, err?: (s: string) => void, env?: object }} [io]
|
|
36
|
+
* @returns {Promise<number>} the exit code
|
|
37
|
+
*/
|
|
38
|
+
export async function runInvariantsCommand(args, io = {}) {
|
|
39
|
+
const cwd = io.cwd || process.cwd();
|
|
40
|
+
const out = io.out || ((s) => process.stdout.write(s));
|
|
41
|
+
const err = io.err || ((s) => process.stderr.write(s));
|
|
42
|
+
const sub = args._[1];
|
|
43
|
+
const flags = args.flags || {};
|
|
44
|
+
const config = resolveAssuranceConfig({ scanRoot: cwd, env: io.env || process.env });
|
|
45
|
+
const emit = (value, lines) => out(flags.json ? `${JSON.stringify(value, null, 2)}\n` : `${lines.join('\n')}\n`);
|
|
46
|
+
|
|
47
|
+
if (sub === 'export') {
|
|
48
|
+
if (typeof flags.invariant !== 'string' || typeof flags.fixture !== 'string') { err(`${USAGE}\n`); return 2; }
|
|
49
|
+
const r = exportFromFiles({
|
|
50
|
+
invariantPath: abs(cwd, flags.invariant), fixturePath: abs(cwd, flags.fixture), ledgerPath: typeof flags.ledger === 'string' ? abs(cwd, flags.ledger) : undefined,
|
|
51
|
+
commit: typeof flags.commit === 'string' ? flags.commit : undefined, seed: flags.seed !== undefined ? Number(flags.seed) : undefined, config,
|
|
52
|
+
});
|
|
53
|
+
if (r.status !== 'ok') { err(`agentic-security invariants export: ${r.status}: ${r.reason ?? ''}\n`); return 1; }
|
|
54
|
+
const text = `${JSON.stringify(r.export, null, 2)}\n`;
|
|
55
|
+
if (typeof flags.output === 'string') {
|
|
56
|
+
fs.writeFileSync(abs(cwd, flags.output), text);
|
|
57
|
+
out(`wrote ${r.export.scenarios.length} scenario(s) to ${flags.output}${r.withheld.length ? `; ${r.withheld.length} withheld (secret-looking content)` : ''}\n`);
|
|
58
|
+
} else out(text);
|
|
59
|
+
return 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (sub === 'coverage') {
|
|
63
|
+
let files = [];
|
|
64
|
+
if (typeof flags.invariant === 'string') files = [abs(cwd, flags.invariant)];
|
|
65
|
+
else if (typeof flags['invariants-dir'] === 'string') {
|
|
66
|
+
try { files = fs.readdirSync(abs(cwd, flags['invariants-dir'])).filter((f) => f.endsWith('.json')).sort().map((f) => path.join(abs(cwd, flags['invariants-dir']), f)); } catch { err('agentic-security invariants coverage: the invariants directory is unreadable\n'); return 1; }
|
|
67
|
+
} else { err(`${USAGE}\n`); return 2; }
|
|
68
|
+
const docs = [];
|
|
69
|
+
for (const f of files) {
|
|
70
|
+
const j = readJsonFile(f);
|
|
71
|
+
if (!j.ok) { err(`agentic-security invariants coverage: ${path.basename(f)}: ${j.reason}\n`); return 1; }
|
|
72
|
+
docs.push(j.value);
|
|
73
|
+
}
|
|
74
|
+
let ledger; let results = [];
|
|
75
|
+
if (typeof flags.ledger === 'string') { const l = readJsonFile(abs(cwd, flags.ledger)); if (!l.ok) { err(`agentic-security invariants coverage: the ledger: ${l.reason}\n`); return 1; } ledger = l.value; }
|
|
76
|
+
if (typeof flags.results === 'string') { const rr = readJsonFile(abs(cwd, flags.results)); if (!rr.ok || !Array.isArray(rr.value)) { err('agentic-security invariants coverage: --results must be a JSON array of { invariantId, result }\n'); return 1; } results = rr.value; }
|
|
77
|
+
const byId = new Map(results.filter((x) => x && typeof x.invariantId === 'string').map((x) => [x.invariantId, x.result]));
|
|
78
|
+
const cov = businessCoverage({ entries: docs.map((invariant) => ({ invariant, result: byId.get(invariant.id) ?? null })), ledger });
|
|
79
|
+
emit(cov, cov.lines);
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (sub === 'regress') {
|
|
84
|
+
const file = args._[2];
|
|
85
|
+
if (!file) { err(`${USAGE}\n`); return 2; }
|
|
86
|
+
const a = readJsonFile(abs(cwd, file));
|
|
87
|
+
if (!a.ok) { err(`agentic-security invariants regress: ${a.reason}\n`); return 1; }
|
|
88
|
+
const r = await runRegressionArtifact(a.value, { config });
|
|
89
|
+
emit(r, [`regression artifact: ${r.status}`, r.summary, ...r.legs.map((l) => ` ${l.matched ? 'ok ' : 'FAIL'} ${l.role}: expected ${l.expected}, got ${l.outcome ?? l.status}`)]);
|
|
90
|
+
return r.status === 'passed' ? 0 : (r.status === 'prerequisite-unmet' || r.status === 'disabled') ? 3 : 1;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
err(`agentic-security invariants: unrecognized sub-command "${sub}"\n${USAGE}\n`);
|
|
94
|
+
return 2;
|
|
95
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// Bounded business-logic coverage (X-407).
|
|
2
|
+
//
|
|
3
|
+
// "We ran scenarios" is not "the business logic is covered". This turns the contracts a project has and the runs it made into a
|
|
4
|
+
// report that says what was exercised AND what was not, and keeps every gap explicit:
|
|
5
|
+
//
|
|
6
|
+
// inventory approved versus proposed (and rejected, superseded, unrecorded) contracts, judged by the VERIFIED approval ledger:
|
|
7
|
+
// a document that merely claims approval is listed as proposed-by-ledger with the claim noted;
|
|
8
|
+
// exercised which actors, states and declared transitions the executed scenarios actually drove, against what each contract
|
|
9
|
+
// declares;
|
|
10
|
+
// gaps untested transitions, actors and states; the sequence depth the scenarios were bounded to; races a contract
|
|
11
|
+
// class cannot model (and the cooperative-scheduling limit where one was run); scenarios that could not be built
|
|
12
|
+
// or did not settle; and absent contracts (a class with no contract at all, or no contracts whatsoever).
|
|
13
|
+
//
|
|
14
|
+
// It never claims exhaustive correctness: the report states, in a field and in the text, that it describes a bounded sample.
|
|
15
|
+
// A run that did not execute contributes nothing to "exercised": an unsupported or inconclusive scenario is a gap, never
|
|
16
|
+
// coverage.
|
|
17
|
+
//
|
|
18
|
+
// Additive output. `invariantCoverageFields` is the one place a surface (the JSON/CLI report) asks for the fields: it returns
|
|
19
|
+
// `{}` unless the `invariant-scenarios` feature is on AND a coverage object was supplied, so a default scan's report is
|
|
20
|
+
// byte-identical to what it was (pinned in `test/fixtures/invariant-compat/`). The key is `invariantCoverage`.
|
|
21
|
+
import { SCHEMA_VERSION, isPlainObject } from '../assurance/schema-kit.js';
|
|
22
|
+
import { featureStatus, resolveAssuranceConfig } from '../assurance/config.js';
|
|
23
|
+
import { INVARIANT_CLASSES } from './schema.js';
|
|
24
|
+
import { verifyLedger } from './lifecycle.js';
|
|
25
|
+
import { FEATURE, HARD_BOUNDS, KINDS_BY_CLASS } from './scenarios.js';
|
|
26
|
+
import { NOT_EXHAUSTIVE } from './export.js';
|
|
27
|
+
|
|
28
|
+
export const COVERAGE_SCHEMA = 'agentic-security/business-coverage';
|
|
29
|
+
export const GAP_CODES = Object.freeze([
|
|
30
|
+
'absent-contract', 'unapproved-contract', 'not-executed', 'untested-transition', 'untested-actor', 'untested-state',
|
|
31
|
+
'bounded-depth', 'unsupported-scenario', 'unsettled-scenario', 'unsupported-race', 'cooperative-scheduling',
|
|
32
|
+
]);
|
|
33
|
+
|
|
34
|
+
const sorted = (set) => [...set].sort();
|
|
35
|
+
|
|
36
|
+
function declaredStates(inv) {
|
|
37
|
+
const wf = inv.forbidden.find((f) => f.op === 'transition-outside');
|
|
38
|
+
if (wf) return sorted(new Set(wf.allowed.flatMap((t) => [t.from, t.to])));
|
|
39
|
+
return sorted(new Set(inv.resources.map((r) => r.id)));
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function exercisedStates(inv, steps) {
|
|
43
|
+
const wf = inv.forbidden.find((f) => f.op === 'transition-outside');
|
|
44
|
+
const keyToId = new Map(inv.resources.map((r) => [r.key, r.id]));
|
|
45
|
+
const out = new Set();
|
|
46
|
+
if (wf) {
|
|
47
|
+
const declared = new Set(declaredStates(inv));
|
|
48
|
+
const reached = steps.filter((s) => s.to && declared.has(s.to));
|
|
49
|
+
if (reached.length) out.add(wf.allowed[0].from); // the seeded initial state is where a workflow starts
|
|
50
|
+
for (const s of reached) out.add(s.to);
|
|
51
|
+
} else {
|
|
52
|
+
for (const s of steps) if (s.key && keyToId.has(s.key)) out.add(keyToId.get(s.key));
|
|
53
|
+
}
|
|
54
|
+
return sorted(out);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* @param {object} p
|
|
59
|
+
* @param {Array<{ invariant: object, result?: object }>} p.entries each contract, with the `verifyInvariant` result of its run (or none)
|
|
60
|
+
* @param {object} [p.ledger] the approval ledger; without a verifying one no contract is approved
|
|
61
|
+
* @param {object} [p.signer] ledger signer (test seam)
|
|
62
|
+
* @returns {object} the coverage report (plain data, deterministic: no clock)
|
|
63
|
+
*/
|
|
64
|
+
export function businessCoverage({ entries, ledger, signer } = {}) {
|
|
65
|
+
const list = (Array.isArray(entries) ? entries : []).filter((e) => isPlainObject(e) && isPlainObject(e.invariant) && Array.isArray(e.invariant.actors));
|
|
66
|
+
const verified = verifyLedger(ledger, { signer });
|
|
67
|
+
const stateOf = (inv) => (verified.ok ? (verified.states[inv.id] ?? 'unrecorded') : 'unrecorded');
|
|
68
|
+
const gaps = [];
|
|
69
|
+
const gap = (code, detail, extra = {}) => gaps.push({ code, detail, ...extra });
|
|
70
|
+
const invariants = [];
|
|
71
|
+
const inventory = { approved: [], proposed: [], rejected: [], superseded: [], unrecorded: [] };
|
|
72
|
+
let maxDepth = 0; let maxRequests = 0; let truncated = 0;
|
|
73
|
+
const totals = { actors: { declared: 0, exercised: 0 }, transitions: { declared: 0, exercised: 0 }, states: { declared: 0, exercised: 0 } };
|
|
74
|
+
|
|
75
|
+
for (const { invariant: inv, result } of list) {
|
|
76
|
+
const state = stateOf(inv);
|
|
77
|
+
const ref = { id: inv.id, key: inv.key, revision: inv.revision, class: inv.class };
|
|
78
|
+
inventory[state === 'approved' ? 'approved' : state === 'proposed' ? 'proposed' : state === 'rejected' ? 'rejected' : state === 'superseded' ? 'superseded' : 'unrecorded'].push(ref);
|
|
79
|
+
const claimsApproval = inv.review?.state === 'approved' && state !== 'approved';
|
|
80
|
+
|
|
81
|
+
const runs = Array.isArray(result?.results) ? result.results : [];
|
|
82
|
+
const executed = runs.filter((r) => r.status === 'completed');
|
|
83
|
+
const decided = executed.filter((r) => r.outcome === 'confirmed' || r.outcome === 'refuted');
|
|
84
|
+
const steps = executed.flatMap((r) => [...(r.exercise?.control || []), ...(r.exercise?.attack || [])]);
|
|
85
|
+
for (const r of executed) {
|
|
86
|
+
maxDepth = Math.max(maxDepth, r.limits?.depth ?? 0); maxRequests = Math.max(maxRequests, r.limits?.requests ?? 0);
|
|
87
|
+
if (r.limits?.truncated) truncated++;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const actorsDeclared = sorted(new Set(inv.actors.map((a) => a.id)));
|
|
91
|
+
const actorsExercised = sorted(new Set(steps.map((s) => s.actor).filter((a) => actorsDeclared.includes(a))));
|
|
92
|
+
const keyOf = new Map(inv.resources.map((r) => [r.id, r.key]));
|
|
93
|
+
const transitionsDeclared = inv.transitions.map((t) => t.id);
|
|
94
|
+
const transitionsExercised = inv.transitions.filter((t) => steps.some((s) => t.actors.includes(s.actor) && s.action === t.action && s.key === keyOf.get(t.resource))).map((t) => t.id);
|
|
95
|
+
const statesDeclared = declaredStates(inv);
|
|
96
|
+
const statesExercised = exercisedStates(inv, steps);
|
|
97
|
+
|
|
98
|
+
const mine = { id: inv.id, key: inv.key };
|
|
99
|
+
if (!executed.length) {
|
|
100
|
+
gap('not-executed', `no scenario for '${inv.key}' was executed${result?.status && result.status !== 'ok' ? ` (${result.status}: ${result.reason ?? 'no reason given'})` : ''}`, mine);
|
|
101
|
+
} else {
|
|
102
|
+
for (const t of transitionsDeclared.filter((x) => !transitionsExercised.includes(x))) gap('untested-transition', `declared transition '${t}' of '${inv.key}' was not exercised by any executed scenario`, { ...mine, transition: t });
|
|
103
|
+
for (const a of actorsDeclared.filter((x) => !actorsExercised.includes(x))) gap('untested-actor', `declared actor '${a}' of '${inv.key}' performed no step in any executed scenario`, { ...mine, actor: a });
|
|
104
|
+
for (const s of statesDeclared.filter((x) => !statesExercised.includes(x))) gap('untested-state', `declared state '${s}' of '${inv.key}' was not reached by any executed scenario`, { ...mine, state: s });
|
|
105
|
+
}
|
|
106
|
+
for (const u of result?.unsupported || []) gap('unsupported-scenario', `${u.kind ?? 'scenario'} for '${inv.key}' could not be built: ${u.reason}`, { ...mine, kind: u.kind ?? null });
|
|
107
|
+
for (const r of runs.filter((x) => x.status !== 'completed' || !['confirmed', 'refuted'].includes(x.outcome))) {
|
|
108
|
+
gap('unsettled-scenario', `${r.kind} for '${inv.key}' did not settle (${r.status === 'completed' ? `outcome '${r.outcome}'` : `status '${r.status}'`})${r.reason ? `: ${String(r.reason).slice(0, 160)}` : ''}`, { ...mine, kind: r.kind });
|
|
109
|
+
}
|
|
110
|
+
// races: a class with no concurrent family cannot be race-tested here; one that has it needs an executed, settled run
|
|
111
|
+
const hasRaceFamily = (KINDS_BY_CLASS[inv.class] || []).includes('concurrent-operations');
|
|
112
|
+
const raceRuns = decided.filter((r) => r.kind === 'concurrent-operations');
|
|
113
|
+
if (!hasRaceFamily) gap('unsupported-race', `no concurrent-operations scenario exists for the '${inv.class}' class, so races against '${inv.key}' are not tested`, mine);
|
|
114
|
+
else if (!raceRuns.length) gap('unsupported-race', `no concurrent-operations scenario for '${inv.key}' settled, so races against it are not tested`, mine);
|
|
115
|
+
else gap('cooperative-scheduling', `races against '${inv.key}' were exercised under cooperative scheduling in one process with a fixed start order; parallel execution and other interleavings are not covered`, mine);
|
|
116
|
+
if (state !== 'approved') gap('unapproved-contract', `'${inv.key}' is ${state}${claimsApproval ? ' (its document claims approval; only the verified ledger counts)' : ''}: a violation of it would be an advisory candidate, not a failure of a requirement`, mine);
|
|
117
|
+
|
|
118
|
+
totals.actors.declared += actorsDeclared.length; totals.actors.exercised += actorsExercised.length;
|
|
119
|
+
totals.transitions.declared += transitionsDeclared.length; totals.transitions.exercised += transitionsExercised.length;
|
|
120
|
+
totals.states.declared += statesDeclared.length; totals.states.exercised += statesExercised.length;
|
|
121
|
+
invariants.push({
|
|
122
|
+
...ref, name: inv.name, state, claimsApproval,
|
|
123
|
+
declared: { actors: actorsDeclared, transitions: transitionsDeclared, states: statesDeclared },
|
|
124
|
+
exercised: { actors: actorsExercised, transitions: transitionsExercised, states: statesExercised },
|
|
125
|
+
scenarios: { total: runs.length, executed: executed.length, decided: decided.length, unsupported: (result?.unsupported || []).length },
|
|
126
|
+
violations: { approved: runs.filter((r) => r.classification?.kind === 'approved-violation').length, candidate: runs.filter((r) => r.classification?.kind === 'candidate-violation').length },
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// contracts that do not exist are gaps too
|
|
131
|
+
if (!list.length) gap('absent-contract', 'no contract was supplied: no business-logic behaviour is covered');
|
|
132
|
+
else for (const cls of INVARIANT_CLASSES) {
|
|
133
|
+
if (!list.some((e) => e.invariant.class === cls)) gap('absent-contract', `no contract exists for the '${cls}' class: that kind of behaviour is not covered at all`, { class: cls });
|
|
134
|
+
}
|
|
135
|
+
const executedAny = invariants.some((i) => i.scenarios.executed > 0);
|
|
136
|
+
if (executedAny) gap('bounded-depth', `scenarios were bounded to at most ${maxDepth} attack step(s) and ${maxRequests} request(s) (hard ceilings: depth ${HARD_BOUNDS.depth}, requests ${HARD_BOUNDS.requests})${truncated ? `; ${truncated} scenario(s) were truncated to fit` : ''}; longer sequences are not explored`, { maxDepth, maxRequests, truncated });
|
|
137
|
+
|
|
138
|
+
const counts = Object.fromEntries(Object.entries(inventory).map(([k, v]) => [k, v.length]));
|
|
139
|
+
const lines = [
|
|
140
|
+
`Business-logic coverage (bounded sample, not exhaustive): ${counts.approved} approved, ${counts.proposed} proposed, ${counts.rejected + counts.superseded + counts.unrecorded} other contract(s)`,
|
|
141
|
+
` Exercised: ${totals.actors.exercised}/${totals.actors.declared} actors, ${totals.states.exercised}/${totals.states.declared} states, ${totals.transitions.exercised}/${totals.transitions.declared} transitions`,
|
|
142
|
+
` Gaps: ${gaps.length}${gaps.length ? ` (${[...new Set(gaps.map((g) => g.code))].sort().join(', ')})` : ''}`,
|
|
143
|
+
` ${NOT_EXHAUSTIVE}`,
|
|
144
|
+
];
|
|
145
|
+
return {
|
|
146
|
+
schema: COVERAGE_SCHEMA, schemaVersion: SCHEMA_VERSION,
|
|
147
|
+
ledger: { verified: verified.ok, note: verified.ok ? 'approval state comes from the verified ledger' : 'no verifying approval ledger: no contract counts as approved' },
|
|
148
|
+
inventory, counts, invariants, totals, gaps,
|
|
149
|
+
bounds: { maxDepth, maxRequests, truncatedScenarios: truncated, hard: HARD_BOUNDS },
|
|
150
|
+
claims: { exhaustive: false, statement: NOT_EXHAUSTIVE },
|
|
151
|
+
lines,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* The fields a report adds for business coverage: `{ invariantCoverage }`, or `{}` when the feature is off or there is nothing to
|
|
157
|
+
* report. ADDITIVE: with the feature off (the default) a report is byte-identical to what it was.
|
|
158
|
+
*/
|
|
159
|
+
export function invariantCoverageFields(coverage, { config } = {}) {
|
|
160
|
+
if (!isPlainObject(coverage) || coverage.schema !== COVERAGE_SCHEMA) return {};
|
|
161
|
+
const cfg = config || resolveAssuranceConfig({ env: process.env });
|
|
162
|
+
if (featureStatus(cfg, FEATURE).status !== 'ok') return {};
|
|
163
|
+
return { invariantCoverage: coverage };
|
|
164
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// Reproducible scenario export, without tenant secrets (X-407.AC03).
|
|
2
|
+
//
|
|
3
|
+
// A reviewer or another tool should be able to take a bounded scenario away, replay it on the same pinned fixture, and get the
|
|
4
|
+
// same verdict. This builds that package from generated scenarios, and keeps three promises:
|
|
5
|
+
//
|
|
6
|
+
// - It carries no tenant secret. The fixture SOURCE is never exported (only its digest: the recipient already holds the
|
|
7
|
+
// fixture, or cannot reproduce anyway); the scenario inputs are scanned for secret-looking keys and token shapes, and a
|
|
8
|
+
// scenario that holds one is WITHHELD with the paths named (never the values), not exported with a hole in it, because a
|
|
9
|
+
// scenario with a value removed is a different scenario and would not replay to the same digest.
|
|
10
|
+
// - It is self-checking. Every scenario carries its content digest and its pinned fixture digest; `verifyScenarioExport`
|
|
11
|
+
// recomputes them (and the replay manifest ids, when a commit was supplied) so an edited export is detected.
|
|
12
|
+
// - It never claims more than it did. The package states that it is a bounded sample of one contract's behaviour, lists the
|
|
13
|
+
// bounds that applied, the scenario families that could not be built, and says plainly that it is not a proof of
|
|
14
|
+
// business-logic correctness.
|
|
15
|
+
//
|
|
16
|
+
// Gate: the `invariant-scenarios` feature, off by default. With it off nothing is built.
|
|
17
|
+
//
|
|
18
|
+
// `findSecrets` is shared with the regression artifact (`repair.js`), which embeds the fixture and the patch and therefore
|
|
19
|
+
// must refuse to embed a credential. It reports WHERE (file and kind), never WHAT. Pattern checks over text: a paraphrased or
|
|
20
|
+
// encoded secret is not caught, and the export is only as clean as the synthetic fixture it was generated from.
|
|
21
|
+
import { digestOf, semanticId } from '../assurance/identity.js';
|
|
22
|
+
import { SCHEMA_VERSION, isPlainObject } from '../assurance/schema-kit.js';
|
|
23
|
+
import { featureStatus, resolveAssuranceConfig } from '../assurance/config.js';
|
|
24
|
+
import { createReplayManifest } from '../replay/replay.js';
|
|
25
|
+
import { generateScenarios, HARD_BOUNDS, FEATURE } from './scenarios.js';
|
|
26
|
+
import { verifyLedger } from './lifecycle.js';
|
|
27
|
+
import { readJsonFile, readFixtureDir } from './project-input.js';
|
|
28
|
+
|
|
29
|
+
export const EXPORT_SCHEMA = 'agentic-security/invariant-scenario-export';
|
|
30
|
+
export const NOT_EXHAUSTIVE = 'This is a bounded sample of one contract on one disposable fixture. A clean result covers only the scenarios listed here; it is not a proof of business-logic correctness.';
|
|
31
|
+
|
|
32
|
+
const SECRET_KEY = /secret|token|passw|api[_-]?key|authorization|cookie|credential|private[_-]?key|ssn/i;
|
|
33
|
+
const SECRET_SHAPES = [
|
|
34
|
+
[/-----BEGIN [A-Z ]*PRIVATE KEY-----/, 'private key block'],
|
|
35
|
+
[/\bAKIA[0-9A-Z]{16}\b/, 'cloud access key id'],
|
|
36
|
+
[/\bgh[pousr]_[A-Za-z0-9]{30,}\b/, 'source host token'],
|
|
37
|
+
[/\bsk-[A-Za-z0-9_-]{10,}/, 'secret key token'],
|
|
38
|
+
[/\bxox[abprs]-[A-Za-z0-9-]{10,}/, 'chat token'],
|
|
39
|
+
[/\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{5,}/, 'signed web token'],
|
|
40
|
+
[/\b(?:password|passwd|secret|api[_-]?key|token|authorization|bearer)\b["']?\s*[:=]\s*["'][^"'\s]{6,}["']/i, 'credential assignment'],
|
|
41
|
+
];
|
|
42
|
+
|
|
43
|
+
/** Secret-looking content in a map of file name to text. Names the file and the kind; never returns the matched text. */
|
|
44
|
+
export function findSecrets(files) {
|
|
45
|
+
const out = [];
|
|
46
|
+
for (const [file, text] of Object.entries(isPlainObject(files) ? files : {})) {
|
|
47
|
+
if (typeof text !== 'string') continue;
|
|
48
|
+
for (const [re, kind] of SECRET_SHAPES) if (re.test(text)) out.push({ where: file, kind });
|
|
49
|
+
}
|
|
50
|
+
return out;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Secret-looking keys or values anywhere in a JSON-able value. Returns paths and kinds, never the values. */
|
|
54
|
+
export function findSecretsIn(value, path = '$', out = []) {
|
|
55
|
+
if (typeof value === 'string') {
|
|
56
|
+
for (const [re, kind] of SECRET_SHAPES) if (re.test(value)) { out.push({ where: path, kind }); break; }
|
|
57
|
+
} else if (Array.isArray(value)) {
|
|
58
|
+
value.forEach((v, i) => findSecretsIn(v, `${path}[${i}]`, out));
|
|
59
|
+
} else if (isPlainObject(value)) {
|
|
60
|
+
for (const [k, v] of Object.entries(value)) {
|
|
61
|
+
if (SECRET_KEY.test(k) && v !== null && v !== undefined && v !== '' && typeof v !== 'boolean') out.push({ where: `${path}.${k}`, kind: 'secret-looking field' });
|
|
62
|
+
else findSecretsIn(v, `${path}.${k}`, out);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return out;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const copy = (v) => JSON.parse(JSON.stringify(v));
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Build the export for one invariant. Never throws.
|
|
72
|
+
*
|
|
73
|
+
* @param {object} p
|
|
74
|
+
* @param {object} p.invariant a valid invariant document
|
|
75
|
+
* @param {object} p.fixture { files } of the disposable fixture (used for its digest and the entry check only; never exported)
|
|
76
|
+
* @param {object} [p.ledger] the approval ledger: only a ledger that verifies can say a contract is approved
|
|
77
|
+
* @param {string} [p.commit] exact commit; with it the replay manifests are included, without it they are listed as needed
|
|
78
|
+
* @param {number} [p.seed]
|
|
79
|
+
* @param {object} [p.bounds]
|
|
80
|
+
* @param {object} [p.config] assurance config
|
|
81
|
+
* @param {object} [p.signer] ledger signer (test seam)
|
|
82
|
+
* @returns {{ status: 'ok'|'disabled'|'unsupported'|'rejected'|'blocked', reason?: string, export?: object, withheld?: object[] }}
|
|
83
|
+
*/
|
|
84
|
+
export function exportScenarios(p = {}) {
|
|
85
|
+
const config = p.config || resolveAssuranceConfig({ env: process.env });
|
|
86
|
+
const gate = featureStatus(config, FEATURE);
|
|
87
|
+
if (gate.status !== 'ok') return { status: 'disabled', reason: `${FEATURE} is not available: ${gate.reason}` };
|
|
88
|
+
const gen = generateScenarios(p.invariant, { fixture: p.fixture, seed: p.seed, bounds: p.bounds, config });
|
|
89
|
+
if (gen.status !== 'ok') return { status: gen.status, reason: gen.reason, errors: gen.errors };
|
|
90
|
+
|
|
91
|
+
// the identifying text that travels with the export is checked too: a credential pasted into a contract's name must not leave
|
|
92
|
+
const meta = findSecretsIn({ name: p.invariant.name, key: p.invariant.key, application: p.invariant.scope.application }, 'invariant');
|
|
93
|
+
if (meta.length) return { status: 'blocked', reason: 'secret-looking content in the contract\'s name, key or application', withheld: [{ scenarioId: null, kind: null, reason: 'secret-looking content in the contract metadata', where: meta }] };
|
|
94
|
+
|
|
95
|
+
const withheld = [];
|
|
96
|
+
const scenarios = [];
|
|
97
|
+
for (const s of gen.scenarios) {
|
|
98
|
+
const hits = findSecretsIn(s.inputs, 'inputs');
|
|
99
|
+
if (hits.length) { withheld.push({ scenarioId: s.id, kind: s.kind, reason: 'secret-looking content in the scenario inputs', where: hits }); continue; }
|
|
100
|
+
const doc = copy(s);
|
|
101
|
+
const entry = { scenario: doc, digest: digestOf(doc) };
|
|
102
|
+
if (p.commit) {
|
|
103
|
+
try { entry.manifest = createReplayManifest({ hypothesisId: s.id, commit: p.commit, fixtureFiles: p.fixture.files, oracleId: 'business-state', entry: s.entry, inputs: s.inputs, budgets: { timeoutMs: s.limits.timeBudgetMs } }); } catch { entry.manifest = null; }
|
|
104
|
+
}
|
|
105
|
+
scenarios.push(entry);
|
|
106
|
+
}
|
|
107
|
+
if (!scenarios.length && withheld.length) return { status: 'blocked', reason: 'every scenario was withheld because it held secret-looking content', withheld };
|
|
108
|
+
|
|
109
|
+
let approval = { state: 'unrecorded', source: 'no ledger supplied' };
|
|
110
|
+
if (p.ledger) {
|
|
111
|
+
const v = verifyLedger(p.ledger, { signer: p.signer });
|
|
112
|
+
approval = v.ok ? { state: v.states[p.invariant.id] || 'unrecorded', source: 'verified approval ledger' } : { state: 'unverified', source: 'the approval ledger does not verify' };
|
|
113
|
+
}
|
|
114
|
+
const body = {
|
|
115
|
+
schema: EXPORT_SCHEMA, schemaVersion: SCHEMA_VERSION,
|
|
116
|
+
invariant: { id: p.invariant.id, key: p.invariant.key, revision: p.invariant.revision, class: p.invariant.class, name: p.invariant.name },
|
|
117
|
+
approval: { ...approval, documentClaims: p.invariant.review?.state ?? null },
|
|
118
|
+
fixture: { digest: gen.scenarios[0]?.fixtureDigest ?? digestOf(p.fixture.files), entry: p.invariant.scope.entry, included: false, note: 'the fixture source is not exported: replay needs the fixture whose digest is listed' },
|
|
119
|
+
oracle: { id: 'business-state', version: '1' },
|
|
120
|
+
bounds: { applied: gen.bounds, hard: HARD_BOUNDS },
|
|
121
|
+
scenarios,
|
|
122
|
+
notBuilt: gen.unsupported,
|
|
123
|
+
withheld,
|
|
124
|
+
secretsExported: false,
|
|
125
|
+
claims: { exhaustive: false, statement: NOT_EXHAUSTIVE },
|
|
126
|
+
reproduce: p.commit
|
|
127
|
+
? 'replay each manifest against the fixture with the listed digest (replayManifest), then compare the outcome with an executed run'
|
|
128
|
+
: 'supply an exact commit to obtain replay manifests; the scenario documents alone pin the fixture digest, the seed and the inputs',
|
|
129
|
+
};
|
|
130
|
+
body.id = semanticId('sexp', { ...body, scenarioDigests: scenarios.map((e) => e.digest) }, ['invariant', 'fixture', 'scenarioDigests', 'bounds']);
|
|
131
|
+
return { status: 'ok', export: body, withheld };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Check an export: every scenario digest, content-addressed id and fixture pin recomputes, no secret-looking content is present,
|
|
136
|
+
* and (given the fixture files) the pinned digest matches. Never throws.
|
|
137
|
+
*/
|
|
138
|
+
export function verifyScenarioExport(doc, { fixtureFiles } = {}) {
|
|
139
|
+
const errors = [];
|
|
140
|
+
const fail = (code, message) => errors.push({ code, message });
|
|
141
|
+
try {
|
|
142
|
+
if (!isPlainObject(doc) || doc.schema !== EXPORT_SCHEMA) return { ok: false, errors: [{ code: 'malformed', message: 'not a scenario export' }] };
|
|
143
|
+
if (doc.claims?.exhaustive !== false || typeof doc.claims?.statement !== 'string') fail('overclaim', 'an export must state that it is not exhaustive');
|
|
144
|
+
if (doc.secretsExported !== false) fail('secrets-flag', 'the export must record that no secret was exported');
|
|
145
|
+
for (const e of doc.scenarios || []) {
|
|
146
|
+
if (digestOf(e.scenario) !== e.digest) fail('scenario-tampered', `scenario ${e.scenario?.id} does not match its digest`);
|
|
147
|
+
if (e.scenario?.fixtureDigest !== doc.fixture?.digest) fail('fixture-pin-mismatch', `scenario ${e.scenario?.id} pins a different fixture than the export`);
|
|
148
|
+
if (e.manifest && e.manifest.fixture?.digest !== doc.fixture?.digest) fail('manifest-fixture-mismatch', `the manifest for ${e.scenario?.id} pins a different fixture`);
|
|
149
|
+
const leaked = findSecretsIn(e.scenario?.inputs, 'inputs');
|
|
150
|
+
if (leaked.length) fail('secret-in-export', `scenario ${e.scenario?.id} holds secret-looking content at ${leaked.map((l) => l.where).join(', ')}`);
|
|
151
|
+
}
|
|
152
|
+
if (fixtureFiles !== undefined && digestOf(fixtureFiles) !== doc.fixture?.digest) fail('fixture-mismatch', 'the supplied fixture does not match the digest the export pins');
|
|
153
|
+
if (doc.fixture?.included !== false) fail('fixture-included', 'the export must not include the fixture source');
|
|
154
|
+
} catch (e) {
|
|
155
|
+
fail('malformed', `the check could not complete: ${String(e?.message || e).slice(0, 120)}`);
|
|
156
|
+
}
|
|
157
|
+
return { ok: errors.length === 0, errors };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* The CLI and MCP entry: build an export from files already resolved (and confined) to absolute paths by the caller.
|
|
162
|
+
* Reads the contract, the fixture directory and, optionally, the ledger through the bounded readers in `project-input.js`.
|
|
163
|
+
* Never throws; every failure is a typed status with its reason.
|
|
164
|
+
*/
|
|
165
|
+
export function exportFromFiles({ invariantPath, fixturePath, ledgerPath, commit, seed, bounds, config, signer } = {}) {
|
|
166
|
+
const inv = readJsonFile(invariantPath);
|
|
167
|
+
if (!inv.ok) return { status: 'rejected', reason: `the invariant file: ${inv.reason}` };
|
|
168
|
+
const fx = readFixtureDir(fixturePath);
|
|
169
|
+
if (!fx.ok) return { status: 'rejected', reason: `the fixture directory: ${fx.errors.join('; ')}` };
|
|
170
|
+
let ledger;
|
|
171
|
+
if (ledgerPath) {
|
|
172
|
+
const l = readJsonFile(ledgerPath);
|
|
173
|
+
if (!l.ok) return { status: 'rejected', reason: `the ledger file: ${l.reason}` };
|
|
174
|
+
ledger = l.value;
|
|
175
|
+
}
|
|
176
|
+
if (commit !== undefined && !/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(String(commit))) return { status: 'rejected', reason: 'commit must be an exact 40 or 64 character hex id' };
|
|
177
|
+
return exportScenarios({ invariant: inv.value, fixture: { files: fx.files }, ledger, commit, seed, bounds, config, signer });
|
|
178
|
+
}
|