@clear-capabilities/agentic-security-scanner 0.162.0 → 0.163.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/bin/agentic-security.js +87 -0
- package/dist/1340.index.js +1871 -0
- package/dist/3477.index.js +727 -0
- package/dist/3499.index.js +23 -3
- package/dist/4113.index.js +1 -1
- package/dist/5014.index.js +833 -0
- package/dist/5756.index.js +2 -2
- package/dist/6257.index.js +1 -1
- package/dist/6907.index.js +248 -0
- package/dist/6994.index.js +1 -1
- package/dist/7033.index.js +33 -5
- package/dist/7439.index.js +1822 -0
- package/dist/7838.index.js +1 -1
- package/dist/9560.index.js +2318 -23
- package/dist/957.index.js +1 -1
- package/dist/9999.index.js +43 -7
- package/dist/agentic-security.mjs +12 -12
- package/dist/agentic-security.mjs.sha256 +1 -1
- package/package.json +33 -6
- package/src/capabilities/CLAUDE.md +114 -0
- package/src/capabilities/address.js +73 -0
- package/src/capabilities/attack-coverage.js +93 -0
- package/src/capabilities/decide.js +256 -0
- package/src/capabilities/delegate.js +69 -0
- package/src/capabilities/egress.js +20 -0
- package/src/capabilities/hook-advice.js +50 -0
- package/src/capabilities/index.js +32 -0
- package/src/capabilities/manifest.js +344 -0
- package/src/capabilities/outbound.js +189 -0
- package/src/capabilities/paths.js +57 -0
- package/src/capabilities/probes.js +211 -0
- package/src/capabilities/proxy.js +182 -0
- package/src/capabilities/reasons.js +67 -0
- package/src/capabilities/receipts.js +356 -0
- package/src/capabilities/records.js +58 -0
- package/src/capabilities/recovery.js +367 -0
- package/src/capabilities/report.js +81 -0
- package/src/capabilities/runner.js +236 -0
- package/src/capabilities/secrets.js +26 -0
- package/src/capabilities/tool-gate.js +119 -0
- package/src/capabilities/tool-registry.js +60 -0
- package/src/dataflow/CLAUDE.md +30 -0
- package/src/dataflow/catalog.js +42 -1
- package/src/dataflow/engine.js +67 -2
- package/src/dataflow/guard-dominance.js +352 -0
- package/src/discovery/CLAUDE.md +9 -0
- package/src/discovery/index.js +15 -1
- package/src/discovery/lenses.js +18 -0
- package/src/engine.js +80 -15
- package/src/integrations/index.js +5 -2
- package/src/ir/CLAUDE.md +14 -0
- package/src/ir/implicit-handler-params.js +74 -0
- package/src/ir/parser-cs.js +20 -0
- package/src/ir/parser-go.js +30 -13
- package/src/ir/parser-java.js +11 -0
- package/src/language/nix-eval-isolation.js +33 -5
- package/src/lineage/CLAUDE.md +31 -0
- package/src/lineage/deployment/adapter-cloud.js +202 -0
- package/src/lineage/deployment/adapter-compose.js +90 -0
- package/src/lineage/deployment/adapter-kit.js +142 -0
- package/src/lineage/deployment/adapter-kubernetes.js +247 -0
- package/src/lineage/deployment/attack-paths.js +443 -0
- package/src/lineage/deployment/boundaries-run.js +223 -0
- package/src/lineage/deployment/boundary-graph.js +567 -0
- package/src/lineage/deployment/cross-repo.js +253 -0
- package/src/lineage/deployment/drift.js +383 -0
- package/src/lineage/deployment/ingest.js +221 -0
- package/src/lineage/deployment/projection.js +244 -0
- package/src/lineage/deployment/trace-correlation.js +541 -0
- package/src/lineage/source-registry.js +19 -0
- package/src/mcp/CLAUDE.md +5 -2
- package/src/mcp/invariant-tools.js +64 -0
- package/src/mcp/portfolio-tools.js +79 -0
- package/src/mcp/redact.js +15 -0
- package/src/mcp/server.js +24 -3
- package/src/mcp/tools.js +13 -1
- package/src/posture/CLAUDE.md +332 -1
- package/src/posture/accuracy-scorecard.js +95 -4
- package/src/posture/assurance/rollback.js +214 -0
- package/src/posture/assurance/rollout.js +200 -0
- package/src/posture/business-logic.js +26 -4
- package/src/posture/evaluation/bakeoff.js +148 -0
- package/src/posture/evaluation/custody.js +200 -0
- package/src/posture/evaluation/deployment-ablation.js +416 -0
- package/src/posture/evaluation/dev-cases.js +65 -0
- package/src/posture/evaluation/economics.js +85 -0
- package/src/posture/evaluation/gates.js +161 -0
- package/src/posture/evaluation/grouping.js +101 -0
- package/src/posture/evaluation/interval.js +132 -0
- package/src/posture/evaluation/invariant-ablation.js +325 -0
- package/src/posture/evaluation/labels.js +200 -0
- package/src/posture/evaluation/protocol.js +308 -0
- package/src/posture/evaluation/release-gate.js +209 -0
- package/src/posture/evaluation/report.js +158 -0
- package/src/posture/evaluation/runner.js +273 -0
- package/src/posture/evaluation/scan-child.js +80 -0
- package/src/posture/evaluation/score.js +174 -0
- package/src/posture/evaluation/synthetic.js +90 -0
- package/src/posture/evaluation/why-missed.js +219 -0
- package/src/posture/fleet.js +11 -2
- package/src/posture/invariants/cli.js +95 -0
- package/src/posture/invariants/coverage.js +164 -0
- package/src/posture/invariants/export.js +178 -0
- package/src/posture/invariants/expressions.js +46 -0
- package/src/posture/invariants/lifecycle.js +289 -0
- package/src/posture/invariants/project-input.js +55 -0
- package/src/posture/invariants/repair.js +340 -0
- package/src/posture/invariants/run.js +45 -0
- package/src/posture/invariants/scenarios.js +311 -0
- package/src/posture/invariants/schema.js +167 -0
- package/src/posture/invariants/shrink.js +407 -0
- package/src/posture/invariants/state-assertions.js +165 -0
- package/src/posture/logic-claims.js +12 -0
- package/src/posture/model-routing.js +40 -0
- package/src/posture/oracles/adapters.js +172 -1
- package/src/posture/oracles/conformance.js +3 -2
- package/src/posture/oracles/oracle.js +23 -3
- package/src/posture/oracles/registry.js +1 -1
- package/src/posture/portfolio/backend.js +279 -0
- package/src/posture/portfolio/bundle.js +314 -0
- package/src/posture/portfolio/cli.js +142 -0
- package/src/posture/portfolio/manifest.js +263 -0
- package/src/posture/portfolio/progress.js +267 -0
- package/src/posture/portfolio/resume.js +130 -0
- package/src/posture/portfolio/retention.js +207 -0
- package/src/posture/portfolio/scheduler.js +488 -0
- package/src/posture/portfolio/signing.js +155 -0
- package/src/posture/portfolio/wording.js +59 -0
- package/src/posture/portfolio/work-units.js +405 -0
- package/src/posture/replay/replay.js +1 -0
- package/src/posture/routing/adapter.js +140 -0
- package/src/posture/routing/calibration.js +278 -0
- package/src/posture/routing/canary.js +141 -0
- package/src/posture/routing/control.js +35 -0
- package/src/posture/routing/decide.js +211 -0
- package/src/posture/routing/drift.js +137 -0
- package/src/posture/routing/economics.js +185 -0
- package/src/posture/routing/feedback.js +207 -0
- package/src/posture/routing/index.js +31 -0
- package/src/posture/routing/outcomes.js +281 -0
- package/src/posture/routing/promotion.js +291 -0
- package/src/posture/routing/receipts.js +59 -0
- package/src/posture/routing/report.js +142 -0
- package/src/posture/routing/shadow.js +53 -0
- package/src/posture/specification-mining.js +26 -0
- package/src/posture/verification/patch-negative.js +16 -4
- package/src/posture/verification/patch-promotion.js +10 -2
- package/src/report/index.js +51 -1
- package/src/sandbox/CLAUDE.md +10 -0
- package/src/sandbox/backend-userspace.js +91 -13
- package/dist/5613.index.js +0 -500
- package/dist/6239.index.js +0 -469
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
// Rollback and compatibility for the assurance features (REL-002.AC02, AC03).
|
|
2
|
+
//
|
|
3
|
+
// Four things live here, and the line between them is the point:
|
|
4
|
+
//
|
|
5
|
+
// 1. READER_POLICY: for every record kind, whether a PREVIOUS reader exists, so a rollback that
|
|
6
|
+
// restores old consumers is supported, or does not, in which case the policy says so and
|
|
7
|
+
// a rollback flags the new records instead of pretending the old consumers can read them.
|
|
8
|
+
// 2. The one known hard compatibility break. A verifier older than CORE-003 rejects every newly
|
|
9
|
+
// signed evidence bundle, because the new `issuance` block is a top-level key its strict
|
|
10
|
+
// unknown-key rule (which exists to stop unsigned fields being stapled on) does not know.
|
|
11
|
+
// `verifyWithPreCore003Reader` reproduces that reader exactly so the break is tested rather than
|
|
12
|
+
// asserted; `exportLegacyBundle` is the migration path for a consumer that cannot upgrade.
|
|
13
|
+
// 3. planRollback / applyRollback: restore the last known-good rollout position and FLAG, never
|
|
14
|
+
// delete or rewrite, the recorded evidence and the assurance claims that rest on it.
|
|
15
|
+
// 4. EvidenceStore / guardedFeatureRun: an append-only record, so a feature that fails cannot take
|
|
16
|
+
// earlier evidence down with it.
|
|
17
|
+
//
|
|
18
|
+
// Nothing here reaches the network or a model. Nothing here edits configuration: a rollback that needs
|
|
19
|
+
// a feature off returns the configuration change for the operator to apply.
|
|
20
|
+
|
|
21
|
+
import * as fs from 'node:fs';
|
|
22
|
+
import * as crypto from 'node:crypto';
|
|
23
|
+
import { canonicalBytes, verifyEvidenceBundle, keyFingerprint } from '../evidence-bundle.js';
|
|
24
|
+
import { digestOfBytes } from './identity.js';
|
|
25
|
+
import { STAGES } from './rollout.js';
|
|
26
|
+
|
|
27
|
+
// ---------------------------------------------------------------- reader policy
|
|
28
|
+
|
|
29
|
+
export const READER_POLICY = Object.freeze({
|
|
30
|
+
verification: { rollbackSupported: true, previousReader: 'toLegacyVerificationView (migrations.js)', note: 'a legacy consumer sees status/verified/reason; verified is true only for confirmed and false only for refuted' },
|
|
31
|
+
evidenceBundle: { rollbackSupported: true, previousReader: 'verifyWithPreCore003Reader', note: 'a pre-CORE-003 verifier can read only bundles re-exported without the issuance block (exportLegacyBundle)' },
|
|
32
|
+
observationBinding: { rollbackSupported: false, previousReader: null, note: 'no previous reader exists: lineage keeps its own observation; the binding is new. Rollback keeps the records and flags them incompatible for any older consumer' },
|
|
33
|
+
capabilityDecision: { rollbackSupported: false, previousReader: null, note: 'the egress decision it adapts has no inverse; a decision is never turned back into a bare allow/deny' },
|
|
34
|
+
routingLabel: { rollbackSupported: false, previousReader: null, note: 'a boolean label has no place for a label source; the inverse would invent one' },
|
|
35
|
+
releaseEvidence: { rollbackSupported: false, previousReader: null, note: 'a legacy check list cannot carry evidence references; the inverse would drop them' },
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// ---------------------------------------------------------------- the known compatibility break
|
|
39
|
+
|
|
40
|
+
// Top-level keys a bundle could carry before CORE-003 added `issuance`. Verbatim from the verifier of that time.
|
|
41
|
+
export const PRE_CORE_003_BUNDLE_KEYS = Object.freeze(['schema', 'finding', 'evidence', 'engine', 'proves', 'doesNotProve', 'signature']);
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* The bundle verifier as it stood before CORE-003: strict unknown top-level keys, signature over the
|
|
45
|
+
* canonical bytes, nothing about issuers. Kept as a faithful copy so the break is a test, not a belief.
|
|
46
|
+
*/
|
|
47
|
+
export function verifyWithPreCore003Reader(bundle, publicKeyPem) {
|
|
48
|
+
if (!bundle || typeof bundle !== 'object') return { ok: false, reason: 'bundle is not an object' };
|
|
49
|
+
const unknownKeys = Object.keys(bundle).filter((k) => !PRE_CORE_003_BUNDLE_KEYS.includes(k));
|
|
50
|
+
if (unknownKeys.length) return { ok: false, reason: `unrecognised top-level key(s) not covered by the signature: ${unknownKeys.join(', ')}` };
|
|
51
|
+
const sig = bundle.signature;
|
|
52
|
+
if (!sig?.value) return { ok: false, reason: 'bundle is unsigned' };
|
|
53
|
+
if (sig.algorithm !== 'ed25519') return { ok: false, reason: `unsupported algorithm: ${sig.algorithm}` };
|
|
54
|
+
try {
|
|
55
|
+
return crypto.verify(null, canonicalBytes(bundle), publicKeyPem, Buffer.from(sig.value, 'base64')) ? { ok: true, reason: null } : { ok: false, reason: 'signature does not match the bundle contents' };
|
|
56
|
+
} catch (e) { return { ok: false, reason: `verification error: ${e.message}` }; }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Migration path for a consumer that cannot upgrade past a pre-CORE-003 verifier: re-sign the SAME content
|
|
61
|
+
* without the issuance block. Three honesty rules:
|
|
62
|
+
* - the original must verify under the key that matches `privateKeyPem` first. A tampered or foreign
|
|
63
|
+
* bundle is refused, not laundered into a fresh valid signature;
|
|
64
|
+
* - the export loses the signed trust-basis statement; a current verifier reports it `none-declared`.
|
|
65
|
+
* Because an old verifier rejects any extra key, that disclosure cannot ride in the bundle, so it is returned
|
|
66
|
+
* as a SIDECAR record the operator keeps beside it;
|
|
67
|
+
* - the export is a new signature by the same local key, so it is still self-issued, still not third-party certification.
|
|
68
|
+
*/
|
|
69
|
+
export function exportLegacyBundle(bundle, privateKeyPem) {
|
|
70
|
+
if (!bundle || typeof bundle !== 'object') return { ok: false, reason: 'bundle is not an object' };
|
|
71
|
+
let publicKeyPem;
|
|
72
|
+
try { publicKeyPem = crypto.createPublicKey(privateKeyPem).export({ type: 'spki', format: 'pem' }); } catch (e) { return { ok: false, reason: `unusable signing key: ${e.message}` }; }
|
|
73
|
+
const original = verifyEvidenceBundle(bundle, publicKeyPem);
|
|
74
|
+
if (!original.ok) return { ok: false, reason: `the original bundle does not verify under this key (${original.reason}); it is not re-signed` };
|
|
75
|
+
const { issuance, signature, ...content } = bundle;
|
|
76
|
+
void signature;
|
|
77
|
+
const sig = crypto.sign(null, canonicalBytes(content), privateKeyPem);
|
|
78
|
+
const exported = { ...content, signature: { algorithm: 'ed25519', canonicalisation: bundle.signature.canonicalisation, value: sig.toString('base64') } };
|
|
79
|
+
return {
|
|
80
|
+
ok: true, bundle: exported,
|
|
81
|
+
sidecar: {
|
|
82
|
+
kind: 'legacy-bundle-export', from: 'core-003', originalBundleDigest: digestOfBytes(canonicalBytes(bundle)), exportedBundleDigest: digestOfBytes(canonicalBytes(exported)),
|
|
83
|
+
droppedIssuance: issuance ?? null, issuerKeyFingerprint: keyFingerprint(publicKeyPem),
|
|
84
|
+
note: 'Re-signed without the issuance block so a pre-CORE-003 verifier accepts it. The trust basis (self-issued local key, not independent certification) is no longer inside the signed bytes; it is recorded only here. Upgrade the verifier and keep the original bundle.',
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// ---------------------------------------------------------------- rollback planning
|
|
90
|
+
|
|
91
|
+
const clone = (x) => JSON.parse(JSON.stringify(x));
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* What a rollback of `feature` would restore and what it would flag. Pure: nothing is changed.
|
|
95
|
+
* store: [{ id, kind, digest, producedUnder: { feature, policyVersion } }]
|
|
96
|
+
* claims: [{ id, evidenceRefs: [store ids] }]
|
|
97
|
+
*/
|
|
98
|
+
export function planRollback({ ledger, feature, store = [], claims = [] }) {
|
|
99
|
+
const cur = ledger?.features?.[feature];
|
|
100
|
+
if (!cur) return { ok: false, reason: `unknown feature '${feature}'` };
|
|
101
|
+
const here = STAGES.indexOf(cur.stage);
|
|
102
|
+
// The most recent known-good entry at an EARLIER stage; the first stage rolls back to "off".
|
|
103
|
+
let target = null;
|
|
104
|
+
for (let i = cur.history.length - 1; i >= 0 && !target; i--) {
|
|
105
|
+
const h = cur.history[i];
|
|
106
|
+
if (h.knownGood && h.to && STAGES.indexOf(h.to) < here) target = h;
|
|
107
|
+
}
|
|
108
|
+
const restore = target
|
|
109
|
+
? { stage: target.to, policyVersion: target.policyVersion, source: `history[${target.seq}]` }
|
|
110
|
+
: { stage: cur.stage, policyVersion: cur.history[0].policyVersion, source: 'history[0]' };
|
|
111
|
+
const configChange = target ? null : { feature, enabled: false, why: 'already at the first stage; the only known-good behaviour below it is the feature being off' };
|
|
112
|
+
|
|
113
|
+
const annotations = {};
|
|
114
|
+
for (const item of store) {
|
|
115
|
+
if (item?.producedUnder?.feature !== feature) continue;
|
|
116
|
+
const flags = []; const reasons = [];
|
|
117
|
+
if ((item.producedUnder.policyVersion ?? 0) > restore.policyVersion) {
|
|
118
|
+
flags.push('stale'); reasons.push(`produced under policy ${item.producedUnder.policyVersion}, newer than the restored policy ${restore.policyVersion}`);
|
|
119
|
+
}
|
|
120
|
+
const policy = READER_POLICY[item.kind];
|
|
121
|
+
if (!policy) { flags.push('incompatible'); reasons.push(`no reader policy is recorded for kind '${item.kind}'`); }
|
|
122
|
+
else if (!policy.rollbackSupported) { flags.push('incompatible'); reasons.push(`kind '${item.kind}' has no previous reader (${policy.note})`); }
|
|
123
|
+
if (flags.length) annotations[item.id] = { flags, reasons };
|
|
124
|
+
}
|
|
125
|
+
const byId = new Map(store.map((i) => [i.id, i]));
|
|
126
|
+
const claimStatus = {};
|
|
127
|
+
for (const c of claims) {
|
|
128
|
+
const reasons = [];
|
|
129
|
+
for (const ref of c.evidenceRefs || []) {
|
|
130
|
+
if (!byId.has(ref)) reasons.push(`${ref}: not in the evidence store (dangling)`);
|
|
131
|
+
else if (annotations[ref]) reasons.push(`${ref}: ${annotations[ref].flags.join('+')}`);
|
|
132
|
+
}
|
|
133
|
+
claimStatus[c.id] = reasons.length ? { status: 'flagged', reasons } : { status: 'valid', reasons: [] };
|
|
134
|
+
}
|
|
135
|
+
return {
|
|
136
|
+
ok: true, feature, from: { stage: cur.stage, policyVersion: cur.policyVersion }, restore, configChange, annotations, claims: claimStatus,
|
|
137
|
+
// The plan never removes anything: every store id is retained.
|
|
138
|
+
retained: store.map((i) => i.id),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Apply a plan to the ledger. Returns a NEW ledger with a `rollback` event appended; history is never
|
|
144
|
+
* rewritten. The evidence store and claims are not touched at all: annotations are returned for the caller to
|
|
145
|
+
* keep beside them.
|
|
146
|
+
*/
|
|
147
|
+
export function applyRollback(ledger, plan, { reason = 'rollback' } = {}) {
|
|
148
|
+
if (!plan?.ok) return { ok: false, ledger, reason: plan?.reason ?? 'no plan' };
|
|
149
|
+
const next = clone(ledger);
|
|
150
|
+
const f = next.features[plan.feature];
|
|
151
|
+
f.history.push({ seq: f.history.length, event: 'rollback', from: f.stage, to: plan.restore.stage, policyVersion: plan.restore.policyVersion, knownGood: true, restoredFrom: plan.restore.source, reason: String(reason), evidence: null });
|
|
152
|
+
f.stage = plan.restore.stage; f.policyVersion = plan.restore.policyVersion;
|
|
153
|
+
return { ok: true, ledger: next, annotations: plan.annotations, claims: plan.claims, configChange: plan.configChange };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ---------------------------------------------------------------- evidence that survives failure
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Append-only evidence store. An existing id can be re-appended only with the identical digest (idempotent
|
|
160
|
+
* redelivery); the same id with a different digest is refused, because overwriting evidence is erasing it.
|
|
161
|
+
* Optionally backed by a JSONL file written with append-only semantics: a torn last line (a crash mid-write)
|
|
162
|
+
* (or any unreadable line) is counted in `unreadable` and skipped on read, and never costs the other lines.
|
|
163
|
+
*/
|
|
164
|
+
export class EvidenceStore {
|
|
165
|
+
constructor({ file = null } = {}) {
|
|
166
|
+
this.file = file; this.items = new Map(); this.failures = []; this.unreadable = 0;
|
|
167
|
+
if (file && fs.existsSync(file)) this._load();
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
_load() {
|
|
171
|
+
const lines = fs.readFileSync(this.file, 'utf8').split('\n');
|
|
172
|
+
lines.forEach((line, i) => {
|
|
173
|
+
if (!line.trim()) return;
|
|
174
|
+
try {
|
|
175
|
+
const rec = JSON.parse(line);
|
|
176
|
+
if (rec.type === 'item') this.items.set(rec.item.id, rec.item);
|
|
177
|
+
else if (rec.type === 'failure') this.failures.push(rec.failure);
|
|
178
|
+
} catch { this.unreadable += 1; }
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
append(item) {
|
|
183
|
+
if (!item || typeof item.id !== 'string' || typeof item.digest !== 'string') return { ok: false, reason: 'an evidence item needs a string id and digest' };
|
|
184
|
+
const have = this.items.get(item.id);
|
|
185
|
+
if (have) return have.digest === item.digest ? { ok: true, duplicate: true } : { ok: false, reason: `evidence '${item.id}' already exists with a different digest; evidence is never overwritten` };
|
|
186
|
+
if (this.file) fs.appendFileSync(this.file, `${JSON.stringify({ type: 'item', item })}\n`);
|
|
187
|
+
this.items.set(item.id, clone(item));
|
|
188
|
+
return { ok: true, duplicate: false };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
recordFailure(failure) {
|
|
192
|
+
this.failures.push(failure);
|
|
193
|
+
if (this.file) fs.appendFileSync(this.file, `${JSON.stringify({ type: 'failure', failure })}\n`);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
all() { return [...this.items.values()].map(clone); }
|
|
197
|
+
|
|
198
|
+
/** Digest over every item in id order; changes if any item is removed or altered. */
|
|
199
|
+
digest() { return digestOfBytes(Buffer.from(JSON.stringify([...this.items.keys()].sort().map((k) => this.items.get(k))))); }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Run a feature body. A throw becomes a typed `blocked` result and a recorded failure event; the store's
|
|
204
|
+
* earlier evidence is untouched, because nothing here has a path that removes an item.
|
|
205
|
+
*/
|
|
206
|
+
export async function guardedFeatureRun(store, feature, fn) {
|
|
207
|
+
try {
|
|
208
|
+
return { status: 'ok', code: null, feature, value: await fn(store) };
|
|
209
|
+
} catch (e) {
|
|
210
|
+
const failure = { feature, error: String(e?.message ?? e).slice(0, 500) };
|
|
211
|
+
store.recordFailure(failure);
|
|
212
|
+
return { status: 'blocked', code: 'feature-failed', reason: `${feature} failed: ${failure.error}`, feature };
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// Staged rollout policy for the assurance features (REL-002.AC01).
|
|
2
|
+
//
|
|
3
|
+
// Every feature in config.js is off by default. This module is the policy that says
|
|
4
|
+
// how a feature moves from "exists in the code" to "an operator may rely on it":
|
|
5
|
+
//
|
|
6
|
+
// offline-fixtures -> shadow-canary -> opt-in-supported
|
|
7
|
+
//
|
|
8
|
+
// offline-fixtures The feature runs against recorded fixtures in tests only. No real target.
|
|
9
|
+
// shadow-canary It may run beside the existing behaviour on real input, but its result is
|
|
10
|
+
// RECORDED, never used: no verdict, exit code or report line changes. A canary is a
|
|
11
|
+
// named, bounded subset an operator chose.
|
|
12
|
+
// opt-in-supported An operator enabled it (config.js), the platform is supported, and every gate for
|
|
13
|
+
// the stage is met. Only here may a result change an outcome.
|
|
14
|
+
//
|
|
15
|
+
// A stage is entered only by `promote`, one step at a time, and only when every documented gate of
|
|
16
|
+
// the stage being entered has evidence that matches the gate's own declared reference. There is no
|
|
17
|
+
// way to skip a stage, to promote a feature the kill switch has stopped, or to promote on a gate
|
|
18
|
+
// that is merely "not failing". Nothing here runs a suite or measures anything: it judges evidence a
|
|
19
|
+
// caller supplies, and a gate that names a suite only accepts evidence for THAT suite.
|
|
20
|
+
//
|
|
21
|
+
// The ledger is append-only history plus a current position. `rollback` (rollback.js) moves the
|
|
22
|
+
// position back to the last known-good entry and appends; it never rewrites or removes history.
|
|
23
|
+
|
|
24
|
+
import { FEATURES, featureStatus } from './config.js';
|
|
25
|
+
|
|
26
|
+
const ROLLOUT_LEDGER_SCHEMA = 'agentic-security/rollout-ledger@1';
|
|
27
|
+
export const STAGES = Object.freeze(['offline-fixtures', 'shadow-canary', 'opt-in-supported']);
|
|
28
|
+
|
|
29
|
+
export const STAGE_MEANING = Object.freeze({
|
|
30
|
+
'offline-fixtures': 'Runs against recorded fixtures in tests only; no real target is touched.',
|
|
31
|
+
'shadow-canary': 'May run beside existing behaviour on real input or a named canary subset; results are recorded and never used, so no verdict, exit code or report line changes.',
|
|
32
|
+
'opt-in-supported': 'An operator enabled it, the platform is supported and every gate is met; only here may a result change an outcome.',
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// Gate evidence kinds. `suite` names a closure step (scripts/release-closure.mjs) whose recorded result is
|
|
36
|
+
// the evidence; `measurement` and `review` name an artifact a person or a bench produced.
|
|
37
|
+
const suite = (id, closureStep, statement) => ({ id, kind: 'suite', closureStep, statement });
|
|
38
|
+
const measure = (id, statement) => ({ id, kind: 'measurement', statement });
|
|
39
|
+
const review = (id, statement) => ({ id, kind: 'review', statement });
|
|
40
|
+
|
|
41
|
+
// The suite that proves each feature's offline behaviour.
|
|
42
|
+
const FEATURE_SUITE = Object.freeze({
|
|
43
|
+
'verification-oracles': 'verification',
|
|
44
|
+
'patch-negative-verification': 'verification',
|
|
45
|
+
'deployment-boundaries': 'deployment',
|
|
46
|
+
'invariant-scenarios': 'invariants',
|
|
47
|
+
'capability-enforcement': 'capabilities',
|
|
48
|
+
'model-routing': 'routing',
|
|
49
|
+
'portfolio-assurance': 'portfolio',
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
// What each feature must show before opt-in. Feature specific, and quoted from the PRD's own targets
|
|
53
|
+
// (section 5) rather than invented here.
|
|
54
|
+
const OPT_IN_SPECIFIC = Object.freeze({
|
|
55
|
+
'verification-oracles': [measure('oracle-conformance', 'Every advertised adapter passes positive, negative, inconclusive and tamper controls (verification-conformance-gate); unavailable prerequisites are reported unsupported')],
|
|
56
|
+
'patch-negative-verification': [measure('repair-evidence-triple', 'Every advertised verified fix carries original-positive, patched-negative and functional-regression evidence bound to the exact environment, patch and oracle')],
|
|
57
|
+
'deployment-boundaries': [measure('paired-ablation', 'A frozen contextual ablation shows additional confirmed defects or fewer false positives with unchanged scoring, and no lost baseline defect')],
|
|
58
|
+
'invariant-scenarios': [review('contract-approval-path', 'Invariant contracts are approved by an authorized reviewer, never by the generator that proposed them')],
|
|
59
|
+
'capability-enforcement': [measure('backend-probes', 'Every mandatory canary-secret and escape fixture is blocked on the advertised backend with zero canary leakage; macOS stays supervise-only')],
|
|
60
|
+
'model-routing': [measure('promotion-thresholds', 'At least 200 paired adjudicated tasks overall and 30 per promoted stratum, lower 95% quality bound above -0.02, median cost 20% lower or quality 0.05 higher at no higher median cost, p95 latency within 1.2x')],
|
|
61
|
+
'portfolio-assurance': [measure('interrupted-run-convergence', 'Interrupted, duplicate-delivery and incremental runs converge to a fresh scoped result with no double-counted progress')],
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
/** The documented gates for entering `stage` with `feature`. Pure and frozen. */
|
|
65
|
+
export function gatesFor(feature, stage) {
|
|
66
|
+
if (!FEATURES[feature] || !STAGES.includes(stage)) return null;
|
|
67
|
+
const own = FEATURE_SUITE[feature];
|
|
68
|
+
if (stage === 'offline-fixtures') {
|
|
69
|
+
return [
|
|
70
|
+
suite('fixtures-green', own, `The ${own} suite passes against recorded fixtures`),
|
|
71
|
+
suite('default-off-unchanged', 'foundation', 'With the feature disabled, scan and report behaviour is unchanged (configuration and kill-switch tests)'),
|
|
72
|
+
];
|
|
73
|
+
}
|
|
74
|
+
if (stage === 'shadow-canary') {
|
|
75
|
+
return [
|
|
76
|
+
suite('fixtures-green', own, `The ${own} suite still passes at the commit being promoted`),
|
|
77
|
+
measure('shadow-no-outcome-change', 'A recorded shadow run on real input changed no verdict, exit code or report line'),
|
|
78
|
+
suite('rollback-rehearsed', 'release-closure-suite', 'Rollback was rehearsed and restored known-good behaviour without erasing evidence'),
|
|
79
|
+
];
|
|
80
|
+
}
|
|
81
|
+
return [
|
|
82
|
+
suite('fixtures-green', own, `The ${own} suite passes at the commit being promoted`),
|
|
83
|
+
suite('compatibility-holds', 'compat-nix', 'Existing Haskell/Nix and core-language behaviour is unchanged'),
|
|
84
|
+
suite('documentation-current', 'documentation-suite', 'Scope, examples and the policy card for this feature are published and drift-checked'),
|
|
85
|
+
review('canary-record', 'A shadow-canary record exists with no unresolved incident'),
|
|
86
|
+
...(OPT_IN_SPECIFIC[feature] || []),
|
|
87
|
+
];
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Every feature's documented gates, for documentation and drift checks. */
|
|
91
|
+
export function describeRollout() {
|
|
92
|
+
return Object.keys(FEATURES).map((feature) => ({
|
|
93
|
+
feature, risk: FEATURES[feature].risk, platforms: [...FEATURES[feature].platforms],
|
|
94
|
+
stages: STAGES.map((stage) => ({ stage, meaning: STAGE_MEANING[stage], gates: gatesFor(feature, stage) })),
|
|
95
|
+
}));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// ---------------------------------------------------------------- ledger
|
|
99
|
+
|
|
100
|
+
const DIGEST = /^(?:sha256:)?[0-9a-f]{64}$/;
|
|
101
|
+
|
|
102
|
+
export function newLedger() {
|
|
103
|
+
const features = {};
|
|
104
|
+
for (const id of Object.keys(FEATURES)) {
|
|
105
|
+
features[id] = { stage: 'offline-fixtures', policyVersion: 1, history: [{ seq: 0, event: 'init', from: null, to: 'offline-fixtures', policyVersion: 1, knownGood: true, evidence: null }] };
|
|
106
|
+
}
|
|
107
|
+
return { schema: ROLLOUT_LEDGER_SCHEMA, features };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const clone = (x) => JSON.parse(JSON.stringify(x));
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Judge supplied evidence against the gates for entering `target`.
|
|
114
|
+
* evidence: { [gateId]: { met: true, ref, digest } }. `ref` must equal the gate's own reference
|
|
115
|
+
* (a suite gate accepts only its closure step id); `digest` must be a sha256.
|
|
116
|
+
* -> { ok, unmet: [{ gate, reason }] }
|
|
117
|
+
*/
|
|
118
|
+
export function evaluateGates(feature, target, evidence = {}) {
|
|
119
|
+
const gates = gatesFor(feature, target);
|
|
120
|
+
if (!gates) return { ok: false, unmet: [{ gate: null, reason: `unknown feature '${feature}' or stage '${target}'` }] };
|
|
121
|
+
const unmet = [];
|
|
122
|
+
for (const g of gates) {
|
|
123
|
+
const e = evidence?.[g.id];
|
|
124
|
+
if (!e || e.met !== true) { unmet.push({ gate: g.id, reason: 'no evidence that the gate is met' }); continue; }
|
|
125
|
+
if (typeof e.digest !== 'string' || !DIGEST.test(e.digest)) { unmet.push({ gate: g.id, reason: 'evidence carries no sha256 digest' }); continue; }
|
|
126
|
+
if (g.kind === 'suite' && e.ref !== g.closureStep) { unmet.push({ gate: g.id, reason: `evidence is for '${e.ref}', not the required suite '${g.closureStep}'` }); continue; }
|
|
127
|
+
if (g.kind !== 'suite' && (typeof e.ref !== 'string' || !e.ref.trim())) { unmet.push({ gate: g.id, reason: 'evidence names no artifact' }); continue; }
|
|
128
|
+
}
|
|
129
|
+
return { ok: unmet.length === 0, unmet };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Build gate evidence from recorded closure steps (id, state, log.sha256). Only a step in state
|
|
134
|
+
* `pass` produces evidence; an unsupported, incomplete or failed step produces none.
|
|
135
|
+
*/
|
|
136
|
+
export function evidenceFromSteps(steps = []) {
|
|
137
|
+
const out = {};
|
|
138
|
+
for (const s of steps) {
|
|
139
|
+
if (s?.state !== 'pass' || !/^[0-9a-f]{64}$/.test(s.log?.sha256 || '')) continue;
|
|
140
|
+
out[`step:${s.id}`] = { met: true, ref: s.id, digest: s.log.sha256 };
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Fill suite-gate evidence for `feature`/`target` from closure-step evidence; other kinds come from `extra`. */
|
|
146
|
+
export function assembleEvidence(feature, target, stepEvidence = {}, extra = {}) {
|
|
147
|
+
const out = { ...extra };
|
|
148
|
+
for (const g of gatesFor(feature, target) || []) {
|
|
149
|
+
if (g.kind === 'suite' && stepEvidence[`step:${g.closureStep}`]) out[g.id] = stepEvidence[`step:${g.closureStep}`];
|
|
150
|
+
}
|
|
151
|
+
return out;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Move one feature up one stage. Returns a NEW ledger; the input is never mutated.
|
|
156
|
+
* Refuses: an unknown feature, skipping a stage, standing still, any unmet gate, and a feature the
|
|
157
|
+
* kill switch has stopped (config given) or an unsupported platform for opt-in.
|
|
158
|
+
*/
|
|
159
|
+
export function promote(ledger, feature, evidence, { config = null } = {}) {
|
|
160
|
+
const cur = ledger?.features?.[feature];
|
|
161
|
+
if (!cur) return { ok: false, ledger, blockers: [{ gate: null, reason: `unknown feature '${feature}'` }] };
|
|
162
|
+
const at = STAGES.indexOf(cur.stage);
|
|
163
|
+
if (at === STAGES.length - 1) return { ok: false, ledger, blockers: [{ gate: null, reason: 'already at the final stage' }] };
|
|
164
|
+
const target = STAGES[at + 1];
|
|
165
|
+
const blockers = [];
|
|
166
|
+
if (config) {
|
|
167
|
+
const st = featureStatus(config, feature);
|
|
168
|
+
if (st.code === 'kill-switch') blockers.push({ gate: null, reason: `the kill switch is set (${st.reason}); a stopped feature is not promoted` });
|
|
169
|
+
if (target === 'opt-in-supported' && st.code === 'platform-unsupported') blockers.push({ gate: null, reason: st.reason });
|
|
170
|
+
}
|
|
171
|
+
const g = evaluateGates(feature, target, evidence);
|
|
172
|
+
blockers.push(...g.unmet);
|
|
173
|
+
if (blockers.length) return { ok: false, ledger, blockers };
|
|
174
|
+
const next = clone(ledger);
|
|
175
|
+
const f = next.features[feature];
|
|
176
|
+
// Monotonic over the whole history: a version number is never reused after a rollback, so a record
|
|
177
|
+
// produced under a rolled-back policy can never be mistaken for one produced under a later policy.
|
|
178
|
+
const policyVersion = Math.max(...cur.history.map((h) => h.policyVersion)) + 1;
|
|
179
|
+
f.history.push({ seq: f.history.length, event: 'promote', from: cur.stage, to: target, policyVersion, knownGood: true, evidence: clone(evidence) });
|
|
180
|
+
f.stage = target; f.policyVersion = policyVersion;
|
|
181
|
+
return { ok: true, ledger: next, blockers: [] };
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* How a feature behaves given its configuration AND its stage. Below opt-in a feature that an operator
|
|
186
|
+
* enabled still only runs in the stage's mode, and says so.
|
|
187
|
+
* 'off' disabled, killed, invalid or unsupported: inert
|
|
188
|
+
* 'fixtures' offline-fixtures: the code path is fixtures-only; a real run is refused
|
|
189
|
+
* 'shadow' shadow-canary: run, record, never change an outcome
|
|
190
|
+
* 'active' opt-in-supported: results may change an outcome
|
|
191
|
+
*/
|
|
192
|
+
export function effectiveMode(config, ledger, feature) {
|
|
193
|
+
const st = featureStatus(config, feature);
|
|
194
|
+
if (st.status !== 'ok') return { mode: 'off', stage: ledger?.features?.[feature]?.stage ?? null, reason: st.reason, code: st.code };
|
|
195
|
+
const stage = ledger?.features?.[feature]?.stage;
|
|
196
|
+
if (!stage) return { mode: 'off', stage: null, reason: 'the feature has no rollout record, so it has not passed any gate', code: 'no-rollout-record' };
|
|
197
|
+
if (stage === 'offline-fixtures') return { mode: 'fixtures', stage, reason: 'enabled, but the feature has not passed the shadow-canary gates; it runs against fixtures only', code: 'below-stage' };
|
|
198
|
+
if (stage === 'shadow-canary') return { mode: 'shadow', stage, reason: 'enabled, but at shadow-canary: results are recorded and never change an outcome', code: 'below-stage' };
|
|
199
|
+
return { mode: 'active', stage, reason: 'enabled at opt-in-supported', code: null };
|
|
200
|
+
}
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
// case. Heuristic: check the test-files corpus for any test asserting
|
|
18
18
|
// a 401 / 403 status against the route's path.
|
|
19
19
|
|
|
20
|
+
import { inferredInvariant } from './invariants/lifecycle.js';
|
|
21
|
+
|
|
20
22
|
const JS_TS_RE = /\.(?:js|jsx|ts|tsx|mjs|cjs)$/i;
|
|
21
23
|
const PY_RE = /\.py$/i;
|
|
22
24
|
|
|
@@ -135,20 +137,25 @@ function emitAuthZMatrixFindings(matrix) {
|
|
|
135
137
|
|
|
136
138
|
// ─── FR-LOGIC-2 State machine extraction ──────────────────────────────────
|
|
137
139
|
|
|
138
|
-
function
|
|
140
|
+
function collectStateSets(fileContents) {
|
|
139
141
|
// Find sites where a literal set of statuses appears (e.g.
|
|
140
142
|
// `STATUSES = ['pending', 'approved', 'rejected']` or an enum-like in TS).
|
|
141
|
-
|
|
142
|
-
const stateSets = []; // [{file, line, name, values: Set<string>}]
|
|
143
|
+
const stateSets = []; // [{file, line, name, values: Set<string>, ordered: string[]}]
|
|
143
144
|
for (const [fp, c] of Object.entries(fileContents || {})) {
|
|
144
145
|
if (typeof c !== 'string' || !JS_TS_RE.test(fp)) continue;
|
|
145
146
|
const re = /\b(STATUSES|STATES|PHASES|ALLOWED_STATUSES|[A-Z_]+_STATUSES)\s*=\s*\[\s*((?:['"][^'"]+['"]\s*,?\s*){2,})\]/g;
|
|
146
147
|
let m;
|
|
147
148
|
while ((m = re.exec(c))) {
|
|
148
149
|
const values = [...m[2].matchAll(/['"]([^'"]+)['"]/g)].map(x => x[1]);
|
|
149
|
-
stateSets.push({ file: fp, line: _lineOf(c, m.index), name: m[1], values: new Set(values) });
|
|
150
|
+
stateSets.push({ file: fp, line: _lineOf(c, m.index), name: m[1], values: new Set(values), ordered: values });
|
|
150
151
|
}
|
|
151
152
|
}
|
|
153
|
+
return stateSets;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function extractStateMachine(fileContents) {
|
|
157
|
+
// Then look for direct writes like `.status = "<not in set>"` and flag.
|
|
158
|
+
const stateSets = collectStateSets(fileContents);
|
|
152
159
|
if (!stateSets.length) return [];
|
|
153
160
|
const findings = [];
|
|
154
161
|
for (const [fp, c] of Object.entries(fileContents || {})) {
|
|
@@ -236,4 +243,19 @@ export function scanBusinessLogic(fileContents) {
|
|
|
236
243
|
}
|
|
237
244
|
|
|
238
245
|
// For tests + the no-dead-modules check.
|
|
246
|
+
// X-402: a declared status list read as an inferred workflow contract. The declared order is the only evidence for the allowed
|
|
247
|
+
// transitions (each value to the next), which is a guess a reviewer must confirm, so the proposal carries a high uncertainty and is
|
|
248
|
+
// advisory until approved in the signed ledger.
|
|
249
|
+
export function mineWorkflowInvariants(fileContents) {
|
|
250
|
+
const out = [];
|
|
251
|
+
for (const set of collectStateSets(fileContents)) {
|
|
252
|
+
const states = set.ordered.filter((v) => /^[A-Za-z0-9_.:/-]{1,64}$/.test(v));
|
|
253
|
+
if (states.length < 2) continue;
|
|
254
|
+
const allowed = states.slice(0, -1).map((from, i) => ({ from, to: states[i + 1] }));
|
|
255
|
+
const inv = inferredInvariant({ class: 'workflow-order', source: 'state-machine', evidence: { file: set.file, line: set.line, set: set.name }, uncertainty: 0.7, action: 'advance', allowed: allowed.slice(0, 16) });
|
|
256
|
+
if (inv) out.push(inv);
|
|
257
|
+
}
|
|
258
|
+
return out;
|
|
259
|
+
}
|
|
260
|
+
|
|
239
261
|
export const _internals = { extractAuthZMatrix, extractStateMachine, findNegativeTestGaps };
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// Bake-off recipe: the manifest, the adapter contract and the claim guard (QA-008.AC02, QA-008.AC03).
|
|
2
|
+
//
|
|
3
|
+
// A bake-off compares this engine with other tools on one workload. The failure modes that make such a comparison worthless are
|
|
4
|
+
// all about what is silently different between the runs, so the recipe pins them in a manifest, and this module refuses to
|
|
5
|
+
// produce a comparative claim when any of them is not identical:
|
|
6
|
+
//
|
|
7
|
+
// workload the same target set, by digest, for every participant
|
|
8
|
+
// scope the same languages and families
|
|
9
|
+
// provider and model named per participant (an engine with no model says so)
|
|
10
|
+
// verification one standard for what counts as a confirmed defect
|
|
11
|
+
// limits the same time and spend ceilings
|
|
12
|
+
// failures a timeout or crash is a recorded failure, scored as a miss, never dropped
|
|
13
|
+
//
|
|
14
|
+
// Comparators are GENERIC SLOTS (`comparator-a`, `comparator-b`, ...). This repository names no other tool: the operator maps a
|
|
15
|
+
// slot to whatever they run. An adapter can declare itself `unavailable` (not installed, no licence, no provider access), and that
|
|
16
|
+
// participant is `not-evaluated`: it is not a zero, not a loss and not excluded quietly. It appears in the report with its
|
|
17
|
+
// reason. A comparative accuracy or cost claim exists only between participants that were actually evaluated on the identical
|
|
18
|
+
// workload under the identical standard, and only for a quantity both measured; otherwise the claim is withheld with the reasons.
|
|
19
|
+
//
|
|
20
|
+
// Scoring is delegated to comparison.js (CWE-only matching, intersection of completed entries), unchanged.
|
|
21
|
+
//
|
|
22
|
+
// Pure: no fs, no clock, no process execution. An adapter's `run` is supplied by the caller.
|
|
23
|
+
|
|
24
|
+
import { digestOf } from '../assurance/identity.js';
|
|
25
|
+
import { compareParticipants, scoreParticipant } from '../comparison.js';
|
|
26
|
+
|
|
27
|
+
const BAKEOFF_SCHEMA = 'agentic-security/bakeoff-manifest';
|
|
28
|
+
const ENGINE_SLOT = 'this-engine';
|
|
29
|
+
const SLOT_RE = /^(?:this-engine|comparator-[a-z])$/;
|
|
30
|
+
export const UNAVAILABLE_REASONS = Object.freeze(['not-installed', 'no-licence', 'no-provider-access', 'not-configured', 'platform-unsupported', 'operator-declined']);
|
|
31
|
+
|
|
32
|
+
const isObj = (x) => x && typeof x === 'object' && !Array.isArray(x);
|
|
33
|
+
const err = (path, message) => ({ code: 'BAD_MANIFEST', path, message });
|
|
34
|
+
|
|
35
|
+
/** @returns {{ok: boolean, errors: object[]}} */
|
|
36
|
+
export function validateBakeoffManifest(m) {
|
|
37
|
+
const errors = [];
|
|
38
|
+
if (!isObj(m)) return { ok: false, errors: [err('', 'manifest must be an object')] };
|
|
39
|
+
if (m.schema !== BAKEOFF_SCHEMA) errors.push(err('schema', `schema must be ${BAKEOFF_SCHEMA}`));
|
|
40
|
+
const w = m.workload;
|
|
41
|
+
if (!isObj(w) || !Array.isArray(w.targetIds) || !w.targetIds.length) errors.push(err('workload.targetIds', 'name the target ids the workload consists of'));
|
|
42
|
+
else {
|
|
43
|
+
if (!/^sha256:[0-9a-f]{64}$/.test(w.workloadDigest || '')) errors.push(err('workload.workloadDigest', 'the workload is pinned by digest'));
|
|
44
|
+
else if (w.workloadDigest !== workloadDigestOf(w)) errors.push(err('workload.workloadDigest', 'the digest does not match the targets and tree digests listed'));
|
|
45
|
+
}
|
|
46
|
+
if (!isObj(m.scope) || !Array.isArray(m.scope.languages) || !Array.isArray(m.scope.families)) errors.push(err('scope', 'declare the languages and families in scope'));
|
|
47
|
+
if (!isObj(m.verification) || !m.verification.standard) errors.push(err('verification.standard', 'one verification standard must be named (what counts as a confirmed defect)'));
|
|
48
|
+
const lim = m.limits;
|
|
49
|
+
if (!isObj(lim) || !(lim.perTargetTimeoutMs > 0) || !(lim.wallClockMs > 0) || typeof lim.spendCeilingUsd !== 'number') errors.push(err('limits', 'perTargetTimeoutMs, wallClockMs and spendCeilingUsd are all required'));
|
|
50
|
+
if (m.failurePolicy !== 'count-as-misses') errors.push(err('failurePolicy', 'failures are scored as misses; the only accepted policy is count-as-misses'));
|
|
51
|
+
const ps = m.participants;
|
|
52
|
+
if (!Array.isArray(ps) || ps.length < 2) errors.push(err('participants', 'a bake-off needs this engine and at least one comparator slot'));
|
|
53
|
+
else {
|
|
54
|
+
const seen = new Set();
|
|
55
|
+
ps.forEach((p, i) => {
|
|
56
|
+
if (!isObj(p) || !SLOT_RE.test(p.slot || '')) { errors.push(err(`participants[${i}].slot`, 'a slot is "this-engine" or "comparator-a".."comparator-z"; no product name belongs in a recipe')); return; }
|
|
57
|
+
if (seen.has(p.slot)) errors.push(err(`participants[${i}].slot`, `duplicate slot ${p.slot}`));
|
|
58
|
+
seen.add(p.slot);
|
|
59
|
+
if (!isObj(p.provider) || !('model' in p.provider)) errors.push(err(`participants[${i}].provider`, 'name the provider and model, or {"model": null} for a tool that uses none'));
|
|
60
|
+
});
|
|
61
|
+
if (!seen.has(ENGINE_SLOT)) errors.push(err('participants', 'this-engine must be one of the participants'));
|
|
62
|
+
}
|
|
63
|
+
return { ok: errors.length === 0, errors };
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The digest a workload is pinned by: its targets and their tree digests, nothing else. */
|
|
67
|
+
export function workloadDigestOf(workload) {
|
|
68
|
+
const targets = Object.fromEntries((workload.targetIds || []).slice().sort().map((id) => [id, workload.treeDigests?.[id] ?? null]));
|
|
69
|
+
return digestOf({ kind: 'bakeoff-workload', targets });
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Decide, per participant, whether it can be evaluated. `adapters` maps a slot to `{ available: boolean, reason?, run? }`.
|
|
74
|
+
* A slot with no adapter, or an adapter that says it is unavailable, is `not-evaluated` with a typed reason.
|
|
75
|
+
*/
|
|
76
|
+
export function resolveParticipants(manifest, adapters = {}) {
|
|
77
|
+
return manifest.participants.map((p) => {
|
|
78
|
+
const a = adapters[p.slot];
|
|
79
|
+
if (!a) return { slot: p.slot, provider: p.provider, state: 'not-evaluated', reason: 'not-configured', detail: 'no adapter was supplied for this slot' };
|
|
80
|
+
if (a.available === false) {
|
|
81
|
+
const reason = UNAVAILABLE_REASONS.includes(a.reason) ? a.reason : 'not-configured';
|
|
82
|
+
return { slot: p.slot, provider: p.provider, state: 'not-evaluated', reason, detail: a.detail || null };
|
|
83
|
+
}
|
|
84
|
+
return { slot: p.slot, provider: p.provider, state: 'evaluated', reason: null, detail: null };
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Judge a finished bake-off.
|
|
90
|
+
* @param {object} o
|
|
91
|
+
* @param {object} o.manifest a valid manifest
|
|
92
|
+
* @param {object[]} o.entries [{id, cwe}] the workload's answer key
|
|
93
|
+
* @param {Object<string, {workloadDigest: string, verificationStandard: string, results: object, costUsd?: number|null, completed?: number}>} o.runs by slot, for evaluated participants
|
|
94
|
+
* @param {Object<string, object>} [o.adapters]
|
|
95
|
+
*/
|
|
96
|
+
export function judgeBakeoff({ manifest, entries, runs = {}, adapters = {} }) {
|
|
97
|
+
const v = validateBakeoffManifest(manifest);
|
|
98
|
+
if (!v.ok) return { ok: false, errors: v.errors };
|
|
99
|
+
const resolved = resolveParticipants(manifest, adapters);
|
|
100
|
+
const participants = resolved.map((r) => {
|
|
101
|
+
if (r.state !== 'evaluated') return r;
|
|
102
|
+
const run = runs[r.slot];
|
|
103
|
+
if (!run) return { ...r, state: 'not-evaluated', reason: 'no-run-recorded', detail: 'the adapter was available but no result was recorded' };
|
|
104
|
+
return { ...r, run: { workloadDigest: run.workloadDigest, verificationStandard: run.verificationStandard, costUsd: run.costUsd ?? null } };
|
|
105
|
+
});
|
|
106
|
+
const evaluated = participants.filter((p) => p.state === 'evaluated');
|
|
107
|
+
const withheld = [];
|
|
108
|
+
for (const p of participants.filter((x) => x.state === 'not-evaluated')) withheld.push({ slot: p.slot, reason: `${p.slot} was not evaluated (${p.reason}); no comparison with it is made` });
|
|
109
|
+
|
|
110
|
+
// Comparability: every evaluated participant must have run the manifest's workload under the manifest's standard.
|
|
111
|
+
const mismatched = evaluated.filter((p) => p.run.workloadDigest !== manifest.workload.workloadDigest || p.run.verificationStandard !== manifest.verification.standard);
|
|
112
|
+
for (const p of mismatched) withheld.push({ slot: p.slot, reason: `${p.slot} ran a different workload or verification standard than the manifest pins; its figures are not comparable` });
|
|
113
|
+
const comparable = evaluated.filter((p) => !mismatched.includes(p));
|
|
114
|
+
|
|
115
|
+
let comparison = null;
|
|
116
|
+
const claims = [];
|
|
117
|
+
if (comparable.length >= 2) {
|
|
118
|
+
const parts = comparable.map((p) => ({ id: p.slot, results: runs[p.slot].results }));
|
|
119
|
+
comparison = compareParticipants(entries, parts);
|
|
120
|
+
if (!comparison.ok) { withheld.push({ slot: null, reason: `scoring refused: ${comparison.reason}` }); comparison = null; }
|
|
121
|
+
} else {
|
|
122
|
+
withheld.push({ slot: null, reason: 'fewer than two participants were evaluated on the identical workload; there is nothing to compare' });
|
|
123
|
+
}
|
|
124
|
+
// End-to-end view per participant: an entry it could not complete is a MISS here (the comparison above is over the shared
|
|
125
|
+
// intersection and is labelled as such). Both views are published; neither replaces the other.
|
|
126
|
+
const endToEnd = {};
|
|
127
|
+
for (const p of comparable) {
|
|
128
|
+
const sc = scoreParticipant(entries, runs[p.slot].results);
|
|
129
|
+
let detected = 0; for (const v of sc.per.values()) if (v.detected) detected++;
|
|
130
|
+
endToEnd[p.slot] = { entries: entries.length, completed: sc.completed, failedOrMissing: entries.length - sc.completed, detected, recallEndToEnd: entries.length ? detected / entries.length : null, rule: 'failures and missing results count as misses' };
|
|
131
|
+
}
|
|
132
|
+
if (comparison) {
|
|
133
|
+
// An accuracy claim is a statement of the measured figures over the shared intersection; it never says "better" for a tie or an
|
|
134
|
+
// interval overlap, and it needs every participant to have completed the shared entries.
|
|
135
|
+
claims.push({ kind: 'accuracy', basis: 'shared-intersection', participants: comparable.map((p) => p.slot), statement: 'figures over the entries every compared participant completed; see the comparison table' });
|
|
136
|
+
const costs = comparable.map((p) => ({ slot: p.slot, usd: runs[p.slot].costUsd ?? null }));
|
|
137
|
+
if (costs.every((c) => typeof c.usd === 'number')) claims.push({ kind: 'cost', participants: costs.map((c) => c.slot), costsUsd: Object.fromEntries(costs.map((c) => [c.slot, c.usd])), statement: 'measured spend per participant under the same limits' });
|
|
138
|
+
else withheld.push({ slot: null, reason: `no cost claim: ${costs.filter((c) => typeof c.usd !== 'number').map((c) => c.slot).join(', ')} did not report a measured cost` });
|
|
139
|
+
}
|
|
140
|
+
const report = {
|
|
141
|
+
schema: 'agentic-security/bakeoff-report', ok: true, errors: [], manifestDigest: digestOf(manifest),
|
|
142
|
+
workloadDigest: manifest.workload.workloadDigest, participants, comparison, endToEnd, claims, withheld,
|
|
143
|
+
superiority: 'none claimed',
|
|
144
|
+
note: 'a participant that was not evaluated is reported as such; it is never a zero, a loss or a silent omission',
|
|
145
|
+
};
|
|
146
|
+
report.reportDigest = digestOf({ ...report, reportDigest: undefined });
|
|
147
|
+
return report;
|
|
148
|
+
}
|