assertledger 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONTRIBUTING.md +31 -0
- package/LICENSE +21 -0
- package/README.fr.md +236 -0
- package/README.md +224 -0
- package/SECURITY.md +51 -0
- package/benchmarks/agentic-profile/README.md +15 -0
- package/benchmarks/agentic-profile/public/README.md +5 -0
- package/benchmarks/self-hosted-core/README.md +113 -0
- package/benchmarks/self-hosted-core/adapter.mjs +293 -0
- package/benchmarks/self-hosted-core/builder.ts +193 -0
- package/benchmarks/self-hosted-core/campaign.ts +233 -0
- package/benchmarks/self-hosted-core/existing-tests-builder.ts +217 -0
- package/benchmarks/self-hosted-core/existing-tests.ts +146 -0
- package/benchmarks/self-hosted-core/liveness.test.mjs +8 -0
- package/conformance/v1/bundle.json +104 -0
- package/conformance/v1/expected/canonical-order-a.json +4 -0
- package/conformance/v1/expected/canonical-order-b.json +4 -0
- package/conformance/v1/expected/create-benchmark-v1-measured.json +575 -0
- package/conformance/v1/expected/create-profile-v1-qualified.json +280 -0
- package/conformance/v1/expected/decide-collection-failure-non-kill.json +192 -0
- package/conformance/v1/expected/decide-compile-failure-non-kill.json +192 -0
- package/conformance/v1/expected/decide-infra-error-non-kill.json +192 -0
- package/conformance/v1/expected/decide-no-test-discovered-non-kill.json +192 -0
- package/conformance/v1/expected/decide-process-crash-non-kill.json +192 -0
- package/conformance/v1/expected/decide-timeout-non-kill.json +192 -0
- package/conformance/v1/expected/decide-verified.json +192 -0
- package/conformance/v1/expected/replay-benchmark-v1-resealed-summary-forgery.json +12 -0
- package/conformance/v1/expected/replay-evidence-raw-tamper.json +6 -0
- package/conformance/v1/expected/replay-evidence-resealed-semantic-forgery.json +6 -0
- package/conformance/v1/inputs/canonical-order-a.json +8 -0
- package/conformance/v1/inputs/canonical-order-b.json +8 -0
- package/conformance/v1/inputs/create-benchmark-v1-measured.json +459 -0
- package/conformance/v1/inputs/create-profile-v1-qualified.json +228 -0
- package/conformance/v1/inputs/decide-collection-failure-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-compile-failure-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-infra-error-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-no-test-discovered-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-process-crash-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-timeout-non-kill.json +143 -0
- package/conformance/v1/inputs/decide-verified.json +143 -0
- package/conformance/v1/inputs/replay-benchmark-v1-resealed-summary-forgery.json +575 -0
- package/conformance/v1/inputs/replay-evidence-raw-tamper.json +201 -0
- package/conformance/v1/inputs/replay-evidence-resealed-semantic-forgery.json +192 -0
- package/conformance/v1/schemas/expected-digests.json +175 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +951 -0
- package/dist/cli.js.map +1 -0
- package/dist/contracts/diagnostics.d.ts +18 -0
- package/dist/contracts/diagnostics.d.ts.map +1 -0
- package/dist/contracts/diagnostics.js +13 -0
- package/dist/contracts/diagnostics.js.map +1 -0
- package/dist/contracts/index.d.ts +3908 -0
- package/dist/contracts/index.d.ts.map +1 -0
- package/dist/contracts/index.js +2569 -0
- package/dist/contracts/index.js.map +1 -0
- package/dist/contracts/runtime-doctor.d.ts +107 -0
- package/dist/contracts/runtime-doctor.d.ts.map +1 -0
- package/dist/contracts/runtime-doctor.js +91 -0
- package/dist/contracts/runtime-doctor.js.map +1 -0
- package/dist/core/index.d.ts +200 -0
- package/dist/core/index.d.ts.map +1 -0
- package/dist/core/index.js +2587 -0
- package/dist/core/index.js.map +1 -0
- package/dist/diagnostics.d.ts +7 -0
- package/dist/diagnostics.d.ts.map +1 -0
- package/dist/diagnostics.js +252 -0
- package/dist/diagnostics.js.map +1 -0
- package/dist/engine/adapters/node-test-profile.d.ts +14 -0
- package/dist/engine/adapters/node-test-profile.d.ts.map +1 -0
- package/dist/engine/adapters/node-test-profile.js +14 -0
- package/dist/engine/adapters/node-test-profile.js.map +1 -0
- package/dist/engine/adapters/node-test-runtime.d.ts +39 -0
- package/dist/engine/adapters/node-test-runtime.d.ts.map +1 -0
- package/dist/engine/adapters/node-test-runtime.js +173 -0
- package/dist/engine/adapters/node-test-runtime.js.map +1 -0
- package/dist/engine/adapters/runtime-facts.d.ts +26 -0
- package/dist/engine/adapters/runtime-facts.d.ts.map +1 -0
- package/dist/engine/adapters/runtime-facts.js +73 -0
- package/dist/engine/adapters/runtime-facts.js.map +1 -0
- package/dist/engine/connection.d.ts +22 -0
- package/dist/engine/connection.d.ts.map +1 -0
- package/dist/engine/connection.js +343 -0
- package/dist/engine/connection.js.map +1 -0
- package/dist/engine/git-regression.d.ts +25 -0
- package/dist/engine/git-regression.d.ts.map +1 -0
- package/dist/engine/git-regression.js +803 -0
- package/dist/engine/git-regression.js.map +1 -0
- package/dist/engine/index.d.ts +55 -0
- package/dist/engine/index.d.ts.map +1 -0
- package/dist/engine/index.js +2782 -0
- package/dist/engine/index.js.map +1 -0
- package/dist/engine/node-test-reporter.d.ts +2 -0
- package/dist/engine/node-test-reporter.d.ts.map +1 -0
- package/dist/engine/node-test-reporter.js +70 -0
- package/dist/engine/node-test-reporter.js.map +1 -0
- package/dist/engine/runtime-doctor.d.ts +16 -0
- package/dist/engine/runtime-doctor.d.ts.map +1 -0
- package/dist/engine/runtime-doctor.js +100 -0
- package/dist/engine/runtime-doctor.js.map +1 -0
- package/dist/evaluation/agentic-corpus.d.ts +161 -0
- package/dist/evaluation/agentic-corpus.d.ts.map +1 -0
- package/dist/evaluation/agentic-corpus.js +710 -0
- package/dist/evaluation/agentic-corpus.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/mcp/index.d.ts +13 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +391 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/stdio.d.ts +3 -0
- package/dist/mcp/stdio.d.ts.map +1 -0
- package/dist/mcp/stdio.js +13 -0
- package/dist/mcp/stdio.js.map +1 -0
- package/dist/sdk/index.d.ts +52 -0
- package/dist/sdk/index.d.ts.map +1 -0
- package/dist/sdk/index.js +224 -0
- package/dist/sdk/index.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +10 -0
- package/dist/version.js.map +1 -0
- package/docs/adapter-protocol.md +196 -0
- package/docs/agentic-benchmark.md +118 -0
- package/docs/agentic-corpus-experiment-h3.md +89 -0
- package/docs/agentic-corpus-plan.md +105 -0
- package/docs/agentic-corpus-provenance.md +59 -0
- package/docs/agentic-test-profile-pilot.md +57 -0
- package/docs/agentic-test-profile-v2.md +116 -0
- package/docs/agentic-test-profile.md +274 -0
- package/docs/architecture.md +157 -0
- package/docs/ci.md +37 -0
- package/docs/client-connections.md +61 -0
- package/docs/conformance-v1.md +72 -0
- package/docs/decisions/0001-typescript-runtime.md +24 -0
- package/docs/developer-experience.md +55 -0
- package/docs/diagnostics.md +35 -0
- package/docs/distribution.md +40 -0
- package/docs/git-regression.md +39 -0
- package/docs/migration-repository-validation-order.md +35 -0
- package/docs/migration-testforge-to-assertledger.md +64 -0
- package/docs/project-intent.md +173 -0
- package/docs/proof-model.md +116 -0
- package/docs/reference.md +336 -0
- package/docs/release-1.0.md +63 -0
- package/docs/repository-audit.md +52 -0
- package/docs/repository-init.md +60 -0
- package/docs/research-basis.md +27 -0
- package/docs/roadmap.md +74 -0
- package/docs/runtime-doctor.md +65 -0
- package/docs/testexplora-calibration.md +71 -0
- package/examples/agentic-benchmark/benchmark-request.mjs +19 -0
- package/examples/agentic-benchmark/structured-phase-adapter-fixture.mjs +35 -0
- package/examples/agentic-profile/profile-benchmark.mjs +34 -0
- package/examples/agentic-profile/profile-manifest.mjs +28 -0
- package/examples/git-history/README.md +44 -0
- package/examples/git-history/create-demo.mjs +128 -0
- package/examples/git-history/escape-string-regexp/LICENSE +9 -0
- package/examples/git-history/escape-string-regexp/before.cjs.txt +11 -0
- package/examples/git-history/escape-string-regexp/fixed.cjs.txt +13 -0
- package/examples/git-history/escape-string-regexp/provenance.json +28 -0
- package/examples/node-test/repository/package.json +5 -0
- package/examples/node-test/repository/src/is-even.js +3 -0
- package/examples/node-test/repository/tests/base.test.js +6 -0
- package/examples/node-test/request.json +93 -0
- package/integrations/skill/SKILL.md +51 -0
- package/package.json +88 -0
- package/schemas/agentic-benchmark-acquisition-replay-result.v1.json +70 -0
- package/schemas/agentic-benchmark-acquisition-request.v1.json +564 -0
- package/schemas/agentic-benchmark-acquisition-result.v1.json +1409 -0
- package/schemas/agentic-benchmark-artifact.v1.json +1251 -0
- package/schemas/agentic-benchmark-replay-result.v1.json +84 -0
- package/schemas/agentic-benchmark-request.v1.json +1034 -0
- package/schemas/agentic-corpus-allocation-commitment-replay-result.v1.json +58 -0
- package/schemas/agentic-corpus-allocation-commitment.v1.json +141 -0
- package/schemas/agentic-corpus-allocation-replay-result.v1.json +34 -0
- package/schemas/agentic-corpus-allocation-request.v1.json +65 -0
- package/schemas/agentic-corpus-allocation-reveal.v1.json +66 -0
- package/schemas/agentic-corpus-allocation.v1.json +167 -0
- package/schemas/agentic-corpus-experiment-artifact.v1.json +329 -0
- package/schemas/agentic-corpus-experiment-plan-replay-result.v1.json +50 -0
- package/schemas/agentic-corpus-experiment-plan.v1.json +424 -0
- package/schemas/agentic-corpus-experiment-replay-request.v1.json +336 -0
- package/schemas/agentic-corpus-experiment-replay-result.v1.json +106 -0
- package/schemas/agentic-corpus-experiment-request.v1.json +204 -0
- package/schemas/agentic-corpus-provenance.v1.json +143 -0
- package/schemas/agentic-corpus-trust-policy.v1.json +133 -0
- package/schemas/agentic-profile-replay-result.v1.json +56 -0
- package/schemas/agentic-profile-replay-result.v2.json +63 -0
- package/schemas/agentic-profile-report.v1.json +961 -0
- package/schemas/agentic-profile-report.v2.json +1674 -0
- package/schemas/agentic-profile-request.v1.json +671 -0
- package/schemas/agentic-profile-request.v2.json +1338 -0
- package/schemas/evidence-manifest.v1.json +636 -0
- package/schemas/replay-result.v1.json +49 -0
- package/schemas/repository-analysis.v1.json +119 -0
- package/schemas/repository-audit.v1.json +811 -0
- package/schemas/repository-init-config.v1.json +183 -0
- package/schemas/repository-init-lock.v1.json +162 -0
- package/schemas/repository-init-result.v1.json +212 -0
- package/schemas/verification-request.v1.json +389 -0
|
@@ -0,0 +1,2587 @@
|
|
|
1
|
+
import { parseAgenticBenchmarkArtifact, parseAgenticBenchmarkAcquisitionResult, parseAgenticBenchmarkRequest, parseAgenticCorpusAllocation, parseAgenticCorpusAllocationCommitment, parseAgenticCorpusAllocationReveal, parseAgenticCorpusAllocationRequest, parseAgenticCorpusExperimentArtifact, parseAgenticCorpusExperimentPlan, parseAgenticCorpusExperimentReplayRequest, parseAgenticCorpusExperimentReceipt, parseAgenticCorpusExperimentRequest, parseAgenticCorpusExperimentStructuredResult, parseAgenticCorpusTrustPolicy, parseAgenticProfileReport, parseAgenticProfileReportV2, parseAgenticProfileRequest, parseAgenticProfileRequestV2, } from "../contracts/index.js";
|
|
2
|
+
const EVIDENCE_SCHEMA_VERSION = "1.0.0";
|
|
3
|
+
const EVIDENCE_POLICY_VERSION = "1.0.0";
|
|
4
|
+
const SHA256_DIGEST_PATTERN = /^sha256:[a-f0-9]{64}$/u;
|
|
5
|
+
const OBSERVATION_OUTCOMES = new Set([
|
|
6
|
+
"PASS",
|
|
7
|
+
"ASSERTION_FAILURE",
|
|
8
|
+
"COLLECTION_FAILURE",
|
|
9
|
+
"COMPILE_FAILURE",
|
|
10
|
+
"PROCESS_CRASH",
|
|
11
|
+
"TIMEOUT",
|
|
12
|
+
"INFRA_ERROR",
|
|
13
|
+
"NO_TEST_DISCOVERED",
|
|
14
|
+
]);
|
|
15
|
+
export class EvidenceValidationError extends Error {
|
|
16
|
+
constructor(message, options) {
|
|
17
|
+
super(message, options);
|
|
18
|
+
this.name = "EvidenceValidationError";
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
function canonicalJson(value, ancestors) {
|
|
22
|
+
if (value === null)
|
|
23
|
+
return "null";
|
|
24
|
+
if (typeof value === "string" || typeof value === "boolean") {
|
|
25
|
+
return JSON.stringify(value);
|
|
26
|
+
}
|
|
27
|
+
if (typeof value === "number") {
|
|
28
|
+
if (!Number.isFinite(value))
|
|
29
|
+
throw new TypeError("Canonical JSON requires finite numbers");
|
|
30
|
+
return JSON.stringify(value);
|
|
31
|
+
}
|
|
32
|
+
if (typeof value === "undefined") {
|
|
33
|
+
throw new TypeError("undefined is not a canonical JSON value");
|
|
34
|
+
}
|
|
35
|
+
if (typeof value !== "object") {
|
|
36
|
+
throw new TypeError(`Unsupported canonical JSON value: ${typeof value}`);
|
|
37
|
+
}
|
|
38
|
+
if (ancestors.has(value))
|
|
39
|
+
throw new TypeError("Canonical JSON cannot contain cycles");
|
|
40
|
+
ancestors.add(value);
|
|
41
|
+
try {
|
|
42
|
+
if (Array.isArray(value)) {
|
|
43
|
+
const items = [];
|
|
44
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
45
|
+
if (!(index in value))
|
|
46
|
+
throw new TypeError("Canonical JSON arrays cannot contain holes");
|
|
47
|
+
items.push(canonicalJson(value[index], ancestors));
|
|
48
|
+
}
|
|
49
|
+
return `[${items.join(",")}]`;
|
|
50
|
+
}
|
|
51
|
+
const prototype = Object.getPrototypeOf(value);
|
|
52
|
+
if (prototype !== Object.prototype && prototype !== null) {
|
|
53
|
+
throw new TypeError("Canonical JSON requires plain objects");
|
|
54
|
+
}
|
|
55
|
+
const record = value;
|
|
56
|
+
const keys = Object.keys(record).sort(compareOrdinal);
|
|
57
|
+
const entries = keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key], ancestors)}`);
|
|
58
|
+
return `{${entries.join(",")}}`;
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
ancestors.delete(value);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
export function canonicalize(value) {
|
|
65
|
+
return canonicalJson(value, new Set());
|
|
66
|
+
}
|
|
67
|
+
function rotateRight(value, amount) {
|
|
68
|
+
return (value >>> amount) | (value << (32 - amount));
|
|
69
|
+
}
|
|
70
|
+
const SHA256_CONSTANTS = [
|
|
71
|
+
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
|
|
72
|
+
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
|
|
73
|
+
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
|
|
74
|
+
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
|
|
75
|
+
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
|
|
76
|
+
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
|
|
77
|
+
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
|
|
78
|
+
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
|
|
79
|
+
];
|
|
80
|
+
function sha256(input) {
|
|
81
|
+
const bytes = typeof input === "string" ? new TextEncoder().encode(input) : input;
|
|
82
|
+
const bitLength = bytes.length * 8;
|
|
83
|
+
const paddedLength = Math.ceil((bytes.length + 9) / 64) * 64;
|
|
84
|
+
const padded = new Uint8Array(paddedLength);
|
|
85
|
+
padded.set(bytes);
|
|
86
|
+
padded[bytes.length] = 0x80;
|
|
87
|
+
const view = new DataView(padded.buffer);
|
|
88
|
+
view.setUint32(paddedLength - 8, Math.floor(bitLength / 0x1_0000_0000), false);
|
|
89
|
+
view.setUint32(paddedLength - 4, bitLength >>> 0, false);
|
|
90
|
+
const state = [
|
|
91
|
+
0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19,
|
|
92
|
+
];
|
|
93
|
+
const words = new Uint32Array(64);
|
|
94
|
+
for (let offset = 0; offset < paddedLength; offset += 64) {
|
|
95
|
+
for (let index = 0; index < 16; index += 1) {
|
|
96
|
+
words[index] = view.getUint32(offset + index * 4, false);
|
|
97
|
+
}
|
|
98
|
+
for (let index = 16; index < 64; index += 1) {
|
|
99
|
+
const previous15 = words[index - 15] ?? 0;
|
|
100
|
+
const previous2 = words[index - 2] ?? 0;
|
|
101
|
+
const sigma0 = rotateRight(previous15, 7) ^ rotateRight(previous15, 18) ^ (previous15 >>> 3);
|
|
102
|
+
const sigma1 = rotateRight(previous2, 17) ^ rotateRight(previous2, 19) ^ (previous2 >>> 10);
|
|
103
|
+
words[index] = ((words[index - 16] ?? 0) + sigma0 + (words[index - 7] ?? 0) + sigma1) >>> 0;
|
|
104
|
+
}
|
|
105
|
+
let [a, b, c, d, e, f, g, h] = state;
|
|
106
|
+
for (let index = 0; index < 64; index += 1) {
|
|
107
|
+
const sum1 = rotateRight(e, 6) ^ rotateRight(e, 11) ^ rotateRight(e, 25);
|
|
108
|
+
const choice = (e & f) ^ (~e & g);
|
|
109
|
+
const temporary1 = (h + sum1 + choice + (SHA256_CONSTANTS[index] ?? 0) + (words[index] ?? 0)) >>> 0;
|
|
110
|
+
const sum0 = rotateRight(a, 2) ^ rotateRight(a, 13) ^ rotateRight(a, 22);
|
|
111
|
+
const majority = (a & b) ^ (a & c) ^ (b & c);
|
|
112
|
+
const temporary2 = (sum0 + majority) >>> 0;
|
|
113
|
+
h = g;
|
|
114
|
+
g = f;
|
|
115
|
+
f = e;
|
|
116
|
+
e = (d + temporary1) >>> 0;
|
|
117
|
+
d = c;
|
|
118
|
+
c = b;
|
|
119
|
+
b = a;
|
|
120
|
+
a = (temporary1 + temporary2) >>> 0;
|
|
121
|
+
}
|
|
122
|
+
const next = [a, b, c, d, e, f, g, h];
|
|
123
|
+
for (let index = 0; index < 8; index += 1) {
|
|
124
|
+
state[index] = ((state[index] ?? 0) + (next[index] ?? 0)) >>> 0;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
return state.map((word) => word.toString(16).padStart(8, "0")).join("");
|
|
128
|
+
}
|
|
129
|
+
export function sha256Canonical(value) {
|
|
130
|
+
return `sha256:${sha256(canonicalize(value))}`;
|
|
131
|
+
}
|
|
132
|
+
function isRecord(value) {
|
|
133
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
134
|
+
}
|
|
135
|
+
function stringField(record, key) {
|
|
136
|
+
const value = record[key];
|
|
137
|
+
if (typeof value !== "string" || value.length === 0)
|
|
138
|
+
throw new TypeError(`${key} must be a non-empty string`);
|
|
139
|
+
return value;
|
|
140
|
+
}
|
|
141
|
+
function boundedStringField(record, key, maximum) {
|
|
142
|
+
const value = stringField(record, key);
|
|
143
|
+
if (value.length > maximum)
|
|
144
|
+
throw new TypeError(`${key} must not exceed ${maximum} characters`);
|
|
145
|
+
return value;
|
|
146
|
+
}
|
|
147
|
+
function digestField(record, key) {
|
|
148
|
+
const value = stringField(record, key);
|
|
149
|
+
if (!SHA256_DIGEST_PATTERN.test(value))
|
|
150
|
+
throw new TypeError(`${key} must be a SHA-256 digest`);
|
|
151
|
+
return value;
|
|
152
|
+
}
|
|
153
|
+
function identifierField(record, key) {
|
|
154
|
+
const value = stringField(record, key);
|
|
155
|
+
if (value.length > 128 || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/u.test(value)) {
|
|
156
|
+
throw new TypeError(`${key} must be a portable ASCII identifier`);
|
|
157
|
+
}
|
|
158
|
+
return value;
|
|
159
|
+
}
|
|
160
|
+
function integerField(record, key, minimum = 0) {
|
|
161
|
+
const value = record[key];
|
|
162
|
+
if (!Number.isSafeInteger(value) || value < minimum)
|
|
163
|
+
throw new TypeError(`${key} must be an integer >= ${minimum}`);
|
|
164
|
+
return value;
|
|
165
|
+
}
|
|
166
|
+
function compareOrdinal(left, right) {
|
|
167
|
+
return left < right ? -1 : left > right ? 1 : 0;
|
|
168
|
+
}
|
|
169
|
+
function jsonField(record, key) {
|
|
170
|
+
const value = record[key];
|
|
171
|
+
canonicalize(value);
|
|
172
|
+
return JSON.parse(JSON.stringify(value));
|
|
173
|
+
}
|
|
174
|
+
function sortedUniqueStrings(record, key, maximumItems, maximumLength) {
|
|
175
|
+
const value = record[key];
|
|
176
|
+
if (!Array.isArray(value) ||
|
|
177
|
+
value.length > maximumItems ||
|
|
178
|
+
value.some((item) => typeof item !== "string" || item.length === 0 || item.length > maximumLength)) {
|
|
179
|
+
throw new TypeError(`${key} exceeds its string-array bounds`);
|
|
180
|
+
}
|
|
181
|
+
return [...new Set(value)].sort(compareOrdinal);
|
|
182
|
+
}
|
|
183
|
+
function parseEvidenceContext(value) {
|
|
184
|
+
if (!isRecord(value))
|
|
185
|
+
throw new TypeError("evidenceContext must be an object");
|
|
186
|
+
if (!isRecord(value.engine))
|
|
187
|
+
throw new TypeError("evidenceContext.engine must be an object");
|
|
188
|
+
if (!isRecord(value.adapter))
|
|
189
|
+
throw new TypeError("evidenceContext.adapter must be an object");
|
|
190
|
+
if (!isRecord(value.execution)) {
|
|
191
|
+
throw new TypeError("evidenceContext.execution must be an object");
|
|
192
|
+
}
|
|
193
|
+
if (!Array.isArray(value.worlds) || value.worlds.length === 0) {
|
|
194
|
+
throw new TypeError("evidenceContext.worlds must be non-empty");
|
|
195
|
+
}
|
|
196
|
+
const worlds = value.worlds
|
|
197
|
+
.map((raw) => {
|
|
198
|
+
if (!isRecord(raw))
|
|
199
|
+
throw new TypeError("evidenceContext world must be an object");
|
|
200
|
+
return {
|
|
201
|
+
id: identifierField(raw, "id"),
|
|
202
|
+
provenance: boundedStringField(raw, "provenance", 1_024),
|
|
203
|
+
digest: digestField(raw, "digest"),
|
|
204
|
+
};
|
|
205
|
+
})
|
|
206
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
207
|
+
if (new Set(worlds.map((world) => world.id)).size !== worlds.length) {
|
|
208
|
+
throw new TypeError("Duplicate evidenceContext world id");
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
engine: {
|
|
212
|
+
name: boundedStringField(value.engine, "name", 128),
|
|
213
|
+
version: boundedStringField(value.engine, "version", 128),
|
|
214
|
+
},
|
|
215
|
+
adapter: {
|
|
216
|
+
name: boundedStringField(value.adapter, "name", 128),
|
|
217
|
+
version: boundedStringField(value.adapter, "version", 128),
|
|
218
|
+
configuration: jsonField(value.adapter, "configuration"),
|
|
219
|
+
},
|
|
220
|
+
execution: {
|
|
221
|
+
isolation: boundedStringField(value.execution, "isolation", 128),
|
|
222
|
+
environmentAllowlist: sortedUniqueStrings(value.execution, "environmentAllowlist", 100, 128),
|
|
223
|
+
budgets: jsonField(value.execution, "budgets"),
|
|
224
|
+
candidateRoots: sortedUniqueStrings(value.execution, "candidateRoots", 100, 512),
|
|
225
|
+
},
|
|
226
|
+
worlds,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function parseInputUnchecked(input) {
|
|
230
|
+
if (!isRecord(input))
|
|
231
|
+
throw new TypeError("Evidence must be an object");
|
|
232
|
+
const schemaVersion = stringField(input, "schemaVersion");
|
|
233
|
+
if (schemaVersion !== EVIDENCE_SCHEMA_VERSION) {
|
|
234
|
+
throw new TypeError(`Unsupported schemaVersion: ${schemaVersion}`);
|
|
235
|
+
}
|
|
236
|
+
const policyValue = input.policy;
|
|
237
|
+
if (!isRecord(policyValue))
|
|
238
|
+
throw new TypeError("policy must be an object");
|
|
239
|
+
const policyVersion = stringField(policyValue, "policyVersion");
|
|
240
|
+
if (policyVersion !== EVIDENCE_POLICY_VERSION) {
|
|
241
|
+
throw new TypeError(`Unsupported policyVersion: ${policyVersion}`);
|
|
242
|
+
}
|
|
243
|
+
const attempts = integerField(policyValue, "requiredAttempts", 1);
|
|
244
|
+
if (attempts > 1_000)
|
|
245
|
+
throw new TypeError("requiredAttempts must not exceed 1000");
|
|
246
|
+
const minimum = integerField(policyValue, "minimumTargetWeightPermille", 1);
|
|
247
|
+
if (minimum > 1_000)
|
|
248
|
+
throw new TypeError("minimumTargetWeightPermille must not exceed 1000");
|
|
249
|
+
const acceptedValue = policyValue.acceptedTargetOutcomes;
|
|
250
|
+
if (!Array.isArray(acceptedValue) ||
|
|
251
|
+
acceptedValue.length !== 1 ||
|
|
252
|
+
acceptedValue[0] !== "ASSERTION_FAILURE") {
|
|
253
|
+
throw new TypeError("acceptedTargetOutcomes must be exactly [ASSERTION_FAILURE]");
|
|
254
|
+
}
|
|
255
|
+
const policy = {
|
|
256
|
+
policyVersion,
|
|
257
|
+
requiredAttempts: attempts,
|
|
258
|
+
minimumTargetWeightPermille: minimum,
|
|
259
|
+
maximumSelectedCandidates: integerField(policyValue, "maximumSelectedCandidates", 1),
|
|
260
|
+
acceptedTargetOutcomes: ["ASSERTION_FAILURE"],
|
|
261
|
+
};
|
|
262
|
+
if (policy.maximumSelectedCandidates > 1_000) {
|
|
263
|
+
throw new TypeError("maximumSelectedCandidates must not exceed 1000");
|
|
264
|
+
}
|
|
265
|
+
if (!Array.isArray(input.worlds) || input.worlds.length === 0)
|
|
266
|
+
throw new TypeError("worlds must be non-empty");
|
|
267
|
+
const worlds = input.worlds
|
|
268
|
+
.map((raw) => {
|
|
269
|
+
if (!isRecord(raw))
|
|
270
|
+
throw new TypeError("world must be an object");
|
|
271
|
+
const kind = raw.kind;
|
|
272
|
+
if (kind !== "REFERENCE" && kind !== "TARGET" && kind !== "NEUTRAL")
|
|
273
|
+
throw new TypeError("Invalid world kind");
|
|
274
|
+
if (typeof raw.required !== "boolean")
|
|
275
|
+
throw new TypeError("world.required must be boolean");
|
|
276
|
+
const world = {
|
|
277
|
+
id: identifierField(raw, "id"),
|
|
278
|
+
kind,
|
|
279
|
+
required: raw.required,
|
|
280
|
+
weight: integerField(raw, "weight"),
|
|
281
|
+
};
|
|
282
|
+
if (kind === "TARGET" && world.weight < 1) {
|
|
283
|
+
throw new TypeError("TARGET world weight must be at least 1");
|
|
284
|
+
}
|
|
285
|
+
if (kind !== "TARGET" && world.weight !== 0) {
|
|
286
|
+
throw new TypeError("Non-target world weight must be 0");
|
|
287
|
+
}
|
|
288
|
+
if (world.weight > 1_000_000)
|
|
289
|
+
throw new TypeError("world weight must not exceed 1000000");
|
|
290
|
+
return world;
|
|
291
|
+
})
|
|
292
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
293
|
+
if (new Set(worlds.map((world) => world.id)).size !== worlds.length)
|
|
294
|
+
throw new TypeError("Duplicate world id");
|
|
295
|
+
for (const kind of ["REFERENCE", "TARGET", "NEUTRAL"]) {
|
|
296
|
+
if (!worlds.some((world) => world.kind === kind && world.required)) {
|
|
297
|
+
throw new TypeError(`At least one required ${kind} world is required`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const worldIds = new Set(worlds.map((world) => world.id));
|
|
301
|
+
const evidenceContext = parseEvidenceContext(input.evidenceContext);
|
|
302
|
+
if (evidenceContext.worlds.length !== worlds.length ||
|
|
303
|
+
evidenceContext.worlds.some((world) => !worldIds.has(world.id))) {
|
|
304
|
+
throw new TypeError("evidenceContext worlds must exactly match decision worlds");
|
|
305
|
+
}
|
|
306
|
+
if (!Array.isArray(input.candidates) || input.candidates.length === 0) {
|
|
307
|
+
throw new TypeError("candidates must be non-empty");
|
|
308
|
+
}
|
|
309
|
+
const candidates = input.candidates
|
|
310
|
+
.map((raw) => {
|
|
311
|
+
if (!isRecord(raw))
|
|
312
|
+
throw new TypeError("candidate must be an object");
|
|
313
|
+
return {
|
|
314
|
+
id: identifierField(raw, "id"),
|
|
315
|
+
digest: digestField(raw, "digest"),
|
|
316
|
+
sizeBytes: integerField(raw, "sizeBytes"),
|
|
317
|
+
};
|
|
318
|
+
})
|
|
319
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
320
|
+
if (new Set(candidates.map((candidate) => candidate.id)).size !== candidates.length)
|
|
321
|
+
throw new TypeError("Duplicate candidate id");
|
|
322
|
+
const candidateIds = new Set(candidates.map((candidate) => candidate.id));
|
|
323
|
+
if (!Array.isArray(input.observations))
|
|
324
|
+
throw new TypeError("observations must be an array");
|
|
325
|
+
const observations = input.observations
|
|
326
|
+
.map((raw) => {
|
|
327
|
+
if (!isRecord(raw))
|
|
328
|
+
throw new TypeError("observation must be an object");
|
|
329
|
+
const candidateId = raw.candidateId;
|
|
330
|
+
if (candidateId !== null &&
|
|
331
|
+
(typeof candidateId !== "string" || !candidateIds.has(candidateId)))
|
|
332
|
+
throw new TypeError("Unknown candidateId");
|
|
333
|
+
const worldId = stringField(raw, "worldId");
|
|
334
|
+
if (!worldIds.has(worldId))
|
|
335
|
+
throw new TypeError("Unknown worldId");
|
|
336
|
+
if (typeof raw.attributed !== "boolean")
|
|
337
|
+
throw new TypeError("attributed must be boolean");
|
|
338
|
+
const duration = raw.durationMs;
|
|
339
|
+
if (duration !== undefined &&
|
|
340
|
+
(typeof duration !== "number" || !Number.isFinite(duration) || duration < 0))
|
|
341
|
+
throw new TypeError("durationMs must be finite and non-negative");
|
|
342
|
+
const exitCode = raw.exitCode;
|
|
343
|
+
if (exitCode !== undefined &&
|
|
344
|
+
exitCode !== null &&
|
|
345
|
+
(!Number.isSafeInteger(exitCode) || exitCode < 0)) {
|
|
346
|
+
throw new TypeError("exitCode must be null or a non-negative integer");
|
|
347
|
+
}
|
|
348
|
+
const stdoutDigest = raw.stdoutDigest === undefined ? undefined : digestField(raw, "stdoutDigest");
|
|
349
|
+
const stderrDigest = raw.stderrDigest === undefined ? undefined : digestField(raw, "stderrDigest");
|
|
350
|
+
const testsDiscovered = integerField(raw, "testsDiscovered");
|
|
351
|
+
const candidateTestsDiscovered = integerField(raw, "candidateTestsDiscovered");
|
|
352
|
+
if (candidateTestsDiscovered > testsDiscovered) {
|
|
353
|
+
throw new TypeError("candidateTestsDiscovered must not exceed testsDiscovered");
|
|
354
|
+
}
|
|
355
|
+
const outcome = boundedStringField(raw, "outcome", 64);
|
|
356
|
+
if (!OBSERVATION_OUTCOMES.has(outcome))
|
|
357
|
+
throw new TypeError("Invalid observation outcome");
|
|
358
|
+
return {
|
|
359
|
+
runId: boundedStringField(raw, "runId", 512),
|
|
360
|
+
candidateId,
|
|
361
|
+
worldId,
|
|
362
|
+
attempt: integerField(raw, "attempt", 1),
|
|
363
|
+
outcome,
|
|
364
|
+
testsDiscovered,
|
|
365
|
+
candidateTestsDiscovered,
|
|
366
|
+
attributed: raw.attributed,
|
|
367
|
+
...(duration === undefined ? {} : { durationMs: duration }),
|
|
368
|
+
...(exitCode === undefined ? {} : { exitCode: exitCode }),
|
|
369
|
+
...(stdoutDigest === undefined ? {} : { stdoutDigest }),
|
|
370
|
+
...(stderrDigest === undefined ? {} : { stderrDigest }),
|
|
371
|
+
};
|
|
372
|
+
})
|
|
373
|
+
.sort(compareObservations);
|
|
374
|
+
if (new Set(observations.map((item) => item.runId)).size !== observations.length)
|
|
375
|
+
throw new TypeError("Duplicate runId");
|
|
376
|
+
return {
|
|
377
|
+
schemaVersion,
|
|
378
|
+
repositoryDigest: digestField(input, "repositoryDigest"),
|
|
379
|
+
evidenceContext,
|
|
380
|
+
policy,
|
|
381
|
+
worlds,
|
|
382
|
+
candidates,
|
|
383
|
+
observations,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
function parseInput(input) {
|
|
387
|
+
try {
|
|
388
|
+
return parseInputUnchecked(input);
|
|
389
|
+
}
|
|
390
|
+
catch (error) {
|
|
391
|
+
if (error instanceof EvidenceValidationError)
|
|
392
|
+
throw error;
|
|
393
|
+
if (error instanceof TypeError) {
|
|
394
|
+
throw new EvidenceValidationError(error.message, { cause: error });
|
|
395
|
+
}
|
|
396
|
+
throw error;
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function compareObservations(left, right) {
|
|
400
|
+
return (compareOrdinal(left.candidateId ?? "", right.candidateId ?? "") ||
|
|
401
|
+
compareOrdinal(left.worldId, right.worldId) ||
|
|
402
|
+
left.attempt - right.attempt ||
|
|
403
|
+
compareOrdinal(left.runId, right.runId));
|
|
404
|
+
}
|
|
405
|
+
const INCONCLUSIVE_OUTCOMES = new Set(["TIMEOUT", "INFRA_ERROR"]);
|
|
406
|
+
function runsFor(input, candidateId, worldId) {
|
|
407
|
+
return input.observations.filter((run) => run.candidateId === candidateId && run.worldId === worldId);
|
|
408
|
+
}
|
|
409
|
+
function isComplete(runs, requiredAttempts) {
|
|
410
|
+
if (runs.length !== requiredAttempts)
|
|
411
|
+
return false;
|
|
412
|
+
const attempts = new Set(runs.map((run) => run.attempt));
|
|
413
|
+
if (attempts.size !== requiredAttempts)
|
|
414
|
+
return false;
|
|
415
|
+
for (let attempt = 1; attempt <= requiredAttempts; attempt += 1)
|
|
416
|
+
if (!attempts.has(attempt))
|
|
417
|
+
return false;
|
|
418
|
+
return true;
|
|
419
|
+
}
|
|
420
|
+
function isCompleteStable(runs, requiredAttempts) {
|
|
421
|
+
if (!isComplete(runs, requiredAttempts))
|
|
422
|
+
return false;
|
|
423
|
+
return runs.every((run) => run.outcome === runs[0]?.outcome);
|
|
424
|
+
}
|
|
425
|
+
function controlsAreValid(input) {
|
|
426
|
+
return input.worlds.every((world) => {
|
|
427
|
+
const runs = runsFor(input, null, world.id);
|
|
428
|
+
return (isCompleteStable(runs, input.policy.requiredAttempts) &&
|
|
429
|
+
runs.every((run) => run.outcome === "PASS" && run.candidateTestsDiscovered === 0 && !run.attributed));
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
function assessCandidate(input, candidate) {
|
|
433
|
+
const worldRuns = input.worlds.map((world) => ({
|
|
434
|
+
world,
|
|
435
|
+
runs: runsFor(input, candidate.id, world.id),
|
|
436
|
+
}));
|
|
437
|
+
const allRuns = worldRuns.flatMap(({ runs }) => runs).sort(compareObservations);
|
|
438
|
+
const ids = (runs) => runs.map((run) => run.runId).sort(compareOrdinal);
|
|
439
|
+
const notRun = (name) => ({
|
|
440
|
+
name,
|
|
441
|
+
status: "NOT_RUN",
|
|
442
|
+
evidenceRunIds: [],
|
|
443
|
+
reasonCodes: ["PREREQUISITE_GATE_FAILED"],
|
|
444
|
+
});
|
|
445
|
+
const gates = [];
|
|
446
|
+
const complete = worldRuns.every(({ runs }) => isComplete(runs, input.policy.requiredAttempts));
|
|
447
|
+
gates.push({
|
|
448
|
+
name: "COMPLETENESS",
|
|
449
|
+
status: complete ? "PASSED" : "FAILED",
|
|
450
|
+
evidenceRunIds: ids(allRuns),
|
|
451
|
+
reasonCodes: complete ? [] : ["CANDIDATE_EVIDENCE_INCOMPLETE"],
|
|
452
|
+
});
|
|
453
|
+
if (!complete) {
|
|
454
|
+
gates.push(notRun("STABILITY"), notRun("DISCOVERY"), notRun("REFERENCE"), notRun("NEUTRAL"), notRun("TARGET_STRENGTH"));
|
|
455
|
+
return {
|
|
456
|
+
...candidate,
|
|
457
|
+
status: "INCONCLUSIVE",
|
|
458
|
+
killedTargetIds: [],
|
|
459
|
+
targetWeightKilled: 0,
|
|
460
|
+
gates,
|
|
461
|
+
reasonCodes: ["CANDIDATE_EVIDENCE_INCOMPLETE"],
|
|
462
|
+
};
|
|
463
|
+
}
|
|
464
|
+
const stable = worldRuns.every(({ runs }) => isCompleteStable(runs, input.policy.requiredAttempts));
|
|
465
|
+
gates.push({
|
|
466
|
+
name: "STABILITY",
|
|
467
|
+
status: stable ? "PASSED" : "FAILED",
|
|
468
|
+
evidenceRunIds: ids(allRuns),
|
|
469
|
+
reasonCodes: stable ? [] : ["OBSERVATIONS_DIVERGE"],
|
|
470
|
+
});
|
|
471
|
+
if (!stable) {
|
|
472
|
+
gates.push(notRun("DISCOVERY"), notRun("REFERENCE"), notRun("NEUTRAL"), notRun("TARGET_STRENGTH"));
|
|
473
|
+
return {
|
|
474
|
+
...candidate,
|
|
475
|
+
status: "UNSTABLE",
|
|
476
|
+
killedTargetIds: [],
|
|
477
|
+
targetWeightKilled: 0,
|
|
478
|
+
gates,
|
|
479
|
+
reasonCodes: ["OBSERVATIONS_DIVERGE"],
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
const discoveryValid = allRuns.every((run) => run.candidateTestsDiscovered >= 1 && run.attributed);
|
|
483
|
+
gates.push({
|
|
484
|
+
name: "DISCOVERY",
|
|
485
|
+
status: discoveryValid ? "PASSED" : "FAILED",
|
|
486
|
+
evidenceRunIds: ids(allRuns),
|
|
487
|
+
reasonCodes: discoveryValid ? [] : ["CANDIDATE_DISCOVERY_INVALID"],
|
|
488
|
+
});
|
|
489
|
+
if (!discoveryValid) {
|
|
490
|
+
gates.push(notRun("REFERENCE"), notRun("NEUTRAL"), notRun("TARGET_STRENGTH"));
|
|
491
|
+
return {
|
|
492
|
+
...candidate,
|
|
493
|
+
status: "INVALID",
|
|
494
|
+
killedTargetIds: [],
|
|
495
|
+
targetWeightKilled: 0,
|
|
496
|
+
gates,
|
|
497
|
+
reasonCodes: ["CANDIDATE_DISCOVERY_INVALID"],
|
|
498
|
+
};
|
|
499
|
+
}
|
|
500
|
+
const referenceRuns = worldRuns
|
|
501
|
+
.filter(({ world }) => world.kind === "REFERENCE")
|
|
502
|
+
.flatMap(({ runs }) => runs);
|
|
503
|
+
const referencePassed = referenceRuns.every((run) => run.outcome === "PASS");
|
|
504
|
+
gates.push({
|
|
505
|
+
name: "REFERENCE",
|
|
506
|
+
status: referencePassed ? "PASSED" : "FAILED",
|
|
507
|
+
evidenceRunIds: ids(referenceRuns),
|
|
508
|
+
reasonCodes: referencePassed ? [] : ["REFERENCE_NOT_GREEN"],
|
|
509
|
+
});
|
|
510
|
+
const neutralRuns = worldRuns
|
|
511
|
+
.filter(({ world }) => world.kind === "NEUTRAL")
|
|
512
|
+
.flatMap(({ runs }) => runs);
|
|
513
|
+
const neutralPassed = neutralRuns.every((run) => run.outcome === "PASS");
|
|
514
|
+
gates.push({
|
|
515
|
+
name: "NEUTRAL",
|
|
516
|
+
status: neutralPassed ? "PASSED" : "FAILED",
|
|
517
|
+
evidenceRunIds: ids(neutralRuns),
|
|
518
|
+
reasonCodes: neutralPassed ? [] : ["NEUTRAL_NOT_GREEN"],
|
|
519
|
+
});
|
|
520
|
+
const killedWorlds = worldRuns
|
|
521
|
+
.filter(({ world, runs }) => world.kind === "TARGET" &&
|
|
522
|
+
runs.every((run) => run.attributed && run.outcome === "ASSERTION_FAILURE"))
|
|
523
|
+
.map(({ world }) => world);
|
|
524
|
+
const killedTargetIds = killedWorlds.map((world) => world.id).sort(compareOrdinal);
|
|
525
|
+
const targetWeightKilled = killedWorlds.reduce((sum, world) => sum + world.weight, 0);
|
|
526
|
+
const allTargets = input.worlds.filter((world) => world.kind === "TARGET");
|
|
527
|
+
const totalWeight = allTargets.reduce((sum, world) => sum + world.weight, 0);
|
|
528
|
+
const requiredKilled = allTargets
|
|
529
|
+
.filter((world) => world.required)
|
|
530
|
+
.every((world) => killedTargetIds.includes(world.id));
|
|
531
|
+
const thresholdMet = totalWeight > 0 &&
|
|
532
|
+
targetWeightKilled * 1_000 >= totalWeight * input.policy.minimumTargetWeightPermille;
|
|
533
|
+
const targetPassed = requiredKilled && thresholdMet;
|
|
534
|
+
const targetRuns = worldRuns
|
|
535
|
+
.filter(({ world }) => world.kind === "TARGET")
|
|
536
|
+
.flatMap(({ runs }) => runs);
|
|
537
|
+
const targetExecutionInconclusive = targetRuns.some((run) => INCONCLUSIVE_OUTCOMES.has(run.outcome));
|
|
538
|
+
gates.push({
|
|
539
|
+
name: "TARGET_STRENGTH",
|
|
540
|
+
status: targetPassed ? "PASSED" : "FAILED",
|
|
541
|
+
evidenceRunIds: ids(targetRuns),
|
|
542
|
+
reasonCodes: targetPassed
|
|
543
|
+
? []
|
|
544
|
+
: [
|
|
545
|
+
targetExecutionInconclusive
|
|
546
|
+
? "CANDIDATE_EXECUTION_INCONCLUSIVE"
|
|
547
|
+
: "TARGET_STRENGTH_INSUFFICIENT",
|
|
548
|
+
],
|
|
549
|
+
});
|
|
550
|
+
const executionInconclusive = allRuns.some((run) => INCONCLUSIVE_OUTCOMES.has(run.outcome));
|
|
551
|
+
const status = executionInconclusive
|
|
552
|
+
? "INCONCLUSIVE"
|
|
553
|
+
: !referencePassed || !neutralPassed
|
|
554
|
+
? "INVALID"
|
|
555
|
+
: targetPassed
|
|
556
|
+
? "ELIGIBLE"
|
|
557
|
+
: "WEAK_ORACLE";
|
|
558
|
+
const reasonCodes = executionInconclusive
|
|
559
|
+
? ["CANDIDATE_EXECUTION_INCONCLUSIVE"]
|
|
560
|
+
: !referencePassed
|
|
561
|
+
? ["REFERENCE_NOT_GREEN"]
|
|
562
|
+
: !neutralPassed
|
|
563
|
+
? ["NEUTRAL_NOT_GREEN"]
|
|
564
|
+
: targetPassed
|
|
565
|
+
? ["POLICY_SATISFIED"]
|
|
566
|
+
: ["TARGET_STRENGTH_INSUFFICIENT"];
|
|
567
|
+
return {
|
|
568
|
+
...candidate,
|
|
569
|
+
status,
|
|
570
|
+
killedTargetIds,
|
|
571
|
+
targetWeightKilled,
|
|
572
|
+
gates,
|
|
573
|
+
reasonCodes,
|
|
574
|
+
};
|
|
575
|
+
}
|
|
576
|
+
export function selectCandidates(assessments, maximum) {
|
|
577
|
+
if (!Number.isSafeInteger(maximum) || maximum <= 0 || !Array.isArray(assessments))
|
|
578
|
+
return [];
|
|
579
|
+
const eligible = assessments.flatMap((raw) => {
|
|
580
|
+
if (!isRecord(raw) ||
|
|
581
|
+
raw.status !== "ELIGIBLE" ||
|
|
582
|
+
typeof raw.id !== "string" ||
|
|
583
|
+
typeof raw.digest !== "string" ||
|
|
584
|
+
!Number.isSafeInteger(raw.sizeBytes) ||
|
|
585
|
+
!Array.isArray(raw.killedTargetIds) ||
|
|
586
|
+
raw.killedTargetIds.some((id) => typeof id !== "string"))
|
|
587
|
+
return [];
|
|
588
|
+
return [
|
|
589
|
+
{
|
|
590
|
+
id: raw.id,
|
|
591
|
+
digest: raw.digest,
|
|
592
|
+
sizeBytes: raw.sizeBytes,
|
|
593
|
+
killedTargetIds: [...new Set(raw.killedTargetIds)].sort(compareOrdinal),
|
|
594
|
+
},
|
|
595
|
+
];
|
|
596
|
+
});
|
|
597
|
+
const selected = [];
|
|
598
|
+
const covered = new Set();
|
|
599
|
+
const remaining = [...eligible];
|
|
600
|
+
while (selected.length < maximum && remaining.length > 0) {
|
|
601
|
+
remaining.sort((left, right) => {
|
|
602
|
+
const leftMarginal = left.killedTargetIds.filter((id) => !covered.has(id)).length;
|
|
603
|
+
const rightMarginal = right.killedTargetIds.filter((id) => !covered.has(id)).length;
|
|
604
|
+
return (rightMarginal - leftMarginal ||
|
|
605
|
+
left.sizeBytes - right.sizeBytes ||
|
|
606
|
+
compareOrdinal(left.digest, right.digest) ||
|
|
607
|
+
compareOrdinal(left.id, right.id));
|
|
608
|
+
});
|
|
609
|
+
const best = remaining.shift();
|
|
610
|
+
if (!best || best.killedTargetIds.every((id) => covered.has(id)))
|
|
611
|
+
break;
|
|
612
|
+
selected.push(best.id);
|
|
613
|
+
for (const id of best.killedTargetIds)
|
|
614
|
+
covered.add(id);
|
|
615
|
+
}
|
|
616
|
+
return selected;
|
|
617
|
+
}
|
|
618
|
+
function digestProjection(manifest) {
|
|
619
|
+
return {
|
|
620
|
+
schemaVersion: manifest.schemaVersion,
|
|
621
|
+
repositoryDigest: manifest.repositoryDigest,
|
|
622
|
+
evidenceContext: manifest.evidenceContext,
|
|
623
|
+
policy: manifest.policy,
|
|
624
|
+
worlds: [...manifest.worlds].sort((a, b) => compareOrdinal(a.id, b.id)),
|
|
625
|
+
candidates: [...manifest.candidates].sort((a, b) => compareOrdinal(a.id, b.id)),
|
|
626
|
+
observations: [...manifest.observations]
|
|
627
|
+
.sort(compareObservations)
|
|
628
|
+
.map(({ durationMs: _durationMs, exitCode: _exitCode, stdoutDigest: _stdoutDigest, stderrDigest: _stderrDigest, ...run }) => run),
|
|
629
|
+
decision: manifest.decision,
|
|
630
|
+
};
|
|
631
|
+
}
|
|
632
|
+
function artifactProjection(manifest) {
|
|
633
|
+
const { artifactDigest: _artifactDigest, ...artifact } = manifest;
|
|
634
|
+
return artifact;
|
|
635
|
+
}
|
|
636
|
+
export function sealManifestArtifact(manifest) {
|
|
637
|
+
return {
|
|
638
|
+
...manifest,
|
|
639
|
+
artifactDigest: sha256Canonical(artifactProjection(manifest)),
|
|
640
|
+
};
|
|
641
|
+
}
|
|
642
|
+
function finishManifest(base) {
|
|
643
|
+
const decisionSealed = {
|
|
644
|
+
...base,
|
|
645
|
+
decisionDigest: sha256Canonical(digestProjection(base)),
|
|
646
|
+
};
|
|
647
|
+
return sealManifestArtifact(decisionSealed);
|
|
648
|
+
}
|
|
649
|
+
export function decideEvidence(input) {
|
|
650
|
+
let evidence;
|
|
651
|
+
try {
|
|
652
|
+
evidence = parseInput(input);
|
|
653
|
+
}
|
|
654
|
+
catch (error) {
|
|
655
|
+
if (!(error instanceof EvidenceValidationError))
|
|
656
|
+
throw error;
|
|
657
|
+
return finishManifest({
|
|
658
|
+
schemaVersion: "1.0.0",
|
|
659
|
+
repositoryDigest: "invalid",
|
|
660
|
+
evidenceContext: {
|
|
661
|
+
engine: { name: "invalid", version: "invalid" },
|
|
662
|
+
adapter: { name: "invalid", version: "invalid", configuration: {} },
|
|
663
|
+
execution: {
|
|
664
|
+
isolation: "invalid",
|
|
665
|
+
environmentAllowlist: [],
|
|
666
|
+
budgets: {},
|
|
667
|
+
candidateRoots: [],
|
|
668
|
+
},
|
|
669
|
+
worlds: [],
|
|
670
|
+
},
|
|
671
|
+
policy: {
|
|
672
|
+
policyVersion: "invalid",
|
|
673
|
+
requiredAttempts: 1,
|
|
674
|
+
minimumTargetWeightPermille: 1_000,
|
|
675
|
+
maximumSelectedCandidates: 0,
|
|
676
|
+
acceptedTargetOutcomes: [],
|
|
677
|
+
},
|
|
678
|
+
worlds: [],
|
|
679
|
+
candidates: [],
|
|
680
|
+
observations: [],
|
|
681
|
+
decision: {
|
|
682
|
+
status: "ENGINE_ERROR",
|
|
683
|
+
selectedCandidateIds: [],
|
|
684
|
+
reasonCodes: ["EVIDENCE_INPUT_INVALID"],
|
|
685
|
+
},
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
const candidates = evidence.candidates.map((candidate) => assessCandidate(evidence, candidate));
|
|
689
|
+
const invalidControls = !controlsAreValid(evidence);
|
|
690
|
+
const hasInconclusiveCandidate = candidates.some((candidate) => candidate.status === "UNSTABLE" || candidate.status === "INCONCLUSIVE");
|
|
691
|
+
const selectedCandidateIds = invalidControls
|
|
692
|
+
? []
|
|
693
|
+
: selectCandidates(candidates, evidence.policy.maximumSelectedCandidates);
|
|
694
|
+
const status = invalidControls
|
|
695
|
+
? "INCONCLUSIVE"
|
|
696
|
+
: selectedCandidateIds.length > 0
|
|
697
|
+
? "VERIFIED"
|
|
698
|
+
: hasInconclusiveCandidate
|
|
699
|
+
? "INCONCLUSIVE"
|
|
700
|
+
: "REJECTED";
|
|
701
|
+
const reasonCodes = [
|
|
702
|
+
...(invalidControls ? ["CONTROL_EVIDENCE_INVALID"] : []),
|
|
703
|
+
...(!invalidControls && selectedCandidateIds.length > 0 ? ["POLICY_SATISFIED"] : []),
|
|
704
|
+
...(!invalidControls && selectedCandidateIds.length === 0 && hasInconclusiveCandidate
|
|
705
|
+
? ["CANDIDATE_EVIDENCE_INCONCLUSIVE"]
|
|
706
|
+
: []),
|
|
707
|
+
...(!invalidControls && selectedCandidateIds.length === 0 && !hasInconclusiveCandidate
|
|
708
|
+
? ["NO_ELIGIBLE_CANDIDATE"]
|
|
709
|
+
: []),
|
|
710
|
+
];
|
|
711
|
+
return finishManifest({
|
|
712
|
+
...evidence,
|
|
713
|
+
candidates,
|
|
714
|
+
decision: { status, selectedCandidateIds, reasonCodes },
|
|
715
|
+
});
|
|
716
|
+
}
|
|
717
|
+
export function verifyDecisionDigest(manifest) {
|
|
718
|
+
if (!isRecord(manifest) || typeof manifest.decisionDigest !== "string")
|
|
719
|
+
return { valid: false };
|
|
720
|
+
try {
|
|
721
|
+
const candidate = manifest;
|
|
722
|
+
return { valid: candidate.decisionDigest === sha256Canonical(digestProjection(candidate)) };
|
|
723
|
+
}
|
|
724
|
+
catch {
|
|
725
|
+
return { valid: false };
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
export function verifyManifestIntegrity(manifest) {
|
|
729
|
+
if (!isRecord(manifest)) {
|
|
730
|
+
return { valid: false, decisionDigestValid: false, artifactDigestValid: false };
|
|
731
|
+
}
|
|
732
|
+
const decisionDigestValid = verifyDecisionDigest(manifest).valid;
|
|
733
|
+
let artifactDigestValid = false;
|
|
734
|
+
try {
|
|
735
|
+
artifactDigestValid =
|
|
736
|
+
typeof manifest.artifactDigest === "string" &&
|
|
737
|
+
manifest.artifactDigest === sha256Canonical(artifactProjection(manifest));
|
|
738
|
+
}
|
|
739
|
+
catch {
|
|
740
|
+
artifactDigestValid = false;
|
|
741
|
+
}
|
|
742
|
+
return {
|
|
743
|
+
valid: decisionDigestValid && artifactDigestValid,
|
|
744
|
+
decisionDigestValid,
|
|
745
|
+
artifactDigestValid,
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
function replayCandidateSemantics(candidates) {
|
|
749
|
+
if (!Array.isArray(candidates))
|
|
750
|
+
throw new TypeError("manifest.candidates must be an array");
|
|
751
|
+
return candidates
|
|
752
|
+
.map((raw) => {
|
|
753
|
+
if (!isRecord(raw))
|
|
754
|
+
throw new TypeError("manifest candidate must be an object");
|
|
755
|
+
return {
|
|
756
|
+
id: stringField(raw, "id"),
|
|
757
|
+
digest: stringField(raw, "digest"),
|
|
758
|
+
sizeBytes: integerField(raw, "sizeBytes"),
|
|
759
|
+
status: stringField(raw, "status"),
|
|
760
|
+
killedTargetIds: jsonField(raw, "killedTargetIds"),
|
|
761
|
+
targetWeightKilled: integerField(raw, "targetWeightKilled"),
|
|
762
|
+
gates: jsonField(raw, "gates"),
|
|
763
|
+
reasonCodes: jsonField(raw, "reasonCodes"),
|
|
764
|
+
};
|
|
765
|
+
})
|
|
766
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
767
|
+
}
|
|
768
|
+
export function replayEvidenceManifest(manifest) {
|
|
769
|
+
const invalid = {
|
|
770
|
+
valid: false,
|
|
771
|
+
decisionDigestValid: false,
|
|
772
|
+
artifactDigestValid: false,
|
|
773
|
+
decisionSemanticsValid: false,
|
|
774
|
+
};
|
|
775
|
+
if (!isRecord(manifest))
|
|
776
|
+
return invalid;
|
|
777
|
+
const integrity = verifyManifestIntegrity(manifest);
|
|
778
|
+
try {
|
|
779
|
+
if (!isRecord(manifest.evidenceContext) ||
|
|
780
|
+
!isRecord(manifest.policy) ||
|
|
781
|
+
!Array.isArray(manifest.worlds) ||
|
|
782
|
+
!Array.isArray(manifest.candidates) ||
|
|
783
|
+
!Array.isArray(manifest.observations) ||
|
|
784
|
+
!isRecord(manifest.decision)) {
|
|
785
|
+
throw new TypeError("Manifest does not contain replayable evidence");
|
|
786
|
+
}
|
|
787
|
+
const candidates = manifest.candidates.map((raw) => {
|
|
788
|
+
if (!isRecord(raw))
|
|
789
|
+
throw new TypeError("manifest candidate must be an object");
|
|
790
|
+
return {
|
|
791
|
+
id: stringField(raw, "id"),
|
|
792
|
+
digest: stringField(raw, "digest"),
|
|
793
|
+
sizeBytes: integerField(raw, "sizeBytes"),
|
|
794
|
+
};
|
|
795
|
+
});
|
|
796
|
+
const replayed = decideEvidence({
|
|
797
|
+
schemaVersion: manifest.schemaVersion,
|
|
798
|
+
repositoryDigest: manifest.repositoryDigest,
|
|
799
|
+
evidenceContext: manifest.evidenceContext,
|
|
800
|
+
policy: manifest.policy,
|
|
801
|
+
worlds: manifest.worlds,
|
|
802
|
+
candidates,
|
|
803
|
+
observations: manifest.observations,
|
|
804
|
+
});
|
|
805
|
+
const decisionSemanticsValid = replayed.decision.status !== "ENGINE_ERROR" &&
|
|
806
|
+
replayed.decisionDigest === manifest.decisionDigest &&
|
|
807
|
+
canonicalize({
|
|
808
|
+
candidates: replayCandidateSemantics(replayed.candidates),
|
|
809
|
+
decision: replayed.decision,
|
|
810
|
+
}) ===
|
|
811
|
+
canonicalize({
|
|
812
|
+
candidates: replayCandidateSemantics(manifest.candidates),
|
|
813
|
+
decision: manifest.decision,
|
|
814
|
+
});
|
|
815
|
+
return {
|
|
816
|
+
valid: integrity.decisionDigestValid && integrity.artifactDigestValid && decisionSemanticsValid,
|
|
817
|
+
decisionDigestValid: integrity.decisionDigestValid,
|
|
818
|
+
artifactDigestValid: integrity.artifactDigestValid,
|
|
819
|
+
decisionSemanticsValid,
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
catch {
|
|
823
|
+
return {
|
|
824
|
+
...invalid,
|
|
825
|
+
decisionDigestValid: integrity.decisionDigestValid,
|
|
826
|
+
artifactDigestValid: integrity.artifactDigestValid,
|
|
827
|
+
};
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
function nearestRank(sorted, percentile) {
|
|
831
|
+
if (sorted.length === 0)
|
|
832
|
+
return null;
|
|
833
|
+
return sorted[Math.ceil(percentile * sorted.length) - 1] ?? null;
|
|
834
|
+
}
|
|
835
|
+
function agenticReportProjection(report) {
|
|
836
|
+
const { reportDigest: _reportDigest, ...projection } = report;
|
|
837
|
+
return projection;
|
|
838
|
+
}
|
|
839
|
+
export function createAgenticProfile(request) {
|
|
840
|
+
request = parseAgenticProfileRequest(request);
|
|
841
|
+
if (!replayEvidenceManifest(request.manifest).valid) {
|
|
842
|
+
throw new EvidenceValidationError("AGENTIC_PROFILE_SOURCE_INVALID");
|
|
843
|
+
}
|
|
844
|
+
const targetWorlds = request.manifest.worlds.filter((world) => world.kind === "TARGET");
|
|
845
|
+
const referenceIds = new Set(request.manifest.worlds.filter((world) => world.kind === "REFERENCE").map((world) => world.id));
|
|
846
|
+
const targetWeights = new Map(targetWorlds.map((world) => [world.id, world.weight]));
|
|
847
|
+
const totalTargetWeight = targetWorlds.reduce((total, world) => total + world.weight, 0);
|
|
848
|
+
const requiredTargetIds = new Set(targetWorlds.filter((world) => world.required).map((world) => world.id));
|
|
849
|
+
const selected = new Set(request.manifest.decision.selectedCandidateIds);
|
|
850
|
+
const marginalWeight = new Map();
|
|
851
|
+
const coveredTargets = new Set();
|
|
852
|
+
for (const id of request.manifest.decision.selectedCandidateIds) {
|
|
853
|
+
const candidate = request.manifest.candidates.find((item) => item.id === id);
|
|
854
|
+
let weight = 0;
|
|
855
|
+
for (const targetId of candidate?.killedTargetIds ?? []) {
|
|
856
|
+
if (!coveredTargets.has(targetId))
|
|
857
|
+
weight += targetWeights.get(targetId) ?? 0;
|
|
858
|
+
coveredTargets.add(targetId);
|
|
859
|
+
}
|
|
860
|
+
marginalWeight.set(id, weight);
|
|
861
|
+
}
|
|
862
|
+
const candidates = request.manifest.candidates
|
|
863
|
+
.map((candidate) => {
|
|
864
|
+
const observations = request.manifest.observations.filter((observation) => observation.candidateId === candidate.id);
|
|
865
|
+
const referenceDurations = observations
|
|
866
|
+
.filter((observation) => referenceIds.has(observation.worldId) && typeof observation.durationMs === "number")
|
|
867
|
+
.map((observation) => observation.durationMs)
|
|
868
|
+
.sort((left, right) => left - right);
|
|
869
|
+
const totalCandidateDurationMs = observations.reduce((total, observation) => total + (observation.durationMs ?? 0), 0);
|
|
870
|
+
const p95Ms = nearestRank(referenceDurations, 0.95);
|
|
871
|
+
const evidenceQualified = request.manifest.decision.status === "VERIFIED" &&
|
|
872
|
+
selected.has(candidate.id) &&
|
|
873
|
+
candidate.status === "ELIGIBLE";
|
|
874
|
+
const timingSufficient = referenceDurations.length >= request.policy.minimumTimingSamples;
|
|
875
|
+
const satisfiedLaneIds = evidenceQualified && timingSufficient && p95Ms !== null
|
|
876
|
+
? request.policy.lanes
|
|
877
|
+
.filter((lane) => p95Ms <= lane.maximumReferenceP95Ms)
|
|
878
|
+
.map((lane) => lane.id)
|
|
879
|
+
: [];
|
|
880
|
+
const classification = !evidenceQualified
|
|
881
|
+
? "NOT_QUALIFIED"
|
|
882
|
+
: !timingSufficient
|
|
883
|
+
? "INSUFFICIENT_TIMING_EVIDENCE"
|
|
884
|
+
: satisfiedLaneIds.length === 0
|
|
885
|
+
? "BUDGET_MISSED"
|
|
886
|
+
: "QUALIFIED";
|
|
887
|
+
return {
|
|
888
|
+
id: candidate.id,
|
|
889
|
+
evidenceStatus: candidate.status,
|
|
890
|
+
classification,
|
|
891
|
+
satisfiedLaneIds,
|
|
892
|
+
bestLaneId: satisfiedLaneIds[0] ?? null,
|
|
893
|
+
targetWeightKilled: candidate.targetWeightKilled,
|
|
894
|
+
totalTargetWeight,
|
|
895
|
+
targetWeightPermille: totalTargetWeight === 0
|
|
896
|
+
? 0
|
|
897
|
+
: Math.floor((candidate.targetWeightKilled * 1_000) / totalTargetWeight),
|
|
898
|
+
requiredTargetsKilled: candidate.killedTargetIds.filter((id) => requiredTargetIds.has(id))
|
|
899
|
+
.length,
|
|
900
|
+
requiredTargetsTotal: requiredTargetIds.size,
|
|
901
|
+
killedTargetIds: [...candidate.killedTargetIds].sort(compareOrdinal),
|
|
902
|
+
marginalTargetWeight: marginalWeight.get(candidate.id) ?? 0,
|
|
903
|
+
consistency: {
|
|
904
|
+
claim: candidate.gates.find((gate) => gate.name === "STABILITY")?.status === "PASSED"
|
|
905
|
+
? "OBSERVED_CONSISTENT"
|
|
906
|
+
: "OBSERVED_INCONSISTENT",
|
|
907
|
+
attempts: request.manifest.policy.requiredAttempts,
|
|
908
|
+
},
|
|
909
|
+
latency: {
|
|
910
|
+
kind: "RECORDED_REFERENCE_WALL_TIME",
|
|
911
|
+
samples: referenceDurations.length,
|
|
912
|
+
minimumMs: referenceDurations[0] ?? null,
|
|
913
|
+
p50Ms: nearestRank(referenceDurations, 0.5),
|
|
914
|
+
p95Ms,
|
|
915
|
+
maximumMs: referenceDurations.at(-1) ?? null,
|
|
916
|
+
totalCandidateDurationMs,
|
|
917
|
+
},
|
|
918
|
+
paretoFrontier: false,
|
|
919
|
+
sizeBytes: candidate.sizeBytes,
|
|
920
|
+
digest: candidate.digest,
|
|
921
|
+
};
|
|
922
|
+
})
|
|
923
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
924
|
+
for (const candidate of candidates) {
|
|
925
|
+
if ((candidate.classification !== "QUALIFIED" && candidate.classification !== "BUDGET_MISSED") ||
|
|
926
|
+
candidate.latency.p95Ms === null) {
|
|
927
|
+
continue;
|
|
928
|
+
}
|
|
929
|
+
const candidateP95Ms = candidate.latency.p95Ms;
|
|
930
|
+
candidate.paretoFrontier = !candidates.some((other) => {
|
|
931
|
+
const otherP95Ms = other.latency.p95Ms;
|
|
932
|
+
return (other.id !== candidate.id &&
|
|
933
|
+
(other.classification === "QUALIFIED" || other.classification === "BUDGET_MISSED") &&
|
|
934
|
+
otherP95Ms !== null &&
|
|
935
|
+
other.targetWeightKilled >= candidate.targetWeightKilled &&
|
|
936
|
+
other.requiredTargetsKilled >= candidate.requiredTargetsKilled &&
|
|
937
|
+
otherP95Ms <= candidateP95Ms &&
|
|
938
|
+
other.latency.totalCandidateDurationMs <= candidate.latency.totalCandidateDurationMs &&
|
|
939
|
+
other.sizeBytes <= candidate.sizeBytes &&
|
|
940
|
+
(other.targetWeightKilled > candidate.targetWeightKilled ||
|
|
941
|
+
other.requiredTargetsKilled > candidate.requiredTargetsKilled ||
|
|
942
|
+
otherP95Ms < candidateP95Ms ||
|
|
943
|
+
other.latency.totalCandidateDurationMs < candidate.latency.totalCandidateDurationMs ||
|
|
944
|
+
other.sizeBytes < candidate.sizeBytes));
|
|
945
|
+
});
|
|
946
|
+
}
|
|
947
|
+
const portfolios = request.policy.lanes.map((lane) => {
|
|
948
|
+
const selectedCandidateIds = [];
|
|
949
|
+
const selectedTargets = new Set();
|
|
950
|
+
let totalReferenceP95Ms = 0;
|
|
951
|
+
while (true) {
|
|
952
|
+
const remainingBudget = lane.maximumReferenceP95Ms - totalReferenceP95Ms;
|
|
953
|
+
const choices = candidates
|
|
954
|
+
.filter((candidate) => candidate.classification === "QUALIFIED" &&
|
|
955
|
+
candidate.satisfiedLaneIds.includes(lane.id) &&
|
|
956
|
+
!selectedCandidateIds.includes(candidate.id) &&
|
|
957
|
+
candidate.latency.p95Ms !== null &&
|
|
958
|
+
candidate.latency.p95Ms <= remainingBudget)
|
|
959
|
+
.map((candidate) => ({
|
|
960
|
+
candidate,
|
|
961
|
+
marginalWeight: candidate.killedTargetIds.reduce((weight, targetId) => weight + (selectedTargets.has(targetId) ? 0 : (targetWeights.get(targetId) ?? 0)), 0),
|
|
962
|
+
}))
|
|
963
|
+
.filter((choice) => choice.marginalWeight > 0)
|
|
964
|
+
.sort((left, right) => {
|
|
965
|
+
const leftCost = left.candidate.latency.p95Ms;
|
|
966
|
+
const rightCost = right.candidate.latency.p95Ms;
|
|
967
|
+
if (leftCost === 0 || rightCost === 0) {
|
|
968
|
+
if (leftCost === 0 && rightCost !== 0)
|
|
969
|
+
return -1;
|
|
970
|
+
if (rightCost === 0 && leftCost !== 0)
|
|
971
|
+
return 1;
|
|
972
|
+
}
|
|
973
|
+
else {
|
|
974
|
+
const leftRatio = left.marginalWeight * rightCost;
|
|
975
|
+
const rightRatio = right.marginalWeight * leftCost;
|
|
976
|
+
if (leftRatio !== rightRatio)
|
|
977
|
+
return leftRatio > rightRatio ? -1 : 1;
|
|
978
|
+
}
|
|
979
|
+
if (left.marginalWeight !== right.marginalWeight) {
|
|
980
|
+
return right.marginalWeight - left.marginalWeight;
|
|
981
|
+
}
|
|
982
|
+
if (leftCost !== rightCost)
|
|
983
|
+
return leftCost - rightCost;
|
|
984
|
+
if (left.candidate.sizeBytes !== right.candidate.sizeBytes) {
|
|
985
|
+
return left.candidate.sizeBytes - right.candidate.sizeBytes;
|
|
986
|
+
}
|
|
987
|
+
const digestOrder = compareOrdinal(left.candidate.digest, right.candidate.digest);
|
|
988
|
+
return digestOrder !== 0
|
|
989
|
+
? digestOrder
|
|
990
|
+
: compareOrdinal(left.candidate.id, right.candidate.id);
|
|
991
|
+
});
|
|
992
|
+
const choice = choices[0];
|
|
993
|
+
if (choice === undefined)
|
|
994
|
+
break;
|
|
995
|
+
selectedCandidateIds.push(choice.candidate.id);
|
|
996
|
+
totalReferenceP95Ms += choice.candidate.latency.p95Ms;
|
|
997
|
+
for (const targetId of choice.candidate.killedTargetIds)
|
|
998
|
+
selectedTargets.add(targetId);
|
|
999
|
+
}
|
|
1000
|
+
const targetWeight = [...selectedTargets].reduce((weight, targetId) => weight + (targetWeights.get(targetId) ?? 0), 0);
|
|
1001
|
+
return {
|
|
1002
|
+
laneId: lane.id,
|
|
1003
|
+
budgetMs: lane.maximumReferenceP95Ms,
|
|
1004
|
+
selectedCandidateIds,
|
|
1005
|
+
totalReferenceP95Ms,
|
|
1006
|
+
targetWeight,
|
|
1007
|
+
targetWeightPermille: totalTargetWeight === 0 ? 0 : Math.floor((targetWeight * 1_000) / totalTargetWeight),
|
|
1008
|
+
requiredTargetsKilled: [...selectedTargets].filter((id) => requiredTargetIds.has(id)).length,
|
|
1009
|
+
requiredTargetsTotal: requiredTargetIds.size,
|
|
1010
|
+
};
|
|
1011
|
+
});
|
|
1012
|
+
const publicCandidates = candidates.map(({ sizeBytes: _sizeBytes, digest: _digest, ...candidate }) => candidate);
|
|
1013
|
+
const qualifiedCandidateIds = publicCandidates
|
|
1014
|
+
.filter((candidate) => candidate.classification === "QUALIFIED")
|
|
1015
|
+
.map((candidate) => candidate.id);
|
|
1016
|
+
const status = qualifiedCandidateIds.length > 0
|
|
1017
|
+
? "QUALIFIED"
|
|
1018
|
+
: publicCandidates.some((candidate) => candidate.classification === "INSUFFICIENT_TIMING_EVIDENCE")
|
|
1019
|
+
? "INSUFFICIENT_TIMING_EVIDENCE"
|
|
1020
|
+
: publicCandidates.some((candidate) => candidate.classification === "BUDGET_MISSED")
|
|
1021
|
+
? "BUDGET_MISSED"
|
|
1022
|
+
: "NOT_QUALIFIED";
|
|
1023
|
+
const base = {
|
|
1024
|
+
schemaVersion: "1.0.0",
|
|
1025
|
+
sourceManifest: request.manifest,
|
|
1026
|
+
sourceArtifactDigest: request.manifest.artifactDigest,
|
|
1027
|
+
policy: request.policy,
|
|
1028
|
+
policyDigest: sha256Canonical(request.policy),
|
|
1029
|
+
status,
|
|
1030
|
+
candidates: publicCandidates,
|
|
1031
|
+
portfolios,
|
|
1032
|
+
qualifiedCandidateIds,
|
|
1033
|
+
limitations: [
|
|
1034
|
+
"Profile strength is limited to the declared worlds and recorded attempts.",
|
|
1035
|
+
"RECORDED_REFERENCE_WALL_TIME is local operational evidence, not a portable cold/warm benchmark.",
|
|
1036
|
+
"Observed consistency does not prove permanent absence of flakiness.",
|
|
1037
|
+
],
|
|
1038
|
+
};
|
|
1039
|
+
return { ...base, reportDigest: sha256Canonical(base) };
|
|
1040
|
+
}
|
|
1041
|
+
export function replayAgenticProfile(value) {
|
|
1042
|
+
const invalid = {
|
|
1043
|
+
valid: false,
|
|
1044
|
+
schemaValid: false,
|
|
1045
|
+
sourceManifestValid: false,
|
|
1046
|
+
policyDigestValid: false,
|
|
1047
|
+
reportDigestValid: false,
|
|
1048
|
+
semanticsValid: false,
|
|
1049
|
+
};
|
|
1050
|
+
let report;
|
|
1051
|
+
try {
|
|
1052
|
+
report = parseAgenticProfileReport(value);
|
|
1053
|
+
}
|
|
1054
|
+
catch {
|
|
1055
|
+
return invalid;
|
|
1056
|
+
}
|
|
1057
|
+
const sourceManifestValid = replayEvidenceManifest(report.sourceManifest).valid;
|
|
1058
|
+
const policyDigestValid = report.policyDigest === sha256Canonical(report.policy);
|
|
1059
|
+
const reportDigestValid = report.reportDigest === sha256Canonical(agenticReportProjection(report));
|
|
1060
|
+
let semanticsValid = false;
|
|
1061
|
+
try {
|
|
1062
|
+
semanticsValid =
|
|
1063
|
+
canonicalize(report) ===
|
|
1064
|
+
canonicalize(createAgenticProfile({
|
|
1065
|
+
schemaVersion: "1.0.0",
|
|
1066
|
+
manifest: report.sourceManifest,
|
|
1067
|
+
policy: report.policy,
|
|
1068
|
+
}));
|
|
1069
|
+
}
|
|
1070
|
+
catch {
|
|
1071
|
+
semanticsValid = false;
|
|
1072
|
+
}
|
|
1073
|
+
return {
|
|
1074
|
+
valid: sourceManifestValid && policyDigestValid && reportDigestValid && semanticsValid,
|
|
1075
|
+
schemaValid: true,
|
|
1076
|
+
sourceManifestValid,
|
|
1077
|
+
policyDigestValid,
|
|
1078
|
+
reportDigestValid,
|
|
1079
|
+
semanticsValid,
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
const BENCHMARK_PHASES = ["PREPARATION", "STARTUP", "COMPILE_OR_COLLECTION", "EXECUTION"];
|
|
1083
|
+
function benchmarkSourceBindingValid(request) {
|
|
1084
|
+
if (request.sourceManifest.decision.status !== "VERIFIED")
|
|
1085
|
+
return false;
|
|
1086
|
+
const referenceWorld = request.sourceManifest.worlds.find((world) => world.id === request.referenceWorldId);
|
|
1087
|
+
if (referenceWorld?.kind !== "REFERENCE" || !referenceWorld.required)
|
|
1088
|
+
return false;
|
|
1089
|
+
const contextWorld = request.sourceManifest.evidenceContext.worlds.find((world) => world.id === request.referenceWorldId);
|
|
1090
|
+
if (contextWorld === undefined)
|
|
1091
|
+
return false;
|
|
1092
|
+
return request.sourceManifest.decision.selectedCandidateIds.every((id) => {
|
|
1093
|
+
const candidate = request.sourceManifest.candidates.find((item) => item.id === id);
|
|
1094
|
+
return candidate?.status === "ELIGIBLE";
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
function benchmarkComparisonScopeProjection(request) {
|
|
1098
|
+
const referenceWorld = request.sourceManifest.evidenceContext.worlds.find((world) => world.id === request.referenceWorldId);
|
|
1099
|
+
if (referenceWorld === undefined) {
|
|
1100
|
+
throw new EvidenceValidationError("AGENTIC_BENCHMARK_REFERENCE_WORLD_INVALID");
|
|
1101
|
+
}
|
|
1102
|
+
return {
|
|
1103
|
+
environmentId: request.fingerprint.environmentId,
|
|
1104
|
+
fingerprintDigest: sha256Canonical(request.fingerprint),
|
|
1105
|
+
protocolDigest: sha256Canonical(request.protocol),
|
|
1106
|
+
sourceDecisionDigest: request.sourceManifest.decisionDigest,
|
|
1107
|
+
referenceWorldId: request.referenceWorldId,
|
|
1108
|
+
referenceWorldDigest: referenceWorld.digest,
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
function compareBenchmarkRuns(left, right) {
|
|
1112
|
+
const candidateOrder = compareOrdinal(left.candidateId, right.candidateId);
|
|
1113
|
+
if (candidateOrder !== 0)
|
|
1114
|
+
return candidateOrder;
|
|
1115
|
+
const regimeOrder = (left.regime === "COLD" ? 0 : 1) - (right.regime === "COLD" ? 0 : 1);
|
|
1116
|
+
if (regimeOrder !== 0)
|
|
1117
|
+
return regimeOrder;
|
|
1118
|
+
const roleOrder = (left.role === "WARMUP" ? 0 : 1) - (right.role === "WARMUP" ? 0 : 1);
|
|
1119
|
+
return roleOrder !== 0 ? roleOrder : left.ordinal - right.ordinal;
|
|
1120
|
+
}
|
|
1121
|
+
function benchmarkQuantiles(values) {
|
|
1122
|
+
if (values.length === 0)
|
|
1123
|
+
return null;
|
|
1124
|
+
const sorted = [...values].sort((left, right) => left - right);
|
|
1125
|
+
return {
|
|
1126
|
+
minimumUs: sorted[0],
|
|
1127
|
+
p50Us: nearestRank(sorted, 0.5),
|
|
1128
|
+
p95Us: nearestRank(sorted, 0.95),
|
|
1129
|
+
maximumUs: sorted.at(-1),
|
|
1130
|
+
};
|
|
1131
|
+
}
|
|
1132
|
+
function benchmarkSummaries(request) {
|
|
1133
|
+
const selected = request.sourceManifest.decision.selectedCandidateIds
|
|
1134
|
+
.map((id) => request.sourceManifest.candidates.find((candidate) => candidate.id === id))
|
|
1135
|
+
.filter((candidate) => candidate !== undefined)
|
|
1136
|
+
.sort((left, right) => compareOrdinal(left.id, right.id));
|
|
1137
|
+
return selected.flatMap((candidate) => ["COLD", "WARM"].map((regime) => {
|
|
1138
|
+
const runs = request.runs.filter((run) => run.candidateId === candidate.id && run.regime === regime);
|
|
1139
|
+
const measurements = runs.filter((run) => run.role === "MEASUREMENT");
|
|
1140
|
+
const accepted = measurements.filter((run) => run.status === "COMPLETE");
|
|
1141
|
+
const failed = runs.filter((run) => run.status === "INCOMPLETE").length;
|
|
1142
|
+
const planned = regime === "COLD" ? request.policy.coldMeasuredSamples : request.policy.warmMeasuredSamples;
|
|
1143
|
+
const status = failed > 0
|
|
1144
|
+
? "OBSERVED_RUN_FAILURE"
|
|
1145
|
+
: accepted.length < request.policy.minimumMeasuredSamplesPerRegime
|
|
1146
|
+
? "INSUFFICIENT_SAMPLES"
|
|
1147
|
+
: "MEASURED";
|
|
1148
|
+
const cpuValues = accepted.flatMap((run) => run.cpuTimeUs === undefined ? [] : [run.cpuTimeUs]);
|
|
1149
|
+
const cpuAvailability = cpuValues.length === 0
|
|
1150
|
+
? "UNAVAILABLE"
|
|
1151
|
+
: cpuValues.length === accepted.length
|
|
1152
|
+
? "COMPLETE"
|
|
1153
|
+
: "PARTIAL";
|
|
1154
|
+
const phaseValues = Object.fromEntries(BENCHMARK_PHASES.map((phase) => [
|
|
1155
|
+
phase,
|
|
1156
|
+
benchmarkQuantiles(accepted.map((run) => run.phases.find((timing) => timing.phase === phase)?.durationUs)),
|
|
1157
|
+
]));
|
|
1158
|
+
return {
|
|
1159
|
+
candidateId: candidate.id,
|
|
1160
|
+
candidateDigest: candidate.digest,
|
|
1161
|
+
regime,
|
|
1162
|
+
status,
|
|
1163
|
+
counters: {
|
|
1164
|
+
planned,
|
|
1165
|
+
recorded: measurements.length,
|
|
1166
|
+
accepted: accepted.length,
|
|
1167
|
+
failed,
|
|
1168
|
+
warmup: runs.filter((run) => run.role === "WARMUP").length,
|
|
1169
|
+
},
|
|
1170
|
+
wall: {
|
|
1171
|
+
total: benchmarkQuantiles(accepted.map((run) => run.totalUs)),
|
|
1172
|
+
phases: {
|
|
1173
|
+
PREPARATION: phaseValues.PREPARATION,
|
|
1174
|
+
STARTUP: phaseValues.STARTUP,
|
|
1175
|
+
COMPILE_OR_COLLECTION: phaseValues.COMPILE_OR_COLLECTION,
|
|
1176
|
+
EXECUTION: phaseValues.EXECUTION,
|
|
1177
|
+
},
|
|
1178
|
+
},
|
|
1179
|
+
cpu: {
|
|
1180
|
+
availability: cpuAvailability,
|
|
1181
|
+
quantiles: cpuAvailability === "COMPLETE" ? benchmarkQuantiles(cpuValues) : null,
|
|
1182
|
+
},
|
|
1183
|
+
};
|
|
1184
|
+
}));
|
|
1185
|
+
}
|
|
1186
|
+
function benchmarkArtifactProjection(artifact) {
|
|
1187
|
+
const { artifactDigest: _artifactDigest, ...projection } = artifact;
|
|
1188
|
+
return projection;
|
|
1189
|
+
}
|
|
1190
|
+
export function createAgenticBenchmark(request) {
|
|
1191
|
+
request = parseAgenticBenchmarkRequest(request);
|
|
1192
|
+
if (!replayEvidenceManifest(request.sourceManifest).valid) {
|
|
1193
|
+
throw new EvidenceValidationError("AGENTIC_BENCHMARK_SOURCE_INVALID");
|
|
1194
|
+
}
|
|
1195
|
+
if (!benchmarkSourceBindingValid(request)) {
|
|
1196
|
+
throw new EvidenceValidationError("AGENTIC_BENCHMARK_SOURCE_BINDING_INVALID");
|
|
1197
|
+
}
|
|
1198
|
+
const runs = [...request.runs].sort(compareBenchmarkRuns);
|
|
1199
|
+
const normalizedRequest = { ...request, runs };
|
|
1200
|
+
const base = {
|
|
1201
|
+
schemaVersion: "1.0.0",
|
|
1202
|
+
sourceManifest: request.sourceManifest,
|
|
1203
|
+
sourceArtifactDigest: request.sourceManifest.artifactDigest,
|
|
1204
|
+
sourceDecisionDigest: request.sourceManifest.decisionDigest,
|
|
1205
|
+
referenceWorldId: request.referenceWorldId,
|
|
1206
|
+
policy: request.policy,
|
|
1207
|
+
protocol: request.protocol,
|
|
1208
|
+
fingerprint: request.fingerprint,
|
|
1209
|
+
policyDigest: sha256Canonical(request.policy),
|
|
1210
|
+
protocolDigest: sha256Canonical(request.protocol),
|
|
1211
|
+
fingerprintDigest: sha256Canonical(request.fingerprint),
|
|
1212
|
+
comparisonScopeDigest: sha256Canonical(benchmarkComparisonScopeProjection(request)),
|
|
1213
|
+
runs,
|
|
1214
|
+
summaries: benchmarkSummaries(normalizedRequest),
|
|
1215
|
+
limitations: [
|
|
1216
|
+
"UNSANDBOXED measurements are trusted-local operational evidence.",
|
|
1217
|
+
"Benchmark comparisons are valid only within the declared comparison scope.",
|
|
1218
|
+
"Observed timings do not alter the source TestForge VERIFIED decision.",
|
|
1219
|
+
],
|
|
1220
|
+
};
|
|
1221
|
+
return { ...base, artifactDigest: sha256Canonical(base) };
|
|
1222
|
+
}
|
|
1223
|
+
export function replayAgenticBenchmark(value) {
|
|
1224
|
+
const invalid = {
|
|
1225
|
+
valid: false,
|
|
1226
|
+
schemaValid: false,
|
|
1227
|
+
sourceManifestValid: false,
|
|
1228
|
+
sourceBindingValid: false,
|
|
1229
|
+
policyDigestValid: false,
|
|
1230
|
+
protocolDigestValid: false,
|
|
1231
|
+
fingerprintDigestValid: false,
|
|
1232
|
+
comparisonScopeDigestValid: false,
|
|
1233
|
+
artifactDigestValid: false,
|
|
1234
|
+
summarySemanticsValid: false,
|
|
1235
|
+
};
|
|
1236
|
+
let artifact;
|
|
1237
|
+
try {
|
|
1238
|
+
artifact = parseAgenticBenchmarkArtifact(value);
|
|
1239
|
+
}
|
|
1240
|
+
catch {
|
|
1241
|
+
return invalid;
|
|
1242
|
+
}
|
|
1243
|
+
const request = {
|
|
1244
|
+
schemaVersion: artifact.schemaVersion,
|
|
1245
|
+
sourceManifest: artifact.sourceManifest,
|
|
1246
|
+
referenceWorldId: artifact.referenceWorldId,
|
|
1247
|
+
policy: artifact.policy,
|
|
1248
|
+
protocol: artifact.protocol,
|
|
1249
|
+
fingerprint: artifact.fingerprint,
|
|
1250
|
+
runs: artifact.runs,
|
|
1251
|
+
};
|
|
1252
|
+
const sourceManifestValid = replayEvidenceManifest(artifact.sourceManifest).valid;
|
|
1253
|
+
const sourceBindingValid = benchmarkSourceBindingValid(request) &&
|
|
1254
|
+
artifact.sourceArtifactDigest === artifact.sourceManifest.artifactDigest &&
|
|
1255
|
+
artifact.sourceDecisionDigest === artifact.sourceManifest.decisionDigest;
|
|
1256
|
+
const policyDigestValid = artifact.policyDigest === sha256Canonical(artifact.policy);
|
|
1257
|
+
const protocolDigestValid = artifact.protocolDigest === sha256Canonical(artifact.protocol);
|
|
1258
|
+
const fingerprintDigestValid = artifact.fingerprintDigest === sha256Canonical(artifact.fingerprint);
|
|
1259
|
+
let comparisonScopeDigestValid = false;
|
|
1260
|
+
try {
|
|
1261
|
+
comparisonScopeDigestValid =
|
|
1262
|
+
artifact.comparisonScopeDigest ===
|
|
1263
|
+
sha256Canonical(benchmarkComparisonScopeProjection(request));
|
|
1264
|
+
}
|
|
1265
|
+
catch {
|
|
1266
|
+
comparisonScopeDigestValid = false;
|
|
1267
|
+
}
|
|
1268
|
+
const artifactDigestValid = artifact.artifactDigest === sha256Canonical(benchmarkArtifactProjection(artifact));
|
|
1269
|
+
let summarySemanticsValid = false;
|
|
1270
|
+
try {
|
|
1271
|
+
const expected = createAgenticBenchmark(request);
|
|
1272
|
+
summarySemanticsValid =
|
|
1273
|
+
canonicalize(artifact.runs) === canonicalize(expected.runs) &&
|
|
1274
|
+
canonicalize(artifact.summaries) === canonicalize(expected.summaries) &&
|
|
1275
|
+
canonicalize(artifact.limitations) === canonicalize(expected.limitations);
|
|
1276
|
+
}
|
|
1277
|
+
catch {
|
|
1278
|
+
summarySemanticsValid = false;
|
|
1279
|
+
}
|
|
1280
|
+
return {
|
|
1281
|
+
valid: sourceManifestValid &&
|
|
1282
|
+
sourceBindingValid &&
|
|
1283
|
+
policyDigestValid &&
|
|
1284
|
+
protocolDigestValid &&
|
|
1285
|
+
fingerprintDigestValid &&
|
|
1286
|
+
comparisonScopeDigestValid &&
|
|
1287
|
+
artifactDigestValid &&
|
|
1288
|
+
summarySemanticsValid,
|
|
1289
|
+
schemaValid: true,
|
|
1290
|
+
sourceManifestValid,
|
|
1291
|
+
sourceBindingValid,
|
|
1292
|
+
policyDigestValid,
|
|
1293
|
+
protocolDigestValid,
|
|
1294
|
+
fingerprintDigestValid,
|
|
1295
|
+
comparisonScopeDigestValid,
|
|
1296
|
+
artifactDigestValid,
|
|
1297
|
+
summarySemanticsValid,
|
|
1298
|
+
};
|
|
1299
|
+
}
|
|
1300
|
+
export function replayAgenticBenchmarkAcquisition(value) {
|
|
1301
|
+
const invalid = {
|
|
1302
|
+
valid: false,
|
|
1303
|
+
schemaValid: false,
|
|
1304
|
+
sourceManifestValid: false,
|
|
1305
|
+
sourceBindingValid: false,
|
|
1306
|
+
artifactReplayValid: false,
|
|
1307
|
+
contextBindingValid: false,
|
|
1308
|
+
resultDigestValid: false,
|
|
1309
|
+
statusSemanticsValid: false,
|
|
1310
|
+
};
|
|
1311
|
+
let result;
|
|
1312
|
+
try {
|
|
1313
|
+
result = parseAgenticBenchmarkAcquisitionResult(value);
|
|
1314
|
+
}
|
|
1315
|
+
catch {
|
|
1316
|
+
return invalid;
|
|
1317
|
+
}
|
|
1318
|
+
const sourceReplay = replayEvidenceManifest(result.sourceManifest);
|
|
1319
|
+
const sourceManifestValid = sourceReplay.valid;
|
|
1320
|
+
const artifactReplayValid = result.benchmarkArtifact === null
|
|
1321
|
+
? result.status === "SOURCE_NOT_VERIFIED"
|
|
1322
|
+
: replayAgenticBenchmark(result.benchmarkArtifact).valid;
|
|
1323
|
+
const sourceBindingValid = result.benchmarkArtifact === null
|
|
1324
|
+
? result.status === "SOURCE_NOT_VERIFIED"
|
|
1325
|
+
: canonicalize(result.benchmarkArtifact.sourceManifest) ===
|
|
1326
|
+
canonicalize(result.sourceManifest) &&
|
|
1327
|
+
result.benchmarkArtifact.sourceArtifactDigest === result.sourceManifest.artifactDigest &&
|
|
1328
|
+
result.benchmarkArtifact.sourceDecisionDigest === result.sourceManifest.decisionDigest;
|
|
1329
|
+
const adapterConfiguration = result.sourceManifest.evidenceContext.adapter.configuration;
|
|
1330
|
+
const phaseAdapter = result.benchmarkArtifact?.fingerprint.tools.find((tool) => tool.role === "phase-adapter");
|
|
1331
|
+
let contextBindingValid = false;
|
|
1332
|
+
try {
|
|
1333
|
+
contextBindingValid =
|
|
1334
|
+
result.acquisitionContext.snapshotRepositoryDigest ===
|
|
1335
|
+
result.sourceManifest.repositoryDigest &&
|
|
1336
|
+
result.sourceManifest.worlds.some((world) => world.id === result.acquisitionContext.referenceWorldId &&
|
|
1337
|
+
world.kind === "REFERENCE" &&
|
|
1338
|
+
world.required) &&
|
|
1339
|
+
result.acquisitionContext.dependencyGraphDigest ===
|
|
1340
|
+
sha256Canonical(result.acquisitionContext.dependencyFiles) &&
|
|
1341
|
+
isRecord(adapterConfiguration) &&
|
|
1342
|
+
adapterConfiguration.kind === "testforge-command" &&
|
|
1343
|
+
adapterConfiguration.executable === result.acquisitionContext.adapter.executable &&
|
|
1344
|
+
canonicalize(adapterConfiguration.arguments) ===
|
|
1345
|
+
canonicalize(result.acquisitionContext.adapter.arguments) &&
|
|
1346
|
+
(result.benchmarkArtifact === null
|
|
1347
|
+
? false
|
|
1348
|
+
: result.acquisitionContext.referenceWorldId ===
|
|
1349
|
+
result.benchmarkArtifact.referenceWorldId &&
|
|
1350
|
+
canonicalize(result.acquisitionContext.candidateIds) ===
|
|
1351
|
+
canonicalize(result.sourceManifest.decision.selectedCandidateIds) &&
|
|
1352
|
+
result.acquisitionContext.dependencyGraphDigest ===
|
|
1353
|
+
result.benchmarkArtifact.fingerprint.dependencyGraphDigest &&
|
|
1354
|
+
result.acquisitionContext.adapter.identityDigest ===
|
|
1355
|
+
result.benchmarkArtifact.fingerprint.phaseReporterDigest &&
|
|
1356
|
+
phaseAdapter !== undefined &&
|
|
1357
|
+
phaseAdapter.digest === result.acquisitionContext.adapter.executableDigest &&
|
|
1358
|
+
phaseAdapter.configurationDigest ===
|
|
1359
|
+
sha256Canonical({
|
|
1360
|
+
arguments: result.acquisitionContext.adapter.arguments,
|
|
1361
|
+
identityFilePath: result.acquisitionContext.adapter.identityFilePath,
|
|
1362
|
+
identityDigest: result.acquisitionContext.adapter.identityDigest,
|
|
1363
|
+
}));
|
|
1364
|
+
}
|
|
1365
|
+
catch {
|
|
1366
|
+
contextBindingValid = false;
|
|
1367
|
+
}
|
|
1368
|
+
const { resultDigest: _resultDigest, ...projection } = result;
|
|
1369
|
+
const resultDigestValid = result.resultDigest === sha256Canonical(projection);
|
|
1370
|
+
let expectedStatus;
|
|
1371
|
+
if (sourceManifestValid && result.sourceManifest.decision.status !== "VERIFIED") {
|
|
1372
|
+
expectedStatus = "SOURCE_NOT_VERIFIED";
|
|
1373
|
+
}
|
|
1374
|
+
else if (sourceManifestValid && result.benchmarkArtifact !== null) {
|
|
1375
|
+
const statuses = result.benchmarkArtifact.summaries.map((summary) => summary.status);
|
|
1376
|
+
expectedStatus = statuses.some((status) => status === "OBSERVED_RUN_FAILURE")
|
|
1377
|
+
? "OBSERVED_RUN_FAILURE"
|
|
1378
|
+
: statuses.some((status) => status === "INSUFFICIENT_SAMPLES")
|
|
1379
|
+
? "INSUFFICIENT_SAMPLES"
|
|
1380
|
+
: "COMPLETE";
|
|
1381
|
+
}
|
|
1382
|
+
const expectedReasonCodes = expectedStatus === undefined
|
|
1383
|
+
? undefined
|
|
1384
|
+
: [expectedStatus === "COMPLETE" ? "ACQUISITION_COMPLETE" : expectedStatus];
|
|
1385
|
+
const statusSemanticsValid = expectedStatus !== undefined &&
|
|
1386
|
+
expectedReasonCodes !== undefined &&
|
|
1387
|
+
expectedStatus === result.status &&
|
|
1388
|
+
canonicalize(result.reasonCodes) === canonicalize(expectedReasonCodes) &&
|
|
1389
|
+
(expectedStatus === "SOURCE_NOT_VERIFIED"
|
|
1390
|
+
? result.benchmarkArtifact === null
|
|
1391
|
+
: result.benchmarkArtifact !== null);
|
|
1392
|
+
return {
|
|
1393
|
+
valid: sourceManifestValid &&
|
|
1394
|
+
sourceBindingValid &&
|
|
1395
|
+
artifactReplayValid &&
|
|
1396
|
+
contextBindingValid &&
|
|
1397
|
+
resultDigestValid &&
|
|
1398
|
+
statusSemanticsValid,
|
|
1399
|
+
schemaValid: true,
|
|
1400
|
+
sourceManifestValid,
|
|
1401
|
+
sourceBindingValid,
|
|
1402
|
+
artifactReplayValid,
|
|
1403
|
+
contextBindingValid,
|
|
1404
|
+
resultDigestValid,
|
|
1405
|
+
statusSemanticsValid,
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
function agenticReportV2Projection(report) {
|
|
1409
|
+
const { reportDigest: _reportDigest, ...projection } = report;
|
|
1410
|
+
return projection;
|
|
1411
|
+
}
|
|
1412
|
+
function profileV2CandidateClassification(scopeMatches, benchmarkStatus, p95Us, request) {
|
|
1413
|
+
if (!scopeMatches)
|
|
1414
|
+
return "COMPARISON_SCOPE_MISMATCH";
|
|
1415
|
+
if (benchmarkStatus === "OBSERVED_RUN_FAILURE")
|
|
1416
|
+
return "OBSERVED_BENCHMARK_FAILURE";
|
|
1417
|
+
if (benchmarkStatus === "INSUFFICIENT_SAMPLES" || p95Us === null) {
|
|
1418
|
+
return "INSUFFICIENT_TIMING_EVIDENCE";
|
|
1419
|
+
}
|
|
1420
|
+
return request.policy.lanes.some((lane) => p95Us <= lane.maximumWarmTotalWallP95Us)
|
|
1421
|
+
? "QUALIFIED"
|
|
1422
|
+
: "BUDGET_MISSED";
|
|
1423
|
+
}
|
|
1424
|
+
export function createAgenticProfileV2(request) {
|
|
1425
|
+
request = parseAgenticProfileRequestV2(request);
|
|
1426
|
+
const benchmarkReplay = replayAgenticBenchmark(request.benchmarkArtifact);
|
|
1427
|
+
if (!benchmarkReplay.valid) {
|
|
1428
|
+
throw new EvidenceValidationError("AGENTIC_PROFILE_V2_BENCHMARK_INVALID");
|
|
1429
|
+
}
|
|
1430
|
+
const benchmark = request.benchmarkArtifact;
|
|
1431
|
+
const manifest = benchmark.sourceManifest;
|
|
1432
|
+
const selectedIds = [...manifest.decision.selectedCandidateIds].sort(compareOrdinal);
|
|
1433
|
+
const selectedSet = new Set(selectedIds);
|
|
1434
|
+
const excludedEligibleCandidateIds = manifest.candidates
|
|
1435
|
+
.filter((candidate) => candidate.status === "ELIGIBLE" && !selectedSet.has(candidate.id))
|
|
1436
|
+
.map((candidate) => candidate.id)
|
|
1437
|
+
.sort(compareOrdinal);
|
|
1438
|
+
const targetWorlds = manifest.worlds.filter((world) => world.kind === "TARGET");
|
|
1439
|
+
const targetWeights = new Map(targetWorlds.map((world) => [world.id, world.weight]));
|
|
1440
|
+
const totalTargetWeight = targetWorlds.reduce((total, world) => total + world.weight, 0);
|
|
1441
|
+
const requiredTargetIds = new Set(targetWorlds.filter((world) => world.required).map((world) => world.id));
|
|
1442
|
+
const scopeMatches = request.policy.requiredComparisonScopeDigest === benchmark.comparisonScopeDigest;
|
|
1443
|
+
const warmSummaries = new Map(benchmark.summaries
|
|
1444
|
+
.filter((summary) => summary.regime === "WARM")
|
|
1445
|
+
.map((summary) => [summary.candidateId, summary]));
|
|
1446
|
+
const candidates = selectedIds.map((id) => {
|
|
1447
|
+
const source = manifest.candidates.find((candidate) => candidate.id === id);
|
|
1448
|
+
const summary = warmSummaries.get(id);
|
|
1449
|
+
if (source === undefined || source.status !== "ELIGIBLE" || summary === undefined) {
|
|
1450
|
+
throw new EvidenceValidationError("AGENTIC_PROFILE_V2_COHORT_INVALID");
|
|
1451
|
+
}
|
|
1452
|
+
const p95Us = summary.status === "MEASURED" ? (summary.wall.total?.p95Us ?? null) : null;
|
|
1453
|
+
return {
|
|
1454
|
+
id,
|
|
1455
|
+
candidateDigest: source.digest,
|
|
1456
|
+
evidenceStatus: "ELIGIBLE",
|
|
1457
|
+
classification: profileV2CandidateClassification(scopeMatches, summary.status, p95Us, request),
|
|
1458
|
+
status: summary.status,
|
|
1459
|
+
reasonCodes: [...source.reasonCodes],
|
|
1460
|
+
strength: {
|
|
1461
|
+
targetWeightKilled: source.targetWeightKilled,
|
|
1462
|
+
totalTargetWeight,
|
|
1463
|
+
targetWeightPermille: totalTargetWeight === 0
|
|
1464
|
+
? 0
|
|
1465
|
+
: Math.floor((source.targetWeightKilled * 1_000) / totalTargetWeight),
|
|
1466
|
+
requiredTargetsKilled: source.killedTargetIds.filter((targetId) => requiredTargetIds.has(targetId)).length,
|
|
1467
|
+
requiredTargetsTotal: requiredTargetIds.size,
|
|
1468
|
+
killedTargetIds: [...source.killedTargetIds].sort(compareOrdinal),
|
|
1469
|
+
},
|
|
1470
|
+
consistency: {
|
|
1471
|
+
claim: source.gates.find((gate) => gate.name === "STABILITY")?.status === "PASSED"
|
|
1472
|
+
? "OBSERVED_CONSISTENT"
|
|
1473
|
+
: "OBSERVED_INCONSISTENT",
|
|
1474
|
+
attempts: manifest.policy.requiredAttempts,
|
|
1475
|
+
},
|
|
1476
|
+
cost: {
|
|
1477
|
+
kind: "BENCHMARK_WARM_TOTAL_WALL_P95",
|
|
1478
|
+
acceptedSamples: summary.counters.accepted,
|
|
1479
|
+
p95Us,
|
|
1480
|
+
},
|
|
1481
|
+
paretoStatus: "NOT_EVALUATED",
|
|
1482
|
+
sizeBytes: source.sizeBytes,
|
|
1483
|
+
};
|
|
1484
|
+
});
|
|
1485
|
+
const anyWarmFailure = candidates.some((candidate) => candidate.status === "OBSERVED_RUN_FAILURE");
|
|
1486
|
+
const anyWarmInsufficient = candidates.some((candidate) => candidate.status === "INSUFFICIENT_SAMPLES" || candidate.cost.p95Us === null);
|
|
1487
|
+
const cohortEvaluable = scopeMatches && !anyWarmFailure && !anyWarmInsufficient;
|
|
1488
|
+
if (cohortEvaluable) {
|
|
1489
|
+
for (const candidate of candidates) {
|
|
1490
|
+
const candidateCost = candidate.cost.p95Us;
|
|
1491
|
+
candidate.paretoStatus = candidates.some((other) => {
|
|
1492
|
+
if (other.id === candidate.id || other.cost.p95Us === null)
|
|
1493
|
+
return false;
|
|
1494
|
+
return (other.strength.targetWeightKilled >= candidate.strength.targetWeightKilled &&
|
|
1495
|
+
other.strength.requiredTargetsKilled >= candidate.strength.requiredTargetsKilled &&
|
|
1496
|
+
other.cost.p95Us <= candidateCost &&
|
|
1497
|
+
(other.strength.targetWeightKilled > candidate.strength.targetWeightKilled ||
|
|
1498
|
+
other.strength.requiredTargetsKilled > candidate.strength.requiredTargetsKilled ||
|
|
1499
|
+
other.cost.p95Us < candidateCost));
|
|
1500
|
+
})
|
|
1501
|
+
? "DOMINATED"
|
|
1502
|
+
: "ON_FRONTIER";
|
|
1503
|
+
}
|
|
1504
|
+
}
|
|
1505
|
+
const portfolios = cohortEvaluable
|
|
1506
|
+
? request.policy.lanes.map((lane) => {
|
|
1507
|
+
const selectedCandidateIds = [];
|
|
1508
|
+
const selectedTargets = new Set();
|
|
1509
|
+
const steps = [];
|
|
1510
|
+
let cumulativeCostUs = 0;
|
|
1511
|
+
let cumulativeTargetWeight = 0;
|
|
1512
|
+
while (true) {
|
|
1513
|
+
const choices = candidates
|
|
1514
|
+
.filter((candidate) => !selectedCandidateIds.includes(candidate.id) &&
|
|
1515
|
+
candidate.cost.p95Us !== null &&
|
|
1516
|
+
candidate.cost.p95Us <= lane.maximumWarmTotalWallP95Us - cumulativeCostUs)
|
|
1517
|
+
.map((candidate) => {
|
|
1518
|
+
const marginalTargetIds = candidate.strength.killedTargetIds.filter((targetId) => !selectedTargets.has(targetId));
|
|
1519
|
+
return {
|
|
1520
|
+
candidate,
|
|
1521
|
+
marginalTargetIds,
|
|
1522
|
+
marginalTargetWeight: marginalTargetIds.reduce((weight, targetId) => weight + (targetWeights.get(targetId) ?? 0), 0),
|
|
1523
|
+
};
|
|
1524
|
+
})
|
|
1525
|
+
.filter((choice) => choice.marginalTargetWeight > 0)
|
|
1526
|
+
.sort((left, right) => {
|
|
1527
|
+
const leftCost = left.candidate.cost.p95Us;
|
|
1528
|
+
const rightCost = right.candidate.cost.p95Us;
|
|
1529
|
+
if (leftCost === 0 || rightCost === 0) {
|
|
1530
|
+
if (leftCost === 0 && rightCost !== 0)
|
|
1531
|
+
return -1;
|
|
1532
|
+
if (rightCost === 0 && leftCost !== 0)
|
|
1533
|
+
return 1;
|
|
1534
|
+
}
|
|
1535
|
+
else {
|
|
1536
|
+
const leftRatio = BigInt(left.marginalTargetWeight) * BigInt(rightCost);
|
|
1537
|
+
const rightRatio = BigInt(right.marginalTargetWeight) * BigInt(leftCost);
|
|
1538
|
+
if (leftRatio !== rightRatio)
|
|
1539
|
+
return leftRatio > rightRatio ? -1 : 1;
|
|
1540
|
+
}
|
|
1541
|
+
if (left.marginalTargetWeight !== right.marginalTargetWeight) {
|
|
1542
|
+
return right.marginalTargetWeight - left.marginalTargetWeight;
|
|
1543
|
+
}
|
|
1544
|
+
if (leftCost !== rightCost)
|
|
1545
|
+
return leftCost - rightCost;
|
|
1546
|
+
if (left.candidate.sizeBytes !== right.candidate.sizeBytes) {
|
|
1547
|
+
return left.candidate.sizeBytes - right.candidate.sizeBytes;
|
|
1548
|
+
}
|
|
1549
|
+
const digestOrder = compareOrdinal(left.candidate.candidateDigest, right.candidate.candidateDigest);
|
|
1550
|
+
return digestOrder !== 0
|
|
1551
|
+
? digestOrder
|
|
1552
|
+
: compareOrdinal(left.candidate.id, right.candidate.id);
|
|
1553
|
+
});
|
|
1554
|
+
const choice = choices[0];
|
|
1555
|
+
if (choice === undefined)
|
|
1556
|
+
break;
|
|
1557
|
+
const choiceCost = choice.candidate.cost.p95Us;
|
|
1558
|
+
selectedCandidateIds.push(choice.candidate.id);
|
|
1559
|
+
cumulativeCostUs += choiceCost;
|
|
1560
|
+
cumulativeTargetWeight += choice.marginalTargetWeight;
|
|
1561
|
+
for (const targetId of choice.marginalTargetIds)
|
|
1562
|
+
selectedTargets.add(targetId);
|
|
1563
|
+
steps.push({
|
|
1564
|
+
ordinal: steps.length + 1,
|
|
1565
|
+
candidateId: choice.candidate.id,
|
|
1566
|
+
marginalTargetIds: [...choice.marginalTargetIds].sort(compareOrdinal),
|
|
1567
|
+
marginalTargetWeight: choice.marginalTargetWeight,
|
|
1568
|
+
candidateWarmTotalWallP95Us: choiceCost,
|
|
1569
|
+
cumulativeCostUs,
|
|
1570
|
+
cumulativeTargetWeight,
|
|
1571
|
+
});
|
|
1572
|
+
}
|
|
1573
|
+
return {
|
|
1574
|
+
laneId: lane.id,
|
|
1575
|
+
maximumWarmTotalWallP95Us: lane.maximumWarmTotalWallP95Us,
|
|
1576
|
+
costModel: "SUM_OF_INDIVIDUAL_P95",
|
|
1577
|
+
selectedCandidateIds,
|
|
1578
|
+
steps,
|
|
1579
|
+
sumIndividualWarmTotalWallP95Us: cumulativeCostUs,
|
|
1580
|
+
targetWeight: cumulativeTargetWeight,
|
|
1581
|
+
targetWeightPermille: totalTargetWeight === 0
|
|
1582
|
+
? 0
|
|
1583
|
+
: Math.floor((cumulativeTargetWeight * 1_000) / totalTargetWeight),
|
|
1584
|
+
requiredTargetsKilled: [...selectedTargets].filter((id) => requiredTargetIds.has(id))
|
|
1585
|
+
.length,
|
|
1586
|
+
requiredTargetsTotal: requiredTargetIds.size,
|
|
1587
|
+
};
|
|
1588
|
+
})
|
|
1589
|
+
: [];
|
|
1590
|
+
const status = !scopeMatches
|
|
1591
|
+
? "COMPARISON_SCOPE_MISMATCH"
|
|
1592
|
+
: anyWarmFailure
|
|
1593
|
+
? "OBSERVED_BENCHMARK_FAILURE"
|
|
1594
|
+
: anyWarmInsufficient
|
|
1595
|
+
? "INSUFFICIENT_TIMING_EVIDENCE"
|
|
1596
|
+
: portfolios.some((portfolio) => portfolio.selectedCandidateIds.length > 0)
|
|
1597
|
+
? "QUALIFIED"
|
|
1598
|
+
: "BUDGET_MISSED";
|
|
1599
|
+
const publicCandidates = candidates.map(({ sizeBytes: _sizeBytes, ...candidate }) => candidate);
|
|
1600
|
+
const base = {
|
|
1601
|
+
schemaVersion: "2.0.0",
|
|
1602
|
+
benchmarkArtifact: benchmark,
|
|
1603
|
+
sourceBenchmarkArtifactDigest: benchmark.artifactDigest,
|
|
1604
|
+
comparisonScopeDigest: benchmark.comparisonScopeDigest,
|
|
1605
|
+
policy: request.policy,
|
|
1606
|
+
policyDigest: sha256Canonical(request.policy),
|
|
1607
|
+
status,
|
|
1608
|
+
candidateUniverse: {
|
|
1609
|
+
kind: "SOURCE_SELECTED_ELIGIBLE",
|
|
1610
|
+
candidateIds: selectedIds,
|
|
1611
|
+
excludedEligibleCandidateIds,
|
|
1612
|
+
},
|
|
1613
|
+
candidates: publicCandidates,
|
|
1614
|
+
portfolios,
|
|
1615
|
+
limitations: [
|
|
1616
|
+
"The optimized universe is limited to source-selected eligible candidates benchmarked by the source artifact.",
|
|
1617
|
+
"Portfolio cost is the sum of individual warm total wall p95 values, not a measured portfolio p95.",
|
|
1618
|
+
"The profile does not alter the source TestForge VERIFIED decision.",
|
|
1619
|
+
],
|
|
1620
|
+
};
|
|
1621
|
+
return { ...base, reportDigest: sha256Canonical(base) };
|
|
1622
|
+
}
|
|
1623
|
+
export function replayAgenticProfileV2(value) {
|
|
1624
|
+
const invalid = {
|
|
1625
|
+
valid: false,
|
|
1626
|
+
schemaValid: false,
|
|
1627
|
+
sourceBenchmarkValid: false,
|
|
1628
|
+
sourceBindingValid: false,
|
|
1629
|
+
policyDigestValid: false,
|
|
1630
|
+
reportDigestValid: false,
|
|
1631
|
+
semanticsValid: false,
|
|
1632
|
+
};
|
|
1633
|
+
let report;
|
|
1634
|
+
try {
|
|
1635
|
+
report = parseAgenticProfileReportV2(value);
|
|
1636
|
+
}
|
|
1637
|
+
catch {
|
|
1638
|
+
return invalid;
|
|
1639
|
+
}
|
|
1640
|
+
const sourceBenchmarkValid = replayAgenticBenchmark(report.benchmarkArtifact).valid;
|
|
1641
|
+
const sourceBindingValid = report.sourceBenchmarkArtifactDigest === report.benchmarkArtifact.artifactDigest &&
|
|
1642
|
+
report.comparisonScopeDigest === report.benchmarkArtifact.comparisonScopeDigest;
|
|
1643
|
+
const policyDigestValid = report.policyDigest === sha256Canonical(report.policy);
|
|
1644
|
+
const reportDigestValid = report.reportDigest === sha256Canonical(agenticReportV2Projection(report));
|
|
1645
|
+
let semanticsValid = false;
|
|
1646
|
+
try {
|
|
1647
|
+
semanticsValid =
|
|
1648
|
+
canonicalize(report) ===
|
|
1649
|
+
canonicalize(createAgenticProfileV2({
|
|
1650
|
+
schemaVersion: "2.0.0",
|
|
1651
|
+
benchmarkArtifact: report.benchmarkArtifact,
|
|
1652
|
+
policy: report.policy,
|
|
1653
|
+
}));
|
|
1654
|
+
}
|
|
1655
|
+
catch {
|
|
1656
|
+
semanticsValid = false;
|
|
1657
|
+
}
|
|
1658
|
+
return {
|
|
1659
|
+
valid: sourceBenchmarkValid &&
|
|
1660
|
+
sourceBindingValid &&
|
|
1661
|
+
policyDigestValid &&
|
|
1662
|
+
reportDigestValid &&
|
|
1663
|
+
semanticsValid,
|
|
1664
|
+
schemaValid: true,
|
|
1665
|
+
sourceBenchmarkValid,
|
|
1666
|
+
sourceBindingValid,
|
|
1667
|
+
policyDigestValid,
|
|
1668
|
+
reportDigestValid,
|
|
1669
|
+
semanticsValid,
|
|
1670
|
+
};
|
|
1671
|
+
}
|
|
1672
|
+
export const AGENTIC_CORPUS_ALLOCATION_ALGORITHM = "SHA256_ASCENDING_SPLIT_V1";
|
|
1673
|
+
const AGENTIC_CORPUS_ALLOCATION_DOMAIN = "TESTFORGE_CORPUS_ALLOCATION_V1";
|
|
1674
|
+
function corpusAllocationProjection(allocation) {
|
|
1675
|
+
const { allocationDigest: _allocationDigest, ...projection } = allocation;
|
|
1676
|
+
return projection;
|
|
1677
|
+
}
|
|
1678
|
+
export function agenticCorpusAllocationDigest(allocation) {
|
|
1679
|
+
return sha256Canonical(corpusAllocationProjection(allocation));
|
|
1680
|
+
}
|
|
1681
|
+
function corpusAllocationScore(seedDigest, caseId) {
|
|
1682
|
+
return `sha256:${sha256(`${AGENTIC_CORPUS_ALLOCATION_DOMAIN}\0${seedDigest}\0${caseId}`)}`;
|
|
1683
|
+
}
|
|
1684
|
+
export function createAgenticCorpusAllocation(request) {
|
|
1685
|
+
const parsed = parseAgenticCorpusAllocationRequest(request);
|
|
1686
|
+
const requestedStrata = [...parsed.strata].sort((left, right) => compareOrdinal(left.sourceId, right.sourceId) ||
|
|
1687
|
+
compareOrdinal(left.sourceIdentityDigest, right.sourceIdentityDigest));
|
|
1688
|
+
const scoredStrata = requestedStrata.map((stratum) => {
|
|
1689
|
+
const scored = [...stratum.caseIds]
|
|
1690
|
+
.map((caseId) => ({
|
|
1691
|
+
caseId,
|
|
1692
|
+
sourceId: stratum.sourceId,
|
|
1693
|
+
sourceIdentityDigest: stratum.sourceIdentityDigest,
|
|
1694
|
+
scoreDigest: corpusAllocationScore(parsed.seedDigest, caseId),
|
|
1695
|
+
}))
|
|
1696
|
+
.sort((left, right) => compareOrdinal(left.scoreDigest, right.scoreDigest) ||
|
|
1697
|
+
compareOrdinal(left.caseId, right.caseId));
|
|
1698
|
+
return {
|
|
1699
|
+
sourceId: stratum.sourceId,
|
|
1700
|
+
sourceIdentityDigest: stratum.sourceIdentityDigest,
|
|
1701
|
+
calibrationCount: stratum.calibrationCount,
|
|
1702
|
+
sourceCaseIds: [...stratum.caseIds].sort(compareOrdinal),
|
|
1703
|
+
calibrationCaseIds: scored.slice(0, stratum.calibrationCount).map(({ caseId }) => caseId),
|
|
1704
|
+
holdoutCaseIds: scored.slice(stratum.calibrationCount).map(({ caseId }) => caseId),
|
|
1705
|
+
scored,
|
|
1706
|
+
};
|
|
1707
|
+
});
|
|
1708
|
+
const calibrationSet = new Set(scoredStrata.flatMap((stratum) => stratum.calibrationCaseIds));
|
|
1709
|
+
const assignments = scoredStrata
|
|
1710
|
+
.flatMap((stratum) => stratum.scored)
|
|
1711
|
+
.sort((left, right) => compareOrdinal(left.scoreDigest, right.scoreDigest) ||
|
|
1712
|
+
compareOrdinal(left.caseId, right.caseId))
|
|
1713
|
+
.map((item) => ({
|
|
1714
|
+
...item,
|
|
1715
|
+
partition: calibrationSet.has(item.caseId) ? "CALIBRATION" : "HOLDOUT",
|
|
1716
|
+
}));
|
|
1717
|
+
const sourceCaseIds = assignments.map(({ caseId }) => caseId).sort(compareOrdinal);
|
|
1718
|
+
const calibrationCaseIds = assignments
|
|
1719
|
+
.filter((item) => item.partition === "CALIBRATION")
|
|
1720
|
+
.map(({ caseId }) => caseId);
|
|
1721
|
+
const holdoutCaseIds = assignments
|
|
1722
|
+
.filter((item) => item.partition === "HOLDOUT")
|
|
1723
|
+
.map(({ caseId }) => caseId);
|
|
1724
|
+
const calibrationCount = scoredStrata.reduce((total, stratum) => total + stratum.calibrationCount, 0);
|
|
1725
|
+
const projection = {
|
|
1726
|
+
schemaVersion: "1.0.0",
|
|
1727
|
+
allocationId: parsed.allocationId,
|
|
1728
|
+
algorithm: AGENTIC_CORPUS_ALLOCATION_ALGORITHM,
|
|
1729
|
+
seedDigest: parsed.seedDigest,
|
|
1730
|
+
calibrationCount,
|
|
1731
|
+
sourceCaseIds,
|
|
1732
|
+
strata: scoredStrata.map(({ scored: _scored, ...stratum }) => stratum),
|
|
1733
|
+
calibrationCaseIds,
|
|
1734
|
+
holdoutCaseIds,
|
|
1735
|
+
assignments,
|
|
1736
|
+
};
|
|
1737
|
+
return parseAgenticCorpusAllocation({
|
|
1738
|
+
...projection,
|
|
1739
|
+
allocationDigest: agenticCorpusAllocationDigest(projection),
|
|
1740
|
+
});
|
|
1741
|
+
}
|
|
1742
|
+
export function replayAgenticCorpusAllocation(allocation) {
|
|
1743
|
+
const invalid = {
|
|
1744
|
+
valid: false,
|
|
1745
|
+
schemaValid: false,
|
|
1746
|
+
allocationDigestValid: false,
|
|
1747
|
+
sourceCaseIdsValid: false,
|
|
1748
|
+
assignmentScoresValid: false,
|
|
1749
|
+
partitionSemanticsValid: false,
|
|
1750
|
+
};
|
|
1751
|
+
let parsed;
|
|
1752
|
+
try {
|
|
1753
|
+
parsed = parseAgenticCorpusAllocation(allocation);
|
|
1754
|
+
}
|
|
1755
|
+
catch {
|
|
1756
|
+
return invalid;
|
|
1757
|
+
}
|
|
1758
|
+
const expected = createAgenticCorpusAllocation({
|
|
1759
|
+
schemaVersion: parsed.schemaVersion,
|
|
1760
|
+
allocationId: parsed.allocationId,
|
|
1761
|
+
seedDigest: parsed.seedDigest,
|
|
1762
|
+
strata: parsed.strata.map((stratum) => ({
|
|
1763
|
+
sourceId: stratum.sourceId,
|
|
1764
|
+
sourceIdentityDigest: stratum.sourceIdentityDigest,
|
|
1765
|
+
calibrationCount: stratum.calibrationCount,
|
|
1766
|
+
caseIds: stratum.sourceCaseIds,
|
|
1767
|
+
})),
|
|
1768
|
+
});
|
|
1769
|
+
const allocationDigestValid = parsed.allocationDigest === agenticCorpusAllocationDigest(parsed);
|
|
1770
|
+
const sourceCaseIdsValid = canonicalize(parsed.sourceCaseIds) === canonicalize(expected.sourceCaseIds);
|
|
1771
|
+
const assignmentScoresValid = canonicalize(parsed.assignments.map(({ caseId, sourceId, sourceIdentityDigest, scoreDigest }) => ({
|
|
1772
|
+
caseId,
|
|
1773
|
+
sourceId,
|
|
1774
|
+
sourceIdentityDigest,
|
|
1775
|
+
scoreDigest,
|
|
1776
|
+
}))) ===
|
|
1777
|
+
canonicalize(expected.assignments.map(({ caseId, sourceId, sourceIdentityDigest, scoreDigest }) => ({
|
|
1778
|
+
caseId,
|
|
1779
|
+
sourceId,
|
|
1780
|
+
sourceIdentityDigest,
|
|
1781
|
+
scoreDigest,
|
|
1782
|
+
})));
|
|
1783
|
+
const partitionSemanticsValid = canonicalize(parsed.strata) === canonicalize(expected.strata) &&
|
|
1784
|
+
canonicalize(parsed.calibrationCaseIds) === canonicalize(expected.calibrationCaseIds) &&
|
|
1785
|
+
canonicalize(parsed.holdoutCaseIds) === canonicalize(expected.holdoutCaseIds) &&
|
|
1786
|
+
canonicalize(parsed.assignments.map(({ caseId, partition }) => ({ caseId, partition }))) ===
|
|
1787
|
+
canonicalize(expected.assignments.map(({ caseId, partition }) => ({ caseId, partition })));
|
|
1788
|
+
return {
|
|
1789
|
+
valid: allocationDigestValid &&
|
|
1790
|
+
sourceCaseIdsValid &&
|
|
1791
|
+
assignmentScoresValid &&
|
|
1792
|
+
partitionSemanticsValid,
|
|
1793
|
+
schemaValid: true,
|
|
1794
|
+
allocationDigestValid,
|
|
1795
|
+
sourceCaseIdsValid,
|
|
1796
|
+
assignmentScoresValid,
|
|
1797
|
+
partitionSemanticsValid,
|
|
1798
|
+
};
|
|
1799
|
+
}
|
|
1800
|
+
const AGENTIC_CORPUS_EXPERIMENT_PAYLOAD_SCHEMA_ID = "https://testforge.dev/payloads/agentic-corpus-experiment-h3.v1.json";
|
|
1801
|
+
const ALLOCATION_SHARE_DOMAIN = "TESTFORGE_CORPUS_ALLOCATION_SHARE_V1";
|
|
1802
|
+
const ALLOCATION_SEED_DOMAIN = "TESTFORGE_CORPUS_ALLOCATION_FINAL_SEED_V1";
|
|
1803
|
+
function digestBytes(bytes) {
|
|
1804
|
+
return `sha256:${sha256(bytes)}`;
|
|
1805
|
+
}
|
|
1806
|
+
function concatenateBytes(parts) {
|
|
1807
|
+
const length = parts.reduce((total, part) => total + part.length, 0);
|
|
1808
|
+
const output = new Uint8Array(length);
|
|
1809
|
+
let offset = 0;
|
|
1810
|
+
for (const part of parts) {
|
|
1811
|
+
output.set(part, offset);
|
|
1812
|
+
offset += part.length;
|
|
1813
|
+
}
|
|
1814
|
+
return output;
|
|
1815
|
+
}
|
|
1816
|
+
function lengthPrefix(bytes) {
|
|
1817
|
+
const prefix = new Uint8Array(4);
|
|
1818
|
+
new DataView(prefix.buffer).setUint32(0, bytes.length, false);
|
|
1819
|
+
return concatenateBytes([prefix, bytes]);
|
|
1820
|
+
}
|
|
1821
|
+
function decodeBase64Url(value) {
|
|
1822
|
+
if (!/^[A-Za-z0-9_-]+$/u.test(value))
|
|
1823
|
+
return null;
|
|
1824
|
+
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
|
|
1825
|
+
const output = [];
|
|
1826
|
+
let buffer = 0;
|
|
1827
|
+
let bits = 0;
|
|
1828
|
+
for (const character of value) {
|
|
1829
|
+
const index = alphabet.indexOf(character);
|
|
1830
|
+
if (index < 0)
|
|
1831
|
+
return null;
|
|
1832
|
+
buffer = buffer * 64 + index;
|
|
1833
|
+
bits += 6;
|
|
1834
|
+
while (bits >= 8) {
|
|
1835
|
+
bits -= 8;
|
|
1836
|
+
output.push(Math.floor(buffer / 2 ** bits) & 0xff);
|
|
1837
|
+
buffer %= 2 ** bits;
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1840
|
+
if (bits > 0 && buffer !== 0)
|
|
1841
|
+
return null;
|
|
1842
|
+
return new Uint8Array(output);
|
|
1843
|
+
}
|
|
1844
|
+
function allocationCommitmentSigningProjection(commitment) {
|
|
1845
|
+
return {
|
|
1846
|
+
schemaVersion: commitment.schemaVersion,
|
|
1847
|
+
commitmentVersion: commitment.commitmentVersion,
|
|
1848
|
+
commitmentId: commitment.commitmentId,
|
|
1849
|
+
allocationId: commitment.allocationId,
|
|
1850
|
+
trustPolicyDigest: commitment.trustPolicyDigest,
|
|
1851
|
+
caseSet: commitment.caseSet,
|
|
1852
|
+
strata: commitment.strata,
|
|
1853
|
+
signers: commitment.signers.map(({ signature: _signature, ...signer }) => signer),
|
|
1854
|
+
};
|
|
1855
|
+
}
|
|
1856
|
+
export function agenticCorpusAllocationCommitmentSigningBytes(commitment) {
|
|
1857
|
+
return new TextEncoder().encode(canonicalize(allocationCommitmentSigningProjection(commitment)));
|
|
1858
|
+
}
|
|
1859
|
+
export function agenticCorpusAllocationCommitmentDigest(commitment) {
|
|
1860
|
+
const { commitmentDigest: _digest, ...projection } = commitment;
|
|
1861
|
+
return sha256Canonical(projection);
|
|
1862
|
+
}
|
|
1863
|
+
export function agenticCorpusAllocationRevealDigest(reveal) {
|
|
1864
|
+
const { revealDigest: _digest, ...projection } = reveal;
|
|
1865
|
+
return sha256Canonical(projection);
|
|
1866
|
+
}
|
|
1867
|
+
export function agenticCorpusAllocationShareCommitmentDigest(keyId, secret) {
|
|
1868
|
+
const secretBytes = decodeBase64Url(secret);
|
|
1869
|
+
if (secretBytes?.length !== 32)
|
|
1870
|
+
throw new TypeError("ALLOCATION_SHARE_INVALID");
|
|
1871
|
+
return digestBytes(concatenateBytes([
|
|
1872
|
+
new TextEncoder().encode(ALLOCATION_SHARE_DOMAIN),
|
|
1873
|
+
lengthPrefix(new TextEncoder().encode(keyId)),
|
|
1874
|
+
lengthPrefix(secretBytes),
|
|
1875
|
+
]));
|
|
1876
|
+
}
|
|
1877
|
+
export function agenticCorpusAllocationFinalSeedDigest(reveal) {
|
|
1878
|
+
const shares = [...reveal.shares].sort((left, right) => compareOrdinal(left.keyId, right.keyId));
|
|
1879
|
+
const secretBytes = shares.map((share) => decodeBase64Url(share.secret));
|
|
1880
|
+
if (secretBytes.some((bytes) => bytes?.length !== 32))
|
|
1881
|
+
throw new TypeError("ALLOCATION_SHARE_INVALID");
|
|
1882
|
+
return digestBytes(concatenateBytes([
|
|
1883
|
+
new TextEncoder().encode(ALLOCATION_SEED_DOMAIN),
|
|
1884
|
+
...secretBytes.map((bytes) => lengthPrefix(bytes)),
|
|
1885
|
+
]));
|
|
1886
|
+
}
|
|
1887
|
+
const INVALID_COMMITMENT_REPLAY = {
|
|
1888
|
+
valid: false,
|
|
1889
|
+
schemaValid: false,
|
|
1890
|
+
externalTrustAnchorValid: false,
|
|
1891
|
+
externalCommitmentAnchorValid: false,
|
|
1892
|
+
commitmentDigestValid: false,
|
|
1893
|
+
caseSetValid: false,
|
|
1894
|
+
commitmentSignaturesValid: false,
|
|
1895
|
+
principalsDistinct: false,
|
|
1896
|
+
revealValid: false,
|
|
1897
|
+
finalSeedDigestValid: false,
|
|
1898
|
+
allocationReplayValid: false,
|
|
1899
|
+
allocationBindingValid: false,
|
|
1900
|
+
};
|
|
1901
|
+
export function replayAgenticCorpusAllocationCommitment(input, dependencies) {
|
|
1902
|
+
let commitment;
|
|
1903
|
+
let reveal;
|
|
1904
|
+
let allocation;
|
|
1905
|
+
let trustPolicy;
|
|
1906
|
+
try {
|
|
1907
|
+
commitment = parseAgenticCorpusAllocationCommitment(input.commitment);
|
|
1908
|
+
reveal = parseAgenticCorpusAllocationReveal(input.reveal);
|
|
1909
|
+
allocation = parseAgenticCorpusAllocation(input.allocation);
|
|
1910
|
+
trustPolicy = parseAgenticCorpusTrustPolicy(input.trustPolicy);
|
|
1911
|
+
}
|
|
1912
|
+
catch {
|
|
1913
|
+
return INVALID_COMMITMENT_REPLAY;
|
|
1914
|
+
}
|
|
1915
|
+
const externalTrustAnchorValid = SHA256_DIGEST_PATTERN.test(input.expectedTrustPolicyDigest ?? "") &&
|
|
1916
|
+
trustPolicy.policyDigest === input.expectedTrustPolicyDigest &&
|
|
1917
|
+
commitment.trustPolicyDigest === input.expectedTrustPolicyDigest;
|
|
1918
|
+
const externalCommitmentAnchorValid = SHA256_DIGEST_PATTERN.test(input.expectedAllocationCommitmentDigest ?? "") &&
|
|
1919
|
+
commitment.commitmentDigest === input.expectedAllocationCommitmentDigest;
|
|
1920
|
+
const commitmentDigestValid = commitment.commitmentDigest === agenticCorpusAllocationCommitmentDigest(commitment);
|
|
1921
|
+
const sortedCaseSet = [...commitment.caseSet].sort((left, right) => compareOrdinal(left.caseId, right.caseId));
|
|
1922
|
+
const caseSetValid = canonicalize(commitment.caseSet) === canonicalize(sortedCaseSet) &&
|
|
1923
|
+
canonicalize(commitment.strata) ===
|
|
1924
|
+
canonicalize([...commitment.strata].sort((left, right) => compareOrdinal(left.sourceId, right.sourceId) ||
|
|
1925
|
+
compareOrdinal(left.sourceIdentityDigest, right.sourceIdentityDigest)));
|
|
1926
|
+
const signatureResult = dependencies?.verifySignatures(commitment, trustPolicy, input.expectedTrustPolicyDigest ?? "") ?? { valid: false, principalsDistinct: false };
|
|
1927
|
+
const revealByKey = new Map(reveal.shares.map((share) => [share.keyId, share]));
|
|
1928
|
+
const revealValid = reveal.commitmentDigest === commitment.commitmentDigest &&
|
|
1929
|
+
reveal.revealDigest === agenticCorpusAllocationRevealDigest(reveal) &&
|
|
1930
|
+
commitment.signers.every((signer) => {
|
|
1931
|
+
const share = revealByKey.get(signer.keyId);
|
|
1932
|
+
try {
|
|
1933
|
+
return (share !== undefined &&
|
|
1934
|
+
signer.shareCommitmentDigest ===
|
|
1935
|
+
agenticCorpusAllocationShareCommitmentDigest(signer.keyId, share.secret));
|
|
1936
|
+
}
|
|
1937
|
+
catch {
|
|
1938
|
+
return false;
|
|
1939
|
+
}
|
|
1940
|
+
});
|
|
1941
|
+
let expectedAllocation = null;
|
|
1942
|
+
let finalSeedDigestValid = false;
|
|
1943
|
+
try {
|
|
1944
|
+
const finalSeedDigest = agenticCorpusAllocationFinalSeedDigest(reveal);
|
|
1945
|
+
finalSeedDigestValid = allocation.seedDigest === finalSeedDigest;
|
|
1946
|
+
expectedAllocation = createAgenticCorpusAllocation({
|
|
1947
|
+
schemaVersion: "1.0.0",
|
|
1948
|
+
allocationId: commitment.allocationId,
|
|
1949
|
+
seedDigest: finalSeedDigest,
|
|
1950
|
+
strata: commitment.strata.map((stratum) => ({
|
|
1951
|
+
...stratum,
|
|
1952
|
+
caseIds: commitment.caseSet
|
|
1953
|
+
.filter((item) => item.sourceId === stratum.sourceId &&
|
|
1954
|
+
item.sourceIdentityDigest === stratum.sourceIdentityDigest)
|
|
1955
|
+
.map((item) => item.caseId),
|
|
1956
|
+
})),
|
|
1957
|
+
});
|
|
1958
|
+
}
|
|
1959
|
+
catch {
|
|
1960
|
+
expectedAllocation = null;
|
|
1961
|
+
}
|
|
1962
|
+
const allocationReplayValid = replayAgenticCorpusAllocation(allocation).valid;
|
|
1963
|
+
const allocationBindingValid = expectedAllocation !== null && canonicalize(allocation) === canonicalize(expectedAllocation);
|
|
1964
|
+
const rails = {
|
|
1965
|
+
schemaValid: true,
|
|
1966
|
+
externalTrustAnchorValid,
|
|
1967
|
+
externalCommitmentAnchorValid,
|
|
1968
|
+
commitmentDigestValid,
|
|
1969
|
+
caseSetValid,
|
|
1970
|
+
commitmentSignaturesValid: signatureResult.valid,
|
|
1971
|
+
principalsDistinct: signatureResult.principalsDistinct,
|
|
1972
|
+
revealValid,
|
|
1973
|
+
finalSeedDigestValid,
|
|
1974
|
+
allocationReplayValid,
|
|
1975
|
+
allocationBindingValid,
|
|
1976
|
+
};
|
|
1977
|
+
return { valid: Object.values(rails).every(Boolean), ...rails };
|
|
1978
|
+
}
|
|
1979
|
+
export function agenticCorpusExperimentPlanDigest(plan) {
|
|
1980
|
+
const { planDigest: _digest, ...projection } = plan;
|
|
1981
|
+
return sha256Canonical(projection);
|
|
1982
|
+
}
|
|
1983
|
+
export function agenticCorpusExperimentCommandDigest(command) {
|
|
1984
|
+
return sha256Canonical(command);
|
|
1985
|
+
}
|
|
1986
|
+
export function agenticCorpusExperimentSelectedSuiteDigest(candidates) {
|
|
1987
|
+
return sha256Canonical({
|
|
1988
|
+
domain: "TESTFORGE_H3_SELECTED_SUITE_V1",
|
|
1989
|
+
candidates: [...candidates].sort((left, right) => compareOrdinal(left.candidateId, right.candidateId) ||
|
|
1990
|
+
compareOrdinal(left.candidateDigest, right.candidateDigest)),
|
|
1991
|
+
});
|
|
1992
|
+
}
|
|
1993
|
+
export function agenticCorpusExperimentCandidateSetDigest(candidates) {
|
|
1994
|
+
return sha256Canonical([...candidates].sort((left, right) => compareOrdinal(left.candidateId, right.candidateId) ||
|
|
1995
|
+
compareOrdinal(left.candidateDigest, right.candidateDigest)));
|
|
1996
|
+
}
|
|
1997
|
+
const INVALID_PLAN_REPLAY = {
|
|
1998
|
+
valid: false,
|
|
1999
|
+
schemaValid: false,
|
|
2000
|
+
externalPlanAnchorValid: false,
|
|
2001
|
+
planDigestValid: false,
|
|
2002
|
+
allocationBindingValid: false,
|
|
2003
|
+
subjectSetValid: false,
|
|
2004
|
+
armsValid: false,
|
|
2005
|
+
commandsValid: false,
|
|
2006
|
+
scheduleValid: false,
|
|
2007
|
+
budgetValid: false,
|
|
2008
|
+
};
|
|
2009
|
+
function exactStringSet(left, right) {
|
|
2010
|
+
return (canonicalize([...new Set(left)].sort(compareOrdinal)) ===
|
|
2011
|
+
canonicalize([...new Set(right)].sort(compareOrdinal)));
|
|
2012
|
+
}
|
|
2013
|
+
export function replayAgenticCorpusExperimentPlan(value, expectedExperimentPlanDigest, context = {}) {
|
|
2014
|
+
let plan;
|
|
2015
|
+
try {
|
|
2016
|
+
plan = parseAgenticCorpusExperimentPlan(value);
|
|
2017
|
+
}
|
|
2018
|
+
catch {
|
|
2019
|
+
return INVALID_PLAN_REPLAY;
|
|
2020
|
+
}
|
|
2021
|
+
const externalPlanAnchorValid = SHA256_DIGEST_PATTERN.test(expectedExperimentPlanDigest) &&
|
|
2022
|
+
plan.planDigest === expectedExperimentPlanDigest;
|
|
2023
|
+
const planDigestValid = plan.planDigest === agenticCorpusExperimentPlanDigest(plan);
|
|
2024
|
+
const allocationBindingValid = context.allocation !== undefined &&
|
|
2025
|
+
context.commitment !== undefined &&
|
|
2026
|
+
plan.trustPolicyDigest === context.expectedTrustPolicyDigest &&
|
|
2027
|
+
plan.allocationCommitmentDigest === context.expectedAllocationCommitmentDigest &&
|
|
2028
|
+
plan.allocationDigest === context.allocation.allocationDigest;
|
|
2029
|
+
const holdoutIds = context.allocation?.holdoutCaseIds ?? [];
|
|
2030
|
+
const commitmentByCase = new Map(context.commitment?.caseSet.map((item) => [item.caseId, item]));
|
|
2031
|
+
const subjectSetValid = exactStringSet(plan.subjects.map((item) => item.caseId), holdoutIds) &&
|
|
2032
|
+
plan.subjects.every((subject) => {
|
|
2033
|
+
const committed = commitmentByCase.get(subject.caseId);
|
|
2034
|
+
return (committed !== undefined &&
|
|
2035
|
+
subject.caseDigest === committed.caseDigest &&
|
|
2036
|
+
subject.provenanceDigest === committed.provenanceDigest &&
|
|
2037
|
+
subject.sourceId === committed.sourceId &&
|
|
2038
|
+
subject.sourceIdentityDigest === committed.sourceIdentityDigest);
|
|
2039
|
+
});
|
|
2040
|
+
const armsValid = canonicalize([...plan.arms.baseline.candidateUniverse].sort((left, right) => compareOrdinal(left.candidateId, right.candidateId))) ===
|
|
2041
|
+
canonicalize([...plan.arms.profile.candidateUniverse].sort((left, right) => compareOrdinal(left.candidateId, right.candidateId))) &&
|
|
2042
|
+
[plan.arms.baseline, plan.arms.profile].every((arm) => new Set(arm.candidateUniverse.map((candidate) => candidate.candidateId)).size ===
|
|
2043
|
+
arm.candidateUniverse.length &&
|
|
2044
|
+
new Set(arm.selectedCandidates.map((candidate) => candidate.candidateId)).size ===
|
|
2045
|
+
arm.selectedCandidates.length &&
|
|
2046
|
+
arm.selectedSuiteDigest ===
|
|
2047
|
+
agenticCorpusExperimentSelectedSuiteDigest(arm.selectedCandidates));
|
|
2048
|
+
const adapterIds = new Set(plan.adapters.map((item) => item.adapterId));
|
|
2049
|
+
const commandById = new Map(plan.commands.map((item) => [item.commandId, item]));
|
|
2050
|
+
const commandsValid = commandById.size === plan.commands.length &&
|
|
2051
|
+
plan.commands.every((command) => {
|
|
2052
|
+
const arm = command.arm === "BASELINE" ? plan.arms.baseline : plan.arms.profile;
|
|
2053
|
+
return (adapterIds.has(command.adapterId) &&
|
|
2054
|
+
(command.subjectState === "REQUIRED_SUITE" ||
|
|
2055
|
+
command.testSuiteDigest === arm.selectedSuiteDigest));
|
|
2056
|
+
});
|
|
2057
|
+
const scheduleKeys = plan.schedule.map((item) => `${item.caseId}\0${item.arm}\0${item.attemptOrdinal}\0${item.subjectState}`);
|
|
2058
|
+
const expectedKeys = plan.subjects.flatMap((subject) => ["BASELINE", "PROFILE"].flatMap((arm) => Array.from({ length: plan.protocol.minimumStableAttempts }, (_, index) => index + 1).flatMap((attemptOrdinal) => ["BUGGY", "FIXED", "REQUIRED_SUITE"].map((subjectState) => `${subject.caseId}\0${arm}\0${attemptOrdinal}\0${subjectState}`))));
|
|
2059
|
+
const scheduleValid = new Set(scheduleKeys).size === scheduleKeys.length &&
|
|
2060
|
+
exactStringSet(scheduleKeys, expectedKeys) &&
|
|
2061
|
+
plan.schedule.every((entry) => commandById.get(entry.commandId)?.subjectState === entry.subjectState &&
|
|
2062
|
+
commandById.get(entry.commandId)?.arm === entry.arm);
|
|
2063
|
+
const baselineCount = plan.schedule.filter((item) => item.arm === "BASELINE").length;
|
|
2064
|
+
const profileCount = plan.schedule.filter((item) => item.arm === "PROFILE").length;
|
|
2065
|
+
const plannedTimeout = (arm) => plan.schedule
|
|
2066
|
+
.filter((item) => item.arm === arm)
|
|
2067
|
+
.reduce((total, item) => total + (commandById.get(item.commandId)?.timeoutMs ?? 0), 0);
|
|
2068
|
+
const baselinePlannedTimeoutMs = plannedTimeout("BASELINE");
|
|
2069
|
+
const profilePlannedTimeoutMs = plannedTimeout("PROFILE");
|
|
2070
|
+
const budgetValid = plan.budget.unit === "PLANNED_PROCESS_EXECUTION" &&
|
|
2071
|
+
plan.budget.baselinePlannedProcessExecutions === baselineCount &&
|
|
2072
|
+
plan.budget.profilePlannedProcessExecutions === profileCount &&
|
|
2073
|
+
plan.budget.equalPlannedProcessExecutions === (baselineCount === profileCount) &&
|
|
2074
|
+
plan.budget.equalPlannedProcessExecutions &&
|
|
2075
|
+
Number.isSafeInteger(baselinePlannedTimeoutMs) &&
|
|
2076
|
+
Number.isSafeInteger(profilePlannedTimeoutMs) &&
|
|
2077
|
+
plan.budget.baselinePlannedTimeoutMs === baselinePlannedTimeoutMs &&
|
|
2078
|
+
plan.budget.profilePlannedTimeoutMs === profilePlannedTimeoutMs &&
|
|
2079
|
+
plan.budget.equalPlannedTimeoutBudget ===
|
|
2080
|
+
(baselinePlannedTimeoutMs === profilePlannedTimeoutMs) &&
|
|
2081
|
+
plan.budget.equalPlannedTimeoutBudget;
|
|
2082
|
+
const rails = {
|
|
2083
|
+
schemaValid: true,
|
|
2084
|
+
externalPlanAnchorValid,
|
|
2085
|
+
planDigestValid,
|
|
2086
|
+
allocationBindingValid,
|
|
2087
|
+
subjectSetValid,
|
|
2088
|
+
armsValid,
|
|
2089
|
+
commandsValid,
|
|
2090
|
+
scheduleValid,
|
|
2091
|
+
budgetValid,
|
|
2092
|
+
};
|
|
2093
|
+
return { valid: Object.values(rails).every(Boolean), ...rails };
|
|
2094
|
+
}
|
|
2095
|
+
export function resolveAgenticCorpusExperimentEvidence(requiredDigests, contents) {
|
|
2096
|
+
const missingDigests = [];
|
|
2097
|
+
const mismatchedDigests = [];
|
|
2098
|
+
for (const digest of [...new Set(requiredDigests)].sort(compareOrdinal)) {
|
|
2099
|
+
const content = contents.get(digest);
|
|
2100
|
+
if (content === undefined)
|
|
2101
|
+
missingDigests.push(digest);
|
|
2102
|
+
else if (digestBytes(typeof content === "string" ? new TextEncoder().encode(content) : content) !==
|
|
2103
|
+
digest) {
|
|
2104
|
+
mismatchedDigests.push(digest);
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
return {
|
|
2108
|
+
valid: missingDigests.length === 0 && mismatchedDigests.length === 0,
|
|
2109
|
+
missingDigests,
|
|
2110
|
+
mismatchedDigests,
|
|
2111
|
+
};
|
|
2112
|
+
}
|
|
2113
|
+
function classifyExperimentAttempt(attempt) {
|
|
2114
|
+
if (attempt.buggy.outcome === "ASSERTION_FAILURE" &&
|
|
2115
|
+
attempt.buggy.attributed &&
|
|
2116
|
+
attempt.fixed.outcome === "PASS" &&
|
|
2117
|
+
attempt.requiredSuite.outcome === "PASS")
|
|
2118
|
+
return "DETECTED";
|
|
2119
|
+
if (attempt.buggy.outcome === "PASS" &&
|
|
2120
|
+
attempt.fixed.outcome === "PASS" &&
|
|
2121
|
+
attempt.requiredSuite.outcome === "PASS")
|
|
2122
|
+
return "NOT_DETECTED";
|
|
2123
|
+
return "INSUFFICIENT";
|
|
2124
|
+
}
|
|
2125
|
+
function classifyExperimentArm(attempts, minimum) {
|
|
2126
|
+
if (attempts.length !== minimum ||
|
|
2127
|
+
attempts.some((attempt, index) => attempt.ordinal !== index + 1)) {
|
|
2128
|
+
return "INSUFFICIENT";
|
|
2129
|
+
}
|
|
2130
|
+
const outcomes = attempts.map(classifyExperimentAttempt);
|
|
2131
|
+
if (outcomes.every((item) => item === "DETECTED"))
|
|
2132
|
+
return "DETECTED";
|
|
2133
|
+
if (outcomes.every((item) => item === "NOT_DETECTED"))
|
|
2134
|
+
return "NOT_DETECTED";
|
|
2135
|
+
return "INSUFFICIENT";
|
|
2136
|
+
}
|
|
2137
|
+
function deriveExperimentResult(payload, plan) {
|
|
2138
|
+
const protocol = plan.protocol;
|
|
2139
|
+
const baselineDetectedCaseIds = [];
|
|
2140
|
+
const profileDetectedCaseIds = [];
|
|
2141
|
+
const insufficientCaseIds = [];
|
|
2142
|
+
for (const item of payload.cases) {
|
|
2143
|
+
const baseline = classifyExperimentArm(item.baselineAttempts, protocol.minimumStableAttempts);
|
|
2144
|
+
const profile = classifyExperimentArm(item.profileAttempts, protocol.minimumStableAttempts);
|
|
2145
|
+
if (baseline === "DETECTED")
|
|
2146
|
+
baselineDetectedCaseIds.push(item.caseId);
|
|
2147
|
+
if (profile === "DETECTED")
|
|
2148
|
+
profileDetectedCaseIds.push(item.caseId);
|
|
2149
|
+
if (baseline === "INSUFFICIENT" || profile === "INSUFFICIENT")
|
|
2150
|
+
insufficientCaseIds.push(item.caseId);
|
|
2151
|
+
}
|
|
2152
|
+
baselineDetectedCaseIds.sort(compareOrdinal);
|
|
2153
|
+
profileDetectedCaseIds.sort(compareOrdinal);
|
|
2154
|
+
insufficientCaseIds.sort(compareOrdinal);
|
|
2155
|
+
const evaluatedCaseIds = payload.cases.map((item) => item.caseId).sort(compareOrdinal);
|
|
2156
|
+
const aggregateNonInferior = insufficientCaseIds.length === 0 &&
|
|
2157
|
+
profileDetectedCaseIds.length + protocol.maximumDetectionDeficitCases >=
|
|
2158
|
+
baselineDetectedCaseIds.length;
|
|
2159
|
+
const subjectsBySource = new Map();
|
|
2160
|
+
for (const subject of plan.subjects) {
|
|
2161
|
+
const key = `${subject.sourceId}\0${subject.sourceIdentityDigest}`;
|
|
2162
|
+
const stratum = subjectsBySource.get(key) ?? {
|
|
2163
|
+
sourceId: subject.sourceId,
|
|
2164
|
+
sourceIdentityDigest: subject.sourceIdentityDigest,
|
|
2165
|
+
caseIds: [],
|
|
2166
|
+
};
|
|
2167
|
+
stratum.caseIds.push(subject.caseId);
|
|
2168
|
+
subjectsBySource.set(key, stratum);
|
|
2169
|
+
}
|
|
2170
|
+
const sourceResults = [...subjectsBySource.values()]
|
|
2171
|
+
.sort((left, right) => compareOrdinal(left.sourceId, right.sourceId) ||
|
|
2172
|
+
compareOrdinal(left.sourceIdentityDigest, right.sourceIdentityDigest))
|
|
2173
|
+
.map((stratum) => {
|
|
2174
|
+
const caseIds = stratum.caseIds.sort(compareOrdinal);
|
|
2175
|
+
const baselineDetectedCount = caseIds.filter((caseId) => baselineDetectedCaseIds.includes(caseId)).length;
|
|
2176
|
+
const profileDetectedCount = caseIds.filter((caseId) => profileDetectedCaseIds.includes(caseId)).length;
|
|
2177
|
+
const hasInsufficient = caseIds.some((caseId) => insufficientCaseIds.includes(caseId));
|
|
2178
|
+
return {
|
|
2179
|
+
sourceId: stratum.sourceId,
|
|
2180
|
+
sourceIdentityDigest: stratum.sourceIdentityDigest,
|
|
2181
|
+
evaluatedCaseIds: caseIds,
|
|
2182
|
+
baselineDetectedCount,
|
|
2183
|
+
profileDetectedCount,
|
|
2184
|
+
nonInferior: !hasInsufficient &&
|
|
2185
|
+
profileDetectedCount + protocol.maximumDetectionDeficitCases >= baselineDetectedCount,
|
|
2186
|
+
};
|
|
2187
|
+
});
|
|
2188
|
+
const nonInferior = aggregateNonInferior && sourceResults.every((source) => source.nonInferior);
|
|
2189
|
+
return {
|
|
2190
|
+
status: insufficientCaseIds.length > 0 ? "INSUFFICIENT" : nonInferior ? "SUPPORTED" : "NOT_SUPPORTED",
|
|
2191
|
+
evaluatedCaseIds,
|
|
2192
|
+
baselineDetectedCaseIds,
|
|
2193
|
+
profileDetectedCaseIds,
|
|
2194
|
+
insufficientCaseIds,
|
|
2195
|
+
baselineDetectedCount: baselineDetectedCaseIds.length,
|
|
2196
|
+
profileDetectedCount: profileDetectedCaseIds.length,
|
|
2197
|
+
maximumDetectionDeficitCases: protocol.maximumDetectionDeficitCases,
|
|
2198
|
+
nonInferior,
|
|
2199
|
+
sourceResults,
|
|
2200
|
+
};
|
|
2201
|
+
}
|
|
2202
|
+
function normalizeExperimentRequest(request) {
|
|
2203
|
+
return {
|
|
2204
|
+
...request,
|
|
2205
|
+
evidenceBindings: [...request.evidenceBindings].sort((left, right) => compareOrdinal(left.runId, right.runId)),
|
|
2206
|
+
payload: {
|
|
2207
|
+
cases: request.payload.cases
|
|
2208
|
+
.map((item) => ({
|
|
2209
|
+
caseId: item.caseId,
|
|
2210
|
+
baselineAttempts: [...item.baselineAttempts].sort((left, right) => left.ordinal - right.ordinal),
|
|
2211
|
+
profileAttempts: [...item.profileAttempts].sort((left, right) => left.ordinal - right.ordinal),
|
|
2212
|
+
}))
|
|
2213
|
+
.sort((left, right) => compareOrdinal(left.caseId, right.caseId)),
|
|
2214
|
+
},
|
|
2215
|
+
limitations: [...request.limitations].sort(compareOrdinal),
|
|
2216
|
+
};
|
|
2217
|
+
}
|
|
2218
|
+
function experimentArtifactProjection(artifact) {
|
|
2219
|
+
const { artifactDigest: _digest, ...projection } = artifact;
|
|
2220
|
+
return projection;
|
|
2221
|
+
}
|
|
2222
|
+
export function agenticCorpusExperimentArtifactDigest(artifact) {
|
|
2223
|
+
return sha256Canonical(experimentArtifactProjection(artifact));
|
|
2224
|
+
}
|
|
2225
|
+
export function createAgenticCorpusExperimentArtifact(value, plan) {
|
|
2226
|
+
const request = normalizeExperimentRequest(parseAgenticCorpusExperimentRequest(value));
|
|
2227
|
+
const projection = {
|
|
2228
|
+
...request,
|
|
2229
|
+
candidateSetDigest: agenticCorpusExperimentCandidateSetDigest(plan.arms.baseline.candidateUniverse),
|
|
2230
|
+
evidenceSetDigest: sha256Canonical(request.evidenceBindings),
|
|
2231
|
+
payloadDigest: sha256Canonical(request.payload),
|
|
2232
|
+
result: deriveExperimentResult(request.payload, plan),
|
|
2233
|
+
};
|
|
2234
|
+
return parseAgenticCorpusExperimentArtifact({
|
|
2235
|
+
...projection,
|
|
2236
|
+
artifactDigest: agenticCorpusExperimentArtifactDigest(projection),
|
|
2237
|
+
});
|
|
2238
|
+
}
|
|
2239
|
+
function receiptDigest(receipt) {
|
|
2240
|
+
const { receiptDigest: _digest, ...projection } = receipt;
|
|
2241
|
+
return sha256Canonical(projection);
|
|
2242
|
+
}
|
|
2243
|
+
function parseCanonicalJsonBytes(bytes) {
|
|
2244
|
+
try {
|
|
2245
|
+
const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
2246
|
+
const parsed = JSON.parse(text);
|
|
2247
|
+
return `${canonicalize(parsed)}\n` === text ? parsed : null;
|
|
2248
|
+
}
|
|
2249
|
+
catch {
|
|
2250
|
+
return null;
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
const INVALID_EXPERIMENT_REPLAY = {
|
|
2254
|
+
valid: false,
|
|
2255
|
+
schemaValid: false,
|
|
2256
|
+
artifactDigestValid: false,
|
|
2257
|
+
externalTrustAnchorValid: false,
|
|
2258
|
+
externalCommitmentAnchorValid: false,
|
|
2259
|
+
commitmentRevealValid: false,
|
|
2260
|
+
allocationReplayValid: false,
|
|
2261
|
+
allocationBindingValid: false,
|
|
2262
|
+
externalPlanAnchorValid: false,
|
|
2263
|
+
planDigestValid: false,
|
|
2264
|
+
planSemanticsValid: false,
|
|
2265
|
+
subjectDigestsValid: false,
|
|
2266
|
+
provenanceReplayValid: false,
|
|
2267
|
+
scheduleValid: false,
|
|
2268
|
+
budgetValid: false,
|
|
2269
|
+
evidenceSetDigestValid: false,
|
|
2270
|
+
receiptBindingsValid: false,
|
|
2271
|
+
candidateEvidenceValid: false,
|
|
2272
|
+
outputBytesResolved: false,
|
|
2273
|
+
structuredResultsValid: false,
|
|
2274
|
+
payloadSupported: false,
|
|
2275
|
+
payloadDigestValid: false,
|
|
2276
|
+
observationSemanticsValid: false,
|
|
2277
|
+
resultSemanticsValid: false,
|
|
2278
|
+
};
|
|
2279
|
+
export function replayAgenticCorpusExperimentArtifact(value, dependencies) {
|
|
2280
|
+
let replayRequest;
|
|
2281
|
+
let trustPolicy;
|
|
2282
|
+
let commitment;
|
|
2283
|
+
let allocation;
|
|
2284
|
+
let plan;
|
|
2285
|
+
if (dependencies === undefined)
|
|
2286
|
+
return INVALID_EXPERIMENT_REPLAY;
|
|
2287
|
+
try {
|
|
2288
|
+
replayRequest = parseAgenticCorpusExperimentReplayRequest(value);
|
|
2289
|
+
trustPolicy = parseAgenticCorpusTrustPolicy(dependencies.trustPolicy);
|
|
2290
|
+
commitment = parseAgenticCorpusAllocationCommitment(dependencies.allocationCommitment);
|
|
2291
|
+
allocation = parseAgenticCorpusAllocation(dependencies.allocation);
|
|
2292
|
+
plan = parseAgenticCorpusExperimentPlan(dependencies.experimentPlan);
|
|
2293
|
+
}
|
|
2294
|
+
catch {
|
|
2295
|
+
return INVALID_EXPERIMENT_REPLAY;
|
|
2296
|
+
}
|
|
2297
|
+
const { artifact } = replayRequest;
|
|
2298
|
+
const artifactDigestValid = artifact.artifactDigest === agenticCorpusExperimentArtifactDigest(artifact);
|
|
2299
|
+
const commitmentReplay = replayAgenticCorpusAllocationCommitment({
|
|
2300
|
+
commitment,
|
|
2301
|
+
reveal: dependencies.allocationReveal,
|
|
2302
|
+
allocation,
|
|
2303
|
+
trustPolicy,
|
|
2304
|
+
expectedTrustPolicyDigest: dependencies.expectedTrustPolicyDigest,
|
|
2305
|
+
expectedAllocationCommitmentDigest: dependencies.expectedAllocationCommitmentDigest,
|
|
2306
|
+
}, { verifySignatures: dependencies.verifyCommitmentSignatures });
|
|
2307
|
+
const planReplay = replayAgenticCorpusExperimentPlan(plan, dependencies.expectedExperimentPlanDigest, {
|
|
2308
|
+
allocation,
|
|
2309
|
+
commitment,
|
|
2310
|
+
expectedTrustPolicyDigest: dependencies.expectedTrustPolicyDigest,
|
|
2311
|
+
expectedAllocationCommitmentDigest: dependencies.expectedAllocationCommitmentDigest,
|
|
2312
|
+
});
|
|
2313
|
+
const allocationBindingValid = artifact.allocationDigest === allocation.allocationDigest &&
|
|
2314
|
+
artifact.planDigest === plan.planDigest &&
|
|
2315
|
+
artifact.trustPolicyDigest === dependencies.expectedTrustPolicyDigest;
|
|
2316
|
+
const subjectEvidenceByCase = new Map(dependencies.subjectEvidence.map((item) => [
|
|
2317
|
+
item.caseDocument.caseId,
|
|
2318
|
+
item,
|
|
2319
|
+
]));
|
|
2320
|
+
const subjectDigestsValid = plan.subjects.length === dependencies.subjectEvidence.length &&
|
|
2321
|
+
plan.subjects.every((subject) => {
|
|
2322
|
+
const evidence = subjectEvidenceByCase.get(subject.caseId);
|
|
2323
|
+
const caseDocument = evidence?.caseDocument;
|
|
2324
|
+
const provenance = evidence?.provenance;
|
|
2325
|
+
return (caseDocument !== undefined &&
|
|
2326
|
+
provenance !== undefined &&
|
|
2327
|
+
caseDocument.h1 === undefined &&
|
|
2328
|
+
caseDocument.h2 === undefined &&
|
|
2329
|
+
caseDocument.h3 === undefined &&
|
|
2330
|
+
caseDocument.h4 === undefined &&
|
|
2331
|
+
subject.caseDigest === sha256Canonical(caseDocument) &&
|
|
2332
|
+
subject.provenanceDigest === provenance.provenanceDigest &&
|
|
2333
|
+
subject.sourceId === provenance.source?.sourceId &&
|
|
2334
|
+
subject.sourceIdentityDigest === provenance.source?.sourceIdentityDigest);
|
|
2335
|
+
});
|
|
2336
|
+
const provenanceReplayValid = plan.subjects.every((subject) => {
|
|
2337
|
+
const evidence = subjectEvidenceByCase.get(subject.caseId);
|
|
2338
|
+
return (evidence !== undefined &&
|
|
2339
|
+
dependencies.replayProvenance(evidence.caseDocument, evidence.provenance, trustPolicy, dependencies.expectedTrustPolicyDigest).valid);
|
|
2340
|
+
});
|
|
2341
|
+
const scheduleRunIds = plan.schedule.map((item) => item.runId).sort(compareOrdinal);
|
|
2342
|
+
const bindingRunIds = artifact.evidenceBindings.map((item) => item.runId).sort(compareOrdinal);
|
|
2343
|
+
const payloadRunIds = artifact.payload.cases
|
|
2344
|
+
.flatMap((item) => [...item.baselineAttempts, ...item.profileAttempts].flatMap((attempt) => Object.values(attempt.runIds)))
|
|
2345
|
+
.sort(compareOrdinal);
|
|
2346
|
+
const scheduleValid = planReplay.scheduleValid &&
|
|
2347
|
+
exactStringSet(scheduleRunIds, bindingRunIds) &&
|
|
2348
|
+
exactStringSet(scheduleRunIds, payloadRunIds);
|
|
2349
|
+
const budgetValid = planReplay.budgetValid && artifact.evidenceBindings.length === plan.schedule.length;
|
|
2350
|
+
const evidenceSetDigestValid = artifact.evidenceSetDigest === sha256Canonical(artifact.evidenceBindings);
|
|
2351
|
+
const contentMap = dependencies.evidenceContents;
|
|
2352
|
+
const candidateUniverse = plan.arms.baseline.candidateUniverse;
|
|
2353
|
+
const candidateEvidenceValid = artifact.candidateSetDigest === agenticCorpusExperimentCandidateSetDigest(candidateUniverse) &&
|
|
2354
|
+
resolveAgenticCorpusExperimentEvidence(candidateUniverse.map((candidate) => candidate.candidateDigest), contentMap).valid;
|
|
2355
|
+
let receiptBindingsValid = true;
|
|
2356
|
+
let outputBytesResolved = true;
|
|
2357
|
+
let structuredResultsValid = true;
|
|
2358
|
+
const observations = new Map();
|
|
2359
|
+
const commandById = new Map(plan.commands.map((item) => [item.commandId, item]));
|
|
2360
|
+
const scheduleByRunId = new Map(plan.schedule.map((item) => [item.runId, item]));
|
|
2361
|
+
const subjectByCaseId = new Map(plan.subjects.map((item) => [item.caseId, item]));
|
|
2362
|
+
for (const binding of artifact.evidenceBindings) {
|
|
2363
|
+
const receiptContent = contentMap.get(binding.receiptContentDigest);
|
|
2364
|
+
const receiptBytes = typeof receiptContent === "string"
|
|
2365
|
+
? new TextEncoder().encode(receiptContent)
|
|
2366
|
+
: receiptContent;
|
|
2367
|
+
if (receiptBytes === undefined || digestBytes(receiptBytes) !== binding.receiptContentDigest) {
|
|
2368
|
+
receiptBindingsValid = false;
|
|
2369
|
+
continue;
|
|
2370
|
+
}
|
|
2371
|
+
const parsedReceipt = parseCanonicalJsonBytes(receiptBytes);
|
|
2372
|
+
let receipt;
|
|
2373
|
+
try {
|
|
2374
|
+
receipt = parseAgenticCorpusExperimentReceipt(parsedReceipt);
|
|
2375
|
+
}
|
|
2376
|
+
catch {
|
|
2377
|
+
receiptBindingsValid = false;
|
|
2378
|
+
continue;
|
|
2379
|
+
}
|
|
2380
|
+
const schedule = scheduleByRunId.get(binding.runId);
|
|
2381
|
+
const command = commandById.get(receipt.commandId);
|
|
2382
|
+
const subject = subjectByCaseId.get(receipt.caseId);
|
|
2383
|
+
if (schedule === undefined || command === undefined || subject === undefined) {
|
|
2384
|
+
receiptBindingsValid = false;
|
|
2385
|
+
continue;
|
|
2386
|
+
}
|
|
2387
|
+
const expectedRepositoryDigest = receipt.subjectState === "BUGGY" ? subject.buggyRevisionDigest : subject.fixedRevisionDigest;
|
|
2388
|
+
const bindingValid = receipt.runId === binding.runId &&
|
|
2389
|
+
receipt.receiptDigest === binding.receiptDigest &&
|
|
2390
|
+
receipt.receiptDigest === receiptDigest(receipt) &&
|
|
2391
|
+
receipt.planDigest === plan.planDigest &&
|
|
2392
|
+
receipt.commandId === schedule.commandId &&
|
|
2393
|
+
receipt.commandDigest === agenticCorpusExperimentCommandDigest(command) &&
|
|
2394
|
+
receipt.caseId === schedule.caseId &&
|
|
2395
|
+
receipt.arm === schedule.arm &&
|
|
2396
|
+
receipt.attemptOrdinal === schedule.attemptOrdinal &&
|
|
2397
|
+
receipt.subjectState === schedule.subjectState &&
|
|
2398
|
+
receipt.subjectState === command.subjectState &&
|
|
2399
|
+
receipt.repositoryDigest === expectedRepositoryDigest &&
|
|
2400
|
+
receipt.testSuiteDigest === command.testSuiteDigest &&
|
|
2401
|
+
receipt.appliedTimeoutMs === command.timeoutMs &&
|
|
2402
|
+
(receipt.subjectState !== "REQUIRED_SUITE" ||
|
|
2403
|
+
receipt.testSuiteDigest === subject.requiredSuiteDigest);
|
|
2404
|
+
if (!bindingValid)
|
|
2405
|
+
receiptBindingsValid = false;
|
|
2406
|
+
const requiredOutputDigests = [
|
|
2407
|
+
receipt.process.stdoutDigest,
|
|
2408
|
+
receipt.process.stderrDigest,
|
|
2409
|
+
receipt.structuredResultDigest,
|
|
2410
|
+
];
|
|
2411
|
+
const resolution = resolveAgenticCorpusExperimentEvidence(requiredOutputDigests, contentMap);
|
|
2412
|
+
if (!resolution.valid) {
|
|
2413
|
+
outputBytesResolved = false;
|
|
2414
|
+
continue;
|
|
2415
|
+
}
|
|
2416
|
+
const stdoutContent = contentMap.get(receipt.process.stdoutDigest);
|
|
2417
|
+
const stderrContent = contentMap.get(receipt.process.stderrDigest);
|
|
2418
|
+
const stdoutBytes = typeof stdoutContent === "string" ? new TextEncoder().encode(stdoutContent) : stdoutContent;
|
|
2419
|
+
const stderrBytes = typeof stderrContent === "string" ? new TextEncoder().encode(stderrContent) : stderrContent;
|
|
2420
|
+
if (stdoutBytes === undefined ||
|
|
2421
|
+
stderrBytes === undefined ||
|
|
2422
|
+
stdoutBytes.length > command.maxStdoutBytes ||
|
|
2423
|
+
stderrBytes.length > command.maxStderrBytes) {
|
|
2424
|
+
outputBytesResolved = false;
|
|
2425
|
+
continue;
|
|
2426
|
+
}
|
|
2427
|
+
const structuredContent = contentMap.get(receipt.structuredResultDigest);
|
|
2428
|
+
const structuredBytes = typeof structuredContent === "string"
|
|
2429
|
+
? new TextEncoder().encode(structuredContent)
|
|
2430
|
+
: structuredContent;
|
|
2431
|
+
const parsedStructured = structuredBytes === undefined ? null : parseCanonicalJsonBytes(structuredBytes);
|
|
2432
|
+
try {
|
|
2433
|
+
const structured = parseAgenticCorpusExperimentStructuredResult(parsedStructured);
|
|
2434
|
+
const processConsistent = (() => {
|
|
2435
|
+
switch (structured.outcome) {
|
|
2436
|
+
case "PASS":
|
|
2437
|
+
return !receipt.process.timedOut && receipt.process.exitCode === 0;
|
|
2438
|
+
case "PROCESS_CRASH":
|
|
2439
|
+
return !receipt.process.timedOut && receipt.process.exitCode === null;
|
|
2440
|
+
case "TIMEOUT":
|
|
2441
|
+
return receipt.process.timedOut;
|
|
2442
|
+
case "ASSERTION_FAILURE":
|
|
2443
|
+
case "COMPILE_FAILURE":
|
|
2444
|
+
case "COLLECTION_FAILURE":
|
|
2445
|
+
case "INFRA_ERROR":
|
|
2446
|
+
case "NO_TEST_DISCOVERED":
|
|
2447
|
+
return (!receipt.process.timedOut &&
|
|
2448
|
+
receipt.process.exitCode !== null &&
|
|
2449
|
+
receipt.process.exitCode !== 0);
|
|
2450
|
+
}
|
|
2451
|
+
})();
|
|
2452
|
+
if (!processConsistent)
|
|
2453
|
+
structuredResultsValid = false;
|
|
2454
|
+
observations.set(binding.runId, {
|
|
2455
|
+
outcome: structured.outcome,
|
|
2456
|
+
attributed: structured.attributed,
|
|
2457
|
+
});
|
|
2458
|
+
}
|
|
2459
|
+
catch {
|
|
2460
|
+
structuredResultsValid = false;
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
const derivedCases = plan.subjects
|
|
2464
|
+
.map((subject) => {
|
|
2465
|
+
const attemptsForArm = (arm) => Array.from({ length: plan.protocol.minimumStableAttempts }, (_, index) => {
|
|
2466
|
+
const ordinal = index + 1;
|
|
2467
|
+
const runFor = (state) => plan.schedule.find((entry) => entry.caseId === subject.caseId &&
|
|
2468
|
+
entry.arm === arm &&
|
|
2469
|
+
entry.attemptOrdinal === ordinal &&
|
|
2470
|
+
entry.subjectState === state)?.runId;
|
|
2471
|
+
const buggyRunId = runFor("BUGGY") ?? "";
|
|
2472
|
+
const fixedRunId = runFor("FIXED") ?? "";
|
|
2473
|
+
const suiteRunId = runFor("REQUIRED_SUITE") ?? "";
|
|
2474
|
+
return {
|
|
2475
|
+
ordinal,
|
|
2476
|
+
buggy: observations.get(buggyRunId) ?? {
|
|
2477
|
+
outcome: "INFRA_ERROR",
|
|
2478
|
+
attributed: false,
|
|
2479
|
+
},
|
|
2480
|
+
fixed: observations.get(fixedRunId) ?? {
|
|
2481
|
+
outcome: "INFRA_ERROR",
|
|
2482
|
+
attributed: false,
|
|
2483
|
+
},
|
|
2484
|
+
requiredSuite: observations.get(suiteRunId) ?? {
|
|
2485
|
+
outcome: "INFRA_ERROR",
|
|
2486
|
+
attributed: false,
|
|
2487
|
+
},
|
|
2488
|
+
runIds: { buggy: buggyRunId, fixed: fixedRunId, requiredSuite: suiteRunId },
|
|
2489
|
+
};
|
|
2490
|
+
});
|
|
2491
|
+
return {
|
|
2492
|
+
caseId: subject.caseId,
|
|
2493
|
+
baselineAttempts: attemptsForArm("BASELINE"),
|
|
2494
|
+
profileAttempts: attemptsForArm("PROFILE"),
|
|
2495
|
+
};
|
|
2496
|
+
})
|
|
2497
|
+
.sort((left, right) => compareOrdinal(left.caseId, right.caseId));
|
|
2498
|
+
const derivedPayload = { cases: derivedCases };
|
|
2499
|
+
const payloadSupported = artifact.payloadSchemaId === AGENTIC_CORPUS_EXPERIMENT_PAYLOAD_SCHEMA_ID;
|
|
2500
|
+
const payloadDigestValid = artifact.payloadDigest === sha256Canonical(artifact.payload);
|
|
2501
|
+
const observationSemanticsValid = canonicalize(artifact.payload) === canonicalize(derivedPayload);
|
|
2502
|
+
const expectedResult = deriveExperimentResult(derivedPayload, plan);
|
|
2503
|
+
const resultSemanticsValid = canonicalize(artifact.result) === canonicalize(expectedResult);
|
|
2504
|
+
const artifactPlanBindingValid = artifact.experimentId === plan.experimentId &&
|
|
2505
|
+
artifact.hypothesis === plan.hypothesis &&
|
|
2506
|
+
artifact.partition === plan.partition;
|
|
2507
|
+
const rails = {
|
|
2508
|
+
schemaValid: true,
|
|
2509
|
+
artifactDigestValid,
|
|
2510
|
+
externalTrustAnchorValid: commitmentReplay.externalTrustAnchorValid,
|
|
2511
|
+
externalCommitmentAnchorValid: commitmentReplay.externalCommitmentAnchorValid,
|
|
2512
|
+
commitmentRevealValid: commitmentReplay.valid,
|
|
2513
|
+
allocationReplayValid: commitmentReplay.allocationReplayValid,
|
|
2514
|
+
allocationBindingValid,
|
|
2515
|
+
externalPlanAnchorValid: planReplay.externalPlanAnchorValid,
|
|
2516
|
+
planDigestValid: planReplay.planDigestValid,
|
|
2517
|
+
planSemanticsValid: planReplay.valid && artifactPlanBindingValid,
|
|
2518
|
+
subjectDigestsValid,
|
|
2519
|
+
provenanceReplayValid,
|
|
2520
|
+
scheduleValid,
|
|
2521
|
+
budgetValid,
|
|
2522
|
+
evidenceSetDigestValid,
|
|
2523
|
+
receiptBindingsValid,
|
|
2524
|
+
candidateEvidenceValid,
|
|
2525
|
+
outputBytesResolved,
|
|
2526
|
+
structuredResultsValid,
|
|
2527
|
+
payloadSupported,
|
|
2528
|
+
payloadDigestValid,
|
|
2529
|
+
observationSemanticsValid,
|
|
2530
|
+
resultSemanticsValid,
|
|
2531
|
+
};
|
|
2532
|
+
return { valid: Object.values(rails).every(Boolean), ...rails };
|
|
2533
|
+
}
|
|
2534
|
+
function normalizeRelativePath(value) {
|
|
2535
|
+
if (typeof value !== "string" || value.length === 0 || value.includes("\0"))
|
|
2536
|
+
throw new TypeError("Path must be a non-empty relative path");
|
|
2537
|
+
if (/^(?:[a-zA-Z]:|[\\/]{1,2})/.test(value) || value.includes(":"))
|
|
2538
|
+
throw new TypeError("Absolute paths and Windows ADS are forbidden");
|
|
2539
|
+
const normalized = value.replaceAll("\\", "/");
|
|
2540
|
+
const segments = normalized.split("/");
|
|
2541
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === ".."))
|
|
2542
|
+
throw new TypeError("Path traversal or ambiguous segments are forbidden");
|
|
2543
|
+
for (const segment of segments) {
|
|
2544
|
+
if (segment.normalize("NFC") !== segment) {
|
|
2545
|
+
throw new TypeError("Ambiguous Unicode path segment must use NFC");
|
|
2546
|
+
}
|
|
2547
|
+
const hasControlCharacter = [...segment].some((character) => {
|
|
2548
|
+
const codePoint = character.codePointAt(0);
|
|
2549
|
+
return codePoint !== undefined && (codePoint <= 0x1f || codePoint === 0x7f);
|
|
2550
|
+
});
|
|
2551
|
+
if (hasControlCharacter || /[<>"|?*]/u.test(segment)) {
|
|
2552
|
+
throw new TypeError("Control or Windows-forbidden filename character");
|
|
2553
|
+
}
|
|
2554
|
+
if (/[. ]$/u.test(segment))
|
|
2555
|
+
throw new TypeError("Ambiguous Windows path segment");
|
|
2556
|
+
if (/^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/iu.test(segment)) {
|
|
2557
|
+
throw new TypeError("Reserved Windows device name");
|
|
2558
|
+
}
|
|
2559
|
+
}
|
|
2560
|
+
return segments.join("/");
|
|
2561
|
+
}
|
|
2562
|
+
export function portablePathKey(path) {
|
|
2563
|
+
return normalizeRelativePath(path).toLowerCase().normalize("NFC");
|
|
2564
|
+
}
|
|
2565
|
+
export function assertSafeRelativePath(path, roots) {
|
|
2566
|
+
const normalized = normalizeRelativePath(path);
|
|
2567
|
+
if (!Array.isArray(roots) || roots.length === 0)
|
|
2568
|
+
throw new TypeError("At least one safe root is required");
|
|
2569
|
+
const normalizedRoots = roots.map((root) => {
|
|
2570
|
+
if (root === ".")
|
|
2571
|
+
return ".";
|
|
2572
|
+
if (typeof root !== "string")
|
|
2573
|
+
throw new TypeError("Safe roots must be strings");
|
|
2574
|
+
return normalizeRelativePath(root.replace(/[\\/]+$/, ""));
|
|
2575
|
+
});
|
|
2576
|
+
const pathKey = portablePathKey(normalized);
|
|
2577
|
+
if (!normalizedRoots.some((root) => {
|
|
2578
|
+
if (root === ".")
|
|
2579
|
+
return true;
|
|
2580
|
+
const rootKey = portablePathKey(root);
|
|
2581
|
+
return pathKey === rootKey || pathKey.startsWith(`${rootKey}/`);
|
|
2582
|
+
})) {
|
|
2583
|
+
throw new TypeError("Path is outside the allowed roots");
|
|
2584
|
+
}
|
|
2585
|
+
return normalized;
|
|
2586
|
+
}
|
|
2587
|
+
//# sourceMappingURL=index.js.map
|