@velum-labs/routekit-eval-setup 1.0.26 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/git-basis-snapshot.d.ts +27 -0
- package/dist/adapters/git-basis-snapshot.js +178 -0
- package/dist/basis-candidates.d.ts +72 -0
- package/dist/basis-candidates.js +157 -0
- package/dist/basis-projection.d.ts +9 -0
- package/dist/basis-projection.js +30 -0
- package/dist/basis-validation.d.ts +38 -0
- package/dist/basis-validation.js +187 -0
- package/dist/effect-api.d.ts +8 -0
- package/dist/effect-api.js +5 -0
- package/dist/errors.d.ts +11 -1
- package/dist/errors.js +5 -0
- package/dist/index.d.ts +15 -3
- package/dist/index.js +10 -2
- package/dist/project-artifacts.d.ts +26 -1
- package/dist/project-artifacts.js +52 -1
- package/dist/project-contracts.d.ts +3594 -0
- package/dist/project-contracts.js +344 -1
- package/dist/project-store.d.ts +3 -1
- package/dist/project-store.js +39 -33
- package/dist/project-workflow.d.ts +2 -2
- package/dist/services/basis-authoring/service.d.ts +253 -0
- package/dist/services/basis-authoring/service.js +547 -0
- package/dist/services/basis-onboarding/service.d.ts +48 -0
- package/dist/services/basis-onboarding/service.js +491 -0
- package/dist/services/basis-repair/service.d.ts +25 -0
- package/dist/services/basis-repair/service.js +249 -0
- package/dist/test/basis-authoring.test.d.ts +1 -0
- package/dist/test/basis-authoring.test.js +194 -0
- package/dist/test/basis-onboarding.test.d.ts +1 -0
- package/dist/test/basis-onboarding.test.js +385 -0
- package/dist/test/basis-repair.test.d.ts +1 -0
- package/dist/test/basis-repair.test.js +338 -0
- package/dist/test/basis-v3.test.d.ts +1 -0
- package/dist/test/basis-v3.test.js +262 -0
- package/dist/test/project-v2.test.d.ts +1 -0
- package/dist/test/project-v2.test.js +99 -0
- package/package.json +4 -3
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { execFile } from "node:child_process";
|
|
3
|
+
import { readFileSync } from "node:fs";
|
|
4
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
5
|
+
import os from "node:os";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { after, test } from "node:test";
|
|
8
|
+
import { promisify } from "node:util";
|
|
9
|
+
import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
|
|
10
|
+
import { routingBasisDigestV3, routingContractDigest } from "@velum-labs/routekit-eval-contracts";
|
|
11
|
+
import { Effect, Layer } from "effect";
|
|
12
|
+
import { EvalProjectArtifactsLive } from "../project-artifacts.js";
|
|
13
|
+
import { inspectBasisCandidatesV3, makeBasisCandidateDatasetV3 } from "../basis-validation.js";
|
|
14
|
+
import { EvalAuthoringTransport } from "../project-authoring.js";
|
|
15
|
+
import { EvalProjectStore, EvalProjectStoreLive } from "../project-store.js";
|
|
16
|
+
import { BasisOnboarding, BasisOnboardingLive } from "../services/basis-onboarding/service.js";
|
|
17
|
+
import { BasisRepair, BasisRepairLive } from "../services/basis-repair/service.js";
|
|
18
|
+
const execFilePromise = promisify(execFile);
|
|
19
|
+
const roots = [];
|
|
20
|
+
after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
|
|
21
|
+
const configuration = {
|
|
22
|
+
workloadDescription: "RouteKit repository changes",
|
|
23
|
+
candidateModels: [
|
|
24
|
+
"openai/FORBIDDEN_ROUTING_MODEL",
|
|
25
|
+
"openai/other"
|
|
26
|
+
],
|
|
27
|
+
classifierModel: "openai/classifier",
|
|
28
|
+
authorModel: "openai/author",
|
|
29
|
+
judgeModel: "openai/judge",
|
|
30
|
+
objective: { kind: "highest-quality" },
|
|
31
|
+
maximumUnknownWeight: 0.3
|
|
32
|
+
};
|
|
33
|
+
const basisFixture = () => JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/routing-basis-v3.example.json", import.meta.url), "utf8"));
|
|
34
|
+
const task = (index) => ({
|
|
35
|
+
caseId: `case-${String(index).padStart(3, "0")}`,
|
|
36
|
+
occurredAt: new Date(Date.UTC(2026, 0, 1) + index * 86_400_000).toISOString(),
|
|
37
|
+
taskDigest: routingContractDigest("routekit.repair-test-task.v1", "task", index),
|
|
38
|
+
summary: index >= 13
|
|
39
|
+
? `FORBIDDEN_FINAL_TASK_${String(index)}`
|
|
40
|
+
: `Construction or validation task ${String(index)}`,
|
|
41
|
+
changedPathDigest: routingContractDigest("routekit.repair-test-paths.v1", "paths", [`src/${String(index)}.ts`]),
|
|
42
|
+
changedPaths: [`src/${String(index)}.ts`]
|
|
43
|
+
});
|
|
44
|
+
async function repository() {
|
|
45
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "routekit-basis-repair-"));
|
|
46
|
+
roots.push(root);
|
|
47
|
+
await execFilePromise("git", ["init", "-q", root]);
|
|
48
|
+
await execFilePromise("git", [
|
|
49
|
+
"-C",
|
|
50
|
+
root,
|
|
51
|
+
"config",
|
|
52
|
+
"user.name",
|
|
53
|
+
"RouteKit Test"
|
|
54
|
+
]);
|
|
55
|
+
await execFilePromise("git", [
|
|
56
|
+
"-C",
|
|
57
|
+
root,
|
|
58
|
+
"config",
|
|
59
|
+
"user.email",
|
|
60
|
+
"routekit@example.test"
|
|
61
|
+
]);
|
|
62
|
+
await writeFile(path.join(root, "README.md"), "# Frozen repair source\n");
|
|
63
|
+
await execFilePromise("git", ["-C", root, "add", "README.md"]);
|
|
64
|
+
await execFilePromise("git", [
|
|
65
|
+
"-C",
|
|
66
|
+
root,
|
|
67
|
+
"commit",
|
|
68
|
+
"-q",
|
|
69
|
+
"-m",
|
|
70
|
+
"initial"
|
|
71
|
+
]);
|
|
72
|
+
return root;
|
|
73
|
+
}
|
|
74
|
+
function candidateFor(repositoryDigest) {
|
|
75
|
+
const source = basisFixture();
|
|
76
|
+
const semantic = {
|
|
77
|
+
...source,
|
|
78
|
+
basisDigest: "0".repeat(64),
|
|
79
|
+
repositoryDigest,
|
|
80
|
+
facetSnapshot: {
|
|
81
|
+
...source.facetSnapshot,
|
|
82
|
+
repositoryDigest
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
const basis = {
|
|
86
|
+
...semantic,
|
|
87
|
+
basisDigest: routingBasisDigestV3(semantic)
|
|
88
|
+
};
|
|
89
|
+
return {
|
|
90
|
+
candidateId: basis.basisId,
|
|
91
|
+
recipeId: "incumbent-enriched-8",
|
|
92
|
+
basis,
|
|
93
|
+
basisDigest: basis.basisDigest,
|
|
94
|
+
generationConfigurationDigest: "a".repeat(64),
|
|
95
|
+
evidenceDigest: "b".repeat(64),
|
|
96
|
+
createdAt: "2026-08-23T00:00:00.000Z"
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
test("repair is isolated, immutable, and excludes validation/final/model evidence", async () => {
|
|
100
|
+
const root = await repository();
|
|
101
|
+
const infrastructure = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive).pipe(Layer.provide(NodeServicesLayer));
|
|
102
|
+
const onboarding = BasisOnboardingLive.pipe(Layer.provide(infrastructure));
|
|
103
|
+
const initialized = await Effect.runPromise(Effect.gen(function* () {
|
|
104
|
+
return yield* (yield* BasisOnboarding).initialize({
|
|
105
|
+
repositoryRoot: root,
|
|
106
|
+
requestedRef: "HEAD",
|
|
107
|
+
allowDirty: false,
|
|
108
|
+
configuration,
|
|
109
|
+
tasks: Array.from({ length: 41 }, (_, index) => task(index))
|
|
110
|
+
});
|
|
111
|
+
}).pipe(Effect.provide(onboarding)));
|
|
112
|
+
assert.equal(initialized.stage, "basis-generation-ready");
|
|
113
|
+
const candidate = candidateFor(initialized.snapshot.repositoryDigest);
|
|
114
|
+
const validationPlanDigest = "c".repeat(64);
|
|
115
|
+
const selectionReportDigest = "d".repeat(64);
|
|
116
|
+
const review = {
|
|
117
|
+
...initialized,
|
|
118
|
+
revision: 3,
|
|
119
|
+
stage: "basis-validation-review",
|
|
120
|
+
candidates: [candidate],
|
|
121
|
+
structureReports: [
|
|
122
|
+
{
|
|
123
|
+
candidateId: candidate.candidateId,
|
|
124
|
+
basisDigest: candidate.basisDigest,
|
|
125
|
+
eligible: true,
|
|
126
|
+
dimensionCount: candidate.basis.dimensions.length,
|
|
127
|
+
definitionBytes: 1,
|
|
128
|
+
findings: [
|
|
129
|
+
{
|
|
130
|
+
severity: "warning",
|
|
131
|
+
code: "reviewed-boundary-warning",
|
|
132
|
+
dimensionIds: [candidate.basis.dimensions[0].id],
|
|
133
|
+
detail: "Clarify one reviewed semantic boundary."
|
|
134
|
+
}
|
|
135
|
+
]
|
|
136
|
+
}
|
|
137
|
+
],
|
|
138
|
+
datasets: [],
|
|
139
|
+
validationPlan: {
|
|
140
|
+
version: 1,
|
|
141
|
+
planDigest: validationPlanDigest,
|
|
142
|
+
repositoryDigest: initialized.snapshot.repositoryDigest,
|
|
143
|
+
splitDigest: initialized.split.splitDigest,
|
|
144
|
+
classifierBindings: [
|
|
145
|
+
{
|
|
146
|
+
candidateId: candidate.candidateId,
|
|
147
|
+
basisDigest: candidate.basisDigest,
|
|
148
|
+
classifierConfigurationDigest: "e".repeat(64)
|
|
149
|
+
}
|
|
150
|
+
],
|
|
151
|
+
candidateIds: [candidate.candidateId],
|
|
152
|
+
datasetDigests: [],
|
|
153
|
+
repetitions: 1,
|
|
154
|
+
plannedPrimaryCalls: 1,
|
|
155
|
+
maximumAttempts: 2,
|
|
156
|
+
requiredTotalInputTokens: 1,
|
|
157
|
+
requiredTotalOutputTokens: 1,
|
|
158
|
+
acknowledgeUnpricedExposure: true
|
|
159
|
+
},
|
|
160
|
+
validationResults: [
|
|
161
|
+
{
|
|
162
|
+
version: 1,
|
|
163
|
+
candidateId: candidate.candidateId,
|
|
164
|
+
basisDigest: candidate.basisDigest,
|
|
165
|
+
validationDatasetDigest: "f".repeat(64),
|
|
166
|
+
classifierConfigurationDigest: "e".repeat(64),
|
|
167
|
+
predictions: [
|
|
168
|
+
{
|
|
169
|
+
caseId: "FORBIDDEN_VALIDATION_PREDICTION",
|
|
170
|
+
basisDigest: candidate.basisDigest,
|
|
171
|
+
scores: candidate.basis.dimensions.map((dimension) => ({
|
|
172
|
+
dimensionId: dimension.id,
|
|
173
|
+
score: 0
|
|
174
|
+
})),
|
|
175
|
+
unknownProbability: 1,
|
|
176
|
+
classifierConfigurationDigest: "e".repeat(64)
|
|
177
|
+
}
|
|
178
|
+
],
|
|
179
|
+
caseCount: 1,
|
|
180
|
+
skippedInvalidCount: 0,
|
|
181
|
+
activeMae: 0.2,
|
|
182
|
+
activeF1: 0.8,
|
|
183
|
+
topDimensionAgreement: 0.8,
|
|
184
|
+
unknownMae: 0.2,
|
|
185
|
+
unknownBrier: 0.04,
|
|
186
|
+
unknownAccuracy: 0.8,
|
|
187
|
+
falseUnknownRate: 0,
|
|
188
|
+
missedUnknownRate: 0.2,
|
|
189
|
+
referenceUnknownRate: 0.2,
|
|
190
|
+
averageActiveDimensionCount: 1,
|
|
191
|
+
selectionScore: 0.3,
|
|
192
|
+
provisional: false
|
|
193
|
+
}
|
|
194
|
+
],
|
|
195
|
+
selection: {
|
|
196
|
+
version: 1,
|
|
197
|
+
selectionReportDigest,
|
|
198
|
+
repositoryDigest: initialized.snapshot.repositoryDigest,
|
|
199
|
+
validationPlanDigest,
|
|
200
|
+
objectiveVersion: "basis-selection-v1",
|
|
201
|
+
rankedCandidateIds: [candidate.candidateId],
|
|
202
|
+
paretoCandidateIds: [candidate.candidateId],
|
|
203
|
+
selectedCandidateId: candidate.candidateId,
|
|
204
|
+
provisional: false
|
|
205
|
+
}
|
|
206
|
+
};
|
|
207
|
+
await Effect.runPromise(Effect.gen(function* () {
|
|
208
|
+
yield* (yield* EvalProjectStore).saveDocument(root, review);
|
|
209
|
+
}).pipe(Effect.provide(infrastructure)));
|
|
210
|
+
const summarySemantic = {
|
|
211
|
+
version: 1,
|
|
212
|
+
sourceCandidateId: candidate.candidateId,
|
|
213
|
+
sourceBasisDigest: candidate.basisDigest,
|
|
214
|
+
errorCategories: [
|
|
215
|
+
{
|
|
216
|
+
code: "boundary-ambiguity",
|
|
217
|
+
dimensionIds: [candidate.basis.dimensions[0].id],
|
|
218
|
+
summary: "Clarify the reviewed boundary without exposing examples."
|
|
219
|
+
}
|
|
220
|
+
]
|
|
221
|
+
};
|
|
222
|
+
const summary = {
|
|
223
|
+
...summarySemantic,
|
|
224
|
+
summaryDigest: routingContractDigest("routekit.basis-repair-summary.v1", "summary", summarySemantic)
|
|
225
|
+
};
|
|
226
|
+
let serializedInput = "";
|
|
227
|
+
const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
|
|
228
|
+
complete: (input) => Effect.sync(() => {
|
|
229
|
+
serializedInput = input.input;
|
|
230
|
+
const fixture = basisFixture();
|
|
231
|
+
const { repositoryDigest: _repositoryDigest, ...facetSnapshot } = fixture.facetSnapshot;
|
|
232
|
+
return JSON.stringify({
|
|
233
|
+
facetSnapshot,
|
|
234
|
+
dimensions: fixture.dimensions
|
|
235
|
+
});
|
|
236
|
+
})
|
|
237
|
+
}));
|
|
238
|
+
const repairPlanLive = BasisRepairLive.pipe(Layer.provide(infrastructure));
|
|
239
|
+
const plan = await Effect.runPromise(Effect.gen(function* () {
|
|
240
|
+
return yield* (yield* BasisRepair).plan({
|
|
241
|
+
repositoryRoot: root,
|
|
242
|
+
sourceCandidateId: candidate.candidateId,
|
|
243
|
+
authorModel: "openai/repair-author",
|
|
244
|
+
summary
|
|
245
|
+
});
|
|
246
|
+
}).pipe(Effect.provide(repairPlanLive)));
|
|
247
|
+
const repairExecuteLive = BasisRepairLive.pipe(Layer.provide(infrastructure));
|
|
248
|
+
const repaired = await Effect.runPromise(Effect.gen(function* () {
|
|
249
|
+
return yield* (yield* BasisRepair)
|
|
250
|
+
.execute({
|
|
251
|
+
repositoryRoot: root,
|
|
252
|
+
plan,
|
|
253
|
+
summary
|
|
254
|
+
})
|
|
255
|
+
.pipe(Effect.provide(transport));
|
|
256
|
+
}).pipe(Effect.provide(repairExecuteLive)));
|
|
257
|
+
assert.equal(repaired.candidate.basis.provenance.origin?.kind, "repair");
|
|
258
|
+
assert.equal(repaired.candidate.basis.provenance.origin?.sourceBasisId, candidate.basis.basisId);
|
|
259
|
+
assert.doesNotMatch(serializedInput, /FORBIDDEN_FINAL_TASK/u);
|
|
260
|
+
assert.doesNotMatch(serializedInput, /FORBIDDEN_VALIDATION_PREDICTION/u);
|
|
261
|
+
assert.doesNotMatch(serializedInput, /FORBIDDEN_ROUTING_MODEL/u);
|
|
262
|
+
assert.doesNotMatch(serializedInput, /activeMae|selectionScore/u);
|
|
263
|
+
const afterRepair = await Effect.runPromise(Effect.gen(function* () {
|
|
264
|
+
return yield* (yield* EvalProjectStore).loadDocument(root);
|
|
265
|
+
}).pipe(Effect.provide(infrastructure)));
|
|
266
|
+
assert.deepEqual(afterRepair, review);
|
|
267
|
+
assert.equal(review.candidates.length, 1);
|
|
268
|
+
assert.equal(review.selection.selectedCandidateId, candidate.candidateId);
|
|
269
|
+
const resumed = await Effect.runPromise(Effect.gen(function* () {
|
|
270
|
+
return yield* (yield* BasisRepair)
|
|
271
|
+
.execute({
|
|
272
|
+
repositoryRoot: root,
|
|
273
|
+
plan,
|
|
274
|
+
summary
|
|
275
|
+
})
|
|
276
|
+
.pipe(Effect.provide(transport));
|
|
277
|
+
}).pipe(Effect.provide(repairExecuteLive)));
|
|
278
|
+
assert.deepEqual(resumed, repaired);
|
|
279
|
+
const labelProtocol = {
|
|
280
|
+
version: 1,
|
|
281
|
+
protocolDigest: "9".repeat(64),
|
|
282
|
+
kind: "reviewed-human",
|
|
283
|
+
author: "Author",
|
|
284
|
+
reviewer: "Reviewer",
|
|
285
|
+
blindness: "candidate-output-hidden",
|
|
286
|
+
activeDimensionThreshold: 0.25,
|
|
287
|
+
unknownThreshold: 0.3
|
|
288
|
+
};
|
|
289
|
+
const candidates = [candidate, repaired.candidate];
|
|
290
|
+
const datasets = candidates.map((entry) => makeBasisCandidateDatasetV3({
|
|
291
|
+
splitDigest: review.split.splitDigest,
|
|
292
|
+
candidate: entry,
|
|
293
|
+
tasks: review.split.validation,
|
|
294
|
+
labelProtocol,
|
|
295
|
+
labels: review.split.validation.map((validationTask) => ({
|
|
296
|
+
caseId: validationTask.caseId,
|
|
297
|
+
basisDigest: entry.basisDigest,
|
|
298
|
+
scores: entry.basis.dimensions.map((dimension, index) => ({
|
|
299
|
+
dimensionId: dimension.id,
|
|
300
|
+
score: index === 0 ? 1 : 0
|
|
301
|
+
})),
|
|
302
|
+
unknownProbability: 0
|
|
303
|
+
}))
|
|
304
|
+
}));
|
|
305
|
+
const validationSemantic = {
|
|
306
|
+
version: 1,
|
|
307
|
+
repositoryDigest: review.snapshot.repositoryDigest,
|
|
308
|
+
splitDigest: review.split.splitDigest,
|
|
309
|
+
classifierBindings: candidates.map((entry, index) => ({
|
|
310
|
+
candidateId: entry.candidateId,
|
|
311
|
+
basisDigest: entry.basisDigest,
|
|
312
|
+
classifierConfigurationDigest: String(index + 1).repeat(64)
|
|
313
|
+
})),
|
|
314
|
+
candidateIds: candidates.map((entry) => entry.candidateId),
|
|
315
|
+
datasetDigests: datasets.map((dataset) => dataset.datasetDigest),
|
|
316
|
+
repetitions: 1,
|
|
317
|
+
plannedPrimaryCalls: candidates.length * review.split.validation.length,
|
|
318
|
+
maximumAttempts: candidates.length * review.split.validation.length * 2,
|
|
319
|
+
requiredTotalInputTokens: 1_000,
|
|
320
|
+
requiredTotalOutputTokens: 1_000,
|
|
321
|
+
acknowledgeUnpricedExposure: true
|
|
322
|
+
};
|
|
323
|
+
const validationPlan = {
|
|
324
|
+
...validationSemantic,
|
|
325
|
+
planDigest: routingContractDigest("routekit.basis-validation-plan.v1", "plan", validationSemantic)
|
|
326
|
+
};
|
|
327
|
+
const repairReady = await Effect.runPromise(Effect.gen(function* () {
|
|
328
|
+
return yield* (yield* BasisOnboarding).prepareRepairValidation({
|
|
329
|
+
repositoryRoot: root,
|
|
330
|
+
repairCandidate: repaired.candidate,
|
|
331
|
+
datasets,
|
|
332
|
+
validationPlan
|
|
333
|
+
});
|
|
334
|
+
}).pipe(Effect.provide(onboarding)));
|
|
335
|
+
assert.equal(repairReady.stage, "basis-validation-ready");
|
|
336
|
+
assert.deepEqual(repairReady.candidates.map((entry) => entry.candidateId), candidates.map((entry) => entry.candidateId));
|
|
337
|
+
assert.equal(inspectBasisCandidatesV3([repaired.candidate])[0].eligible, true);
|
|
338
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
|
+
import { test } from "node:test";
|
|
4
|
+
import { routingContractDigest } from "@velum-labs/routekit-eval-contracts";
|
|
5
|
+
import { DEFAULT_BASIS_CANDIDATE_RECIPES_V3, selectConstructionTasksForRecipeV3, splitBasisTasksChronologicallyV3 } from "../basis-candidates.js";
|
|
6
|
+
import { approveSelectedBasisV3, evaluateBasisCandidateValidationV3, inspectBasisCandidatesV3, makeBasisCandidateDatasetV3, selectValidatedBasisCandidatesV3 } from "../basis-validation.js";
|
|
7
|
+
import { projectRoutingBasisV3 } from "../basis-projection.js";
|
|
8
|
+
const basisFixture = () => JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/routing-basis-v3.example.json", import.meta.url), "utf8"));
|
|
9
|
+
const task = (index) => ({
|
|
10
|
+
caseId: `case-${String(index).padStart(3, "0")}`,
|
|
11
|
+
occurredAt: new Date(Date.UTC(2026, 0, 1) + index * 86_400_000).toISOString(),
|
|
12
|
+
taskDigest: routingContractDigest("routekit.test-task.v1", "task", index),
|
|
13
|
+
summary: `Task ${String(index)}`,
|
|
14
|
+
changedPathDigest: routingContractDigest("routekit.test-paths.v1", "paths", [`src/${String(index)}.ts`]),
|
|
15
|
+
changedPaths: [`src/${String(index)}.ts`]
|
|
16
|
+
});
|
|
17
|
+
const candidate = () => {
|
|
18
|
+
const basis = basisFixture();
|
|
19
|
+
return {
|
|
20
|
+
candidateId: basis.basisId,
|
|
21
|
+
recipeId: "incumbent-enriched-8",
|
|
22
|
+
basis,
|
|
23
|
+
basisDigest: basis.basisDigest,
|
|
24
|
+
generationConfigurationDigest: "a".repeat(64),
|
|
25
|
+
evidenceDigest: "b".repeat(64),
|
|
26
|
+
createdAt: "2026-08-23T00:00:00.000Z"
|
|
27
|
+
};
|
|
28
|
+
};
|
|
29
|
+
const labelProtocol = {
|
|
30
|
+
version: 1,
|
|
31
|
+
protocolDigest: "c".repeat(64),
|
|
32
|
+
kind: "reviewed-human",
|
|
33
|
+
author: "Author",
|
|
34
|
+
reviewer: "Reviewer",
|
|
35
|
+
blindness: "candidate-output-hidden",
|
|
36
|
+
activeDimensionThreshold: 0.25,
|
|
37
|
+
unknownThreshold: 0.3
|
|
38
|
+
};
|
|
39
|
+
test("default basis portfolio is incumbent plus the exact seven challengers", () => {
|
|
40
|
+
assert.deepEqual(DEFAULT_BASIS_CANDIDATE_RECIPES_V3.map((recipe) => recipe.id), [
|
|
41
|
+
"incumbent-enriched-8",
|
|
42
|
+
"structure-only-8",
|
|
43
|
+
"hybrid-paths-20-diverse-8",
|
|
44
|
+
"hybrid-paths-40-diverse-6",
|
|
45
|
+
"hybrid-paths-40-diverse-8",
|
|
46
|
+
"hybrid-paths-40-diverse-10",
|
|
47
|
+
"hybrid-paths-40-recent-8",
|
|
48
|
+
"hybrid-paths-80-diverse-8"
|
|
49
|
+
]);
|
|
50
|
+
assert.equal(DEFAULT_BASIS_CANDIDATE_RECIPES_V3.length, 8);
|
|
51
|
+
assert.equal(DEFAULT_BASIS_CANDIDATE_RECIPES_V3[1]
|
|
52
|
+
?.includeChangedPathStatistics, false);
|
|
53
|
+
});
|
|
54
|
+
test("chronological split keeps held-out cohorts disjoint and marks sparse histories provisional", () => {
|
|
55
|
+
const full = splitBasisTasksChronologicallyV3(Array.from({ length: 50 }, (_, index) => task(index)));
|
|
56
|
+
assert.equal(full.construction.length, 10);
|
|
57
|
+
assert.equal(full.validation.length, 12);
|
|
58
|
+
assert.equal(full.final.length, 28);
|
|
59
|
+
assert.equal(full.provisional, false);
|
|
60
|
+
const ids = [
|
|
61
|
+
...full.construction,
|
|
62
|
+
...full.validation,
|
|
63
|
+
...full.final
|
|
64
|
+
].map((entry) => entry.caseId);
|
|
65
|
+
assert.equal(new Set(ids).size, ids.length);
|
|
66
|
+
const sparse = splitBasisTasksChronologicallyV3(Array.from({ length: 10 }, (_, index) => task(index)));
|
|
67
|
+
assert.equal(sparse.provisional, true);
|
|
68
|
+
assert.ok(sparse.warnings.some((warning) => warning.includes("validation")));
|
|
69
|
+
assert.ok(sparse.warnings.some((warning) => warning.includes("final")));
|
|
70
|
+
const diverseRecipe = DEFAULT_BASIS_CANDIDATE_RECIPES_V3.find((recipe) => recipe.id === "hybrid-paths-20-diverse-8");
|
|
71
|
+
const recentRecipe = DEFAULT_BASIS_CANDIDATE_RECIPES_V3.find((recipe) => recipe.id === "hybrid-paths-40-recent-8");
|
|
72
|
+
assert.deepEqual(selectConstructionTasksForRecipeV3(full, diverseRecipe).map((entry) => entry.caseId), full.construction.map((entry) => entry.caseId));
|
|
73
|
+
assert.deepEqual(selectConstructionTasksForRecipeV3(full, recentRecipe).map((entry) => entry.caseId), full.construction.map((entry) => entry.caseId));
|
|
74
|
+
});
|
|
75
|
+
test("candidate dataset identity binds task, label, split, basis, and label protocol", () => {
|
|
76
|
+
const record = candidate();
|
|
77
|
+
const tasks = [task(1)];
|
|
78
|
+
const labels = [
|
|
79
|
+
{
|
|
80
|
+
caseId: tasks[0].caseId,
|
|
81
|
+
basisDigest: record.basisDigest,
|
|
82
|
+
scores: record.basis.dimensions.map((dimension, index) => ({
|
|
83
|
+
dimensionId: dimension.id,
|
|
84
|
+
score: index === 0 ? 1 : 0
|
|
85
|
+
})),
|
|
86
|
+
unknownProbability: 0
|
|
87
|
+
}
|
|
88
|
+
];
|
|
89
|
+
const original = makeBasisCandidateDatasetV3({
|
|
90
|
+
splitDigest: "d".repeat(64),
|
|
91
|
+
candidate: record,
|
|
92
|
+
tasks,
|
|
93
|
+
labelProtocol,
|
|
94
|
+
labels
|
|
95
|
+
});
|
|
96
|
+
const changedTask = makeBasisCandidateDatasetV3({
|
|
97
|
+
splitDigest: "d".repeat(64),
|
|
98
|
+
candidate: record,
|
|
99
|
+
tasks: [{ ...tasks[0], summary: "Changed task" }],
|
|
100
|
+
labelProtocol,
|
|
101
|
+
labels
|
|
102
|
+
});
|
|
103
|
+
const changedLabel = makeBasisCandidateDatasetV3({
|
|
104
|
+
splitDigest: "d".repeat(64),
|
|
105
|
+
candidate: record,
|
|
106
|
+
tasks,
|
|
107
|
+
labelProtocol,
|
|
108
|
+
labels: [{ ...labels[0], unknownProbability: 0.2 }]
|
|
109
|
+
});
|
|
110
|
+
assert.notEqual(original.datasetDigest, changedTask.datasetDigest);
|
|
111
|
+
assert.notEqual(original.datasetDigest, changedLabel.datasetDigest);
|
|
112
|
+
});
|
|
113
|
+
test("V3 basis projection preserves dimension ids and evaluator boundaries", () => {
|
|
114
|
+
const record = candidate();
|
|
115
|
+
const projection = projectRoutingBasisV3(record.basis);
|
|
116
|
+
assert.equal(projection.basisDigest, record.basisDigest);
|
|
117
|
+
assert.deepEqual(projection.dimensions.map((dimension) => dimension.id), record.basis.dimensions.map((dimension) => dimension.id));
|
|
118
|
+
assert.deepEqual(projection.dimensions[0], {
|
|
119
|
+
id: record.basis.dimensions[0].id,
|
|
120
|
+
description: record.basis.dimensions[0].description,
|
|
121
|
+
includes: record.basis.dimensions[0].inclusions,
|
|
122
|
+
excludes: record.basis.dimensions[0].exclusions
|
|
123
|
+
});
|
|
124
|
+
assert.notEqual(projection.projectedBasisDigest, record.basisDigest);
|
|
125
|
+
});
|
|
126
|
+
test("validation, Pareto selection, and approval are exact-digest bound", () => {
|
|
127
|
+
const record = candidate();
|
|
128
|
+
const structure = inspectBasisCandidatesV3([record])[0];
|
|
129
|
+
assert.equal(structure.eligible, true);
|
|
130
|
+
const validationTask = task(1);
|
|
131
|
+
const scores = record.basis.dimensions.map((dimension, index) => ({
|
|
132
|
+
dimensionId: dimension.id,
|
|
133
|
+
score: index === 0 ? 1 : 0
|
|
134
|
+
}));
|
|
135
|
+
const dataset = makeBasisCandidateDatasetV3({
|
|
136
|
+
splitDigest: "d".repeat(64),
|
|
137
|
+
candidate: record,
|
|
138
|
+
tasks: [validationTask],
|
|
139
|
+
labelProtocol,
|
|
140
|
+
labels: [
|
|
141
|
+
{
|
|
142
|
+
caseId: validationTask.caseId,
|
|
143
|
+
basisDigest: record.basisDigest,
|
|
144
|
+
scores,
|
|
145
|
+
unknownProbability: 0
|
|
146
|
+
}
|
|
147
|
+
]
|
|
148
|
+
});
|
|
149
|
+
const validationPlan = {
|
|
150
|
+
version: 1,
|
|
151
|
+
planDigest: "e".repeat(64),
|
|
152
|
+
repositoryDigest: record.basis.repositoryDigest,
|
|
153
|
+
splitDigest: "d".repeat(64),
|
|
154
|
+
classifierBindings: [
|
|
155
|
+
{
|
|
156
|
+
candidateId: record.candidateId,
|
|
157
|
+
basisDigest: record.basisDigest,
|
|
158
|
+
classifierConfigurationDigest: "f".repeat(64)
|
|
159
|
+
}
|
|
160
|
+
],
|
|
161
|
+
candidateIds: [record.candidateId],
|
|
162
|
+
datasetDigests: [dataset.datasetDigest],
|
|
163
|
+
repetitions: 1,
|
|
164
|
+
plannedPrimaryCalls: 1,
|
|
165
|
+
maximumAttempts: 2,
|
|
166
|
+
requiredTotalInputTokens: 1,
|
|
167
|
+
requiredTotalOutputTokens: 1,
|
|
168
|
+
acknowledgeUnpricedExposure: true
|
|
169
|
+
};
|
|
170
|
+
const validation = evaluateBasisCandidateValidationV3({
|
|
171
|
+
candidate: record,
|
|
172
|
+
dataset,
|
|
173
|
+
validationPlan,
|
|
174
|
+
predictions: [
|
|
175
|
+
{
|
|
176
|
+
caseId: validationTask.caseId,
|
|
177
|
+
basisDigest: record.basisDigest,
|
|
178
|
+
scores,
|
|
179
|
+
unknownProbability: 0,
|
|
180
|
+
classifierConfigurationDigest: validationPlan.classifierBindings[0]
|
|
181
|
+
.classifierConfigurationDigest
|
|
182
|
+
}
|
|
183
|
+
],
|
|
184
|
+
provisional: false
|
|
185
|
+
});
|
|
186
|
+
assert.equal(validation.selectionScore, 0);
|
|
187
|
+
const selection = selectValidatedBasisCandidatesV3({
|
|
188
|
+
repositoryDigest: record.basis.repositoryDigest,
|
|
189
|
+
validationPlan,
|
|
190
|
+
candidates: [record],
|
|
191
|
+
structureReports: [structure],
|
|
192
|
+
validationResults: [validation]
|
|
193
|
+
});
|
|
194
|
+
assert.equal(selection.selectedCandidateId, record.candidateId);
|
|
195
|
+
assert.deepEqual(selection.paretoCandidateIds, [record.candidateId]);
|
|
196
|
+
const snapshot = {
|
|
197
|
+
repositoryRootDigest: "1".repeat(64),
|
|
198
|
+
requestedRef: "HEAD",
|
|
199
|
+
commit: "2".repeat(64),
|
|
200
|
+
tree: "3".repeat(64),
|
|
201
|
+
repositoryDigest: record.basis.repositoryDigest,
|
|
202
|
+
capturedAt: "2026-08-23T00:00:00.000Z",
|
|
203
|
+
dirty: false,
|
|
204
|
+
sourceInventoryDigest: "4".repeat(64)
|
|
205
|
+
};
|
|
206
|
+
assert.doesNotThrow(() => approveSelectedBasisV3({
|
|
207
|
+
projectId: "project",
|
|
208
|
+
approvedBy: "Reviewer",
|
|
209
|
+
approvedAt: "2026-08-23T00:00:00.000Z",
|
|
210
|
+
snapshot,
|
|
211
|
+
candidate: record,
|
|
212
|
+
structure,
|
|
213
|
+
validation,
|
|
214
|
+
selection
|
|
215
|
+
}));
|
|
216
|
+
assert.throws(() => approveSelectedBasisV3({
|
|
217
|
+
projectId: "project",
|
|
218
|
+
approvedBy: "Reviewer",
|
|
219
|
+
approvedAt: "2026-08-23T00:00:00.000Z",
|
|
220
|
+
snapshot: { ...snapshot, dirty: true },
|
|
221
|
+
candidate: record,
|
|
222
|
+
structure,
|
|
223
|
+
validation,
|
|
224
|
+
selection
|
|
225
|
+
}), /dirty/);
|
|
226
|
+
const provisionalSelectionSemantic = {
|
|
227
|
+
version: selection.version,
|
|
228
|
+
repositoryDigest: selection.repositoryDigest,
|
|
229
|
+
validationPlanDigest: selection.validationPlanDigest,
|
|
230
|
+
objectiveVersion: selection.objectiveVersion,
|
|
231
|
+
rankedCandidateIds: selection.rankedCandidateIds,
|
|
232
|
+
paretoCandidateIds: selection.paretoCandidateIds,
|
|
233
|
+
selectedCandidateId: selection.selectedCandidateId,
|
|
234
|
+
provisional: true
|
|
235
|
+
};
|
|
236
|
+
assert.throws(() => approveSelectedBasisV3({
|
|
237
|
+
projectId: "project",
|
|
238
|
+
approvedBy: "Reviewer",
|
|
239
|
+
approvedAt: "2026-08-23T00:00:00.000Z",
|
|
240
|
+
snapshot,
|
|
241
|
+
candidate: record,
|
|
242
|
+
structure,
|
|
243
|
+
validation: { ...validation, provisional: true },
|
|
244
|
+
selection: {
|
|
245
|
+
...provisionalSelectionSemantic,
|
|
246
|
+
selectionReportDigest: routingContractDigest("routekit.basis-selection-report.v1", "selection", provisionalSelectionSemantic)
|
|
247
|
+
}
|
|
248
|
+
}), /acknowledgement/);
|
|
249
|
+
assert.throws(() => approveSelectedBasisV3({
|
|
250
|
+
projectId: "project",
|
|
251
|
+
approvedBy: "Reviewer",
|
|
252
|
+
approvedAt: "2026-08-23T00:00:00.000Z",
|
|
253
|
+
snapshot,
|
|
254
|
+
candidate: record,
|
|
255
|
+
structure,
|
|
256
|
+
validation,
|
|
257
|
+
selection: {
|
|
258
|
+
...selection,
|
|
259
|
+
selectedCandidateId: "changed"
|
|
260
|
+
}
|
|
261
|
+
}), /digest/);
|
|
262
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|