@velum-labs/routekit-eval-setup 1.0.21 → 1.0.23

package/dist/project-artifacts.d.ts

@@ -14,7 +14,7 @@ export type EvalProjectArtifactsShape = {
     readonly loadEvaluationsApproval: (repositoryRoot: string) => Effect.Effect<EvalArtifactApproval | undefined, EvalProjectArtifactError, never>;
     readonly saveEvaluationsApproval: (repositoryRoot: string, approval: EvalArtifactApproval) => Effect.Effect<void, EvalProjectArtifactError, never>;
     readonly savePlan: (repositoryRoot: string, plan: EvalExecutionPlan) => Effect.Effect<void, EvalProjectArtifactError, never>;
-    readonly materializePlanSuites: (repositoryRoot: string, plan: EvalExecutionPlan, proposal: EvalEvaluationProposal) => Effect.Effect<void, EvalProjectArtifactError, never>;
+    readonly materializePlanSuites: (repositoryRoot: string, plan: EvalExecutionPlan, proposal: EvalEvaluationProposal, mode?: "create" | "rematerialize") => Effect.Effect<void, EvalProjectArtifactError, never>;
     readonly planSuitePath: (repositoryRoot: string, planId: string, dimensionId: string) => Effect.Effect<string, EvalProjectArtifactError, never>;
     readonly compositionSuitePath: (repositoryRoot: string, planId: string) => Effect.Effect<string, EvalProjectArtifactError, never>;
     readonly loadPlan: (repositoryRoot: string, planId: string) => Effect.Effect<EvalExecutionPlan | undefined, EvalProjectArtifactError, never>;

package/dist/project-artifacts.js

@@ -30,8 +30,16 @@ for (const model of manifest.candidateModels) {
         ? testCase.prompt
         : [testCase.prompt, "", "Reference material:", "-----", testCase.context, "-----"].join("\\n");
       const run = await candidate.run({ prompt, caseId: testCase.id });
-      run.toComplete();
+      let candidateCompletionError: unknown;
+      try {
+        run.toComplete();
+      } catch (error) {
+        candidateCompletionError = error;
+      }
       await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+      if (candidateCompletionError !== undefined) {
+        throw candidateCompletionError;
+      }
     });
   }
 }
@@ -226,7 +234,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
         loadEvaluationsApproval: (repositoryRoot) => read(repositoryRoot, EVALUATIONS_APPROVAL, Schema.decodeUnknownEffect(EvalArtifactApprovalSchema)),
         saveEvaluationsApproval: (repositoryRoot, approval) => write(repositoryRoot, EVALUATIONS_APPROVAL, approval),
         savePlan: (repositoryRoot, plan) => write(repositoryRoot, paths.join("plans", `${plan.planId}.json`), plan, false),
-        materializePlanSuites: (repositoryRoot, plan, proposal) => Effect.gen(function* () {
+        materializePlanSuites: (repositoryRoot, plan, proposal, mode = "create") => Effect.gen(function* () {
             yield* requireArtifactId("plan", plan.planId);
             const suites = new Map(proposal.suites.map((suite) => [suite.dimensionId, suite]));
             for (const selection of plan.selectedCaseIds) {
@@ -246,8 +254,8 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     return testCase;
                 });
                 const suiteRoot = paths.join("plans", plan.planId, "dimensions", selection.dimensionId);
-                yield* writeText(repositoryRoot, paths.join(suiteRoot, `${selection.dimensionId}.eval.ts`), renderDimensionSuite(), false);
-                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "cases.json"), cases, false);
+                yield* writeText(repositoryRoot, paths.join(suiteRoot, `${selection.dimensionId}.eval.ts`), renderDimensionSuite(), mode === "rematerialize");
+                yield* write(repositoryRoot, paths.join(suiteRoot, "data", "cases.json"), cases, mode === "rematerialize");
                 yield* write(repositoryRoot, paths.join(suiteRoot, "routekit.eval-manifest.json"), {
                     version: EVAL_PROJECT_VERSION,
                     profileId: selection.dimensionId,
@@ -257,7 +265,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     caseIds: cases.map((testCase) => testCase.id),
                     maxOutputTokens: suite.maximumOutputTokens,
                     expectedCallCount: cases.length * plan.candidateModels.length * 2
-                }, false);
+                }, mode === "rematerialize");
             }
             const compositionById = new Map(proposal.compositionSuite.cases.map((testCase) => [testCase.id, testCase]));
             const compositionCases = plan.selectedCompositionCaseIds.map((caseId) => {
@@ -268,8 +276,8 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                 return testCase;
             });
             const compositionRoot = paths.join("plans", plan.planId, "composition");
-            yield* writeText(repositoryRoot, paths.join(compositionRoot, "composition.eval.ts"), renderDimensionSuite(), false);
-            yield* write(repositoryRoot, paths.join(compositionRoot, "data", "cases.json"), compositionCases, false);
+            yield* writeText(repositoryRoot, paths.join(compositionRoot, "composition.eval.ts"), renderDimensionSuite(), mode === "rematerialize");
+            yield* write(repositoryRoot, paths.join(compositionRoot, "data", "cases.json"), compositionCases, mode === "rematerialize");
             yield* write(repositoryRoot, paths.join(compositionRoot, "routekit.eval-manifest.json"), {
                 version: EVAL_PROJECT_VERSION,
                 profileId: "composition",
@@ -279,7 +287,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                 caseIds: compositionCases.map((testCase) => testCase.id),
                 maxOutputTokens: proposal.compositionSuite.maximumOutputTokens,
                 expectedCallCount: compositionCases.length * plan.candidateModels.length * 2
-            }, false);
+            }, mode === "rematerialize");
         }).pipe(Effect.mapError((cause) => cause instanceof EvalProjectArtifactError
             ? cause
             : artifactFailure("writing", artifactPath(repositoryRoot, paths.join("plans", plan.planId)), cause))),

package/dist/project-workflow.js

@@ -782,6 +782,13 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             plan.judgeModel !== state.configuration.judgeModel) {
             return yield* transitionError(state.stage, "execution plan is stale or does not match the approved project");
         }
+        const proposal = yield* artifacts.loadEvaluationProposal(root);
+        if (proposal === undefined ||
+            proposal.evaluationDigest !== plan.evaluationDigest ||
+            proposal.basisDigest !== plan.basisDigest) {
+            return yield* transitionError(state.stage, "the approved evaluation proposal for the execution plan is missing or stale");
+        }
+        yield* artifacts.materializePlanSuites(root, plan, proposal, "rematerialize");
         const now = yield* isoNow;
         const runId = crypto.randomUUID();
         const next = {

package/dist/test/project-workflow.test.js

@@ -306,6 +306,14 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
     assert.equal((await stat(planPath)).mode & 0o777, 0o600);
     assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
+    const firstDimension = reviewedDimensions[0];
+    const generatedSuite = await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, `${firstDimension.id}.eval.ts`), "utf8");
+    const completionAssertion = generatedSuite.indexOf("run.toComplete();");
+    const judgeCall = generatedSuite.indexOf("await judge.autoEvals");
+    const completionRethrow = generatedSuite.indexOf("throw candidateCompletionError;");
+    assert.ok(completionAssertion >= 0);
+    assert.ok(judgeCall > completionAssertion);
+    assert.ok(completionRethrow > judgeCall);
     for (const dimension of reviewedDimensions) {
         assert.equal((await stat(path.join(root, ".routekit", "evals", "dimensions", dimension.id, "suite.json")))
             .mode & 0o777, 0o600);
@@ -613,6 +621,55 @@ test("failed runs preserve plan revisions for retry while mismatched plans remai
     assert.equal(rerun.ready.state.evaluationDigest, plan.evaluationDigest);
     assert.equal(rerun.second.plan.planId, plan.planId);
 });
+test("startRun rematerializes stale dimension and composition suite templates", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "pilot");
+    const selection = plan.selectedCaseIds[0];
+    const planRoot = path.join(root, ".routekit", "evals", "plans", plan.planId);
+    const dimensionRoot = path.join(planRoot, "dimensions", selection.dimensionId);
+    const dimensionSuitePath = path.join(dimensionRoot, `${selection.dimensionId}.eval.ts`);
+    const compositionRoot = path.join(planRoot, "composition");
+    const compositionSuitePath = path.join(compositionRoot, "composition.eval.ts");
+    const dimensionManifestPath = path.join(dimensionRoot, "routekit.eval-manifest.json");
+    const compositionManifestPath = path.join(compositionRoot, "routekit.eval-manifest.json");
+    const staleTemplate = `// generated by RouteKit 1.0.21
+run.toComplete();
+await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+`;
+    await Promise.all([
+        writeFile(dimensionSuitePath, staleTemplate),
+        writeFile(compositionSuitePath, staleTemplate)
+    ]);
+    const [dimensionManifestBefore, compositionManifestBefore] = await Promise.all([
+        readFile(dimensionManifestPath, "utf8"),
+        readFile(compositionManifestPath, "utf8")
+    ]);
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.startRun(root, plan.planId);
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    const [dimensionSuite, compositionSuite, dimensionManifestAfter, compositionManifestAfter] = await Promise.all([
+        readFile(dimensionSuitePath, "utf8"),
+        readFile(compositionSuitePath, "utf8"),
+        readFile(dimensionManifestPath, "utf8"),
+        readFile(compositionManifestPath, "utf8")
+    ]);
+    for (const suite of [dimensionSuite, compositionSuite]) {
+        const completionAssertion = suite.indexOf("run.toComplete();");
+        const judgeCall = suite.indexOf("await judge.autoEvals");
+        const completionRethrow = suite.indexOf("throw candidateCompletionError;");
+        assert.doesNotMatch(suite, /generated by RouteKit 1\.0\.21/u);
+        assert.ok(completionAssertion >= 0);
+        assert.ok(judgeCall > completionAssertion);
+        assert.ok(completionRethrow > judgeCall);
+    }
+    assert.equal(dimensionManifestAfter, dimensionManifestBefore);
+    assert.equal(compositionManifestAfter, compositionManifestBefore);
+    const dimensionManifest = JSON.parse(dimensionManifestAfter);
+    const compositionManifest = JSON.parse(compositionManifestAfter);
+    assert.equal(dimensionManifest.expectedCallCount, selection.caseIds.length * plan.candidateModels.length * 2);
+    assert.equal(compositionManifest.expectedCallCount, plan.selectedCompositionCaseIds.length * plan.candidateModels.length * 2);
+});
 test("failed run reports persist the nested qualification error chain", async () => {
     const root = await makeRepository();
     const plan = await createPlan(root, "full");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.21",
+  "version": "1.0.23",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.21",
-    "@velum-labs/routekit-runtime": "1.0.21"
+    "@velum-labs/routekit-eval-contracts": "1.0.23",
+    "@velum-labs/routekit-runtime": "1.0.23"
   },
   "keywords": [
     "routekit",