@velum-labs/routekit-eval-setup 1.0.20 → 1.0.22

package/dist/project-artifacts.js

@@ -30,8 +30,16 @@ for (const model of manifest.candidateModels) {
         ? testCase.prompt
         : [testCase.prompt, "", "Reference material:", "-----", testCase.context, "-----"].join("\\n");
       const run = await candidate.run({ prompt, caseId: testCase.id });
-      run.toComplete();
+      let candidateCompletionError: unknown;
+      try {
+        run.toComplete();
+      } catch (error) {
+        candidateCompletionError = error;
+      }
       await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+      if (candidateCompletionError !== undefined) {
+        throw candidateCompletionError;
+      }
     });
   }
 }

package/dist/project-workflow.js

@@ -787,7 +787,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         const next = {
             version: state.version,
             projectId: state.projectId,
-            revision: state.revision + 1,
+            revision: state.revision,
             createdAt: state.createdAt,
             updatedAt: now,
             sourceInventory: state.sourceInventory,
@@ -879,7 +879,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         const next = {
             version: state.version,
             projectId: state.projectId,
-            revision: state.revision + 1,
+            revision: state.revision,
             createdAt: state.createdAt,
             updatedAt: report.finishedAt,
             sourceInventory: state.sourceInventory,

package/dist/test/project-workflow.test.js

@@ -306,6 +306,14 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
     assert.equal((await stat(planPath)).mode & 0o777, 0o600);
     assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
+    const firstDimension = reviewedDimensions[0];
+    const generatedSuite = await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, `${firstDimension.id}.eval.ts`), "utf8");
+    const completionAssertion = generatedSuite.indexOf("run.toComplete();");
+    const judgeCall = generatedSuite.indexOf("await judge.autoEvals");
+    const completionRethrow = generatedSuite.indexOf("throw candidateCompletionError;");
+    assert.ok(completionAssertion >= 0);
+    assert.ok(judgeCall > completionAssertion);
+    assert.ok(completionRethrow > judgeCall);
     for (const dimension of reviewedDimensions) {
         assert.equal((await stat(path.join(root, ".routekit", "evals", "dimensions", dimension.id, "suite.json")))
             .mode & 0o777, 0o600);
@@ -542,6 +550,77 @@ test("run lifecycle retains complete sanitized evidence and activates only after
     const reportPath = path.join(root, ".routekit", "evals", "runs", outcome.started.runId, "report.json");
     assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
 });
+test("failed runs preserve plan revisions for retry while mismatched plans remain stale", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "full");
+    const mismatchedPlans = [
+        {
+            ...plan,
+            planId: `${plan.planId}-project`,
+            projectId: "different-project-id"
+        },
+        {
+            ...plan,
+            planId: `${plan.planId}-evaluation`,
+            evaluationDigest: "different-evaluation-digest"
+        },
+        {
+            ...plan,
+            planId: `${plan.planId}-models`,
+            candidateModels: [...plan.candidateModels].reverse()
+        }
+    ];
+    await Promise.all(mismatchedPlans.map((mismatchedPlan) => writeFile(path.join(root, ".routekit", "evals", "plans", `${mismatchedPlan.planId}.json`), JSON.stringify(mismatchedPlan))));
+    const rerun = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        for (const mismatchedPlan of mismatchedPlans) {
+            const mismatch = yield* Effect.exit(workflow.startRun(root, mismatchedPlan.planId));
+            assert.equal(mismatch._tag, "Failure");
+            if (mismatch._tag === "Failure") {
+                assert.match(String(mismatch.cause), /execution plan is stale/u);
+            }
+        }
+        const first = yield* workflow.startRun(root, plan.planId);
+        const running = yield* workflow.status(root);
+        const ready = yield* workflow.failRun(root, {
+            version: 1,
+            status: "failed",
+            runId: first.runId,
+            planId: plan.planId,
+            projectId: running.state.projectId,
+            startedAt: "2026-08-21T03:47:27.000Z",
+            finishedAt: "2026-08-21T03:47:48.000Z",
+            basisDigest: plan.basisDigest,
+            evaluationDigest: plan.evaluationDigest,
+            target: {
+                kind: "configured",
+                identity: "routekit-generation:48",
+                publishAllowed: true
+            },
+            cleanup: { sessionOpened: true, sessionClosed: true },
+            comparisons: [],
+            ledger: {
+                expectedCalls: plan.expectedCallCount,
+                observedCalls: 0,
+                observedCandidateRows: 0,
+                knownInputTokens: 0,
+                knownOutputTokens: 0,
+                unknownTokenMeasurements: 0,
+                knownPricedSubtotalUsd: 0,
+                unpricedCalls: 0
+            },
+            failure: "qualification timed out before observing any calls",
+            errors: []
+        });
+        const second = yield* workflow.startRun(root, plan.planId);
+        return { ready, second };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(rerun.ready.state.stage, "ready");
+    assert.equal(rerun.ready.state.revision, plan.projectRevision);
+    assert.equal(rerun.ready.state.basisDigest, plan.basisDigest);
+    assert.equal(rerun.ready.state.evaluationDigest, plan.evaluationDigest);
+    assert.equal(rerun.second.plan.planId, plan.planId);
+});
 test("failed run reports persist the nested qualification error chain", async () => {
     const root = await makeRepository();
     const plan = await createPlan(root, "full");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.20",
+  "version": "1.0.22",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.20",
-    "@velum-labs/routekit-runtime": "1.0.20"
+    "@velum-labs/routekit-eval-contracts": "1.0.22",
+    "@velum-labs/routekit-runtime": "1.0.22"
   },
   "keywords": [
     "routekit",