@velum-labs/routekit-eval-setup 1.0.21 → 1.0.22

package/dist/project-artifacts.js

@@ -30,8 +30,16 @@ for (const model of manifest.candidateModels) {
         ? testCase.prompt
         : [testCase.prompt, "", "Reference material:", "-----", testCase.context, "-----"].join("\\n");
       const run = await candidate.run({ prompt, caseId: testCase.id });
-      run.toComplete();
+      let candidateCompletionError: unknown;
+      try {
+        run.toComplete();
+      } catch (error) {
+        candidateCompletionError = error;
+      }
       await judge.autoEvals({ criteria: testCase.rubric, prompt, run });
+      if (candidateCompletionError !== undefined) {
+        throw candidateCompletionError;
+      }
     });
   }
 }

package/dist/test/project-workflow.test.js

@@ -306,6 +306,14 @@ test("reviewed artifacts are digest-bound and produce an immutable exact-call pl
     const planPath = path.join(root, ".routekit", "evals", "plans", `${result.plan.planId}.json`);
     assert.equal((await stat(planPath)).mode & 0o777, 0o600);
     assert.deepEqual(JSON.parse(await readFile(planPath, "utf8")), result.plan);
+    const firstDimension = reviewedDimensions[0];
+    const generatedSuite = await readFile(path.join(root, ".routekit", "evals", "plans", result.plan.planId, "dimensions", firstDimension.id, `${firstDimension.id}.eval.ts`), "utf8");
+    const completionAssertion = generatedSuite.indexOf("run.toComplete();");
+    const judgeCall = generatedSuite.indexOf("await judge.autoEvals");
+    const completionRethrow = generatedSuite.indexOf("throw candidateCompletionError;");
+    assert.ok(completionAssertion >= 0);
+    assert.ok(judgeCall > completionAssertion);
+    assert.ok(completionRethrow > judgeCall);
     for (const dimension of reviewedDimensions) {
         assert.equal((await stat(path.join(root, ".routekit", "evals", "dimensions", dimension.id, "suite.json")))
             .mode & 0o777, 0o600);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.21",
+  "version": "1.0.22",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.21",
-    "@velum-labs/routekit-runtime": "1.0.21"
+    "@velum-labs/routekit-eval-contracts": "1.0.22",
+    "@velum-labs/routekit-runtime": "1.0.22"
   },
   "keywords": [
     "routekit",