@velum-labs/routekit-eval-setup 1.0.16 → 1.0.17

package/dist/index.d.ts

@@ -7,8 +7,8 @@ export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export type { EvalAuthoringCompletion, EvalAuthoringSource, EvalAuthoringTransportShape, EvalProjectAuthorShape } from "./project-authoring.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export type { EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProjectArtifactsStatus, EvalProjectStatus, EvalRunCleanup, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget } from "./project-contracts.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRunCleanup as EvalRunCleanupSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
+export type { EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProjectArtifactsStatus, EvalProjectStatus, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export type { EvalProjectWorkflowError, EvalProjectWorkflowShape } from "./project-workflow.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";

package/dist/index.js

@@ -4,7 +4,7 @@ export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRunCleanup as EvalRunCleanupSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
 export { questionForStage, withOpenQuestion } from "./questions.js";

package/dist/project-contracts.d.ts

@@ -747,6 +747,12 @@ export declare const EvalRunQualification: Schema.Struct<{
 }>;
 export type EvalRunQualification = typeof EvalRunQualification.Type;
 export declare function summarizeEvalRunLedger(comparisons: readonly EvalComparisonResultType[], expectedCalls: number, classifierObservations?: readonly EvalClassifierObservation[]): EvalRunLedger;
+export declare const EvalRunFailureError: Schema.Struct<{
+    readonly name: Schema.String;
+    readonly message: Schema.String;
+    readonly stack: Schema.optionalKey<Schema.String>;
+}>;
+export type EvalRunFailureError = typeof EvalRunFailureError.Type;
 /**
  * Sanitized durable result of one immutable execution plan.
  *
@@ -1078,6 +1084,11 @@ export declare const EvalRunReport: Schema.Union<readonly [Schema.Struct<{
 }>, Schema.Struct<{
     readonly status: Schema.Literal<"failed">;
     readonly failure: Schema.String;
+    readonly errors: Schema.optionalKey<Schema.$Array<Schema.Struct<{
+        readonly name: Schema.String;
+        readonly message: Schema.String;
+        readonly stack: Schema.optionalKey<Schema.String>;
+    }>>>;
     readonly version: Schema.Literal<1>;
     readonly runId: Schema.String;
     readonly planId: Schema.String;

package/dist/project-contracts.js

@@ -366,6 +366,11 @@ const EvalRunReportCommon = {
     qualification: Schema.optionalKey(EvalRunQualification),
     ledger: EvalRunLedger
 };
+export const EvalRunFailureError = Schema.Struct({
+    name: Schema.String,
+    message: Schema.String,
+    stack: Schema.optionalKey(Schema.String)
+});
 const QualifiedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,
     status: Schema.Literal("passed"),
@@ -379,7 +384,8 @@ const CompletedEvalRunReport = Schema.Struct({
 const FailedEvalRunReport = Schema.Struct({
     ...EvalRunReportCommon,
     status: Schema.Literal("failed"),
-    failure: Schema.String
+    failure: Schema.String,
+    errors: Schema.optionalKey(Schema.Array(EvalRunFailureError))
 });
 /**
  * Sanitized durable result of one immutable execution plan.

package/dist/test/project-workflow.test.js

@@ -476,6 +476,77 @@ test("run lifecycle retains complete sanitized evidence and activates only after
     const reportPath = path.join(root, ".routekit", "evals", "runs", outcome.started.runId, "report.json");
     assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
 });
+test("failed run reports persist the nested qualification error chain", async () => {
+    const root = await makeRepository();
+    const plan = await createPlan(root, "full");
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        const started = yield* workflow.startRun(root, plan.planId);
+        const status = yield* workflow.status(root);
+        const report = {
+            version: 1,
+            status: "failed",
+            runId: started.runId,
+            planId: plan.planId,
+            projectId: status.state.projectId,
+            startedAt: "2026-08-21T03:47:27.000Z",
+            finishedAt: "2026-08-21T03:47:48.000Z",
+            basisDigest: plan.basisDigest,
+            evaluationDigest: plan.evaluationDigest,
+            target: {
+                kind: "configured",
+                identity: "routekit-generation:48",
+                publishAllowed: true
+            },
+            cleanup: { sessionOpened: true, sessionClosed: true },
+            comparisons: [],
+            ledger: {
+                expectedCalls: plan.expectedCallCount,
+                observedCalls: 0,
+                observedCandidateRows: 0,
+                knownInputTokens: 0,
+                knownOutputTokens: 0,
+                unknownTokenMeasurements: 0,
+                knownPricedSubtotalUsd: 0,
+                unpricedCalls: 0
+            },
+            failure: 'qualification dimension "provider-protocol-translation" failed ' +
+                "(per-test timeout 600000ms); observed call ids: none",
+            errors: [
+                {
+                    name: "EvalServiceComparisonError",
+                    message: "RouteKit Eval comparison failed",
+                    stack: "EvalServiceComparisonError: RouteKit Eval comparison failed"
+                },
+                {
+                    name: "RouteKitEvalGatewayBridgeStartError",
+                    message: "Could not start the scoped RouteKit Eval gateway bridge.",
+                    stack: "RouteKitEvalGatewayBridgeStartError: Could not start the scoped RouteKit Eval gateway bridge."
+                }
+            ]
+        };
+        yield* workflow.failRun(root, report);
+        return {
+            loaded: yield* workflow.result(root, started.runId),
+            runId: started.runId
+        };
+    }).pipe(Effect.provide(ProjectWorkflowTestLive)));
+    assert.equal(outcome.loaded?.status, "failed");
+    assert.deepEqual(outcome.loaded?.status === "failed" ? outcome.loaded.errors : undefined, [
+        {
+            name: "EvalServiceComparisonError",
+            message: "RouteKit Eval comparison failed",
+            stack: "EvalServiceComparisonError: RouteKit Eval comparison failed"
+        },
+        {
+            name: "RouteKitEvalGatewayBridgeStartError",
+            message: "Could not start the scoped RouteKit Eval gateway bridge.",
+            stack: "RouteKitEvalGatewayBridgeStartError: Could not start the scoped RouteKit Eval gateway bridge."
+        }
+    ]);
+    const raw = JSON.parse(await readFile(path.join(root, ".routekit", "evals", "runs", outcome.runId, "report.json"), "utf8"));
+    assert.equal(raw.errors?.length, 2);
+});
 test("incomplete evidence and external qualification fail closed", async () => {
     const incompleteRoot = await makeRepository();
     const incompletePlan = await createPlan(incompleteRoot, "full");

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.0.16",
+  "version": "1.0.17",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -32,8 +32,8 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.0.16",
-    "@velum-labs/routekit-runtime": "1.0.16"
+    "@velum-labs/routekit-eval-contracts": "1.0.17",
+    "@velum-labs/routekit-runtime": "1.0.17"
   },
   "keywords": [
     "routekit",