@velum-labs/routekit-eval-setup 1.1.1 → 1.3.0

package/dist/project-workflow.d.ts

@@ -1,18 +1,19 @@
 import { Context, Effect, Layer, Path } from "effect";
 import type { EvalProjectArtifactError, EvalSetupInspectionError } from "./errors.js";
-import { EvalDimensionContrastError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
+import { EvalDimensionContrastError, type EvalModelCatalogError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
 import { EvalProjectArtifacts } from "./project-artifacts.js";
 import type { EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectStatus, EvalRunReport } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
-export type EvalProjectWorkflowError = EvalProjectStoreError | EvalProjectArtifactError | EvalSetupInspectionError | EvalDimensionContrastError | EvalProjectTransitionError;
+import { EvalModelCatalog } from "./services/model-catalog/service.js";
+export type EvalProjectWorkflowError = EvalProjectStoreError | EvalProjectArtifactError | EvalSetupInspectionError | EvalModelCatalogError | EvalDimensionContrastError | EvalProjectTransitionError;
 export type EvalProjectWorkflowShape = {
     readonly setup: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly status: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus | undefined, EvalProjectStoreError | EvalProjectArtifactError>;
-    readonly answer: (repositoryRoot: string, answer: string) => Effect.Effect<EvalProjectStatus, EvalProjectStoreError | EvalProjectArtifactError | EvalProjectTransitionError>;
+    readonly answer: (repositoryRoot: string, answer: string) => Effect.Effect<EvalProjectStatus, EvalProjectStoreError | EvalProjectArtifactError | EvalProjectTransitionError | EvalModelCatalogError>;
     readonly proposeDimensions: (repositoryRoot: string, dimensions: readonly EvalProposedDimension[]) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveDimensions: (repositoryRoot: string, basisDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
-    readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "candidateReasoningEfforts" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveEvaluations: (repositoryRoot: string, evaluationDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
     readonly startRun: (repositoryRoot: string, planId: string) => Effect.Effect<{
@@ -27,6 +28,6 @@ export type EvalProjectWorkflowShape = {
 declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
 export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
 }
-export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
-export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
 export {};

package/dist/project-workflow.js

@@ -3,9 +3,11 @@ import { RoutingActivationConstraints, RoutingObjectivePolicy } from "@velum-lab
 import { Clock, Context, Effect, Layer, Path, Schema } from "effect";
 import { EvalDimensionContrastError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
+import { candidateReasoningEffortFromAnswer } from "./model-selection.js";
 import { EvalProjectArtifacts, evaluationProposalDigest, routingBasisDigest } from "./project-artifacts.js";
 import { EVAL_PROJECT_VERSION, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
+import { EvalModelCatalog } from "./services/model-catalog/service.js";
 const isoNow = Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis).toISOString());
 const questionForProgress = (progress) => {
     switch (progress._tag) {
@@ -21,6 +23,12 @@ const questionForProgress = (progress) => {
                 prompt: "Which explicit provider/model IDs may RouteKit route to?",
                 options: []
             };
+        case "CandidateReasoningEffortsRequired":
+            return {
+                id: "candidate-reasoning-efforts",
+                prompt: `Choose an advertised reasoning effort for ${progress.currentCandidateModel}.`,
+                options: [...progress.supportedReasoningEfforts[progress.currentCandidateModel]]
+            };
         case "ClassifierModelRequired":
             return {
                 id: "classifier-model",
@@ -124,7 +132,24 @@ const parseModel = (state, answer, role) => Effect.gen(function* () {
     const model = yield* nonEmptyAnswer(state, answer);
     return yield* validateModel(state, model, role);
 });
+const supportedCandidateReasoningEfforts = (candidates, input) => {
+    const supported = {};
+    for (const model of candidates) {
+        const efforts = [...new Set(input[model] ?? [])];
+        if (efforts.length > 0)
+            supported[model] = efforts;
+    }
+    return supported;
+};
+const firstReasoningCandidate = (candidates, supported) => candidates.find((model) => supported[model] !== undefined);
+const nextReasoningCandidate = (candidates, supported, selected) => candidates.find((model) => supported[model] !== undefined && selected[model] === undefined);
 const sameStrings = (left, right) => left.length === right.length && left.every((value, index) => value === right[index]);
+const sameReasoningEfforts = (left, right) => {
+    const leftEntries = Object.entries(left ?? {});
+    const rightEntries = Object.entries(right ?? {});
+    return (leftEntries.length === rightEntries.length &&
+        leftEntries.every(([model, effort]) => right?.[model] === effort));
+};
 const sameLedger = (left, right) => left.expectedCalls === right.expectedCalls &&
     left.observedCalls === right.observedCalls &&
     left.observedCandidateRows === right.observedCandidateRows &&
@@ -300,6 +325,7 @@ const parseObjective = (state, answer) => Effect.gen(function* () {
 const configurationFrom = (progress, constraints) => ({
     workloadDescription: progress.workloadDescription,
     candidateModels: progress.candidateModels,
+    candidateReasoningEfforts: progress.candidateReasoningEfforts ?? {},
     classifierModel: progress.classifierModel,
     authorModel: progress.authorModel,
     judgeModel: progress.judgeModel,
@@ -324,7 +350,7 @@ const parseConstraints = (state, answer) => Effect.gen(function* () {
     const constraints = yield* Schema.decodeUnknownEffect(RoutingActivationConstraints)(json).pipe(Effect.mapError(() => transitionError(state.stage, "routing constraints contain invalid dimension quality or failure-rate values")));
     return Object.keys(constraints).length === 0 ? undefined : constraints;
 });
-const advanceSetup = (state, answer, now) => Effect.gen(function* () {
+const advanceSetup = (state, answer, now, discoverReasoningEfforts) => Effect.gen(function* () {
     const common = {
         version: state.version,
         projectId: state.projectId,
@@ -343,16 +369,58 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     workloadDescription: yield* nonEmptyAnswer(state, answer)
                 }
             };
-        case "CandidateModelsRequired":
+        case "CandidateModelsRequired": {
+            const candidateModels = yield* parseCandidateModels(state, answer);
+            const supportedReasoningEfforts = supportedCandidateReasoningEfforts(candidateModels, yield* discoverReasoningEfforts(candidateModels));
+            const currentCandidateModel = firstReasoningCandidate(candidateModels, supportedReasoningEfforts);
             return {
                 ...common,
                 stage: "setup-required",
-                progress: {
-                    _tag: "ClassifierModelRequired",
-                    workloadDescription: state.progress.workloadDescription,
-                    candidateModels: yield* parseCandidateModels(state, answer)
-                }
+                progress: currentCandidateModel === undefined
+                    ? {
+                        _tag: "ClassifierModelRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels,
+                        candidateReasoningEfforts: {}
+                    }
+                    : {
+                        _tag: "CandidateReasoningEffortsRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels,
+                        candidateReasoningEfforts: {},
+                        currentCandidateModel,
+                        supportedReasoningEfforts
+                    }
             };
+        }
+        case "CandidateReasoningEffortsRequired": {
+            const supportedReasoningEfforts = state.progress.supportedReasoningEfforts;
+            const currentCandidateModel = state.progress.currentCandidateModel;
+            const candidateReasoningEfforts = {
+                ...state.progress.candidateReasoningEfforts,
+                [currentCandidateModel]: yield* Effect.try({
+                    try: () => candidateReasoningEffortFromAnswer(answer, supportedReasoningEfforts[currentCandidateModel]),
+                    catch: (cause) => transitionError(state.stage, cause instanceof Error ? cause.message : String(cause))
+                })
+            };
+            const nextCandidateModel = nextReasoningCandidate(state.progress.candidateModels, supportedReasoningEfforts, candidateReasoningEfforts);
+            return {
+                ...common,
+                stage: "setup-required",
+                progress: nextCandidateModel === undefined
+                    ? {
+                        _tag: "ClassifierModelRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels: state.progress.candidateModels,
+                        candidateReasoningEfforts
+                    }
+                    : {
+                        ...state.progress,
+                        candidateReasoningEfforts,
+                        currentCandidateModel: nextCandidateModel
+                    }
+            };
+        }
         case "ClassifierModelRequired":
             return {
                 ...common,
@@ -361,6 +429,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "AuthorModelRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: yield* parseModel(state, answer, "classifier")
                 }
             };
@@ -372,6 +441,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "JudgeModelRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: yield* parseModel(state, answer, "author")
                 }
@@ -384,6 +454,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "RoutingObjectiveRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: yield* parseModel(state, answer, "judge")
@@ -398,6 +469,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "MaximumUnknownWeightRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: state.progress.judgeModel,
@@ -413,6 +485,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "RoutingConstraintsRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: state.progress.judgeModel,
@@ -438,6 +511,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
     const store = yield* EvalProjectStore;
     const artifacts = yield* EvalProjectArtifacts;
     const inspector = yield* EvalRepositoryInspector;
+    const modelCatalog = yield* EvalModelCatalog;
     const paths = yield* Path.Path;
     const resolveRoot = (repositoryRoot) => paths.resolve(repositoryRoot);
     const statusOf = (root, state) => Effect.gen(function* () {
@@ -514,7 +588,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         if (state.stage !== "setup-required") {
             return yield* transitionError(state.stage, "project setup has no unanswered question");
         }
-        const next = yield* advanceSetup(state, answerText, yield* isoNow);
+        const next = yield* advanceSetup(state, answerText, yield* isoNow, modelCatalog.reasoningEfforts);
         yield* store.save(root, next);
         return yield* statusOf(root, next);
     });
@@ -663,6 +737,11 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             version: EVAL_PROJECT_VERSION,
             basisDigest: state.basisDigest,
             candidateModels: state.configuration.candidateModels,
+            ...(state.configuration.candidateReasoningEfforts === undefined
+                ? {}
+                : {
+                    candidateReasoningEfforts: state.configuration.candidateReasoningEfforts
+                }),
             judgeModel: state.configuration.judgeModel,
             suites: input.suites,
             decompositionBenchmark: input.decompositionBenchmark,
@@ -764,6 +843,11 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             basisDigest: state.basisDigest,
             evaluationDigest: state.evaluationDigest,
             candidateModels: state.configuration.candidateModels,
+            ...(state.configuration.candidateReasoningEfforts === undefined
+                ? {}
+                : {
+                    candidateReasoningEfforts: state.configuration.candidateReasoningEfforts
+                }),
             classifierModel: state.configuration.classifierModel,
             authorModel: state.configuration.authorModel,
             judgeModel: state.configuration.judgeModel,
@@ -792,6 +876,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             plan.basisDigest !== state.basisDigest ||
             plan.evaluationDigest !== state.evaluationDigest ||
             !sameStrings(plan.candidateModels, state.configuration.candidateModels) ||
+            !sameReasoningEfforts(plan.candidateReasoningEfforts, state.configuration.candidateReasoningEfforts) ||
             plan.classifierModel !== state.configuration.classifierModel ||
             plan.authorModel !== state.configuration.authorModel ||
             plan.judgeModel !== state.configuration.judgeModel) {

package/dist/questions.d.ts

@@ -1,6 +1,6 @@
 import type { EvalSetupStage, EvalSetupState } from "@velum-labs/routekit-eval-contracts";
 import type { RepositoryInspection, SetupQuestion } from "./types.js";
-export declare const questionForStage: (stage: EvalSetupStage, inspection?: RepositoryInspection) => SetupQuestion | undefined;
+export declare const questionForStage: (stage: EvalSetupStage, inspection?: RepositoryInspection, reasoningEfforts?: readonly string[]) => SetupQuestion | undefined;
 export declare const withOpenQuestion: (state: EvalSetupState, inspection?: RepositoryInspection) => {
     readonly state: EvalSetupState;
     readonly question?: SetupQuestion;

package/dist/questions.js

@@ -2,7 +2,7 @@ const firstThree = (values, fallback) => {
     const unique = [...new Set(values.filter((value) => value.trim().length > 0))].slice(0, 3);
     return [unique[0] ?? fallback[0], unique[1] ?? fallback[1], unique[2] ?? fallback[2]];
 };
-export const questionForStage = (stage, inspection) => {
+export const questionForStage = (stage, inspection, reasoningEfforts = []) => {
     switch (stage) {
         case "surface":
             return {
@@ -40,6 +40,12 @@ export const questionForStage = (stage, inspection) => {
                     "Help me find three explicit model IDs"
                 ]
             };
+        case "reasoning-effort":
+            return {
+                id: stage,
+                prompt: "Choose an explicit reasoning effort advertised by the live model catalog.",
+                options: [...new Set(reasoningEfforts)]
+            };
         case "spend-approval":
             return {
                 id: stage,

package/dist/services/model-catalog/service.d.ts

@@ -0,0 +1,10 @@
+import type { EvalReasoningEffort } from "@velum-labs/routekit-eval-contracts";
+import { Context, type Effect } from "effect";
+import type { EvalModelCatalogError } from "../../errors.js";
+export type EvalModelCatalogShape = {
+    readonly reasoningEfforts: (candidateModels: readonly string[]) => Effect.Effect<Readonly<Record<string, readonly EvalReasoningEffort[]>>, EvalModelCatalogError>;
+};
+declare const EvalModelCatalog_base: Context.ServiceClass<EvalModelCatalog, "@velum-labs/routekit-eval-setup/EvalModelCatalog", EvalModelCatalogShape>;
+export declare class EvalModelCatalog extends EvalModelCatalog_base {
+}
+export {};

package/dist/services/model-catalog/service.js

@@ -0,0 +1,3 @@
+import { Context } from "effect";
+export class EvalModelCatalog extends Context.Service()("@velum-labs/routekit-eval-setup/EvalModelCatalog") {
+}

package/dist/test/model-selection.test.js

@@ -1,12 +1,17 @@
 import assert from "node:assert/strict";
 import { test } from "node:test";
-import { modelSelectionFromAnswer } from "../model-selection.js";
+import { candidateReasoningEffortFromAnswer, modelSelectionFromAnswer } from "../model-selection.js";
 test("model selection accepts exactly two candidates followed by a distinct judge", () => {
     assert.deepEqual(modelSelectionFromAnswer("openai/gpt-a anthropic/claude-b google/gemini-judge"), {
         candidates: ["openai/gpt-a", "anthropic/claude-b"],
         judgeModel: "google/gemini-judge"
     });
 });
+test("reasoning effort selection accepts exactly one advertised opaque value", () => {
+    const supported = ["none", "low", "provider-next"];
+    assert.equal(candidateReasoningEffortFromAnswer(" provider-next ", supported), "provider-next");
+    assert.throws(() => candidateReasoningEffortFromAnswer("max", supported), /must be one of/iu);
+});
 test("model selection rejects canned answers, duplicates, aliases, and extra ids", () => {
     assert.throws(() => modelSelectionFromAnswer("Current model, a cheaper candidate, and a stronger candidate"), /exactly three explicit provider\/model ids/iu);
     assert.throws(() => modelSelectionFromAnswer("openai/a openai/a openai/judge"), /three unique model ids/iu);

package/dist/test/project-workflow.test.js

@@ -5,16 +5,30 @@ import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { Cause, Effect, Layer } from "effect";
-import { EvalDimensionContrastError } from "../errors.js";
+import { EvalDimensionContrastError, EvalModelCatalogError } from "../errors.js";
 import { EvalRepositoryInspectorLive } from "../inspection.js";
 import { EvalProjectArtifactsLive } from "../project-artifacts.js";
 import { scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
 import { EvalProjectStoreLive } from "../project-store.js";
 import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
+import { EvalModelCatalog } from "../services/model-catalog/service.js";
 const roots = [];
 after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
 const ProjectDependenciesLive = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive, EvalRepositoryInspectorLive).pipe(Layer.provide(NodeServicesLayer));
-const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(NodeServicesLayer));
+const catalogRequests = [];
+const catalogEfforts = (candidateModels) => Effect.sync(() => {
+    catalogRequests.push([...candidateModels]);
+    return {
+        ...(candidateModels.includes("openai/gpt-5.6-luna")
+            ? { "openai/gpt-5.6-luna": ["low", "medium", "ultra"] }
+            : {}),
+        ...(candidateModels.includes("openai/gpt-5.6-terra")
+            ? { "openai/gpt-5.6-terra": ["none", "low", "high"] }
+            : {})
+    };
+});
+const EvalModelCatalogTest = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({ reasoningEfforts: catalogEfforts }));
+const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(NodeServicesLayer));
 const makeRepository = async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-project-"));
     roots.push(root);
@@ -47,22 +61,47 @@ test("setup creates one durable repository project without invoking model-backed
 });
 test("answers advance exactly one question and persist a complete compositional configuration", async () => {
     const root = await makeRepository();
+    catalogRequests.length = 0;
     const statuses = await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
         const workload = yield* workflow.answer(root, "Customer support and documentation requests");
         const candidates = yield* workflow.answer(root, "openai/gpt-5.6-luna, openai/gpt-5.6-terra openai/gpt-5.6-sol");
+        const lunaEffort = yield* workflow.answer(root, candidates.question.options[1]);
+        const terraEffort = yield* workflow.answer(root, lunaEffort.question.options[1]);
         const classifier = yield* workflow.answer(root, "openai/gpt-5.6-luna");
         const author = yield* workflow.answer(root, "openai/gpt-5.6-terra");
         const judge = yield* workflow.answer(root, "openai/gpt-5.6-terra");
         const objective = yield* workflow.answer(root, '{"kind":"highest-quality"}');
         const unknown = yield* workflow.answer(root, "0.2");
         const completed = yield* workflow.answer(root, "{}");
-        return { workload, candidates, classifier, author, judge, objective, unknown, completed };
+        return {
+            workload,
+            candidates,
+            lunaEffort,
+            terraEffort,
+            classifier,
+            author,
+            judge,
+            objective,
+            unknown,
+            completed
+        };
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
     assert.equal(statuses.workload.question?.id, "candidate-models");
     assert.equal(statuses.workload.state.revision, 1);
-    assert.equal(statuses.candidates.question?.id, "classifier-model");
+    assert.deepEqual(catalogRequests, [[
+            "openai/gpt-5.6-luna",
+            "openai/gpt-5.6-terra",
+            "openai/gpt-5.6-sol"
+        ]]);
+    assert.equal(statuses.candidates.question?.id, "candidate-reasoning-efforts");
+    assert.match(statuses.candidates.question?.prompt ?? "", /openai\/gpt-5\.6-luna/u);
+    assert.deepEqual(statuses.candidates.question?.options, ["low", "medium", "ultra"]);
+    assert.equal(statuses.lunaEffort.question?.id, "candidate-reasoning-efforts");
+    assert.match(statuses.lunaEffort.question?.prompt ?? "", /openai\/gpt-5\.6-terra/u);
+    assert.deepEqual(statuses.lunaEffort.question?.options, ["none", "low", "high"]);
+    assert.equal(statuses.terraEffort.question?.id, "classifier-model");
     assert.equal(statuses.classifier.question?.id, "author-model");
     assert.equal(statuses.author.question?.id, "judge-model");
     assert.equal(statuses.judge.question?.id, "routing-objective");
@@ -79,6 +118,10 @@ test("answers advance exactly one question and persist a complete compositional
         "openai/gpt-5.6-terra",
         "openai/gpt-5.6-sol"
     ]);
+    assert.deepEqual(statuses.completed.state.configuration.candidateReasoningEfforts, {
+        "openai/gpt-5.6-luna": "medium",
+        "openai/gpt-5.6-terra": "low"
+    });
     assert.equal(statuses.completed.state.configuration.classifierModel, "openai/gpt-5.6-luna");
     assert.equal(statuses.completed.state.configuration.authorModel, "openai/gpt-5.6-terra");
     assert.equal(statuses.completed.state.configuration.judgeModel, "openai/gpt-5.6-terra");
@@ -91,7 +134,7 @@ test("answers advance exactly one question and persist a complete compositional
         const workflow = yield* EvalProjectWorkflow;
         return yield* workflow.status(root);
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
-    assert.equal(resumed?.state.revision, 8);
+    assert.equal(resumed?.state.revision, 10);
     assert.equal(resumed?.state.stage, "dimensions-review");
 });
 test("invalid answers fail without changing the durable project revision", async () => {
@@ -112,6 +155,30 @@ test("invalid answers fail without changing the durable project revision", async
         assert.equal(afterDuplicate?.question?.id, "candidate-models");
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
 });
+test("candidate selection fails closed when catalog reasoning metadata is unavailable", async () => {
+    const root = await makeRepository();
+    const unavailableCatalog = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({
+        reasoningEfforts: () => Effect.fail(new EvalModelCatalogError({
+            detail: "models.list did not advertise candidate reasoning capabilities"
+        }))
+    }));
+    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(NodeServicesLayer));
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.setup(root);
+        yield* workflow.answer(root, "Production workload");
+        const answer = yield* Effect.exit(workflow.answer(root, "openai/reasoning openai/plain"));
+        assert.equal(answer._tag, "Failure");
+        const current = yield* workflow.status(root);
+        assert.equal(current?.state.revision, 1);
+        assert.equal(current?.state.stage, "setup-required");
+        if (current?.state.stage !== "setup-required") {
+            assert.fail("expected setup-required state");
+        }
+        assert.equal(current.state.progress._tag, "CandidateModelsRequired");
+        assert.equal(current.question?.id, "candidate-models");
+    }).pipe(Effect.provide(workflowLayer)));
+});
 test("corrupt project state preserves the typed project-store failure", async () => {
     const root = await makeRepository();
     await mkdir(path.join(root, ".routekit", "evals"), { recursive: true });
@@ -195,7 +262,11 @@ async function completeProjectSetup(root, candidateModels = "openai/gpt-5.6-luna
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
         yield* workflow.answer(root, "Production workload");
-        yield* workflow.answer(root, candidateModels);
+        let status = yield* workflow.answer(root, candidateModels);
+        while (status.state.stage === "setup-required" &&
+            status.state.progress._tag === "CandidateReasoningEffortsRequired") {
+            status = yield* workflow.answer(root, status.question.options[0]);
+        }
         yield* workflow.answer(root, "openai/gpt-5.6-luna");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
@@ -732,6 +803,10 @@ run.toComplete();
     assert.equal(compositionManifestAfter, compositionManifestBefore);
     const dimensionManifest = JSON.parse(dimensionManifestAfter);
     const compositionManifest = JSON.parse(compositionManifestAfter);
+    assert.deepEqual(dimensionManifest.candidateReasoningEfforts, plan.candidateReasoningEfforts ?? {});
+    assert.deepEqual(compositionManifest.candidateReasoningEfforts, plan.candidateReasoningEfforts ?? {});
+    assert.match(dimensionSuite, /parameters: \{ reasoning: \{ effort \} \}/u);
+    assert.match(compositionSuite, /parameters: \{ reasoning: \{ effort \} \}/u);
     assert.equal(dimensionManifest.expectedCallCount, selection.caseIds.length * plan.candidateModels.length * 2);
     assert.equal(compositionManifest.expectedCallCount, plan.selectedCompositionCaseIds.length * plan.candidateModels.length * 2);
 });

package/dist/test/questions.test.js

@@ -8,13 +8,14 @@ test("setup exposes exactly one question for each unresolved stage", () => {
         "criteria",
         "constraints",
         "candidates",
+        "reasoning-effort",
         "spend-approval",
         "publish"
     ];
     for (const stage of stages) {
         const question = questionForStage(stage);
         assert.equal(question?.id, stage);
-        assert.equal(question?.options.length, 3);
+        assert.equal(question?.options.length === 3 || stage === "reasoning-effort", true);
     }
     assert.equal(questionForStage("completed"), undefined);
 });
@@ -46,3 +47,8 @@ test("candidate question requires two candidates and a distinct judge", () => {
     assert.match(question?.prompt ?? "", /exactly three unique provider\/model IDs/iu);
     assert.doesNotMatch(question?.options.join("\n") ?? "", /save without choosing|current model,/iu);
 });
+test("reasoning-effort stage exposes an explicit picker without a hidden default", () => {
+    const question = questionForStage("reasoning-effort", undefined, ["low", "provider-next"]);
+    assert.match(question?.prompt ?? "", /explicit reasoning effort/iu);
+    assert.deepEqual(question?.options, ["low", "provider-next"]);
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.1.1",
+  "version": "1.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -22,7 +22,6 @@
   },
   "files": [
     "dist",
-    "skills",
     "LICENSE"
   ],
   "publishConfig": {
@@ -32,9 +31,9 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.1.1",
-    "@velum-labs/routekit-eval-core": "1.1.1",
-    "@velum-labs/routekit-runtime": "1.1.1"
+    "@velum-labs/routekit-eval-contracts": "1.3.0",
+    "@velum-labs/routekit-eval-core": "1.3.0",
+    "@velum-labs/routekit-runtime": "1.3.0"
   },
   "keywords": [
     "routekit",

package/dist/test/skill.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/test/skill.test.js

@@ -1,35 +0,0 @@
-import assert from "node:assert/strict";
-import { readFile } from "node:fs/promises";
-import path from "node:path";
-import { test } from "node:test";
-import { fileURLToPath } from "node:url";
-const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
-test("onboarding skill uses the public CLI and preserves approval boundaries", async () => {
-    const skill = await readFile(path.join(packageRoot, "skills", "setup-eval-routing", "SKILL.md"), "utf8");
-    assert.match(skill, /setup-eval-routing/u);
-    assert.match(skill, /one question per turn/iu);
-    assert.match(skill, /Never spend or publish silently/u);
-    assert.match(skill, /public `routekit eval` CLI/u);
-    assert.match(skill, /\$ROUTEKIT eval --help/u);
-    for (const command of ["setup", "status", "answer", "validate", "estimate", "run", "publish"]) {
-        assert.match(skill, new RegExp(`eval ${command}`, "u"));
-    }
-    for (const term of [
-        "routing basis",
-        "workload dimension",
-        "request decomposition",
-        "evidence matrix",
-        "routing activation"
-    ]) {
-        assert.match(skill, new RegExp(term, "iu"));
-    }
-    assert.match(skill, /exclusive in-scope request or a distinct near-miss/iu);
-    assert.match(skill, /product-behavior axes are mixed with repository-change\/process axes/iu);
-    assert.match(skill, /high weight on almost every ticket/iu);
-    assert.match(skill, /Unknown weight absorbs the remainder/u);
-    assert.doesNotMatch(skill, /\beval prepare\b/u);
-    assert.doesNotMatch(skill, /\barea catalog\b/iu);
-    assert.doesNotMatch(skill, /Use RouteKit's `EvalSetup` operations/u);
-    assert.doesNotMatch(skill, /test:e2e:eval-routing/u);
-    assert.doesNotMatch(skill, /OPENROUTER_API_KEY/u);
-});