@velum-labs/routekit-eval-setup 1.2.0 → 1.3.0

package/dist/effect-api.d.ts

@@ -15,6 +15,8 @@ export { EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeE
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export type { EvalProjectWorkflowError, EvalProjectWorkflowShape } from "./project-workflow.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
+export type { EvalModelCatalogShape } from "./services/model-catalog/service.js";
+export { EvalModelCatalog } from "./services/model-catalog/service.js";
 export { EvalSetupRunner, EvalSetupRunnerNoop } from "./runner.js";
 export { EvalSetup, EvalSetupLive, makeEvalSetup } from "./service.js";
 export { EvalSetupStateStore, EvalSetupStateStoreLive, initialSetupState, makeFileEvalSetupStateStore } from "./state-store.js";

package/dist/effect-api.js

@@ -9,6 +9,7 @@ export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDiges
 export { EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
+export { EvalModelCatalog } from "./services/model-catalog/service.js";
 export { EvalSetupRunner, EvalSetupRunnerNoop } from "./runner.js";
 export { EvalSetup, EvalSetupLive, makeEvalSetup } from "./service.js";
 export { EvalSetupStateStore, EvalSetupStateStoreLive, initialSetupState, makeFileEvalSetupStateStore } from "./state-store.js";

package/dist/errors.d.ts

@@ -67,6 +67,15 @@ export declare class EvalProjectTransitionError extends EvalProjectTransitionErr
 }> {
     get message(): string;
 }
+declare const EvalModelCatalogError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "EvalModelCatalogError";
+} & Readonly<A>;
+export declare class EvalModelCatalogError extends EvalModelCatalogError_base<{
+    readonly detail: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
 declare const EvalDimensionContrastError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
     readonly _tag: "EvalDimensionContrastError";
 } & Readonly<A>;

package/dist/errors.js

@@ -34,6 +34,11 @@ export class EvalProjectTransitionError extends Data.TaggedError("EvalProjectTra
         return `RouteKit Eval project cannot continue from ${this.state}: ${this.detail}`;
     }
 }
+export class EvalModelCatalogError extends Data.TaggedError("EvalModelCatalogError") {
+    get message() {
+        return `RouteKit Eval could not discover candidate reasoning capabilities: ${this.detail}`;
+    }
+}
 export class EvalDimensionContrastError extends Data.TaggedError("EvalDimensionContrastError") {
     get message() {
         return `RouteKit Eval cannot approve routing dimension ${JSON.stringify(this.dimensionId)}: ${this.detail}`;

package/dist/index.d.ts

@@ -1,5 +1,7 @@
 export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
-export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalModelCatalogError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export type { EvalModelCatalogShape } from "./services/model-catalog/service.js";
+export { EvalModelCatalog } from "./services/model-catalog/service.js";
 export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
 export type { CapturedRepositorySnapshotV3, FrozenRepositorySourceV3 } from "./adapters/git-basis-snapshot.js";
 export { readFrozenRepositorySourcesV3 } from "./adapters/git-basis-snapshot.js";

package/dist/index.js

@@ -1,5 +1,6 @@
 export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
-export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalModelCatalogError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { EvalModelCatalog } from "./services/model-catalog/service.js";
 export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
 export { readFrozenRepositorySourcesV3 } from "./adapters/git-basis-snapshot.js";
 export { authoringRequest, hostDirectory } from "./host-metadata.js";

package/dist/model-selection.d.ts

@@ -1,6 +1,8 @@
+import { type EvalReasoningEffort } from "@velum-labs/routekit-eval-contracts";
 export type EvalModelSelection = {
     readonly candidates: readonly [string, string];
     readonly judgeModel: string;
 };
 export declare const assertEvalModelSelection: (candidates: readonly string[], judgeModel: string) => EvalModelSelection;
 export declare const modelSelectionFromAnswer: (answer: string) => EvalModelSelection;
+export declare const candidateReasoningEffortFromAnswer: (answer: string, supported: readonly EvalReasoningEffort[]) => EvalReasoningEffort;

package/dist/model-selection.js

@@ -35,3 +35,10 @@ export const modelSelectionFromAnswer = (answer) => {
     }
     return assertEvalModelSelection(modelIds.slice(0, 2), modelIds[2] ?? "");
 };
+export const candidateReasoningEffortFromAnswer = (answer, supported) => {
+    const effort = answer.trim();
+    if (!supported.includes(effort)) {
+        throw new Error(`reasoning effort must be one of ${supported.join(", ")}`);
+    }
+    return effort;
+};

package/dist/project-artifacts.js

@@ -37,13 +37,20 @@ import manifest from "./routekit.eval-manifest.json" with { type: "json" };
 
 assert.equal(cases.length, manifest.caseCount);
 assert.deepEqual(cases.map((testCase) => testCase.id), manifest.caseIds);
+const candidateReasoningEfforts = manifest.candidateReasoningEfforts ?? {};
 const judge = setupJudge({
   agent: setupAgent({ model: manifest.judgeModel }),
   minScore: 0.8
 });
 
 for (const model of manifest.candidateModels) {
-  const candidate = setupAgent({ model });
+  const effort = candidateReasoningEfforts[model];
+  const candidate = setupAgent({
+    model,
+    ...(effort === undefined
+      ? {}
+      : { parameters: { reasoning: { effort } } })
+  });
   for (const testCase of cases) {
     test(\`\${model} / \${testCase.id}\`, async () => {
       ${renderCandidatePrompt()}
@@ -89,6 +96,9 @@ function assertEvaluationProposal(proposal) {
             version: proposal.version,
             basisDigest: proposal.basisDigest,
             candidateModels: proposal.candidateModels,
+            ...(proposal.candidateReasoningEfforts === undefined
+                ? {}
+                : { candidateReasoningEfforts: proposal.candidateReasoningEfforts }),
             judgeModel: proposal.judgeModel,
             suites: proposal.suites,
             decompositionBenchmark: proposal.decompositionBenchmark,
@@ -102,6 +112,9 @@ function assertEvaluationProposal(proposal) {
     if (new Set(proposal.candidateModels).size !== proposal.candidateModels.length) {
         throw new Error("evaluation proposal candidate models must be unique");
     }
+    if (Object.keys(proposal.candidateReasoningEfforts ?? {}).some((model) => !proposal.candidateModels.includes(model))) {
+        throw new Error("evaluation proposal reasoning efforts must belong to candidate models");
+    }
     if (proposal.judgeModel.trim().length === 0) {
         throw new Error("evaluation proposal judge must be explicit");
     }
@@ -285,6 +298,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     version: EVAL_PROJECT_VERSION,
                     profileId: suite.dimensionId,
                     candidateModels: proposal.candidateModels,
+                    candidateReasoningEfforts: proposal.candidateReasoningEfforts ?? {},
                     judgeModel: proposal.judgeModel,
                     caseCount: suite.cases.length,
                     caseIds: suite.cases.map((testCase) => testCase.id),
@@ -324,6 +338,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                     version: EVAL_PROJECT_VERSION,
                     profileId: selection.dimensionId,
                     candidateModels: plan.candidateModels,
+                    candidateReasoningEfforts: plan.candidateReasoningEfforts ?? {},
                     judgeModel: plan.judgeModel,
                     caseCount: cases.length,
                     caseIds: cases.map((testCase) => testCase.id),
@@ -346,6 +361,7 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
                 version: EVAL_PROJECT_VERSION,
                 profileId: "composition",
                 candidateModels: plan.candidateModels,
+                candidateReasoningEfforts: plan.candidateReasoningEfforts ?? {},
                 judgeModel: plan.judgeModel,
                 caseCount: compositionCases.length,
                 caseIds: compositionCases.map((testCase) => testCase.id),

package/dist/project-authoring.d.ts

@@ -59,7 +59,7 @@ export type EvalProjectAuthorShape = {
         readonly sourceInventory: readonly string[];
         readonly configuration: EvalProjectConfiguration;
         readonly basis: RoutingBasis;
-    }) => Effect.Effect<Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "judgeModel">, EvalProjectAuthoringError>;
+    }) => Effect.Effect<Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "candidateReasoningEfforts" | "judgeModel">, EvalProjectAuthoringError>;
 };
 declare const EvalProjectAuthor_base: Context.ServiceClass<EvalProjectAuthor, "@velum-labs/routekit-eval-setup/EvalProjectAuthor", EvalProjectAuthorShape>;
 export declare class EvalProjectAuthor extends EvalProjectAuthor_base {

package/dist/project-contracts.d.ts

@@ -3,34 +3,45 @@ import { Schema } from "effect";
 export declare const EVAL_PROJECT_VERSION: 1;
 export declare const EVAL_PROJECT_V2_VERSION: 2;
 export declare const EvalProjectQuestion: Schema.Struct<{
-    readonly id: Schema.Literals<readonly ["workload-description", "candidate-models", "classifier-model", "author-model", "judge-model", "routing-objective", "maximum-unknown-weight", "routing-constraints"]>;
+    readonly id: Schema.Literals<readonly ["workload-description", "candidate-models", "candidate-reasoning-efforts", "classifier-model", "author-model", "judge-model", "routing-objective", "maximum-unknown-weight", "routing-constraints"]>;
     readonly prompt: Schema.String;
     readonly options: Schema.$Array<Schema.String>;
 }>;
 export type EvalProjectQuestion = typeof EvalProjectQuestion.Type;
 export declare const EvalProjectSetupProgress: Schema.Union<readonly [Schema.TaggedStruct<"WorkloadDescriptionRequired", {}>, Schema.TaggedStruct<"CandidateModelsRequired", {
     readonly workloadDescription: Schema.String;
+}>, Schema.TaggedStruct<"CandidateReasoningEffortsRequired", {
+    readonly workloadDescription: Schema.String;
+    readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
+    readonly currentCandidateModel: Schema.String;
+    readonly supportedReasoningEfforts: Schema.$Record<Schema.String, Schema.$Array<Schema.NonEmptyString>>;
 }>, Schema.TaggedStruct<"ClassifierModelRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
 }>, Schema.TaggedStruct<"AuthorModelRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
 }>, Schema.TaggedStruct<"JudgeModelRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
 }>, Schema.TaggedStruct<"RoutingObjectiveRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
     readonly judgeModel: Schema.String;
 }>, Schema.TaggedStruct<"MaximumUnknownWeightRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
     readonly judgeModel: Schema.String;
@@ -58,6 +69,7 @@ export declare const EvalProjectSetupProgress: Schema.Union<readonly [Schema.Tag
 }>, Schema.TaggedStruct<"RoutingConstraintsRequired", {
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
     readonly judgeModel: Schema.String;
@@ -88,6 +100,7 @@ export type EvalProjectSetupProgress = typeof EvalProjectSetupProgress.Type;
 export declare const EvalProjectConfiguration: Schema.Struct<{
     readonly workloadDescription: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
     readonly judgeModel: Schema.String;
@@ -154,27 +167,38 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly stage: Schema.Literal<"setup-required">;
     readonly progress: Schema.Union<readonly [Schema.TaggedStruct<"WorkloadDescriptionRequired", {}>, Schema.TaggedStruct<"CandidateModelsRequired", {
         readonly workloadDescription: Schema.String;
+    }>, Schema.TaggedStruct<"CandidateReasoningEffortsRequired", {
+        readonly workloadDescription: Schema.String;
+        readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
+        readonly currentCandidateModel: Schema.String;
+        readonly supportedReasoningEfforts: Schema.$Record<Schema.String, Schema.$Array<Schema.NonEmptyString>>;
     }>, Schema.TaggedStruct<"ClassifierModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
     }>, Schema.TaggedStruct<"AuthorModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
     }>, Schema.TaggedStruct<"JudgeModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
     }>, Schema.TaggedStruct<"RoutingObjectiveRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
     }>, Schema.TaggedStruct<"MaximumUnknownWeightRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -202,6 +226,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     }>, Schema.TaggedStruct<"RoutingConstraintsRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -239,6 +264,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -280,6 +306,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -322,6 +349,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -365,6 +393,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -410,6 +439,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -455,6 +485,7 @@ export declare const EvalProjectState: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -1501,6 +1532,7 @@ export declare const EvalProjectStateV2: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -1699,6 +1731,7 @@ export declare const EvalProjectStateV2: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -1950,6 +1983,7 @@ export declare const EvalProjectStateV2: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2243,6 +2277,7 @@ export declare const EvalProjectStateV2: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2548,6 +2583,7 @@ export declare const EvalProjectStateV2: Schema.Union<readonly [Schema.Struct<{
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2584,27 +2620,38 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly stage: Schema.Literal<"setup-required">;
     readonly progress: Schema.Union<readonly [Schema.TaggedStruct<"WorkloadDescriptionRequired", {}>, Schema.TaggedStruct<"CandidateModelsRequired", {
         readonly workloadDescription: Schema.String;
+    }>, Schema.TaggedStruct<"CandidateReasoningEffortsRequired", {
+        readonly workloadDescription: Schema.String;
+        readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
+        readonly currentCandidateModel: Schema.String;
+        readonly supportedReasoningEfforts: Schema.$Record<Schema.String, Schema.$Array<Schema.NonEmptyString>>;
     }>, Schema.TaggedStruct<"ClassifierModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.$Record<Schema.String, Schema.NonEmptyString>;
     }>, Schema.TaggedStruct<"AuthorModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
     }>, Schema.TaggedStruct<"JudgeModelRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
     }>, Schema.TaggedStruct<"RoutingObjectiveRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
     }>, Schema.TaggedStruct<"MaximumUnknownWeightRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2632,6 +2679,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     }>, Schema.TaggedStruct<"RoutingConstraintsRequired", {
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2669,6 +2717,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2710,6 +2759,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2752,6 +2802,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2795,6 +2846,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2840,6 +2892,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -2885,6 +2938,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -3013,6 +3067,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -3211,6 +3266,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -3462,6 +3518,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -3755,6 +3812,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -4060,6 +4118,7 @@ export declare const EvalProjectDocument: Schema.Union<readonly [Schema.Union<re
     readonly configuration: Schema.Struct<{
         readonly workloadDescription: Schema.String;
         readonly candidateModels: Schema.$Array<Schema.String>;
+        readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
         readonly classifierModel: Schema.String;
         readonly authorModel: Schema.String;
         readonly judgeModel: Schema.String;
@@ -4190,6 +4249,7 @@ export declare const EvalEvaluationProposal: Schema.Struct<{
     readonly evaluationDigest: Schema.String;
     readonly basisDigest: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly judgeModel: Schema.String;
     readonly suites: Schema.$Array<Schema.Struct<{
         readonly version: Schema.Literal<1>;
@@ -4262,6 +4322,7 @@ export declare const EvalExecutionPlan: Schema.Struct<{
     readonly basisDigest: Schema.String;
     readonly evaluationDigest: Schema.String;
     readonly candidateModels: Schema.$Array<Schema.String>;
+    readonly candidateReasoningEfforts: Schema.optionalKey<Schema.$Record<Schema.String, Schema.NonEmptyString>>;
     readonly classifierModel: Schema.String;
     readonly authorModel: Schema.String;
     readonly judgeModel: Schema.String;

package/dist/project-contracts.js

@@ -1,4 +1,4 @@
-import { DecompositionResult, DimensionScoreLabelV3, DimensionScorePredictionV3, EvalComparisonResult, PublishedRoutingActivation, RoutingBasisApprovalV3, RoutingBasisV3, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
+import { DecompositionResult, DimensionScoreLabelV3, DimensionScorePredictionV3, EvalCandidateReasoningEfforts, EvalComparisonResult, EvalReasoningEffort, PublishedRoutingActivation, RoutingBasisApprovalV3, RoutingBasisV3, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
 import { Schema } from "effect";
 export const EVAL_PROJECT_VERSION = 1;
 export const EVAL_PROJECT_V2_VERSION = 2;
@@ -6,6 +6,7 @@ export const EvalProjectQuestion = Schema.Struct({
     id: Schema.Literals([
         "workload-description",
         "candidate-models",
+        "candidate-reasoning-efforts",
         "classifier-model",
         "author-model",
         "judge-model",
@@ -20,24 +21,35 @@ const WorkloadDescriptionRequired = Schema.TaggedStruct("WorkloadDescriptionRequ
 const CandidateModelsRequired = Schema.TaggedStruct("CandidateModelsRequired", {
     workloadDescription: Schema.String
 });
+const CandidateReasoningEffortsRequired = Schema.TaggedStruct("CandidateReasoningEffortsRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: EvalCandidateReasoningEfforts,
+    currentCandidateModel: Schema.String,
+    supportedReasoningEfforts: Schema.Record(Schema.String, Schema.Array(EvalReasoningEffort))
+});
 const ClassifierModelRequired = Schema.TaggedStruct("ClassifierModelRequired", {
     workloadDescription: Schema.String,
-    candidateModels: Schema.Array(Schema.String)
+    candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: EvalCandidateReasoningEfforts
 });
 const AuthorModelRequired = Schema.TaggedStruct("AuthorModelRequired", {
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String
 });
 const JudgeModelRequired = Schema.TaggedStruct("JudgeModelRequired", {
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String
 });
 const RoutingObjectiveRequired = Schema.TaggedStruct("RoutingObjectiveRequired", {
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String,
     judgeModel: Schema.String
@@ -45,6 +57,7 @@ const RoutingObjectiveRequired = Schema.TaggedStruct("RoutingObjectiveRequired",
 const MaximumUnknownWeightRequired = Schema.TaggedStruct("MaximumUnknownWeightRequired", {
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String,
     judgeModel: Schema.String,
@@ -53,6 +66,7 @@ const MaximumUnknownWeightRequired = Schema.TaggedStruct("MaximumUnknownWeightRe
 const RoutingConstraintsRequired = Schema.TaggedStruct("RoutingConstraintsRequired", {
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String,
     judgeModel: Schema.String,
@@ -62,6 +76,7 @@ const RoutingConstraintsRequired = Schema.TaggedStruct("RoutingConstraintsRequir
 export const EvalProjectSetupProgress = Schema.Union([
     WorkloadDescriptionRequired,
     CandidateModelsRequired,
+    CandidateReasoningEffortsRequired,
     ClassifierModelRequired,
     AuthorModelRequired,
     JudgeModelRequired,
@@ -72,6 +87,7 @@ export const EvalProjectSetupProgress = Schema.Union([
 export const EvalProjectConfiguration = Schema.Struct({
     workloadDescription: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String,
     judgeModel: Schema.String,
@@ -550,6 +566,7 @@ export const EvalEvaluationProposal = Schema.Struct({
     evaluationDigest: Schema.String,
     basisDigest: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     judgeModel: Schema.String,
     suites: Schema.Array(EvalDimensionSuite),
     decompositionBenchmark: EvalDecompositionBenchmark,
@@ -572,6 +589,7 @@ export const EvalExecutionPlan = Schema.Struct({
     basisDigest: Schema.String,
     evaluationDigest: Schema.String,
     candidateModels: Schema.Array(Schema.String),
+    candidateReasoningEfforts: Schema.optionalKey(EvalCandidateReasoningEfforts),
     classifierModel: Schema.String,
     authorModel: Schema.String,
     judgeModel: Schema.String,

package/dist/project-workflow.d.ts

@@ -1,18 +1,19 @@
 import { Context, Effect, Layer, Path } from "effect";
 import type { EvalProjectArtifactError, EvalSetupInspectionError } from "./errors.js";
-import { EvalDimensionContrastError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
+import { EvalDimensionContrastError, type EvalModelCatalogError, type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
 import { EvalProjectArtifacts } from "./project-artifacts.js";
 import type { EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectStatus, EvalRunReport } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
-export type EvalProjectWorkflowError = EvalProjectStoreError | EvalProjectArtifactError | EvalSetupInspectionError | EvalDimensionContrastError | EvalProjectTransitionError;
+import { EvalModelCatalog } from "./services/model-catalog/service.js";
+export type EvalProjectWorkflowError = EvalProjectStoreError | EvalProjectArtifactError | EvalSetupInspectionError | EvalModelCatalogError | EvalDimensionContrastError | EvalProjectTransitionError;
 export type EvalProjectWorkflowShape = {
     readonly setup: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly status: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus | undefined, EvalProjectStoreError | EvalProjectArtifactError>;
-    readonly answer: (repositoryRoot: string, answer: string) => Effect.Effect<EvalProjectStatus, EvalProjectStoreError | EvalProjectArtifactError | EvalProjectTransitionError>;
+    readonly answer: (repositoryRoot: string, answer: string) => Effect.Effect<EvalProjectStatus, EvalProjectStoreError | EvalProjectArtifactError | EvalProjectTransitionError | EvalModelCatalogError>;
     readonly proposeDimensions: (repositoryRoot: string, dimensions: readonly EvalProposedDimension[]) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveDimensions: (repositoryRoot: string, basisDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
-    readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "candidateReasoningEfforts" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly approveEvaluations: (repositoryRoot: string, evaluationDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
     readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
     readonly startRun: (repositoryRoot: string, planId: string) => Effect.Effect<{
@@ -27,6 +28,6 @@ export type EvalProjectWorkflowShape = {
 declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
 export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
 }
-export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
-export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector | EvalModelCatalog>;
 export {};

package/dist/project-workflow.js

@@ -3,9 +3,11 @@ import { RoutingActivationConstraints, RoutingObjectivePolicy } from "@velum-lab
 import { Clock, Context, Effect, Layer, Path, Schema } from "effect";
 import { EvalDimensionContrastError, EvalProjectTransitionError } from "./errors.js";
 import { EvalRepositoryInspector } from "./inspection.js";
+import { candidateReasoningEffortFromAnswer } from "./model-selection.js";
 import { EvalProjectArtifacts, evaluationProposalDigest, routingBasisDigest } from "./project-artifacts.js";
 import { EVAL_PROJECT_VERSION, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 import { EvalProjectStore } from "./project-store.js";
+import { EvalModelCatalog } from "./services/model-catalog/service.js";
 const isoNow = Effect.map(Clock.currentTimeMillis, (millis) => new Date(millis).toISOString());
 const questionForProgress = (progress) => {
     switch (progress._tag) {
@@ -21,6 +23,12 @@ const questionForProgress = (progress) => {
                 prompt: "Which explicit provider/model IDs may RouteKit route to?",
                 options: []
             };
+        case "CandidateReasoningEffortsRequired":
+            return {
+                id: "candidate-reasoning-efforts",
+                prompt: `Choose an advertised reasoning effort for ${progress.currentCandidateModel}.`,
+                options: [...progress.supportedReasoningEfforts[progress.currentCandidateModel]]
+            };
         case "ClassifierModelRequired":
             return {
                 id: "classifier-model",
@@ -124,7 +132,24 @@ const parseModel = (state, answer, role) => Effect.gen(function* () {
     const model = yield* nonEmptyAnswer(state, answer);
     return yield* validateModel(state, model, role);
 });
+const supportedCandidateReasoningEfforts = (candidates, input) => {
+    const supported = {};
+    for (const model of candidates) {
+        const efforts = [...new Set(input[model] ?? [])];
+        if (efforts.length > 0)
+            supported[model] = efforts;
+    }
+    return supported;
+};
+const firstReasoningCandidate = (candidates, supported) => candidates.find((model) => supported[model] !== undefined);
+const nextReasoningCandidate = (candidates, supported, selected) => candidates.find((model) => supported[model] !== undefined && selected[model] === undefined);
 const sameStrings = (left, right) => left.length === right.length && left.every((value, index) => value === right[index]);
+const sameReasoningEfforts = (left, right) => {
+    const leftEntries = Object.entries(left ?? {});
+    const rightEntries = Object.entries(right ?? {});
+    return (leftEntries.length === rightEntries.length &&
+        leftEntries.every(([model, effort]) => right?.[model] === effort));
+};
 const sameLedger = (left, right) => left.expectedCalls === right.expectedCalls &&
     left.observedCalls === right.observedCalls &&
     left.observedCandidateRows === right.observedCandidateRows &&
@@ -300,6 +325,7 @@ const parseObjective = (state, answer) => Effect.gen(function* () {
 const configurationFrom = (progress, constraints) => ({
     workloadDescription: progress.workloadDescription,
     candidateModels: progress.candidateModels,
+    candidateReasoningEfforts: progress.candidateReasoningEfforts ?? {},
     classifierModel: progress.classifierModel,
     authorModel: progress.authorModel,
     judgeModel: progress.judgeModel,
@@ -324,7 +350,7 @@ const parseConstraints = (state, answer) => Effect.gen(function* () {
     const constraints = yield* Schema.decodeUnknownEffect(RoutingActivationConstraints)(json).pipe(Effect.mapError(() => transitionError(state.stage, "routing constraints contain invalid dimension quality or failure-rate values")));
     return Object.keys(constraints).length === 0 ? undefined : constraints;
 });
-const advanceSetup = (state, answer, now) => Effect.gen(function* () {
+const advanceSetup = (state, answer, now, discoverReasoningEfforts) => Effect.gen(function* () {
     const common = {
         version: state.version,
         projectId: state.projectId,
@@ -343,16 +369,58 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     workloadDescription: yield* nonEmptyAnswer(state, answer)
                 }
             };
-        case "CandidateModelsRequired":
+        case "CandidateModelsRequired": {
+            const candidateModels = yield* parseCandidateModels(state, answer);
+            const supportedReasoningEfforts = supportedCandidateReasoningEfforts(candidateModels, yield* discoverReasoningEfforts(candidateModels));
+            const currentCandidateModel = firstReasoningCandidate(candidateModels, supportedReasoningEfforts);
             return {
                 ...common,
                 stage: "setup-required",
-                progress: {
-                    _tag: "ClassifierModelRequired",
-                    workloadDescription: state.progress.workloadDescription,
-                    candidateModels: yield* parseCandidateModels(state, answer)
-                }
+                progress: currentCandidateModel === undefined
+                    ? {
+                        _tag: "ClassifierModelRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels,
+                        candidateReasoningEfforts: {}
+                    }
+                    : {
+                        _tag: "CandidateReasoningEffortsRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels,
+                        candidateReasoningEfforts: {},
+                        currentCandidateModel,
+                        supportedReasoningEfforts
+                    }
             };
+        }
+        case "CandidateReasoningEffortsRequired": {
+            const supportedReasoningEfforts = state.progress.supportedReasoningEfforts;
+            const currentCandidateModel = state.progress.currentCandidateModel;
+            const candidateReasoningEfforts = {
+                ...state.progress.candidateReasoningEfforts,
+                [currentCandidateModel]: yield* Effect.try({
+                    try: () => candidateReasoningEffortFromAnswer(answer, supportedReasoningEfforts[currentCandidateModel]),
+                    catch: (cause) => transitionError(state.stage, cause instanceof Error ? cause.message : String(cause))
+                })
+            };
+            const nextCandidateModel = nextReasoningCandidate(state.progress.candidateModels, supportedReasoningEfforts, candidateReasoningEfforts);
+            return {
+                ...common,
+                stage: "setup-required",
+                progress: nextCandidateModel === undefined
+                    ? {
+                        _tag: "ClassifierModelRequired",
+                        workloadDescription: state.progress.workloadDescription,
+                        candidateModels: state.progress.candidateModels,
+                        candidateReasoningEfforts
+                    }
+                    : {
+                        ...state.progress,
+                        candidateReasoningEfforts,
+                        currentCandidateModel: nextCandidateModel
+                    }
+            };
+        }
         case "ClassifierModelRequired":
             return {
                 ...common,
@@ -361,6 +429,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "AuthorModelRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: yield* parseModel(state, answer, "classifier")
                 }
             };
@@ -372,6 +441,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "JudgeModelRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: yield* parseModel(state, answer, "author")
                 }
@@ -384,6 +454,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "RoutingObjectiveRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: yield* parseModel(state, answer, "judge")
@@ -398,6 +469,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "MaximumUnknownWeightRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: state.progress.judgeModel,
@@ -413,6 +485,7 @@ const advanceSetup = (state, answer, now) => Effect.gen(function* () {
                     _tag: "RoutingConstraintsRequired",
                     workloadDescription: state.progress.workloadDescription,
                     candidateModels: state.progress.candidateModels,
+                    candidateReasoningEfforts: state.progress.candidateReasoningEfforts,
                     classifierModel: state.progress.classifierModel,
                     authorModel: state.progress.authorModel,
                     judgeModel: state.progress.judgeModel,
@@ -438,6 +511,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
     const store = yield* EvalProjectStore;
     const artifacts = yield* EvalProjectArtifacts;
     const inspector = yield* EvalRepositoryInspector;
+    const modelCatalog = yield* EvalModelCatalog;
     const paths = yield* Path.Path;
     const resolveRoot = (repositoryRoot) => paths.resolve(repositoryRoot);
     const statusOf = (root, state) => Effect.gen(function* () {
@@ -514,7 +588,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
         if (state.stage !== "setup-required") {
             return yield* transitionError(state.stage, "project setup has no unanswered question");
         }
-        const next = yield* advanceSetup(state, answerText, yield* isoNow);
+        const next = yield* advanceSetup(state, answerText, yield* isoNow, modelCatalog.reasoningEfforts);
         yield* store.save(root, next);
         return yield* statusOf(root, next);
     });
@@ -663,6 +737,11 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             version: EVAL_PROJECT_VERSION,
             basisDigest: state.basisDigest,
             candidateModels: state.configuration.candidateModels,
+            ...(state.configuration.candidateReasoningEfforts === undefined
+                ? {}
+                : {
+                    candidateReasoningEfforts: state.configuration.candidateReasoningEfforts
+                }),
             judgeModel: state.configuration.judgeModel,
             suites: input.suites,
             decompositionBenchmark: input.decompositionBenchmark,
@@ -764,6 +843,11 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             basisDigest: state.basisDigest,
             evaluationDigest: state.evaluationDigest,
             candidateModels: state.configuration.candidateModels,
+            ...(state.configuration.candidateReasoningEfforts === undefined
+                ? {}
+                : {
+                    candidateReasoningEfforts: state.configuration.candidateReasoningEfforts
+                }),
             classifierModel: state.configuration.classifierModel,
             authorModel: state.configuration.authorModel,
             judgeModel: state.configuration.judgeModel,
@@ -792,6 +876,7 @@ export const makeEvalProjectWorkflow = Effect.gen(function* () {
             plan.basisDigest !== state.basisDigest ||
             plan.evaluationDigest !== state.evaluationDigest ||
             !sameStrings(plan.candidateModels, state.configuration.candidateModels) ||
+            !sameReasoningEfforts(plan.candidateReasoningEfforts, state.configuration.candidateReasoningEfforts) ||
             plan.classifierModel !== state.configuration.classifierModel ||
             plan.authorModel !== state.configuration.authorModel ||
             plan.judgeModel !== state.configuration.judgeModel) {

package/dist/questions.d.ts

@@ -1,6 +1,6 @@
 import type { EvalSetupStage, EvalSetupState } from "@velum-labs/routekit-eval-contracts";
 import type { RepositoryInspection, SetupQuestion } from "./types.js";
-export declare const questionForStage: (stage: EvalSetupStage, inspection?: RepositoryInspection) => SetupQuestion | undefined;
+export declare const questionForStage: (stage: EvalSetupStage, inspection?: RepositoryInspection, reasoningEfforts?: readonly string[]) => SetupQuestion | undefined;
 export declare const withOpenQuestion: (state: EvalSetupState, inspection?: RepositoryInspection) => {
     readonly state: EvalSetupState;
     readonly question?: SetupQuestion;

package/dist/questions.js

@@ -2,7 +2,7 @@ const firstThree = (values, fallback) => {
     const unique = [...new Set(values.filter((value) => value.trim().length > 0))].slice(0, 3);
     return [unique[0] ?? fallback[0], unique[1] ?? fallback[1], unique[2] ?? fallback[2]];
 };
-export const questionForStage = (stage, inspection) => {
+export const questionForStage = (stage, inspection, reasoningEfforts = []) => {
     switch (stage) {
         case "surface":
             return {
@@ -40,6 +40,12 @@ export const questionForStage = (stage, inspection) => {
                     "Help me find three explicit model IDs"
                 ]
             };
+        case "reasoning-effort":
+            return {
+                id: stage,
+                prompt: "Choose an explicit reasoning effort advertised by the live model catalog.",
+                options: [...new Set(reasoningEfforts)]
+            };
         case "spend-approval":
             return {
                 id: stage,

package/dist/services/model-catalog/service.d.ts

@@ -0,0 +1,10 @@
+import type { EvalReasoningEffort } from "@velum-labs/routekit-eval-contracts";
+import { Context, type Effect } from "effect";
+import type { EvalModelCatalogError } from "../../errors.js";
+export type EvalModelCatalogShape = {
+    readonly reasoningEfforts: (candidateModels: readonly string[]) => Effect.Effect<Readonly<Record<string, readonly EvalReasoningEffort[]>>, EvalModelCatalogError>;
+};
+declare const EvalModelCatalog_base: Context.ServiceClass<EvalModelCatalog, "@velum-labs/routekit-eval-setup/EvalModelCatalog", EvalModelCatalogShape>;
+export declare class EvalModelCatalog extends EvalModelCatalog_base {
+}
+export {};

package/dist/services/model-catalog/service.js

@@ -0,0 +1,3 @@
+import { Context } from "effect";
+export class EvalModelCatalog extends Context.Service()("@velum-labs/routekit-eval-setup/EvalModelCatalog") {
+}

package/dist/test/model-selection.test.js

@@ -1,12 +1,17 @@
 import assert from "node:assert/strict";
 import { test } from "node:test";
-import { modelSelectionFromAnswer } from "../model-selection.js";
+import { candidateReasoningEffortFromAnswer, modelSelectionFromAnswer } from "../model-selection.js";
 test("model selection accepts exactly two candidates followed by a distinct judge", () => {
     assert.deepEqual(modelSelectionFromAnswer("openai/gpt-a anthropic/claude-b google/gemini-judge"), {
         candidates: ["openai/gpt-a", "anthropic/claude-b"],
         judgeModel: "google/gemini-judge"
     });
 });
+test("reasoning effort selection accepts exactly one advertised opaque value", () => {
+    const supported = ["none", "low", "provider-next"];
+    assert.equal(candidateReasoningEffortFromAnswer(" provider-next ", supported), "provider-next");
+    assert.throws(() => candidateReasoningEffortFromAnswer("max", supported), /must be one of/iu);
+});
 test("model selection rejects canned answers, duplicates, aliases, and extra ids", () => {
     assert.throws(() => modelSelectionFromAnswer("Current model, a cheaper candidate, and a stronger candidate"), /exactly three explicit provider\/model ids/iu);
     assert.throws(() => modelSelectionFromAnswer("openai/a openai/a openai/judge"), /three unique model ids/iu);

package/dist/test/project-workflow.test.js

@@ -5,16 +5,30 @@ import path from "node:path";
 import { after, test } from "node:test";
 import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
 import { Cause, Effect, Layer } from "effect";
-import { EvalDimensionContrastError } from "../errors.js";
+import { EvalDimensionContrastError, EvalModelCatalogError } from "../errors.js";
 import { EvalRepositoryInspectorLive } from "../inspection.js";
 import { EvalProjectArtifactsLive } from "../project-artifacts.js";
 import { scopedExpectedCalls, summarizeEvalRunLedger } from "../project-contracts.js";
 import { EvalProjectStoreLive } from "../project-store.js";
 import { EvalProjectWorkflow, EvalProjectWorkflowLive } from "../project-workflow.js";
+import { EvalModelCatalog } from "../services/model-catalog/service.js";
 const roots = [];
 after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
 const ProjectDependenciesLive = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive, EvalRepositoryInspectorLive).pipe(Layer.provide(NodeServicesLayer));
-const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(NodeServicesLayer));
+const catalogRequests = [];
+const catalogEfforts = (candidateModels) => Effect.sync(() => {
+    catalogRequests.push([...candidateModels]);
+    return {
+        ...(candidateModels.includes("openai/gpt-5.6-luna")
+            ? { "openai/gpt-5.6-luna": ["low", "medium", "ultra"] }
+            : {}),
+        ...(candidateModels.includes("openai/gpt-5.6-terra")
+            ? { "openai/gpt-5.6-terra": ["none", "low", "high"] }
+            : {})
+    };
+});
+const EvalModelCatalogTest = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({ reasoningEfforts: catalogEfforts }));
+const ProjectWorkflowTestLive = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(EvalModelCatalogTest), Layer.provide(NodeServicesLayer));
 const makeRepository = async () => {
     const root = await mkdtemp(path.join(os.tmpdir(), "routekit-eval-project-"));
     roots.push(root);
@@ -47,22 +61,47 @@ test("setup creates one durable repository project without invoking model-backed
 });
 test("answers advance exactly one question and persist a complete compositional configuration", async () => {
     const root = await makeRepository();
+    catalogRequests.length = 0;
     const statuses = await Effect.runPromise(Effect.gen(function* () {
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
         const workload = yield* workflow.answer(root, "Customer support and documentation requests");
         const candidates = yield* workflow.answer(root, "openai/gpt-5.6-luna, openai/gpt-5.6-terra openai/gpt-5.6-sol");
+        const lunaEffort = yield* workflow.answer(root, candidates.question.options[1]);
+        const terraEffort = yield* workflow.answer(root, lunaEffort.question.options[1]);
         const classifier = yield* workflow.answer(root, "openai/gpt-5.6-luna");
         const author = yield* workflow.answer(root, "openai/gpt-5.6-terra");
         const judge = yield* workflow.answer(root, "openai/gpt-5.6-terra");
         const objective = yield* workflow.answer(root, '{"kind":"highest-quality"}');
         const unknown = yield* workflow.answer(root, "0.2");
         const completed = yield* workflow.answer(root, "{}");
-        return { workload, candidates, classifier, author, judge, objective, unknown, completed };
+        return {
+            workload,
+            candidates,
+            lunaEffort,
+            terraEffort,
+            classifier,
+            author,
+            judge,
+            objective,
+            unknown,
+            completed
+        };
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
     assert.equal(statuses.workload.question?.id, "candidate-models");
     assert.equal(statuses.workload.state.revision, 1);
-    assert.equal(statuses.candidates.question?.id, "classifier-model");
+    assert.deepEqual(catalogRequests, [[
+            "openai/gpt-5.6-luna",
+            "openai/gpt-5.6-terra",
+            "openai/gpt-5.6-sol"
+        ]]);
+    assert.equal(statuses.candidates.question?.id, "candidate-reasoning-efforts");
+    assert.match(statuses.candidates.question?.prompt ?? "", /openai\/gpt-5\.6-luna/u);
+    assert.deepEqual(statuses.candidates.question?.options, ["low", "medium", "ultra"]);
+    assert.equal(statuses.lunaEffort.question?.id, "candidate-reasoning-efforts");
+    assert.match(statuses.lunaEffort.question?.prompt ?? "", /openai\/gpt-5\.6-terra/u);
+    assert.deepEqual(statuses.lunaEffort.question?.options, ["none", "low", "high"]);
+    assert.equal(statuses.terraEffort.question?.id, "classifier-model");
     assert.equal(statuses.classifier.question?.id, "author-model");
     assert.equal(statuses.author.question?.id, "judge-model");
     assert.equal(statuses.judge.question?.id, "routing-objective");
@@ -79,6 +118,10 @@ test("answers advance exactly one question and persist a complete compositional
         "openai/gpt-5.6-terra",
         "openai/gpt-5.6-sol"
     ]);
+    assert.deepEqual(statuses.completed.state.configuration.candidateReasoningEfforts, {
+        "openai/gpt-5.6-luna": "medium",
+        "openai/gpt-5.6-terra": "low"
+    });
     assert.equal(statuses.completed.state.configuration.classifierModel, "openai/gpt-5.6-luna");
     assert.equal(statuses.completed.state.configuration.authorModel, "openai/gpt-5.6-terra");
     assert.equal(statuses.completed.state.configuration.judgeModel, "openai/gpt-5.6-terra");
@@ -91,7 +134,7 @@ test("answers advance exactly one question and persist a complete compositional
         const workflow = yield* EvalProjectWorkflow;
         return yield* workflow.status(root);
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
-    assert.equal(resumed?.state.revision, 8);
+    assert.equal(resumed?.state.revision, 10);
     assert.equal(resumed?.state.stage, "dimensions-review");
 });
 test("invalid answers fail without changing the durable project revision", async () => {
@@ -112,6 +155,30 @@ test("invalid answers fail without changing the durable project revision", async
         assert.equal(afterDuplicate?.question?.id, "candidate-models");
     }).pipe(Effect.provide(ProjectWorkflowTestLive)));
 });
+test("candidate selection fails closed when catalog reasoning metadata is unavailable", async () => {
+    const root = await makeRepository();
+    const unavailableCatalog = Layer.succeed(EvalModelCatalog, EvalModelCatalog.of({
+        reasoningEfforts: () => Effect.fail(new EvalModelCatalogError({
+            detail: "models.list did not advertise candidate reasoning capabilities"
+        }))
+    }));
+    const workflowLayer = EvalProjectWorkflowLive.pipe(Layer.provide(ProjectDependenciesLive), Layer.provide(unavailableCatalog), Layer.provide(NodeServicesLayer));
+    await Effect.runPromise(Effect.gen(function* () {
+        const workflow = yield* EvalProjectWorkflow;
+        yield* workflow.setup(root);
+        yield* workflow.answer(root, "Production workload");
+        const answer = yield* Effect.exit(workflow.answer(root, "openai/reasoning openai/plain"));
+        assert.equal(answer._tag, "Failure");
+        const current = yield* workflow.status(root);
+        assert.equal(current?.state.revision, 1);
+        assert.equal(current?.state.stage, "setup-required");
+        if (current?.state.stage !== "setup-required") {
+            assert.fail("expected setup-required state");
+        }
+        assert.equal(current.state.progress._tag, "CandidateModelsRequired");
+        assert.equal(current.question?.id, "candidate-models");
+    }).pipe(Effect.provide(workflowLayer)));
+});
 test("corrupt project state preserves the typed project-store failure", async () => {
     const root = await makeRepository();
     await mkdir(path.join(root, ".routekit", "evals"), { recursive: true });
@@ -195,7 +262,11 @@ async function completeProjectSetup(root, candidateModels = "openai/gpt-5.6-luna
         const workflow = yield* EvalProjectWorkflow;
         yield* workflow.setup(root);
         yield* workflow.answer(root, "Production workload");
-        yield* workflow.answer(root, candidateModels);
+        let status = yield* workflow.answer(root, candidateModels);
+        while (status.state.stage === "setup-required" &&
+            status.state.progress._tag === "CandidateReasoningEffortsRequired") {
+            status = yield* workflow.answer(root, status.question.options[0]);
+        }
         yield* workflow.answer(root, "openai/gpt-5.6-luna");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
         yield* workflow.answer(root, "openai/gpt-5.6-terra");
@@ -732,6 +803,10 @@ run.toComplete();
     assert.equal(compositionManifestAfter, compositionManifestBefore);
     const dimensionManifest = JSON.parse(dimensionManifestAfter);
     const compositionManifest = JSON.parse(compositionManifestAfter);
+    assert.deepEqual(dimensionManifest.candidateReasoningEfforts, plan.candidateReasoningEfforts ?? {});
+    assert.deepEqual(compositionManifest.candidateReasoningEfforts, plan.candidateReasoningEfforts ?? {});
+    assert.match(dimensionSuite, /parameters: \{ reasoning: \{ effort \} \}/u);
+    assert.match(compositionSuite, /parameters: \{ reasoning: \{ effort \} \}/u);
     assert.equal(dimensionManifest.expectedCallCount, selection.caseIds.length * plan.candidateModels.length * 2);
     assert.equal(compositionManifest.expectedCallCount, plan.selectedCompositionCaseIds.length * plan.candidateModels.length * 2);
 });

package/dist/test/questions.test.js

@@ -8,13 +8,14 @@ test("setup exposes exactly one question for each unresolved stage", () => {
         "criteria",
         "constraints",
         "candidates",
+        "reasoning-effort",
         "spend-approval",
         "publish"
     ];
     for (const stage of stages) {
         const question = questionForStage(stage);
         assert.equal(question?.id, stage);
-        assert.equal(question?.options.length, 3);
+        assert.equal(question?.options.length === 3 || stage === "reasoning-effort", true);
     }
     assert.equal(questionForStage("completed"), undefined);
 });
@@ -46,3 +47,8 @@ test("candidate question requires two candidates and a distinct judge", () => {
     assert.match(question?.prompt ?? "", /exactly three unique provider\/model IDs/iu);
     assert.doesNotMatch(question?.options.join("\n") ?? "", /save without choosing|current model,/iu);
 });
+test("reasoning-effort stage exposes an explicit picker without a hidden default", () => {
+    const question = questionForStage("reasoning-effort", undefined, ["low", "provider-next"]);
+    assert.match(question?.prompt ?? "", /explicit reasoning effort/iu);
+    assert.deepEqual(question?.options, ["low", "provider-next"]);
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@velum-labs/routekit-eval-setup",
   "private": false,
-  "version": "1.2.0",
+  "version": "1.3.0",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/velum-labs/routekit.git",
@@ -31,9 +31,9 @@
   },
   "dependencies": {
     "effect": "4.0.0-rc.108",
-    "@velum-labs/routekit-eval-contracts": "1.2.0",
-    "@velum-labs/routekit-eval-core": "1.2.0",
-    "@velum-labs/routekit-runtime": "1.2.0"
+    "@velum-labs/routekit-eval-contracts": "1.3.0",
+    "@velum-labs/routekit-eval-core": "1.3.0",
+    "@velum-labs/routekit-runtime": "1.3.0"
   },
   "keywords": [
     "routekit",