@velum-labs/routekit-eval-setup 1.0.0

package/dist/project-contracts.js

@@ -0,0 +1,396 @@
+import { DecompositionResult, EvalComparisonResult, PublishedRoutingActivation, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy } from "@velum-labs/routekit-eval-contracts";
+import { Schema } from "effect";
+export const EVAL_PROJECT_VERSION = 1;
+export const EvalProjectQuestion = Schema.Struct({
+    id: Schema.Literals([
+        "workload-description",
+        "candidate-models",
+        "classifier-model",
+        "author-model",
+        "judge-model",
+        "routing-objective",
+        "maximum-unknown-weight",
+        "routing-constraints"
+    ]),
+    prompt: Schema.String,
+    options: Schema.Array(Schema.String)
+});
+const WorkloadDescriptionRequired = Schema.TaggedStruct("WorkloadDescriptionRequired", {});
+const CandidateModelsRequired = Schema.TaggedStruct("CandidateModelsRequired", {
+    workloadDescription: Schema.String
+});
+const ClassifierModelRequired = Schema.TaggedStruct("ClassifierModelRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String)
+});
+const AuthorModelRequired = Schema.TaggedStruct("AuthorModelRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String
+});
+const JudgeModelRequired = Schema.TaggedStruct("JudgeModelRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String
+});
+const RoutingObjectiveRequired = Schema.TaggedStruct("RoutingObjectiveRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String,
+    judgeModel: Schema.String
+});
+const MaximumUnknownWeightRequired = Schema.TaggedStruct("MaximumUnknownWeightRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String,
+    judgeModel: Schema.String,
+    objective: RoutingObjectivePolicy
+});
+const RoutingConstraintsRequired = Schema.TaggedStruct("RoutingConstraintsRequired", {
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String,
+    judgeModel: Schema.String,
+    objective: RoutingObjectivePolicy,
+    maximumUnknownWeight: Schema.Finite
+});
+export const EvalProjectSetupProgress = Schema.Union([
+    WorkloadDescriptionRequired,
+    CandidateModelsRequired,
+    ClassifierModelRequired,
+    AuthorModelRequired,
+    JudgeModelRequired,
+    RoutingObjectiveRequired,
+    MaximumUnknownWeightRequired,
+    RoutingConstraintsRequired
+]);
+export const EvalProjectConfiguration = Schema.Struct({
+    workloadDescription: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String,
+    judgeModel: Schema.String,
+    objective: RoutingObjectivePolicy,
+    maximumUnknownWeight: Schema.Finite,
+    constraints: Schema.optionalKey(RoutingActivationConstraints)
+});
+const NonNegativeInteger = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 && Number.isInteger(value) ? undefined : "value must be a non-negative integer")));
+const NonNegativeFinite = Schema.Finite.pipe(Schema.check(Schema.makeFilter((value) => value >= 0 ? undefined : "value must be a non-negative finite number")));
+const ProjectCommon = {
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    projectId: Schema.String,
+    revision: NonNegativeInteger,
+    createdAt: Schema.String,
+    updatedAt: Schema.String,
+    sourceInventory: Schema.Array(Schema.String)
+};
+const SetupRequiredState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("setup-required"),
+    progress: EvalProjectSetupProgress
+});
+const DimensionsReviewState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("dimensions-review"),
+    configuration: EvalProjectConfiguration
+});
+const EvaluationsReviewState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("evaluations-review"),
+    configuration: EvalProjectConfiguration,
+    basisDigest: Schema.String
+});
+const ReadyState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("ready"),
+    configuration: EvalProjectConfiguration,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String
+});
+const RunningState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("running"),
+    configuration: EvalProjectConfiguration,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    planId: Schema.String,
+    runId: Schema.String
+});
+const QualifiedState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("qualified"),
+    configuration: EvalProjectConfiguration,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    runId: Schema.String,
+    reportPath: Schema.String
+});
+const ActivatedState = Schema.Struct({
+    ...ProjectCommon,
+    stage: Schema.Literal("activated"),
+    configuration: EvalProjectConfiguration,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    runId: Schema.String,
+    reportPath: Schema.String,
+    evidenceDigest: Schema.String,
+    targetIdentity: Schema.String
+});
+export const EvalProjectState = Schema.Union([
+    SetupRequiredState,
+    DimensionsReviewState,
+    EvaluationsReviewState,
+    ReadyState,
+    RunningState,
+    QualifiedState,
+    ActivatedState
+]);
+export const EvalDimensionCase = Schema.Struct({
+    id: Schema.String,
+    prompt: Schema.String,
+    context: Schema.optionalKey(Schema.String),
+    rubric: Schema.String
+});
+export const EvalDimensionSuite = Schema.Struct({
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    dimensionId: Schema.String,
+    maximumOutputTokens: NonNegativeInteger,
+    cases: Schema.Array(EvalDimensionCase)
+});
+export const EvalDecompositionBenchmarkCase = Schema.Struct({
+    id: Schema.String,
+    request: Schema.String,
+    expected: DecompositionResult
+});
+export const EvalDecompositionBenchmark = Schema.Struct({
+    maximumVectorL1Error: Schema.Finite,
+    cases: Schema.Array(EvalDecompositionBenchmarkCase)
+});
+export const EvalCompositionCase = Schema.Struct({
+    id: Schema.String,
+    prompt: Schema.String,
+    context: Schema.optionalKey(Schema.String),
+    rubric: Schema.String,
+    decomposition: DecompositionResult,
+    requirements: RequestRoutingRequirements
+});
+export const EvalCompositionSuite = Schema.Struct({
+    maximumOutputTokens: NonNegativeInteger,
+    minimumWinnerScoreGap: Schema.Finite,
+    minimumWinnerAgreement: Schema.Finite,
+    cases: Schema.Array(EvalCompositionCase)
+});
+export const EvalEvaluationProposal = Schema.Struct({
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    evaluationDigest: Schema.String,
+    basisDigest: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    judgeModel: Schema.String,
+    suites: Schema.Array(EvalDimensionSuite),
+    decompositionBenchmark: EvalDecompositionBenchmark,
+    compositionSuite: EvalCompositionSuite
+});
+export const EvalArtifactApproval = Schema.Struct({
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    kind: Schema.Literals(["routing-basis", "evaluations"]),
+    digest: Schema.String,
+    approvedAt: Schema.String
+});
+export const EvalPlanScope = Schema.Literals(["pilot", "full"]);
+export const EvalExecutionPlan = Schema.Struct({
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    planId: Schema.String,
+    projectId: Schema.String,
+    projectRevision: NonNegativeInteger,
+    createdAt: Schema.String,
+    scope: EvalPlanScope,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    candidateModels: Schema.Array(Schema.String),
+    classifierModel: Schema.String,
+    authorModel: Schema.String,
+    judgeModel: Schema.String,
+    selectedCaseIds: Schema.Array(Schema.Struct({
+        dimensionId: Schema.String,
+        caseIds: Schema.Array(Schema.String)
+    })),
+    selectedDecompositionCaseIds: Schema.Array(Schema.String),
+    selectedCompositionCaseIds: Schema.Array(Schema.String),
+    maximumOutputTokens: NonNegativeInteger,
+    expectedDimensionCandidateCalls: NonNegativeInteger,
+    expectedDimensionJudgeCalls: NonNegativeInteger,
+    expectedClassifierCalls: NonNegativeInteger,
+    expectedCompositionCandidateCalls: NonNegativeInteger,
+    expectedCompositionJudgeCalls: NonNegativeInteger,
+    expectedCandidateCalls: NonNegativeInteger,
+    expectedJudgeCalls: NonNegativeInteger,
+    expectedCallCount: NonNegativeInteger
+});
+export const EvalRunTarget = Schema.Struct({
+    kind: Schema.Literals(["configured", "external"]),
+    identity: Schema.String,
+    publishAllowed: Schema.Boolean
+});
+export const EvalRunCleanup = Schema.Struct({
+    sessionOpened: Schema.Boolean,
+    sessionClosed: Schema.Boolean,
+    detail: Schema.optionalKey(Schema.String)
+});
+export const EvalRunLedger = Schema.Struct({
+    expectedCalls: NonNegativeInteger,
+    observedCalls: NonNegativeInteger,
+    observedCandidateRows: NonNegativeInteger,
+    knownInputTokens: NonNegativeInteger,
+    knownOutputTokens: NonNegativeInteger,
+    unknownTokenMeasurements: NonNegativeInteger,
+    knownPricedSubtotalUsd: NonNegativeFinite,
+    unpricedCalls: NonNegativeInteger
+});
+export const EvalClassifierObservation = Schema.Struct({
+    caseId: Schema.String,
+    weights: DecompositionResult.fields.weights,
+    unknownWeight: DecompositionResult.fields.unknownWeight,
+    vectorL1Error: NonNegativeFinite,
+    passed: Schema.Boolean,
+    classifierCallId: Schema.optionalKey(Schema.String),
+    measurement: Schema.Struct({
+        costUsd: Schema.optionalKey(NonNegativeFinite),
+        durationMs: Schema.optionalKey(NonNegativeFinite),
+        inputTokens: Schema.optionalKey(NonNegativeInteger),
+        outputTokens: Schema.optionalKey(NonNegativeInteger)
+    })
+});
+export const EvalCompositionCaseResult = Schema.Struct({
+    caseId: Schema.String,
+    predictedWinner: Schema.String,
+    observedWinner: Schema.String,
+    predictedScoreGap: NonNegativeFinite,
+    observedScoreGap: NonNegativeFinite,
+    passed: Schema.Boolean
+});
+export const EvalRunQualification = Schema.Struct({
+    decomposition: Schema.Struct({
+        expectedCases: NonNegativeInteger,
+        passedCases: NonNegativeInteger,
+        maximumObservedL1Error: NonNegativeFinite,
+        observations: Schema.Array(EvalClassifierObservation)
+    }),
+    composition: Schema.Struct({
+        expectedCases: NonNegativeInteger,
+        comparableCases: NonNegativeInteger,
+        agreeingCases: NonNegativeInteger,
+        winnerAgreement: NonNegativeFinite,
+        cases: Schema.Array(EvalCompositionCaseResult)
+    })
+});
+export function summarizeEvalRunLedger(comparisons, expectedCalls, classifierObservations = []) {
+    let observedCalls = 0;
+    let observedCandidateRows = 0;
+    let knownInputTokens = 0;
+    let knownOutputTokens = 0;
+    let unknownTokenMeasurements = 0;
+    let knownPricedSubtotalUsd = 0;
+    let unpricedCalls = 0;
+    for (const comparison of comparisons) {
+        const calls = comparison.calls ??
+            comparison.models.flatMap((model) => model.cases.map((testCase) => ({
+                role: "candidate",
+                model: model.model,
+                caseId: testCase.caseId,
+                measurement: testCase.measurement
+            })));
+        for (const call of calls) {
+            observedCalls += 1;
+            if (call.role === "candidate")
+                observedCandidateRows += 1;
+            if (call.measurement.inputTokens === undefined ||
+                call.measurement.outputTokens === undefined) {
+                unknownTokenMeasurements += 1;
+            }
+            else {
+                knownInputTokens += call.measurement.inputTokens;
+                knownOutputTokens += call.measurement.outputTokens;
+            }
+            if (call.measurement.costUsd === undefined) {
+                unpricedCalls += 1;
+            }
+            else {
+                knownPricedSubtotalUsd += call.measurement.costUsd;
+            }
+        }
+    }
+    for (const observation of classifierObservations) {
+        observedCalls += 1;
+        if (observation.measurement.inputTokens === undefined ||
+            observation.measurement.outputTokens === undefined) {
+            unknownTokenMeasurements += 1;
+        }
+        else {
+            knownInputTokens += observation.measurement.inputTokens;
+            knownOutputTokens += observation.measurement.outputTokens;
+        }
+        if (observation.measurement.costUsd === undefined) {
+            unpricedCalls += 1;
+        }
+        else {
+            knownPricedSubtotalUsd += observation.measurement.costUsd;
+        }
+    }
+    return {
+        expectedCalls,
+        observedCalls,
+        observedCandidateRows,
+        knownInputTokens,
+        knownOutputTokens,
+        unknownTokenMeasurements,
+        knownPricedSubtotalUsd,
+        unpricedCalls
+    };
+}
+const EvalRunReportCommon = {
+    version: Schema.Literal(EVAL_PROJECT_VERSION),
+    runId: Schema.String,
+    planId: Schema.String,
+    projectId: Schema.String,
+    startedAt: Schema.String,
+    finishedAt: Schema.String,
+    basisDigest: Schema.String,
+    evaluationDigest: Schema.String,
+    target: EvalRunTarget,
+    cleanup: EvalRunCleanup,
+    comparisons: Schema.Array(EvalComparisonResult),
+    qualification: Schema.optionalKey(EvalRunQualification),
+    ledger: EvalRunLedger
+};
+const QualifiedEvalRunReport = Schema.Struct({
+    ...EvalRunReportCommon,
+    status: Schema.Literal("passed"),
+    activation: PublishedRoutingActivation
+});
+const CompletedEvalRunReport = Schema.Struct({
+    ...EvalRunReportCommon,
+    status: Schema.Literal("completed"),
+    activation: PublishedRoutingActivation
+});
+const FailedEvalRunReport = Schema.Struct({
+    ...EvalRunReportCommon,
+    status: Schema.Literal("failed"),
+    failure: Schema.String
+});
+/**
+ * Sanitized durable result of one immutable execution plan.
+ *
+ * Prompts, model responses, credentials, headers, and raw child output are
+ * deliberately absent from this contract. Dollar totals are known-priced
+ * subtotals only; callers must render cost as unknown when
+ * `unpricedCalls` is non-zero.
+ */
+export const EvalRunReport = Schema.Union([
+    QualifiedEvalRunReport,
+    CompletedEvalRunReport,
+    FailedEvalRunReport
+]);

package/dist/project-store.d.ts

@@ -0,0 +1,13 @@
+import { Context, Effect, FileSystem, Layer, Path } from "effect";
+import { EvalProjectStoreError } from "./errors.js";
+import { EvalProjectState } from "./project-contracts.js";
+export type EvalProjectStoreShape = {
+    readonly load: (repositoryRoot: string) => Effect.Effect<EvalProjectState | undefined, EvalProjectStoreError>;
+    readonly save: (repositoryRoot: string, state: EvalProjectState) => Effect.Effect<void, EvalProjectStoreError>;
+};
+declare const EvalProjectStore_base: Context.ServiceClass<EvalProjectStore, "@velum-labs/routekit-eval-setup/EvalProjectStore", EvalProjectStoreShape>;
+export declare class EvalProjectStore extends EvalProjectStore_base {
+}
+export declare const makeFileEvalProjectStore: Effect.Effect<EvalProjectStoreShape, never, Path.Path | FileSystem.FileSystem>;
+export declare const EvalProjectStoreLive: Layer.Layer<EvalProjectStore, never, Path.Path | FileSystem.FileSystem>;
+export {};

package/dist/project-store.js

@@ -0,0 +1,53 @@
+import { writeFileAtomicEffect } from "@velum-labs/routekit-runtime/effect";
+import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect";
+import { EvalProjectStoreError } from "./errors.js";
+import { EvalProjectState } from "./project-contracts.js";
+const PROJECT_FILE = "project.json";
+export class EvalProjectStore extends Context.Service()("@velum-labs/routekit-eval-setup/EvalProjectStore") {
+}
+const storeFailure = (operation, path, cause) => new EvalProjectStoreError({
+    operation,
+    path,
+    detail: cause instanceof Error ? cause.message : String(cause),
+    cause
+});
+export const makeFileEvalProjectStore = Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const paths = yield* Path.Path;
+    const projectPath = (root) => paths.join(paths.resolve(root), ".routekit", "evals", PROJECT_FILE);
+    return EvalProjectStore.of({
+        load: (repositoryRoot) => Effect.gen(function* () {
+            const root = yield* fs
+                .realPath(repositoryRoot)
+                .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
+            const target = projectPath(root);
+            const exists = yield* fs
+                .exists(target)
+                .pipe(Effect.mapError((cause) => storeFailure("checking", target, cause)));
+            if (!exists)
+                return undefined;
+            const encoded = yield* fs
+                .readFileString(target)
+                .pipe(Effect.mapError((cause) => storeFailure("reading", target, cause)));
+            const json = yield* Effect.try({
+                try: () => JSON.parse(encoded),
+                catch: (cause) => storeFailure("parsing", target, cause)
+            });
+            return yield* Schema.decodeUnknownEffect(EvalProjectState)(json).pipe(Effect.mapError((cause) => storeFailure("decoding", target, cause)));
+        }),
+        save: (repositoryRoot, state) => Effect.gen(function* () {
+            const root = yield* fs
+                .realPath(repositoryRoot)
+                .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
+            const target = projectPath(root);
+            const directory = paths.dirname(target);
+            yield* fs
+                .makeDirectory(directory, { recursive: true, mode: 0o700 })
+                .pipe(Effect.mapError((cause) => storeFailure("creating", directory, cause)));
+            yield* writeFileAtomicEffect(target, `${JSON.stringify(state, null, 2)}\n`, {
+                mode: 0o600
+            }).pipe(Effect.mapError((cause) => storeFailure("committing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
+        })
+    });
+});
+export const EvalProjectStoreLive = Layer.effect(EvalProjectStore, makeFileEvalProjectStore);

package/dist/project-workflow.d.ts

@@ -0,0 +1,33 @@
+import type { RoutingBasis } from "@velum-labs/routekit-eval-contracts";
+import { Context, Effect, Layer, Path } from "effect";
+import type { EvalProjectArtifactError, EvalSetupInspectionError } from "./errors.js";
+import { type EvalProjectStoreError, EvalProjectTransitionError } from "./errors.js";
+import { EvalRepositoryInspector } from "./inspection.js";
+import { EvalProjectArtifacts } from "./project-artifacts.js";
+import type { EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProjectStatus, EvalRunReport } from "./project-contracts.js";
+import { EvalProjectStore } from "./project-store.js";
+export type EvalProjectWorkflowError = EvalProjectStoreError | EvalProjectArtifactError | EvalSetupInspectionError | EvalProjectTransitionError;
+export type EvalProjectWorkflowShape = {
+    readonly setup: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly status: (repositoryRoot: string) => Effect.Effect<EvalProjectStatus | undefined, EvalProjectStoreError | EvalProjectArtifactError>;
+    readonly answer: (repositoryRoot: string, answer: string) => Effect.Effect<EvalProjectStatus, EvalProjectStoreError | EvalProjectArtifactError | EvalProjectTransitionError>;
+    readonly proposeDimensions: (repositoryRoot: string, dimensions: RoutingBasis["dimensions"]) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly approveDimensions: (repositoryRoot: string, basisDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly proposeEvaluations: (repositoryRoot: string, proposal: Omit<EvalEvaluationProposal, "version" | "evaluationDigest" | "basisDigest" | "candidateModels" | "judgeModel">) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly approveEvaluations: (repositoryRoot: string, evaluationDigest: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly createPlan: (repositoryRoot: string, scope: EvalPlanScope) => Effect.Effect<EvalExecutionPlan, EvalProjectWorkflowError>;
+    readonly startRun: (repositoryRoot: string, planId: string) => Effect.Effect<{
+        readonly plan: EvalExecutionPlan;
+        readonly runId: string;
+    }, EvalProjectWorkflowError>;
+    readonly finishRun: (repositoryRoot: string, report: EvalRunReport) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly failRun: (repositoryRoot: string, report: EvalRunReport) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+    readonly result: (repositoryRoot: string, runId?: string) => Effect.Effect<EvalRunReport | undefined, EvalProjectWorkflowError>;
+    readonly markActivated: (repositoryRoot: string, runId: string, targetIdentity: string) => Effect.Effect<EvalProjectStatus, EvalProjectWorkflowError>;
+};
+declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
+export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
+}
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalRepositoryInspector | Path.Path | EvalProjectArtifacts | EvalProjectStore>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalRepositoryInspector | Path.Path | EvalProjectArtifacts | EvalProjectStore>;
+export {};