@velum-labs/routekit-eval-setup 1.0.26 → 1.1.0

package/dist/project-contracts.js

@@ -1,6 +1,7 @@
-import { DecompositionResult, EvalComparisonResult, PublishedRoutingActivation, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
+import { DecompositionResult, DimensionScoreLabelV3, DimensionScorePredictionV3, EvalComparisonResult, PublishedRoutingActivation, RoutingBasisApprovalV3, RoutingBasisV3, RoutingActivationConstraints, RequestRoutingRequirements, RoutingObjectivePolicy, WorkloadDimension } from "@velum-labs/routekit-eval-contracts";
 import { Schema } from "effect";
 export const EVAL_PROJECT_VERSION = 1;
+export const EVAL_PROJECT_V2_VERSION = 2;
 export const EvalProjectQuestion = Schema.Struct({
     id: Schema.Literals([
         "workload-description",
@@ -167,6 +168,348 @@ export const EvalProjectState = Schema.Union([
     QualifiedState,
     ActivatedState
 ]);
+export const FrozenRepositorySnapshotV3 = Schema.Struct({
+    repositoryRootDigest: Schema.String,
+    requestedRef: Schema.String,
+    commit: Schema.String,
+    tree: Schema.String,
+    repositoryDigest: Schema.String,
+    capturedAt: Schema.String,
+    dirty: Schema.Boolean,
+    dirtyPatchDigest: Schema.optionalKey(Schema.String),
+    sourceInventoryDigest: Schema.String
+});
+export const BasisTaskRecordV3 = Schema.Struct({
+    caseId: Schema.String,
+    occurredAt: Schema.String,
+    taskDigest: Schema.String,
+    summary: Schema.String,
+    changedPathDigest: Schema.String,
+    changedPaths: Schema.Array(Schema.String)
+});
+export const BasisTaskSplitV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    splitDigest: Schema.String,
+    construction: Schema.Array(BasisTaskRecordV3),
+    validation: Schema.Array(BasisTaskRecordV3),
+    final: Schema.Array(BasisTaskRecordV3),
+    provisional: Schema.Boolean,
+    warnings: Schema.Array(Schema.String)
+});
+export const BasisCandidateRecipeV3 = Schema.Struct({
+    id: Schema.String,
+    dimensionCount: NonNegativeInteger,
+    historySize: NonNegativeInteger,
+    sampling: Schema.Literals(["none", "diverse", "recent"]),
+    includeChangedPathStatistics: Schema.Boolean,
+    deterministic: Schema.Boolean
+});
+export const BasisGenerationPlanV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    planDigest: Schema.String,
+    repositoryDigest: Schema.String,
+    splitDigest: Schema.String,
+    authorModel: Schema.String,
+    recipes: Schema.Array(BasisCandidateRecipeV3),
+    plannedPrimaryCalls: NonNegativeInteger,
+    maximumAttempts: NonNegativeInteger,
+    requiredTotalInputTokens: NonNegativeInteger,
+    requiredTotalOutputTokens: NonNegativeInteger
+});
+export const BasisGenerationJobV3 = Schema.Struct({
+    recipeId: Schema.String,
+    status: Schema.Literals(["pending", "completed", "failed"]),
+    candidateId: Schema.optionalKey(Schema.String),
+    attemptCount: NonNegativeInteger,
+    inputTokens: NonNegativeInteger,
+    outputTokens: NonNegativeInteger,
+    callIds: Schema.Array(Schema.String),
+    error: Schema.optionalKey(Schema.String)
+});
+export const BasisGenerationLedgerV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    planDigest: Schema.String,
+    ledgerDigest: Schema.String,
+    jobs: Schema.Array(BasisGenerationJobV3)
+});
+export const BasisCandidateRecordV3 = Schema.Struct({
+    candidateId: Schema.String,
+    recipeId: Schema.String,
+    basis: RoutingBasisV3,
+    basisDigest: Schema.String,
+    authorModel: Schema.optionalKey(Schema.String),
+    generationConfigurationDigest: Schema.String,
+    evidenceDigest: Schema.String,
+    createdAt: Schema.String
+});
+export const BasisCandidateCheckpointV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    generationPlanDigest: Schema.String,
+    job: BasisGenerationJobV3,
+    candidate: BasisCandidateRecordV3
+});
+export const BasisStructureFindingV3 = Schema.Struct({
+    severity: Schema.Literals(["hard", "warning"]),
+    code: Schema.String,
+    dimensionIds: Schema.Array(Schema.String),
+    detail: Schema.String
+});
+export const BasisStructureReportV3 = Schema.Struct({
+    candidateId: Schema.String,
+    basisDigest: Schema.String,
+    eligible: Schema.Boolean,
+    dimensionCount: NonNegativeInteger,
+    definitionBytes: NonNegativeInteger,
+    findings: Schema.Array(BasisStructureFindingV3)
+});
+export const BasisLabelProtocolV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    protocolDigest: Schema.String,
+    kind: Schema.Literals(["reviewed-human", "reviewed-reference-mapper"]),
+    author: Schema.String,
+    reviewer: Schema.String,
+    blindness: Schema.Literal("candidate-output-hidden"),
+    activeDimensionThreshold: Schema.Finite,
+    unknownThreshold: Schema.Finite
+});
+export const BasisCandidateDatasetV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    datasetDigest: Schema.String,
+    splitDigest: Schema.String,
+    basisDigest: Schema.String,
+    labelProtocol: BasisLabelProtocolV3,
+    tasks: Schema.Array(BasisTaskRecordV3),
+    labels: Schema.Array(DimensionScoreLabelV3)
+});
+export const BasisValidationClassifierBindingV3 = Schema.Struct({
+    candidateId: Schema.String,
+    basisDigest: Schema.String,
+    classifierConfigurationDigest: Schema.String
+});
+export const BasisValidationPlanV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    planDigest: Schema.String,
+    repositoryDigest: Schema.String,
+    splitDigest: Schema.String,
+    classifierBindings: Schema.Array(BasisValidationClassifierBindingV3),
+    candidateIds: Schema.Array(Schema.String),
+    datasetDigests: Schema.Array(Schema.String),
+    repetitions: NonNegativeInteger,
+    plannedPrimaryCalls: NonNegativeInteger,
+    maximumAttempts: NonNegativeInteger,
+    requiredTotalInputTokens: NonNegativeInteger,
+    requiredTotalOutputTokens: NonNegativeInteger,
+    maximumPricedExposureUsd: Schema.optionalKey(NonNegativeFinite),
+    acknowledgeUnpricedExposure: Schema.Boolean
+});
+export const BasisValidationResultV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    candidateId: Schema.String,
+    basisDigest: Schema.String,
+    validationDatasetDigest: Schema.String,
+    classifierConfigurationDigest: Schema.String,
+    predictions: Schema.Array(DimensionScorePredictionV3),
+    caseCount: NonNegativeInteger,
+    skippedInvalidCount: NonNegativeInteger,
+    activeMae: NonNegativeFinite,
+    activeF1: NonNegativeFinite,
+    topDimensionAgreement: NonNegativeFinite,
+    unknownMae: NonNegativeFinite,
+    unknownBrier: NonNegativeFinite,
+    unknownAccuracy: NonNegativeFinite,
+    falseUnknownRate: NonNegativeFinite,
+    missedUnknownRate: NonNegativeFinite,
+    referenceUnknownRate: NonNegativeFinite,
+    averageActiveDimensionCount: NonNegativeFinite,
+    selectionScore: NonNegativeFinite,
+    provisional: Schema.Boolean
+});
+export const BasisValidationJobCheckpointV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    validationPlanDigest: Schema.String,
+    candidateId: Schema.String,
+    basisDigest: Schema.String,
+    validationDatasetDigest: Schema.String,
+    caseId: Schema.String,
+    repetition: NonNegativeInteger,
+    classifierConfigurationDigest: Schema.String,
+    status: Schema.Literals(["completed", "failed"]),
+    prediction: Schema.optionalKey(DimensionScorePredictionV3),
+    error: Schema.optionalKey(Schema.Struct({
+        name: Schema.String,
+        message: Schema.String,
+        code: Schema.optionalKey(Schema.String),
+        attemptCount: Schema.optionalKey(NonNegativeInteger)
+    })),
+    attemptCount: NonNegativeInteger,
+    callIds: Schema.Array(Schema.String),
+    knownInputTokens: NonNegativeInteger,
+    knownOutputTokens: NonNegativeInteger,
+    knownCostUsd: Schema.optionalKey(NonNegativeFinite),
+    unpricedCalls: NonNegativeInteger,
+    unknownUsageCalls: NonNegativeInteger
+});
+export const BasisSelectionReportV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    selectionReportDigest: Schema.String,
+    repositoryDigest: Schema.String,
+    validationPlanDigest: Schema.String,
+    objectiveVersion: Schema.Literal("basis-selection-v1"),
+    rankedCandidateIds: Schema.Array(Schema.String),
+    paretoCandidateIds: Schema.Array(Schema.String),
+    selectedCandidateId: Schema.String,
+    provisional: Schema.Boolean
+});
+export const BasisGenerationArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    sourceInventory: Schema.Array(Schema.String),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3
+});
+export const BasisCandidatesArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    generationPlanDigest: Schema.String,
+    generationLedger: BasisGenerationLedgerV3,
+    candidates: Schema.Array(BasisCandidateRecordV3),
+    structureReports: Schema.Array(BasisStructureReportV3)
+});
+export const BasisValidationArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    generationPlanDigest: Schema.String,
+    datasets: Schema.Array(BasisCandidateDatasetV3),
+    validationPlan: BasisValidationPlanV3
+});
+export const BasisValidationResultsArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    validationPlanDigest: Schema.String,
+    validationResults: Schema.Array(BasisValidationResultV3),
+    selection: BasisSelectionReportV3
+});
+export const BasisQualificationArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    validationPlanDigest: Schema.String,
+    validationResults: Schema.Array(BasisValidationResultV3),
+    selection: BasisSelectionReportV3,
+    approval: RoutingBasisApprovalV3,
+    approvalDigest: Schema.String
+});
+export const BasisEvaluationProjectionV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    basisDigest: Schema.String,
+    projectedBasisDigest: Schema.String,
+    dimensions: Schema.Array(WorkloadDimension)
+});
+export const BasisRepairSummaryV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    summaryDigest: Schema.String,
+    sourceCandidateId: Schema.String,
+    sourceBasisDigest: Schema.String,
+    errorCategories: Schema.Array(Schema.Struct({
+        code: Schema.String,
+        dimensionIds: Schema.Array(Schema.String),
+        summary: Schema.String
+    }))
+});
+export const BasisRepairPlanV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    planDigest: Schema.String,
+    repositoryDigest: Schema.String,
+    sourceCandidateId: Schema.String,
+    sourceBasisDigest: Schema.String,
+    summaryDigest: Schema.String,
+    authorModel: Schema.String,
+    plannedPrimaryCalls: Schema.Literal(1),
+    maximumAttempts: Schema.Literal(2),
+    requiredTotalInputTokens: NonNegativeInteger,
+    requiredTotalOutputTokens: NonNegativeInteger
+});
+export const BasisRepairArtifactsV3 = Schema.Struct({
+    version: Schema.Literal(1),
+    plan: BasisRepairPlanV3,
+    summary: BasisRepairSummaryV3,
+    candidate: BasisCandidateRecordV3
+});
+const ProjectV2Common = {
+    version: Schema.Literal(EVAL_PROJECT_V2_VERSION),
+    projectId: Schema.String,
+    revision: NonNegativeInteger,
+    createdAt: Schema.String,
+    updatedAt: Schema.String,
+    sourceInventory: Schema.Array(Schema.String),
+    configuration: EvalProjectConfiguration
+};
+const BasisGenerationReadyStateV2 = Schema.Struct({
+    ...ProjectV2Common,
+    stage: Schema.Literal("basis-generation-ready"),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3
+});
+const BasisCandidatesReviewStateV2 = Schema.Struct({
+    ...ProjectV2Common,
+    stage: Schema.Literal("basis-candidates-review"),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3,
+    candidates: Schema.Array(BasisCandidateRecordV3),
+    structureReports: Schema.Array(BasisStructureReportV3)
+});
+const BasisValidationReadyStateV2 = Schema.Struct({
+    ...ProjectV2Common,
+    stage: Schema.Literal("basis-validation-ready"),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3,
+    candidates: Schema.Array(BasisCandidateRecordV3),
+    structureReports: Schema.Array(BasisStructureReportV3),
+    datasets: Schema.Array(BasisCandidateDatasetV3),
+    validationPlan: BasisValidationPlanV3
+});
+const BasisQualifiedStateV2 = Schema.Struct({
+    ...ProjectV2Common,
+    stage: Schema.Literal("basis-qualified"),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3,
+    candidates: Schema.Array(BasisCandidateRecordV3),
+    structureReports: Schema.Array(BasisStructureReportV3),
+    datasets: Schema.Array(BasisCandidateDatasetV3),
+    validationPlan: BasisValidationPlanV3,
+    validationResults: Schema.Array(BasisValidationResultV3),
+    selection: BasisSelectionReportV3,
+    approval: RoutingBasisApprovalV3
+});
+const BasisValidationReviewStateV2 = Schema.Struct({
+    ...ProjectV2Common,
+    stage: Schema.Literal("basis-validation-review"),
+    snapshot: FrozenRepositorySnapshotV3,
+    split: BasisTaskSplitV3,
+    generationPlan: BasisGenerationPlanV3,
+    generationLedger: BasisGenerationLedgerV3,
+    candidates: Schema.Array(BasisCandidateRecordV3),
+    structureReports: Schema.Array(BasisStructureReportV3),
+    datasets: Schema.Array(BasisCandidateDatasetV3),
+    validationPlan: BasisValidationPlanV3,
+    validationResults: Schema.Array(BasisValidationResultV3),
+    selection: BasisSelectionReportV3
+});
+export const EvalProjectStateV2 = Schema.Union([
+    BasisGenerationReadyStateV2,
+    BasisCandidatesReviewStateV2,
+    BasisValidationReadyStateV2,
+    BasisValidationReviewStateV2,
+    BasisQualifiedStateV2
+]);
+export const EvalProjectDocument = Schema.Union([
+    EvalProjectState,
+    EvalProjectStateV2
+]);
 export const EvalDimensionCase = Schema.Struct({
     id: Schema.String,
     prompt: Schema.String,

package/dist/project-store.d.ts

@@ -1,7 +1,9 @@
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import { EvalProjectStoreError } from "./errors.js";
-import { EvalProjectState } from "./project-contracts.js";
+import { type EvalProjectDocument as EvalProjectDocumentType, EvalProjectState } from "./project-contracts.js";
 export type EvalProjectStoreShape = {
+    readonly loadDocument: (repositoryRoot: string) => Effect.Effect<EvalProjectDocumentType | undefined, EvalProjectStoreError>;
+    readonly saveDocument: (repositoryRoot: string, state: EvalProjectDocumentType) => Effect.Effect<void, EvalProjectStoreError>;
     readonly load: (repositoryRoot: string) => Effect.Effect<EvalProjectState | undefined, EvalProjectStoreError>;
     readonly save: (repositoryRoot: string, state: EvalProjectState) => Effect.Effect<void, EvalProjectStoreError>;
 };

package/dist/project-store.js

@@ -1,7 +1,7 @@
 import { writeFileAtomicEffect } from "@velum-labs/routekit-runtime/effect";
 import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect";
 import { EvalProjectStoreError } from "./errors.js";
-import { EvalProjectState } from "./project-contracts.js";
+import { EvalProjectDocument } from "./project-contracts.js";
 const PROJECT_FILE = "project.json";
 export class EvalProjectStore extends Context.Service()("@velum-labs/routekit-eval-setup/EvalProjectStore") {
 }
@@ -15,39 +15,45 @@ export const makeFileEvalProjectStore = Effect.gen(function* () {
     const fs = yield* FileSystem.FileSystem;
     const paths = yield* Path.Path;
     const projectPath = (root) => paths.join(paths.resolve(root), ".routekit", "evals", PROJECT_FILE);
+    const loadDocument = (repositoryRoot) => Effect.gen(function* () {
+        const root = yield* fs
+            .realPath(repositoryRoot)
+            .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
+        const target = projectPath(root);
+        const exists = yield* fs
+            .exists(target)
+            .pipe(Effect.mapError((cause) => storeFailure("checking", target, cause)));
+        if (!exists)
+            return undefined;
+        const encoded = yield* fs
+            .readFileString(target)
+            .pipe(Effect.mapError((cause) => storeFailure("reading", target, cause)));
+        const json = yield* Effect.try({
+            try: () => JSON.parse(encoded),
+            catch: (cause) => storeFailure("parsing", target, cause)
+        });
+        return yield* Schema.decodeUnknownEffect(EvalProjectDocument)(json).pipe(Effect.mapError((cause) => storeFailure("decoding", target, cause)));
+    });
+    const saveDocument = (repositoryRoot, state) => Effect.gen(function* () {
+        const root = yield* fs
+            .realPath(repositoryRoot)
+            .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
+        const target = projectPath(root);
+        const directory = paths.dirname(target);
+        yield* fs
+            .makeDirectory(directory, { recursive: true, mode: 0o700 })
+            .pipe(Effect.mapError((cause) => storeFailure("creating", directory, cause)));
+        yield* writeFileAtomicEffect(target, `${JSON.stringify(state, null, 2)}\n`, {
+            mode: 0o600
+        }).pipe(Effect.mapError((cause) => storeFailure("committing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
+    });
     return EvalProjectStore.of({
-        load: (repositoryRoot) => Effect.gen(function* () {
-            const root = yield* fs
-                .realPath(repositoryRoot)
-                .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
-            const target = projectPath(root);
-            const exists = yield* fs
-                .exists(target)
-                .pipe(Effect.mapError((cause) => storeFailure("checking", target, cause)));
-            if (!exists)
-                return undefined;
-            const encoded = yield* fs
-                .readFileString(target)
-                .pipe(Effect.mapError((cause) => storeFailure("reading", target, cause)));
-            const json = yield* Effect.try({
-                try: () => JSON.parse(encoded),
-                catch: (cause) => storeFailure("parsing", target, cause)
-            });
-            return yield* Schema.decodeUnknownEffect(EvalProjectState)(json).pipe(Effect.mapError((cause) => storeFailure("decoding", target, cause)));
-        }),
-        save: (repositoryRoot, state) => Effect.gen(function* () {
-            const root = yield* fs
-                .realPath(repositoryRoot)
-                .pipe(Effect.mapError((cause) => storeFailure("resolving", repositoryRoot, cause)));
-            const target = projectPath(root);
-            const directory = paths.dirname(target);
-            yield* fs
-                .makeDirectory(directory, { recursive: true, mode: 0o700 })
-                .pipe(Effect.mapError((cause) => storeFailure("creating", directory, cause)));
-            yield* writeFileAtomicEffect(target, `${JSON.stringify(state, null, 2)}\n`, {
-                mode: 0o600
-            }).pipe(Effect.mapError((cause) => storeFailure("committing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
-        })
+        loadDocument,
+        saveDocument,
+        load: (repositoryRoot) => loadDocument(repositoryRoot).pipe(Effect.flatMap((document) => document === undefined || document.version === 1
+            ? Effect.succeed(document)
+            : Effect.fail(storeFailure("decoding", projectPath(repositoryRoot), new Error("Classification V3 project requires the V2 workflow API"))))),
+        save: (repositoryRoot, state) => saveDocument(repositoryRoot, state)
     });
 });
 export const EvalProjectStoreLive = Layer.effect(EvalProjectStore, makeFileEvalProjectStore);

package/dist/project-workflow.d.ts

@@ -27,6 +27,6 @@ export type EvalProjectWorkflowShape = {
 declare const EvalProjectWorkflow_base: Context.ServiceClass<EvalProjectWorkflow, "@velum-labs/routekit-eval-setup/EvalProjectWorkflow", EvalProjectWorkflowShape>;
 export declare class EvalProjectWorkflow extends EvalProjectWorkflow_base {
 }
-export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalRepositoryInspector | Path.Path | EvalProjectArtifacts | EvalProjectStore>;
-export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalRepositoryInspector | Path.Path | EvalProjectArtifacts | EvalProjectStore>;
+export declare const makeEvalProjectWorkflow: Effect.Effect<EvalProjectWorkflowShape, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
+export declare const EvalProjectWorkflowLive: Layer.Layer<EvalProjectWorkflow, never, EvalProjectArtifacts | Path.Path | EvalProjectStore | EvalRepositoryInspector>;
 export {};