@velum-labs/routekit-eval-setup 1.0.26 → 1.1.0

package/dist/basis-validation.js

@@ -0,0 +1,187 @@
+import { assertRoutingBasisV3, routingBasisApprovalDigestV3, routingContractDigest } from "@velum-labs/routekit-eval-contracts";
+import { evaluateClassificationMetricsV3, inspectRoutingBasisStructureV3, selectRoutingBasisCandidatesV3 } from "@velum-labs/routekit-eval-core";
+import { projectRoutingBasisV3 } from "./basis-projection.js";
+function sameIds(left, right) {
+    return (left.length === right.length &&
+        left.every((value, index) => value === right[index]));
+}
+export function inspectBasisCandidatesV3(candidates, aliasSimilarityThreshold = 0.58) {
+    return candidates.map((candidate) => {
+        if (candidate.basisDigest !== candidate.basis.basisDigest ||
+            candidate.candidateId !== candidate.basis.basisId) {
+            throw new Error(`basis candidate ${JSON.stringify(candidate.candidateId)} has inconsistent identity`);
+        }
+        assertRoutingBasisV3(candidate.basis);
+        const report = inspectRoutingBasisStructureV3(candidate.basis, aliasSimilarityThreshold);
+        let projectionFinding;
+        try {
+            projectRoutingBasisV3(candidate.basis);
+        }
+        catch (cause) {
+            projectionFinding = {
+                severity: "hard",
+                code: "evaluation-projection-invalid",
+                dimensionIds: candidate.basis.dimensions.map((dimension) => dimension.id),
+                detail: cause instanceof Error
+                    ? cause.message
+                    : "basis cannot be projected into existing evaluation dimensions"
+            };
+        }
+        return {
+            candidateId: candidate.candidateId,
+            basisDigest: candidate.basisDigest,
+            eligible: report.eligible && projectionFinding === undefined,
+            dimensionCount: report.dimensionCount,
+            definitionBytes: report.definitionBytes,
+            findings: [
+                ...report.findings,
+                ...(projectionFinding === undefined ? [] : [projectionFinding])
+            ]
+        };
+    });
+}
+export function makeBasisCandidateDatasetV3(input) {
+    const taskIds = input.tasks.map((task) => task.caseId);
+    const labelIds = input.labels.map((label) => label.caseId);
+    if (!sameIds(taskIds, labelIds)) {
+        throw new Error("candidate labels must cover validation tasks in exact order");
+    }
+    const dimensionIds = input.candidate.basis.dimensions.map((dimension) => dimension.id);
+    for (const label of input.labels) {
+        if (label.basisDigest !== input.candidate.basisDigest ||
+            !sameIds(label.scores.map((score) => score.dimensionId), dimensionIds)) {
+            throw new Error(`candidate label ${JSON.stringify(label.caseId)} does not cover the exact basis`);
+        }
+    }
+    const semantic = {
+        version: 1,
+        splitDigest: input.splitDigest,
+        basisDigest: input.candidate.basisDigest,
+        labelProtocol: input.labelProtocol,
+        tasks: input.tasks,
+        labels: input.labels
+    };
+    return {
+        ...semantic,
+        datasetDigest: routingContractDigest("routekit.basis-validation-dataset.v1", "dataset", semantic)
+    };
+}
+export function evaluateBasisCandidateValidationV3(input) {
+    const classifierBinding = input.validationPlan.classifierBindings.find((binding) => binding.candidateId === input.candidate.candidateId);
+    if (input.dataset.basisDigest !== input.candidate.basisDigest ||
+        !input.validationPlan.candidateIds.includes(input.candidate.candidateId) ||
+        !input.validationPlan.datasetDigests.includes(input.dataset.datasetDigest) ||
+        classifierBinding?.basisDigest !== input.candidate.basisDigest) {
+        throw new Error("validation inputs do not match the immutable plan");
+    }
+    if (input.predictions.some((prediction) => prediction.classifierConfigurationDigest !==
+        classifierBinding.classifierConfigurationDigest)) {
+        throw new Error("validation predictions use a different classifier configuration");
+    }
+    const metrics = evaluateClassificationMetricsV3(input.dataset.labels, input.predictions, {
+        activeDimensionThreshold: input.dataset.labelProtocol.activeDimensionThreshold,
+        unknownThreshold: input.dataset.labelProtocol.unknownThreshold
+    });
+    return {
+        version: 1,
+        candidateId: input.candidate.candidateId,
+        basisDigest: input.candidate.basisDigest,
+        validationDatasetDigest: input.dataset.datasetDigest,
+        classifierConfigurationDigest: classifierBinding.classifierConfigurationDigest,
+        predictions: input.predictions,
+        ...metrics,
+        selectionScore: metrics.activeMae +
+            0.5 * metrics.unknownMae +
+            0.15 * metrics.referenceUnknownRate,
+        provisional: input.provisional
+    };
+}
+export function selectValidatedBasisCandidatesV3(input) {
+    const structureByCandidate = new Map(input.structureReports.map((report) => [report.candidateId, report]));
+    const validationByCandidate = new Map(input.validationResults.map((result) => [result.candidateId, result]));
+    const selection = selectRoutingBasisCandidatesV3(input.candidates.map((candidate) => {
+        const structure = structureByCandidate.get(candidate.candidateId);
+        const validation = validationByCandidate.get(candidate.candidateId);
+        if (structure === undefined || validation === undefined) {
+            throw new Error(`candidate ${JSON.stringify(candidate.candidateId)} is missing structural or validation evidence`);
+        }
+        return {
+            basis: candidate.basis,
+            structure: {
+                basisId: candidate.candidateId,
+                basisDigest: candidate.basisDigest,
+                eligible: structure.eligible,
+                dimensionCount: structure.dimensionCount,
+                definitionBytes: structure.definitionBytes,
+                findings: structure.findings
+            },
+            activeMae: validation.activeMae,
+            unknownMae: validation.unknownMae,
+            referenceUnknownRate: validation.referenceUnknownRate,
+            activeF1: validation.activeF1,
+            topDimensionAgreement: validation.topDimensionAgreement,
+            provisional: validation.provisional
+        };
+    }));
+    if (selection.selectedBasisId === undefined) {
+        throw new Error("no structurally eligible basis candidate can be selected");
+    }
+    const semantic = {
+        version: 1,
+        repositoryDigest: input.repositoryDigest,
+        validationPlanDigest: input.validationPlan.planDigest,
+        objectiveVersion: "basis-selection-v1",
+        rankedCandidateIds: selection.ranked.map((candidate) => candidate.basis.basisId),
+        paretoCandidateIds: selection.paretoBasisIds,
+        selectedCandidateId: selection.selectedBasisId,
+        provisional: input.validationResults.some((result) => result.provisional)
+    };
+    return {
+        ...semantic,
+        selectionReportDigest: routingContractDigest("routekit.basis-selection-report.v1", "selection", semantic)
+    };
+}
+export function approveSelectedBasisV3(input) {
+    assertRoutingBasisV3(input.candidate.basis);
+    const { selectionReportDigest: _selectionReportDigest, ...selectionSemantic } = input.selection;
+    if (routingContractDigest("routekit.basis-selection-report.v1", "selection", selectionSemantic) !== input.selection.selectionReportDigest) {
+        throw new Error("basis selection report digest does not match its contents");
+    }
+    if (input.snapshot.dirty) {
+        throw new Error("a dirty repository snapshot cannot be approved");
+    }
+    if (!input.structure.eligible) {
+        throw new Error("a structurally invalid basis cannot be approved");
+    }
+    if (input.selection.selectedCandidateId !== input.candidate.candidateId ||
+        input.structure.candidateId !== input.candidate.candidateId ||
+        input.validation.candidateId !== input.candidate.candidateId ||
+        input.validation.basisDigest !== input.candidate.basisDigest ||
+        input.snapshot.repositoryDigest !== input.candidate.basis.repositoryDigest) {
+        throw new Error("basis approval inputs contain stale or mismatched identities");
+    }
+    if (input.selection.provisional &&
+        (input.provisionalAcknowledgement?.trim().length ?? 0) === 0) {
+        throw new Error("provisional basis approval requires an acknowledgement");
+    }
+    const approval = {
+        version: 1,
+        projectId: input.projectId,
+        candidateId: input.candidate.candidateId,
+        basisDigest: input.candidate.basisDigest,
+        repositoryDigest: input.snapshot.repositoryDigest,
+        validationDatasetDigest: input.validation.validationDatasetDigest,
+        selectionReportDigest: input.selection.selectionReportDigest,
+        approvedAt: input.approvedAt,
+        approvedBy: input.approvedBy,
+        ...(input.provisionalAcknowledgement === undefined
+            ? {}
+            : {
+                provisionalAcknowledgement: input.provisionalAcknowledgement
+            })
+    };
+    return {
+        approval,
+        approvalDigest: routingBasisApprovalDigestV3(approval)
+    };
+}

package/dist/effect-api.d.ts

@@ -1,3 +1,11 @@
+export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
+export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
+export type { BasisAuthoringShape } from "./services/basis-authoring/service.js";
+export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";
+export type { BasisOnboardingShape } from "./services/basis-onboarding/service.js";
+export { BasisOnboarding, BasisOnboardingLive, makeBasisOnboarding } from "./services/basis-onboarding/service.js";
+export type { BasisRepairShape } from "./services/basis-repair/service.js";
+export { BasisRepair, BasisRepairLive, makeBasisRepair } from "./services/basis-repair/service.js";
 export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository } from "./inspection.js";
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";

package/dist/effect-api.js

@@ -1,3 +1,8 @@
+export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
+export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
+export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";
+export { BasisOnboarding, BasisOnboardingLive, makeBasisOnboarding } from "./services/basis-onboarding/service.js";
+export { BasisRepair, BasisRepairLive, makeBasisRepair } from "./services/basis-repair/service.js";
 export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository } from "./inspection.js";
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";

package/dist/errors.d.ts

@@ -91,7 +91,17 @@ declare const EvalProjectAuthoringError_base: new <A extends Record<string, any>
     readonly _tag: "EvalProjectAuthoringError";
 } & Readonly<A>;
 export declare class EvalProjectAuthoringError extends EvalProjectAuthoringError_base<{
-    readonly operation: "reading-sources" | "authoring-dimensions" | "authoring-evaluations";
+    readonly operation: "reading-sources" | "authoring-basis" | "authoring-dimensions" | "authoring-evaluations";
+    readonly detail: string;
+    readonly cause?: unknown;
+}> {
+    get message(): string;
+}
+declare const EvalBasisSnapshotError_base: new <A extends Record<string, any> = {}>(args: import("effect/Types").VoidIfEmpty<{ readonly [P in keyof A as P extends "_tag" ? never : P]: A[P]; }>) => import("effect/Cause").YieldableError & {
+    readonly _tag: "EvalBasisSnapshotError";
+} & Readonly<A>;
+export declare class EvalBasisSnapshotError extends EvalBasisSnapshotError_base<{
+    readonly operation: "resolve-root" | "inspect-status" | "resolve-ref" | "read-tree" | "read-dirty-content";
     readonly detail: string;
     readonly cause?: unknown;
 }> {

package/dist/errors.js

@@ -49,3 +49,8 @@ export class EvalProjectAuthoringError extends Data.TaggedError("EvalProjectAuth
         return `RouteKit Eval could not complete ${this.operation}: ${this.detail}`;
     }
 }
+export class EvalBasisSnapshotError extends Data.TaggedError("EvalBasisSnapshotError") {
+    get message() {
+        return `RouteKit Eval could not ${this.operation}: ${this.detail}`;
+    }
+}

package/dist/index.d.ts

@@ -1,15 +1,27 @@
-export { EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
+export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
+export type { CapturedRepositorySnapshotV3, FrozenRepositorySourceV3 } from "./adapters/git-basis-snapshot.js";
+export { readFrozenRepositorySourcesV3 } from "./adapters/git-basis-snapshot.js";
 export type { EvalHostMetadata, HostEligibility } from "./host-metadata.js";
 export { authoringRequest, hostDirectory } from "./host-metadata.js";
+export { BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT, BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT, DEFAULT_BASIS_CANDIDATE_RECIPES_V3, selectConstructionTasksForRecipeV3, splitBasisTasksChronologicallyV3 } from "./basis-candidates.js";
+export { approveSelectedBasisV3, evaluateBasisCandidateValidationV3, inspectBasisCandidatesV3, makeBasisCandidateDatasetV3, selectValidatedBasisCandidatesV3 } from "./basis-validation.js";
 export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository } from "./inspection.js";
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export type { OriEvalAuthoringApi, OriEvalResult } from "./ori-result.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export type { EvalAuthoringCompletion, EvalAuthoringSource, EvalAuthoringTransportShape, EvalProjectAuthorShape } from "./project-authoring.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export type { EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget, EvalScopedExpectedCalls } from "./project-contracts.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
+export type { BasisCandidateDatasetV3, BasisCandidatesArtifactsV3, BasisCandidateCheckpointV3, BasisCandidateRecipeV3, BasisCandidateRecordV3, BasisGenerationArtifactsV3, BasisEvaluationProjectionV3, BasisGenerationJobV3, BasisGenerationLedgerV3, BasisGenerationPlanV3, BasisLabelProtocolV3, BasisQualificationArtifactsV3, BasisRepairArtifactsV3, BasisRepairPlanV3, BasisRepairSummaryV3, BasisSelectionReportV3, BasisStructureFindingV3, BasisStructureReportV3, BasisTaskRecordV3, BasisTaskSplitV3, BasisValidationPlanV3, BasisValidationClassifierBindingV3, BasisValidationJobCheckpointV3, BasisValidationResultsArtifactsV3, BasisValidationArtifactsV3, BasisValidationResultV3, EvalClassifierObservation, EvalCompositionCase, EvalCompositionCaseResult, EvalCompositionSuite, EvalDecompositionBenchmark, EvalDecompositionBenchmarkCase, EvalDimensionCase, EvalDimensionContrast, EvalDimensionSuite, EvalEvaluationProposal, EvalExecutionPlan, EvalPlanScope, EvalProposedDimension, EvalProjectArtifactsStatus, EvalProjectDocument, EvalProjectStateV2, EvalProjectStatus, EvalRoutingBasisProposal, EvalRunCleanup, EvalRunFailureError, EvalRunLedger, EvalRunQualification, EvalRunReport, EvalRunTarget, EvalScopedExpectedCalls, FrozenRepositorySnapshotV3 } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
+export type { BasisAuthoringShape } from "./services/basis-authoring/service.js";
+export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";
+export type { BasisOnboardingShape } from "./services/basis-onboarding/service.js";
+export { BasisOnboarding, BasisOnboardingLive, makeBasisOnboarding } from "./services/basis-onboarding/service.js";
+export type { BasisRepairShape } from "./services/basis-repair/service.js";
+export { BasisRepair, BasisRepairLive, makeBasisRepair } from "./services/basis-repair/service.js";
 export type { EvalProjectWorkflowError, EvalProjectWorkflowShape } from "./project-workflow.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
 export { questionForStage, withOpenQuestion } from "./questions.js";

package/dist/index.js

@@ -1,11 +1,19 @@
-export { EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { projectRoutingBasisV3, projectRoutingDimensionV3 } from "./basis-projection.js";
+export { EvalBasisSnapshotError, EvalDimensionContrastError, EvalProjectArtifactError, EvalProjectAuthoringError, EvalProjectStoreError, EvalProjectTransitionError, EvalSetupInspectionError, EvalSetupRunnerError, EvalSetupScaffoldError, EvalSetupStateError, EvalSetupTransitionError } from "./errors.js";
+export { captureFrozenRepositorySnapshotV3 } from "./adapters/git-basis-snapshot.js";
+export { readFrozenRepositorySourcesV3 } from "./adapters/git-basis-snapshot.js";
 export { authoringRequest, hostDirectory } from "./host-metadata.js";
+export { BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT, BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT, DEFAULT_BASIS_CANDIDATE_RECIPES_V3, selectConstructionTasksForRecipeV3, splitBasisTasksChronologicallyV3 } from "./basis-candidates.js";
+export { approveSelectedBasisV3, evaluateBasisCandidateValidationV3, inspectBasisCandidatesV3, makeBasisCandidateDatasetV3, selectValidatedBasisCandidatesV3 } from "./basis-validation.js";
 export { EvalRepositoryInspector, EvalRepositoryInspectorLive, inspectRepository } from "./inspection.js";
 export { OriEvalAuthoring, oriAuthoringFromApi } from "./ori-authoring.js";
 export { EvalProjectArtifacts, EvalProjectArtifactsLive, evaluationProposalDigest, makeFileEvalProjectArtifacts, routingBasisDigest } from "./project-artifacts.js";
 export { EVAL_AUTHORING_CASES_PER_DIMENSION, EVAL_AUTHORING_EVALUATION_OUTPUT_TOKENS, EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_SOURCE_BYTES, EVAL_AUTHORING_SOURCE_FILES, EvalAuthoringTransport, EvalProjectAuthor, EvalProjectAuthorLive, makeEvalProjectAuthor, readProjectAuthoringSources, selectProjectAuthoringSourceFiles } from "./project-authoring.js";
-export { EVAL_PROJECT_VERSION, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectSetupProgress, EvalProjectState, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
+export { EVAL_PROJECT_VERSION, EVAL_PROJECT_V2_VERSION, BasisCandidateDatasetV3 as BasisCandidateDatasetV3Schema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisCandidateRecipeV3 as BasisCandidateRecipeV3Schema, BasisCandidateRecordV3 as BasisCandidateRecordV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisGenerationJobV3 as BasisGenerationJobV3Schema, BasisGenerationLedgerV3 as BasisGenerationLedgerV3Schema, BasisGenerationPlanV3 as BasisGenerationPlanV3Schema, BasisLabelProtocolV3 as BasisLabelProtocolV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisRepairPlanV3 as BasisRepairPlanV3Schema, BasisRepairSummaryV3 as BasisRepairSummaryV3Schema, BasisSelectionReportV3 as BasisSelectionReportV3Schema, BasisStructureReportV3 as BasisStructureReportV3Schema, BasisTaskRecordV3 as BasisTaskRecordV3Schema, BasisTaskSplitV3 as BasisTaskSplitV3Schema, BasisValidationPlanV3 as BasisValidationPlanV3Schema, BasisValidationClassifierBindingV3 as BasisValidationClassifierBindingV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, BasisValidationResultV3 as BasisValidationResultV3Schema, EvalArtifactApproval, EvalCompositionSuite as EvalCompositionSuiteSchema, EvalDecompositionBenchmark as EvalDecompositionBenchmarkSchema, EvalDimensionCase as EvalDimensionCaseSchema, EvalDimensionContrast as EvalDimensionContrastSchema, EvalDimensionSuite as EvalDimensionSuiteSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalPlanScope as EvalPlanScopeSchema, EvalProposedDimension as EvalProposedDimensionSchema, EvalProjectConfiguration, EvalProjectQuestion, EvalProjectDocument as EvalProjectDocumentSchema, EvalProjectSetupProgress, EvalProjectState, EvalProjectStateV2 as EvalProjectStateV2Schema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunCleanup as EvalRunCleanupSchema, EvalRunFailureError as EvalRunFailureErrorSchema, EvalRunLedger as EvalRunLedgerSchema, EvalRunReport as EvalRunReportSchema, EvalRunTarget as EvalRunTargetSchema, FrozenRepositorySnapshotV3 as FrozenRepositorySnapshotV3Schema, scopedExpectedCalls, summarizeEvalRunLedger } from "./project-contracts.js";
 export { EvalProjectStore, EvalProjectStoreLive, makeFileEvalProjectStore } from "./project-store.js";
+export { BasisAuthoring, BasisAuthoringLive, makeBasisAuthoring } from "./services/basis-authoring/service.js";
+export { BasisOnboarding, BasisOnboardingLive, makeBasisOnboarding } from "./services/basis-onboarding/service.js";
+export { BasisRepair, BasisRepairLive, makeBasisRepair } from "./services/basis-repair/service.js";
 export { EvalProjectWorkflow, EvalProjectWorkflowLive, makeEvalProjectWorkflow } from "./project-workflow.js";
 export { questionForStage, withOpenQuestion } from "./questions.js";
 export { EvalSetupRunner, EvalSetupRunnerNoop } from "./runner.js";

package/dist/project-artifacts.d.ts

@@ -1,10 +1,35 @@
 import { type RoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { Context, Effect, FileSystem, Layer, Path } from "effect";
 import { EvalProjectArtifactError } from "./errors.js";
-import { type EvalArtifactApproval, type EvalEvaluationProposal, type EvalExecutionPlan, type EvalRoutingBasisProposal, type EvalRunReport } from "./project-contracts.js";
+import { type EvalArtifactApproval, type BasisCandidatesArtifactsV3, type BasisEvaluationProjectionV3, type BasisCandidateCheckpointV3, type BasisGenerationArtifactsV3, type BasisQualificationArtifactsV3, type BasisRepairArtifactsV3, type BasisValidationJobCheckpointV3, type BasisValidationResultsArtifactsV3, type BasisValidationArtifactsV3, type EvalEvaluationProposal, type EvalExecutionPlan, type EvalRoutingBasisProposal, type EvalRunReport } from "./project-contracts.js";
 export declare function routingBasisDigest(dimensions: RoutingBasis["dimensions"]): string;
 export declare function evaluationProposalDigest(proposal: Omit<EvalEvaluationProposal, "evaluationDigest">): string;
 export type EvalProjectArtifactsShape = {
+    readonly loadBasisGenerationV3: (repositoryRoot: string) => Effect.Effect<BasisGenerationArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisGenerationV3: (repositoryRoot: string, artifacts: BasisGenerationArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisCandidatesV3: (repositoryRoot: string) => Effect.Effect<BasisCandidatesArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisCandidatesV3: (repositoryRoot: string, artifacts: BasisCandidatesArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisCandidateCheckpointV3: (repositoryRoot: string, recipeId: string) => Effect.Effect<BasisCandidateCheckpointV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisCandidateCheckpointV3: (repositoryRoot: string, checkpoint: BasisCandidateCheckpointV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisValidationV3: (repositoryRoot: string) => Effect.Effect<BasisValidationArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisValidationV3: (repositoryRoot: string, artifacts: BasisValidationArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisValidationJobCheckpointV3: (repositoryRoot: string, input: {
+        readonly validationPlanDigest: string;
+        readonly candidateId: string;
+        readonly caseId: string;
+        readonly repetition: number;
+    }) => Effect.Effect<BasisValidationJobCheckpointV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisValidationJobCheckpointV3: (repositoryRoot: string, checkpoint: BasisValidationJobCheckpointV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisValidationResultsV3: (repositoryRoot: string) => Effect.Effect<BasisValidationResultsArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisValidationResultsV3: (repositoryRoot: string, artifacts: BasisValidationResultsArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisQualificationV3: (repositoryRoot: string) => Effect.Effect<BasisQualificationArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisQualificationV3: (repositoryRoot: string, artifacts: BasisQualificationArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisEvaluationProjectionV3: (repositoryRoot: string) => Effect.Effect<BasisEvaluationProjectionV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisEvaluationProjectionV3: (repositoryRoot: string, projection: BasisEvaluationProjectionV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly loadBasisRepairV3: (repositoryRoot: string) => Effect.Effect<BasisRepairArtifactsV3 | undefined, EvalProjectArtifactError>;
+    readonly saveBasisRepairV3: (repositoryRoot: string, artifacts: BasisRepairArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly saveBasisRepairValidationV3: (repositoryRoot: string, artifacts: BasisValidationArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
+    readonly saveBasisRepairValidationResultsV3: (repositoryRoot: string, artifacts: BasisValidationResultsArtifactsV3) => Effect.Effect<void, EvalProjectArtifactError>;
     readonly loadBasisProposal: (repositoryRoot: string) => Effect.Effect<EvalRoutingBasisProposal | undefined, EvalProjectArtifactError, never>;
     readonly saveBasisProposal: (repositoryRoot: string, basis: EvalRoutingBasisProposal) => Effect.Effect<void, EvalProjectArtifactError, never>;
     readonly loadBasisApproval: (repositoryRoot: string) => Effect.Effect<EvalArtifactApproval | undefined, EvalProjectArtifactError, never>;

package/dist/project-artifacts.js

@@ -3,11 +3,20 @@ import { assertRoutingBasis } from "@velum-labs/routekit-eval-contracts";
 import { writeFileAtomicEffect } from "@velum-labs/routekit-runtime/effect";
 import { Context, Effect, Exit, FileSystem, Layer, Path, Schema } from "effect";
 import { EvalProjectArtifactError } from "./errors.js";
-import { EVAL_PROJECT_VERSION, EvalArtifactApproval as EvalArtifactApprovalSchema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunReport as EvalRunReportSchema } from "./project-contracts.js";
+import { EVAL_PROJECT_VERSION, EvalArtifactApproval as EvalArtifactApprovalSchema, BasisCandidatesArtifactsV3 as BasisCandidatesArtifactsV3Schema, BasisEvaluationProjectionV3 as BasisEvaluationProjectionV3Schema, BasisCandidateCheckpointV3 as BasisCandidateCheckpointV3Schema, BasisGenerationArtifactsV3 as BasisGenerationArtifactsV3Schema, BasisQualificationArtifactsV3 as BasisQualificationArtifactsV3Schema, BasisRepairArtifactsV3 as BasisRepairArtifactsV3Schema, BasisValidationJobCheckpointV3 as BasisValidationJobCheckpointV3Schema, BasisValidationResultsArtifactsV3 as BasisValidationResultsArtifactsV3Schema, BasisValidationArtifactsV3 as BasisValidationArtifactsV3Schema, EvalEvaluationProposal as EvalEvaluationProposalSchema, EvalExecutionPlan as EvalExecutionPlanSchema, EvalRoutingBasisProposal as EvalRoutingBasisProposalSchema, EvalRunReport as EvalRunReportSchema } from "./project-contracts.js";
 const BASIS_PROPOSAL = "routing-basis.proposed.json";
 const BASIS_APPROVAL = "routing-basis.approval.json";
 const EVALUATIONS_PROPOSAL = "evaluations.proposed.json";
 const EVALUATIONS_APPROVAL = "evaluations.approval.json";
+const BASIS_GENERATION_V3 = "basis/generation.v3.json";
+const BASIS_CANDIDATES_V3 = "basis/candidates.v3.json";
+const BASIS_VALIDATION_V3 = "basis/validation.v3.json";
+const BASIS_VALIDATION_RESULTS_V3 = "basis/validation-results.v3.json";
+const BASIS_QUALIFICATION_V3 = "basis/qualification.v3.json";
+const BASIS_EVALUATION_PROJECTION_V3 = "basis/evaluation-projection.v3.json";
+const BASIS_REPAIR_V3 = "basis/repair.v3.json";
+const BASIS_REPAIR_VALIDATION_V3 = "basis/repair-validation.v3.json";
+const BASIS_REPAIR_VALIDATION_RESULTS_V3 = "basis/repair-validation-results.v3.json";
 const ARTIFACT_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/u;
 const renderCandidatePrompt = () => `const prompt = [
         testCase.prompt,
@@ -157,6 +166,18 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
     const paths = yield* Path.Path;
     const root = (repositoryRoot) => paths.join(paths.resolve(repositoryRoot), ".routekit", "evals");
     const artifactPath = (repositoryRoot, name) => paths.join(root(repositoryRoot), name);
+    const candidateCheckpointName = (recipeId) => ARTIFACT_ID.test(recipeId)
+        ? Effect.succeed(paths.join("basis", "candidates", `${recipeId}.v3.json`))
+        : Effect.fail(artifactFailure("reading", recipeId, new Error("recipe id must be a bounded identifier, not a path")));
+    const validationCheckpointName = (input) => {
+        const identity = {
+            validationPlanDigest: input.validationPlanDigest,
+            candidateId: input.candidateId,
+            caseId: input.caseId,
+            repetition: input.repetition
+        };
+        return paths.join("basis", "validation", `${digest(identity)}.v3.json`);
+    };
     const writeAtomic = (target, content) => writeFileAtomicEffect(target, content, {
         mode: 0o600
     }).pipe(Effect.mapError((cause) => artifactFailure("writing", target, cause)), Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, paths));
@@ -190,7 +211,37 @@ export const makeFileEvalProjectArtifacts = Effect.gen(function* () {
         yield* writeAtomic(target, content);
     });
     const write = (repositoryRoot, name, value, overwrite = true) => writeText(repositoryRoot, name, `${JSON.stringify(value, null, 2)}\n`, overwrite);
+    const saveImmutable = (repositoryRoot, name, value, decode) => Effect.gen(function* () {
+        const existing = yield* read(repositoryRoot, name, decode);
+        if (existing !== undefined) {
+            if (JSON.stringify(existing) !== JSON.stringify(value)) {
+                return yield* artifactFailure("writing", artifactPath(repositoryRoot, name), new Error("immutable artifact already exists with different contents"));
+            }
+            return;
+        }
+        yield* write(repositoryRoot, name, value, false);
+    });
     return EvalProjectArtifacts.of({
+        loadBasisGenerationV3: (repositoryRoot) => read(repositoryRoot, BASIS_GENERATION_V3, Schema.decodeUnknownEffect(BasisGenerationArtifactsV3Schema)),
+        saveBasisGenerationV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_GENERATION_V3, artifacts, Schema.decodeUnknownEffect(BasisGenerationArtifactsV3Schema)),
+        loadBasisCandidatesV3: (repositoryRoot) => read(repositoryRoot, BASIS_CANDIDATES_V3, Schema.decodeUnknownEffect(BasisCandidatesArtifactsV3Schema)),
+        saveBasisCandidatesV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_CANDIDATES_V3, artifacts, Schema.decodeUnknownEffect(BasisCandidatesArtifactsV3Schema)),
+        loadBasisCandidateCheckpointV3: (repositoryRoot, recipeId) => candidateCheckpointName(recipeId).pipe(Effect.flatMap((name) => read(repositoryRoot, name, Schema.decodeUnknownEffect(BasisCandidateCheckpointV3Schema)))),
+        saveBasisCandidateCheckpointV3: (repositoryRoot, checkpoint) => candidateCheckpointName(checkpoint.job.recipeId).pipe(Effect.flatMap((name) => saveImmutable(repositoryRoot, name, checkpoint, Schema.decodeUnknownEffect(BasisCandidateCheckpointV3Schema)))),
+        loadBasisValidationV3: (repositoryRoot) => read(repositoryRoot, BASIS_VALIDATION_V3, Schema.decodeUnknownEffect(BasisValidationArtifactsV3Schema)),
+        saveBasisValidationV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_VALIDATION_V3, artifacts, Schema.decodeUnknownEffect(BasisValidationArtifactsV3Schema)),
+        loadBasisValidationJobCheckpointV3: (repositoryRoot, input) => read(repositoryRoot, validationCheckpointName(input), Schema.decodeUnknownEffect(BasisValidationJobCheckpointV3Schema)),
+        saveBasisValidationJobCheckpointV3: (repositoryRoot, checkpoint) => saveImmutable(repositoryRoot, validationCheckpointName(checkpoint), checkpoint, Schema.decodeUnknownEffect(BasisValidationJobCheckpointV3Schema)),
+        loadBasisValidationResultsV3: (repositoryRoot) => read(repositoryRoot, BASIS_VALIDATION_RESULTS_V3, Schema.decodeUnknownEffect(BasisValidationResultsArtifactsV3Schema)),
+        saveBasisValidationResultsV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_VALIDATION_RESULTS_V3, artifacts, Schema.decodeUnknownEffect(BasisValidationResultsArtifactsV3Schema)),
+        loadBasisQualificationV3: (repositoryRoot) => read(repositoryRoot, BASIS_QUALIFICATION_V3, Schema.decodeUnknownEffect(BasisQualificationArtifactsV3Schema)),
+        saveBasisQualificationV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_QUALIFICATION_V3, artifacts, Schema.decodeUnknownEffect(BasisQualificationArtifactsV3Schema)),
+        loadBasisEvaluationProjectionV3: (repositoryRoot) => read(repositoryRoot, BASIS_EVALUATION_PROJECTION_V3, Schema.decodeUnknownEffect(BasisEvaluationProjectionV3Schema)),
+        saveBasisEvaluationProjectionV3: (repositoryRoot, projection) => saveImmutable(repositoryRoot, BASIS_EVALUATION_PROJECTION_V3, projection, Schema.decodeUnknownEffect(BasisEvaluationProjectionV3Schema)),
+        loadBasisRepairV3: (repositoryRoot) => read(repositoryRoot, BASIS_REPAIR_V3, Schema.decodeUnknownEffect(BasisRepairArtifactsV3Schema)),
+        saveBasisRepairV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_REPAIR_V3, artifacts, Schema.decodeUnknownEffect(BasisRepairArtifactsV3Schema)),
+        saveBasisRepairValidationV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_REPAIR_VALIDATION_V3, artifacts, Schema.decodeUnknownEffect(BasisValidationArtifactsV3Schema)),
+        saveBasisRepairValidationResultsV3: (repositoryRoot, artifacts) => saveImmutable(repositoryRoot, BASIS_REPAIR_VALIDATION_RESULTS_V3, artifacts, Schema.decodeUnknownEffect(BasisValidationResultsArtifactsV3Schema)),
         loadBasisProposal: (repositoryRoot) => read(repositoryRoot, BASIS_PROPOSAL, Schema.decodeUnknownEffect(EvalRoutingBasisProposalSchema)).pipe(Effect.tap((basis) => basis === undefined
             ? Effect.void
             : Effect.try({