@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/repository-routing-quality-policy.js

@@ -0,0 +1,191 @@
+import { assertRoutingBasisV3, routingContractDigest } from "@velum-labs/routekit-eval-contracts";
+import { assertRepositoryReviewedDimensionLabelV1, assertRepositoryRoutingEligibilityPolicyV1 } from "./repository-routing-benchmark-protocol.js";
+const duplicateValues = (values) => {
+    const seen = new Set();
+    const duplicates = new Set();
+    for (const value of values) {
+        if (seen.has(value))
+            duplicates.add(value);
+        seen.add(value);
+    }
+    return [...duplicates].sort();
+};
+const labelPurityRejectionReasons = (input) => {
+    const ordered = [...input.scores].sort((left, right) => right.score - left.score || left.dimensionId.localeCompare(right.dimensionId));
+    const primaryScore = ordered[0]?.score ?? 0;
+    const secondaryScore = ordered[1]?.score ?? 0;
+    const activeCount = ordered.filter((score) => score.score >= input.policy.activeDimensionThreshold).length;
+    return [
+        ...(JSON.stringify(input.scores.map((score) => score.dimensionId)) ===
+            JSON.stringify(input.basisDimensionIds)
+            ? []
+            : [`${input.source} does not cover basis dimensions in canonical order`]),
+        ...(primaryScore >= input.policy.activeDimensionThreshold
+            ? []
+            : [`${input.source} primary dimension is below the active threshold`]),
+        ...(activeCount === 1
+            ? []
+            : [`${input.source} activates ${String(activeCount)} dimensions instead of exactly one`]),
+        ...(primaryScore - secondaryScore >= input.policy.minimumDominanceMargin
+            ? []
+            : [`${input.source} primary dimension does not dominate its nearest competitor`]),
+        ...(input.unknownProbability <= input.policy.maximumUnknownProbability
+            ? []
+            : [`${input.source} unknown probability exceeds the routing threshold`])
+    ];
+};
+const assignmentFor = (input) => {
+    if (input.reviewed === undefined) {
+        return {
+            caseId: input.benchmarkCase.id,
+            basisDigest: input.basis.basisDigest,
+            labelDigest: routingContractDigest("routekit.repository-routing-label.v1", "missing", {
+                caseId: input.benchmarkCase.id,
+                basisDigest: input.basis.basisDigest
+            }),
+            primaryScore: 0,
+            secondaryScore: 0,
+            unknownProbability: 1,
+            eligible: false,
+            rejectionReasons: ["case has no independently reviewed dimension label"]
+        };
+    }
+    assertRepositoryReviewedDimensionLabelV1(input.reviewed);
+    const basisDimensionIds = input.basis.dimensions.map((dimension) => dimension.id);
+    const labelDimensionIds = input.reviewed.label.scores.map((score) => score.dimensionId);
+    const ordered = [...input.reviewed.label.scores].sort((left, right) => right.score - left.score || left.dimensionId.localeCompare(right.dimensionId));
+    const primary = ordered[0];
+    const secondary = ordered[1];
+    const primaryScore = primary?.score ?? 0;
+    const secondaryScore = secondary?.score ?? 0;
+    const activeDimensionIds = ordered
+        .filter((score) => score.score >= input.policy.activeDimensionThreshold)
+        .map((score) => score.dimensionId);
+    const approvedReviews = input.reviewed.reviews.filter((review) => review.outcome === "approve" && review.primaryDimensionId === primary?.dimensionId);
+    const criticPurityRejections = input.reviewed.reviews.flatMap((review) => labelPurityRejectionReasons({
+        source: `independent ${review.model} review`,
+        scores: review.independentLabel.scores,
+        unknownProbability: review.independentLabel.unknownProbability,
+        basisDimensionIds,
+        policy: input.policy
+    }));
+    const rejectionReasons = [
+        ...(input.benchmarkCase.status === "valid-library"
+            ? []
+            : [`benchmark case status is ${input.benchmarkCase.status}, not valid-library`]),
+        ...(input.reviewed.caseId === input.benchmarkCase.id
+            ? []
+            : ["dimension label case id does not match benchmark case"]),
+        ...(input.reviewed.basisDigest === input.basis.basisDigest
+            ? []
+            : ["dimension label basis digest does not match routing basis"]),
+        ...(JSON.stringify(labelDimensionIds) === JSON.stringify(basisDimensionIds)
+            ? []
+            : ["dimension label does not cover basis dimensions in canonical order"]),
+        ...(primary === undefined
+            ? ["dimension label has no primary dimension"]
+            : basisDimensionIds.includes(primary.dimensionId)
+                ? []
+                : ["dimension label primary dimension is absent from the routing basis"]),
+        ...(primaryScore >= input.policy.activeDimensionThreshold
+            ? []
+            : ["primary dimension score is below the active threshold"]),
+        ...(activeDimensionIds.length === 1
+            ? []
+            : [
+                `case activates ${String(activeDimensionIds.length)} dimensions; routing evals require exactly one`
+            ]),
+        ...(primaryScore - secondaryScore >= input.policy.minimumDominanceMargin
+            ? []
+            : ["primary dimension does not dominate the nearest confusable dimension"]),
+        ...(input.reviewed.label.unknownProbability <= input.policy.maximumUnknownProbability
+            ? []
+            : ["dimension label unknown probability exceeds the routing threshold"]),
+        ...(approvedReviews.length === 2
+            ? []
+            : ["independent Terra and Luna reviews did not both approve the primary dimension"]),
+        ...criticPurityRejections
+    ];
+    return {
+        caseId: input.benchmarkCase.id,
+        basisDigest: input.basis.basisDigest,
+        labelDigest: routingContractDigest("routekit.repository-routing-label.v1", "reviewed-label", input.reviewed),
+        ...(primary === undefined ? {} : { primaryDimensionId: primary.dimensionId }),
+        primaryScore,
+        secondaryScore,
+        unknownProbability: input.reviewed.label.unknownProbability,
+        eligible: rejectionReasons.length === 0,
+        rejectionReasons
+    };
+};
+export function compileRepositoryRoutingBenchmarkV1(input) {
+    assertRoutingBasisV3(input.basis);
+    assertRepositoryRoutingEligibilityPolicyV1(input.policy);
+    for (const label of input.labels) {
+        assertRepositoryReviewedDimensionLabelV1(label);
+    }
+    const duplicateCaseIds = duplicateValues(input.library.cases.map((benchmarkCase) => benchmarkCase.id));
+    const duplicateLabelIds = duplicateValues(input.labels.map((label) => label.caseId));
+    if (duplicateCaseIds.length > 0 || duplicateLabelIds.length > 0) {
+        throw new Error([
+            ...(duplicateCaseIds.length === 0
+                ? []
+                : [`duplicate benchmark cases: ${duplicateCaseIds.join(", ")}`]),
+            ...(duplicateLabelIds.length === 0
+                ? []
+                : [`duplicate dimension labels: ${duplicateLabelIds.join(", ")}`])
+        ].join("; "));
+    }
+    const caseIds = new Set(input.library.cases.map((benchmarkCase) => benchmarkCase.id));
+    const unknownLabelIds = input.labels
+        .map((label) => label.caseId)
+        .filter((caseId) => !caseIds.has(caseId))
+        .sort();
+    if (unknownLabelIds.length > 0) {
+        throw new Error(`dimension labels reference unknown benchmark cases: ${unknownLabelIds.join(", ")}`);
+    }
+    const labels = new Map(input.labels.map((label) => [label.caseId, label]));
+    const assignments = input.library.cases
+        .map((benchmarkCase) => assignmentFor({
+        basis: input.basis,
+        benchmarkCase,
+        reviewed: labels.get(benchmarkCase.id),
+        policy: input.policy
+    }))
+        .sort((left, right) => left.caseId.localeCompare(right.caseId));
+    const dimensions = input.basis.dimensions.map((dimension) => {
+        const caseIdsForDimension = assignments
+            .filter((assignment) => assignment.eligible && assignment.primaryDimensionId === dimension.id)
+            .map((assignment) => assignment.caseId)
+            .sort();
+        return {
+            dimensionId: dimension.id,
+            caseIds: caseIdsForDimension,
+            minimumRequired: input.policy.minimumCasesPerDimension,
+            routingReady: caseIdsForDimension.length >= input.policy.minimumCasesPerDimension
+        };
+    });
+    const eligibleCaseIds = assignments
+        .filter((assignment) => assignment.eligible)
+        .map((assignment) => assignment.caseId)
+        .sort();
+    const eligible = new Set(eligibleCaseIds);
+    const libraryOnlyCaseIds = input.library.cases
+        .map((benchmarkCase) => benchmarkCase.id)
+        .filter((caseId) => !eligible.has(caseId))
+        .sort();
+    const coverageGaps = dimensions
+        .filter((dimension) => !dimension.routingReady)
+        .map((dimension) => `${dimension.dimensionId} has ${String(dimension.caseIds.length)}/${String(dimension.minimumRequired)} routing-eligible cases`);
+    return {
+        version: 1,
+        basis: input.basis,
+        policy: input.policy,
+        assignments,
+        dimensions,
+        eligibleCaseIds,
+        libraryOnlyCaseIds,
+        routingReady: coverageGaps.length === 0,
+        coverageGaps
+    };
+}

package/dist/repository-seed-qualification-progress-protocol.d.ts

@@ -0,0 +1,205 @@
+import { Schema } from "effect";
+/**
+ * Durable execution units below the final seed checkpoint. These rows are
+ * scientifically incomplete by construction and become admission evidence only
+ * after every required phase and independent batch is assembled and validated.
+ *
+ * This protocol intentionally lives outside both the task-seed and execution
+ * protocols: qualification progress composes those two stable boundaries but
+ * neither boundary owns or depends on the resumable orchestration state.
+ */
+export declare const RepositorySeedQualificationProgressV1: Schema.Struct<{
+    readonly version: Schema.Literal<1>;
+    readonly bindingDigest: Schema.String;
+    readonly batchesRequired: Schema.Finite;
+    readonly repetitionsPerBatch: Schema.Finite;
+    readonly completedBatches: Schema.$Array<Schema.Struct<{
+        readonly batch: Schema.Finite;
+        readonly seed: Schema.Struct<{
+            readonly version: Schema.Literal<1>;
+            readonly id: Schema.String;
+            readonly status: Schema.Literals<readonly ["candidate", "qualified", "rejected"]>;
+            readonly source: Schema.Union<readonly [Schema.Struct<{
+                readonly kind: Schema.Literal<"historical-fix">;
+                readonly changeEpisodeId: Schema.String;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"historical-feature">;
+                readonly changeEpisodeId: Schema.String;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"historical-refactor">;
+                readonly changeEpisodeId: Schema.String;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"existing-test">;
+                readonly testIds: Schema.$Array<Schema.String>;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"public-api">;
+                readonly surfaceIds: Schema.$Array<Schema.String>;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"schema-contract">;
+                readonly protocolIds: Schema.$Array<Schema.String>;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"production-request">;
+                readonly requestId: Schema.String;
+            }>, Schema.Struct<{
+                readonly kind: Schema.Literal<"reviewed-synthetic-gap">;
+                readonly reviewId: Schema.String;
+            }>]>;
+            readonly summary: Schema.String;
+            readonly capabilityEvidence: Schema.$Array<Schema.Struct<{
+                readonly kind: Schema.Literals<readonly ["package", "public-surface", "symbol", "protocol", "schema", "test", "command", "history-episode", "workload-signal", "documentation"]>;
+                readonly id: Schema.String;
+                readonly path: Schema.optionalKey<Schema.String>;
+            }>>;
+            readonly initialState: Schema.Struct<{
+                readonly commit: Schema.String;
+                readonly parentCommit: Schema.optionalKey<Schema.String>;
+            }>;
+            readonly targetBehavior: Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly description: Schema.String;
+                readonly critical: Schema.Boolean;
+                readonly evidence: Schema.$Array<Schema.Struct<{
+                    readonly kind: Schema.Literals<readonly ["package", "public-surface", "symbol", "protocol", "schema", "test", "command", "history-episode", "workload-signal", "documentation"]>;
+                    readonly id: Schema.String;
+                    readonly path: Schema.optionalKey<Schema.String>;
+                }>>;
+            }>>;
+            readonly referenceCommit: Schema.optionalKey<Schema.String>;
+            readonly baselineObservations: Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly kind: Schema.Literals<readonly ["test-command", "schema", "property", "event-trace", "structural"]>;
+                readonly subjectId: Schema.String;
+                readonly outcome: Schema.Literals<readonly ["pass", "fail", "unknown"]>;
+                readonly detail: Schema.String;
+            }>>;
+            readonly preChangeObservations: Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly kind: Schema.Literals<readonly ["test-command", "schema", "property", "event-trace", "structural"]>;
+                readonly subjectId: Schema.String;
+                readonly outcome: Schema.Literals<readonly ["pass", "fail", "unknown"]>;
+                readonly detail: Schema.String;
+            }>>;
+            readonly postChangeObservations: Schema.$Array<Schema.Struct<{
+                readonly id: Schema.String;
+                readonly kind: Schema.Literals<readonly ["test-command", "schema", "property", "event-trace", "structural"]>;
+                readonly subjectId: Schema.String;
+                readonly outcome: Schema.Literals<readonly ["pass", "fail", "unknown"]>;
+                readonly detail: Schema.String;
+            }>>;
+            readonly candidateTestIds: Schema.$Array<Schema.String>;
+            readonly environment: Schema.Struct<{
+                readonly packageManager: Schema.optionalKey<Schema.Struct<{
+                    readonly name: Schema.Literal<"pnpm">;
+                    readonly version: Schema.String;
+                }>>;
+                readonly lockfilePath: Schema.optionalKey<Schema.String>;
+                readonly protectedControlPaths: Schema.$Array<Schema.String>;
+                readonly trustedPreparationRecipes: Schema.$Array<Schema.Struct<{
+                    readonly id: Schema.String;
+                    readonly kind: Schema.Literals<readonly ["build", "test", "check", "schema", "custom"]>;
+                    readonly cwd: Schema.String;
+                    readonly executable: Schema.String;
+                    readonly args: Schema.$Array<Schema.String>;
+                    readonly timeoutMs: Schema.Finite;
+                }>>;
+                readonly candidateValidationRecipes: Schema.$Array<Schema.Struct<{
+                    readonly id: Schema.String;
+                    readonly kind: Schema.Literals<readonly ["build", "test", "check", "schema", "custom"]>;
+                    readonly cwd: Schema.String;
+                    readonly executable: Schema.String;
+                    readonly args: Schema.$Array<Schema.String>;
+                    readonly timeoutMs: Schema.Finite;
+                }>>;
+                readonly baselineGradeRecipes: Schema.$Array<Schema.Struct<{
+                    readonly id: Schema.String;
+                    readonly kind: Schema.Literals<readonly ["build", "test", "check", "schema", "custom"]>;
+                    readonly cwd: Schema.String;
+                    readonly executable: Schema.String;
+                    readonly args: Schema.$Array<Schema.String>;
+                    readonly timeoutMs: Schema.Finite;
+                }>>;
+                readonly gradeRecipes: Schema.$Array<Schema.Struct<{
+                    readonly id: Schema.String;
+                    readonly kind: Schema.Literals<readonly ["build", "test", "check", "schema", "custom"]>;
+                    readonly cwd: Schema.String;
+                    readonly executable: Schema.String;
+                    readonly args: Schema.$Array<Schema.String>;
+                    readonly timeoutMs: Schema.Finite;
+                }>>;
+            }>;
+            readonly confidence: Schema.Literals<readonly ["high", "medium", "review-required"]>;
+            readonly risks: Schema.$Array<Schema.Literals<readonly ["ambiguous-intent", "bundled-change", "external-service", "flaky-baseline", "reference-only-oracle", "weak-test-delta", "unrealistic-mutation", "private-context", "cross-batch-instability", "review-required"]>>;
+            readonly rejectionReasons: Schema.$Array<Schema.String>;
+        }>;
+    }>>;
+    readonly activeUnits: Schema.$Array<Schema.Struct<{
+        readonly batch: Schema.Finite;
+        readonly phase: Schema.Literals<readonly ["baseline", "pre-change", "post-change"]>;
+        readonly repetition: Schema.Finite;
+        readonly results: Schema.$Array<Schema.Struct<{
+            readonly gradeExecuted: Schema.Boolean;
+            readonly trustedPreparationResults: Schema.$Array<Schema.Struct<{
+                readonly recipeId: Schema.String;
+                readonly commit: Schema.String;
+                readonly repetition: Schema.Finite;
+                readonly stage: Schema.Literals<readonly ["trusted-preparation", "candidate-validation", "grade"]>;
+                readonly exitCode: Schema.NullOr<Schema.Finite>;
+                readonly timedOut: Schema.Boolean;
+                readonly outputTruncated: Schema.Boolean;
+                readonly stdout: Schema.String;
+                readonly stderr: Schema.String;
+                readonly durationMs: Schema.Finite;
+                readonly testReporter: Schema.optionalKey<Schema.Literal<"vitest-v1">>;
+            }>>;
+            readonly candidateValidationResults: Schema.$Array<Schema.Struct<{
+                readonly recipeId: Schema.String;
+                readonly commit: Schema.String;
+                readonly repetition: Schema.Finite;
+                readonly stage: Schema.Literals<readonly ["trusted-preparation", "candidate-validation", "grade"]>;
+                readonly exitCode: Schema.NullOr<Schema.Finite>;
+                readonly timedOut: Schema.Boolean;
+                readonly outputTruncated: Schema.Boolean;
+                readonly stdout: Schema.String;
+                readonly stderr: Schema.String;
+                readonly durationMs: Schema.Finite;
+                readonly testReporter: Schema.optionalKey<Schema.Literal<"vitest-v1">>;
+            }>>;
+            readonly isolationTimings: Schema.optionalKey<Schema.Struct<{
+                readonly checkoutDurationMs: Schema.Finite;
+                readonly overlayDurationMs: Schema.Finite;
+                readonly cleanupDurationMs: Schema.Finite;
+            }>>;
+            readonly preparation: Schema.optionalKey<Schema.Struct<{
+                readonly kind: Schema.Literal<"pristine-install-template">;
+                readonly templateKey: Schema.String;
+                readonly sourceCommit: Schema.String;
+                readonly cloneDurationMs: Schema.Finite;
+                readonly sharedResults: Schema.$Array<Schema.Struct<{
+                    readonly recipeId: Schema.String;
+                    readonly commit: Schema.String;
+                    readonly repetition: Schema.Finite;
+                    readonly stage: Schema.Literals<readonly ["trusted-preparation", "candidate-validation", "grade"]>;
+                    readonly exitCode: Schema.NullOr<Schema.Finite>;
+                    readonly timedOut: Schema.Boolean;
+                    readonly outputTruncated: Schema.Boolean;
+                    readonly stdout: Schema.String;
+                    readonly stderr: Schema.String;
+                    readonly durationMs: Schema.Finite;
+                    readonly testReporter: Schema.optionalKey<Schema.Literal<"vitest-v1">>;
+                }>>;
+            }>>;
+            readonly recipeId: Schema.String;
+            readonly commit: Schema.String;
+            readonly repetition: Schema.Finite;
+            readonly stage: Schema.Literals<readonly ["trusted-preparation", "candidate-validation", "grade"]>;
+            readonly exitCode: Schema.NullOr<Schema.Finite>;
+            readonly timedOut: Schema.Boolean;
+            readonly outputTruncated: Schema.Boolean;
+            readonly stdout: Schema.String;
+            readonly stderr: Schema.String;
+            readonly durationMs: Schema.Finite;
+            readonly testReporter: Schema.optionalKey<Schema.Literal<"vitest-v1">>;
+        }>>;
+    }>>;
+}>;
+export type RepositorySeedQualificationProgressV1 = typeof RepositorySeedQualificationProgressV1.Type;

package/dist/repository-seed-qualification-progress-protocol.js

@@ -0,0 +1,28 @@
+import { Schema } from "effect";
+import { RepositoryCommandResultV1 } from "./repository-execution-protocol.js";
+import { RepositorySeedQualificationPhaseV1, RepositoryTaskSeedV1 } from "./repository-task-seed-protocol.js";
+/**
+ * Durable execution units below the final seed checkpoint. These rows are
+ * scientifically incomplete by construction and become admission evidence only
+ * after every required phase and independent batch is assembled and validated.
+ *
+ * This protocol intentionally lives outside both the task-seed and execution
+ * protocols: qualification progress composes those two stable boundaries but
+ * neither boundary owns or depends on the resumable orchestration state.
+ */
+export const RepositorySeedQualificationProgressV1 = Schema.Struct({
+    version: Schema.Literal(1),
+    bindingDigest: Schema.String,
+    batchesRequired: Schema.Finite,
+    repetitionsPerBatch: Schema.Finite,
+    completedBatches: Schema.Array(Schema.Struct({
+        batch: Schema.Finite,
+        seed: RepositoryTaskSeedV1
+    })),
+    activeUnits: Schema.Array(Schema.Struct({
+        batch: Schema.Finite,
+        phase: RepositorySeedQualificationPhaseV1,
+        repetition: Schema.Finite,
+        results: Schema.Array(RepositoryCommandResultV1)
+    }))
+});