@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/routing-benchmark/service.d.ts

@@ -0,0 +1,362 @@
+import { type RoutingBasisV3 } from "@velum-labs/routekit-eval-contracts";
+import { Context, Effect, Layer } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import type { RepositoryBenchmarkCaseV1 } from "../../repository-benchmark-protocol.js";
+import { type RepositoryRoutingBenchmarkV1, type RepositoryRoutingEligibilityPolicyV1 } from "../../repository-routing-benchmark-protocol.js";
+import { type RepositoryRoutingModelPlanV1 } from "../../repository-routing-model-protocol.js";
+import { RepositoryRoutingLabelingPlanV1 } from "../../repository-routing-plan-protocol.js";
+import { type RepositoryDimensionLabelingResultV1 } from "../dimension-labeling/service.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+export type RepositoryRoutingLabelingExecutionResultV1 = {
+    readonly version: 1;
+    readonly plan: RepositoryRoutingLabelingPlanV1;
+    readonly labeling: RepositoryDimensionLabelingResultV1;
+    readonly routing: RepositoryRoutingBenchmarkV1;
+};
+export type RoutingBenchmarkShape = {
+    readonly planLabeling: (input: {
+        readonly foundryPlanDigest: string;
+        readonly benchmarkCase: RepositoryBenchmarkCaseV1;
+        readonly basis: RoutingBasisV3;
+        readonly modelPlan: RepositoryRoutingModelPlanV1;
+        readonly policy: RepositoryRoutingEligibilityPolicyV1;
+    }) => Effect.Effect<RepositoryRoutingLabelingPlanV1, RepositoryFoundryError>;
+    readonly executeLabeling: (input: {
+        readonly plan: RepositoryRoutingLabelingPlanV1;
+        readonly benchmarkCase: RepositoryBenchmarkCaseV1;
+    }) => Effect.Effect<RepositoryRoutingLabelingExecutionResultV1, RepositoryFoundryError>;
+};
+declare const RoutingBenchmark_base: Context.ServiceClass<RoutingBenchmark, "@velum-labs/routekit-eval-setup/RoutingBenchmark", RoutingBenchmarkShape>;
+export declare class RoutingBenchmark extends RoutingBenchmark_base {
+}
+export declare const planRepositoryRoutingLabelingV1: (input: {
+    readonly foundryPlanDigest: string;
+    readonly benchmarkCase: RepositoryBenchmarkCaseV1;
+    readonly basis: RoutingBasisV3;
+    readonly modelPlan: RepositoryRoutingModelPlanV1;
+    readonly policy: RepositoryRoutingEligibilityPolicyV1;
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly planDigest: string;
+    readonly foundryPlanDigest: string;
+    readonly caseId: string;
+    readonly behavioralFingerprint: string;
+    readonly basis: {
+        readonly version: 3;
+        readonly basisDigest: string;
+        readonly dimensions: readonly {
+            readonly id: string;
+            readonly name: string;
+            readonly description: string;
+            readonly inclusions: readonly string[];
+            readonly exclusions: readonly string[];
+            readonly confusableDimensionIds: readonly string[];
+            readonly boundaryRules: readonly string[];
+            readonly repositoryFacets: {
+                readonly componentIds: readonly string[];
+                readonly pathAnchors: readonly string[];
+                readonly symbolAnchors: readonly string[];
+            };
+            readonly positiveExamples: readonly string[];
+            readonly negativeExamples: readonly string[];
+            readonly codeSummaries: readonly string[];
+            readonly representativeSnippets: readonly string[];
+        }[];
+        readonly repositoryDigest: string;
+        readonly basisId: string;
+        readonly facetSnapshot: {
+            readonly repositoryId: string;
+            readonly repositoryDigest: string;
+            readonly name: string;
+            readonly purpose: string;
+            readonly languages: readonly string[];
+            readonly frameworks: readonly string[];
+            readonly components: readonly {
+                readonly id: string;
+                readonly name: string;
+                readonly purpose: string;
+                readonly paths: readonly string[];
+            }[];
+        };
+        readonly provenance: {
+            readonly recipeId: string;
+            readonly generatorVersion: string;
+            readonly sourceDigests: readonly string[];
+            readonly authorModel?: string | undefined;
+            readonly constructionTaskDigest?: string | undefined;
+            readonly origin?: {
+                readonly kind: "generated" | "incumbent-enriched" | "repair";
+                readonly sourceBasisId?: string | undefined;
+                readonly sourceBasisVersion?: number | undefined;
+            } | undefined;
+        };
+        readonly validation?: {
+            readonly validationDatasetDigest: string;
+            readonly caseCount: number;
+            readonly activeMae: number;
+            readonly activeF1: number;
+            readonly topDimensionAgreement: number;
+            readonly unknownMae: number;
+            readonly referenceUnknownRate: number;
+            readonly selectionScore: number;
+        } | undefined;
+    };
+    readonly modelPlan: {
+        readonly version: 1;
+        readonly assignments: readonly {
+            readonly role: "dimension-labeler" | "dimension-critic-terra" | "dimension-critic-luna";
+            readonly model: string;
+            readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+        }[];
+    };
+    readonly policy: {
+        readonly activeDimensionThreshold: number;
+        readonly maximumUnknownProbability: number;
+        readonly minimumDominanceMargin: number;
+        readonly minimumCasesPerDimension: number;
+    };
+    readonly budget: {
+        readonly semanticModelCalls: number;
+        readonly plannedModelCalls: number;
+        readonly maximumTransientRetries: number;
+        readonly requiredTotalInputTokens: number;
+        readonly requiredTotalOutputTokens: number;
+        readonly perCallOutputTokens: number;
+        readonly wallTimeMs: number;
+    };
+}, RepositoryFoundryError, never>;
+export declare const executeRepositoryRoutingLabelingV1: (input: {
+    readonly plan: RepositoryRoutingLabelingPlanV1;
+    readonly benchmarkCase: RepositoryBenchmarkCaseV1;
+}) => Effect.Effect<{
+    version: 1;
+    plan: {
+        readonly version: 1;
+        readonly planDigest: string;
+        readonly foundryPlanDigest: string;
+        readonly caseId: string;
+        readonly behavioralFingerprint: string;
+        readonly basis: {
+            readonly version: 3;
+            readonly basisDigest: string;
+            readonly dimensions: readonly {
+                readonly id: string;
+                readonly name: string;
+                readonly description: string;
+                readonly inclusions: readonly string[];
+                readonly exclusions: readonly string[];
+                readonly confusableDimensionIds: readonly string[];
+                readonly boundaryRules: readonly string[];
+                readonly repositoryFacets: {
+                    readonly componentIds: readonly string[];
+                    readonly pathAnchors: readonly string[];
+                    readonly symbolAnchors: readonly string[];
+                };
+                readonly positiveExamples: readonly string[];
+                readonly negativeExamples: readonly string[];
+                readonly codeSummaries: readonly string[];
+                readonly representativeSnippets: readonly string[];
+            }[];
+            readonly repositoryDigest: string;
+            readonly basisId: string;
+            readonly facetSnapshot: {
+                readonly repositoryId: string;
+                readonly repositoryDigest: string;
+                readonly name: string;
+                readonly purpose: string;
+                readonly languages: readonly string[];
+                readonly frameworks: readonly string[];
+                readonly components: readonly {
+                    readonly id: string;
+                    readonly name: string;
+                    readonly purpose: string;
+                    readonly paths: readonly string[];
+                }[];
+            };
+            readonly provenance: {
+                readonly recipeId: string;
+                readonly generatorVersion: string;
+                readonly sourceDigests: readonly string[];
+                readonly authorModel?: string | undefined;
+                readonly constructionTaskDigest?: string | undefined;
+                readonly origin?: {
+                    readonly kind: "generated" | "incumbent-enriched" | "repair";
+                    readonly sourceBasisId?: string | undefined;
+                    readonly sourceBasisVersion?: number | undefined;
+                } | undefined;
+            };
+            readonly validation?: {
+                readonly validationDatasetDigest: string;
+                readonly caseCount: number;
+                readonly activeMae: number;
+                readonly activeF1: number;
+                readonly topDimensionAgreement: number;
+                readonly unknownMae: number;
+                readonly referenceUnknownRate: number;
+                readonly selectionScore: number;
+            } | undefined;
+        };
+        readonly modelPlan: {
+            readonly version: 1;
+            readonly assignments: readonly {
+                readonly role: "dimension-labeler" | "dimension-critic-terra" | "dimension-critic-luna";
+                readonly model: string;
+                readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+            }[];
+        };
+        readonly policy: {
+            readonly activeDimensionThreshold: number;
+            readonly maximumUnknownProbability: number;
+            readonly minimumDominanceMargin: number;
+            readonly minimumCasesPerDimension: number;
+        };
+        readonly budget: {
+            readonly semanticModelCalls: number;
+            readonly plannedModelCalls: number;
+            readonly maximumTransientRetries: number;
+            readonly requiredTotalInputTokens: number;
+            readonly requiredTotalOutputTokens: number;
+            readonly perCallOutputTokens: number;
+            readonly wallTimeMs: number;
+        };
+    };
+    labeling: {
+        version: 1;
+        reviewed: {
+            readonly version: 1;
+            readonly caseId: string;
+            readonly label: {
+                readonly caseId: string;
+                readonly basisDigest: string;
+                readonly scores: readonly {
+                    readonly dimensionId: string;
+                    readonly score: number;
+                }[];
+                readonly unknownProbability: number;
+            };
+            readonly basisDigest: string;
+            readonly authorModel: string;
+            readonly reviews: readonly {
+                readonly outcome: "approve" | "reject";
+                readonly detail: string;
+                readonly model: string;
+                readonly evidenceIds: readonly string[];
+                readonly reviewerId: string;
+                readonly primaryDimensionId: string;
+                readonly independentLabel: {
+                    readonly caseId: string;
+                    readonly basisDigest: string;
+                    readonly scores: readonly {
+                        readonly dimensionId: string;
+                        readonly score: number;
+                    }[];
+                    readonly unknownProbability: number;
+                };
+                readonly callId?: string | undefined;
+            }[];
+            readonly authorEvidenceIds: readonly string[];
+            readonly authorDetail: string;
+            readonly authorCallId?: string | undefined;
+        };
+        modelCalls: {
+            readonly operationId: string;
+            readonly role: "dimension-labeler" | "dimension-critic-terra" | "dimension-critic-luna";
+            readonly model: string;
+            readonly reasoningEffort: import("../../repository-language-model-protocol.js").RepositoryFoundryReasoningEffortV1;
+            readonly callId?: string;
+        }[];
+    };
+    routing: {
+        readonly version: 1;
+        readonly basis: {
+            readonly version: 3;
+            readonly basisDigest: string;
+            readonly dimensions: readonly {
+                readonly id: string;
+                readonly name: string;
+                readonly description: string;
+                readonly inclusions: readonly string[];
+                readonly exclusions: readonly string[];
+                readonly confusableDimensionIds: readonly string[];
+                readonly boundaryRules: readonly string[];
+                readonly repositoryFacets: {
+                    readonly componentIds: readonly string[];
+                    readonly pathAnchors: readonly string[];
+                    readonly symbolAnchors: readonly string[];
+                };
+                readonly positiveExamples: readonly string[];
+                readonly negativeExamples: readonly string[];
+                readonly codeSummaries: readonly string[];
+                readonly representativeSnippets: readonly string[];
+            }[];
+            readonly repositoryDigest: string;
+            readonly basisId: string;
+            readonly facetSnapshot: {
+                readonly repositoryId: string;
+                readonly repositoryDigest: string;
+                readonly name: string;
+                readonly purpose: string;
+                readonly languages: readonly string[];
+                readonly frameworks: readonly string[];
+                readonly components: readonly {
+                    readonly id: string;
+                    readonly name: string;
+                    readonly purpose: string;
+                    readonly paths: readonly string[];
+                }[];
+            };
+            readonly provenance: {
+                readonly recipeId: string;
+                readonly generatorVersion: string;
+                readonly sourceDigests: readonly string[];
+                readonly authorModel?: string | undefined;
+                readonly constructionTaskDigest?: string | undefined;
+                readonly origin?: {
+                    readonly kind: "generated" | "incumbent-enriched" | "repair";
+                    readonly sourceBasisId?: string | undefined;
+                    readonly sourceBasisVersion?: number | undefined;
+                } | undefined;
+            };
+            readonly validation?: {
+                readonly validationDatasetDigest: string;
+                readonly caseCount: number;
+                readonly activeMae: number;
+                readonly activeF1: number;
+                readonly topDimensionAgreement: number;
+                readonly unknownMae: number;
+                readonly referenceUnknownRate: number;
+                readonly selectionScore: number;
+            } | undefined;
+        };
+        readonly policy: {
+            readonly activeDimensionThreshold: number;
+            readonly maximumUnknownProbability: number;
+            readonly minimumDominanceMargin: number;
+            readonly minimumCasesPerDimension: number;
+        };
+        readonly assignments: readonly {
+            readonly caseId: string;
+            readonly rejectionReasons: readonly string[];
+            readonly basisDigest: string;
+            readonly eligible: boolean;
+            readonly unknownProbability: number;
+            readonly labelDigest: string;
+            readonly primaryScore: number;
+            readonly secondaryScore: number;
+            readonly primaryDimensionId?: string | undefined;
+        }[];
+        readonly dimensions: readonly {
+            readonly dimensionId: string;
+            readonly caseIds: readonly string[];
+            readonly minimumRequired: number;
+            readonly routingReady: boolean;
+        }[];
+        readonly eligibleCaseIds: readonly string[];
+        readonly libraryOnlyCaseIds: readonly string[];
+        readonly routingReady: boolean;
+        readonly coverageGaps: readonly string[];
+    };
+}, RepositoryFoundryError, RepositoryFoundryLanguageModel>;
+export declare const makeRoutingBenchmark: Effect.Effect<RoutingBenchmarkShape, never, RepositoryFoundryLanguageModel>;
+export declare const RoutingBenchmarkLive: Layer.Layer<RoutingBenchmark, never, RepositoryFoundryLanguageModel>;
+export {};

package/dist/services/routing-benchmark/service.js

@@ -0,0 +1,96 @@
+import { assertRoutingBasisV3 } from "@velum-labs/routekit-eval-contracts";
+import { Context, Effect, Layer, Schema } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import { compileValidatedBenchmarkLibraryV1 } from "../../repository-quality-policy.js";
+import { assertRepositoryRoutingEligibilityPolicyV1 } from "../../repository-routing-benchmark-protocol.js";
+import { compileRepositoryRoutingBenchmarkV1 } from "../../repository-routing-quality-policy.js";
+import { assertRepositoryRoutingModelPlanV1 } from "../../repository-routing-model-protocol.js";
+import { assertRepositoryRoutingLabelingPlanV1, RepositoryRoutingLabelingPlanV1, repositoryRoutingLabelingBudgetV1, repositoryRoutingLabelingPlanDigestV1 } from "../../repository-routing-plan-protocol.js";
+import { authorRepositoryReviewedDimensionLabelV1 } from "../dimension-labeling/service.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+export class RoutingBenchmark extends Context.Service()("@velum-labs/routekit-eval-setup/RoutingBenchmark") {
+}
+const failure = (operation, detail, cause) => new RepositoryFoundryError({
+    operation,
+    detail,
+    ...(cause === undefined ? {} : { cause })
+});
+export const planRepositoryRoutingLabelingV1 = Effect.fn("RoutingBenchmark.planLabeling")(function* (input) {
+    yield* Effect.try({
+        try: () => {
+            assertRoutingBasisV3(input.basis);
+            assertRepositoryRoutingModelPlanV1(input.modelPlan);
+            assertRepositoryRoutingEligibilityPolicyV1(input.policy);
+            if (input.foundryPlanDigest.trim().length === 0 ||
+                input.benchmarkCase.status !== "valid-library") {
+                throw new Error("routing labeling requires the exact foundry plan and an admitted benchmark-library case");
+            }
+        },
+        catch: (cause) => failure("plan-generation", "repository routing labeling request failed validation", cause)
+    });
+    return yield* Effect.try({
+        try: () => {
+            // Validation decodes the persisted schema before checking its digest.
+            // Normalize at creation too: valid JSON property order must not create
+            // an immediately stale plan. Keep the existing V1 digest representation.
+            const { planDigest: _unsetDigest, ...withoutDigest } = Schema.decodeUnknownSync(RepositoryRoutingLabelingPlanV1)({
+                version: 1,
+                planDigest: "",
+                foundryPlanDigest: input.foundryPlanDigest,
+                caseId: input.benchmarkCase.id,
+                behavioralFingerprint: input.benchmarkCase.behavioralFingerprint,
+                basis: input.basis,
+                modelPlan: input.modelPlan,
+                policy: input.policy,
+                budget: repositoryRoutingLabelingBudgetV1()
+            });
+            const plan = {
+                ...withoutDigest,
+                planDigest: repositoryRoutingLabelingPlanDigestV1(withoutDigest)
+            };
+            assertRepositoryRoutingLabelingPlanV1(plan);
+            return plan;
+        },
+        catch: (cause) => failure("plan-generation", "repository routing labeling plan failed validation", cause)
+    });
+});
+export const executeRepositoryRoutingLabelingV1 = Effect.fn("RoutingBenchmark.executeLabeling")(function* (input) {
+    yield* Effect.try({
+        try: () => {
+            assertRepositoryRoutingLabelingPlanV1(input.plan);
+            if (input.benchmarkCase.id !== input.plan.caseId ||
+                input.benchmarkCase.behavioralFingerprint !== input.plan.behavioralFingerprint ||
+                input.benchmarkCase.status !== "valid-library") {
+                throw new Error("benchmark case changed after the immutable routing plan was created");
+            }
+        },
+        catch: (cause) => failure("execute-generation", "repository routing labeling plan is stale", cause)
+    });
+    const labeling = yield* authorRepositoryReviewedDimensionLabelV1({
+        operationId: `repository-routing-${input.plan.planDigest.slice(0, 20)}`,
+        basis: input.plan.basis,
+        benchmarkCase: input.benchmarkCase,
+        modelPlan: input.plan.modelPlan
+    });
+    const library = compileValidatedBenchmarkLibraryV1([input.benchmarkCase]);
+    const routing = compileRepositoryRoutingBenchmarkV1({
+        basis: input.plan.basis,
+        library,
+        labels: [labeling.reviewed],
+        policy: input.plan.policy
+    });
+    return {
+        version: 1,
+        plan: input.plan,
+        labeling,
+        routing
+    };
+});
+export const makeRoutingBenchmark = Effect.gen(function* () {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    return RoutingBenchmark.of({
+        planLabeling: planRepositoryRoutingLabelingV1,
+        executeLabeling: (input) => executeRepositoryRoutingLabelingV1(input).pipe(Effect.provideService(RepositoryFoundryLanguageModel, languageModel))
+    });
+});
+export const RoutingBenchmarkLive = Layer.effect(RoutingBenchmark, makeRoutingBenchmark);

package/dist/services/specification-critic/service.d.ts

@@ -0,0 +1,92 @@
+import { Context, Effect, Layer } from "effect";
+import type { RepositoryCandidateVisibleTaskV1 } from "../../repository-benchmark-protocol.js";
+import { type RepositorySpecificationCritiqueV1, type RepositorySpecificationReviewV1 } from "../../repository-specification-critique-protocol.js";
+import type { RepositoryTaskFamilyV1 } from "../../repository-task-family-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+export type SpecificationCriticShape = {
+    readonly critique: (input: {
+        readonly repositoryRoot: string;
+        readonly seed: RepositoryTaskSeedV1;
+        readonly family: RepositoryTaskFamilyV1;
+        readonly visible: RepositoryCandidateVisibleTaskV1;
+        readonly reviews: readonly RepositorySpecificationReviewV1[];
+        readonly requestedDimension?: string;
+    }) => Effect.Effect<RepositorySpecificationCritiqueV1, import("../../errors.js").RepositoryFoundryError>;
+};
+declare const SpecificationCritic_base: Context.ServiceClass<SpecificationCritic, "@velum-labs/routekit-eval-setup/SpecificationCritic", SpecificationCriticShape>;
+export declare class SpecificationCritic extends SpecificationCritic_base {
+}
+export declare const critiqueRepositoryTaskSpecificationV1: (input: {
+    readonly repositoryRoot: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly family: RepositoryTaskFamilyV1;
+    readonly visible: RepositoryCandidateVisibleTaskV1;
+    readonly reviews: readonly RepositorySpecificationReviewV1[];
+    readonly requestedDimension?: string;
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly caseId: string;
+    readonly outcome: "pass" | "fail" | "review-required";
+    readonly mechanicalFindings: readonly {
+        readonly axis: "missing-precondition" | "unavailable-context" | "multiple-interpretations" | "hidden-answer-leakage" | "unrealistic-phrasing" | "unsupported-assumption";
+        readonly severity: "warning" | "blocker";
+        readonly detail: string;
+    }[];
+    readonly reviews: readonly {
+        readonly detail: string;
+        readonly findings: readonly {
+            readonly axis: "missing-precondition" | "unavailable-context" | "multiple-interpretations" | "hidden-answer-leakage" | "unrealistic-phrasing" | "unsupported-assumption";
+            readonly severity: "warning" | "blocker";
+            readonly detail: string;
+        }[];
+        readonly verdict: "sufficient" | "insufficient";
+        readonly reviewerId: string;
+        readonly independentlyProduced: boolean;
+        readonly reviewerKind: "independent-solver" | "human-reviewer" | "model-critic";
+        readonly criticalBehaviorCoverage?: readonly {
+            readonly behaviorId: string;
+            readonly outcome: "covered" | "uncertain" | "omitted";
+            readonly detail: string;
+        }[] | undefined;
+        readonly dimensionFit?: {
+            readonly requestedDimension: string;
+            readonly outcome: "uncertain" | "fit" | "mismatch";
+            readonly detail: string;
+            readonly evidencePaths: readonly string[];
+        } | undefined;
+        readonly referenceCompatibility?: {
+            readonly outcome: "uncertain" | "contradictory" | "compatible";
+            readonly detail: string;
+            readonly scopeCoverage: readonly {
+                readonly scopeId: string;
+                readonly outcome: "supported" | "uncertain" | "contradictory";
+                readonly detail: string;
+                readonly evidence: readonly {
+                    readonly sourceId: string;
+                    readonly startLine: number;
+                    readonly endLine: number;
+                }[];
+            }[];
+            readonly questionResolutions?: readonly {
+                readonly outcome: "unresolved" | "answered" | "not-required";
+                readonly detail: string;
+                readonly evidence: readonly {
+                    readonly sourceId: string;
+                    readonly startLine: number;
+                    readonly endLine: number;
+                }[];
+                readonly affectedScopeIds: readonly string[];
+                readonly questionId: string;
+                readonly visibleScopeExclusion?: {
+                    readonly quotation: string;
+                    readonly rationale: string;
+                } | undefined;
+            }[] | undefined;
+        } | undefined;
+    }[];
+    readonly independentApprovalCount: number;
+    readonly rejectionReasons: readonly string[];
+}, import("../../errors.js").RepositoryFoundryError, never>;
+export declare const makeSpecificationCritic: Effect.Effect<SpecificationCriticShape, never, never>;
+export declare const SpecificationCriticLive: Layer.Layer<SpecificationCritic, never, never>;
+export {};