@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/pipeline-challenge/service.d.ts

@@ -0,0 +1,551 @@
+import { Effect } from "effect";
+import { type AdmitCheckpointV1, type ChallengeCheckpointV1, type PipelineStageRunV1, type PipelineStageInputV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+type MaterializedHeldOut = ChallengeCheckpointV1["heldOut"][number];
+export declare const runPipelineChallengeCapabilityV1: (input: PipelineStageInputV1<ChallengeCheckpointV1> & {
+    readonly materialized?: MaterializedHeldOut;
+    readonly resumeOracle?: ChallengeCheckpointV1["oracle"];
+    readonly onMaterialized?: (solution: MaterializedHeldOut) => Effect.Effect<void>;
+    readonly onOracleProgress?: (solution: MaterializedHeldOut, oracle: ChallengeCheckpointV1["oracle"]) => Effect.Effect<void>;
+}) => Effect.Effect<{
+    readonly heldOut: readonly {
+        readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+        readonly id: string;
+        readonly expectedClass: "valid" | "wrong";
+        readonly commit: string;
+        readonly family: string;
+        readonly fileOverrides?: readonly ({
+            readonly path: string;
+            readonly content: string;
+        } | {
+            readonly path: string;
+            readonly delete: true;
+        })[] | undefined;
+    }[];
+    readonly adequacy: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly admitted: boolean;
+        readonly validSolutionCount: number;
+        readonly wrongSolutionCount: number;
+        readonly falseRejectedSolutionIds: readonly string[];
+        readonly falseAcceptedSolutionIds: readonly string[];
+        readonly unstableSolutionIds: readonly string[];
+        readonly unobservedClauseIds: readonly string[];
+        readonly insufficientObservationPairs: readonly string[];
+        readonly mutationFamilyKills: readonly {
+            readonly family: string;
+            readonly killed: number;
+            readonly total: number;
+        }[];
+        readonly rejectionReasons: readonly string[];
+    };
+    readonly oracle: {
+        readonly version: 1;
+        readonly fixtures: readonly ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        readonly caseId: string;
+        readonly hardFailurePolicy: "any-critical-hard-clause-fails";
+        readonly clauses: readonly ({
+            readonly kind: "candidate-policy";
+            readonly policyId: string;
+            readonly forbiddenPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "test-command";
+            readonly recipeId: string;
+            readonly expectedExitCode: number;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "json-field";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly operator: "equals" | "present" | "absent";
+            readonly pointer: string;
+            readonly expectedJson?: string | undefined;
+        } | {
+            readonly kind: "ordered-events";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly eventTypes: readonly string[];
+            readonly terminalEvent?: string | undefined;
+        } | {
+            readonly kind: "property";
+            readonly propertyId: string;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "differential";
+            readonly implementationIds: readonly string[];
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "architecture";
+            readonly ruleId: string;
+            readonly expectedPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "transformation-completeness";
+            readonly ruleId: string;
+            readonly forbiddenLegacyPaths: readonly string[];
+            readonly requiredNewPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "semantic-residual";
+            readonly criterion: string;
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        })[];
+        readonly solutions: readonly {
+            readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+            readonly id: string;
+            readonly expectedClass: "valid" | "wrong";
+            readonly family: string;
+            readonly commit?: string | undefined;
+        }[];
+        readonly observations: readonly {
+            readonly outcome: "pass" | "fail" | "unknown";
+            readonly solutionId: string;
+            readonly clauseId: string;
+            readonly repetition: number;
+            readonly detail: string;
+            readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
+        }[];
+        readonly knownLimitations: readonly string[];
+        readonly executionBindingDigest?: string | undefined;
+        readonly executionRepetitions?: number | undefined;
+    };
+    readonly verdict: "passed" | "failed";
+    readonly fold: number;
+    readonly version: 1;
+    readonly stage: "challenge";
+    readonly parentDigest: string;
+    readonly createdAt: string;
+}, RepositoryFoundryError | import("../../case-pipeline-protocol.js").PipelineBudgetExhaustedError, RepositoryFoundryLanguageModel>;
+export declare const runPipelineAdmitStageV1: PipelineStageRunV1<AdmitCheckpointV1>;
+export declare const runPipelineChallengeStageV1: PipelineStageRunV1<ChallengeCheckpointV1>;
+export declare const runPipelineChallengeV1: PipelineStageRunV1<{
+    readonly heldOut: readonly {
+        readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+        readonly id: string;
+        readonly expectedClass: "valid" | "wrong";
+        readonly commit: string;
+        readonly family: string;
+        readonly fileOverrides?: readonly ({
+            readonly path: string;
+            readonly content: string;
+        } | {
+            readonly path: string;
+            readonly delete: true;
+        })[] | undefined;
+    }[];
+    readonly adequacy: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly admitted: boolean;
+        readonly validSolutionCount: number;
+        readonly wrongSolutionCount: number;
+        readonly falseRejectedSolutionIds: readonly string[];
+        readonly falseAcceptedSolutionIds: readonly string[];
+        readonly unstableSolutionIds: readonly string[];
+        readonly unobservedClauseIds: readonly string[];
+        readonly insufficientObservationPairs: readonly string[];
+        readonly mutationFamilyKills: readonly {
+            readonly family: string;
+            readonly killed: number;
+            readonly total: number;
+        }[];
+        readonly rejectionReasons: readonly string[];
+    };
+    readonly oracle: {
+        readonly version: 1;
+        readonly fixtures: readonly ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        readonly caseId: string;
+        readonly hardFailurePolicy: "any-critical-hard-clause-fails";
+        readonly clauses: readonly ({
+            readonly kind: "candidate-policy";
+            readonly policyId: string;
+            readonly forbiddenPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "test-command";
+            readonly recipeId: string;
+            readonly expectedExitCode: number;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "json-field";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly operator: "equals" | "present" | "absent";
+            readonly pointer: string;
+            readonly expectedJson?: string | undefined;
+        } | {
+            readonly kind: "ordered-events";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly eventTypes: readonly string[];
+            readonly terminalEvent?: string | undefined;
+        } | {
+            readonly kind: "property";
+            readonly propertyId: string;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "differential";
+            readonly implementationIds: readonly string[];
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "architecture";
+            readonly ruleId: string;
+            readonly expectedPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "transformation-completeness";
+            readonly ruleId: string;
+            readonly forbiddenLegacyPaths: readonly string[];
+            readonly requiredNewPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "semantic-residual";
+            readonly criterion: string;
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        })[];
+        readonly solutions: readonly {
+            readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+            readonly id: string;
+            readonly expectedClass: "valid" | "wrong";
+            readonly family: string;
+            readonly commit?: string | undefined;
+        }[];
+        readonly observations: readonly {
+            readonly outcome: "pass" | "fail" | "unknown";
+            readonly solutionId: string;
+            readonly clauseId: string;
+            readonly repetition: number;
+            readonly detail: string;
+            readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
+        }[];
+        readonly knownLimitations: readonly string[];
+        readonly executionBindingDigest?: string | undefined;
+        readonly executionRepetitions?: number | undefined;
+    };
+    readonly verdict: "passed" | "failed";
+    readonly fold: number;
+    readonly version: 1;
+    readonly stage: "challenge";
+    readonly parentDigest: string;
+    readonly createdAt: string;
+}>;
+export declare const runPipelineAdmitV1: PipelineStageRunV1<{
+    readonly benchmarkCase: {
+        readonly id: string;
+        readonly version: 1;
+        readonly rejectionReasons: readonly string[];
+        readonly status: "rejected" | "valid-library" | "routing-eligible" | "routing-active";
+        readonly seedId: string;
+        readonly lane: "request" | "repository-agent";
+        readonly qualityFindings: readonly {
+            readonly axis: "coverage" | "stability" | "authenticity" | "specification-sufficiency" | "oracle-adequacy" | "distinctness" | "realism" | "leakage" | "triviality";
+            readonly outcome: "pass" | "fail" | "review-required";
+            readonly detail: string;
+            readonly evidence: readonly {
+                readonly kind: "symbol" | "package" | "public-surface" | "protocol" | "schema" | "test" | "command" | "history-episode" | "workload-signal" | "documentation";
+                readonly id: string;
+                readonly path?: string | undefined;
+            }[];
+            readonly evidenceIds: readonly string[];
+        }[];
+        readonly capabilityId: string;
+        readonly familyId: string;
+        readonly visible: {
+            readonly lane: "request";
+            readonly id: string;
+            readonly request: {
+                readonly endpoint: "chat" | "responses" | "anthropic";
+                readonly method: "POST";
+                readonly path: string;
+                readonly headers: readonly {
+                    readonly name: string;
+                    readonly value: string;
+                }[];
+                readonly bodyJson: string;
+            };
+        } | {
+            readonly lane: "repository-agent";
+            readonly id: string;
+            readonly requestText: string;
+            readonly checkoutCommit: string;
+            readonly allowedTools: readonly ("test" | "read" | "search" | "edit" | "shell")[];
+            readonly constraints: readonly string[];
+        };
+        readonly hidden: {
+            readonly version: 1;
+            readonly fixtures: readonly ({
+                readonly kind: "historical-regression";
+                readonly expectationMode: "changes";
+                readonly id: string;
+                readonly description: string;
+                readonly expectedBehavior: string;
+                readonly source: "historical" | "generated" | "reviewed";
+                readonly testPath: string;
+            } | {
+                readonly kind: "boundary";
+                readonly expectationMode: "changes" | "preserved";
+                readonly id: string;
+                readonly description: string;
+                readonly expectedBehavior: string;
+                readonly source: "historical" | "generated" | "reviewed";
+                readonly testPath: string;
+            } | {
+                readonly kind: "counterfactual";
+                readonly expectationMode: "changes";
+                readonly id: string;
+                readonly description: string;
+                readonly expectedBehavior: string;
+                readonly source: "historical" | "generated" | "reviewed";
+                readonly testPath: string;
+            } | {
+                readonly kind: "metamorphic";
+                readonly expectationMode: "preserved";
+                readonly id: string;
+                readonly description: string;
+                readonly expectedBehavior: string;
+                readonly source: "historical" | "generated" | "reviewed";
+                readonly testPath: string;
+            })[];
+            readonly caseId: string;
+            readonly hardFailurePolicy: "any-critical-hard-clause-fails";
+            readonly clauses: readonly ({
+                readonly kind: "candidate-policy";
+                readonly policyId: string;
+                readonly forbiddenPaths: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "test-command";
+                readonly recipeId: string;
+                readonly expectedExitCode: number;
+                readonly fixtureIds: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "json-field";
+                readonly id: string;
+                readonly description: string;
+                readonly critical: boolean;
+                readonly operator: "equals" | "present" | "absent";
+                readonly pointer: string;
+                readonly expectedJson?: string | undefined;
+            } | {
+                readonly kind: "ordered-events";
+                readonly id: string;
+                readonly description: string;
+                readonly critical: boolean;
+                readonly eventTypes: readonly string[];
+                readonly terminalEvent?: string | undefined;
+            } | {
+                readonly kind: "property";
+                readonly propertyId: string;
+                readonly fixtureIds: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "differential";
+                readonly implementationIds: readonly string[];
+                readonly fixtureIds: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "architecture";
+                readonly ruleId: string;
+                readonly expectedPaths: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "transformation-completeness";
+                readonly ruleId: string;
+                readonly forbiddenLegacyPaths: readonly string[];
+                readonly requiredNewPaths: readonly string[];
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            } | {
+                readonly kind: "semantic-residual";
+                readonly criterion: string;
+                readonly id: string;
+                readonly critical: boolean;
+                readonly description: string;
+            })[];
+            readonly solutions: readonly {
+                readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+                readonly id: string;
+                readonly expectedClass: "valid" | "wrong";
+                readonly family: string;
+                readonly commit?: string | undefined;
+            }[];
+            readonly observations: readonly {
+                readonly outcome: "pass" | "fail" | "unknown";
+                readonly solutionId: string;
+                readonly clauseId: string;
+                readonly repetition: number;
+                readonly detail: string;
+                readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
+            }[];
+            readonly knownLimitations: readonly string[];
+            readonly executionBindingDigest?: string | undefined;
+            readonly executionRepetitions?: number | undefined;
+        };
+        readonly adequacy: {
+            readonly version: 1;
+            readonly caseId: string;
+            readonly admitted: boolean;
+            readonly validSolutionCount: number;
+            readonly wrongSolutionCount: number;
+            readonly falseRejectedSolutionIds: readonly string[];
+            readonly falseAcceptedSolutionIds: readonly string[];
+            readonly unstableSolutionIds: readonly string[];
+            readonly unobservedClauseIds: readonly string[];
+            readonly insufficientObservationPairs: readonly string[];
+            readonly mutationFamilyKills: readonly {
+                readonly family: string;
+                readonly killed: number;
+                readonly total: number;
+            }[];
+            readonly rejectionReasons: readonly string[];
+        };
+        readonly behavioralFingerprint: string;
+        readonly trajectoryPolicy?: {
+            readonly version: 1;
+            readonly caseId: string;
+            readonly purpose: "routing-signal";
+            readonly validationRecipeIds: readonly string[];
+            readonly protectedPaths: readonly string[];
+            readonly thresholds: {
+                readonly maximumToolCalls: number;
+                readonly maximumFailedToolCalls: number;
+                readonly maximumRepeatedFailedActions: number;
+            };
+            readonly requirements: {
+                readonly passingValidationAfterLastMutation: boolean;
+                readonly noProtectedPathMutations: boolean;
+            };
+            readonly rationale: string;
+        } | undefined;
+    };
+    readonly verdict: "admitted" | "rejected";
+    readonly reasons: readonly string[];
+    readonly evidence: {
+        readonly validCount: number;
+        readonly wrongCount: number;
+        readonly heldOutWrongCount: number;
+        readonly semanticKills: readonly string[];
+    };
+    readonly version: 1;
+    readonly stage: "admit";
+    readonly parentDigest: string;
+    readonly createdAt: string;
+}>;
+export {};