@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/pipeline-tournament/service.d.ts

@@ -0,0 +1,258 @@
+import { Effect } from "effect";
+import { type PipelineStageRunV1, type PipelineStageInputV1, type TournamentCheckpointV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+/**
+ * Runs real, repeated isolated execution and hill-climbs only from measured
+ * escapes. Checkpoints are written after every accepted incumbent replacement.
+ */
+export declare const runPipelineTournamentCapabilityV1: (input: PipelineStageInputV1<TournamentCheckpointV1> & {
+    readonly maximumIterations?: number;
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly valid: readonly {
+        readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+        readonly id: string;
+        readonly expectedClass: "valid" | "wrong";
+        readonly commit: string;
+        readonly family: string;
+        readonly fileOverrides?: readonly ({
+            readonly path: string;
+            readonly content: string;
+        } | {
+            readonly path: string;
+            readonly delete: true;
+        })[] | undefined;
+    }[];
+    readonly wrong: readonly {
+        readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+        readonly id: string;
+        readonly expectedClass: "valid" | "wrong";
+        readonly commit: string;
+        readonly family: string;
+        readonly fileOverrides?: readonly ({
+            readonly path: string;
+            readonly content: string;
+        } | {
+            readonly path: string;
+            readonly delete: true;
+        })[] | undefined;
+    }[];
+    readonly stage: "tournament";
+    readonly oracle: {
+        readonly version: 1;
+        readonly fixtures: readonly ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        readonly caseId: string;
+        readonly hardFailurePolicy: "any-critical-hard-clause-fails";
+        readonly clauses: readonly ({
+            readonly kind: "candidate-policy";
+            readonly policyId: string;
+            readonly forbiddenPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "test-command";
+            readonly recipeId: string;
+            readonly expectedExitCode: number;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "json-field";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly operator: "equals" | "present" | "absent";
+            readonly pointer: string;
+            readonly expectedJson?: string | undefined;
+        } | {
+            readonly kind: "ordered-events";
+            readonly id: string;
+            readonly description: string;
+            readonly critical: boolean;
+            readonly eventTypes: readonly string[];
+            readonly terminalEvent?: string | undefined;
+        } | {
+            readonly kind: "property";
+            readonly propertyId: string;
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "differential";
+            readonly implementationIds: readonly string[];
+            readonly fixtureIds: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "architecture";
+            readonly ruleId: string;
+            readonly expectedPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "transformation-completeness";
+            readonly ruleId: string;
+            readonly forbiddenLegacyPaths: readonly string[];
+            readonly requiredNewPaths: readonly string[];
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        } | {
+            readonly kind: "semantic-residual";
+            readonly criterion: string;
+            readonly id: string;
+            readonly critical: boolean;
+            readonly description: string;
+        })[];
+        readonly solutions: readonly {
+            readonly kind: "historical-reference" | "independent-valid" | "simplified-valid" | "behavior-preserving-valid" | "historical-defect" | "mutation" | "model-near-miss" | "reward-hacking";
+            readonly id: string;
+            readonly expectedClass: "valid" | "wrong";
+            readonly family: string;
+            readonly commit?: string | undefined;
+        }[];
+        readonly observations: readonly {
+            readonly outcome: "pass" | "fail" | "unknown";
+            readonly solutionId: string;
+            readonly clauseId: string;
+            readonly repetition: number;
+            readonly detail: string;
+            readonly inconclusiveReason?: "unattributed-test-failure" | undefined;
+        }[];
+        readonly knownLimitations: readonly string[];
+        readonly executionBindingDigest?: string | undefined;
+        readonly executionRepetitions?: number | undefined;
+    };
+    readonly parentDigest: string;
+    readonly createdAt: string;
+    readonly adequacy: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly admitted: boolean;
+        readonly validSolutionCount: number;
+        readonly wrongSolutionCount: number;
+        readonly falseRejectedSolutionIds: readonly string[];
+        readonly falseAcceptedSolutionIds: readonly string[];
+        readonly unstableSolutionIds: readonly string[];
+        readonly unobservedClauseIds: readonly string[];
+        readonly insufficientObservationPairs: readonly string[];
+        readonly mutationFamilyKills: readonly {
+            readonly family: string;
+            readonly killed: number;
+            readonly total: number;
+        }[];
+        readonly rejectionReasons: readonly string[];
+    };
+    readonly suite: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly fixtures: readonly ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        readonly overlays: readonly {
+            readonly path: string;
+            readonly content: string;
+            readonly fixtureIds: readonly string[];
+        }[];
+    };
+    readonly history: readonly {
+        readonly rejectionReasons: readonly string[];
+        readonly sequence: number;
+        readonly decision: "rejected" | "accepted";
+        readonly targeted: readonly string[];
+        readonly candidateAdequacy?: {
+            readonly version: 1;
+            readonly caseId: string;
+            readonly admitted: boolean;
+            readonly validSolutionCount: number;
+            readonly wrongSolutionCount: number;
+            readonly falseRejectedSolutionIds: readonly string[];
+            readonly falseAcceptedSolutionIds: readonly string[];
+            readonly unstableSolutionIds: readonly string[];
+            readonly unobservedClauseIds: readonly string[];
+            readonly insufficientObservationPairs: readonly string[];
+            readonly mutationFamilyKills: readonly {
+                readonly family: string;
+                readonly killed: number;
+                readonly total: number;
+            }[];
+            readonly rejectionReasons: readonly string[];
+        } | undefined;
+    }[];
+    readonly iterations: number;
+    readonly critiques?: readonly {
+        readonly solutionId: string;
+        readonly classification: "valid" | "wrong" | "uncertain";
+        readonly detail: string;
+        readonly violatedBehaviorIds: readonly string[];
+    }[] | undefined;
+}, RepositoryFoundryError | import("../../case-pipeline-protocol.js").PipelineBudgetExhaustedError, RepositoryFoundryLanguageModel>;
+export declare const runPipelineTournamentStageV1: PipelineStageRunV1<TournamentCheckpointV1>;