@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/oracle-coverage/service.d.ts

@@ -0,0 +1,28 @@
+import { Effect } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import type { RepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+import type { RepositoryFoundryModelPlanV1 } from "../../repository-language-model-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+export declare const reviewRepositoryOracleCoverageV1: (input: {
+    readonly operationId: string;
+    readonly modelPlan: RepositoryFoundryModelPlanV1;
+    readonly scope: readonly {
+        readonly id: string;
+        readonly description: string;
+    }[];
+    readonly suite: RepositoryHiddenFixtureSuiteV1;
+    readonly evidence: Readonly<Record<string, unknown>>;
+}) => Effect.Effect<import("../language-model/service.js").RepositoryFoundryStructuredGenerationV1<{
+    readonly verdict: "approve" | "revise";
+    readonly detail: string;
+    readonly scopeCoverage: readonly {
+        readonly scopeId: string;
+        readonly outcome: "covered" | "gap" | "uncertain";
+        readonly detail: string;
+        readonly counterexample: string;
+        readonly assertionEvidence: readonly {
+            readonly fixtureId: string;
+            readonly excerpt: string;
+        }[];
+    }[];
+}>, RepositoryFoundryError, RepositoryFoundryLanguageModel>;

package/dist/services/oracle-coverage/service.js

@@ -0,0 +1,50 @@
+import { Effect } from "effect";
+import { RepositoryFoundryError } from "../../errors.js";
+import { repositoryFoundryRoleOutputCeilingV1 } from "../../repository-foundry-plan-protocol.js";
+import { assertRepositoryOracleCoverageReviewV1, RepositoryOracleCoverageReviewV1 } from "../../repository-oracle-coverage-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+const INSTRUCTIONS = `You independently review executable oracle coverage before the oracle is frozen.
+Inspect the complete supplied scope, visible contract, pinned repository sources and current fixture overlays.
+The reference preflight establishes executability, not coverage. Do not trust the author's scopeCoverage labels.
+For every supplied scope clause, ask whether an otherwise correct implementation could violate only that clause
+and still pass every current fixture. Analyze the actual inputs, setup, assertions and tested paths. A compound
+wrong patch can fail for an unrelated unfixed bug; that does not prove the intended regression is detected.
+Preservation clauses need observable assertions under the conditions that could change them, including
+interactions between newly supported inputs and existing state, deadlines, eligibility or precedence. A happy-path
+assertion outside those conditions does not establish preservation. Check cross-file behavior and relevant
+input/state partitions, not just a single historical example.
+Return exactly one scopeCoverage entry for every scope id. Mark covered only when the actual assertions
+distinguish an isolated violation; cite short verbatim excerpts from the corresponding supplied fixture overlay
+and explain why its setup makes that violation observable. Leave counterexample empty for covered entries.
+For gap or uncertain, give a concrete input/state/behavior scenario that should be added or clarified in
+counterexample. Do not invent behavior beyond the pinned scope or prescribe the reference implementation.
+Approve only when every scope clause is covered. Otherwise request revise. Do not write code or inspect any
+valid solution, adversary, future tournament, or held-out material. Return only the structured review.`;
+export const reviewRepositoryOracleCoverageV1 = Effect.fn("OracleCoverage.review")(function* (input) {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const reviewed = yield* languageModel.generateStructured({
+        plan: input.modelPlan,
+        role: "oracle-critic",
+        operationId: input.operationId,
+        instructions: INSTRUCTIONS,
+        input: {
+            ...input.evidence,
+            behavioralScope: input.scope,
+            fixtureSuite: input.suite
+        },
+        schemaName: "routekit_repository_oracle_coverage_review_v1",
+        outputSchema: RepositoryOracleCoverageReviewV1,
+        maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("oracle-critic")
+    });
+    yield* Effect.try({
+        try: () => assertRepositoryOracleCoverageReviewV1(reviewed.value, input.scope.map((clause) => clause.id), input.suite),
+        catch: (cause) => cause instanceof RepositoryFoundryError
+            ? cause
+            : new RepositoryFoundryError({
+                operation: "compile-reviewed-case",
+                detail: "oracle coverage review could not be validated",
+                cause
+            })
+    });
+    return reviewed;
+});

package/dist/services/oracle-coverage-witness/service.d.ts

@@ -0,0 +1,130 @@
+import { Effect } from "effect";
+import { repositoryCommandTestEvidenceV1 } from "../../adapters/repository-command-evidence.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { type RepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+import type { RepositoryFoundryModelPlanV1 } from "../../repository-language-model-protocol.js";
+import type { RepositoryOracleCoverageReviewV1 } from "../../repository-oracle-coverage-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+type TestEvidence = ReturnType<typeof repositoryCommandTestEvidenceV1>;
+export type RepositoryOracleCoverageWitnessV1 = {
+    readonly scopeId: string;
+    readonly hypothesis: string;
+    readonly outcome: "already-detected" | "confirmed-escape";
+    readonly explanation: string;
+    readonly referenceCommit: string;
+    readonly fileOverrides: readonly {
+        readonly path: string;
+        readonly content: string;
+    }[];
+    readonly fixture: RepositoryHiddenFixtureSuiteV1["fixtures"][number];
+    readonly overlay: RepositoryHiddenFixtureSuiteV1["overlays"][number];
+    readonly referenceEvidence: TestEvidence;
+    readonly mutantEvidence: TestEvidence;
+    readonly incumbentEvidence: readonly {
+        readonly fixtureId: string;
+        readonly evidence: TestEvidence;
+    }[];
+    readonly executionOperationId: string;
+};
+export type RepositoryOracleCoverageWitnessRevisionV1 = {
+    readonly scopeId: string;
+    readonly fixtureId: string;
+    readonly originalExecutionOperationId: string;
+    readonly executionOperationId: string;
+    readonly referenceCommit: string;
+    readonly beforeOverlay: RepositoryHiddenFixtureSuiteV1["overlays"][number];
+    readonly overlay: RepositoryHiddenFixtureSuiteV1["overlays"][number];
+    readonly referenceEvidence: TestEvidence;
+    readonly mutantEvidence?: TestEvidence;
+    readonly status: "validated" | "rejected";
+    readonly detail: string;
+};
+/**
+ * A witness preserves a demonstrated defect-detection obligation, not immutable
+ * test mechanics. This only validates a proposed revision; its caller owns
+ * adoption after the complete candidate tournament succeeds.
+ */
+export declare const validateRepositoryOracleCoverageWitnessRevisionsV1: (input: {
+    readonly operationId: string;
+    readonly repositoryRoot: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly suite: RepositoryHiddenFixtureSuiteV1;
+    readonly retained: readonly {
+        readonly witness: RepositoryOracleCoverageWitnessV1;
+        readonly overlay: RepositoryHiddenFixtureSuiteV1["overlays"][number];
+    }[];
+}) => Effect.Effect<readonly RepositoryOracleCoverageWitnessRevisionV1[], RepositoryFoundryError, never>;
+/** Resolve a fixed set of critic hypotheses through execution, without minting a model approval. */
+export declare const resolveRepositoryOracleCoverageWitnessesV1: (input: {
+    readonly operationId: string;
+    readonly witnessPrefix: string;
+    readonly repositoryRoot: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly suite: RepositoryHiddenFixtureSuiteV1;
+    readonly review: RepositoryOracleCoverageReviewV1;
+    readonly modelPlan: RepositoryFoundryModelPlanV1;
+    readonly allowedTestPaths: ReadonlySet<string>;
+    readonly allowedSolutionPaths: ReadonlySet<string>;
+    readonly referenceSources: readonly {
+        readonly path: string;
+        readonly content: string;
+    }[];
+    readonly context: Readonly<Record<string, unknown>>;
+    readonly maximumAttempts: number;
+    readonly oracleCoverageWitnessVersion?: 1 | 2;
+    readonly oracleRepairFeedbackVersion?: 1;
+    readonly onGenerationAttempt: (attempt: number) => Effect.Effect<void>;
+}) => Effect.Effect<{
+    suite: {
+        fixtures: ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        overlays: {
+            readonly path: string;
+            readonly content: string;
+            readonly fixtureIds: readonly string[];
+        }[];
+        version: 1;
+        caseId: string;
+    };
+    witnesses: RepositoryOracleCoverageWitnessV1[];
+    modelCalls: {
+        readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+        readonly role: "oracle-critic" | "repository-analyst" | "specification-writer" | "specification-critic-a" | "specification-critic-b" | "trajectory-designer" | "trajectory-critic" | "oracle-designer" | "valid-solution-generator-a" | "valid-solution-generator-b" | "adversary" | "solution-critic-a" | "solution-critic-b" | "held-out-adversary" | "held-out-solution-critic-a" | "held-out-solution-critic-b" | "quality-reviewer" | "hill-climb-planner";
+        readonly model: string;
+        readonly operationId: string;
+        readonly callId?: string | undefined;
+    }[];
+}, RepositoryFoundryError, RepositoryFoundryLanguageModel>;
+export {};