@velum-labs/routekit-eval-setup 1.4.0 → 1.5.0

package/dist/services/eval-environment/service.d.ts

@@ -0,0 +1,173 @@
+import { Effect } from "effect";
+import { type AgenticEnvironmentPlanV1 } from "../../agentic-capabilities-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import type { RepositorySeedEnvironmentV1, RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+export type EvalEnvironmentV1 = {
+    readonly plan: AgenticEnvironmentPlanV1;
+    readonly environment: RepositorySeedEnvironmentV1;
+};
+/** Read-only planning. Execution is explicit and happens only when probing or qualifying. */
+export declare const planEvalEnvironmentV1: (input: {
+    readonly repositoryRoot: string;
+    readonly candidate: string;
+    readonly revision: string;
+    readonly sourceDigest: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly recipeIds: readonly string[];
+    readonly executionMode: AgenticEnvironmentPlanV1["executionMode"];
+    readonly requiredArtifacts?: readonly string[];
+    readonly runtime: AgenticEnvironmentPlanV1["runtime"];
+    readonly justification: string;
+}) => Effect.Effect<{
+    plan: {
+        readonly version: 1;
+        readonly planId: string;
+        readonly candidate: string;
+        readonly revision: string;
+        readonly sourceDigest: string;
+        readonly commitSha: string;
+        readonly recipeIds: readonly string[];
+        readonly executionMode: "source" | "emitted" | "repository-default";
+        readonly targetRoots: readonly string[];
+        readonly dependencyClosure: readonly string[];
+        readonly requiredArtifacts: readonly string[];
+        readonly lockfileDigest: string | null;
+        readonly packageManager: {
+            readonly name: string;
+            readonly version: string;
+        } | null;
+        readonly runtime: {
+            readonly name: string;
+            readonly version: string;
+            readonly platform: string;
+            readonly architecture: string;
+        };
+        readonly commands: readonly {
+            readonly id: string;
+            readonly purpose: "generate" | "build" | "grade" | "install" | "probe";
+            readonly executable: string;
+            readonly args: readonly string[];
+            readonly cwd: string | null;
+            readonly timeoutMs: number;
+        }[];
+        readonly expectedCost: {
+            readonly wallTimeMs: number;
+            readonly spendUsd: string | null;
+        };
+        readonly justification: string;
+        readonly justificationEvidence: readonly string[];
+    };
+    environment: {
+        readonly protectedControlPaths: readonly string[];
+        readonly trustedPreparationRecipes: readonly {
+            readonly id: string;
+            readonly kind: "schema" | "test" | "build" | "check" | "custom";
+            readonly cwd: string;
+            readonly executable: string;
+            readonly args: readonly string[];
+            readonly timeoutMs: number;
+        }[];
+        readonly candidateValidationRecipes: readonly {
+            readonly id: string;
+            readonly kind: "schema" | "test" | "build" | "check" | "custom";
+            readonly cwd: string;
+            readonly executable: string;
+            readonly args: readonly string[];
+            readonly timeoutMs: number;
+        }[];
+        readonly baselineGradeRecipes: readonly {
+            readonly id: string;
+            readonly kind: "schema" | "test" | "build" | "check" | "custom";
+            readonly cwd: string;
+            readonly executable: string;
+            readonly args: readonly string[];
+            readonly timeoutMs: number;
+        }[];
+        readonly gradeRecipes: readonly {
+            readonly id: string;
+            readonly kind: "schema" | "test" | "build" | "check" | "custom";
+            readonly cwd: string;
+            readonly executable: string;
+            readonly args: readonly string[];
+            readonly timeoutMs: number;
+        }[];
+        readonly packageManager?: {
+            readonly name: "pnpm";
+            readonly version: string;
+        } | undefined;
+        readonly lockfilePath?: string | undefined;
+    };
+}, RepositoryFoundryError, never>;
+/** Preparation failures are not failing grades; this runs the same isolated runner used by qualification. */
+export declare const probeEvalEnvironmentV1: (input: {
+    readonly repositoryRoot: string;
+    readonly prepared: EvalEnvironmentV1;
+    readonly commit?: string;
+}) => Effect.Effect<{
+    outcome: "completed" | "environment_blocked" | "scientific_rejection";
+    gradeExecuted: boolean;
+    results: readonly {
+        readonly recipeId: string;
+        readonly commit: string;
+        readonly repetition: number;
+        readonly stdout: string;
+        readonly stderr: string;
+        readonly exitCode: number | null;
+        readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+        readonly timedOut: boolean;
+        readonly outputTruncated: boolean;
+        readonly durationMs: number;
+        readonly trustedPreparationResults: readonly {
+            readonly recipeId: string;
+            readonly commit: string;
+            readonly repetition: number;
+            readonly stdout: string;
+            readonly stderr: string;
+            readonly exitCode: number | null;
+            readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+            readonly timedOut: boolean;
+            readonly outputTruncated: boolean;
+            readonly durationMs: number;
+            readonly testReporter?: "vitest-v1" | undefined;
+        }[];
+        readonly candidateValidationResults: readonly {
+            readonly recipeId: string;
+            readonly commit: string;
+            readonly repetition: number;
+            readonly stdout: string;
+            readonly stderr: string;
+            readonly exitCode: number | null;
+            readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+            readonly timedOut: boolean;
+            readonly outputTruncated: boolean;
+            readonly durationMs: number;
+            readonly testReporter?: "vitest-v1" | undefined;
+        }[];
+        readonly gradeExecuted: boolean;
+        readonly testReporter?: "vitest-v1" | undefined;
+        readonly isolationTimings?: {
+            readonly checkoutDurationMs: number;
+            readonly overlayDurationMs: number;
+            readonly cleanupDurationMs: number;
+        } | undefined;
+        readonly preparation?: {
+            readonly kind: "pristine-install-template";
+            readonly templateKey: string;
+            readonly sourceCommit: string;
+            readonly cloneDurationMs: number;
+            readonly sharedResults: readonly {
+                readonly recipeId: string;
+                readonly commit: string;
+                readonly repetition: number;
+                readonly stdout: string;
+                readonly stderr: string;
+                readonly exitCode: number | null;
+                readonly stage: "trusted-preparation" | "candidate-validation" | "grade";
+                readonly timedOut: boolean;
+                readonly outputTruncated: boolean;
+                readonly durationMs: number;
+                readonly testReporter?: "vitest-v1" | undefined;
+            }[];
+        } | undefined;
+    }[];
+}, RepositoryFoundryError, never>;

package/dist/services/eval-environment/service.js

@@ -0,0 +1,127 @@
+import { Effect } from "effect";
+import { agenticDigestV1, decodeAgenticEnvironmentPlanV1 } from "../../agentic-capabilities-protocol.js";
+import { captureGitTreeSnapshotV1, readGitChangeEpisodeV1, readGitTreeFileV1 } from "../../adapters/git-task-history.js";
+import { replayRepositoryCommandsV1 } from "../../adapters/repository-command-runner.js";
+import { repositoryCommandProbeOutcomeV1 } from "../../adapters/repository-command-evidence.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { deriveHistoricalSeedEnvironmentV1 } from "../task-seed/service.js";
+/** Read-only planning. Execution is explicit and happens only when probing or qualifying. */
+export const planEvalEnvironmentV1 = Effect.fn("EvalEnvironment.plan")(function* (input) {
+    const recipes = input.seed.environment.gradeRecipes.filter((recipe) => input.recipeIds.includes(recipe.id));
+    if (recipes.length === 0 || recipes.length !== new Set(input.recipeIds).size) {
+        return yield* new RepositoryFoundryError({
+            operation: "run-command",
+            detail: "environment plan must select known repository-owned grade recipes"
+        });
+    }
+    const testPaths = input.seed.capabilityEvidence
+        .filter((entry) => entry.kind === "test")
+        .flatMap((entry) => entry.path === undefined ? [] : [entry.path]);
+    if (testPaths.some((testPath) => !recipes.some((recipe) => recipe.cwd === "." || testPath.startsWith(`${recipe.cwd}/`)))) {
+        return yield* new RepositoryFoundryError({
+            operation: "run-command",
+            detail: "environment plan must cover every reviewed test path; select the remaining repository-owned recipes or create a separately scoped candidate"
+        });
+    }
+    if (input.seed.referenceCommit === undefined) {
+        return yield* new RepositoryFoundryError({
+            operation: "run-command",
+            detail: "historical environment planning requires a pinned reference change"
+        });
+    }
+    const episode = yield* readGitChangeEpisodeV1({
+        repositoryRoot: input.repositoryRoot,
+        commit: input.seed.referenceCommit
+    });
+    if (episode.parentCommit !== input.seed.initialState.commit ||
+        ("changeEpisodeId" in input.seed.source && episode.id !== input.seed.source.changeEpisodeId)) {
+        return yield* new RepositoryFoundryError({
+            operation: "run-command",
+            detail: "environment plan must bind the same historical change and initial tree as the candidate"
+        });
+    }
+    const environment = yield* deriveHistoricalSeedEnvironmentV1({
+        repositoryRoot: input.repositoryRoot,
+        commit: input.seed.initialState.commit,
+        gradeRecipes: recipes,
+        executionMode: input.executionMode,
+        testPaths,
+        // Capability citations are not the complete Git delta. In particular they
+        // omit changed manifests/config, which must remain implementation inputs
+        // rather than being overwritten as supposedly unchanged protected controls.
+        changedPaths: episode.changedPaths.flatMap(({ path, previousPath }) => previousPath === undefined ? [path] : [previousPath, path])
+    });
+    const snapshot = yield* captureGitTreeSnapshotV1({
+        repositoryRoot: input.repositoryRoot,
+        requestedRef: input.seed.initialState.commit
+    });
+    const lockfile = environment.lockfilePath === undefined
+        ? undefined
+        : yield* readGitTreeFileV1({ snapshot, path: environment.lockfilePath });
+    const command = (purpose, recipe) => ({
+        id: recipe.id,
+        purpose,
+        executable: recipe.executable,
+        args: recipe.args,
+        cwd: recipe.cwd === "." ? null : recipe.cwd,
+        timeoutMs: recipe.timeoutMs
+    });
+    const commands = [
+        ...environment.trustedPreparationRecipes.map((recipe) => command("install", recipe)),
+        ...environment.candidateValidationRecipes.map((recipe) => command("build", recipe)),
+        ...environment.gradeRecipes.map((recipe) => command("grade", recipe))
+    ];
+    const planBody = {
+        version: 1,
+        candidate: input.candidate,
+        revision: input.revision,
+        sourceDigest: input.sourceDigest,
+        commitSha: input.seed.initialState.commit,
+        recipeIds: recipes.map((recipe) => recipe.id),
+        executionMode: input.executionMode,
+        targetRoots: [...new Set(recipes.map((recipe) => recipe.cwd))],
+        dependencyClosure: [
+            ...new Set(environment.candidateValidationRecipes.map((recipe) => recipe.cwd))
+        ],
+        requiredArtifacts: input.requiredArtifacts ?? [],
+        lockfileDigest: lockfile === undefined ? null : agenticDigestV1(lockfile),
+        packageManager: environment.packageManager ?? null,
+        runtime: input.runtime,
+        commands,
+        expectedCost: {
+            wallTimeMs: commands.reduce((sum, entry) => sum + entry.timeoutMs, 0),
+            spendUsd: null
+        },
+        justification: input.justification,
+        justificationEvidence: [input.candidate]
+    };
+    const plan = yield* Effect.try({
+        try: () => decodeAgenticEnvironmentPlanV1({
+            ...planBody,
+            planId: `environment-${agenticDigestV1(planBody)}`
+        }),
+        catch: (cause) => new RepositoryFoundryError({
+            operation: "run-command",
+            detail: "environment plan exceeds the supported bounded protocol",
+            cause
+        })
+    });
+    return { plan, environment };
+});
+/** Preparation failures are not failing grades; this runs the same isolated runner used by qualification. */
+export const probeEvalEnvironmentV1 = Effect.fn("EvalEnvironment.probe")(function* (input) {
+    const results = yield* replayRepositoryCommandsV1({
+        repositoryRoot: input.repositoryRoot,
+        commit: input.commit ?? input.prepared.plan.commitSha,
+        recipes: input.prepared.environment.gradeRecipes,
+        trustedPreparationRecipes: input.prepared.environment.trustedPreparationRecipes,
+        candidateValidationRecipes: input.prepared.environment.candidateValidationRecipes,
+        repetitions: 1
+    });
+    const outcome = repositoryCommandProbeOutcomeV1(results);
+    return {
+        outcome,
+        gradeExecuted: outcome !== "environment_blocked",
+        results
+    };
+});

package/dist/services/evidence-reconstruction/service.d.ts

@@ -0,0 +1,36 @@
+import { Context, Effect } from "effect";
+import type { RepositoryCommandResultV1, RepositoryHistoricalOracleInputV1 } from "../../repository-execution-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import type { RepositoryFoundryGenerationPlanV1 } from "../../repository-foundry-plan-protocol.js";
+import type { RepositoryFoundryAuthoringArtifactV1 } from "../../repository-foundry-progress-protocol.js";
+import type { RepositoryHiddenOracleV1 } from "../../repository-oracle-protocol.js";
+import type { EvalAuthoringTransportShape } from "../../project-authoring.js";
+type StoredArtifact = RepositoryFoundryAuthoringArtifactV1 & {
+    readonly planDigest: string;
+    readonly caseId: string;
+    readonly updatedAt: string;
+};
+declare const RepositoryFoundryEvidenceReconstruction_base: Context.ServiceClass<RepositoryFoundryEvidenceReconstruction, "@velum-labs/routekit-eval-setup/RepositoryFoundryEvidenceReconstruction", {
+    readonly commands: (input: unknown) => Effect.Effect<readonly RepositoryCommandResultV1[], RepositoryFoundryError>;
+    readonly oracle: (input: RepositoryHistoricalOracleInputV1) => Effect.Effect<RepositoryHiddenOracleV1, RepositoryFoundryError>;
+    readonly assertConsumed: Effect.Effect<void, RepositoryFoundryError>;
+}>;
+/**
+ * Read-only reconstruction of interrupted, pre-review work. This capability
+ * never supplies admission authority. Finalization must revalidate the frozen
+ * suite in real isolated checkouts before any new review is issued.
+ */
+export declare class RepositoryFoundryEvidenceReconstruction extends RepositoryFoundryEvidenceReconstruction_base {
+}
+export declare const makeRepositoryFoundryEvidenceReconstructionV1: (input: {
+    readonly plan: RepositoryFoundryGenerationPlanV1;
+    readonly artifacts: readonly StoredArtifact[];
+}) => Effect.Effect<{
+    service: {
+        readonly commands: (input: unknown) => Effect.Effect<readonly RepositoryCommandResultV1[], RepositoryFoundryError>;
+        readonly oracle: (input: RepositoryHistoricalOracleInputV1) => Effect.Effect<RepositoryHiddenOracleV1, RepositoryFoundryError>;
+        readonly assertConsumed: Effect.Effect<void, RepositoryFoundryError>;
+    };
+    transport: EvalAuthoringTransportShape;
+}, RepositoryFoundryError, never>;
+export {};

package/dist/services/evidence-reconstruction/service.js

@@ -0,0 +1,145 @@
+import { Context, Effect } from "effect";
+import { EvalProjectAuthoringError, RepositoryFoundryError } from "../../errors.js";
+import { assertRepositoryHiddenOracleV1 } from "../../repository-oracle-protocol.js";
+import { repositoryOracleExecutionBindingV1 } from "../../repository-oracle-execution-binding.js";
+/**
+ * Read-only reconstruction of interrupted, pre-review work. This capability
+ * never supplies admission authority. Finalization must revalidate the frozen
+ * suite in real isolated checkouts before any new review is issued.
+ */
+export class RepositoryFoundryEvidenceReconstruction extends Context.Service()("@velum-labs/routekit-eval-setup/RepositoryFoundryEvidenceReconstruction") {
+}
+const fail = (detail, cause) => new RepositoryFoundryError({
+    operation: "compile-reviewed-case",
+    detail: `saved evidence reconstruction: ${detail}`,
+    ...(cause === undefined ? {} : { cause })
+});
+const record = (value) => value !== null && typeof value === "object" && !Array.isArray(value)
+    ? value
+    : {};
+const normalize = (value) => Array.isArray(value)
+    ? value.map(normalize)
+    : value !== null && typeof value === "object"
+        ? Object.fromEntries(Object.entries(value)
+            .filter(([, entry]) => entry !== undefined)
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, entry]) => [key, normalize(entry)]))
+        : value;
+const same = (left, right) => JSON.stringify(normalize(left)) === JSON.stringify(normalize(right));
+const replayIdentity = (value) => {
+    const { preparationCache: _runtimeOnlyCache, ...identity } = record(value);
+    return identity;
+};
+export const makeRepositoryFoundryEvidenceReconstructionV1 = Effect.fn("EvidenceReconstruction.make")(function* (input) {
+    const sorted = [...input.artifacts].sort((left, right) => left.updatedAt.localeCompare(right.updatedAt));
+    if (sorted.length === 0 ||
+        sorted.some((artifact) => artifact.planDigest !== input.plan.planDigest ||
+            artifact.caseId !== input.plan.caseId ||
+            !Number.isFinite(Date.parse(artifact.updatedAt))))
+        return yield* fail("every artifact must belong to the exact parent plan and case");
+    const modelRecords = sorted.filter((artifact) => artifact.response.callId !== undefined);
+    if (modelRecords.length === 0 ||
+        modelRecords.some((artifact) => artifact.validation !== "validated" ||
+            artifact.role === "quality-reviewer" ||
+            !input.plan.modelPlan.assignments.some((assignment) => assignment.role === artifact.role &&
+                assignment.model === artifact.model &&
+                assignment.reasoningEffort === artifact.reasoningEffort)))
+        return yield* fail("saved model calls are incomplete, unbound, or already include final review");
+    const replayRecords = sorted.filter((artifact) => [
+        "routekit_repository_seed_qualification_phase_v1",
+        "routekit_repository_fixture_preflight_v1",
+        ...(input.plan.oracleCoverageWitnessVersion !== undefined
+            ? ["routekit_repository_coverage_witness_replay_v1"]
+            : [])
+    ].includes(artifact.schemaName));
+    const oracleRecords = sorted.filter((artifact) => artifact.schemaName === "routekit_repository_oracle_execution_v1" &&
+        record(artifact.value).complete === true);
+    const usedCalls = new Set();
+    const usedReplays = new Set();
+    let oracleIndex = 0;
+    const check = (body, detail) => Effect.try({
+        try: body,
+        catch: (cause) => fail(detail, cause)
+    });
+    const service = RepositoryFoundryEvidenceReconstruction.of({
+        commands: (untyped) => check(() => {
+            const request = replayIdentity(untyped);
+            const saved = replayRecords.find((artifact) => {
+                if (usedReplays.has(artifact.operationId))
+                    return false;
+                const recordedInput = record(artifact.request.input);
+                return same(replayIdentity(artifact.schemaName === "routekit_repository_seed_qualification_phase_v1"
+                    ? recordedInput
+                    : recordedInput.replayInput), request);
+            });
+            if (saved === undefined)
+                throw new Error("no exact saved replay request");
+            const results = record(saved.value).results;
+            if (!Array.isArray(results) || results.length === 0)
+                throw new Error("saved replay has no completed results");
+            usedReplays.add(saved.operationId);
+            return structuredClone(results);
+        }, "command reconstruction requires an unused exact request and complete recorded output"),
+        oracle: (request) => check(() => {
+            const saved = oracleRecords[oracleIndex];
+            if (saved === undefined)
+                throw new Error("no complete tournament at this ordinal");
+            const savedRequest = record(saved.request.input);
+            const oracle = record(saved.value).oracle;
+            assertRepositoryHiddenOracleV1(oracle);
+            const binding = repositoryOracleExecutionBindingV1(request);
+            const bound = oracle.executionBindingDigest !== undefined;
+            if (bound &&
+                (oracle.executionBindingDigest !== binding.executionBindingDigest ||
+                    (oracle.executionRepetitions !== undefined &&
+                        oracle.executionRepetitions !== binding.repetitions)))
+                throw new Error("saved tournament executable input binding differs");
+            if (request.repositoryRoot !== input.plan.repository.root ||
+                request.caseId !== input.plan.caseId ||
+                request.caseId !== oracle.caseId ||
+                request.seed.initialState.commit !== savedRequest.initialCommit ||
+                request.seed.referenceCommit !== savedRequest.referenceCommit ||
+                !same(request.hiddenFixtureSuite?.fixtures ?? [], oracle.fixtures) ||
+                request.additionalSolutions.length + 2 !== oracle.solutions.length ||
+                (bound
+                    ? !same(binding.solutionMetadata, oracle.solutions)
+                    // Legacy artifacts had no full executable binding. Preserve their
+                    // stricter full-solution comparison, not a metadata-only fallback.
+                    // Reconstructed checkpoints still require fresh replay validation.
+                    : !same(binding.solutions, oracle.solutions)))
+                throw new Error("saved tournament population or frozen fixture identity differs");
+            oracleIndex += 1;
+            return structuredClone(oracle);
+        }, "tournament reconstruction requires the same ordered population and frozen fixtures"),
+        assertConsumed: Effect.suspend(() => usedCalls.size === modelRecords.length &&
+            usedReplays.size === replayRecords.length &&
+            oracleIndex === oracleRecords.length
+            ? Effect.void
+            : Effect.fail(fail("reconstruction left model calls, replays or complete tournaments unused")))
+    });
+    const transport = {
+        complete: () => Effect.fail(new EvalProjectAuthoringError({
+            operation: "authoring-evaluations",
+            detail: "saved evidence reconstruction cannot make a model call"
+        })),
+        completeDetailed: (request) => check(() => {
+            const saved = modelRecords.find((artifact) => artifact.operationId === request.operationId && !usedCalls.has(artifact.operationId));
+            if (saved === undefined ||
+                request.model !== saved.model ||
+                request.reasoningEffort !== saved.reasoningEffort ||
+                request.schemaName !== saved.schemaName ||
+                request.instructions !== saved.request.instructions ||
+                request.maximumOutputTokens !== saved.request.maximumOutputTokens ||
+                !same(request.jsonSchema, saved.request.jsonSchema) ||
+                !same(JSON.parse(request.input), saved.request.input))
+                throw new Error("saved model request envelope differs or was already used");
+            usedCalls.add(saved.operationId);
+            return structuredClone(saved.response);
+        }, "model reconstruction requires an unused exact model, instructions, input, schema and limit").pipe(Effect.mapError((cause) => new EvalProjectAuthoringError({
+            operation: "authoring-evaluations",
+            detail: cause.detail,
+            cause
+        })))
+    };
+    return { service, transport };
+});

package/dist/services/fixture-builder/service.d.ts

@@ -0,0 +1,62 @@
+import { Context, Effect, Layer } from "effect";
+import { type RepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+export type FixtureBuilderShape = {
+    readonly compileReviewed: (input: {
+        readonly caseId: string;
+        readonly seed: RepositoryTaskSeedV1;
+        readonly suite: RepositoryHiddenFixtureSuiteV1;
+    }) => Effect.Effect<RepositoryHiddenFixtureSuiteV1>;
+};
+declare const FixtureBuilder_base: Context.ServiceClass<FixtureBuilder, "@velum-labs/routekit-eval-setup/FixtureBuilder", FixtureBuilderShape>;
+export declare class FixtureBuilder extends FixtureBuilder_base {
+}
+export declare const compileReviewedHiddenFixtureSuiteV1: (input: {
+    readonly caseId: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly suite: RepositoryHiddenFixtureSuiteV1;
+}) => Effect.Effect<{
+    readonly version: 1;
+    readonly caseId: string;
+    readonly fixtures: readonly ({
+        readonly kind: "historical-regression";
+        readonly expectationMode: "changes";
+        readonly id: string;
+        readonly description: string;
+        readonly expectedBehavior: string;
+        readonly source: "historical" | "generated" | "reviewed";
+        readonly testPath: string;
+    } | {
+        readonly kind: "boundary";
+        readonly expectationMode: "changes" | "preserved";
+        readonly id: string;
+        readonly description: string;
+        readonly expectedBehavior: string;
+        readonly source: "historical" | "generated" | "reviewed";
+        readonly testPath: string;
+    } | {
+        readonly kind: "counterfactual";
+        readonly expectationMode: "changes";
+        readonly id: string;
+        readonly description: string;
+        readonly expectedBehavior: string;
+        readonly source: "historical" | "generated" | "reviewed";
+        readonly testPath: string;
+    } | {
+        readonly kind: "metamorphic";
+        readonly expectationMode: "preserved";
+        readonly id: string;
+        readonly description: string;
+        readonly expectedBehavior: string;
+        readonly source: "historical" | "generated" | "reviewed";
+        readonly testPath: string;
+    })[];
+    readonly overlays: readonly {
+        readonly path: string;
+        readonly content: string;
+        readonly fixtureIds: readonly string[];
+    }[];
+}, never, never>;
+export declare const makeFixtureBuilder: Effect.Effect<FixtureBuilderShape, never, never>;
+export declare const FixtureBuilderLive: Layer.Layer<FixtureBuilder, never, never>;
+export {};

package/dist/services/fixture-builder/service.js

@@ -0,0 +1,36 @@
+import { Context, Effect, Layer } from "effect";
+import { assertRepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+export class FixtureBuilder extends Context.Service()("@velum-labs/routekit-eval-setup/FixtureBuilder") {
+}
+export const compileReviewedHiddenFixtureSuiteV1 = Effect.fn("FixtureBuilder.compileReviewed")(function* (input) {
+    assertRepositoryHiddenFixtureSuiteV1(input.suite);
+    if (input.suite.caseId !== input.caseId) {
+        throw new Error("hidden fixture suite case id does not match its task");
+    }
+    const candidateTestPaths = new Set(input.seed.capabilityEvidence
+        .filter((evidence) => evidence.kind === "test" && evidence.path !== undefined)
+        .map((evidence) => evidence.path));
+    const overlayByFixtureId = new Map(input.suite.overlays.map((overlay) => [overlay.fixtureIds[0], overlay]));
+    for (const fixture of input.suite.fixtures) {
+        if (!candidateTestPaths.has(fixture.testPath)) {
+            throw new Error(`hidden fixture targets a path outside the seed's reviewed test scope: ${JSON.stringify(fixture.testPath)}`);
+        }
+        if (fixture.kind === "counterfactual" &&
+            fixture.expectationMode !== "changes") {
+            throw new Error("counterfactual fixtures must identify an expected behavior change");
+        }
+        if (fixture.kind === "metamorphic" &&
+            fixture.expectationMode !== "preserved") {
+            throw new Error("metamorphic fixtures must identify a preserved invariant");
+        }
+        const overlay = overlayByFixtureId.get(fixture.id);
+        if (overlay === undefined || overlay.path !== fixture.testPath) {
+            throw new Error(`hidden fixture ${JSON.stringify(fixture.id)} must own one independently executable overlay at its declared test path`);
+        }
+    }
+    return input.suite;
+});
+export const makeFixtureBuilder = Effect.succeed(FixtureBuilder.of({
+    compileReviewed: compileReviewedHiddenFixtureSuiteV1
+}));
+export const FixtureBuilderLive = Layer.effect(FixtureBuilder, makeFixtureBuilder);

package/dist/services/fixture-validation/service.d.ts

@@ -0,0 +1,75 @@
+import { Effect } from "effect";
+import { type PipelineWorkCheckpointStoreV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { RepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+import type { RepositoryFoundryModelCallV1, RepositoryFoundryModelPlanV1 } from "../../repository-language-model-protocol.js";
+import { type RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+/** Private, bounded author feedback; never stringify a structured cause as [object Object]. */
+export declare const repositoryFixturePreflightFeedbackV1: (error: RepositoryFoundryError) => string;
+/** Reference executability preflight only; never replaces independent oracle admission evidence. */
+export declare const validateRepositoryFixturesV1: (input: {
+    readonly repositoryRoot: string;
+    readonly operationId: string;
+    readonly seed: RepositoryTaskSeedV1;
+    readonly suite: RepositoryHiddenFixtureSuiteV1;
+    readonly allowedTestPaths: ReadonlySet<string>;
+    readonly context: unknown;
+    readonly modelPlan: RepositoryFoundryModelPlanV1;
+    readonly maximumRepairAttempts: number;
+    readonly checkpointStore?: PipelineWorkCheckpointStoreV1;
+    readonly oracleCoverageWitnessVersion?: 1 | 2;
+    readonly oracleRepairFeedbackVersion?: 1;
+    readonly onRepairAttempt?: () => Effect.Effect<void>;
+    readonly onRepairCall?: (call: RepositoryFoundryModelCallV1) => Effect.Effect<void>;
+}) => Effect.Effect<{
+    suite: {
+        readonly version: 1;
+        readonly caseId: string;
+        readonly fixtures: readonly ({
+            readonly kind: "historical-regression";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "boundary";
+            readonly expectationMode: "changes" | "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "counterfactual";
+            readonly expectationMode: "changes";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        } | {
+            readonly kind: "metamorphic";
+            readonly expectationMode: "preserved";
+            readonly id: string;
+            readonly description: string;
+            readonly expectedBehavior: string;
+            readonly source: "historical" | "generated" | "reviewed";
+            readonly testPath: string;
+        })[];
+        readonly overlays: readonly {
+            readonly path: string;
+            readonly content: string;
+            readonly fixtureIds: readonly string[];
+        }[];
+    };
+    modelCalls: {
+        readonly reasoningEffort: "high" | "medium" | "low" | "xhigh" | "max" | "ultra";
+        readonly role: "oracle-critic" | "repository-analyst" | "specification-writer" | "specification-critic-a" | "specification-critic-b" | "trajectory-designer" | "trajectory-critic" | "oracle-designer" | "valid-solution-generator-a" | "valid-solution-generator-b" | "adversary" | "solution-critic-a" | "solution-critic-b" | "held-out-adversary" | "held-out-solution-critic-a" | "held-out-solution-critic-b" | "quality-reviewer" | "hill-climb-planner";
+        readonly model: string;
+        readonly operationId: string;
+        readonly callId?: string | undefined;
+    }[];
+}, RepositoryFoundryError, RepositoryFoundryLanguageModel>;