@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/pipeline-controls/service.js

@@ -0,0 +1,483 @@
+import { Clock, Effect, Schema } from "effect";
+import { repositoryCommandTestEvidenceV1 } from "../../adapters/repository-command-evidence.js";
+import { replayRepositoryCommandsV1 } from "../../adapters/repository-command-runner.js";
+import { loadPinnedSolutionEditSourcesV1, materializeSolutionFileOverridesV1, RepositorySolutionFileOutputV1 } from "../../adapters/repository-solution-edits.js";
+import { checkpointDigestV1, pipelineStageToProgressStageV1, SolutionCritiqueV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { repositoryFoundryRoleOutputCeilingV1 } from "../../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelAssignmentV1 } from "../../repository-language-model-protocol.js";
+import { generateWithFeedbackV1, normalizePipelineFailureV1 } from "../budgeted-model/service.js";
+import { reportFoundryStageV1 } from "../foundry-progress/service.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+const STAGE = "controls";
+const REQUIRED_VALID_CONTROLS = 1;
+const REQUIRED_WRONG_CONTROLS = 1;
+const CONTROL_CRITIC_ROLES = ["solution-critic-a", "solution-critic-b"];
+const SolutionSubmissionV1 = Schema.Struct({
+    id: Schema.String,
+    family: Schema.String,
+    kind: Schema.Literals([
+        "independent-valid",
+        "simplified-valid",
+        "behavior-preserving-valid",
+        "mutation",
+        "model-near-miss"
+    ]),
+    fileOverrides: Schema.Array(RepositorySolutionFileOutputV1)
+});
+const SolutionCritiqueSubmissionV1 = Schema.Struct({
+    critiques: Schema.Array(SolutionCritiqueV1)
+});
+const EXACT_EDIT_INSTRUCTIONS = `Return edits against the ORIGINAL pinned pre-change source. For an existing
+file, prefer {path, edits:[{search, replace}]}. Every search must be nonempty verbatim text that occurs exactly
+once in both the supplied complete source and the pinned file, include enough unchanged context to be unique,
+and refer to the original source rather than another edit's output. Edits in one file must not overlap. Preserve
+all bytes outside the requested replacements. A complete small replacement may use {path, content}. Do not
+modify tests, manifests, lockfiles, generated controls, or any path outside allowedSolutionPaths. Return one
+structured solution submission. Invalid structure, unmatched anchors, protected paths, failed materialization,
+failed validation commands, or failed isolated execution are host findings: revise your own prior submission
+and submit again.`;
+const VALID_A_INSTRUCTIONS = `You are blind solver A for a repository coding benchmark. Solve only the visible
+task from the complete dimension contract, grounded public scope, and pinned pre-change sources. Produce one
+behaviorally correct implementation. Preserve every stated boundary and unrelated behavior. You cannot see the
+reference change, hidden fixtures, oracle, another solver, or any tournament result. Do not infer or request
+those materials. The family field is a concise description of your own implementation mechanism and is never
+used as correctness evidence.`;
+const ADVERSARY_INSTRUCTIONS = `You are a blind adversarial solver for a repository coding benchmark. Using
+only the visible task, complete dimension contract, grounded public scope, and pinned pre-change sources,
+produce one plausible implementation that builds and passes the repository's ordinary sanity commands but is
+behaviorally wrong for one explicitly stated visible requirement. Choose kind mutation or model-near-miss.
+Preserve unrelated behavior. Do not modify tests or protected controls. You cannot see the reference change,
+hidden fixtures, oracle, valid solvers, or tournament outcomes. Do not rely on test detection, fixture names,
+source fingerprints, process timing, or harness manipulation. The family field describes the concrete defect
+mechanism and is not compared with any other family.`;
+const CONTROL_CRITIC_INSTRUCTIONS = `You are an independent fixture-blind correctness critic for repository
+benchmark controls. Classify every supplied implementation against only the visible task, complete dimension
+contract, grounded public scope, target behaviors, and pinned pre-change sources. Inspect the actual complete
+file overrides. You are not shown proposal labels, the reference change, hidden fixtures, oracle, another
+critic, or tournament outcomes. A different implementation mechanism is not a correctness defect. Classify
+wrong only when the implementation violates a named user-observable target behavior; list every such behavior
+id. Use uncertain when the supplied source and contract do not justify valid or wrong. Return exactly one
+critique for every supplied solution. The host records this review as advisory development adjudication;
+execution, not your opinion, remains the admission gate.`;
+const failure = (detail, cause) => new RepositoryFoundryError({
+    operation: "run-pipeline-stage",
+    detail,
+    ...(cause === undefined ? {} : { cause })
+});
+const describe = (cause) => cause instanceof Error && cause.message.trim().length > 0 ? cause.message : String(cause);
+const assignmentOf = (input) => Effect.try({
+    try: () => repositoryFoundryModelAssignmentV1(input.plan, input.role),
+    catch: (cause) => failure(`could not resolve ${input.role} model assignment`, cause)
+});
+const solutionPathsOf = (input) => {
+    const protectedPaths = new Set(input.seed.environment.protectedControlPaths);
+    const testPaths = new Set(input.seed.capabilityEvidence
+        .filter((evidence) => evidence.kind === "test" && evidence.path !== undefined)
+        .map((evidence) => evidence.path));
+    return new Set(input.contextSources
+        .map((source) => source.path)
+        .filter((path) => !protectedPaths.has(path) && !testPaths.has(path)));
+};
+const submissionFindings = (input) => {
+    const findings = [];
+    const expectedKind = input.expected === "valid"
+        ? input.submission.kind === "independent-valid" ||
+            input.submission.kind === "simplified-valid" ||
+            input.submission.kind === "behavior-preserving-valid"
+        : input.submission.kind === "mutation" || input.submission.kind === "model-near-miss";
+    if (input.submission.id.trim().length === 0) {
+        findings.push({ kind: "contract", message: "solution id must be non-empty" });
+    }
+    else if (input.submission.id === "historical-reference" ||
+        input.submission.id === "historical-defect" ||
+        input.existingIds.has(input.submission.id)) {
+        findings.push({
+            kind: "contract",
+            message: `solution id ${JSON.stringify(input.submission.id)} is reserved or already accepted`
+        });
+    }
+    if (input.submission.family.trim().length === 0) {
+        findings.push({ kind: "contract", message: "solution family must describe its mechanism" });
+    }
+    if (!expectedKind) {
+        findings.push({
+            kind: "contract",
+            message: input.expected === "valid"
+                ? "a valid solver must use independent-valid, simplified-valid, or behavior-preserving-valid"
+                : "the blind adversary must use mutation or model-near-miss"
+        });
+    }
+    if (input.submission.fileOverrides.length === 0) {
+        findings.push({ kind: "contract", message: "solution must contain at least one source edit" });
+    }
+    const seen = new Set();
+    for (const override of input.submission.fileOverrides) {
+        if (seen.has(override.path)) {
+            findings.push({
+                kind: "materialization",
+                path: override.path,
+                message: "solution contains duplicate overrides for one path"
+            });
+        }
+        seen.add(override.path);
+        if (input.protectedPaths.has(override.path)) {
+            findings.push({
+                kind: "contract",
+                path: override.path,
+                message: "solution modifies a protected repository control"
+            });
+        }
+        else if (!input.allowedPaths.has(override.path)) {
+            findings.push({
+                kind: "contract",
+                path: override.path,
+                message: "solution modifies a path that was not supplied to this blind role"
+            });
+        }
+    }
+    return findings;
+};
+const commandDetail = (result) => [
+    `recipe=${result.recipeId}`,
+    `stage=${result.stage}`,
+    `exit=${String(result.exitCode)}`,
+    `timeout=${String(result.timedOut)}`,
+    `gradeExecuted=${String(result.gradeExecuted)}`,
+    `stdout=${result.stdout.slice(-2_000)}`,
+    `stderr=${result.stderr.slice(-2_000)}`
+].join("; ");
+const executionFindings = (input) => {
+    const findings = [];
+    for (const result of input.results) {
+        const preparationFailure = result.trustedPreparationResults.find((candidate) => candidate.exitCode !== 0 || candidate.timedOut);
+        if (preparationFailure !== undefined) {
+            findings.push({
+                kind: "execution",
+                message: `trusted preparation did not complete: recipe=${preparationFailure.recipeId}; exit=${String(preparationFailure.exitCode)}; timeout=${String(preparationFailure.timedOut)}`
+            });
+        }
+        for (const validation of result.candidateValidationResults) {
+            if (validation.exitCode !== 0 || validation.timedOut) {
+                findings.push({
+                    kind: "compile",
+                    message: `candidate validation failed: recipe=${validation.recipeId}; exit=${String(validation.exitCode)}; timeout=${String(validation.timedOut)}; stdout=${validation.stdout.slice(-2_000)}; stderr=${validation.stderr.slice(-2_000)}`
+                });
+            }
+        }
+        const evidence = repositoryCommandTestEvidenceV1(result, result.recipeId);
+        if (evidence.outcome !== "pass") {
+            findings.push({
+                kind: "execution",
+                message: `${commandDetail(result)}; evidence=${evidence.detail}`
+            });
+        }
+    }
+    const observed = new Set(input.results.filter((result) => result.gradeExecuted).map((result) => result.recipeId));
+    for (const recipeId of input.expectedRecipeIds) {
+        if (!observed.has(recipeId)) {
+            findings.push({
+                kind: "execution",
+                message: `isolated sanity execution did not reach reviewed recipe ${recipeId}`
+            });
+        }
+    }
+    return findings;
+};
+const materializeSubmission = (input) => Effect.try({
+    try: () => {
+        const materialized = materializeSolutionFileOverridesV1({
+            fileOverrides: input.submission.fileOverrides,
+            sources: input.sources,
+            allowedPaths: input.allowedPaths
+        });
+        return {
+            id: input.submission.id,
+            family: input.submission.family,
+            kind: input.submission.kind,
+            expectedClass: input.expected,
+            commit: input.initialCommit,
+            fileOverrides: materialized.fileOverrides
+        };
+    },
+    catch: (cause) => cause instanceof RepositoryFoundryError
+        ? cause
+        : failure("solution exact-edit materialization failed", cause)
+});
+const checkpointOf = (input) => ({
+    version: 1,
+    stage: STAGE,
+    parentDigest: input.parentDigest,
+    createdAt: input.createdAt,
+    valid: [...input.valid],
+    wrong: [...input.wrong],
+    critiques: [...input.critiques],
+    regenerations: input.regenerations
+});
+const critiqueFindings = (input) => {
+    const findings = [];
+    const ids = input.critiques.map(({ solutionId }) => solutionId);
+    if (ids.length !== input.solutionIds.size ||
+        new Set(ids).size !== ids.length ||
+        ids.some((id) => !input.solutionIds.has(id))) {
+        findings.push({
+            kind: "contract",
+            message: "critic must return exactly one classification for every supplied solution"
+        });
+    }
+    for (const critique of input.critiques) {
+        if (critique.detail.trim().length === 0) {
+            findings.push({
+                kind: "critique",
+                message: `critic rationale is empty for ${JSON.stringify(critique.solutionId)}`
+            });
+        }
+        if (critique.classification === "wrong" &&
+            (critique.violatedBehaviorIds.length === 0 ||
+                critique.violatedBehaviorIds.some((id) => !input.behaviorIds.has(id)))) {
+            findings.push({
+                kind: "critique",
+                message: `wrong classification for ${JSON.stringify(critique.solutionId)} must cite supplied target behavior ids`
+            });
+        }
+        if (critique.classification === "valid" && critique.violatedBehaviorIds.length > 0) {
+            findings.push({
+                kind: "critique",
+                message: `valid classification for ${JSON.stringify(critique.solutionId)} cannot cite violated behaviors`
+            });
+        }
+    }
+    return findings;
+};
+/**
+ * Run fixture-blind correctness review only when execution has false-rejected
+ * a generated valid control. Callers may supply prior completed rounds when
+ * resuming or when an older controls checkpoint already contains them.
+ */
+export const runPipelineControlCorrectnessCriticsV1 = Effect.fnUntraced(function* (input) {
+    const solutionIds = new Set(input.solutions.map(({ id }) => id));
+    const behaviorIds = new Set(input.seed.targetBehavior.map(({ id }) => id));
+    const critiques = (input.priorCritiques ?? []).filter(({ solutionId }) => solutionIds.has(solutionId));
+    let regenerations = 0;
+    const completedCriticRounds = input.solutions.length === 0 ? 0 : Math.floor(critiques.length / input.solutions.length);
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    for (let criticIndex = completedCriticRounds; criticIndex < CONTROL_CRITIC_ROLES.length; criticIndex += 1) {
+        const role = CONTROL_CRITIC_ROLES[criticIndex];
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `fixture-blind correctness adjudication ${String(criticIndex + 1)} of ${String(CONTROL_CRITIC_ROLES.length)}`);
+        const assignment = yield* assignmentOf({ plan: input.context.modelPlan, role });
+        const review = yield* generateWithFeedbackV1({
+            ...(input.workCheckpointStore === undefined ? {} : {
+                checkpoint: { store: input.workCheckpointStore, binding: input.seed }
+            }),
+            languageModel,
+            assignment,
+            operationId: `${input.context.caseId}:pipeline:controls:${role}`,
+            instructions: CONTROL_CRITIC_INSTRUCTIONS,
+            input: {
+                dimension: input.context.dimension,
+                visibleTask: input.spec.visible,
+                analysis: input.spec.analysis,
+                groundedScope: input.spec.scope,
+                targetBehavior: input.seed.targetBehavior.map(({ id, description, critical }) => ({
+                    id,
+                    description,
+                    critical
+                })),
+                preChangeSources: input.spec.contextSources,
+                solutions: input.solutions.map(({ id, fileOverrides }) => ({
+                    id,
+                    fileOverrides
+                }))
+            },
+            schemaName: "routekit_pipeline_control_correctness_review_v1",
+            outputSchema: SolutionCritiqueSubmissionV1,
+            maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1(role),
+            maxAttempts: Number.MAX_SAFE_INTEGER,
+            validate: ({ critiques: proposed }) => Effect.succeed(critiqueFindings({
+                critiques: proposed,
+                solutionIds,
+                behaviorIds
+            }))
+        });
+        critiques.push(...review.value.critiques);
+        regenerations += review.attempts - 1;
+        if (input.onRound !== undefined)
+            yield* input.onRound(critiques, regenerations);
+    }
+    return { critiques, regenerations };
+});
+/**
+ * Generate fixture-blind valid and wrong controls. Each proposal is exact-edit
+ * materialized and run in an isolated checkout before it is accepted. Model
+ * labels remain advisory; the tournament stage performs semantic adjudication.
+ */
+export const runPipelineControlsCapabilityV1 = (input) => Effect.gen(function* () {
+    const seedCheckpoint = input.checkpoints.seed;
+    const spec = input.checkpoints.spec;
+    const oracle = input.checkpoints.oracle;
+    if (seedCheckpoint === undefined || spec === undefined || oracle === undefined) {
+        return yield* failure("controls requires accepted seed, specification, and oracle checkpoints");
+    }
+    const seed = seedCheckpoint.seed;
+    if (seed.status !== "qualified" || seed.referenceCommit === undefined) {
+        return yield* failure("controls requires a replay-qualified pinned reference");
+    }
+    const parentDigest = checkpointDigestV1(oracle);
+    const resumed = input.checkpoints.controls?.parentDigest === parentDigest
+        ? input.checkpoints.controls
+        : undefined;
+    const createdAt = resumed?.createdAt ?? new Date(yield* Clock.currentTimeMillis).toISOString();
+    const valid = [...(resumed?.valid ?? [])];
+    const wrong = [...(resumed?.wrong ?? [])];
+    const critiques = [...(resumed?.critiques ?? [])];
+    let regenerations = resumed?.regenerations ?? 0;
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const allowedPaths = solutionPathsOf({ seed, contextSources: spec.contextSources });
+    if (allowedPaths.size === 0) {
+        return yield* failure("specification exposes no unprotected implementation source to blind solvers");
+    }
+    const protectedPaths = new Set(seed.environment.protectedControlPaths);
+    const editSources = yield* loadPinnedSolutionEditSourcesV1({
+        repositoryRoot: input.context.repositoryRoot,
+        initialCommit: seed.initialState.commit,
+        visibleSources: spec.contextSources,
+        allowedPaths
+    });
+    if (editSources.length === 0) {
+        return yield* failure("no complete pinned pre-change edit source could be materialized");
+    }
+    // Blind controls run on the initial tree, without reference-test overlays.
+    // Strong grade recipes may name files introduced by the historical fix.
+    // Only the qualified original baseline is an executable, non-leaking sanity
+    // contract here; hidden behavior is measured later by the tournament.
+    const sanityRecipes = seed.environment.baselineGradeRecipes;
+    if (sanityRecipes.length === 0)
+        return yield* failure("blind controls require qualified initial baseline recipes");
+    const expectedRecipeIds = new Set(sanityRecipes.map((recipe) => recipe.id));
+    const generateControl = Effect.fnUntraced(function* (request) {
+        const assignment = yield* assignmentOf({ plan: input.context.modelPlan, role: request.role });
+        let accepted;
+        const generated = yield* generateWithFeedbackV1({
+            ...(input.workCheckpointStore === undefined ? {} : {
+                checkpoint: {
+                    store: input.workCheckpointStore,
+                    binding: {
+                        parentDigest,
+                        seed,
+                        editSources,
+                        expected: request.expected,
+                        existingIds: [...valid, ...wrong].map(({ id }) => id).sort()
+                    }
+                }
+            }),
+            languageModel,
+            assignment,
+            operationId: `${input.context.caseId}:pipeline:controls:${request.role}:${String(request.sequence)}${input.replacementRound === undefined ? "" : `:replacement-${String(input.replacementRound)}`}`,
+            instructions: `${request.instructions}\n\n${EXACT_EDIT_INSTRUCTIONS}`,
+            input: {
+                dimension: input.context.dimension,
+                visibleTask: spec.visible,
+                analysis: spec.analysis,
+                groundedScope: spec.scope,
+                targetBehavior: seed.targetBehavior.map(({ id, description, critical }) => ({
+                    id,
+                    description,
+                    critical
+                })),
+                repositoryEnvironment: {
+                    packageManager: seed.environment.packageManager,
+                    candidateValidationRecipes: seed.environment.candidateValidationRecipes,
+                    sanityRecipes
+                },
+                allowedSolutionPaths: [...allowedPaths].sort(),
+                preChangeSources: spec.contextSources
+            },
+            schemaName: `routekit_pipeline_${request.role.replaceAll("-", "_")}_v1`,
+            outputSchema: SolutionSubmissionV1,
+            maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1(request.role),
+            maxAttempts: Number.MAX_SAFE_INTEGER,
+            validate: (submission) => Effect.gen(function* () {
+                const findings = submissionFindings({
+                    submission,
+                    expected: request.expected,
+                    existingIds: new Set([...valid, ...wrong].map((solution) => solution.id)),
+                    allowedPaths,
+                    protectedPaths
+                });
+                if (findings.length > 0)
+                    return findings;
+                const materialized = yield* Effect.result(materializeSubmission({
+                    submission,
+                    expected: request.expected,
+                    initialCommit: seed.initialState.commit,
+                    sources: editSources,
+                    allowedPaths
+                }));
+                if (materialized._tag === "Failure") {
+                    return [
+                        { kind: "materialization", message: describe(materialized.failure) }
+                    ];
+                }
+                const execution = yield* Effect.result(replayRepositoryCommandsV1({
+                    repositoryRoot: input.context.repositoryRoot,
+                    commit: seed.initialState.commit,
+                    recipes: sanityRecipes,
+                    trustedPreparationRecipes: seed.environment.trustedPreparationRecipes,
+                    candidateValidationRecipes: seed.environment.candidateValidationRecipes,
+                    fileOverrides: materialized.success.fileOverrides,
+                    repetitions: 1
+                }));
+                if (execution._tag === "Failure") {
+                    return [
+                        { kind: "execution", message: describe(execution.failure) }
+                    ];
+                }
+                const observedFindings = executionFindings({
+                    results: execution.success,
+                    expectedRecipeIds
+                });
+                if (observedFindings.length === 0)
+                    accepted = materialized.success;
+                return observedFindings;
+            })
+        });
+        regenerations += generated.attempts - 1;
+        if (accepted === undefined) {
+            return yield* failure(`accepted ${request.role} response had no materialized execution`);
+        }
+        return accepted;
+    });
+    let added = 0;
+    while (input.kind !== "wrong" &&
+        valid.length < REQUIRED_VALID_CONTROLS &&
+        added < (input.maximumNewControls ?? Number.MAX_SAFE_INTEGER)) {
+        const role = "valid-solution-generator-a";
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `blind valid control ${String(valid.length + 1)} of ${String(REQUIRED_VALID_CONTROLS)}`);
+        valid.push(yield* generateControl({
+            role,
+            expected: "valid",
+            instructions: VALID_A_INSTRUCTIONS,
+            sequence: valid.length + 1
+        }));
+        added += 1;
+        yield* input.checkpoint(checkpointOf({ parentDigest, createdAt, valid, wrong, critiques, regenerations }));
+    }
+    while (input.kind !== "valid" &&
+        wrong.length < REQUIRED_WRONG_CONTROLS &&
+        added < (input.maximumNewControls ?? Number.MAX_SAFE_INTEGER)) {
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `blind wrong control ${String(wrong.length + 1)} of ${String(REQUIRED_WRONG_CONTROLS)}`);
+        wrong.push(yield* generateControl({
+            role: "adversary",
+            expected: "wrong",
+            instructions: ADVERSARY_INSTRUCTIONS,
+            sequence: wrong.length + 1
+        }));
+        added += 1;
+        yield* input.checkpoint(checkpointOf({ parentDigest, createdAt, valid, wrong, critiques, regenerations }));
+    }
+    yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `controls ready; valid=${String(valid.length)}; wrong=${String(wrong.length)}; advisoryCritiques=${String(critiques.length)}`);
+    return checkpointOf({ parentDigest, createdAt, valid, wrong, critiques, regenerations });
+}).pipe(Effect.mapError((error) => normalizePipelineFailureV1(error)));
+/** Short name used by the orchestrator. */
+export const runPipelineControlsStageV1 = (input) => runPipelineControlsCapabilityV1(input);
+export const runPipelineControlsV1 = runPipelineControlsStageV1;

package/dist/services/pipeline-oracle/service.d.ts

@@ -0,0 +1,8 @@
+import { type OracleCheckpointV1, type PipelineStageRunV1 } from "../../case-pipeline-protocol.js";
+/**
+ * Authors and reference-preflights the initial oracle. Proactive coverage
+ * criticism and speculative witness generation are intentionally deferred to
+ * the tournament, where actual correct and wrong solutions provide measured
+ * escapes. Only executed observations can later establish admission evidence.
+ */
+export declare const runPipelineOracleStageV1: PipelineStageRunV1<OracleCheckpointV1>;