@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/pipeline-tournament/service.js

@@ -0,0 +1,476 @@
+import { Clock, Effect, Option, Schema } from "effect";
+import { checkpointDigestV1, pipelineStageToProgressStageV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { assertRepositoryHiddenFixtureSuiteV1, RepositoryHiddenFixtureSuiteV1 } from "../../repository-fixture-protocol.js";
+import { repositoryFoundryRoleOutputCeilingV1 } from "../../repository-foundry-plan-protocol.js";
+import { repositoryFoundryModelAssignmentV1 } from "../../repository-language-model-protocol.js";
+import { RepositoryHiddenOracleV1 } from "../../repository-oracle-protocol.js";
+import { evaluateRepositoryOracleAdequacyV1 } from "../../repository-quality-policy.js";
+import { evaluateRepositoryOracleScopeCoverageV1 } from "../../repository-oracle-scope-policy.js";
+import { normalizePipelineFailureV1 } from "../budgeted-model/service.js";
+import { reportFoundryHeartbeatV1, reportFoundryStageV1 } from "../foundry-progress/service.js";
+import { groundedStructuredGenerationV1 } from "../grounded-authoring/service.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+import { buildHistoricalHiddenOracleV1 } from "../oracle-builder/service.js";
+import { runPipelineControlCorrectnessCriticsV1, runPipelineControlsCapabilityV1 } from "../pipeline-controls/service.js";
+const STAGE = "tournament";
+const TournamentRepairProposalV1 = Schema.Struct({
+    targetedSolutionIds: Schema.Array(Schema.String),
+    suite: RepositoryHiddenFixtureSuiteV1,
+    rationale: Schema.String
+});
+const REPAIR_INSTRUCTIONS = `You are the grounded oracle-repair role for a repository benchmark.
+The host has executed the incumbent suite repeatedly in isolated checkouts. Repair only the measured false
+accepts, false rejects, or inconclusive fixture observations supplied in the task. Inspect both pinned Git trees
+and use run_fixture for every changed or added overlay before submitting. Preserve all useful incumbent fixtures
+and every grounded behavior obligation. Reject wrong solutions only through user-observable assertions; accept
+all behaviorally valid implementations. Do not inspect solution family names, source fingerprints, textual
+similarity, or historical patch identity. Do not modify solutions. A model opinion never overrides execution.
+Return one complete replacement suite, every measured solution id in targetedSolutionIds, and a rationale.`;
+const failure = (detail, cause) => new RepositoryFoundryError({
+    operation: "run-pipeline-stage",
+    detail,
+    ...(cause === undefined ? {} : { cause })
+});
+const describe = (cause) => cause instanceof Error && cause.message.trim().length > 0 ? cause.message : String(cause);
+const sortedUnique = (values) => [...new Set(values)].sort();
+const unresolvedIds = (adequacy) => sortedUnique([
+    ...adequacy.falseAcceptedSolutionIds,
+    ...adequacy.falseRejectedSolutionIds,
+    ...adequacy.unstableSolutionIds
+]);
+const difference = (left, right) => {
+    const other = new Set(right);
+    return left.filter((value) => !other.has(value));
+};
+const advisoryControlClassification = (critiques, solutionId) => {
+    const classifications = critiques
+        .filter((critique) => critique.solutionId === solutionId)
+        .map(({ classification }) => classification);
+    if (classifications.length >= 2 &&
+        classifications.every((classification) => classification === "wrong")) {
+        return "wrong";
+    }
+    if (classifications.length >= 2 &&
+        classifications.every((classification) => classification === "valid")) {
+        return "valid";
+    }
+    return "uncertain";
+};
+const candidateRegressionReasons = (incumbent, candidate, incumbentScope, candidateScope) => [
+    ...difference(candidateScope, incumbentScope).map((id) => `introduced unproved grounded scope: ${id}`),
+    ...difference(candidate.falseAcceptedSolutionIds, incumbent.falseAcceptedSolutionIds).map((id) => `introduced false acceptance: ${id}`),
+    ...difference(candidate.falseRejectedSolutionIds, incumbent.falseRejectedSolutionIds).map((id) => `introduced false rejection: ${id}`),
+    ...difference(candidate.unstableSolutionIds, incumbent.unstableSolutionIds).map((id) => `introduced unstable execution: ${id}`),
+    ...difference(candidate.unobservedClauseIds, incumbent.unobservedClauseIds).map((id) => `introduced unobserved critical clause: ${id}`),
+    ...difference(candidate.insufficientObservationPairs, incumbent.insufficientObservationPairs).map((id) => `introduced insufficient repeated observation: ${id}`)
+];
+/** A candidate is accepted only when execution removes a defect without introducing any regression. */
+const candidateRejectionReasons = (incumbent, candidate) => {
+    const before = unresolvedIds(incumbent.adequacy).length +
+        incumbent.scopeCoverage.unresolvedScopeIds.length;
+    const after = unresolvedIds(candidate.adequacy).length +
+        candidate.scopeCoverage.unresolvedScopeIds.length;
+    return [
+        ...candidateRegressionReasons(incumbent.adequacy, candidate.adequacy, incumbent.scopeCoverage.unresolvedScopeIds, candidate.scopeCoverage.unresolvedScopeIds),
+        ...(after < before
+            ? []
+            : [
+                `executed candidate did not reduce unresolved classifications and scope (${String(before)} -> ${String(after)})`
+            ])
+    ];
+};
+const validateProposal = (input) => {
+    const findings = [];
+    try {
+        const decoded = Schema.decodeUnknownSync(RepositoryHiddenFixtureSuiteV1)(input.proposal.suite);
+        assertRepositoryHiddenFixtureSuiteV1(decoded);
+    }
+    catch (cause) {
+        findings.push({ kind: "contract", message: describe(cause) });
+    }
+    if (input.proposal.suite.caseId !== input.caseId) {
+        findings.push({
+            kind: "contract",
+            message: "replacement suite must retain the pipeline case id"
+        });
+    }
+    const proposedTargets = new Set(input.proposal.targetedSolutionIds);
+    for (const id of input.targeted) {
+        if (!proposedTargets.has(id)) {
+            findings.push({
+                kind: "contract",
+                message: `replacement did not account for measured solution ${JSON.stringify(id)}`
+            });
+        }
+    }
+    if (input.proposal.rationale.trim().length === 0) {
+        findings.push({ kind: "contract", message: "replacement rationale must be non-empty" });
+    }
+    const coverageByScope = new Map(input.scopeCoverage.map((coverage) => [coverage.scopeId, coverage]));
+    for (const scope of input.scope) {
+        const coverage = coverageByScope.get(scope.id);
+        if (coverage === undefined ||
+            !coverage.fixtureIds.some((fixtureId) => input.proposal.suite.fixtures.some((fixture) => fixture.id === fixtureId &&
+                (scope.kind === "preserved" || fixture.expectationMode === "changes")))) {
+            findings.push({
+                kind: "contract",
+                message: scope.kind === "changed"
+                    ? `replacement must retain at least one mapped changes fixture for grounded scope ${JSON.stringify(scope.id)}`
+                    : `replacement must retain at least one mapped fixture for grounded scope ${JSON.stringify(scope.id)}`
+            });
+        }
+    }
+    return findings;
+};
+const runTournament = Effect.fnUntraced(function* (input) {
+    // A challenger must never replace the incumbent's checkpoint before the
+    // non-regression decision. Retain its partial execution in the work store.
+    const bindingDigest = checkpointDigestV1({
+        version: 1,
+        caseId: input.caseId,
+        seed: input.seed,
+        suite: input.suite,
+        solutions: input.solutions,
+        repetitions: input.repetitions
+    });
+    const workName = `pipeline-execution-tournament-${checkpointDigestV1(input.checkpoint?.operationId ?? input.caseId)}`;
+    const retained = input.checkpoint === undefined
+        ? Option.none()
+        : yield* input.checkpoint.store.readWorkCheckpoint(workName, Schema.Struct({
+            version: Schema.Literal(1),
+            bindingDigest: Schema.String,
+            oracle: RepositoryHiddenOracleV1
+        }));
+    const resumeOracle = Option.isSome(retained) && retained.value.bindingDigest === bindingDigest
+        ? retained.value.oracle
+        : input.resumeOracle;
+    if (input.checkpoint !== undefined && resumeOracle !== undefined) {
+        yield* reportFoundryHeartbeatV1({
+            role: "oracle-executor",
+            detail: `tournament challenger execution restored; observations=${String(resumeOracle.observations.length)}`
+        });
+    }
+    const oracle = yield* buildHistoricalHiddenOracleV1({
+        repositoryRoot: input.repositoryRoot,
+        caseId: input.caseId,
+        seed: input.seed,
+        additionalSolutions: input.solutions,
+        hiddenFixtureSuite: input.suite,
+        repetitions: input.repetitions,
+        oracleConcurrency: input.concurrency,
+        ...(resumeOracle === undefined
+            ? {}
+            : { resumeOracle }),
+        ...(input.onReplayCompleted === undefined && input.checkpoint === undefined
+            ? {}
+            : {
+                onReplayCompleted: (partialOracle) => Effect.gen(function* () {
+                    if (input.checkpoint !== undefined) {
+                        yield* input.checkpoint.store.writeWorkCheckpoint(workName, {
+                            version: 1, bindingDigest, oracle: partialOracle
+                        }).pipe(Effect.uninterruptible, Effect.orDie);
+                    }
+                    if (input.onReplayCompleted !== undefined)
+                        yield* input.onReplayCompleted({
+                            oracle: partialOracle,
+                            adequacy: evaluateRepositoryOracleAdequacyV1(partialOracle)
+                        });
+                })
+            })
+    });
+    return { oracle, adequacy: evaluateRepositoryOracleAdequacyV1(oracle) };
+});
+const checkpointOf = (input) => ({
+    version: 1,
+    stage: STAGE,
+    parentDigest: input.parentDigest,
+    createdAt: input.createdAt,
+    suite: input.suite,
+    adequacy: input.result.adequacy,
+    oracle: input.result.oracle,
+    valid: [...input.valid],
+    wrong: [...input.wrong],
+    critiques: [...(input.critiques ?? [])],
+    history: [...input.history],
+    iterations: input.iterations
+});
+/**
+ * Runs real, repeated isolated execution and hill-climbs only from measured
+ * escapes. Checkpoints are written after every accepted incumbent replacement.
+ */
+export const runPipelineTournamentCapabilityV1 = (input) => Effect.gen(function* () {
+    const seedCheckpoint = input.checkpoints.seed;
+    const spec = input.checkpoints.spec;
+    const oracleCheckpoint = input.checkpoints.oracle;
+    const controls = input.checkpoints.controls;
+    if (seedCheckpoint === undefined ||
+        spec === undefined ||
+        oracleCheckpoint === undefined ||
+        controls === undefined) {
+        return yield* failure("tournament requires seed, specification, oracle, and controls checkpoints");
+    }
+    const seed = seedCheckpoint.seed;
+    if (seed.status !== "qualified" || seed.referenceCommit === undefined) {
+        return yield* failure("tournament requires a replay-qualified pinned reference");
+    }
+    const parentDigest = checkpointDigestV1(controls);
+    const prior = input.checkpoints.tournament;
+    const resumable = prior !== undefined && prior.parentDigest === parentDigest;
+    const createdAt = resumable
+        ? prior.createdAt
+        : new Date(yield* Clock.currentTimeMillis).toISOString();
+    let suite = resumable ? prior.suite : oracleCheckpoint.suite;
+    let valid = resumable ? [...prior.valid] : [...controls.valid];
+    const wrong = resumable ? [...prior.wrong] : [...controls.wrong];
+    let critiques = resumable
+        ? [...(prior.critiques ?? controls.critiques)]
+        : [...controls.critiques];
+    let history = resumable ? [...prior.history] : [];
+    let iterations = resumable ? prior.iterations : 0;
+    // Check the same grounded execution obligation here and at final admission.
+    // Otherwise a generically adequate but scope-blind oracle freezes before
+    // its known, repairable evidence gap is discovered at the terminal gate.
+    const measureScope = (result) => {
+        const scopeCoverage = evaluateRepositoryOracleScopeCoverageV1({
+            spec, oracleCheckpoint, oracle: result.oracle,
+            repetitions: input.context.oracleRepetitions, label: "development"
+        });
+        return {
+            oracle: result.oracle,
+            scopeCoverage,
+            adequacy: {
+                ...result.adequacy,
+                admitted: result.adequacy.admitted && scopeCoverage.reasons.length === 0,
+                rejectionReasons: [...result.adequacy.rejectionReasons, ...scopeCoverage.reasons]
+            }
+        };
+    };
+    const snapshot = (result) => checkpointOf({
+        parentDigest,
+        createdAt,
+        suite,
+        valid,
+        wrong,
+        critiques,
+        history,
+        iterations,
+        result
+    });
+    yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), resumable ? "resuming repeated isolated tournament" : "running repeated isolated tournament");
+    let incumbent = yield* runTournament({
+        repositoryRoot: input.context.repositoryRoot,
+        caseId: input.context.caseId,
+        seed,
+        suite,
+        solutions: [...valid, ...wrong],
+        repetitions: input.context.oracleRepetitions,
+        concurrency: input.context.oracleConcurrency,
+        ...(resumable ? { resumeOracle: prior.oracle } : {}),
+        onReplayCompleted: (result) => input.checkpoint(snapshot(measureScope(result)))
+    }).pipe(Effect.map(measureScope));
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const assignment = repositoryFoundryModelAssignmentV1(input.context.modelPlan, "hill-climb-planner");
+    let boundedIterations = 0;
+    for (;;) {
+        if (!incumbent.adequacy.admitted &&
+            boundedIterations >= (input.maximumIterations ?? Number.MAX_SAFE_INTEGER)) {
+            const partial = snapshot(incumbent);
+            yield* input.checkpoint(partial);
+            return partial;
+        }
+        boundedIterations += 1;
+        if (incumbent.adequacy.admitted) {
+            const complete = snapshot(incumbent);
+            yield* input.checkpoint(complete);
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `development tournament adequate; iterations=${String(iterations)}`);
+            return complete;
+        }
+        const proposedValidFalseRejects = incumbent.adequacy.falseRejectedSolutionIds.filter((id) => valid.some((solution) => solution.id === id));
+        if (proposedValidFalseRejects.length > 0) {
+            const reviewed = yield* runPipelineControlCorrectnessCriticsV1({
+                ...(input.workCheckpointStore === undefined ? {} : {
+                    workCheckpointStore: input.workCheckpointStore
+                }),
+                context: input.context,
+                seed,
+                spec,
+                solutions: valid.filter((solution) => proposedValidFalseRejects.includes(solution.id)),
+                priorCritiques: critiques,
+                onRound: (roundCritiques) => {
+                    const reviewedIds = new Set(roundCritiques.map(({ solutionId }) => solutionId));
+                    critiques = [
+                        ...critiques.filter(({ solutionId }) => !reviewedIds.has(solutionId)),
+                        ...roundCritiques
+                    ];
+                    return input.checkpoint(snapshot(incumbent));
+                }
+            });
+            critiques = [
+                ...critiques.filter(({ solutionId }) => !proposedValidFalseRejects.includes(solutionId)),
+                ...reviewed.critiques
+            ];
+        }
+        const mislabeledValid = proposedValidFalseRejects.filter((id) => advisoryControlClassification(critiques, id) === "wrong");
+        if (mislabeledValid.length > 0) {
+            const incumbentValid = valid;
+            const incumbentCritiques = critiques;
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `replacing executed invalid controls without changing the oracle: ${mislabeledValid.join(",")}`);
+            const retainedValid = valid.filter((solution) => !mislabeledValid.includes(solution.id));
+            const replacementInput = {
+                context: input.context,
+                map: input.map,
+                replacementRound: iterations + 1,
+                ...(input.workCheckpointStore === undefined ? {} : {
+                    workCheckpointStore: input.workCheckpointStore
+                }),
+                checkpoints: {
+                    ...input.checkpoints,
+                    controls: {
+                        ...controls,
+                        valid: retainedValid,
+                        wrong,
+                        critiques: []
+                    }
+                },
+                checkpoint: () => Effect.void
+            };
+            const replacementControls = yield* runPipelineControlsCapabilityV1(replacementInput);
+            valid = [...replacementControls.valid];
+            critiques = [...replacementControls.critiques];
+            const replacementResult = yield* runTournament({
+                ...(input.workCheckpointStore === undefined ? {} : {
+                    checkpoint: {
+                        store: input.workCheckpointStore,
+                        operationId: `${input.context.caseId}:tournament:replacement:${String(iterations + 1)}`
+                    }
+                }),
+                repositoryRoot: input.context.repositoryRoot,
+                caseId: input.context.caseId,
+                seed,
+                suite,
+                solutions: [...valid, ...wrong],
+                repetitions: input.context.oracleRepetitions,
+                concurrency: input.context.oracleConcurrency
+            }).pipe(Effect.map(measureScope));
+            const replacementRejectionReasons = candidateRegressionReasons({
+                ...incumbent.adequacy,
+                falseRejectedSolutionIds: incumbent.adequacy.falseRejectedSolutionIds.filter((id) => !mislabeledValid.includes(id))
+            }, replacementResult.adequacy, incumbent.scopeCoverage.unresolvedScopeIds, replacementResult.scopeCoverage.unresolvedScopeIds);
+            const sequence = iterations + 1;
+            history = [
+                ...history,
+                {
+                    sequence,
+                    decision: replacementRejectionReasons.length === 0 ? "accepted" : "rejected",
+                    targeted: mislabeledValid,
+                    rejectionReasons: replacementRejectionReasons,
+                    candidateAdequacy: replacementResult.adequacy
+                }
+            ];
+            iterations = sequence;
+            if (replacementRejectionReasons.length === 0) {
+                incumbent = replacementResult;
+            }
+            else {
+                valid = incumbentValid;
+                critiques = incumbentCritiques;
+            }
+            yield* input.checkpoint(snapshot(incumbent));
+            continue;
+        }
+        const targeted = sortedUnique([
+            ...unresolvedIds(incumbent.adequacy),
+            ...(incumbent.scopeCoverage.unresolvedScopeIds.length === 0
+                ? [] : ["historical-reference", "historical-defect"])
+        ]);
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `repairing measured oracle escapes; iteration=${String(iterations + 1)}; targets=${targeted.join(",")}`);
+        const authored = yield* groundedStructuredGenerationV1({
+            checkpointStore: input.workCheckpointStore,
+            languageModel,
+            assignment,
+            operationId: `${input.context.caseId}:pipeline:tournament:repair:${String(iterations + 1)}`,
+            instructions: REPAIR_INSTRUCTIONS,
+            task: {
+                dimension: input.context.dimension,
+                visibleTask: spec.visible,
+                analysis: spec.analysis,
+                groundedScope: spec.scope,
+                scopeCoverage: oracleCheckpoint.scopeCoverage,
+                measuredScopeGaps: incumbent.scopeCoverage,
+                targetBehavior: seed.targetBehavior,
+                initialCommit: seed.initialState.commit,
+                referenceCommit: seed.referenceCommit,
+                incumbentSuite: suite,
+                measuredAdequacy: incumbent.adequacy,
+                measuredObservations: incumbent.oracle.observations.filter((observation) => targeted.includes(observation.solutionId)),
+                priorHistory: history
+            },
+            outputSchema: TournamentRepairProposalV1,
+            schemaName: "routekit_pipeline_tournament_repair_v1",
+            tools: {
+                repositoryRoot: input.context.repositoryRoot,
+                commits: { initial: seed.initialState.commit, reference: seed.referenceCommit },
+                allowedTestPaths: new Set(oracleCheckpoint.suite.fixtures.map((fixture) => fixture.testPath)),
+                seed,
+                runFixture: true
+            },
+            validateSubmission: (proposal) => Effect.succeed(validateProposal({
+                proposal,
+                caseId: input.context.caseId,
+                targeted,
+                scope: spec.scope,
+                scopeCoverage: oracleCheckpoint.scopeCoverage
+            })),
+            maxTurns: Math.max(1, Math.trunc(input.context.budget.maxModelCalls) + 1),
+            maximumOutputTokens: repositoryFoundryRoleOutputCeilingV1("hill-climb-planner")
+        });
+        const candidate = yield* runTournament({
+            ...(input.workCheckpointStore === undefined ? {} : {
+                checkpoint: {
+                    store: input.workCheckpointStore,
+                    operationId: `${input.context.caseId}:tournament:repair:${String(iterations + 1)}`
+                }
+            }),
+            repositoryRoot: input.context.repositoryRoot,
+            caseId: input.context.caseId,
+            seed,
+            suite: authored.value.suite,
+            solutions: [...valid, ...wrong],
+            repetitions: input.context.oracleRepetitions,
+            concurrency: input.context.oracleConcurrency
+        }).pipe(Effect.map(measureScope));
+        const rejectionReasons = candidateRejectionReasons(incumbent, candidate);
+        const sequence = iterations + 1;
+        if (rejectionReasons.length > 0) {
+            history = [
+                ...history,
+                {
+                    sequence,
+                    decision: "rejected",
+                    targeted,
+                    rejectionReasons,
+                    candidateAdequacy: candidate.adequacy
+                }
+            ];
+            iterations = sequence;
+            yield* input.checkpoint(snapshot(incumbent));
+            continue;
+        }
+        suite = authored.value.suite;
+        incumbent = candidate;
+        history = [
+            ...history,
+            {
+                sequence,
+                decision: "accepted",
+                targeted,
+                rejectionReasons: [],
+                candidateAdequacy: candidate.adequacy
+            }
+        ];
+        iterations = sequence;
+        yield* input.checkpoint(snapshot(incumbent));
+    }
+}).pipe(Effect.mapError(normalizePipelineFailureV1));
+export const runPipelineTournamentStageV1 = (input) => runPipelineTournamentCapabilityV1(input);