@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/services/pipeline-seed/service.js

@@ -0,0 +1,428 @@
+import { Clock, Effect, Schema } from "effect";
+import { discoverIntegratedRepositoryHistoryV1 } from "../../adapters/integrated-repository-history.js";
+import { rankRepositorySeedCandidatesV1 } from "../../adapters/repository-seed-selection.js";
+import { checkpointDigestV1, PipelineStageError, pipelineStageToProgressStageV1 } from "../../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { REPOSITORY_FOUNDRY_STANDARD_OUTPUT_CEILING } from "../../repository-foundry-plan-protocol.js";
+import { generateWithFeedbackV1 } from "../budgeted-model/service.js";
+import { reportFoundryStageV1 } from "../foundry-progress/service.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+import { extractHistoricalTaskSeedsV1, qualifyHistoricalTaskSeedAcrossBatchesV1 } from "../task-seed/service.js";
+const STAGE = "seed";
+const QUALIFICATION_BATCHES = 2;
+const QUALIFICATION_REPETITIONS_PER_BATCH = 2;
+/**
+ * The ranker may return a useful prefix. Omitting candidates is not a terminal
+ * exact-count failure: omitted candidates retain their lexical prefilter order
+ * and are still replay-qualified.
+ */
+export const PipelineSeedRankingV1 = Schema.Struct({
+    candidates: Schema.Array(Schema.Struct({
+        commit: Schema.String,
+        score: Schema.Finite,
+        rationale: Schema.String
+    }))
+});
+const PipelineSeedDimensionFitV1 = Schema.Struct({
+    commit: Schema.String,
+    requestedDimension: Schema.String,
+    outcome: Schema.Literals(["fit", "mismatch", "uncertain"]),
+    detail: Schema.String,
+    includes: Schema.Array(Schema.Struct({
+        requirement: Schema.String,
+        outcome: Schema.Literals(["supports", "does-not-support", "uncertain"]),
+        detail: Schema.String
+    })),
+    excludes: Schema.Array(Schema.Struct({
+        requirement: Schema.String,
+        outcome: Schema.Literals(["respected", "violated", "uncertain"]),
+        detail: Schema.String
+    }))
+});
+const stageFailure = (detail, terminal, cause) => new RepositoryFoundryError({
+    operation: "run-pipeline-stage",
+    detail,
+    cause: new PipelineStageError({ stage: STAGE, detail, terminal, cause })
+});
+const describeError = (error) => error.detail.split("\n", 1)[0]?.slice(0, 1_000) ?? error.message.slice(0, 1_000);
+const dimensionRetrievalQuery = (dimension) => [dimension.id, dimension.description, ...dimension.includes, ...dimension.excludes].join("\n");
+const episodeModelInput = (candidate, map) => ({
+    commit: candidate.episode.commit,
+    subject: candidate.episode.subject,
+    body: candidate.episode.body,
+    occurredAt: candidate.episode.occurredAt,
+    changedPaths: candidate.episode.changedPaths,
+    implementationPaths: candidate.episode.implementationPaths,
+    protocolPaths: candidate.episode.protocolPaths,
+    testPaths: candidate.episode.testPaths,
+    testNames: map.tests
+        .filter((test) => candidate.episode.testPaths.includes(test.path))
+        .flatMap((test) => test.names),
+    lexicalPrefilter: {
+        relevance: candidate.relevance,
+        matchedTerms: candidate.matchedTerms
+    }
+});
+const explicitCandidate = Effect.fnUntraced(function* (input) {
+    const ranked = rankRepositorySeedCandidatesV1(input.map, dimensionRetrievalQuery(input.dimension));
+    const candidate = ranked.find(({ episode }) => episode.commit === input.referenceCommit);
+    if (candidate === undefined) {
+        return yield* stageFailure(`the requested reference ${input.referenceCommit} is not an eligible executable historical change`, true);
+    }
+    return { map: input.map, candidates: [candidate] };
+});
+const discoveredCandidates = Effect.fnUntraced(function* (input) {
+    const retained = yield* discoverIntegratedRepositoryHistoryV1({
+        repositoryRoot: input.map.repository.root,
+        requestedRef: input.map.repository.requestedRef,
+        requestedTipCommit: input.map.repository.commit,
+        requestedTipTree: input.map.repository.tree,
+        integrationCommits: input.map.historyEpisodes.map(({ commit }) => commit)
+    });
+    const excludedRoots = input.map.blindSpots.find(({ id }) => id === "blind-excluded-package-roots")?.paths ?? [];
+    const permittedRetained = retained.filter(({ episode }) => !episode.changedPaths.some(({ path }) => excludedRoots.some((root) => path === root || path.startsWith(`${root}/`))));
+    const map = {
+        ...input.map,
+        historyEpisodes: [
+            ...input.map.historyEpisodes,
+            ...permittedRetained.map(({ episode }) => episode)
+        ]
+    };
+    const candidates = rankRepositorySeedCandidatesV1(map, dimensionRetrievalQuery(input.dimension));
+    return { map, candidates, retained: permittedRetained };
+});
+const rankWithModel = Effect.fnUntraced(function* (input) {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const permittedCommits = new Set(input.candidates.map(({ episode }) => episode.commit));
+    const ranked = yield* generateWithFeedbackV1({
+        languageModel,
+        assignment: input.assignment,
+        operationId: `${input.caseId}:seed:rank`,
+        instructions: [
+            "Rank the supplied historical changes for their fit to the COMPLETE routing dimension contract.",
+            "The dimension description, every include, and every exclude are binding context.",
+            "Use concrete changed paths, protocols, tests, and behavior evidence rather than commit wording alone.",
+            "Scores are comparative fit signals, not admission evidence; execution will replay-qualify candidates.",
+            "Return the best candidates first with a finite score and a concise evidence-based rationale.",
+            "You may omit candidates that are clearly weaker. Do not invent commits."
+        ].join("\n"),
+        input: {
+            dimension: input.dimension,
+            repository: input.map.repository,
+            ...(input.requestedRef === undefined ? {} : { requestedRef: input.requestedRef }),
+            ...(input.referenceCommit === undefined
+                ? {}
+                : { requiredReferenceCommit: input.referenceCommit }),
+            candidates: input.candidates.map((candidate) => episodeModelInput(candidate, input.map))
+        },
+        schemaName: "PipelineSeedRankingV1",
+        outputSchema: PipelineSeedRankingV1,
+        maximumOutputTokens: REPOSITORY_FOUNDRY_STANDARD_OUTPUT_CEILING,
+        maxAttempts: Number.MAX_SAFE_INTEGER,
+        validate: (proposal) => Effect.sync(() => {
+            const findings = [];
+            const seen = new Set();
+            for (const candidate of proposal.candidates) {
+                if (!permittedCommits.has(candidate.commit)) {
+                    findings.push({
+                        kind: "contract",
+                        message: `ranking contains a commit that was not supplied: ${candidate.commit}`
+                    });
+                }
+                if (seen.has(candidate.commit)) {
+                    findings.push({
+                        kind: "contract",
+                        message: `ranking contains duplicate commit ${candidate.commit}`
+                    });
+                }
+                if (candidate.rationale.trim().length === 0) {
+                    findings.push({
+                        kind: "contract",
+                        message: `ranking rationale is empty for commit ${candidate.commit}`
+                    });
+                }
+                seen.add(candidate.commit);
+            }
+            if (![...seen].some((commit) => permittedCommits.has(commit))) {
+                findings.push({
+                    kind: "contract",
+                    message: "ranking must contain at least one of the supplied commits"
+                });
+            }
+            return findings;
+        })
+    });
+    return ranked.value;
+});
+const assessQualifiedDimensionFit = Effect.fnUntraced(function* (input) {
+    const languageModel = yield* RepositoryFoundryLanguageModel;
+    const assessed = yield* generateWithFeedbackV1({
+        languageModel,
+        assignment: input.assignment,
+        operationId: `${input.caseId}:seed:dimension-fit:${input.candidate.episode.commit}`,
+        instructions: [
+            "Assess this replay-qualified historical change against the complete routing dimension boundary.",
+            "The description, every include, and every exclude are binding context. Address each supplied include and exclude explicitly.",
+            "Use the historical change facts, extracted target behavior, and executed replay observations together.",
+            "Replay qualification proves that the change is executable; it does not by itself prove dimension fit.",
+            "A historical change is not expected to cover every example listed under includes. Return fit when the authentic change directly exercises a coherent non-empty slice of the requested dimension and respects every exclusion.",
+            "Return mismatch when it supports none of the includes or belongs to an excluded domain. Return uncertain when the supplied evidence cannot establish either conclusion.",
+            "This assessment guides candidate selection. It is advisory and never substitutes for later executable evidence."
+        ].join("\n"),
+        input: {
+            dimension: input.dimension,
+            candidate: episodeModelInput(input.candidate, input.map),
+            seed: input.seed,
+            qualification: input.qualification
+        },
+        schemaName: "PipelineSeedDimensionFitV1",
+        outputSchema: PipelineSeedDimensionFitV1,
+        maximumOutputTokens: REPOSITORY_FOUNDRY_STANDARD_OUTPUT_CEILING,
+        maxAttempts: Number.MAX_SAFE_INTEGER,
+        validate: (proposal) => Effect.sync(() => {
+            const findings = [];
+            if (proposal.commit !== input.candidate.episode.commit) {
+                findings.push({
+                    kind: "contract",
+                    message: `dimension assessment must identify commit ${input.candidate.episode.commit}`
+                });
+            }
+            if (proposal.requestedDimension !== input.dimension.id) {
+                findings.push({
+                    kind: "contract",
+                    message: `dimension assessment must identify requested dimension ${input.dimension.id}`
+                });
+            }
+            const includes = new Set(proposal.includes.map(({ requirement }) => requirement));
+            const excludes = new Set(proposal.excludes.map(({ requirement }) => requirement));
+            if (includes.size !== proposal.includes.length) {
+                findings.push({
+                    kind: "contract",
+                    message: "dimension assessment contains a duplicate include"
+                });
+            }
+            if (excludes.size !== proposal.excludes.length) {
+                findings.push({
+                    kind: "contract",
+                    message: "dimension assessment contains a duplicate exclude"
+                });
+            }
+            for (const requirement of input.dimension.includes) {
+                if (!includes.has(requirement)) {
+                    findings.push({
+                        kind: "contract",
+                        message: `dimension assessment omitted include ${JSON.stringify(requirement)}`
+                    });
+                }
+            }
+            for (const requirement of input.dimension.excludes) {
+                if (!excludes.has(requirement)) {
+                    findings.push({
+                        kind: "contract",
+                        message: `dimension assessment omitted exclude ${JSON.stringify(requirement)}`
+                    });
+                }
+            }
+            if (proposal.includes.some(({ requirement }) => !input.dimension.includes.includes(requirement)) ||
+                proposal.excludes.some(({ requirement }) => !input.dimension.excludes.includes(requirement))) {
+                findings.push({
+                    kind: "contract",
+                    message: "dimension assessment contains an include or exclude that was not supplied"
+                });
+            }
+            if (proposal.outcome === "fit" &&
+                (proposal.includes.every(({ outcome }) => outcome !== "supports") ||
+                    proposal.excludes.some(({ outcome }) => outcome !== "respected"))) {
+                findings.push({
+                    kind: "contract",
+                    message: "a fit assessment requires at least one supported include and every exclude to be respected"
+                });
+            }
+            for (const assessment of [...proposal.includes, ...proposal.excludes]) {
+                if (assessment.detail.trim().length === 0) {
+                    findings.push({
+                        kind: "contract",
+                        message: `dimension assessment detail is empty for ${JSON.stringify(assessment.requirement)}`
+                    });
+                }
+            }
+            if (proposal.detail.trim().length === 0) {
+                findings.push({
+                    kind: "contract",
+                    message: "dimension assessment detail must be non-empty"
+                });
+            }
+            return findings;
+        })
+    });
+    return assessed.value;
+});
+const orderCandidates = (candidates, ranking) => {
+    const byCommit = new Map(candidates.map((candidate) => [candidate.episode.commit, candidate]));
+    const ordered = [];
+    const seen = new Set();
+    for (const ranked of ranking.candidates) {
+        const candidate = byCommit.get(ranked.commit);
+        if (candidate === undefined || seen.has(ranked.commit))
+            continue;
+        seen.add(ranked.commit);
+        ordered.push({ ...candidate, score: ranked.score, rationale: ranked.rationale.trim() });
+    }
+    for (const candidate of candidates) {
+        if (seen.has(candidate.episode.commit))
+            continue;
+        ordered.push({
+            ...candidate,
+            score: candidate.relevance,
+            rationale: "The model omitted this candidate; it remains in lexical prefilter order for execution-backed replay qualification."
+        });
+    }
+    return ordered;
+};
+const qualifyCandidate = Effect.fnUntraced(function* (input) {
+    const episode = input.map.historyEpisodes.find(({ commit }) => commit === input.candidate.episode.commit);
+    if (episode === undefined) {
+        return yield* stageFailure(`the selected Git reference no longer exposes historical episode ${input.candidate.episode.commit}`, false);
+    }
+    const seeds = yield* extractHistoricalTaskSeedsV1({
+        ...input.map,
+        historyEpisodes: [episode]
+    });
+    const seed = seeds.find(({ referenceCommit }) => referenceCommit === episode.commit);
+    if (seed === undefined) {
+        return yield* stageFailure(`no executable task seed could be extracted for ${episode.commit}`, false);
+    }
+    const qualification = yield* qualifyHistoricalTaskSeedAcrossBatchesV1({
+        repositoryRoot: input.map.repository.root,
+        seed,
+        batches: QUALIFICATION_BATCHES,
+        repetitionsPerBatch: QUALIFICATION_REPETITIONS_PER_BATCH
+    });
+    return { map: input.map, seed, qualification };
+});
+const isQualified = (qualification) => qualification.stability.stable && qualification.seed.status === "qualified";
+const rejectionReason = (input) => {
+    const reasons = [
+        ...input.qualification.stability.rejectionReasons,
+        ...input.seed.rejectionReasons
+    ];
+    return reasons.length > 0
+        ? reasons.join("; ").slice(0, 1_000)
+        : "executed replay qualification did not produce a stable qualified seed";
+};
+/** Rank history once, then let repeated pinned execution select the first viable seed. */
+export const runPipelineSeedStageV1 = (input) => Effect.gen(function* () {
+    yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; status=discovering-candidates`);
+    const discovery = input.context.referenceCommit === undefined
+        ? yield* discoveredCandidates({ map: input.map, dimension: input.context.dimension })
+        : yield* explicitCandidate({
+            map: input.map,
+            referenceCommit: input.context.referenceCommit,
+            dimension: input.context.dimension
+        });
+    if (discovery.candidates.length === 0) {
+        return yield* stageFailure(`repository history contains no executable candidate for dimension ${input.context.dimension.id}`, true);
+    }
+    const candidates = input.context.referenceCommit === undefined
+        ? yield* Effect.gen(function* () {
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; status=ranking; candidates=${String(discovery.candidates.length)}`);
+            const ranking = yield* rankWithModel({
+                caseId: input.context.caseId,
+                dimension: input.context.dimension,
+                ...(input.context.requestedRef === undefined
+                    ? {}
+                    : { requestedRef: input.context.requestedRef }),
+                candidates: discovery.candidates,
+                map: discovery.map,
+                assignment: {
+                    ...input.context.modelPlan.assignments.find(({ role }) => role === "repository-analyst"),
+                    role: "repository-analyst"
+                }
+            });
+            return orderCandidates(discovery.candidates, ranking);
+        })
+        : discovery.candidates.map((candidate) => ({
+            ...candidate,
+            score: candidate.relevance,
+            rationale: "The reference commit was explicitly requested, so execution-backed replay qualification and complete dimension assessment determine eligibility without comparative model ranking."
+        }));
+    const rejected = [];
+    const dimensionFit = new Map();
+    const assignment = {
+        ...input.context.modelPlan.assignments.find(({ role }) => role === "repository-analyst"),
+        role: "repository-analyst"
+    };
+    for (const [index, candidate] of candidates.entries()) {
+        const commit = candidate.episode.commit;
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; attempt=${String(index + 1)}; status=replay-qualifying`);
+        const attempted = yield* Effect.result(qualifyCandidate({ map: discovery.map, candidate }));
+        if (attempted._tag === "Failure") {
+            const reason = describeError(attempted.failure);
+            rejected.push({ commit, reason });
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; status=rejected; reason=${reason}`);
+            continue;
+        }
+        if (!isQualified(attempted.success.qualification)) {
+            const reason = rejectionReason({
+                qualification: attempted.success.qualification,
+                seed: attempted.success.seed
+            });
+            rejected.push({ commit, reason });
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; status=rejected; reason=${reason}`);
+            continue;
+        }
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; status=assessing-dimension-fit`);
+        const fit = yield* assessQualifiedDimensionFit({
+            caseId: input.context.caseId,
+            dimension: input.context.dimension,
+            candidate,
+            map: attempted.success.map,
+            seed: attempted.success.seed,
+            qualification: attempted.success.qualification,
+            assignment
+        });
+        if (fit.outcome !== "fit") {
+            const reason = `complete dimension assessment was ${fit.outcome}: ${fit.detail}; ` +
+                `includes=${fit.includes
+                    .map(({ requirement, outcome }) => `${JSON.stringify(requirement)}:${outcome}`)
+                    .join(",")}; ` +
+                `excludes=${fit.excludes
+                    .map(({ requirement, outcome }) => `${JSON.stringify(requirement)}:${outcome}`)
+                    .join(",")}`;
+            rejected.push({ commit, reason: reason.slice(0, 1_000) });
+            yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; status=rejected; reason=${reason.slice(0, 1_000)}`);
+            continue;
+        }
+        dimensionFit.set(commit, fit.detail);
+        const createdAt = new Date(yield* Clock.currentTimeMillis).toISOString();
+        const checkpoint = {
+            version: 1,
+            stage: STAGE,
+            parentDigest: checkpointDigestV1(input.context),
+            createdAt,
+            seed: attempted.success.qualification.seed,
+            qualification: attempted.success.qualification,
+            repository: attempted.success.map.repository,
+            selection: {
+                candidates: candidates.map((entry) => ({
+                    commit: entry.episode.commit,
+                    subject: entry.episode.subject,
+                    score: entry.score,
+                    rationale: dimensionFit.get(entry.episode.commit) === undefined
+                        ? entry.rationale
+                        : `${entry.rationale} Dimension fit: ${dimensionFit.get(entry.episode.commit)}`
+                })),
+                selectedCommit: commit,
+                rejected
+            }
+        };
+        yield* input.checkpoint(checkpoint);
+        yield* reportFoundryStageV1(pipelineStageToProgressStageV1(STAGE), `case=${input.context.caseId}; candidate=${commit}; status=selected`);
+        return checkpoint;
+    }
+    return yield* stageFailure(`no historical reference both replay-qualified and fit dimension ${input.context.dimension.id}: ${rejected
+        .map(({ commit, reason }) => `${commit}: ${reason}`)
+        .join("; ")}`, true);
+}).pipe(Effect.withSpan("PipelineSeed.run"));
+/** Short name used by the orchestrator. */
+export const runPipelineSeedV1 = runPipelineSeedStageV1;

package/dist/services/pipeline-spec/service.d.ts

@@ -0,0 +1,103 @@
+import { Effect } from "effect";
+import { type PipelineStageRunV1, type PipelineStageInputV1, type SpecCheckpointV1 } from "../../case-pipeline-protocol.js";
+import type { RepositoryTaskSeedV1 } from "../../repository-task-seed-protocol.js";
+import { RepositoryFoundryError } from "../../errors.js";
+import { RepositoryFoundryLanguageModel } from "../language-model/service.js";
+/** Source-only authoring also accepts an unqualified draft seed. It never executes repository recipes. */
+export declare const draftPipelineSpecificationV1: (input: PipelineStageInputV1<{
+    readonly visible: {
+        readonly lane: "request";
+        readonly id: string;
+        readonly request: {
+            readonly endpoint: "chat" | "responses" | "anthropic";
+            readonly method: "POST";
+            readonly path: string;
+            readonly headers: readonly {
+                readonly name: string;
+                readonly value: string;
+            }[];
+            readonly bodyJson: string;
+        };
+    } | {
+        readonly lane: "repository-agent";
+        readonly id: string;
+        readonly requestText: string;
+        readonly checkoutCommit: string;
+        readonly allowedTools: readonly ("test" | "read" | "search" | "edit" | "shell")[];
+        readonly constraints: readonly string[];
+    };
+    readonly analysis: {
+        readonly summary: string;
+        readonly userObservableBehaviors: readonly string[];
+        readonly constraints: readonly string[];
+        readonly risks: readonly string[];
+    };
+    readonly scope: readonly {
+        readonly id: string;
+        readonly kind: "preserved" | "changed";
+        readonly behaviorIds: readonly string[];
+        readonly description: string;
+    }[];
+    readonly contextSources: readonly {
+        readonly path: string;
+        readonly content: string;
+    }[];
+    readonly reviews: readonly unknown[];
+    readonly revisions: number;
+    readonly leakFindings: readonly string[];
+    readonly version: 1;
+    readonly stage: "spec";
+    readonly parentDigest: string;
+    readonly createdAt: string;
+}> & {
+    readonly seed: RepositoryTaskSeedV1;
+    readonly parentDigest: string;
+    readonly maximumReviewRounds?: number;
+}) => Effect.Effect<{
+    readonly visible: {
+        readonly lane: "request";
+        readonly id: string;
+        readonly request: {
+            readonly endpoint: "chat" | "responses" | "anthropic";
+            readonly method: "POST";
+            readonly path: string;
+            readonly headers: readonly {
+                readonly name: string;
+                readonly value: string;
+            }[];
+            readonly bodyJson: string;
+        };
+    } | {
+        readonly lane: "repository-agent";
+        readonly id: string;
+        readonly requestText: string;
+        readonly checkoutCommit: string;
+        readonly allowedTools: readonly ("test" | "read" | "search" | "edit" | "shell")[];
+        readonly constraints: readonly string[];
+    };
+    readonly analysis: {
+        readonly summary: string;
+        readonly userObservableBehaviors: readonly string[];
+        readonly constraints: readonly string[];
+        readonly risks: readonly string[];
+    };
+    readonly scope: readonly {
+        readonly id: string;
+        readonly kind: "preserved" | "changed";
+        readonly behaviorIds: readonly string[];
+        readonly description: string;
+    }[];
+    readonly contextSources: readonly {
+        readonly path: string;
+        readonly content: string;
+    }[];
+    readonly reviews: readonly unknown[];
+    readonly revisions: number;
+    readonly leakFindings: readonly string[];
+    readonly version: 1;
+    readonly stage: "spec";
+    readonly parentDigest: string;
+    readonly createdAt: string;
+}, RepositoryFoundryError | import("../../case-pipeline-protocol.js").PipelineBudgetExhaustedError, RepositoryFoundryLanguageModel>;
+/** Legacy callers still require and preserve their qualified seed checkpoint. */
+export declare const runPipelineSpecStageV1: PipelineStageRunV1<SpecCheckpointV1>;