@velum-labs/routekit-eval-setup 1.0.26 → 1.1.0

package/dist/services/basis-repair/service.js

@@ -0,0 +1,249 @@
+import { assertRoutingBasisV3, routingContractDigest } from "@velum-labs/routekit-eval-contracts";
+import { Clock, Context, Effect, Layer, Schema } from "effect";
+import { readFrozenRepositorySourcesV3 } from "../../adapters/git-basis-snapshot.js";
+import { BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT, BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT } from "../../basis-candidates.js";
+import { EvalProjectAuthoringError, EvalProjectTransitionError } from "../../errors.js";
+import { EvalProjectArtifacts } from "../../project-artifacts.js";
+import { EvalAuthoringTransport } from "../../project-authoring.js";
+import { EVAL_PROJECT_V2_VERSION } from "../../project-contracts.js";
+import { EvalProjectStore } from "../../project-store.js";
+import { AUTHORED_BASIS_JSON_SCHEMA, AuthoredBasis, withBasisDigest } from "../basis-authoring/service.js";
+const REPAIR_INSTRUCTIONS = [
+    "Repair one immutable RouteKit Classification V3 Routing Basis.",
+    "Return the same flat number of peer workload dimensions as the source basis.",
+    "Use only the supplied source basis, structural warnings, reviewed error categories, frozen repository sources, and construction tasks.",
+    "Do not infer or mention validation labels, validation predictions, metric values, final tasks, final changed paths, routing models, model winners, provider traces, credentials, or hidden reasoning.",
+    "Preserve the repository identity and improve semantic separability without introducing repository-facet, file-type, test-layer, documentation-layer, or implementation-layer dimensions.",
+    "Return only strict structured JSON."
+].join("\n");
+const failure = (detail, cause) => new EvalProjectAuthoringError({
+    operation: "authoring-basis",
+    detail,
+    ...(cause === undefined ? {} : { cause })
+});
+const transition = (state, detail) => new EvalProjectTransitionError({ state, detail });
+function expectedSummaryDigest(summary) {
+    return routingContractDigest("routekit.basis-repair-summary.v1", "summary", {
+        version: summary.version,
+        sourceCandidateId: summary.sourceCandidateId,
+        sourceBasisDigest: summary.sourceBasisDigest,
+        errorCategories: summary.errorCategories
+    });
+}
+export class BasisRepair extends Context.Service()("@velum-labs/routekit-eval-setup/BasisRepair") {
+}
+export const makeBasisRepair = Effect.gen(function* () {
+    const store = yield* EvalProjectStore;
+    const artifacts = yield* EvalProjectArtifacts;
+    const requireReview = (repositoryRoot) => store.loadDocument(repositoryRoot).pipe(Effect.flatMap((document) => document === undefined ||
+        document.version !== EVAL_PROJECT_V2_VERSION ||
+        (document.stage !== "basis-validation-review" &&
+            document.stage !== "basis-qualified")
+        ? Effect.fail(transition(document?.stage ?? "missing", "basis repair requires completed validation review"))
+        : Effect.succeed(document)));
+    return BasisRepair.of({
+        plan: (input) => Effect.gen(function* () {
+            const state = yield* requireReview(input.repositoryRoot);
+            if (input.sourceCandidateId !== state.selection.selectedCandidateId) {
+                return yield* transition(state.stage, "repair must start from the persisted selected original candidate");
+            }
+            const source = state.candidates.find((candidate) => candidate.candidateId === input.sourceCandidateId);
+            if (source === undefined ||
+                input.summary.sourceCandidateId !== source.candidateId ||
+                input.summary.sourceBasisDigest !== source.basisDigest ||
+                input.summary.summaryDigest !==
+                    expectedSummaryDigest(input.summary)) {
+                return yield* transition(state.stage, "repair summary does not bind the exact selected candidate");
+            }
+            const semantic = {
+                version: 1,
+                repositoryDigest: state.snapshot.repositoryDigest,
+                sourceCandidateId: source.candidateId,
+                sourceBasisDigest: source.basisDigest,
+                summaryDigest: input.summary.summaryDigest,
+                authorModel: input.authorModel,
+                plannedPrimaryCalls: 1,
+                maximumAttempts: 2,
+                requiredTotalInputTokens: BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT * 2,
+                requiredTotalOutputTokens: BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT * 2
+            };
+            return {
+                ...semantic,
+                planDigest: routingContractDigest("routekit.basis-repair-plan.v1", "plan", semantic)
+            };
+        }),
+        execute: (input) => Effect.gen(function* () {
+            const transport = yield* EvalAuthoringTransport;
+            const state = yield* requireReview(input.repositoryRoot);
+            const source = state.candidates.find((candidate) => candidate.candidateId === input.plan.sourceCandidateId);
+            if (source === undefined ||
+                input.plan.sourceCandidateId !==
+                    state.selection.selectedCandidateId ||
+                input.summary.sourceCandidateId !== source.candidateId ||
+                input.summary.sourceBasisDigest !== source.basisDigest ||
+                input.summary.summaryDigest !==
+                    expectedSummaryDigest(input.summary)) {
+                return yield* transition(state.stage, "repair inputs do not bind the selected original candidate");
+            }
+            const expectedSemantic = {
+                version: 1,
+                repositoryDigest: state.snapshot.repositoryDigest,
+                sourceCandidateId: source.candidateId,
+                sourceBasisDigest: source.basisDigest,
+                summaryDigest: input.summary.summaryDigest,
+                authorModel: input.plan.authorModel,
+                plannedPrimaryCalls: 1,
+                maximumAttempts: 2,
+                requiredTotalInputTokens: BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT * 2,
+                requiredTotalOutputTokens: BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT * 2
+            };
+            const expectedPlan = {
+                ...expectedSemantic,
+                planDigest: routingContractDigest("routekit.basis-repair-plan.v1", "plan", expectedSemantic)
+            };
+            if (JSON.stringify(expectedPlan) !== JSON.stringify(input.plan)) {
+                return yield* transition(state.stage, "repair plan differs from the frozen reviewed inputs");
+            }
+            const existing = yield* artifacts.loadBasisRepairV3(input.repositoryRoot);
+            if (existing !== undefined) {
+                if (existing.plan.planDigest !== input.plan.planDigest ||
+                    existing.summary.summaryDigest !== input.summary.summaryDigest) {
+                    return yield* transition(state.stage, "a different immutable repair artifact already exists");
+                }
+                return existing;
+            }
+            const structure = state.structureReports.find((report) => report.candidateId === source.candidateId);
+            const { validation: _sourceValidation, ...sourceSemanticBasis } = source.basis;
+            const sources = yield* readFrozenRepositorySourcesV3({
+                repositoryRoot: input.repositoryRoot,
+                tree: state.snapshot.tree,
+                sourceInventory: state.sourceInventory
+            });
+            const sourceDigest = routingContractDigest("routekit.basis-repair-sources.v1", "sources", sources);
+            let candidate;
+            let lastCause;
+            for (let attempt = 1; attempt <= 2 && candidate === undefined; attempt += 1) {
+                const outcome = yield* transport.complete({
+                    operationId: `basis-repair-${state.projectId}-${input.plan.planDigest.slice(0, 16)}-${String(attempt)}`,
+                    model: input.plan.authorModel,
+                    instructions: attempt === 1
+                        ? REPAIR_INSTRUCTIONS
+                        : `${REPAIR_INSTRUCTIONS}\nThe previous output failed schema or semantic validation. Correct only the contract.`,
+                    input: JSON.stringify({
+                        sourceBasis: sourceSemanticBasis,
+                        structuralWarnings: structure?.findings.map(({ severity, code, dimensionIds, detail }) => ({
+                            severity,
+                            code,
+                            dimensionIds,
+                            detail
+                        })) ?? [],
+                        reviewedErrorCategories: input.summary.errorCategories,
+                        frozenRepository: {
+                            repositoryDigest: state.snapshot.repositoryDigest,
+                            commit: state.snapshot.commit,
+                            tree: state.snapshot.tree,
+                            sources
+                        },
+                        constructionTasks: state.split.construction
+                    }),
+                    schemaName: "routekit_repaired_routing_basis_v3",
+                    jsonSchema: AUTHORED_BASIS_JSON_SCHEMA,
+                    maximumOutputTokens: BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT
+                }).pipe(Effect.flatMap((text) => Effect.gen(function* () {
+                    const parsed = yield* Effect.try({
+                        try: () => JSON.parse(text),
+                        catch: (cause) => failure("repair model returned invalid JSON", cause)
+                    });
+                    return yield* Schema.decodeUnknownEffect(AuthoredBasis)(parsed).pipe(Effect.mapError((cause) => failure("repair model returned an invalid basis shape", cause)));
+                })), Effect.match({
+                    onFailure: (cause) => ({ ok: false, cause }),
+                    onSuccess: (authored) => ({ ok: true, authored })
+                }));
+                if (!outcome.ok) {
+                    lastCause = outcome.cause;
+                    continue;
+                }
+                const candidateId = `repair-${input.plan.planDigest.slice(0, 16)}`;
+                const facetSnapshot = {
+                    ...outcome.authored.facetSnapshot,
+                    repositoryDigest: state.snapshot.repositoryDigest
+                };
+                const basis = yield* Effect.try({
+                    try: () => {
+                        const value = withBasisDigest({
+                            version: 3,
+                            basisId: candidateId,
+                            repositoryDigest: state.snapshot.repositoryDigest,
+                            facetSnapshot,
+                            dimensions: outcome.authored
+                                .dimensions,
+                            provenance: {
+                                generatorVersion: "basis-repair-v1",
+                                recipeId: candidateId,
+                                authorModel: input.plan.authorModel,
+                                sourceDigests: [
+                                    state.snapshot.repositoryDigest,
+                                    state.snapshot.sourceInventoryDigest,
+                                    source.basisDigest,
+                                    input.summary.summaryDigest,
+                                    sourceDigest
+                                ],
+                                origin: {
+                                    kind: "repair",
+                                    sourceBasisId: source.basis.basisId,
+                                    sourceBasisVersion: source.basis.version
+                                }
+                            }
+                        });
+                        if (value.dimensions.length !==
+                            source.basis.dimensions.length) {
+                            throw new Error("repair must preserve the source dimension count");
+                        }
+                        assertRoutingBasisV3(value);
+                        return value;
+                    },
+                    catch: (cause) => failure("repaired basis failed semantic validation", cause)
+                }).pipe(Effect.match({
+                    onFailure: (cause) => ({ ok: false, cause }),
+                    onSuccess: (value) => ({ ok: true, value })
+                }));
+                if (!basis.ok) {
+                    lastCause = basis.cause;
+                    continue;
+                }
+                candidate = {
+                    candidateId,
+                    recipeId: candidateId,
+                    basis: basis.value,
+                    basisDigest: basis.value.basisDigest,
+                    authorModel: input.plan.authorModel,
+                    generationConfigurationDigest: routingContractDigest("routekit.basis-repair-configuration.v1", "configuration", {
+                        instructions: REPAIR_INSTRUCTIONS,
+                        schema: AUTHORED_BASIS_JSON_SCHEMA,
+                        maximumOutputTokens: BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT,
+                        planDigest: input.plan.planDigest
+                    }),
+                    evidenceDigest: routingContractDigest("routekit.basis-repair-evidence.v1", "evidence", {
+                        repositoryDigest: state.snapshot.repositoryDigest,
+                        sourceBasisDigest: source.basisDigest,
+                        summaryDigest: input.summary.summaryDigest,
+                        sourceDigest
+                    }),
+                    createdAt: new Date(yield* Clock.currentTimeMillis).toISOString()
+                };
+            }
+            if (candidate === undefined) {
+                return yield* failure("basis repair exhausted its two approved attempts", lastCause);
+            }
+            const artifactsValue = {
+                version: 1,
+                plan: input.plan,
+                summary: input.summary,
+                candidate
+            };
+            yield* artifacts.saveBasisRepairV3(input.repositoryRoot, artifactsValue);
+            return artifactsValue;
+        })
+    });
+});
+export const BasisRepairLive = Layer.effect(BasisRepair, makeBasisRepair);

package/dist/test/basis-authoring.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/basis-authoring.test.js

@@ -0,0 +1,194 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { readFileSync } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { promisify } from "node:util";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { Effect, Layer } from "effect";
+import { BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT, BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT } from "../basis-candidates.js";
+import { EvalProjectArtifactsLive } from "../project-artifacts.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { EvalProjectStoreLive } from "../project-store.js";
+import { BasisAuthoring, BasisAuthoringLive } from "../services/basis-authoring/service.js";
+import { BasisOnboarding, BasisOnboardingLive } from "../services/basis-onboarding/service.js";
+const execFilePromise = promisify(execFile);
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const configuration = {
+    workloadDescription: "RouteKit repository changes",
+    candidateModels: ["openai/a", "openai/b"],
+    classifierModel: "openai/classifier",
+    authorModel: "openai/author",
+    judgeModel: "openai/judge",
+    objective: { kind: "highest-quality" },
+    maximumUnknownWeight: 0.3
+};
+const basisFixture = () => JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/routing-basis-v3.example.json", import.meta.url), "utf8"));
+async function repository() {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-basis-author-"));
+    roots.push(root);
+    await execFilePromise("git", ["init", "-q", root]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "config",
+        "user.name",
+        "RouteKit Test"
+    ]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "config",
+        "user.email",
+        "routekit@example.test"
+    ]);
+    await writeFile(path.join(root, "README.md"), "# Frozen source\n");
+    await execFilePromise("git", ["-C", root, "add", "README.md"]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "commit",
+        "-q",
+        "-m",
+        "initial"
+    ]);
+    return root;
+}
+function dimensionsFor(count) {
+    const fixture = basisFixture();
+    const source = fixture.dimensions.map((dimension) => ({
+        ...dimension,
+        confusableDimensionIds: []
+    }));
+    while (source.length < count) {
+        const index = source.length + 1;
+        source.push({
+            ...source[0],
+            id: `synthetic-${String(index)}`,
+            name: `Synthetic responsibility ${String(index)}`,
+            description: `A distinct material responsibility ${String(index)} for authoring tests.`,
+            inclusions: [`Work that materially owns synthetic responsibility ${String(index)}.`],
+            exclusions: ["Every other material responsibility in this test basis."],
+            confusableDimensionIds: [],
+            boundaryRules: [
+                `Use synthetic-${String(index)} only for its distinct test responsibility.`
+            ],
+            positiveExamples: [`Implement synthetic responsibility ${String(index)}.`],
+            negativeExamples: ["Change an unrelated repository responsibility."],
+            codeSummaries: [`Synthetic responsibility ${String(index)}.`]
+        });
+    }
+    return source.slice(0, count);
+}
+test("basis authoring makes seven bounded calls and resumes from durable checkpoints", async () => {
+    const root = await repository();
+    const requests = [];
+    const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: (input) => Effect.sync(() => {
+            const decoded = JSON.parse(input.input);
+            requests.push({
+                recipeId: decoded.recipe.id,
+                source: decoded.frozenRepository.sources[0]?.content ?? ""
+            });
+            const fixture = basisFixture();
+            const { repositoryDigest: _repositoryDigest, ...facetSnapshot } = fixture.facetSnapshot;
+            return JSON.stringify({
+                facetSnapshot,
+                dimensions: dimensionsFor(decoded.recipe.dimensionCount)
+            });
+        })
+    }));
+    const infrastructure = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive, transport).pipe(Layer.provide(NodeServicesLayer));
+    const onboarding = BasisOnboardingLive.pipe(Layer.provide(infrastructure));
+    const live = BasisAuthoringLive.pipe(Layer.provide(onboarding), Layer.provide(infrastructure));
+    await Effect.runPromise(Effect.gen(function* () {
+        const service = yield* BasisOnboarding;
+        yield* service.initialize({
+            repositoryRoot: root,
+            requestedRef: "HEAD",
+            allowDirty: false,
+            configuration,
+            tasks: []
+        });
+    }).pipe(Effect.provide(onboarding)));
+    await writeFile(path.join(root, "README.md"), "# Mutable worktree content\n");
+    const first = await Effect.runPromise(Effect.gen(function* () {
+        const service = yield* BasisAuthoring;
+        return yield* service.generate({
+            repositoryRoot: root,
+            incumbent: basisFixture()
+        });
+    }).pipe(Effect.provide(live)));
+    assert.equal(first.stage, "basis-candidates-review");
+    assert.equal(first.candidates.length, 8);
+    assert.equal(requests.length, 7);
+    assert.deepEqual(requests.map((request) => request.recipeId), first.generationPlan.recipes.slice(1).map((recipe) => recipe.id));
+    assert.ok(requests.every((request) => request.source.includes("Frozen source")));
+    assert.ok(requests.every((request) => !request.source.includes("Mutable worktree content")));
+    assert.equal(first.candidates[0]?.authorModel, undefined);
+    assert.ok(first.candidates
+        .slice(1)
+        .every((candidate) => candidate.authorModel === configuration.authorModel));
+    assert.ok(first.generationLedger.jobs.every((job) => job.status === "completed" && job.attemptCount === 1));
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const service = yield* BasisAuthoring;
+        return yield* service.generate({
+            repositoryRoot: root,
+            incumbent: basisFixture()
+        });
+    }).pipe(Effect.provide(live)));
+    assert.equal(resumed.projectId, first.projectId);
+    assert.equal(requests.length, 7);
+});
+test("basis authoring accumulates both reservations when a malformed candidate is retried", async () => {
+    const root = await repository();
+    let calls = 0;
+    const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: (input) => Effect.sync(() => {
+            calls += 1;
+            if (calls === 1)
+                return "{}";
+            const decoded = JSON.parse(input.input);
+            const fixture = basisFixture();
+            const { repositoryDigest: _repositoryDigest, ...facetSnapshot } = fixture.facetSnapshot;
+            return JSON.stringify({
+                facetSnapshot,
+                dimensions: dimensionsFor(decoded.recipe.dimensionCount)
+            });
+        })
+    }));
+    const infrastructure = Layer.mergeAll(EvalProjectStoreLive, EvalProjectArtifactsLive, transport).pipe(Layer.provide(NodeServicesLayer));
+    const onboarding = BasisOnboardingLive.pipe(Layer.provide(infrastructure));
+    const live = BasisAuthoringLive.pipe(Layer.provide(onboarding), Layer.provide(infrastructure));
+    await Effect.runPromise(Effect.gen(function* () {
+        const service = yield* BasisOnboarding;
+        yield* service.initialize({
+            repositoryRoot: root,
+            requestedRef: "HEAD",
+            allowDirty: false,
+            configuration,
+            tasks: []
+        });
+    }).pipe(Effect.provide(onboarding)));
+    const result = await Effect.runPromise(Effect.gen(function* () {
+        const service = yield* BasisAuthoring;
+        return yield* service.generate({
+            repositoryRoot: root,
+            incumbent: basisFixture()
+        });
+    }).pipe(Effect.provide(live)));
+    assert.equal(result.stage, "basis-candidates-review");
+    assert.equal(calls, 8);
+    assert.deepEqual(result.generationLedger.jobs[0], {
+        recipeId: result.generationPlan.recipes[1].id,
+        status: "completed",
+        candidateId: result.candidates[1].candidateId,
+        attemptCount: 2,
+        inputTokens: BASIS_AUTHOR_INPUT_TOKENS_PER_ATTEMPT * 2,
+        outputTokens: BASIS_AUTHOR_OUTPUT_TOKENS_PER_ATTEMPT * 2,
+        callIds: []
+    });
+});

package/dist/test/basis-onboarding.test.d.ts

@@ -0,0 +1 @@
+export {};