@velum-labs/routekit-eval-setup 1.0.26 → 1.1.0

package/dist/test/basis-onboarding.test.js

@@ -0,0 +1,385 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { after, test } from "node:test";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { layer as NodeServicesLayer } from "@effect/platform-node/NodeServices";
+import { routingBasisDigestV3, routingContractDigest } from "@velum-labs/routekit-eval-contracts";
+import { Effect, Layer } from "effect";
+import { captureFrozenRepositorySnapshotV3 } from "../adapters/git-basis-snapshot.js";
+import { evaluateBasisCandidateValidationV3, makeBasisCandidateDatasetV3 } from "../basis-validation.js";
+import { EvalProjectArtifacts, EvalProjectArtifactsLive } from "../project-artifacts.js";
+import { EvalProjectStoreLive } from "../project-store.js";
+import { BasisOnboarding, BasisOnboardingLive } from "../services/basis-onboarding/service.js";
+const execFilePromise = promisify(execFile);
+const roots = [];
+after(async () => Promise.all(roots.map((root) => rm(root, { recursive: true, force: true }))));
+const configuration = {
+    workloadDescription: "RouteKit repository changes",
+    candidateModels: ["openai/a", "openai/b"],
+    classifierModel: "openai/classifier",
+    authorModel: "openai/author",
+    judgeModel: "openai/judge",
+    objective: { kind: "highest-quality" },
+    maximumUnknownWeight: 0.3
+};
+const basisFixture = () => JSON.parse(readFileSync(new URL("../../../../test/fixtures/routing-v3/examples/routing-basis-v3.example.json", import.meta.url), "utf8"));
+const task = (index) => ({
+    caseId: `case-${String(index).padStart(3, "0")}`,
+    occurredAt: new Date(Date.UTC(2026, 0, 1) + index * 86_400_000).toISOString(),
+    taskDigest: routingContractDigest("routekit.test-task.v1", "task", index),
+    summary: `Task ${String(index)}`,
+    changedPathDigest: routingContractDigest("routekit.test-paths.v1", "paths", [`src/${String(index)}.ts`]),
+    changedPaths: [`src/${String(index)}.ts`]
+});
+const candidateFor = (recipeId, repositoryDigest, deterministic) => {
+    const source = basisFixture();
+    const semanticBasis = {
+        ...source,
+        basisId: recipeId,
+        basisDigest: "0".repeat(64),
+        repositoryDigest,
+        facetSnapshot: {
+            ...source.facetSnapshot,
+            repositoryDigest
+        },
+        provenance: {
+            ...source.provenance,
+            recipeId,
+            ...(deterministic
+                ? {
+                    origin: {
+                        kind: "incumbent-enriched",
+                        sourceBasisId: source.basisId,
+                        sourceBasisVersion: source.version
+                    }
+                }
+                : {
+                    authorModel: configuration.authorModel,
+                    origin: { kind: "generated" }
+                })
+        }
+    };
+    const basis = {
+        ...semanticBasis,
+        basisDigest: routingBasisDigestV3(semanticBasis)
+    };
+    return {
+        candidateId: basis.basisId,
+        recipeId,
+        basis,
+        basisDigest: basis.basisDigest,
+        ...(deterministic ? {} : { authorModel: configuration.authorModel }),
+        generationConfigurationDigest: routingContractDigest("routekit.test-generation.v1", "configuration", recipeId),
+        evidenceDigest: routingContractDigest("routekit.test-evidence.v1", "evidence", { recipeId, repositoryDigest }),
+        createdAt: "2026-08-23T00:00:00.000Z"
+    };
+};
+async function repository() {
+    const root = await mkdtemp(path.join(os.tmpdir(), "routekit-basis-onboarding-"));
+    roots.push(root);
+    await execFilePromise("git", ["init", "-q", root]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "config",
+        "user.name",
+        "RouteKit Test"
+    ]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "config",
+        "user.email",
+        "routekit@example.test"
+    ]);
+    await writeFile(path.join(root, "README.md"), "# Repository\n");
+    await execFilePromise("git", ["-C", root, "add", "README.md"]);
+    await execFilePromise("git", [
+        "-C",
+        root,
+        "commit",
+        "-q",
+        "-m",
+        "initial"
+    ]);
+    return root;
+}
+const OnboardingTestLive = BasisOnboardingLive.pipe(Layer.provide(EvalProjectArtifactsLive), Layer.provide(EvalProjectStoreLive), Layer.provide(NodeServicesLayer));
+test("Git snapshot capture binds commit, tree, inventory, and clean status", async () => {
+    const root = await repository();
+    const first = await Effect.runPromise(captureFrozenRepositorySnapshotV3({
+        repositoryRoot: root,
+        requestedRef: "HEAD",
+        allowDirty: false,
+        capturedAt: "2026-08-23T00:00:00.000Z"
+    }));
+    const second = await Effect.runPromise(captureFrozenRepositorySnapshotV3({
+        repositoryRoot: root,
+        requestedRef: "HEAD",
+        allowDirty: false,
+        capturedAt: "2026-08-23T00:01:00.000Z"
+    }));
+    assert.equal(first.snapshot.dirty, false);
+    assert.equal(first.snapshot.repositoryDigest, second.snapshot.repositoryDigest);
+    assert.deepEqual(first.sourceInventory, ["README.md"]);
+    assert.match(first.snapshot.commit, /^[0-9a-f]{40,64}$/u);
+    assert.match(first.snapshot.tree, /^[0-9a-f]{40,64}$/u);
+});
+test("Git snapshot refuses dirty worktrees unless lab-only dirty evidence is explicit", async () => {
+    const root = await repository();
+    await writeFile(path.join(root, "README.md"), "# Changed\n");
+    await writeFile(path.join(root, "untracked.txt"), "private test content\n");
+    await assert.rejects(Effect.runPromise(captureFrozenRepositorySnapshotV3({
+        repositoryRoot: root,
+        requestedRef: "HEAD",
+        allowDirty: false,
+        capturedAt: "2026-08-23T00:00:00.000Z"
+    })), /repository is dirty/);
+    const captured = await Effect.runPromise(captureFrozenRepositorySnapshotV3({
+        repositoryRoot: root,
+        requestedRef: "HEAD",
+        allowDirty: true,
+        capturedAt: "2026-08-23T00:00:00.000Z"
+    }));
+    assert.equal(captured.snapshot.dirty, true);
+    assert.match(captured.snapshot.dirtyPatchDigest ?? "", /^[0-9a-f]{64}$/u);
+    assert.deepEqual(captured.sourceInventory, ["README.md"]);
+});
+test("basis onboarding initializes once, reserves seven author jobs, and resumes", async () => {
+    const root = await repository();
+    const first = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.initialize({
+            repositoryRoot: root,
+            requestedRef: "HEAD",
+            allowDirty: false,
+            configuration,
+            tasks: []
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(first.stage, "basis-generation-ready");
+    assert.equal(first.generationPlan.recipes.length, 8);
+    assert.equal(first.generationPlan.plannedPrimaryCalls, 7);
+    assert.equal(first.generationPlan.maximumAttempts, 14);
+    assert.equal(first.split.provisional, true);
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.initialize({
+            repositoryRoot: root,
+            requestedRef: "HEAD",
+            allowDirty: false,
+            configuration: { ...configuration, authorModel: "openai/changed" },
+            tasks: []
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(resumed.projectId, first.projectId);
+    assert.equal(resumed.generationPlan.planDigest, first.generationPlan.planDigest);
+    assert.equal(JSON.parse(await readFile(path.join(root, ".routekit", "evals", "project.json"), "utf8")).version, 2);
+});
+test("basis onboarding persists exact immutable artifacts through qualification", async () => {
+    const root = await repository();
+    const tasks = Array.from({ length: 50 }, (_, index) => task(index));
+    const initialized = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.initialize({
+            repositoryRoot: root,
+            requestedRef: "HEAD",
+            allowDirty: false,
+            configuration,
+            tasks
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(initialized.stage, "basis-generation-ready");
+    const candidates = initialized.generationPlan.recipes.map((recipe) => candidateFor(recipe.id, initialized.snapshot.repositoryDigest, recipe.deterministic));
+    const jobs = initialized.generationLedger.jobs.map((job) => ({
+        ...job,
+        status: "completed",
+        candidateId: candidates.find((candidate) => candidate.recipeId === job.recipeId).candidateId,
+        attemptCount: 1,
+        inputTokens: 100,
+        outputTokens: 50,
+        callIds: [`call-${job.recipeId}`]
+    }));
+    const ledgerSemantic = {
+        version: 1,
+        planDigest: initialized.generationPlan.planDigest,
+        jobs
+    };
+    const generationLedger = {
+        ...ledgerSemantic,
+        ledgerDigest: routingContractDigest("routekit.basis-generation-ledger.v1", "ledger", ledgerSemantic)
+    };
+    const reviewed = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.recordCandidates({
+            repositoryRoot: root,
+            generationLedger,
+            candidates
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(reviewed.stage, "basis-candidates-review");
+    assert.equal(reviewed.revision, 1);
+    assert.equal(reviewed.structureReports.length, 8);
+    assert.ok(reviewed.structureReports.every((report) => report.eligible));
+    const labelProtocol = {
+        version: 1,
+        protocolDigest: "c".repeat(64),
+        kind: "reviewed-human",
+        author: "Author",
+        reviewer: "Reviewer",
+        blindness: "candidate-output-hidden",
+        activeDimensionThreshold: 0.25,
+        unknownThreshold: 0.3
+    };
+    const datasets = reviewed.candidates.map((candidate) => makeBasisCandidateDatasetV3({
+        splitDigest: reviewed.split.splitDigest,
+        candidate,
+        tasks: reviewed.split.validation,
+        labelProtocol,
+        labels: reviewed.split.validation.map((entry) => ({
+            caseId: entry.caseId,
+            basisDigest: candidate.basisDigest,
+            scores: candidate.basis.dimensions.map((dimension, index) => ({
+                dimensionId: dimension.id,
+                score: index === 0 ? 1 : 0
+            })),
+            unknownProbability: 0
+        }))
+    }));
+    const validationPlanSemantic = {
+        version: 1,
+        repositoryDigest: reviewed.snapshot.repositoryDigest,
+        splitDigest: reviewed.split.splitDigest,
+        classifierBindings: reviewed.candidates.map((candidate) => ({
+            candidateId: candidate.candidateId,
+            basisDigest: candidate.basisDigest,
+            classifierConfigurationDigest: routingContractDigest("routekit.test-classifier-configuration.v1", "configuration", candidate.basisDigest)
+        })),
+        candidateIds: reviewed.structureReports
+            .filter((report) => report.eligible)
+            .map((report) => report.candidateId),
+        datasetDigests: datasets.map((dataset) => dataset.datasetDigest),
+        repetitions: 1,
+        plannedPrimaryCalls: datasets.length * reviewed.split.validation.length,
+        maximumAttempts: datasets.length * reviewed.split.validation.length * 2,
+        requiredTotalInputTokens: 1_000_000,
+        requiredTotalOutputTokens: 100_000,
+        acknowledgeUnpricedExposure: true
+    };
+    const validationPlan = {
+        ...validationPlanSemantic,
+        planDigest: routingContractDigest("routekit.basis-validation-plan.v1", "plan", validationPlanSemantic)
+    };
+    const validationReady = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.prepareValidation({
+            repositoryRoot: root,
+            datasets,
+            validationPlan
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(validationReady.stage, "basis-validation-ready");
+    assert.equal(validationReady.revision, 2);
+    const validationResults = validationReady.candidates.map((candidate, index) => {
+        const dataset = validationReady.datasets[index];
+        return evaluateBasisCandidateValidationV3({
+            candidate,
+            dataset,
+            validationPlan: validationReady.validationPlan,
+            predictions: dataset.labels.map((label) => ({
+                ...label,
+                classifierConfigurationDigest: validationReady.validationPlan.classifierBindings[index]
+                    .classifierConfigurationDigest
+            })),
+            provisional: validationReady.split.provisional
+        });
+    });
+    const validationReview = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.recordValidation({
+            repositoryRoot: root,
+            validationResults
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(validationReview.stage, "basis-validation-review");
+    assert.equal(validationReview.revision, 3);
+    assert.equal(await stat(path.join(root, ".routekit", "evals", "basis", "validation-results.v3.json")).then((info) => info.mode & 0o777), 0o600);
+    const qualified = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.qualify({
+            repositoryRoot: root,
+            candidateId: validationReview.selection.selectedCandidateId,
+            basisDigest: validationReview.candidates.find((candidate) => candidate.candidateId ===
+                validationReview.selection.selectedCandidateId).basisDigest,
+            approvedBy: "Reviewer"
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(qualified.stage, "basis-qualified");
+    assert.equal(qualified.revision, 4);
+    assert.equal(qualified.approval.approvedBy, "Reviewer");
+    assert.equal(qualified.validationResults.length, 8);
+    assert.deepEqual(qualified.validationResults, validationResults);
+    assert.equal(await stat(path.join(root, ".routekit", "evals", "basis", "evaluation-projection.v3.json")).then((info) => info.mode & 0o777), 0o600);
+    const resumed = await Effect.runPromise(Effect.gen(function* () {
+        const onboarding = yield* BasisOnboarding;
+        return yield* onboarding.qualify({
+            repositoryRoot: root,
+            candidateId: qualified.approval.candidateId,
+            basisDigest: qualified.approval.basisDigest,
+            approvedBy: "Reviewer"
+        });
+    }).pipe(Effect.provide(OnboardingTestLive)));
+    assert.equal(resumed.revision, qualified.revision);
+    for (const artifact of [
+        "generation.v3.json",
+        "candidates.v3.json",
+        "validation.v3.json",
+        "qualification.v3.json"
+    ]) {
+        const artifactPath = path.join(root, ".routekit", "evals", "basis", artifact);
+        assert.equal((await stat(artifactPath)).mode & 0o777, 0o600);
+    }
+});
+test("basis validation checkpoints round-trip by immutable job identity", async () => {
+    const root = await repository();
+    const checkpoint = {
+        version: 1,
+        validationPlanDigest: "a".repeat(64),
+        candidateId: "candidate",
+        basisDigest: "b".repeat(64),
+        validationDatasetDigest: "c".repeat(64),
+        caseId: "case-001",
+        repetition: 0,
+        classifierConfigurationDigest: "d".repeat(64),
+        status: "completed",
+        prediction: {
+            caseId: "case-001",
+            basisDigest: "b".repeat(64),
+            scores: [{ dimensionId: "dimension", score: 1 }],
+            unknownProbability: 0,
+            classifierConfigurationDigest: "d".repeat(64)
+        },
+        attemptCount: 1,
+        callIds: ["call-1"],
+        knownInputTokens: 10,
+        knownOutputTokens: 5,
+        knownCostUsd: 0.01,
+        unpricedCalls: 0,
+        unknownUsageCalls: 0
+    };
+    const loaded = await Effect.runPromise(Effect.gen(function* () {
+        const artifacts = yield* EvalProjectArtifacts;
+        yield* artifacts.saveBasisValidationJobCheckpointV3(root, checkpoint);
+        return yield* artifacts.loadBasisValidationJobCheckpointV3(root, {
+            validationPlanDigest: checkpoint.validationPlanDigest,
+            candidateId: checkpoint.candidateId,
+            caseId: checkpoint.caseId,
+            repetition: checkpoint.repetition
+        });
+    }).pipe(Effect.provide(EvalProjectArtifactsLive.pipe(Layer.provide(NodeServicesLayer)))));
+    assert.deepEqual(loaded, checkpoint);
+});

package/dist/test/basis-repair.test.d.ts

@@ -0,0 +1 @@
+export {};