@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-model-authoring.test.js

@@ -0,0 +1,544 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { test } from "node:test";
+import { Deferred, Effect, Layer } from "effect";
+import { EvalProjectAuthoringError } from "../errors.js";
+import { assertRepositoryFoundryModelPlanV1, gpt56RepositoryFoundryModelPlanV1, REPOSITORY_FOUNDRY_MODEL_ROLES, repositoryFoundryModelAssignmentV1, repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { EvalAuthoringTransport } from "../project-authoring.js";
+import { authorRepositoryCaseSpecificationV1, repositoryGenerationContextCandidatesV1 } from "../services/case-authoring/service.js";
+import { RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+const assertStrictObjectSchemas = (value) => {
+    if (value === null || typeof value !== "object")
+        return;
+    if (Array.isArray(value)) {
+        for (const entry of value)
+            assertStrictObjectSchemas(entry);
+        return;
+    }
+    const node = value;
+    if (node.type === "object" || node.properties !== undefined) {
+        assert.equal(node.additionalProperties, false);
+        assert.deepEqual([...(node.required ?? [])].sort(), Object.keys(node.properties ?? {}).sort());
+    }
+    for (const child of Object.values(node))
+        assertStrictObjectSchemas(child);
+};
+const execFilePromise = promisify(execFile);
+const git = async (root, args) => (await execFilePromise("git", ["-C", root, ...args], {
+    encoding: "utf8"
+})).stdout.trim();
+const fixture = async () => {
+    const root = await mkdtemp(path.join(tmpdir(), "routekit-foundry-model-"));
+    await git(root, ["init", "-q"]);
+    await git(root, ["config", "user.email", "eval@example.test"]);
+    await git(root, ["config", "user.name", "Eval Fixture"]);
+    await mkdir(path.join(root, "src"), { recursive: true });
+    await writeFile(path.join(root, "package.json"), `${JSON.stringify({
+        name: "stream-fixture",
+        private: true,
+        type: "module",
+        exports: "./src/index.ts"
+    }, null, 2)}\n`);
+    await writeFile(path.join(root, "src", "stream.ts"), [
+        "export type Event = { type: string };",
+        "export const normalize = (events: Event[]) => events;"
+    ].join("\n"));
+    await writeFile(path.join(root, "src", "stream.test.ts"), "test('keeps completion last', () => {});\n");
+    await writeFile(path.join(root, "src", "index.ts"), 'export { normalize, type Event } from "./stream.js";\n');
+    await git(root, ["add", "."]);
+    await git(root, ["commit", "-qm", "initial stream adapter"]);
+    const initialCommit = await git(root, ["rev-parse", "HEAD"]);
+    return { root, initialCommit };
+};
+const seed = (initialCommit) => ({
+    version: 1,
+    id: "seed-stream-terminal-order",
+    status: "qualified",
+    source: {
+        kind: "historical-fix",
+        changeEpisodeId: "change-stream-terminal-order"
+    },
+    summary: "Preserve provider events while ensuring the completion event is emitted last",
+    capabilityEvidence: [
+        { kind: "symbol", id: "stream", path: "src/stream.ts" },
+        { kind: "test", id: "stream-test", path: "src/stream.test.ts" }
+    ],
+    initialState: { commit: initialCommit },
+    targetBehavior: [
+        {
+            id: "terminal-last",
+            description: "Preserve every provider event while ensuring exactly one completion event is emitted last.",
+            critical: true,
+            evidence: [{ kind: "test", id: "stream-test", path: "src/stream.test.ts" }]
+        }
+    ],
+    referenceCommit: "b".repeat(40),
+    baselineObservations: [
+        {
+            id: "baseline",
+            kind: "test-command",
+            subjectId: "stream-test",
+            outcome: "pass",
+            detail: "control passed"
+        }
+    ],
+    preChangeObservations: [
+        {
+            id: "pre",
+            kind: "test-command",
+            subjectId: "stream-test",
+            outcome: "fail",
+            detail: "completion was not last"
+        }
+    ],
+    postChangeObservations: [
+        {
+            id: "post",
+            kind: "test-command",
+            subjectId: "stream-test",
+            outcome: "pass",
+            detail: "completion was last"
+        }
+    ],
+    candidateTestIds: ["stream-test"],
+    environment: {
+        protectedControlPaths: ["src/stream.test.ts"],
+        trustedPreparationRecipes: [],
+        candidateValidationRecipes: [],
+        baselineGradeRecipes: [],
+        gradeRecipes: []
+    },
+    confidence: "high",
+    risks: [],
+    rejectionReasons: []
+});
+const behaviorMap = (root, commit) => ({
+    version: 1,
+    repository: {
+        root,
+        requestedRef: commit,
+        commit,
+        tree: "fixture-tree"
+    },
+    packages: [
+        {
+            id: "package-stream-fixture",
+            name: "stream-fixture",
+            path: ".",
+            manifestPath: "package.json",
+            dependencyPackageIds: [],
+            publicExports: ["."]
+        }
+    ],
+    publicSurfaces: [
+        {
+            id: "surface-stream-fixture",
+            packageId: "package-stream-fixture",
+            path: "src/index.ts",
+            kind: "package-export",
+            symbols: ["normalize", "Event"]
+        }
+    ],
+    protocols: [],
+    commands: [],
+    tests: [],
+    testBindings: [],
+    historyEpisodes: [],
+    workloadSignals: [],
+    blindSpots: []
+});
+const family = {
+    version: 1,
+    id: "family-stream-normalization",
+    capabilityId: "provider-stream-translation",
+    jobStatement: "Repair provider stream normalization while preserving the public event contract.",
+    lane: "repository-agent",
+    seedIds: ["seed-stream-terminal-order"],
+    inputStateFacets: ["provider event sequence"],
+    requestedChangeFacets: ["terminal ordering"],
+    observableFacets: ["event preservation", "completion ordering"],
+    admissibleSolutionPolicy: "behavior-equivalent",
+    fixtureAxes: ["missing completion", "duplicate completion", "trailing output"],
+    adversaryFamilies: ["event drop", "duplicate terminal"],
+    excludedNearMisses: ["format-only changes"],
+    repositoryCoverage: [{ kind: "symbol", id: "stream", path: "src/stream.ts" }],
+    confidence: "high"
+};
+const completionFor = (input) => {
+    switch (input.foundryRole) {
+        case "repository-analyst":
+            return {
+                summary: "The adapter must retain the provider event sequence and normalize completion ordering.",
+                userObservableBehaviors: [
+                    "all provider events remain visible",
+                    "one completion event is last"
+                ],
+                constraints: ["preserve the exported event contract"],
+                risks: ["streams without a completion event must remain valid"],
+                relevantPaths: ["src/stream.ts", "src/stream.test.ts", "src/index.ts"]
+            };
+        case "specification-writer":
+            return {
+                requestText: "Repair the provider stream adapter so every provider event remains observable while exactly one completion event is emitted at the end of a completed stream. Streams that never contain a completion event must otherwise remain unchanged, and the exported event contract must remain compatible.",
+                constraints: [
+                    "Preserve streams without a completion event.",
+                    "Do not change the exported event shape."
+                ]
+            };
+        case "specification-critic-a":
+        case "specification-critic-b":
+            return {
+                verdict: "sufficient",
+                detail: "The request states the observable terminal-ordering and preservation behavior without prescribing an implementation.",
+                criticalBehaviorCoverage: JSON.parse(input.input).targetBehavior.filter((clause) => clause.critical).map((clause) => ({
+                    behaviorId: clause.id, outcome: "covered", detail: "The request states the required behavior through a faithful paraphrase."
+                })),
+                ...(JSON.parse(input.input).requestedDimension === undefined ? {} : {
+                    dimensionFit: {
+                        requestedDimension: JSON.parse(input.input).requestedDimension,
+                        outcome: "fit", detail: "The requested behavior exercises this area in the repository.",
+                        evidencePaths: ["src/stream.ts"]
+                    }
+                }),
+                findings: []
+            };
+        default:
+            throw new Error(`unexpected model role in specification test: ${input.foundryRole}`);
+    }
+};
+test("foundry plans pin GPT-6 Astra and independently configured critics without model fallback", () => {
+    const plan = repositoryFoundryModelPlanV1({
+        primaryModel: "openai/gpt-6-astra",
+        criticAModel: "codex/gpt-6-astra",
+        criticBModel: "bedrock/anthropic.claude-opus-4-6-v1:0",
+        reviewerModel: "openrouter/google/gemini-3.1-pro-preview",
+        primaryReasoningEffort: "max",
+        reviewReasoningEffort: "high"
+    });
+    assert.doesNotThrow(() => assertRepositoryFoundryModelPlanV1(plan));
+    assert.deepEqual(plan.assignments.map((assignment) => assignment.role), REPOSITORY_FOUNDRY_MODEL_ROLES);
+    for (const role of [
+        "repository-analyst",
+        "oracle-designer",
+        "valid-solution-generator-a",
+        "adversary"
+    ]) {
+        assert.deepEqual(repositoryFoundryModelAssignmentV1(plan, role), {
+            role,
+            model: "openai/gpt-6-astra",
+            reasoningEffort: "max"
+        });
+    }
+    assert.equal(repositoryFoundryModelAssignmentV1(plan, "solution-critic-a").model, "codex/gpt-6-astra");
+    assert.equal(repositoryFoundryModelAssignmentV1(plan, "held-out-solution-critic-b").model, "bedrock/anthropic.claude-opus-4-6-v1:0");
+    assert.equal(repositoryFoundryModelAssignmentV1(plan, "quality-reviewer").model, "openrouter/google/gemini-3.1-pro-preview");
+});
+test("foundry plans explicitly assign the selected primary model to every unspecified critic role", () => {
+    const plan = repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" });
+    assert.equal(plan.assignments.length, REPOSITORY_FOUNDRY_MODEL_ROLES.length);
+    assert.ok(plan.assignments.every((assignment) => assignment.model === "openai/gpt-6-astra" && assignment.reasoningEffort === "high"));
+});
+test("foundry plans reject implicit models and routing aliases for every authoring role", () => {
+    for (const model of [
+        "",
+        "gpt-6-astra",
+        "auto",
+        "openai/auto",
+        "openai/default",
+        "openai/router",
+        "router/gpt-6-astra",
+        "openai/ AUTO",
+        " openai/gpt-6-astra",
+        "openai/gpt-6-astra "
+    ]) {
+        for (const role of REPOSITORY_FOUNDRY_MODEL_ROLES) {
+            const plan = repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" });
+            assert.throws(() => assertRepositoryFoundryModelPlanV1({
+                ...plan,
+                assignments: plan.assignments.map((assignment) => assignment.role === role ? { ...assignment, model } : assignment)
+            }), /(?:explicit|concrete) provider\/model/u, `${role}: ${JSON.stringify(model)}`);
+        }
+    }
+});
+test("foundry plans require one explicit assignment per role and high primary reasoning", () => {
+    const plan = repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" });
+    assert.throws(() => assertRepositoryFoundryModelPlanV1({ ...plan, assignments: plan.assignments.slice(1) }), /exactly one assignment/u);
+    assert.throws(() => assertRepositoryFoundryModelPlanV1({
+        ...plan,
+        assignments: [...plan.assignments.slice(1), plan.assignments[1]]
+    }), /exactly one assignment/u);
+    assert.throws(() => repositoryFoundryModelPlanV1({
+        primaryModel: "openai/gpt-6-astra",
+        primaryReasoningEffort: "low"
+    }), /high or stronger reasoning/u);
+});
+test("existing GPT-5.6 model plan builder remains compatible", () => {
+    const input = {
+        primaryModel: "openai/gpt-5.6-sol",
+        criticAModel: "openai/gpt-5.6-luna",
+        criticBModel: "openai/gpt-5.6-terra",
+        reviewerModel: "openai/gpt-5.6-terra"
+    };
+    assert.deepEqual(gpt56RepositoryFoundryModelPlanV1(input), repositoryFoundryModelPlanV1(input));
+});
+test("model-authored specifications run both isolated critics before either completes", async () => {
+    const { root, initialCommit } = await fixture();
+    const calls = [];
+    const criticsStarted = new Set();
+    const bothCriticsStarted = Deferred.makeUnsafe();
+    try {
+        const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+            completeDetailed: (input) => Effect.gen(function* () {
+                assertStrictObjectSchemas(input.jsonSchema);
+                calls.push(input);
+                if (input.foundryRole?.startsWith("specification-critic-")) {
+                    criticsStarted.add(input.foundryRole);
+                    if (criticsStarted.size === 2)
+                        yield* Deferred.succeed(bothCriticsStarted, undefined);
+                    yield* Deferred.await(bothCriticsStarted).pipe(Effect.timeout("2 seconds"), Effect.orDie);
+                    assert.equal(criticsStarted.size, 2, "neither critic may finish before its sibling starts");
+                }
+                return {
+                    text: JSON.stringify(completionFor(input)),
+                    callId: `call-${input.foundryRole}`
+                };
+            }),
+            complete: (input) => Effect.succeed(JSON.stringify(completionFor(input)))
+        }));
+        const languageModel = RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport));
+        const result = await Effect.runPromise(authorRepositoryCaseSpecificationV1({
+            repositoryRoot: root,
+            operationId: "foundry-case-stream",
+            requestedDimension: "Provider stream translation",
+            caseId: "case-stream-terminal-order",
+            map: behaviorMap(root, initialCommit),
+            seed: seed(initialCommit),
+            family,
+            modelPlan: gpt56RepositoryFoundryModelPlanV1({
+                primaryModel: "openai/gpt-5.6-sol",
+                criticAModel: "openai/gpt-5.6-luna",
+                criticBModel: "openai/gpt-5.6-terra"
+            })
+        }).pipe(Effect.provide(languageModel)));
+        assert.equal(result.visible.lane, "repository-agent");
+        assert.match(result.visible.requestText, /exactly one completion event/u);
+        assert.equal(result.reviews.length, 2);
+        assert.ok(calls.every((call) => JSON.parse(call.input).requestedDimension === "Provider stream translation"));
+        assert.ok(result.reviews.every((review) => review.dimensionFit?.outcome === "fit"));
+        assert.equal(result.modelCalls.length, 4);
+        assert.deepEqual(calls.map((call) => call.foundryRole), [
+            "repository-analyst",
+            "specification-writer",
+            "specification-critic-a",
+            "specification-critic-b"
+        ]);
+        assert.deepEqual(calls.map((call) => call.model), ["openai/gpt-5.6-sol", "openai/gpt-5.6-sol", "openai/gpt-5.6-luna", "openai/gpt-5.6-terra"]);
+        assert.ok(calls.every((call) => call.reasoningEffort === "high"));
+        const writerInput = JSON.parse(calls[1]?.input ?? "{}");
+        assert.equal(writerInput.initialCommit, initialCommit);
+        assert.deepEqual(writerInput.selectedRepositoryContext?.map((source) => source.path), ["src/stream.ts", "src/stream.test.ts", "src/index.ts"]);
+        assert.equal(calls[1]?.input.includes("b".repeat(40)), false, "the specification writer must not receive the historical reference commit");
+        assert.equal(calls[2]?.input.includes("call-specification-critic"), false, "critics must not receive each other's outputs");
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});
+test("a failed specification critic interrupts its in-flight sibling and fails authoring", async () => {
+    const { root, initialCommit } = await fixture();
+    const siblingStarted = Deferred.makeUnsafe();
+    const calls = [];
+    let siblingInterrupted = false;
+    const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: (input) => Effect.gen(function* () {
+            calls.push(input.foundryRole ?? "unknown");
+            if (input.foundryRole === "specification-critic-a") {
+                yield* Deferred.await(siblingStarted);
+                return yield* Effect.fail(new EvalProjectAuthoringError({
+                    operation: "authoring-evaluations", detail: "critic request failed"
+                }));
+            }
+            if (input.foundryRole === "specification-critic-b") {
+                yield* Deferred.succeed(siblingStarted, undefined);
+                return yield* Effect.never.pipe(Effect.onInterrupt(() => Effect.sync(() => {
+                    siblingInterrupted = true;
+                })));
+            }
+            return JSON.stringify(completionFor(input));
+        })
+    }));
+    try {
+        await assert.rejects(Effect.runPromise(authorRepositoryCaseSpecificationV1({
+            repositoryRoot: root,
+            operationId: "foundry-critic-cancellation",
+            caseId: "case-stream-terminal-order",
+            map: behaviorMap(root, initialCommit),
+            seed: seed(initialCommit),
+            family,
+            modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" })
+        }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport))), Effect.timeout("5 seconds"))), /critic request failed/u);
+        assert.equal(siblingInterrupted, true);
+        assert.deepEqual(calls, [
+            "repository-analyst", "specification-writer", "specification-critic-a", "specification-critic-b"
+        ]);
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});
+test("specification criticism rejects dimension mismatch and missing critical coverage before returning a draft", async () => {
+    const { root, initialCommit } = await fixture();
+    try {
+        for (const issue of ["dimension-mismatch", "missing-dimension", "omitted-critical", "missing-critical-review"]) {
+            const response = (input) => {
+                const value = completionFor(input);
+                if (input.foundryRole !== "specification-critic-b")
+                    return value;
+                if (issue === "dimension-mismatch") {
+                    return { ...value, dimensionFit: { ...value.dimensionFit, outcome: "mismatch" } };
+                }
+                if (issue === "missing-dimension") {
+                    const { dimensionFit: _dimensionFit, ...rest } = value;
+                    return rest;
+                }
+                return { ...value, criticalBehaviorCoverage: issue === "missing-critical-review" ? [] : [{
+                            behaviorId: "terminal-last", outcome: "omitted", detail: "The required preservation condition is absent."
+                        }] };
+            };
+            const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+                complete: (input) => Effect.succeed(JSON.stringify(response(input))),
+                completeDetailed: (input) => Effect.succeed({ text: JSON.stringify(response(input)) })
+            }));
+            await assert.rejects(Effect.runPromise(authorRepositoryCaseSpecificationV1({
+                repositoryRoot: root,
+                operationId: `reject-${issue}`,
+                caseId: "case-stream-terminal-order",
+                requestedDimension: "Provider stream translation",
+                map: behaviorMap(root, initialCommit),
+                seed: seed(initialCommit),
+                family,
+                modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" })
+            }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport))))), /generated specification failed independent review/u, issue);
+        }
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});
+test("repository analyst path selection fails closed before specification writing", async () => {
+    const { root, initialCommit } = await fixture();
+    const calls = [];
+    try {
+        const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+            complete: (input) => {
+                calls.push(input);
+                const output = completionFor(input);
+                return Effect.succeed(JSON.stringify(input.foundryRole === "repository-analyst"
+                    ? {
+                        ...output,
+                        relevantPaths: ["src/not-in-the-repository.ts"]
+                    }
+                    : output));
+            }
+        }));
+        await assert.rejects(Effect.runPromise(authorRepositoryCaseSpecificationV1({
+            repositoryRoot: root,
+            operationId: "foundry-invalid-context",
+            caseId: "case-stream-invalid-context",
+            map: behaviorMap(root, initialCommit),
+            seed: seed(initialCommit),
+            family,
+            modelPlan: gpt56RepositoryFoundryModelPlanV1({
+                primaryModel: "openai/gpt-5.6-sol",
+                criticAModel: "openai/gpt-5.6-luna",
+                criticBModel: "openai/gpt-5.6-terra"
+            })
+        }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport))))), /select-generation-context.*between one and eight unique paths/u);
+        assert.deepEqual(calls.map((call) => call.foundryRole), ["repository-analyst"]);
+    }
+    finally {
+        await rm(root, { recursive: true, force: true });
+    }
+});
+test("generation context ownership prefers the deepest workspace package", () => {
+    const nestedSeed = {
+        ...seed("a".repeat(40)),
+        capabilityEvidence: [
+            {
+                kind: "symbol",
+                id: "nested-implementation",
+                path: "packages/feature/src/feature.ts"
+            },
+            {
+                kind: "test",
+                id: "nested-test",
+                path: "packages/feature/src/feature.test.ts"
+            }
+        ]
+    };
+    const map = {
+        ...behaviorMap("/repo", "a".repeat(40)),
+        packages: [
+            {
+                id: "package-root",
+                name: "root",
+                path: ".",
+                manifestPath: "package.json",
+                dependencyPackageIds: [],
+                publicExports: ["."]
+            },
+            {
+                id: "package-feature",
+                name: "feature",
+                path: "packages/feature",
+                manifestPath: "packages/feature/package.json",
+                dependencyPackageIds: [],
+                publicExports: ["."]
+            }
+        ],
+        publicSurfaces: [
+            {
+                id: "surface-root",
+                packageId: "package-root",
+                path: "src/index.ts",
+                kind: "package-export",
+                symbols: []
+            },
+            {
+                id: "surface-feature",
+                packageId: "package-feature",
+                path: "packages/feature/src/index.ts",
+                kind: "package-export",
+                symbols: ["feature"]
+            }
+        ],
+        protocols: [
+            {
+                id: "protocol-root",
+                path: "src/root-protocol.ts",
+                name: "RootProtocol",
+                kind: "internal-boundary"
+            },
+            {
+                id: "protocol-feature",
+                path: "packages/feature/src/feature-protocol.ts",
+                name: "FeatureProtocol",
+                kind: "internal-boundary"
+            }
+        ]
+    };
+    const candidates = repositoryGenerationContextCandidatesV1({
+        map,
+        seed: nestedSeed
+    }).map((candidate) => candidate.path);
+    assert.ok(candidates.includes("packages/feature/package.json"));
+    assert.ok(candidates.includes("packages/feature/src/index.ts"));
+    assert.ok(candidates.includes("packages/feature/src/feature-protocol.ts"));
+    assert.equal(candidates.includes("package.json"), false);
+    assert.equal(candidates.includes("src/index.ts"), false);
+    assert.equal(candidates.includes("src/root-protocol.ts"), false);
+});

package/dist/test/repository-model.test.d.ts

@@ -0,0 +1 @@
+export {};