@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/agentic-environment.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/agentic-environment.test.js

@@ -0,0 +1,213 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect } from "effect";
+import { discoverEvalCandidatesV1 } from "../services/eval-candidate/service.js";
+import { planEvalEnvironmentV1, probeEvalEnvironmentV1 } from "../services/eval-environment/service.js";
+import { buildRepositoryBehaviorMapV1 } from "../services/repository-model/service.js";
+import { RepositoryCommandObservability } from "../services/command-observability/service.js";
+import { extractHistoricalTaskSeedsV1 } from "../services/task-seed/service.js";
+const exec = promisify(execFile);
+const runtime = {
+    name: "node",
+    version: process.versions.node,
+    platform: process.platform,
+    architecture: process.arch
+};
+async function fixture(t, kind, options = {}) {
+    const root = await mkdtemp(join(tmpdir(), "agentic-environment-"));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const write = async (path, value) => {
+        await mkdir(dirname(join(root, path)), { recursive: true });
+        await writeFile(join(root, path), typeof value === "string" ? value : JSON.stringify(value));
+    };
+    const git = async (...args) => (await exec("git", ["-C", root, ...args])).stdout.trim();
+    await git("init", "-q");
+    await git("config", "user.email", "eval@example.test");
+    await git("config", "user.name", "Eval Fixture");
+    await write("package.json", { name: "fixture", private: true, packageManager: "pnpm@11.15.1" });
+    await write("pnpm-workspace.yaml", "packages:\n  - packages/*\n");
+    await write("pnpm-lock.yaml", "lockfileVersion: '9.0'\nimporters: {}\n");
+    for (const name of ["main", "shared", "unrelated"]) {
+        await write(`packages/${name}/package.json`, {
+            name: `fixture-${name}`,
+            type: "module",
+            exports: kind === "emitted" ? "./dist/index.js" : "./src/index.ts",
+            scripts: {
+                build: "tsc -b",
+                test: kind === "emitted"
+                    ? "node --test dist/test/*.test.js"
+                    : kind === "tsx"
+                        ? 'tsx --test "src/test/*.test.ts" "src/deployment/*.test.ts"'
+                        : "node --test src/test/*.test.ts"
+            },
+            ...(name === "main" ? { dependencies: { "fixture-shared": "workspace:*" } } : {})
+        });
+        await write(`packages/${name}/src/index.ts`, "export const clamp = (value: number) => Math.min(100, value);\n");
+        await write(`packages/${name}/src/test/clamp.test.ts`, 'import assert from "node:assert/strict"; import test from "node:test"; import { clamp } from "../index.ts"; test("upper bound", () => assert.equal(clamp(101), 100));\n');
+    }
+    await git("add", ".");
+    await git("commit", "-qm", "feat: add clamp");
+    await write("packages/main/src/index.ts", "export const clamp = (value: number) => Math.max(0, Math.min(100, value));\n");
+    await write("packages/main/src/test/clamp.test.ts", 'import assert from "node:assert/strict"; import test from "node:test"; import { clamp } from "../index.ts"; test("lower bound", () => assert.equal(clamp(-1), 0)); test("upper bound", () => assert.equal(clamp(101), 100));\n');
+    if (options.changedManifest) {
+        const manifest = JSON.parse(await readFile(join(root, "packages/main/package.json"), "utf8"));
+        await write("packages/main/package.json", { ...manifest, description: "Updated public behavior." });
+    }
+    if (options.crossPackageTests) {
+        await write("packages/shared/src/test/clamp.test.ts", 'import assert from "node:assert/strict"; import test from "node:test"; import { clamp } from "../index.ts"; test("upper preservation", () => assert.equal(clamp(101), 100));\n');
+    }
+    await git("add", ".");
+    await git("commit", "-qm", "fix: clamp lower bound");
+    const map = await Effect.runPromise(buildRepositoryBehaviorMapV1({ repositoryRoot: root, requestedRef: "HEAD" }));
+    return { root, map };
+}
+test("environment controls come from the actual Git delta, including manifests omitted by capability citations", async (t) => {
+    const { root, map } = await fixture(t, "source", { changedManifest: true });
+    const found = await Effect.runPromise(discoverEvalCandidatesV1({
+        map, coverageCell: "regression", query: "clamp lower bound"
+    }));
+    const proposal = found.candidates[0];
+    assert.equal(proposal.seed.capabilityEvidence.some((entry) => entry.path === "packages/main/package.json"), false);
+    const prepared = await Effect.runPromise(planEvalEnvironmentV1({
+        repositoryRoot: root, candidate: proposal.candidate.candidate,
+        revision: proposal.candidate.revision, sourceDigest: proposal.candidate.sourceDigest,
+        seed: proposal.seed, recipeIds: proposal.seed.environment.gradeRecipes.map((recipe) => recipe.id),
+        executionMode: "source", runtime, justification: "Keep actual changed configuration editable."
+    }));
+    assert.equal(prepared.environment.protectedControlPaths.includes("packages/main/package.json"), false);
+    assert.equal(prepared.environment.protectedControlPaths.includes("package.json"), true);
+    const legacy = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
+    assert.equal(legacy[0].environment.protectedControlPaths.includes("packages/main/package.json"), false);
+});
+test("environment planning refuses a recipe subset that cannot execute the complete reviewed test scope", async (t) => {
+    const { root, map } = await fixture(t, "source", { crossPackageTests: true });
+    const found = await Effect.runPromise(discoverEvalCandidatesV1({
+        map, coverageCell: "regression", query: "clamp lower bound"
+    }));
+    const proposal = found.candidates[0];
+    const mainRecipe = proposal.seed.environment.gradeRecipes.find((recipe) => recipe.cwd === "packages/main");
+    assert.ok(mainRecipe);
+    let executions = 0;
+    await assert.rejects(Effect.runPromise(planEvalEnvironmentV1({
+        repositoryRoot: root, candidate: proposal.candidate.candidate,
+        revision: proposal.candidate.revision, sourceDigest: proposal.candidate.sourceDigest,
+        seed: proposal.seed, recipeIds: [mainRecipe.id], executionMode: "source",
+        runtime, justification: "Attempt to omit reviewed shared-package tests."
+    }).pipe(Effect.provideService(RepositoryCommandObservability, {
+        observe: () => { executions += 1; }
+    }))), /must cover every reviewed test path/);
+    assert.equal(executions, 0);
+});
+test("environment planning rejects a reference change whose parent differs from the immutable candidate", async (t) => {
+    const { root, map } = await fixture(t, "source");
+    const found = await Effect.runPromise(discoverEvalCandidatesV1({
+        map, coverageCell: "regression", query: "clamp lower bound"
+    }));
+    const proposal = found.candidates[0];
+    await assert.rejects(Effect.runPromise(planEvalEnvironmentV1({
+        repositoryRoot: root, candidate: proposal.candidate.candidate,
+        revision: proposal.candidate.revision, sourceDigest: proposal.candidate.sourceDigest,
+        seed: { ...proposal.seed, initialState: { commit: proposal.seed.referenceCommit } },
+        recipeIds: proposal.seed.environment.gradeRecipes.map((recipe) => recipe.id),
+        executionMode: "source", runtime, justification: "Wrong immutable source binding."
+    })), /must bind the same historical change/);
+});
+for (const kind of ["source", "emitted", "broken", "tsx"]) {
+    test(`${kind}: source candidate discovery invokes zero install/build/test commands`, async (t) => {
+        const { root, map } = await fixture(t, kind);
+        let commands = 0;
+        const found = await Effect.runPromise(discoverEvalCandidatesV1({
+            map,
+            coverageCell: "regression",
+            query: "clamp lower bound"
+        }).pipe(Effect.provideService(RepositoryCommandObservability, {
+            observe: () => {
+                commands += 1;
+            }
+        })));
+        assert.equal(commands, 0);
+        assert.equal(found.candidates.length, 1);
+        const proposal = found.candidates[0];
+        assert.equal(proposal.candidate.state, "draft");
+        assert.equal(proposal.seed.status, "candidate");
+        assert.deepEqual(proposal.seed.baselineObservations, []);
+        assert.deepEqual(proposal.seed.environment.trustedPreparationRecipes, []);
+        const prepared = await Effect.runPromise(planEvalEnvironmentV1({
+            repositoryRoot: root,
+            candidate: proposal.candidate.candidate,
+            revision: proposal.candidate.revision,
+            sourceDigest: proposal.candidate.sourceDigest,
+            seed: proposal.seed,
+            recipeIds: proposal.seed.environment.gradeRecipes.map((recipe) => recipe.id),
+            executionMode: kind === "emitted" ? "emitted" : "source",
+            runtime,
+            justification: "Run only the repository-owned clamp regression recipe."
+        }));
+        assert.equal(commands, 0);
+        assert.equal(prepared.plan.version, 1);
+        assert.ok(prepared.plan.commands.every((command) => command.cwd !== "packages/unrelated"));
+        const builds = prepared.plan.commands.filter((command) => command.purpose === "build");
+        assert.deepEqual(builds.map((command) => command.cwd), kind === "emitted" ? ["packages/shared", "packages/main"] : []);
+        assert.ok(prepared.plan.commands[0].args.includes("fixture-main..."));
+        if (kind === "tsx") {
+            for (const recipe of [
+                ...prepared.environment.baselineGradeRecipes,
+                ...prepared.environment.gradeRecipes
+            ]) {
+                assert.equal(recipe.cwd, "packages/main");
+                assert.equal(recipe.executable, "corepack");
+                assert.deepEqual(recipe.args, [
+                    "pnpm@11.15.1", "exec", "tsx", "--test", "src/test/clamp.test.ts"
+                ]);
+                assert.equal(recipe.args.some((arg) => arg.includes("*")), false);
+            }
+        }
+        if (kind === "source") {
+            // No dependencies are needed by this fixture's actual source imports.
+            const probe = await Effect.runPromise(probeEvalEnvironmentV1({
+                repositoryRoot: root,
+                prepared: {
+                    ...prepared,
+                    environment: { ...prepared.environment, trustedPreparationRecipes: [] }
+                },
+                commit: map.repository.commit
+            }));
+            assert.equal(probe.outcome, "completed");
+            assert.equal(probe.gradeExecuted, true);
+            const old = await Effect.runPromise(extractHistoricalTaskSeedsV1(map));
+            assert.ok(old[0].environment.candidateValidationRecipes.length > 0, "legacy callers retain their qualification environment");
+        }
+        if (kind === "broken") {
+            const blocked = await Effect.runPromise(probeEvalEnvironmentV1({
+                repositoryRoot: root,
+                prepared: {
+                    ...prepared,
+                    environment: {
+                        ...prepared.environment,
+                        trustedPreparationRecipes: [
+                            {
+                                id: "broken-preparation",
+                                kind: "custom",
+                                cwd: "packages/main",
+                                executable: "node",
+                                args: [
+                                    "-e",
+                                    "process.stderr.write('fixture environment unavailable'); process.exit(2)"
+                                ],
+                                timeoutMs: 5_000
+                            }
+                        ]
+                    }
+                }
+            }));
+            assert.equal(blocked.outcome, "environment_blocked");
+            assert.equal(blocked.gradeExecuted, false);
+            assert.match(blocked.results[0].stderr, /fixture environment unavailable/u);
+        }
+    });
+}

package/dist/test/case-pipeline-foundation.test.d.ts

@@ -0,0 +1 @@
+export {};