@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/repository-authoring-artifacts.test.js

@@ -0,0 +1,185 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { Effect, Layer, Schema } from "effect";
+import { EVAL_AUTHORING_REQUEST_BYTES, EVAL_AUTHORING_REVIEW_REQUEST_BYTES, EvalAuthoringTransport } from "../project-authoring.js";
+import { evalAuthoringRequestByteLimit, evalAuthoringResponsesRequestBody } from "../adapters/authoring-responses-request.js";
+import { decodeRepositoryReviewEvidenceV2 } from "../adapters/repository-review-evidence.js";
+import { repositoryFoundryModelPlanV1 } from "../repository-language-model-protocol.js";
+import { REPOSITORY_FOUNDRY_TRANSIENT_RETRY_RESERVE, repositoryFoundryGenerationBudgetV1 } from "../repository-foundry-plan-protocol.js";
+import { prepareRepositoryCaseQualityReviewV1 } from "../services/case-generation/service.js";
+import { RepositoryFoundryProgressLive } from "../services/foundry-progress/service.js";
+import { RepositoryFoundryLanguageModel, RepositoryFoundryLanguageModelLive } from "../services/language-model/service.js";
+test("complete review capacity preserves full evidence and legacy budget behavior", () => {
+    const original = {
+        visible: { requestText: "Offline request-sizing fixture; never an admitted benchmark." },
+        fullEvidence: "Complete unreduced observation.\n".repeat(20_000),
+        evidenceInventory: ["visible", "fullEvidence"]
+    };
+    const checkpoint = {
+        operationId: "offline-review-sizing",
+        modelPlan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" }),
+        qualityReviewInput: original,
+        sourceBases: []
+    };
+    assert.throws(() => prepareRepositoryCaseQualityReviewV1({ checkpoint }), /independent quality review input ceiling/u);
+    const prepared = prepareRepositoryCaseQualityReviewV1({ checkpoint, reviewInputVersion: 2 });
+    assert.equal(prepared.evidence.encoding, "routekit-lossless-review-json-v2");
+    assert.ok(prepared.requestBytes > EVAL_AUTHORING_REQUEST_BYTES);
+    assert.ok(prepared.requestBytes <= EVAL_AUTHORING_REVIEW_REQUEST_BYTES);
+    assert.deepEqual(decodeRepositoryReviewEvidenceV2(prepared.evidence), original);
+    const legacyBudget = repositoryFoundryGenerationBudgetV1(2);
+    const expandedBudget = repositoryFoundryGenerationBudgetV1(2, undefined, undefined, undefined, undefined, 2);
+    assert.equal(legacyBudget.requiredTotalInputTokens, legacyBudget.plannedModelCalls * EVAL_AUTHORING_REQUEST_BYTES);
+    assert.deepEqual({ ...expandedBudget, requiredTotalInputTokens: legacyBudget.requiredTotalInputTokens }, legacyBudget);
+    assert.equal(expandedBudget.requiredTotalInputTokens - legacyBudget.requiredTotalInputTokens, (1 + REPOSITORY_FOUNDRY_TRANSIENT_RETRY_RESERVE) *
+        (EVAL_AUTHORING_REVIEW_REQUEST_BYTES - EVAL_AUTHORING_REQUEST_BYTES));
+});
+test("only complete review roles can request expanded capacity and no capacity metadata enters the wire", async () => {
+    const calls = [];
+    const generate = (role, reviewInputVersion, large = true) => Effect.runPromise(Effect.gen(function* () {
+        const model = yield* RepositoryFoundryLanguageModel;
+        return yield* model.generateAssignedStructured({
+            assignment: { role, model: "openai/gpt-6-astra", reasoningEffort: "high" },
+            operationId: "offline-complete-review-capacity",
+            instructions: "Offline transport test; do not confer benchmark admission.",
+            input: { complete: "e".repeat(large ? 600_000 : 100) },
+            schemaName: "routekit_repository_generation_quality_review_v1",
+            outputSchema: Schema.Struct({ accepted: Schema.Boolean }),
+            maximumOutputTokens: 1024,
+            ...(reviewInputVersion === undefined ? {} : { reviewInputVersion })
+        });
+    }).pipe(Effect.provide(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: (request) => {
+            calls.push(request);
+            return Effect.succeed('{"accepted":true}');
+        }
+    })))))));
+    await assert.rejects(generate("quality-reviewer"), /bounded request/u);
+    await assert.rejects(generate("adversary", 2), /versioned complete evidence review/u);
+    assert.equal(calls.length, 0);
+    await generate("quality-reviewer", 2);
+    assert.equal(calls.length, 1);
+    assert.equal(JSON.parse(calls[0].input).complete.length, 600_000);
+    assert.equal(evalAuthoringRequestByteLimit(calls[0]), EVAL_AUTHORING_REVIEW_REQUEST_BYTES);
+    const { reviewInputVersion: _version, ...legacy } = calls[0];
+    assert.equal(evalAuthoringResponsesRequestBody(calls[0]), evalAuthoringResponsesRequestBody(legacy));
+    await generate("quality-reviewer", undefined, false);
+    assert.equal(calls[1].reviewInputVersion, undefined);
+    assert.equal(evalAuthoringRequestByteLimit(calls[1]), EVAL_AUTHORING_REQUEST_BYTES);
+});
+test("every foundry role can compact a large request without changing evidence or call limits", async () => {
+    const input = {
+        observations: Array.from({ length: 1800 }, (_, index) => ({
+            solutionId: `independent-control-${index % 20}`,
+            clauseId: `required-contract-${index % 12}`,
+            repetition: (index % 2) + 1,
+            outcome: index % 9 === 0 ? "unknown" : index % 3 === 0 ? "fail" : "pass",
+            detail: `recipe=isolated-complete-regression-fixtures; observedRecipe=isolated-complete-regression-fixtures; gradeExecuted=true; timeout=false; durationMs=${index}; preparationTemplate=${"a".repeat(64)}; evidence=complete host-owned execution observations`
+        }))
+    };
+    assert.ok(Buffer.byteLength(JSON.stringify(input)) > EVAL_AUTHORING_REQUEST_BYTES);
+    const calls = [];
+    const artifacts = [];
+    const result = await Effect.runPromise(Effect.gen(function* () {
+        const model = yield* RepositoryFoundryLanguageModel;
+        return yield* model.generateStructured({
+            plan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" }),
+            role: "hill-climb-planner",
+            operationId: "wide-repair-input",
+            instructions: "Repair only documented failures. Preserve all passing requirements.",
+            input,
+            schemaName: "bounded_test_proposal",
+            outputSchema: Schema.Struct({ accepted: Schema.Boolean }),
+            maximumOutputTokens: 1000
+        });
+    }).pipe(Effect.provide(Layer.merge(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: () => Effect.die("detailed transport required"),
+        completeDetailed: (request) => {
+            calls.push(request);
+            assert.ok(Buffer.byteLength(evalAuthoringResponsesRequestBody(request)) <=
+                EVAL_AUTHORING_REQUEST_BYTES);
+            assert.deepEqual(decodeRepositoryReviewEvidenceV2(JSON.parse(request.input)), input);
+            assert.match(request.instructions, /^Repair only documented failures/u);
+            assert.equal(request.maximumOutputTokens, 1000);
+            return Effect.succeed({
+                text: '{"accepted":true}',
+                callId: "large-input-offline-fixture"
+            });
+        }
+    })))), RepositoryFoundryProgressLive(() => Effect.void, (artifact) => Effect.sync(() => {
+        artifacts.push(artifact);
+    }))))));
+    assert.equal(result.value.accepted, true);
+    assert.equal(calls.length, 1);
+    assert.deepEqual(artifacts.at(-1)?.request.input, JSON.parse(calls[0].input));
+    assert.equal(artifacts.at(-1)?.request.instructions, calls[0].instructions);
+});
+test("irreducible foundry input fails before a model call or model-call progress event", async () => {
+    const events = [];
+    let calls = 0;
+    const result = await Effect.runPromiseExit(Effect.gen(function* () {
+        const model = yield* RepositoryFoundryLanguageModel;
+        return yield* model.generateStructured({
+            plan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" }),
+            role: "oracle-designer",
+            operationId: "irreducible-input",
+            instructions: "Bounded offline fixture.",
+            input: { largeUniqueField: "x".repeat(EVAL_AUTHORING_REQUEST_BYTES + 1) },
+            schemaName: "unreachable",
+            outputSchema: Schema.Struct({ accepted: Schema.Boolean }),
+            maximumOutputTokens: 1000
+        });
+    }).pipe(Effect.provide(Layer.merge(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+        complete: () => {
+            calls++;
+            return Effect.succeed('{"accepted":true}');
+        }
+    })))), RepositoryFoundryProgressLive((event) => Effect.sync(() => {
+        events.push(event);
+    }))))));
+    assert.equal(result._tag, "Failure");
+    assert.equal(calls, 0);
+    assert.equal(events.filter((event) => event.kind === "model-call").length, 0);
+});
+test("role artifacts are captured before schema validation without entering progress events", async () => {
+    for (const text of ['{"requestText":"VISIBLE TASK"}', "INVALID PRIVATE RESPONSE"]) {
+        const artifacts = [];
+        const events = [];
+        const transport = Layer.succeed(EvalAuthoringTransport, EvalAuthoringTransport.of({
+            complete: () => Effect.succeed(text)
+        }));
+        const result = await Effect.runPromiseExit(Effect.gen(function* () {
+            const model = yield* RepositoryFoundryLanguageModel;
+            return yield* model.generateStructured({
+                plan: repositoryFoundryModelPlanV1({ primaryModel: "openai/gpt-6-astra" }),
+                role: "specification-writer",
+                operationId: "artifact-test",
+                instructions: "PRIVATE INSTRUCTIONS",
+                input: { repository: "PRIVATE SOURCE" },
+                schemaName: "specification",
+                outputSchema: Schema.Struct({ requestText: Schema.String }),
+                maximumOutputTokens: 100
+            });
+        }).pipe(Effect.provide(Layer.merge(RepositoryFoundryLanguageModelLive.pipe(Layer.provide(transport)), RepositoryFoundryProgressLive((event) => Effect.sync(() => {
+            events.push(event);
+        }), (entry) => Effect.sync(() => {
+            artifacts.push(entry);
+        }))))));
+        assert.equal(artifacts[0]?.validation, "received");
+        assert.equal(artifacts[0]?.response.text, text);
+        assert.equal(artifacts[0]?.request.instructions, "PRIVATE INSTRUCTIONS");
+        assert.doesNotMatch(JSON.stringify(events), /PRIVATE|VISIBLE TASK/u);
+        assert.deepEqual(events.map((event) => event.phase), ["start", "complete"]);
+        assert.equal(events.at(-1)?.modelCalls, 1, "invalid JSON still consumed a returned model call");
+        assert.ok(events.every((event) => event.operationId === "artifact-test"));
+        if (text.startsWith("{")) {
+            assert.equal(result._tag, "Success");
+            assert.equal(artifacts[1]?.validation, "validated");
+            assert.deepEqual(artifacts[1]?.value, { requestText: "VISIBLE TASK" });
+        }
+        else {
+            assert.equal(result._tag, "Failure");
+            assert.equal(artifacts.length, 1);
+        }
+    }
+});

package/dist/test/repository-bundle.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/test/repository-bundle.test.js

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { compilePortableRepositoryBundleV2, makePortableRepositoryBundleV2 } from "../services/repository-bundle/service.js";
+const commit = "a".repeat(40);
+const artifacts = [
+    {
+        path: ".routekit/evals/case.eval.ts",
+        kind: "eval-definition",
+        mediaType: "text/typescript",
+        content: "export {};\n"
+    },
+    {
+        path: ".routekit/evals/case.json",
+        kind: "public-metadata",
+        mediaType: "application/json",
+        content: "{}\n"
+    }
+];
+test("repository bundle core emits stable descriptors and exact sorted public bytes", () => {
+    const first = compilePortableRepositoryBundleV2({
+        admitted: true,
+        sourceCommitSha: commit,
+        artifacts
+    });
+    const second = compilePortableRepositoryBundleV2({
+        admitted: true,
+        sourceCommitSha: commit,
+        artifacts: [...artifacts].reverse()
+    });
+    assert.deepEqual(first, second);
+    assert.deepEqual(first.files.map(({ path }) => path), [".routekit/evals/case.eval.ts", ".routekit/evals/case.json"]);
+    assert.deepEqual(first.bundle, makePortableRepositoryBundleV2({ admitted: true, sourceCommitSha: commit, artifacts }));
+});
+test("repository bundle core rejects nonportable, credential-like, duplicate and unsupported dot paths", () => {
+    for (const artifact of [
+        { ...artifacts[0], path: "../case.eval.ts" },
+        { ...artifacts[0], path: ".github/workflows/case.yml" },
+        { ...artifacts[0], path: ".routekit/credentials.json" },
+        { ...artifacts[0], content: "authorization: Bearer secret-value-123456" }
+    ]) {
+        assert.throws(() => compilePortableRepositoryBundleV2({
+            admitted: true,
+            sourceCommitSha: commit,
+            artifacts: [artifact]
+        }));
+    }
+    assert.throws(() => compilePortableRepositoryBundleV2({
+        admitted: true,
+        sourceCommitSha: commit,
+        artifacts: [artifacts[0], artifacts[0]]
+    }));
+});

package/dist/test/repository-case-generation.test.d.ts

@@ -0,0 +1 @@
+export {};