@velum-labs/routekit-eval-setup 1.3.2 → 1.5.0

package/dist/test/grounded-authoring.test.js

@@ -0,0 +1,565 @@
+import assert from "node:assert/strict";
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { promisify } from "node:util";
+import { Effect, Option, Schema } from "effect";
+import { makeCaseCheckpointStoreV1 } from "../case-checkpoint-store.js";
+import { PipelineBudgetExhaustedError } from "../case-pipeline-protocol.js";
+import { RepositoryFoundryError } from "../errors.js";
+import { BudgetedLanguageModelLive, budgetSnapshotV1, makePipelineStateAccessorV1 } from "../services/budgeted-model/service.js";
+import { GROUNDED_AUTHORING_UNRESOLVED_PREFIX, GROUNDED_FULL_RESULT_WINDOW, GROUNDED_INPUT_BYTES_LIMIT, GROUNDED_MAX_TOOL_TURNS_BEFORE_SUBMIT, GROUNDED_READ_BYTES_PER_TURN, groundedStructuredGenerationV1 } from "../services/grounded-authoring/service.js";
+import { RepositoryFoundryLanguageModel } from "../services/language-model/service.js";
+const execute = promisify(execFile);
+const Answer = Schema.Struct({ answer: Schema.String });
+const assignment = {
+    role: "oracle-author",
+    model: "openai/gpt-6-astra",
+    reasoningEffort: "high"
+};
+const goodCode = `import assert from "node:assert/strict";
+import test from "node:test";
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { value } from "./subject.mjs";
+test("frozen public behavior", () => {
+  assert.equal(existsSync("candidate-ran.txt"), false);
+  writeFileSync("candidate-ran.txt", "fresh candidate only");
+  assert.equal(readFileSync("control.json", "utf8"), "reference");
+  assert.equal(value, 1);
+});
+`;
+const brokenCode = `${goodCode}\nconst syntaxError = ;\n`;
+const decoderCode = `export function decodeSse(chunk) {
+  return chunk.split("\\n\\n").filter(Boolean).map((event) => event.replace(/^data: /u, ""));
+}
+`;
+/** Roughly 300 KB with multi-byte characters so byte slicing must respect UTF-8 boundaries. */
+const bigText = Array.from({ length: 6_000 }, (_, index) => `line ${String(index).padStart(5, "0")} café 中文 ${"x".repeat(32)}`).join("\n");
+async function repository(t) {
+    const root = await realpath(await mkdtemp(join(tmpdir(), "grounded-authoring-test-")));
+    t.after(() => rm(root, { recursive: true, force: true }));
+    const git = (...args) => execute("git", ["-C", root, ...args]);
+    await git("init", "-q");
+    await mkdir(join(root, "lib"), { recursive: true });
+    await writeFile(join(root, "case.test.mjs"), goodCode);
+    await writeFile(join(root, "lib", "sse-decoder.mjs"), decoderCode);
+    await writeFile(join(root, "big.txt"), bigText);
+    const commit = async (value, control) => {
+        await writeFile(join(root, "subject.mjs"), `export const value = ${String(value)};\n`);
+        await writeFile(join(root, "control.json"), control);
+        await git("add", ".");
+        await git("-c", "core.hooksPath=/dev/null", "-c", "commit.gpgSign=false", "-c", "user.name=Grounded Authoring", "-c", "user.email=grounded@example.test", "commit", "-qm", "fixture");
+        return (await git("rev-parse", "HEAD")).stdout.trim();
+    };
+    const initialCommit = await commit(0, "initial");
+    const referenceCommit = await commit(1, "reference");
+    // The working tree is dirty; the loop must only ever see pinned objects.
+    await writeFile(join(root, "control.json"), "untrusted working tree");
+    await writeFile(join(root, "lib", "sse-decoder.mjs"), "export const decodeSse = undefined;\n");
+    const grade = {
+        id: "grade",
+        kind: "test",
+        cwd: ".",
+        executable: "node",
+        args: ["--test", "case.test.mjs"],
+        timeoutMs: 5000
+    };
+    const observation = (id, subjectId, outcome) => ({
+        id,
+        kind: "test-command",
+        subjectId,
+        outcome,
+        detail: "Synthetic qualified seed evidence."
+    });
+    const seed = {
+        version: 1,
+        id: "synthetic-seed",
+        status: "qualified",
+        source: { kind: "historical-fix", changeEpisodeId: "synthetic-change" },
+        summary: "Synthetic public behavior.",
+        capabilityEvidence: [{ kind: "test", id: "case", path: "case.test.mjs" }],
+        initialState: { commit: initialCommit },
+        referenceCommit,
+        targetBehavior: [{ id: "behavior", description: "Return one.", critical: true, evidence: [] }],
+        baselineObservations: [observation("baseline", "baseline", "pass")],
+        preChangeObservations: [observation("pre", "grade", "fail")],
+        postChangeObservations: [observation("post", "grade", "pass")],
+        candidateTestIds: ["case"],
+        environment: {
+            protectedControlPaths: ["control.json"],
+            trustedPreparationRecipes: [],
+            candidateValidationRecipes: [
+                {
+                    id: "syntax",
+                    kind: "check",
+                    cwd: ".",
+                    executable: "node",
+                    args: ["--check", "case.test.mjs"],
+                    timeoutMs: 5000
+                }
+            ],
+            baselineGradeRecipes: [{ ...grade, id: "baseline" }],
+            gradeRecipes: [grade]
+        },
+        confidence: "high",
+        risks: [],
+        rejectionReasons: []
+    };
+    return { root, seed, initialCommit, referenceCommit };
+}
+/** A scripted model: each reply may read the previous tool result from the transcript. */
+const scriptedModel = (script) => {
+    const seen = [];
+    const generateAssignedStructured = (request) => Effect.gen(function* () {
+        const input = request.input;
+        seen.push({
+            operationId: request.operationId,
+            schemaName: request.schemaName,
+            instructions: request.instructions,
+            input
+        });
+        const raw = script(input, seen.length);
+        const value = yield* Schema.decodeUnknownEffect(request.outputSchema)({ turn: raw }).pipe(Effect.mapError((cause) => new RepositoryFoundryError({
+            operation: "invoke-language-model",
+            detail: `language-model role ${request.assignment.role} violated ${request.schemaName}`,
+            cause
+        })));
+        return {
+            value,
+            call: {
+                operationId: request.operationId,
+                role: request.assignment.role,
+                model: request.assignment.model,
+                reasoningEffort: request.assignment.reasoningEffort,
+                callId: `call-${String(seen.length)}`
+            }
+        };
+    });
+    const shape = {
+        generateAssignedStructured,
+        generateStructured: (request) => generateAssignedStructured({
+            ...request,
+            assignment: { role: request.role, model: assignment.model, reasoningEffort: "high" }
+        })
+    };
+    return { shape, seen };
+};
+const lastResult = (input) => (input.transcript.at(-1)?.result ?? {});
+const initialState = () => ({
+    version: 1,
+    caseId: "case-1",
+    repositoryRoot: "/repo",
+    dimension: { id: "dim", description: "A dimension.", includes: [], excludes: [] },
+    model: assignment.model,
+    stage: "oracle",
+    status: "running",
+    startedAt: "2026-09-08T00:00:00.000Z",
+    updatedAt: "2026-09-08T00:00:00.000Z",
+    modelCalls: 0,
+    elapsedMs: 0,
+    resumeCount: 0,
+    folds: 0
+});
+test("grounded work resumes a pending submission with prior execution feedback and no new model call", async (t) => {
+    const repo = await repository(t);
+    const store = makeCaseCheckpointStoreV1({ caseDirectory: join(repo.root, "private-work") });
+    const model = scriptedModel((_input, turn) => turn === 1
+        ? { action: "read_files", commit: "reference", paths: ["lib/sse-decoder.mjs"], reason: "ground" }
+        : { action: "submit", result: { answer: turn === 2 ? "draft" : "repaired" } });
+    let interrupted = false;
+    let validations = 0;
+    const input = {
+        languageModel: model.shape,
+        assignment,
+        operationId: "resume:author",
+        instructions: "Ground then submit.",
+        task: { goal: "durable feedback" },
+        outputSchema: Answer,
+        schemaName: "Answer",
+        checkpointStore: store,
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+        },
+        validateSubmission: (value) => Effect.gen(function* () {
+            validations += 1;
+            if (value.answer === "draft")
+                return [{ kind: "execution", message: "Preserve this concrete failure." }];
+            if (!interrupted) {
+                interrupted = true;
+                return yield* new RepositoryFoundryError({
+                    operation: "run-pipeline-stage",
+                    detail: "simulated interruption during preflight"
+                });
+            }
+            return [];
+        })
+    };
+    const first = await Effect.runPromise(groundedStructuredGenerationV1(input).pipe(Effect.result));
+    assert.equal(first._tag, "Failure");
+    assert.equal(model.seen.length, 3);
+    const resumed = await Effect.runPromise(groundedStructuredGenerationV1({
+        ...input,
+        // Restore through a new store object, as in a new capability process.
+        checkpointStore: makeCaseCheckpointStoreV1({ caseDirectory: store.caseDirectory })
+    }));
+    assert.equal(model.seen.length, 3, "do not repurchase a received pending action");
+    assert.equal(resumed.turns, 3);
+    assert.deepEqual(resumed.transcript.map((entry) => entry.action.action), ["read_files", "submit", "submit"]);
+    assert.deepEqual(resumed.transcript[1].result.findings, [
+        { kind: "execution", message: "Preserve this concrete failure." }
+    ]);
+    assert.equal(resumed.value.answer, "repaired");
+    assert.equal(validations, 3, "only incomplete preflight repeats");
+    const completed = await Effect.runPromise(groundedStructuredGenerationV1(input));
+    assert.equal(completed.value.answer, "repaired");
+    assert.equal(model.seen.length, 3);
+    assert.equal(validations, 3, "accepted work survives interruption before the stage checkpoint");
+});
+test("grounded work binds task, pinned trees, policy and schema instead of trusting an operation name", async (t) => {
+    const repo = await repository(t);
+    const store = makeCaseCheckpointStoreV1({ caseDirectory: join(repo.root, "private-work") });
+    const model = scriptedModel((_input, turn) => ({ action: "submit", result: { answer: String(turn) } }));
+    const input = {
+        languageModel: model.shape,
+        assignment,
+        operationId: "same-operation",
+        instructions: "Submit.",
+        task: { revision: 1 },
+        outputSchema: Answer,
+        schemaName: "Answer",
+        checkpointStore: store,
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+        }
+    };
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(input))).value.answer, "1");
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(input))).value.answer, "1");
+    const changedTask = { ...input, task: { revision: 2 } };
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(changedTask))).value.answer, "2");
+    const changedPolicy = { ...changedTask, instructions: "A revised validation policy." };
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(changedPolicy))).value.answer, "3");
+    const changedTree = { ...changedPolicy, tools: { ...input.tools, commits: {
+                initial: repo.referenceCommit, reference: repo.initialCommit
+            } } };
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(changedTree))).value.answer, "4");
+    const changedSchema = { ...changedTree, outputSchema: Schema.Struct({
+            answer: Schema.String, optional: Schema.optionalKey(Schema.String)
+        }) };
+    assert.equal((await Effect.runPromise(groundedStructuredGenerationV1(changedSchema))).value.answer, "5");
+    assert.equal(model.seen.length, 5);
+});
+test("grounded work preserves received fixture actions across real interruption without accepting unexecuted work", async (t) => {
+    const repo = await repository(t);
+    const store = makeCaseCheckpointStoreV1({ caseDirectory: join(repo.root, "private-work") });
+    const model = scriptedModel((_input, turn) => turn === 1
+        ? {
+            action: "run_fixture", fixtureId: "durable-fixture", testPath: "case.test.mjs",
+            content: goodCode, expectationMode: "preserved", kind: "boundary",
+            description: "Pinned behavior.", expectedBehavior: "Return one."
+        }
+        : { action: "submit", result: { answer: "executed" } });
+    let stop = true;
+    const input = {
+        languageModel: model.shape, assignment, operationId: "interrupted-fixture",
+        instructions: "Probe before submit.", task: { goal: "execute fixture" },
+        outputSchema: Answer, schemaName: "Answer",
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit },
+            seed: repo.seed, runFixture: true
+        },
+        checkpointStore: {
+            ...store,
+            writeWorkCheckpoint: (name, value) => store.writeWorkCheckpoint(name, value).pipe(Effect.andThen(Effect.suspend(() => {
+                if (!stop)
+                    return Effect.void;
+                stop = false;
+                return Effect.interrupt;
+            })))
+        }
+    };
+    const first = await Effect.runPromise(groundedStructuredGenerationV1(input).pipe(Effect.timeoutOption(2000))).catch(() => undefined);
+    assert.ok(first === undefined || Option.isNone(first));
+    assert.equal(model.seen.length, 1);
+    const result = await Effect.runPromise(groundedStructuredGenerationV1({ ...input, checkpointStore: store }));
+    assert.equal(result.turns, 2);
+    assert.equal(model.seen.length, 2, "fixture action is restored, not regenerated");
+    assert.equal(result.transcript[0].result.outcome, "pass");
+});
+test("the grounded loop searches, reads, continues a truncated file, runs fixtures, and accepts a validated submission", async (t) => {
+    const repo = await repository(t);
+    const fixtureAction = (fixtureId, content) => ({
+        action: "run_fixture",
+        fixtureId,
+        testPath: "case.test.mjs",
+        content,
+        expectationMode: "preserved",
+        kind: "metamorphic",
+        description: "Frozen public behavior.",
+        expectedBehavior: "Return one."
+    });
+    const model = scriptedModel((input, turn) => {
+        const previous = lastResult(input);
+        switch (turn) {
+            case 1:
+                return { action: "search", commit: "reference", pattern: "decodeSse", pathPrefix: "lib" };
+            case 2:
+                return {
+                    action: "read_files",
+                    commit: "reference",
+                    paths: ["lib/sse-decoder.mjs", "big.txt", "lib/missing-decoder.mjs"],
+                    reason: "the fixture needs the SSE decoder"
+                };
+            case 3: {
+                const files = previous.files;
+                const big = files.find((file) => file.path === "big.txt");
+                return {
+                    action: "read_range",
+                    commit: "reference",
+                    path: "big.txt",
+                    offset: big.nextOffset,
+                    length: 100_000
+                };
+            }
+            case 4:
+                return fixtureAction("draft", brokenCode);
+            case 5:
+                return fixtureAction("draft", goodCode);
+            case 6:
+                return { action: "submit", result: { answer: "short" } };
+            default:
+                return { action: "submit", result: { answer: "grounded and complete" } };
+        }
+    });
+    const result = await Effect.runPromise(groundedStructuredGenerationV1({
+        languageModel: model.shape,
+        assignment,
+        operationId: "oracle:author",
+        instructions: "Author the hidden oracle.",
+        task: { goal: "write a fixture that uses the SSE decoder" },
+        outputSchema: Answer,
+        schemaName: "Answer",
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit },
+            seed: repo.seed,
+            runFixture: true
+        },
+        validateSubmission: (value) => Effect.succeed(value.answer.length < 12
+            ? [{ kind: "contract", message: "answer must be complete", path: "answer" }]
+            : [])
+    }));
+    assert.deepEqual(result.value, { answer: "grounded and complete" });
+    assert.equal(result.turns, 7);
+    assert.equal(result.transcript.length, 7);
+    assert.deepEqual(result.transcript.map((entry) => entry.action.action), ["search", "read_files", "read_range", "run_fixture", "run_fixture", "submit", "submit"]);
+    assert.deepEqual(model.seen.map((entry) => entry.operationId), Array.from({ length: 7 }, (_, index) => `oracle:author:turn-${String(index + 1)}`));
+    assert.deepEqual(model.seen.map((entry) => entry.input.remainingTurns), [24, 23, 22, 21, 20, 19, 18]);
+    assert.ok(model.seen.every((entry) => entry.input.remainingCalls === undefined));
+    const search = result.transcript[0].result;
+    assert.equal(search.total, 1);
+    assert.equal(search.matches[0].path, "lib/sse-decoder.mjs");
+    assert.equal(search.matches[0].line, 1);
+    assert.match(search.matches[0].text, /export function decodeSse/u);
+    const read = result.transcript[1].result;
+    assert.equal(read.files[0].path, "lib/sse-decoder.mjs");
+    assert.equal(read.files[0].content, decoderCode, "pinned content, not the dirty working tree");
+    assert.equal(read.files[0].truncated, false);
+    const big = read.files[1];
+    assert.equal(big.truncated, true);
+    assert.equal(big.totalBytes, Buffer.byteLength(bigText));
+    assert.ok(big.nextOffset > 0 && big.nextOffset <= GROUNDED_READ_BYTES_PER_TURN);
+    assert.ok(!big.content.includes("�"), "truncation respects UTF-8 boundaries");
+    assert.equal(Buffer.byteLength(big.content), big.nextOffset);
+    assert.ok(read.bytesReturned <= GROUNDED_READ_BYTES_PER_TURN);
+    assert.equal(read.files[2].error, "not-found");
+    assert.ok(read.files[2].closest.includes("lib/sse-decoder.mjs"));
+    const range = result.transcript[2].result;
+    assert.equal(range.offset, big.nextOffset);
+    assert.equal(range.truncated, true);
+    assert.equal(range.nextOffset, range.offset + range.length);
+    assert.ok(range.length > GROUNDED_READ_BYTES_PER_TURN - 4 &&
+        range.length <= GROUNDED_READ_BYTES_PER_TURN);
+    assert.equal(Buffer.from(bigText).subarray(range.offset, range.nextOffset).toString("utf8"), range.content);
+    assert.ok(!range.content.includes("�"));
+    const broken = result.transcript[3].result;
+    assert.equal(broken.outcome, "unknown");
+    assert.match(broken.stderrTail, /SyntaxError/u);
+    assert.ok(broken.durationMs >= 0);
+    assert.equal(broken.stages[0].stage, "candidate-validation");
+    const passing = result.transcript[4].result;
+    assert.equal(passing.outcome, "pass");
+    assert.match(passing.stdoutTail, /# pass 1/u);
+    assert.deepEqual(result.transcript[5].result, {
+        accepted: false,
+        findings: [{ kind: "contract", message: "answer must be complete", path: "answer" }],
+        note: "revise and submit again; the transcript keeps your prior submission"
+    });
+    assert.deepEqual(result.transcript[6].result, { accepted: true });
+    const seventh = model.seen[6].input.transcript.at(-1);
+    assert.deepEqual(seventh.action, { action: "submit", result: { answer: "short" } });
+    assert.deepEqual(seventh.result.findings, [
+        { kind: "contract", message: "answer must be complete", path: "answer" }
+    ]);
+    assert.ok(model.seen[3].schemaName === "Answer");
+});
+test("the grounded loop requires a proposal after bounded tool exploration", async (t) => {
+    const repo = await repository(t);
+    const model = scriptedModel((_input, turn) => turn <= GROUNDED_MAX_TOOL_TURNS_BEFORE_SUBMIT
+        ? { action: "list_files", commit: "initial" }
+        : { action: "submit", result: { answer: "bounded proposal" } });
+    const result = await Effect.runPromise(groundedStructuredGenerationV1({
+        languageModel: model.shape,
+        assignment,
+        operationId: "bounded:author",
+        instructions: "Inspect, then answer.",
+        task: { goal: "answer without unbounded browsing" },
+        outputSchema: Answer,
+        schemaName: "Answer",
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+        }
+    }));
+    assert.equal(result.turns, GROUNDED_MAX_TOOL_TURNS_BEFORE_SUBMIT + 1);
+    assert.deepEqual(result.value, { answer: "bounded proposal" });
+    assert.match(model.seen[GROUNDED_MAX_TOOL_TURNS_BEFORE_SUBMIT].instructions, /bounded exploration phase is complete/u);
+});
+test("grounded replay retains only adjacent evidence, bounds every request, and keeps unresolved loops recoverable", async (t) => {
+    const repo = await repository(t);
+    const reads = GROUNDED_FULL_RESULT_WINDOW + 2;
+    const model = scriptedModel((_input, turn) => turn <= reads
+        ? { action: "read_files", commit: "initial", paths: ["big.txt"], reason: "context" }
+        : { action: "submit", result: { answer: "done" } });
+    const result = await Effect.runPromise(groundedStructuredGenerationV1({
+        languageModel: model.shape,
+        assignment,
+        operationId: "spec",
+        instructions: "Read.",
+        task: { goal: "read the big file" },
+        outputSchema: Answer,
+        schemaName: "Answer",
+        maxToolTurnsBeforeSubmit: reads,
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+        }
+    }));
+    assert.equal(result.turns, reads + 1);
+    const last = model.seen[reads].input;
+    assert.equal(last.transcript.length, reads);
+    const full = last.transcript.filter((entry) => "result" in entry);
+    const summarized = last.transcript.filter((entry) => "resultBytes" in entry);
+    assert.equal(full.length + summarized.length, reads);
+    assert.equal(full.length, GROUNDED_FULL_RESULT_WINDOW, "only the adjacent before/after evidence window remains complete");
+    assert.ok(summarized.length >= 2);
+    assert.deepEqual(summarized[0].action, {
+        action: "read_files",
+        commit: "initial",
+        paths: ["big.txt"]
+    });
+    assert.equal(typeof summarized[0].resultBytes, "number");
+    assert.match(String(summarized[0].note), /re-request/u);
+    assert.equal(last.hostNotes.length, 0);
+    assert.ok(Buffer.byteLength(JSON.stringify(last)) <= GROUNDED_INPUT_BYTES_LIMIT);
+    assert.ok(model.seen.every((entry) => Buffer.byteLength(JSON.stringify(entry.input)) <= GROUNDED_INPUT_BYTES_LIMIT), "repository exploration must not grow the next model request without bound");
+    const earlier = model.seen[GROUNDED_FULL_RESULT_WINDOW].input;
+    assert.ok(earlier.transcript.every((entry) => "resultBytes" in entry || "result" in entry));
+    const unresolved = scriptedModel(() => ({
+        action: "list_files",
+        commit: "reference",
+        glob: "**/*.mjs"
+    }));
+    const error = await Effect.runPromise(groundedStructuredGenerationV1({
+        languageModel: unresolved.shape,
+        assignment,
+        operationId: "spec",
+        instructions: "Read.",
+        task: {},
+        outputSchema: Answer,
+        schemaName: "Answer",
+        maxTurns: 2,
+        tools: {
+            repositoryRoot: repo.root,
+            commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+        }
+    }).pipe(Effect.flip));
+    assert.ok(error instanceof RepositoryFoundryError);
+    assert.equal(error.operation, "run-pipeline-stage");
+    assert.ok(error.detail.startsWith(GROUNDED_AUTHORING_UNRESOLVED_PREFIX));
+    assert.equal(unresolved.seen.length, 2);
+    const listing = unresolved.seen[1].input.transcript[0].result;
+    assert.deepEqual(listing.paths, ["case.test.mjs", "lib/sse-decoder.mjs", "subject.mjs"]);
+    assert.equal(listing.total, 3);
+});
+test("through the budgeted model every turn is counted, invalid output is fed back, and exhaustion is typed", async (t) => {
+    const repo = await repository(t);
+    const directory = join(await mkdtemp(join(tmpdir(), "grounded-authoring-store-")), "case");
+    t.after(() => rm(join(directory, ".."), { recursive: true, force: true }));
+    const store = makeCaseCheckpointStoreV1({ caseDirectory: directory });
+    const state = await Effect.runPromise(makePipelineStateAccessorV1(initialState()));
+    const model = scriptedModel((_input, turn) => {
+        if (turn === 1)
+            return { action: "fly" };
+        if (turn === 2) {
+            return { action: "search", commit: "initial", pattern: "value", pathPrefix: "subject.mjs" };
+        }
+        return { action: "submit", result: { answer: "done" } };
+    });
+    const layer = BudgetedLanguageModelLive({
+        store,
+        state,
+        budget: { maxModelCalls: 3, maxWallMs: 60 * 60_000 },
+        startedAtMs: Date.now(),
+        inner: model.shape
+    });
+    const run = (operationId) => Effect.gen(function* () {
+        const languageModel = yield* RepositoryFoundryLanguageModel;
+        return yield* groundedStructuredGenerationV1({
+            languageModel,
+            assignment,
+            operationId,
+            instructions: "Search.",
+            task: { goal: "find value" },
+            outputSchema: Answer,
+            schemaName: "Answer",
+            tools: {
+                repositoryRoot: repo.root,
+                commits: { initial: repo.initialCommit, reference: repo.referenceCommit }
+            }
+        });
+    });
+    const outcome = await Effect.runPromise(Effect.gen(function* () {
+        const result = yield* run("controls");
+        const snapshot = yield* budgetSnapshotV1;
+        return { result, snapshot };
+    }).pipe(Effect.provide(layer)));
+    assert.equal(outcome.result.turns, 3);
+    assert.equal(outcome.snapshot.modelCalls, 3);
+    assert.deepEqual(outcome.result.transcript.map((entry) => entry.action.action), ["invalid", "search", "submit"]);
+    const invalid = outcome.result.transcript[0].result;
+    assert.equal(invalid.error, "invalid-output");
+    assert.match(invalid.detail, /violated Answer/u);
+    assert.deepEqual(model.seen.map((entry) => entry.input.remainingCalls), [3, 2, 1]);
+    const search = outcome.result.transcript[1].result;
+    assert.match(search.matches[0].text, /export const value = 0/u);
+    const ledger = await Effect.runPromise(store.readLedger);
+    assert.deepEqual(ledger.map((entry) => entry.phase), [
+        "start",
+        "received",
+        "failed",
+        "start",
+        "received",
+        "validated",
+        "start",
+        "received",
+        "validated"
+    ]);
+    const exhausted = await Effect.runPromise(run("controls-2").pipe(Effect.provide(layer), Effect.flip));
+    assert.ok(exhausted instanceof PipelineBudgetExhaustedError);
+    assert.equal(model.seen.length, 3);
+});

package/dist/test/integrated-repository-history.test.d.ts

@@ -0,0 +1 @@
+export {};