lastlight-evals 0.7.30 → 0.9.0

package/dist/mechanism.test.js

@@ -7,21 +7,27 @@
  *   - agentic-pi's `githubApiBaseUrl` seam actually routes Octokit at it;
  *   - workspace seeding + execution grading flip red→green correctly.
  */
-import { describe, it, expect } from "vitest";
+import { describe, it, expect, afterAll } from "vitest";
 import { GitHubClient } from "agentic-pi/dist/extensions/github/client.js";
 import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from "node:fs";
+import { fileURLToPath } from "node:url";
 import { join } from "node:path";
 import { tmpdir } from "node:os";
 import { startFakeGitHub } from "./fake-github.js";
+import { appliedRepoConfigKeys, loadRepoConfigFixture, resolveEvalRepoConfig, } from "./repo-config.js";
 import { seedWorkspace, seedWorkspaceFromGit, prFilesFromGit, injectRepoContext } from "./seed.js";
 import { gitDiffAgainstBase } from "./run-instance.js";
 import { execFileSync } from "node:child_process";
-import { gradeExecution, gradeBehavioral, gradeTriage, gradeReview, fBeta } from "./grade.js";
+import { gradeExecution, gradeBehavioral, gradeTriage, gradeReview, gradeMarkers, fBeta } from "./grade.js";
+import { prContextPatch } from "./pr-context.js";
+import { defaultFixConfig } from "lastlight-core/evals";
 import { computeMartianRanking } from "./report.js";
 import { loadMergedConfig, resolvePhaseModel } from "./config.js";
 import { modelsArm, configArm, releaseOverlayGuard } from "./arm.js";
 import { collectMetrics } from "./metrics.js";
 const staticAuth = { getToken: async () => "fake-token", expiresAt: null, canRefresh: false };
+/** The shipped `datasets/` root — resolved from this file, not the cwd. */
+const DATASETS = join(fileURLToPath(new URL(".", import.meta.url)), "..", "datasets");
 describe("fake GitHub + agentic-pi github tools (baseUrl seam)", () => {
     it("serves seeded issues and records mutations made through the real GitHubClient", async () => {
         const fake = await startFakeGitHub({
@@ -199,6 +205,191 @@ describe("fake GitHub — PR + review endpoints (pr-review tier)", () => {
         }
     });
 });
+describe("fake GitHub — the repo-config layer seam (issue #180)", () => {
+    const seed = (repoConfig) => startFakeGitHub({ owner: "acme", repo: "widget", defaultBranch: "trunk", repoConfig });
+    it("reports `absent` for a repo with no .lastlight/ — the pre-#180 default", async () => {
+        // Every eval case that predates this feature takes this path, so it has to
+        // be a plain negative answer (never an error) AND report the trust ref.
+        const fake = await seed();
+        try {
+            expect(await fake.fetchRepoConfigTree("acme", "widget")).toEqual({
+                status: "absent",
+                defaultBranch: "trunk",
+            });
+            expect(fake.repoConfigFetches()).toBe(1);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("serves a seeded tree as `ok`, honouring includePath and the file cap", async () => {
+        const fake = await seed([
+            { path: "lastlight.yml", content: "models: {}\n" },
+            { path: "workflows/prompts/answer.md", content: "repo prompt" },
+            // Shared real estate: build-handoff docs live under `.lastlight/` too, and
+            // `fetchRepoLayer` filters them out BEFORE spending the byte budget.
+            { path: "issue-42/architect-plan.md", content: "not part of the layer" },
+        ]);
+        try {
+            const all = await fake.fetchRepoConfigTree("acme", "widget");
+            expect(all.status).toBe("ok");
+            if (all.status !== "ok")
+                return;
+            expect(all.defaultBranch).toBe("trunk");
+            expect(all.treeSha).toMatch(/^[0-9a-f]{40}$/);
+            expect(all.files.map((f) => f.path)).toEqual([
+                "lastlight.yml",
+                "workflows/prompts/answer.md",
+                "issue-42/architect-plan.md",
+            ]);
+            expect(all.files[0].mode).toBe("100644");
+            expect(all.files[1].content.toString("utf8")).toBe("repo prompt");
+            expect(all.truncated).toBe(false);
+            const filtered = await fake.fetchRepoConfigTree("acme", "widget", {
+                includePath: (p) => p.startsWith("workflows/"),
+            });
+            expect(filtered.status === "ok" && filtered.files.map((f) => f.path)).toEqual([
+                "workflows/prompts/answer.md",
+            ]);
+            const capped = await fake.fetchRepoConfigTree("acme", "widget", { maxFiles: 1 });
+            expect(capped.status === "ok" && capped.files).toHaveLength(1);
+            expect(capped.status === "ok" && capped.truncated).toBe(true);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("answers `not-modified` to a matching treeSha/etag (the warm-cache path)", async () => {
+        const fake = await seed([{ path: "lastlight.yml", content: "models: {}\n" }]);
+        try {
+            const first = await fake.fetchRepoConfigTree("acme", "widget");
+            if (first.status !== "ok")
+                throw new Error("expected ok");
+            // Content-exact conditional: same subtree sha ⇒ nothing downloaded.
+            const again = await fake.fetchRepoConfigTree("acme", "widget", { treeSha: first.treeSha });
+            expect(again).toEqual({ status: "not-modified", defaultBranch: "trunk", treeSha: first.treeSha, etag: first.etag });
+            // Root-tree ETag conditional (what octokit surfaces as a 304).
+            const byEtag = await fake.fetchRepoConfigTree("acme", "widget", { etag: first.etag, treeSha: first.treeSha });
+            expect(byEtag.status).toBe("not-modified");
+            // A stale sha still downloads.
+            expect((await fake.fetchRepoConfigTree("acme", "widget", { treeSha: "stale" })).status).toBe("ok");
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("is loud about a repo it was never seeded with (a wiring bug, not 'no config')", async () => {
+        const fake = await seed([{ path: "lastlight.yml", content: "models: {}\n" }]);
+        try {
+            await expect(fake.fetchRepoConfigTree("other", "repo")).rejects.toThrow(/no repo-config fixture/);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+});
+describe("repo-config layer — fixture → core's real resolver (issue #180)", () => {
+    const cacheRoots = [];
+    const cacheRoot = () => {
+        const dir = mkdtempSync(join(tmpdir(), "ll-eval-repocfg-"));
+        cacheRoots.push(dir);
+        return dir;
+    };
+    afterAll(() => {
+        for (const dir of cacheRoots)
+            rmSync(dir, { recursive: true, force: true });
+    });
+    const resolve = (fake, repo, workflowName = "answer") => resolveEvalRepoConfig({
+        repo,
+        workflowName,
+        client: fake,
+        defaultModel: "anthropic/claude-haiku-4-5",
+        cacheRoot: cacheRoot(),
+    });
+    it("reads a case's fixture from <datasetDir>/lastlight/<id>/, laid out as the repo commits it", () => {
+        const files = loadRepoConfigFixture(join(DATASETS, "repo-config"), "repoconfig__prompt-override");
+        expect(files?.map((f) => f.path)).toEqual([
+            "agent-context/repo-notes.md",
+            "lastlight.yml",
+            "workflows/prompts/answer.md",
+        ]);
+        // No fixture directory ⇒ no declaration. This is how every other tier stays
+        // on the operator config without knowing the feature exists.
+        expect(loadRepoConfigFixture(join(DATASETS, "repo-config"), "repoconfig__no-layer")).toBeUndefined();
+        expect(loadRepoConfigFixture(undefined, "anything")).toBeUndefined();
+    });
+    it("unpacks the shipped fixture, merges the in-bounds keys and drops the rest", async () => {
+        // The real production chain: fetchRepoLayer → sanitizeRepoFiles → unpack →
+        // resolveRepoConfig. We assert its OUTPUT, not a re-implementation of it.
+        const files = loadRepoConfigFixture(join(DATASETS, "repo-config"), "repoconfig__prompt-override");
+        const fake = await startFakeGitHub({ owner: "lastlight-evals", repo: "lantern", repoConfig: files });
+        try {
+            const { repoConfig, refusal } = await resolve(fake, "lastlight-evals/lantern");
+            expect(refusal).toBeUndefined();
+            expect(repoConfig).toBeDefined();
+            // Always the default branch — the trust ref, never a PR head.
+            expect(repoConfig.defaultBranch).toBe("main");
+            // The assets the repo contributed, unpacked under a real host path the
+            // runner's per-run asset resolver layers on top of the built-ins.
+            expect(repoConfig.assets).toEqual(["agent-context/repo-notes.md", "workflows/prompts/answer.md"]);
+            const prompt = join(repoConfig.assetRoot, "workflows", "prompts", "answer.md");
+            expect(readFileSync(prompt, "utf8")).toContain("REPO PROMPT APPLIED");
+            // In-bounds config merged, with `repo` provenance…
+            expect(repoConfig.disabled.crons).toEqual(["weekly-security-scan"]);
+            expect(appliedRepoConfigKeys(repoConfig)).toEqual(["disabled.crons"]);
+            // …and the out-of-bounds key dropped with a warning, not an exception.
+            expect(repoConfig.warnings.map((w) => w.code)).toContain("key-not-allowed");
+            // The arm's model survives a layer that doesn't set `models:` — otherwise
+            // a repo layer would silently rewrite the comparison axis.
+            expect(repoConfig.models.default).toBe("anthropic/claude-haiku-4-5");
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("degrades to the operator config for a repo with no layer, and when the seam throws", async () => {
+        const plain = await startFakeGitHub({ owner: "lastlight-evals", repo: "plain" });
+        try {
+            // `absent` ⇒ no layer at all ⇒ `runWorkflow` is called exactly as it was
+            // before the feature existed.
+            expect(await resolve(plain, "lastlight-evals/plain")).toEqual({});
+            // A client that FAILS, and one that doesn't implement the seam at all
+            // (a third-party mock predating #180 — a synchronous TypeError, not a
+            // rejection), must both degrade rather than take the run down.
+            for (const broken of [{ fetchRepoConfigTree: () => Promise.reject(new Error("boom")) }, {}]) {
+                const out = await resolveEvalRepoConfig({
+                    repo: "lastlight-evals/plain",
+                    workflowName: "answer",
+                    client: broken,
+                    defaultModel: "anthropic/claude-haiku-4-5",
+                    cacheRoot: cacheRoot(),
+                });
+                expect(out.repoConfig).toBeUndefined();
+                expect(out.refusal).toBeUndefined();
+            }
+        }
+        finally {
+            await plain.close();
+        }
+    });
+    it("surfaces a repo's own disabled.workflows as a refusal, not a layer", async () => {
+        const fake = await startFakeGitHub({
+            owner: "acme",
+            repo: "widget",
+            repoConfig: [{ path: "lastlight.yml", content: "disabled:\n  workflows:\n    - answer\n" }],
+        });
+        try {
+            const { repoConfig, refusal } = await resolve(fake, "acme/widget", "answer");
+            expect(repoConfig).toBeUndefined();
+            expect(refusal).toMatch(/disables the "answer" workflow/);
+            // A workflow the repo did NOT opt out of still resolves normally.
+            expect((await resolve(fake, "acme/widget", "pr-review")).refusal).toBeUndefined();
+        }
+        finally {
+            await fake.close();
+        }
+    });
+});
 describe("prFilesFromGit — GitHub /pulls/:n/files payload from a real git diff", () => {
     it("reports added/modified files with counts and per-file patch hunks", () => {
         const dir = mkdtempSync(join(tmpdir(), "prfiles-"));
@@ -774,4 +965,231 @@ describe("collectMetrics — subscription-model cost fallback", () => {
         }
     });
 });
+// ── The PR state machine seam (issues #251, #252) ───────────────────────────
+describe("PR context — core's own projection, not a copy", () => {
+    const args = {
+        repo: "acme/widget",
+        prNumber: 412,
+        title: "Bump tiny-case from 2.4.0 to 3.0.0",
+        body: "Bumps tiny-case.",
+        branch: "dependabot/npm_and_yarn/tiny-case-3",
+    };
+    it("projects the CI evidence the fix prompts render", () => {
+        const ctx = prContextPatch({
+            ...args,
+            seed: {
+                checks_state: "failing",
+                base_checks_state: "passing",
+                attempt: 2,
+                ci_jobs: [
+                    { name: "CI / test", log_excerpt: "SyntaxError: no export named 'kebab'", failing_step: "npm test" },
+                ],
+            },
+        });
+        // `{{ciSection}}` is the block the whole diagnosis rests on; `{{#if ciSection}}`
+        // is how the prompt gates it, so an empty string here means the agent is
+        // asked to diagnose a failure it was told nothing about.
+        expect(String(ctx.ciSection)).toContain("CI FAILURES");
+        expect(String(ctx.ciSection)).toContain("kebab");
+        expect(ctx.attempt).toBe(2);
+        expect(ctx.maxAttempts).toBe(defaultFixConfig().maxAttempts);
+        expect(ctx.checksState).toBe("failing");
+        expect(ctx.baseChecksState).toBe("passing");
+        expect(ctx.ciLogsAvailable).toBe(true);
+        // Both harness-owned paths reach the prompt as variables, so a fork that
+        // hardcoded the old ones is not silently in play.
+        expect(String(ctx.verifyScript)).toContain(".git/");
+        expect(String(ctx.notesFile)).toContain(".git/");
+    });
+    it("carries the flaky promotion the third attempt turns on", () => {
+        const ctx = prContextPatch({
+            ...args,
+            seed: { attempt: 3, flaky_deferrals: 2, checks_state: "failing" },
+        });
+        // The signal that stops attempt 3 from re-reporting `flaky` and then
+        // running the fix phase against a verdict that says "change nothing".
+        expect(ctx.flakyPromoted).toBe(true);
+        expect(ctx.flakyDeferrals).toBe(2);
+        expect(ctx.maxFlakyDeferrals).toBe(defaultFixConfig().maxFlakyDeferrals);
+    });
+    it("projects the merge gate's verdict AND its reason, from one decision", () => {
+        const green = prContextPatch({
+            ...args,
+            seed: { checks_state: "passing", settled_check_count: 3 },
+        });
+        expect(green.mayMerge).toBe(true);
+        expect(typeof green.mayMergeReason).toBe("string");
+        const pending = prContextPatch({ ...args, seed: { checks_state: "pending" } });
+        expect(pending.mayMerge).toBe(false);
+        // The reason is produced BY the decision — the panel, the log line and the
+        // prompt are three renderings of it, never three prose variants.
+        expect(String(pending.mayMergeReason).length).toBeGreaterThan(0);
+    });
+    it("seeds the policy blocks a prompt renders, under the case's overrides", () => {
+        const ctx = prContextPatch({ ...args, seed: { dependencies: { autoMergeMaxImpact: "low" } } });
+        expect(ctx.dependencies.autoMergeMaxImpact).toBe("low");
+        // Everything the case did NOT override is the shipped default, so a case
+        // states only what it is actually testing.
+        expect(ctx.fix.maxAttempts).toBe(defaultFixConfig().maxAttempts);
+    });
+    it("a case with no seed still gets a coherent snapshot", () => {
+        const ctx = prContextPatch(args);
+        expect(ctx.attempt).toBe(1);
+        expect(ctx.ciSection).toBe("");
+        expect(ctx.prNumber).toBe(412);
+    });
+});
+// ── Marker grading (fix / dependency-merge) ─────────────────────────────────
+describe("gradeMarkers", () => {
+    const diagnose = (cls) => `DIAGNOSIS_COMPLETE: pr=412 attempt=1 class=${cls} cause=renamed export ci_vs_local=none`;
+    it("grades the diagnosis class off the real marker line", () => {
+        const phases = [{ output: `Looked at the log.\n${diagnose("reproducible")}` }];
+        expect(gradeMarkers({ diagnosis_class: "reproducible" }, phases).ok).toBe(true);
+        expect(gradeMarkers({ diagnosis_class: "flaky" }, phases).ok).toBe(false);
+    });
+    it("does not accept a bare mention of the tag", () => {
+        // The disagreement that let a silent no-op run pass as a diagnosis (#251):
+        // the phase gate tested `output.includes(tag)` while the parser required
+        // `<TAG>:`. Grading goes through CORE's parser, so the two agree here.
+        const phases = [{ output: "I'll finish with DIAGNOSIS_COMPLETE once I understand this." }];
+        const g = gradeMarkers({ diagnosis_class: "reproducible" }, phases);
+        expect(g.ok).toBe(false);
+        expect(g.checks[0].detail).toContain("no marker");
+    });
+    it("takes the LAST marker across phases — a loop's later iteration supersedes", () => {
+        const phases = [
+            { output: diagnose("flaky") },
+            { output: diagnose("reproducible") },
+        ];
+        expect(gradeMarkers({ diagnosis_class: "reproducible" }, phases).ok).toBe(true);
+    });
+    it("accepts any of a set where two verdicts are both defensible", () => {
+        const phases = [{ output: diagnose("env-mismatch") }];
+        expect(gradeMarkers({ diagnosis_class_any_of: ["reproducible", "env-mismatch"] }, phases).ok).toBe(true);
+        expect(gradeMarkers({ diagnosis_class_any_of: ["flaky"] }, phases).ok).toBe(false);
+    });
+    it("grades the fix outcome and its gate", () => {
+        const phases = [{ output: "CI_FIX_COMPLETE: pr=412 attempt=1 outcome=pushed tried=rename gate=green" }];
+        expect(gradeMarkers({ fix_outcome: "pushed", fix_gate: "green" }, phases).ok).toBe(true);
+        expect(gradeMarkers({ fix_gate: "red" }, phases).ok).toBe(false);
+    });
+    it("grades the merge assessment's impact and action", () => {
+        const phases = [
+            { output: "ASSESSMENT_COMPLETE: pr=501 verdict=TRIVIAL impact=low action=automerge" },
+        ];
+        expect(gradeMarkers({ assessment_impact: "low", assessment_action: "automerge" }, phases).ok).toBe(true);
+        expect(gradeMarkers({ assessment_impact: "high" }, phases).ok).toBe(false);
+    });
+    it("is a no-op for a case that declares no expectations", () => {
+        expect(gradeMarkers(undefined, [{ output: "anything" }]).ok).toBe(true);
+    });
+});
+// ── The merge + Actions endpoints the two new tiers need ────────────────────
+describe("fake GitHub — merge, auto-merge, diff and Actions", () => {
+    const pr = {
+        number: 501,
+        title: "Bump @types/node from 22 to 26",
+        body: "bump",
+        base_ref: "main",
+        head_ref: "dependabot/types-node-26",
+        base_commit: "0".repeat(40),
+        head_commit: "0".repeat(40),
+        user: "dependabot[bot]",
+        files: [
+            {
+                sha: "0".repeat(40),
+                filename: "package.json",
+                status: "modified",
+                additions: 1,
+                deletions: 1,
+                changes: 2,
+                patch: '@@ -12,7 +12,7 @@\n-    "@types/node": "^22.10.2",\n+    "@types/node": "^26.1.1",',
+            },
+        ],
+    };
+    it("enables auto-merge through the GraphQL mutation the real tool uses", async () => {
+        // There is NO REST endpoint for this. A REST-only fake 404s the merge
+        // workflow's preferred path and the agent silently falls back to a direct
+        // merge — every case would then measure the fallback.
+        const fake = await startFakeGitHub({ owner: "acme", repo: "app", pulls: [pr] });
+        try {
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            const res = (await gh.enablePullRequestAutoMerge("acme", "app", 501, "squash"));
+            expect(res.ok).toBe(true);
+            expect(fake.autoMergeOf(501)?.method).toBe("squash");
+            // Auto-merge is NOT a merge: the PR is still open, waiting on its checks.
+            expect(fake.mergeOf(501)).toBeUndefined();
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("records a direct merge separately from auto-merge", async () => {
+        const fake = await startFakeGitHub({ owner: "acme", repo: "app", pulls: [pr] });
+        try {
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            await gh.mergePullRequest("acme", "app", 501, { merge_method: "squash" });
+            expect(fake.mergeOf(501)?.method).toBe("squash");
+            expect(fake.autoMergeOf(501)).toBeUndefined();
+            // …and the two are gradeable as the different decisions they are.
+            expect(gradeBehavioral({ pr_merged: true, auto_merge_enabled: false }, fake, { issueNumber: 501, branch: "main" }).ok).toBe(true);
+            expect(gradeBehavioral({ pr_merged: false }, fake, { issueNumber: 501, branch: "main" }).ok).toBe(false);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("serves the PR patch to the diff media type, not JSON", async () => {
+        const fake = await startFakeGitHub({ owner: "acme", repo: "app", pulls: [pr] });
+        try {
+            fake.setPullFiles(501, pr.files);
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            const diff = (await gh.getPullRequestDiff("acme", "app", 501));
+            // A JSON body here would hand the impact assessment an object where it
+            // expects a patch — it would then reason about the wrong artifact.
+            expect(String(diff)).toContain("diff --git a/package.json");
+            expect(String(diff)).toContain("@types/node");
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("serves the CI reads from the same fixture the prompt was built from", async () => {
+        const fake = await startFakeGitHub({
+            owner: "acme",
+            repo: "app",
+            pulls: [pr],
+            actions: {
+                headSha: "f1a0b22",
+                jobs: [{ name: "CI / test", log: "SyntaxError: no export named 'kebab'", failingStep: "npm test" }],
+            },
+        });
+        try {
+            const gh = new GitHubClient(staticAuth, { baseUrl: fake.url });
+            const runs = (await gh.listWorkflowRuns("acme", "app", {}));
+            expect(runs.workflow_runs.length).toBe(1);
+            const jobs = (await gh.listWorkflowRunJobs("acme", "app", runs.workflow_runs[0].id));
+            expect(jobs.jobs[0].name).toBe("CI / test");
+            const logs = (await gh.getJobLogs("acme", "app", jobs.jobs[0].id));
+            // An agent that digs into the logs must read the SAME failure the prompt
+            // told it about — one seed, two surfaces.
+            expect(JSON.stringify(logs)).toContain("kebab");
+        }
+        finally {
+            await fake.close();
+        }
+    });
+    it("keeps an un-fixtured Actions route a loud 404", async () => {
+        // The rest of this file's contract: an unimplemented route 404s rather than
+        // returning a plausible empty answer.
+        const fake = await startFakeGitHub({ owner: "acme", repo: "app", pulls: [pr] });
+        try {
+            const res = await fetch(`${fake.url}/repos/acme/app/actions/runs`);
+            expect(res.status).toBe(404);
+        }
+        finally {
+            await fake.close();
+        }
+    });
+});
 //# sourceMappingURL=mechanism.test.js.map