lastlight-evals 0.8.0 → 0.9.0

package/datasets/fix/repos/fix__flaky-timeout/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__flaky-timeout/README.md

@@ -0,0 +1,3 @@
+# poller
+
+A polling helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__flaky-timeout/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^4.2.0"
+  }
+}

package/datasets/fix/repos/fix__flaky-timeout/src/poll.js

@@ -0,0 +1,9 @@
+/** Resolve once `check()` returns true, polling every `intervalMs`. */
+export async function pollUntil(check, { intervalMs = 10, timeoutMs = 200 } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    if (await check()) return true;
+    if (Date.now() > deadline) throw new Error("pollUntil: timed out");
+    await new Promise((r) => setTimeout(r, intervalMs));
+  }
+}

package/datasets/fix/repos/fix__flaky-timeout/test/poll.test.js

@@ -0,0 +1,8 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { pollUntil } from "../src/poll.js";
+
+test("resolves once the condition holds", async () => {
+  let n = 0;
+  assert.equal(await pollUntil(() => ++n > 2), true);
+});

package/datasets/fix/repos/fix__infra-dependent-e2e/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test
+
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      # Needs the deployed staging API and its credentials.
+      - run: node --test e2e/
+        env:
+          STAGING_BASE_URL: ${{ secrets.STAGING_BASE_URL }}
+          STAGING_TOKEN: ${{ secrets.STAGING_TOKEN }}

package/datasets/fix/repos/fix__infra-dependent-e2e/README.md

@@ -0,0 +1,3 @@
+# retryer
+
+A retry helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__infra-dependent-e2e/e2e/smoke.test.js

@@ -0,0 +1,10 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+// Runs only in the `e2e` job, against the deployed staging API.
+test("staging answers /health", async () => {
+  const base = process.env.STAGING_BASE_URL;
+  assert.ok(base, "STAGING_BASE_URL is not set");
+  const res = await fetch(new URL("/health", base));
+  assert.equal(res.status, 200);
+});

package/datasets/fix/repos/fix__infra-dependent-e2e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "retryer",
+  "version": "3.2.1",
+  "type": "module",
+  "description": "Retry helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-backoff": "^3.8.0"
+  }
+}

package/datasets/fix/repos/fix__infra-dependent-e2e/src/retry.js

@@ -0,0 +1,13 @@
+/** Retry `fn` up to `attempts` times with a fixed delay. */
+export async function retry(fn, { attempts = 3, delayMs = 0 } = {}) {
+  let last;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      return await fn();
+    } catch (err) {
+      last = err;
+      if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
+    }
+  }
+  throw last;
+}

package/datasets/fix/repos/fix__infra-dependent-e2e/test/retry.test.js

@@ -0,0 +1,8 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { retry } from "../src/retry.js";
+
+test("retries until it succeeds", async () => {
+  let n = 0;
+  assert.equal(await retry(() => (++n < 3 ? Promise.reject(new Error("no")) : "ok")), "ok");
+});

package/datasets/fix/repos/fix__reproducible-api-removal/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__reproducible-api-removal/README.md

@@ -0,0 +1,3 @@
+# slugger
+
+A slug helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__reproducible-api-removal/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "slugger",
+  "version": "1.4.0",
+  "type": "module",
+  "description": "Slug helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "imports": {
+    "#tiny-case": "./vendor/tiny-case/index.js"
+  },
+  "dependencies": {
+    "tiny-case": "^2.4.0"
+  }
+}

package/datasets/fix/repos/fix__reproducible-api-removal/src/slug.js

@@ -0,0 +1,6 @@
+import { kebab } from "#tiny-case";
+
+/** Turn a title into a URL slug. */
+export function slug(title) {
+  return kebab(title);
+}

package/datasets/fix/repos/fix__reproducible-api-removal/test/slug.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { slug } from "../src/slug.js";
+
+test("slugs a title", () => {
+  assert.equal(slug("Hello There World"), "hello-there-world");
+});

package/datasets/fix/repos/fix__reproducible-api-removal/vendor/tiny-case/index.js

@@ -0,0 +1,4 @@
+// tiny-case@2.4.0
+export function kebab(input) {
+  return String(input).trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__upstream-broken-base-red/README.md

@@ -0,0 +1,3 @@
+# money
+
+Currency formatting. `npm test` runs the suite.

package/datasets/fix/repos/fix__upstream-broken-base-red/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "money",
+  "version": "5.0.0",
+  "type": "module",
+  "description": "Currency formatting.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-intl": "^6.4.0"
+  }
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/src/format.js

@@ -0,0 +1,6 @@
+/** Format minor units as a currency string. */
+export function formatMoney(minorUnits, currency = "USD") {
+  return new Intl.NumberFormat("en-US", { style: "currency", currency }).format(
+    minorUnits / 100,
+  );
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/src/rates.js

@@ -0,0 +1,6 @@
+/** Convert minor units at a rate, rounding to whole minor units. */
+export function convert(minorUnits, rate) {
+  // Regression landed on main in #874: the rounding was dropped, so this
+  // returns a fractional amount and every downstream total is wrong.
+  return minorUnits * rate;
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/test/format.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { formatMoney } from "../src/format.js";
+
+test("formats dollars", () => {
+  assert.equal(formatMoney(12345), "$123.45");
+});

package/datasets/fix/repos/fix__upstream-broken-base-red/test/rates.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { convert } from "../src/rates.js";
+
+test("converts to whole minor units", () => {
+  assert.equal(convert(1000, 1.2345), 1235);
+});

package/datasets/fix/repos-head/fix__env-mismatch-node-version/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "grouper",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Grouping helper.",
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-group": "^2.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__flaky-promoted/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^5.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__flaky-timeout/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^5.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__infra-dependent-e2e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "retryer",
+  "version": "3.2.1",
+  "type": "module",
+  "description": "Retry helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-backoff": "^4.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__reproducible-api-removal/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "slugger",
+  "version": "1.4.0",
+  "type": "module",
+  "description": "Slug helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "imports": {
+    "#tiny-case": "./vendor/tiny-case/index.js"
+  },
+  "dependencies": {
+    "tiny-case": "^3.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__reproducible-api-removal/vendor/tiny-case/index.js

@@ -0,0 +1,4 @@
+// tiny-case@3.0.0 — `kebab` was renamed to `kebabCase` in 3.0.0.
+export function kebabCase(input) {
+  return String(input).trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}

package/datasets/fix/repos-head/fix__upstream-broken-base-red/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "money",
+  "version": "5.0.0",
+  "type": "module",
+  "description": "Currency formatting.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-intl": "^7.0.0"
+  }
+}

package/datasets/fix/tier.json

@@ -0,0 +1,5 @@
+{
+  "name": "fix",
+  "defaultWorkflow": "dependabot-ci-fix",
+  "description": "The fix loop on a RED dependency PR: does the agent reach the right diagnosis class, and does it act on it? One case per class in `DIAGNOSIS_CLASSES`, plus the promoted-flaky attempt."
+}

package/dist/fake-github.js

@@ -181,6 +181,62 @@ export async function startFakeGitHub(opts) {
         }
         return { status: "ok", defaultBranch, treeSha: repoConfigTreeSha, etag: repoConfigEtag, files, truncated };
     }
+    // ── Actions fixture ───────────────────────────────────────────────────
+    // Ids are synthetic but stable within a run, which is all the tools need:
+    // `github_list_workflow_runs` hands the agent a run id and it asks for that
+    // run's jobs, then that job's logs.
+    const ACTIONS_RUN_ID = 900001;
+    const actionsJobs = (opts.actions?.jobs ?? []).map((j, i) => ({
+        id: 800001 + i,
+        name: j.name,
+        conclusion: j.conclusion ?? "failure",
+        log: j.log,
+        workflowPath: j.workflowPath,
+        failingStep: j.failingStep,
+    }));
+    function serializeRun() {
+        const a = opts.actions;
+        return {
+            id: ACTIONS_RUN_ID,
+            name: a.workflowName ?? "CI",
+            head_sha: a.headSha,
+            head_branch: a.headBranch ?? defaultBranch,
+            path: a.jobs[0]?.workflowPath ?? ".github/workflows/ci.yml",
+            event: "pull_request",
+            status: "completed",
+            conclusion: "failure",
+            created_at: NOW,
+            updated_at: NOW,
+            html_url: `https://github.com/${owner}/${repo}/actions/runs/${ACTIONS_RUN_ID}`,
+        };
+    }
+    function serializeJob(j) {
+        return {
+            id: j.id,
+            run_id: ACTIONS_RUN_ID,
+            name: j.name,
+            status: "completed",
+            conclusion: j.conclusion,
+            started_at: NOW,
+            completed_at: NOW,
+            html_url: `https://github.com/${owner}/${repo}/actions/runs/${ACTIONS_RUN_ID}/job/${j.id}`,
+            steps: j.failingStep
+                ? [{ name: j.failingStep, status: "completed", conclusion: "failure", number: 1 }]
+                : [],
+        };
+    }
+    /**
+     * The PR's patch, assembled from the per-file `patch` hunks already seeded
+     * for `GET /pulls/:n/files`. One fixture, two representations — a case that
+     * declares its changed files gets both the file list and the diff, and they
+     * cannot disagree.
+     */
+    function diffOf(prNumber) {
+        const files = pullFiles.get(prNumber) ?? [];
+        return files
+            .map((f) => `diff --git a/${f.filename} b/${f.filename}\n${f.patch ?? ""}`)
+            .join("\n");
+    }
     const repoBase = `/repos/${owner}/${repo}`;
     const server = createServer((req, res) => {
         const method = req.method ?? "GET";
@@ -207,7 +263,8 @@ export async function startFakeGitHub(opts) {
                 res.end(text);
             };
             try {
-                const handled = route(method, path, body, json);
+                const accept = String(req.headers.accept ?? "");
+                const handled = route(method, path, body, json, accept, res);
                 if (!handled)
                     json(404, { message: `fake-github: no route for ${method} ${path}` });
             }
@@ -216,7 +273,73 @@ export async function startFakeGitHub(opts) {
             }
         });
     });
-    function route(method, path, body, json) {
+    function route(method, path, body, json, 
+    /** Request `Accept` — the diff media type is how `pulls.get` asks for a patch. */
+    accept, res) {
+        // ── GraphQL: the auto-merge mutation ────────────────────────────────
+        //
+        // `github_enable_auto_merge` has NO REST equivalent — GitHub exposes
+        // `enablePullRequestAutoMerge` only through GraphQL, so a REST-only fake
+        // 404s the one call the merge workflow's preferred path depends on, and the
+        // agent silently falls back to a direct merge. That would make every
+        // dependency-merge case measure the fallback.
+        if (method === "POST" && path === "/graphql") {
+            const q = String(body?.query ?? "");
+            const vars = (body?.variables ??
+                {});
+            if (q.includes("enablePullRequestAutoMerge")) {
+                // The node id the client resolved via REST is `PR_<number>` here (see
+                // `serializePull`), so the mutation can find the PR the same way.
+                const num = Number(String(vars.id ?? "").replace(/^PR_/, ""));
+                const pr = pulls.find((p) => p.number === num);
+                if (!pr) {
+                    json(200, { data: null, errors: [{ message: `Could not resolve to a node: ${vars.id}` }] });
+                    return true;
+                }
+                const method_ = String(vars.mergeMethod ?? "SQUASH").toLowerCase();
+                pr.autoMerge = { method: method_, enabledAt: NOW };
+                json(200, {
+                    data: {
+                        enablePullRequestAutoMerge: {
+                            pullRequest: { number: pr.number, autoMergeRequest: { enabledAt: NOW } },
+                        },
+                    },
+                });
+                return true;
+            }
+            json(200, { data: null, errors: [{ message: `fake-github: unsupported GraphQL operation` }] });
+            return true;
+        }
+        // ── Actions (CI reads) ──────────────────────────────────────────────
+        //
+        // Served from the `actions` fixture, which is built from the same seed as
+        // the prompt's CI summary — so `github_get_job_logs` corroborates
+        // `{{ciSection}}` instead of contradicting it. With NO fixture these stay
+        // 404, which is the loud default the rest of this file keeps.
+        if (opts.actions && method === "GET" && path.startsWith(`${repoBase}/actions/`)) {
+            const rest = path.slice(`${repoBase}/actions/`.length);
+            if (rest === "runs" || /^workflows\/[^/]+\/runs$/.test(rest)) {
+                json(200, { total_count: 1, workflow_runs: [serializeRun()] });
+                return true;
+            }
+            const jobsOf = /^runs\/(\d+)\/jobs$/.exec(rest);
+            if (jobsOf) {
+                json(200, { total_count: actionsJobs.length, jobs: actionsJobs.map(serializeJob) });
+                return true;
+            }
+            const logsOf = /^jobs\/(\d+)\/logs$/.exec(rest);
+            if (logsOf) {
+                const job = actionsJobs.find((j) => j.id === Number(logsOf[1]));
+                if (!job)
+                    return false;
+                // Real GitHub 302s to a signed blob URL and octokit follows it; a direct
+                // 200 with the text is the same thing to the caller.
+                res.writeHead(200, { "content-type": "text/plain; charset=utf-8" });
+                res.end(job.log);
+                return true;
+            }
+            return false;
+        }
         // GET /repos/:owner/:repo
         if (method === "GET" && path === repoBase) {
             json(200, {
@@ -353,8 +476,8 @@ export async function startFakeGitHub(opts) {
                 return true;
             }
         }
-        // Per-PR routes: /repos/:owner/:repo/pulls/:n[/reviews|/comments|/files]
-        const pullMatch = path.match(new RegExp(`^${escapeRe(repoBase)}/pulls/(\\d+)(/reviews|/comments|/files)?$`));
+        // Per-PR routes: /repos/:owner/:repo/pulls/:n[/reviews|/comments|/files|/merge]
+        const pullMatch = path.match(new RegExp(`^${escapeRe(repoBase)}/pulls/(\\d+)(/reviews|/comments|/files|/merge)?$`));
         if (pullMatch) {
             const num = Number(pullMatch[1]);
             const sub = pullMatch[2];
@@ -363,11 +486,36 @@ export async function startFakeGitHub(opts) {
                 json(404, { message: `pull ${num} not found` });
                 return true;
             }
-            // /pulls/:n
+            // /pulls/:n — JSON, or the raw patch when the caller asked for the diff
+            // media type. `github_get_pull_request_diff` is `pulls.get` with
+            // `mediaType: { format: "diff" }`, so serving JSON to it would hand the
+            // agent an object where it expects a patch — the impact assessment then
+            // reasons about the wrong artifact entirely.
             if (!sub && method === "GET") {
+                if (/vnd\.github(\.[^+\s]*)?\.diff/.test(accept) || /vnd\.github\.v3\.diff/.test(accept)) {
+                    // `charset=utf-8` is load-bearing: octokit decodes a response body as
+                    // TEXT only for `application/json`, `text/*` or a `charset=utf-8`
+                    // content-type, and hands anything else back as an ArrayBuffer.
+                    res.writeHead(200, { "content-type": "application/vnd.github.v3.diff; charset=utf-8" });
+                    res.end(diffOf(pr.number));
+                    return true;
+                }
                 json(200, serializePull(pr));
                 return true;
             }
+            // /pulls/:n/merge — the direct merge (`github_merge_pull_request`).
+            if (sub === "/merge" && method === "PUT") {
+                const b = (body ?? {});
+                if (pr.merged) {
+                    json(405, { message: "Pull Request is not mergeable" });
+                    return true;
+                }
+                pr.merged = true;
+                pr.state = "closed";
+                pr.mergedBy = { method: b.merge_method ?? "merge" };
+                json(200, { sha: pr.head.sha, merged: true, message: "Pull Request successfully merged" });
+                return true;
+            }
             // /pulls/:n/reviews — list existing, or SUBMIT one (create_pull_request_review).
             if (sub === "/reviews") {
                 if (method === "GET") {
@@ -419,10 +567,15 @@ export async function startFakeGitHub(opts) {
     function serializePull(pr) {
         return {
             number: pr.number,
+            // The GraphQL node id `enablePullRequestAutoMerge` is given. The real one
+            // is opaque; the client only ever round-trips it from here to the
+            // mutation, so a derivable form keeps the fake stateless about it.
+            node_id: `PR_${pr.number}`,
             title: pr.title,
             body: pr.body,
             state: pr.state,
             merged: pr.merged,
+            auto_merge: pr.autoMerge ? { enabled_by: { login: "last-light[bot]" }, merge_method: pr.autoMerge.method } : null,
             head: pr.head,
             base: pr.base,
             user: pr.user,
@@ -473,6 +626,8 @@ export async function startFakeGitHub(opts) {
         setPullFiles: (n, files) => pullFiles.set(n, files),
         fetchRepoConfigTree,
         repoConfigFetches: () => repoConfigFetches,
+        mergeOf: (n) => pulls.find((p) => p.number === n)?.mergedBy,
+        autoMergeOf: (n) => pulls.find((p) => p.number === n)?.autoMerge,
     };
 }
 /** Inverse of {@link eventToState} — the grader reports the review's event. */