lastlight-evals 0.7.30 → 0.9.0

package/datasets/fix/repos/fix__flaky-promoted/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__flaky-promoted/README.md

@@ -0,0 +1,3 @@
+# poller
+
+A polling helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__flaky-promoted/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^4.2.0"
+  }
+}

package/datasets/fix/repos/fix__flaky-promoted/src/poll.js

@@ -0,0 +1,9 @@
+/** Resolve once `check()` returns true, polling every `intervalMs`. */
+export async function pollUntil(check, { intervalMs = 10, timeoutMs = 200 } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    if (await check()) return true;
+    if (Date.now() > deadline) throw new Error("pollUntil: timed out");
+    await new Promise((r) => setTimeout(r, intervalMs));
+  }
+}

package/datasets/fix/repos/fix__flaky-promoted/test/poll.test.js

@@ -0,0 +1,8 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { pollUntil } from "../src/poll.js";
+
+test("resolves once the condition holds", async () => {
+  let n = 0;
+  assert.equal(await pollUntil(() => ++n > 2), true);
+});

package/datasets/fix/repos/fix__flaky-timeout/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__flaky-timeout/README.md

@@ -0,0 +1,3 @@
+# poller
+
+A polling helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__flaky-timeout/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^4.2.0"
+  }
+}

package/datasets/fix/repos/fix__flaky-timeout/src/poll.js

@@ -0,0 +1,9 @@
+/** Resolve once `check()` returns true, polling every `intervalMs`. */
+export async function pollUntil(check, { intervalMs = 10, timeoutMs = 200 } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    if (await check()) return true;
+    if (Date.now() > deadline) throw new Error("pollUntil: timed out");
+    await new Promise((r) => setTimeout(r, intervalMs));
+  }
+}

package/datasets/fix/repos/fix__flaky-timeout/test/poll.test.js

@@ -0,0 +1,8 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { pollUntil } from "../src/poll.js";
+
+test("resolves once the condition holds", async () => {
+  let n = 0;
+  assert.equal(await pollUntil(() => ++n > 2), true);
+});

package/datasets/fix/repos/fix__infra-dependent-e2e/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test
+
+  e2e:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      # Needs the deployed staging API and its credentials.
+      - run: node --test e2e/
+        env:
+          STAGING_BASE_URL: ${{ secrets.STAGING_BASE_URL }}
+          STAGING_TOKEN: ${{ secrets.STAGING_TOKEN }}

package/datasets/fix/repos/fix__infra-dependent-e2e/README.md

@@ -0,0 +1,3 @@
+# retryer
+
+A retry helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__infra-dependent-e2e/e2e/smoke.test.js

@@ -0,0 +1,10 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+// Runs only in the `e2e` job, against the deployed staging API.
+test("staging answers /health", async () => {
+  const base = process.env.STAGING_BASE_URL;
+  assert.ok(base, "STAGING_BASE_URL is not set");
+  const res = await fetch(new URL("/health", base));
+  assert.equal(res.status, 200);
+});

package/datasets/fix/repos/fix__infra-dependent-e2e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "retryer",
+  "version": "3.2.1",
+  "type": "module",
+  "description": "Retry helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-backoff": "^3.8.0"
+  }
+}

package/datasets/fix/repos/fix__infra-dependent-e2e/src/retry.js

@@ -0,0 +1,13 @@
+/** Retry `fn` up to `attempts` times with a fixed delay. */
+export async function retry(fn, { attempts = 3, delayMs = 0 } = {}) {
+  let last;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      return await fn();
+    } catch (err) {
+      last = err;
+      if (delayMs) await new Promise((r) => setTimeout(r, delayMs));
+    }
+  }
+  throw last;
+}

package/datasets/fix/repos/fix__infra-dependent-e2e/test/retry.test.js

@@ -0,0 +1,8 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { retry } from "../src/retry.js";
+
+test("retries until it succeeds", async () => {
+  let n = 0;
+  assert.equal(await retry(() => (++n < 3 ? Promise.reject(new Error("no")) : "ok")), "ok");
+});

package/datasets/fix/repos/fix__reproducible-api-removal/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__reproducible-api-removal/README.md

@@ -0,0 +1,3 @@
+# slugger
+
+A slug helper. `npm test` runs the suite.

package/datasets/fix/repos/fix__reproducible-api-removal/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "slugger",
+  "version": "1.4.0",
+  "type": "module",
+  "description": "Slug helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "imports": {
+    "#tiny-case": "./vendor/tiny-case/index.js"
+  },
+  "dependencies": {
+    "tiny-case": "^2.4.0"
+  }
+}

package/datasets/fix/repos/fix__reproducible-api-removal/src/slug.js

@@ -0,0 +1,6 @@
+import { kebab } from "#tiny-case";
+
+/** Turn a title into a URL slug. */
+export function slug(title) {
+  return kebab(title);
+}

package/datasets/fix/repos/fix__reproducible-api-removal/test/slug.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { slug } from "../src/slug.js";
+
+test("slugs a title", () => {
+  assert.equal(slug("Hello There World"), "hello-there-world");
+});

package/datasets/fix/repos/fix__reproducible-api-removal/vendor/tiny-case/index.js

@@ -0,0 +1,4 @@
+// tiny-case@2.4.0
+export function kebab(input) {
+  return String(input).trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/.github/workflows/ci.yml

@@ -0,0 +1,15 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+      - run: npm test

package/datasets/fix/repos/fix__upstream-broken-base-red/README.md

@@ -0,0 +1,3 @@
+# money
+
+Currency formatting. `npm test` runs the suite.

package/datasets/fix/repos/fix__upstream-broken-base-red/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "money",
+  "version": "5.0.0",
+  "type": "module",
+  "description": "Currency formatting.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-intl": "^6.4.0"
+  }
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/src/format.js

@@ -0,0 +1,6 @@
+/** Format minor units as a currency string. */
+export function formatMoney(minorUnits, currency = "USD") {
+  return new Intl.NumberFormat("en-US", { style: "currency", currency }).format(
+    minorUnits / 100,
+  );
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/src/rates.js

@@ -0,0 +1,6 @@
+/** Convert minor units at a rate, rounding to whole minor units. */
+export function convert(minorUnits, rate) {
+  // Regression landed on main in #874: the rounding was dropped, so this
+  // returns a fractional amount and every downstream total is wrong.
+  return minorUnits * rate;
+}

package/datasets/fix/repos/fix__upstream-broken-base-red/test/format.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { formatMoney } from "../src/format.js";
+
+test("formats dollars", () => {
+  assert.equal(formatMoney(12345), "$123.45");
+});

package/datasets/fix/repos/fix__upstream-broken-base-red/test/rates.test.js

@@ -0,0 +1,7 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { convert } from "../src/rates.js";
+
+test("converts to whole minor units", () => {
+  assert.equal(convert(1000, 1.2345), 1235);
+});

package/datasets/fix/repos-head/fix__env-mismatch-node-version/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "grouper",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Grouping helper.",
+  "engines": {
+    "node": ">=22"
+  },
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-group": "^2.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__flaky-promoted/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^5.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__flaky-timeout/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "poller",
+  "version": "2.1.0",
+  "type": "module",
+  "description": "Polling helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "devDependencies": {
+    "tap-reporter": "^5.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__infra-dependent-e2e/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "retryer",
+  "version": "3.2.1",
+  "type": "module",
+  "description": "Retry helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-backoff": "^4.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__reproducible-api-removal/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "slugger",
+  "version": "1.4.0",
+  "type": "module",
+  "description": "Slug helper.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "imports": {
+    "#tiny-case": "./vendor/tiny-case/index.js"
+  },
+  "dependencies": {
+    "tiny-case": "^3.0.0"
+  }
+}

package/datasets/fix/repos-head/fix__reproducible-api-removal/vendor/tiny-case/index.js

@@ -0,0 +1,4 @@
+// tiny-case@3.0.0 — `kebab` was renamed to `kebabCase` in 3.0.0.
+export function kebabCase(input) {
+  return String(input).trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "");
+}

package/datasets/fix/repos-head/fix__upstream-broken-base-red/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "money",
+  "version": "5.0.0",
+  "type": "module",
+  "description": "Currency formatting.",
+  "scripts": {
+    "test": "node --test"
+  },
+  "dependencies": {
+    "tiny-intl": "^7.0.0"
+  }
+}

package/datasets/fix/tier.json

@@ -0,0 +1,5 @@
+{
+  "name": "fix",
+  "defaultWorkflow": "dependabot-ci-fix",
+  "description": "The fix loop on a RED dependency PR: does the agent reach the right diagnosis class, and does it act on it? One case per class in `DIAGNOSIS_CLASSES`, plus the promoted-flaky attempt."
+}

package/datasets/repo-config/instances.json

@@ -0,0 +1,35 @@
+[
+  {
+    "instance_id": "repoconfig__prompt-override",
+    "repo": "lastlight-evals/lantern",
+    "workflow": "answer",
+    "problem_statement": "What does the `default` key under `models` mean?",
+    "issue": {
+      "number": 201,
+      "title": "What does the `default` key under `models` do?",
+      "body": "Reading the config docs I see a `models:` block with a `default:` key and a few per-task keys next to it. Which one wins for a task that isn't listed? Just want to understand the precedence — not a bug.",
+      "labels": [],
+      "user": "erin"
+    },
+    "expect_github": {
+      "labels_added": ["question", "repo-prompt-applied"]
+    }
+  },
+  {
+    "instance_id": "repoconfig__no-layer",
+    "repo": "lastlight-evals/plain",
+    "workflow": "answer",
+    "problem_statement": "What does the `default` key under `models` mean?",
+    "issue": {
+      "number": 202,
+      "title": "What does the `default` key under `models` do?",
+      "body": "Reading the config docs I see a `models:` block with a `default:` key and a few per-task keys next to it. Which one wins for a task that isn't listed? Just want to understand the precedence — not a bug.",
+      "labels": [],
+      "user": "erin"
+    },
+    "expect_github": {
+      "labels_added": ["question"],
+      "labels_absent": ["repo-prompt-applied"]
+    }
+  }
+]

package/datasets/repo-config/lastlight/repoconfig__prompt-override/agent-context/repo-notes.md

@@ -0,0 +1,9 @@
+# Lantern — repository notes
+
+Lantern is a tiny demo repository. It ships its own Last Light configuration in
+`.lastlight/`, so runs against it are shaped by the repository and not only by
+the deployment that hosts the agent.
+
+(Deliberately carries no instruction about the marker label this eval case
+grades on: the marker must come from the repo's prompt override alone, or a pass
+would not prove the *prompt* reached the agent.)

package/datasets/repo-config/lastlight/repoconfig__prompt-override/lastlight.yml

@@ -0,0 +1,22 @@
+# Per-repository Last Light configuration (issue #180).
+#
+# This file is committed by the repository and read ONLY from its default
+# branch, so a pull request can never reconfigure the agent that reviews it.
+# It may set a bounded subset of keys — the deployment's
+# `repoConfig.allowKeys` — and anything outside that is dropped with a warning
+# rather than failing the run.
+
+# In bounds: this repo opts itself out of the deployment's weekly security cron.
+# Inert for the `answer` workflow this eval case runs, which is deliberate — it
+# proves the config half parses, merges and records `repo` provenance without
+# perturbing what the case actually measures (the prompt override below).
+disabled:
+  crons:
+    - weekly-security-scan
+
+# OUT of bounds: a repo may not pick the deployment's sandbox backend. Dropped
+# with a `key-not-allowed` warning and surfaced on the result's
+# `repoLayer.warnings`, so the failure rule — warn, drop the bad bits, run
+# anyway — is measured here too.
+sandbox:
+  backend: docker

package/datasets/repo-config/lastlight/repoconfig__prompt-override/workflows/prompts/answer.md

@@ -0,0 +1,27 @@
+You are answering a question on **{{owner}}/{{repo}}** issue #{{issueNumber}}.
+
+This prompt is committed by the repository itself, at
+`.lastlight/workflows/prompts/answer.md`. It **replaces** Last Light's built-in
+answer prompt for runs against this repository, so follow it exactly and ignore
+every other answering procedure — including the `issue-answer` skill, which you
+should not read.
+
+## The question
+
+**Title:** {{issueTitle}}
+
+{{issueBody}}
+
+## What to do
+
+1. Answer from the question alone. Do **not** clone anything, read any files,
+   search the web, or read a skill — this repository keeps its answers short and
+   its runs cheap.
+2. Label issue #{{issueNumber}} with **both** `question` and
+   `repo-prompt-applied`: one `github_ensure_labels` call for
+   `[{name: "question", color: "d876e3"}, {name: "repo-prompt-applied", color: "0e8a16"}]`,
+   then one `github_add_labels` call adding both names. Leave the issue open.
+3. Make your final message exactly two lines: the first line
+   `REPO PROMPT APPLIED`, the second a one-sentence answer to the question.
+
+Do not post a comment yourself — the harness delivers your final message.

package/datasets/repo-config/tier.json

@@ -0,0 +1,5 @@
+{
+  "name": "repo-config",
+  "defaultWorkflow": "answer",
+  "description": "Per-repository config layer (#180) — does a repo's committed .lastlight/ actually reach the agent? Cheap: one agent call per case, no clone, no judge."
+}