lastlight-evals 0.7.30 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +38 -1
  2. package/datasets/dependency-merge/README.md +69 -0
  3. package/datasets/dependency-merge/instances.json +168 -0
  4. package/datasets/dependency-merge/tier.json +5 -0
  5. package/datasets/fix/README.md +57 -0
  6. package/datasets/fix/instances.json +229 -0
  7. package/datasets/fix/repos/fix__env-mismatch-node-version/.github/workflows/ci.yml +15 -0
  8. package/datasets/fix/repos/fix__env-mismatch-node-version/README.md +3 -0
  9. package/datasets/fix/repos/fix__env-mismatch-node-version/package.json +15 -0
  10. package/datasets/fix/repos/fix__env-mismatch-node-version/src/group.js +4 -0
  11. package/datasets/fix/repos/fix__env-mismatch-node-version/test/group.test.js +9 -0
  12. package/datasets/fix/repos/fix__flaky-promoted/.github/workflows/ci.yml +15 -0
  13. package/datasets/fix/repos/fix__flaky-promoted/README.md +3 -0
  14. package/datasets/fix/repos/fix__flaky-promoted/package.json +12 -0
  15. package/datasets/fix/repos/fix__flaky-promoted/src/poll.js +9 -0
  16. package/datasets/fix/repos/fix__flaky-promoted/test/poll.test.js +8 -0
  17. package/datasets/fix/repos/fix__flaky-timeout/.github/workflows/ci.yml +15 -0
  18. package/datasets/fix/repos/fix__flaky-timeout/README.md +3 -0
  19. package/datasets/fix/repos/fix__flaky-timeout/package.json +12 -0
  20. package/datasets/fix/repos/fix__flaky-timeout/src/poll.js +9 -0
  21. package/datasets/fix/repos/fix__flaky-timeout/test/poll.test.js +8 -0
  22. package/datasets/fix/repos/fix__infra-dependent-e2e/.github/workflows/ci.yml +28 -0
  23. package/datasets/fix/repos/fix__infra-dependent-e2e/README.md +3 -0
  24. package/datasets/fix/repos/fix__infra-dependent-e2e/e2e/smoke.test.js +10 -0
  25. package/datasets/fix/repos/fix__infra-dependent-e2e/package.json +12 -0
  26. package/datasets/fix/repos/fix__infra-dependent-e2e/src/retry.js +13 -0
  27. package/datasets/fix/repos/fix__infra-dependent-e2e/test/retry.test.js +8 -0
  28. package/datasets/fix/repos/fix__reproducible-api-removal/.github/workflows/ci.yml +15 -0
  29. package/datasets/fix/repos/fix__reproducible-api-removal/README.md +3 -0
  30. package/datasets/fix/repos/fix__reproducible-api-removal/package.json +15 -0
  31. package/datasets/fix/repos/fix__reproducible-api-removal/src/slug.js +6 -0
  32. package/datasets/fix/repos/fix__reproducible-api-removal/test/slug.test.js +7 -0
  33. package/datasets/fix/repos/fix__reproducible-api-removal/vendor/tiny-case/index.js +4 -0
  34. package/datasets/fix/repos/fix__upstream-broken-base-red/.github/workflows/ci.yml +15 -0
  35. package/datasets/fix/repos/fix__upstream-broken-base-red/README.md +3 -0
  36. package/datasets/fix/repos/fix__upstream-broken-base-red/package.json +12 -0
  37. package/datasets/fix/repos/fix__upstream-broken-base-red/src/format.js +6 -0
  38. package/datasets/fix/repos/fix__upstream-broken-base-red/src/rates.js +6 -0
  39. package/datasets/fix/repos/fix__upstream-broken-base-red/test/format.test.js +7 -0
  40. package/datasets/fix/repos/fix__upstream-broken-base-red/test/rates.test.js +7 -0
  41. package/datasets/fix/repos-head/fix__env-mismatch-node-version/package.json +15 -0
  42. package/datasets/fix/repos-head/fix__flaky-promoted/package.json +12 -0
  43. package/datasets/fix/repos-head/fix__flaky-timeout/package.json +12 -0
  44. package/datasets/fix/repos-head/fix__infra-dependent-e2e/package.json +12 -0
  45. package/datasets/fix/repos-head/fix__reproducible-api-removal/package.json +15 -0
  46. package/datasets/fix/repos-head/fix__reproducible-api-removal/vendor/tiny-case/index.js +4 -0
  47. package/datasets/fix/repos-head/fix__upstream-broken-base-red/package.json +12 -0
  48. package/datasets/fix/tier.json +5 -0
  49. package/datasets/repo-config/instances.json +35 -0
  50. package/datasets/repo-config/lastlight/repoconfig__prompt-override/agent-context/repo-notes.md +9 -0
  51. package/datasets/repo-config/lastlight/repoconfig__prompt-override/lastlight.yml +22 -0
  52. package/datasets/repo-config/lastlight/repoconfig__prompt-override/workflows/prompts/answer.md +27 -0
  53. package/datasets/repo-config/tier.json +5 -0
  54. package/dist/fake-github.js +228 -5
  55. package/dist/fake-github.js.map +1 -1
  56. package/dist/grade.js +90 -1
  57. package/dist/grade.js.map +1 -1
  58. package/dist/mechanism.test.js +420 -2
  59. package/dist/mechanism.test.js.map +1 -1
  60. package/dist/pr-context.js +162 -0
  61. package/dist/pr-context.js.map +1 -0
  62. package/dist/repo-config.js +141 -0
  63. package/dist/repo-config.js.map +1 -0
  64. package/dist/run-instance.js +134 -7
  65. package/dist/run-instance.js.map +1 -1
  66. package/dist/run.js +8 -0
  67. package/dist/run.js.map +1 -1
  68. package/dist/schema.js +0 -13
  69. package/dist/schema.js.map +1 -1
  70. package/dist/seed.js +7 -0
  71. package/dist/seed.js.map +1 -1
  72. package/package.json +4 -4
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "lastlight-evals",
3
- "version": "0.7.30",
3
+ "version": "0.9.0",
4
4
  "description": "Eval harness for Last Light workflows — drives the real production workflows against a mocked GitHub and grades deterministically (SWE-bench compatible).",
5
5
  "type": "module",
6
6
  "bin": {
@@ -31,10 +31,10 @@
31
31
  "@clack/prompts": "^1.7.0",
32
32
  "chalk": "^6.0.0",
33
33
  "yaml": "^2.5.0",
34
- "lastlight-core": "0.21.14"
34
+ "lastlight-core": "0.23.0"
35
35
  },
36
36
  "peerDependencies": {
37
- "agentic-pi": "^0.3.1"
37
+ "agentic-pi": "^0.4.0"
38
38
  },
39
39
  "devDependencies": {
40
40
  "@types/node": "^26.1.1",
@@ -42,7 +42,7 @@
42
42
  "typescript": "^5.7.0",
43
43
  "vitest": "^4.1.10",
44
44
  "wrangler": "^4.114.0",
45
- "agentic-pi": "0.3.1"
45
+ "agentic-pi": "0.4.0"
46
46
  },
47
47
  "scripts": {
48
48
  "build": "npm run build:harness && npm run build:dashboard",