@jinn-network/client 0.1.8 → 0.1.9-canary.159b38f5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (229) hide show
  1. package/dist/adapters/mech/adapter.d.ts +21 -1
  2. package/dist/adapters/mech/adapter.js +67 -10
  3. package/dist/adapters/mech/adapter.js.map +1 -1
  4. package/dist/adapters/mech/contracts.js +62 -28
  5. package/dist/adapters/mech/contracts.js.map +1 -1
  6. package/dist/adapters/mech/safe-revert.d.ts +4 -0
  7. package/dist/adapters/mech/safe-revert.js +5 -1
  8. package/dist/adapters/mech/safe-revert.js.map +1 -1
  9. package/dist/adapters/mech/safe.js +5 -1
  10. package/dist/adapters/mech/safe.js.map +1 -1
  11. package/dist/api/discovery-endpoint.d.ts +1 -0
  12. package/dist/api/discovery-endpoint.js +24 -0
  13. package/dist/api/discovery-endpoint.js.map +1 -1
  14. package/dist/api/fleet-build.d.ts +1 -7
  15. package/dist/api/fleet-build.js +0 -7
  16. package/dist/api/fleet-build.js.map +1 -1
  17. package/dist/api/gather-status.d.ts +8 -2
  18. package/dist/api/gather-status.js +29 -117
  19. package/dist/api/gather-status.js.map +1 -1
  20. package/dist/api/loop-completion-build.d.ts +79 -0
  21. package/dist/api/loop-completion-build.js +155 -0
  22. package/dist/api/loop-completion-build.js.map +1 -0
  23. package/dist/api/setup-endpoints.d.ts +13 -0
  24. package/dist/api/setup-endpoints.js +39 -20
  25. package/dist/api/setup-endpoints.js.map +1 -1
  26. package/dist/api/solvernets-endpoints.js +3 -0
  27. package/dist/api/solvernets-endpoints.js.map +1 -1
  28. package/dist/api/status-build.d.ts +43 -33
  29. package/dist/api/status-build.js +3 -26
  30. package/dist/api/status-build.js.map +1 -1
  31. package/dist/api/status-rollup-build.d.ts +0 -4
  32. package/dist/api/status-rollup-build.js +0 -4
  33. package/dist/api/status-rollup-build.js.map +1 -1
  34. package/dist/build-info.json +4 -4
  35. package/dist/build-meta.json +1 -1
  36. package/dist/cli/commands/codedigest-revert-check.js +6 -2
  37. package/dist/cli/commands/codedigest-revert-check.js.map +1 -1
  38. package/dist/cli/commands/doctor.d.ts +3 -0
  39. package/dist/cli/commands/doctor.js +35 -0
  40. package/dist/cli/commands/doctor.js.map +1 -1
  41. package/dist/cli/commands/eval.d.ts +76 -0
  42. package/dist/cli/commands/eval.js +401 -0
  43. package/dist/cli/commands/eval.js.map +1 -0
  44. package/dist/cli/commands/rewards.d.ts +2 -0
  45. package/dist/cli/commands/rewards.js +27 -0
  46. package/dist/cli/commands/rewards.js.map +1 -1
  47. package/dist/cli/commands/solver-nets.js +68 -0
  48. package/dist/cli/commands/solver-nets.js.map +1 -1
  49. package/dist/cli/commands/status.js +0 -1
  50. package/dist/cli/commands/status.js.map +1 -1
  51. package/dist/cli/index.js +2 -0
  52. package/dist/cli/index.js.map +1 -1
  53. package/dist/config.d.ts +31 -7
  54. package/dist/config.js +72 -7
  55. package/dist/config.js.map +1 -1
  56. package/dist/daemon/ai-units-gate.d.ts +6 -6
  57. package/dist/daemon/ai-units-gate.js +11 -10
  58. package/dist/daemon/ai-units-gate.js.map +1 -1
  59. package/dist/daemon/daemon.js +24 -22
  60. package/dist/daemon/daemon.js.map +1 -1
  61. package/dist/daemon/eviction-loop.d.ts +7 -0
  62. package/dist/daemon/eviction-loop.js +16 -0
  63. package/dist/daemon/eviction-loop.js.map +1 -1
  64. package/dist/dashboard/assets/index-BVAWkLwY.css +1 -0
  65. package/dist/dashboard/assets/index-DEPqtGOA.js +167 -0
  66. package/dist/dashboard/index.html +2 -2
  67. package/dist/discovery/http.d.ts +7 -0
  68. package/dist/discovery/http.js +241 -25
  69. package/dist/discovery/http.js.map +1 -1
  70. package/dist/discovery/onchain.js +155 -1
  71. package/dist/discovery/onchain.js.map +1 -1
  72. package/dist/discovery/types.d.ts +106 -0
  73. package/dist/discovery/types.js +40 -0
  74. package/dist/discovery/types.js.map +1 -1
  75. package/dist/discovery/with-fallback.js +14 -0
  76. package/dist/discovery/with-fallback.js.map +1 -1
  77. package/dist/earning/bootstrap.d.ts +23 -0
  78. package/dist/earning/bootstrap.js +76 -27
  79. package/dist/earning/bootstrap.js.map +1 -1
  80. package/dist/earning/faucet.d.ts +1 -1
  81. package/dist/earning/faucet.js +2 -2
  82. package/dist/earning/faucet.js.map +1 -1
  83. package/dist/earning/safe-adapter.js +11 -0
  84. package/dist/earning/safe-adapter.js.map +1 -1
  85. package/dist/eval/eval-harness-run.d.ts +63 -0
  86. package/dist/eval/eval-harness-run.js +123 -0
  87. package/dist/eval/eval-harness-run.js.map +1 -0
  88. package/dist/eval/orchestrator.d.ts +163 -0
  89. package/dist/eval/orchestrator.js +232 -0
  90. package/dist/eval/orchestrator.js.map +1 -0
  91. package/dist/eval/paired.d.ts +68 -0
  92. package/dist/eval/paired.js +93 -0
  93. package/dist/eval/paired.js.map +1 -0
  94. package/dist/eval/resolve-slate-tasks.d.ts +35 -0
  95. package/dist/eval/resolve-slate-tasks.js +56 -0
  96. package/dist/eval/resolve-slate-tasks.js.map +1 -0
  97. package/dist/eval/screen-discovery.d.ts +22 -0
  98. package/dist/eval/screen-discovery.js +71 -0
  99. package/dist/eval/screen-discovery.js.map +1 -0
  100. package/dist/eval/screen-progress.d.ts +41 -0
  101. package/dist/eval/screen-progress.js +60 -0
  102. package/dist/eval/screen-progress.js.map +1 -0
  103. package/dist/eval/screen-runner.d.ts +30 -0
  104. package/dist/eval/screen-runner.js +289 -0
  105. package/dist/eval/screen-runner.js.map +1 -0
  106. package/dist/eval/screen.d.ts +107 -0
  107. package/dist/eval/screen.js +159 -0
  108. package/dist/eval/screen.js.map +1 -0
  109. package/dist/eval/slope.d.ts +29 -0
  110. package/dist/eval/slope.js +46 -0
  111. package/dist/eval/slope.js.map +1 -0
  112. package/dist/eval/train-sequence.d.ts +35 -0
  113. package/dist/eval/train-sequence.js +59 -0
  114. package/dist/eval/train-sequence.js.map +1 -0
  115. package/dist/eval/wilson.d.ts +45 -0
  116. package/dist/eval/wilson.js +48 -0
  117. package/dist/eval/wilson.js.map +1 -0
  118. package/dist/harnesses/engine/canonical-json.js +5 -3
  119. package/dist/harnesses/engine/canonical-json.js.map +1 -1
  120. package/dist/harnesses/engine/engine.d.ts +9 -0
  121. package/dist/harnesses/engine/engine.js +50 -5
  122. package/dist/harnesses/engine/engine.js.map +1 -1
  123. package/dist/harnesses/engine/persistence.d.ts +17 -0
  124. package/dist/harnesses/engine/persistence.js +28 -0
  125. package/dist/harnesses/engine/persistence.js.map +1 -1
  126. package/dist/harnesses/impls/hermes-agent/adapter.d.ts +2 -0
  127. package/dist/harnesses/impls/hermes-agent/adapter.js +8 -5
  128. package/dist/harnesses/impls/hermes-agent/adapter.js.map +1 -1
  129. package/dist/harnesses/impls/hermes-agent/bootstrap.d.ts +1 -0
  130. package/dist/harnesses/impls/hermes-agent/bootstrap.js +6 -1
  131. package/dist/harnesses/impls/hermes-agent/bootstrap.js.map +1 -1
  132. package/dist/harnesses/impls/hermes-agent/harness.d.ts +17 -3
  133. package/dist/harnesses/impls/hermes-agent/harness.js +68 -5
  134. package/dist/harnesses/impls/hermes-agent/harness.js.map +1 -1
  135. package/dist/harnesses/impls/index.d.ts +2 -0
  136. package/dist/harnesses/impls/index.js +2 -0
  137. package/dist/harnesses/impls/index.js.map +1 -1
  138. package/dist/harnesses/impls/learner/adapters/claude-code.js +5 -0
  139. package/dist/harnesses/impls/learner/adapters/claude-code.js.map +1 -1
  140. package/dist/harnesses/impls/learner/harness.d.ts +17 -1
  141. package/dist/harnesses/impls/learner/harness.js +51 -1
  142. package/dist/harnesses/impls/learner/harness.js.map +1 -1
  143. package/dist/harnesses/impls/learner/harvest.d.ts +2 -0
  144. package/dist/harnesses/impls/learner/harvest.js +7 -1
  145. package/dist/harnesses/impls/learner/harvest.js.map +1 -1
  146. package/dist/harnesses/impls/learner/plugin-path.js +1 -0
  147. package/dist/harnesses/impls/learner/plugin-path.js.map +1 -1
  148. package/dist/harnesses/impls/swe-rebench-v2-evaluator/harness.js +3 -1
  149. package/dist/harnesses/impls/swe-rebench-v2-evaluator/harness.js.map +1 -1
  150. package/dist/harnesses/impls/swe-rebench-v2-evaluator/index.d.ts +2 -2
  151. package/dist/harnesses/impls/swe-rebench-v2-evaluator/index.js +3 -1
  152. package/dist/harnesses/impls/swe-rebench-v2-evaluator/index.js.map +1 -1
  153. package/dist/harnesses/types.d.ts +14 -0
  154. package/dist/learner/revert-decision.d.ts +16 -1
  155. package/dist/learner/revert-decision.js +38 -18
  156. package/dist/learner/revert-decision.js.map +1 -1
  157. package/dist/learner/revert-stats.d.ts +14 -0
  158. package/dist/learner/revert-stats.js +42 -0
  159. package/dist/learner/revert-stats.js.map +1 -1
  160. package/dist/local-provider-url.d.ts +3 -0
  161. package/dist/local-provider-url.js +28 -0
  162. package/dist/local-provider-url.js.map +1 -0
  163. package/dist/main.js +22 -0
  164. package/dist/main.js.map +1 -1
  165. package/dist/plugins/learner/.claude-plugin/plugin.json +1 -1
  166. package/dist/plugins/learner/.codex-plugin/plugin.json +1 -1
  167. package/dist/plugins/learner/hooks/session-start +30 -1
  168. package/dist/plugins/learner/skills/learn/consolidator-prompt.md +4 -0
  169. package/dist/preflight/deployment-readiness.d.ts +147 -0
  170. package/dist/preflight/deployment-readiness.js +366 -0
  171. package/dist/preflight/deployment-readiness.js.map +1 -0
  172. package/dist/preflight/pidfile-liveness.d.ts +7 -1
  173. package/dist/preflight/pidfile-liveness.js +14 -0
  174. package/dist/preflight/pidfile-liveness.js.map +1 -1
  175. package/dist/rpc/transport.d.ts +36 -0
  176. package/dist/rpc/transport.js +123 -24
  177. package/dist/rpc/transport.js.map +1 -1
  178. package/dist/scripts/swe-rebench-v2-seed-pool.json +2 -1
  179. package/dist/solver-types/_swe-rebench-v2-held-out-slate.d.ts +76 -0
  180. package/dist/solver-types/_swe-rebench-v2-held-out-slate.js +156 -0
  181. package/dist/solver-types/_swe-rebench-v2-held-out-slate.js.map +1 -0
  182. package/dist/solver-types/_swe-rebench-v2-pool-recovery.d.ts +81 -0
  183. package/dist/solver-types/_swe-rebench-v2-pool-recovery.js +116 -0
  184. package/dist/solver-types/_swe-rebench-v2-pool-recovery.js.map +1 -0
  185. package/dist/solver-types/_swe-rebench-v2-state.d.ts +9 -0
  186. package/dist/solver-types/_swe-rebench-v2-state.js +14 -0
  187. package/dist/solver-types/_swe-rebench-v2-state.js.map +1 -1
  188. package/dist/solver-types/slates/held-out-slate.swe-rebench-v2.v1.json +20 -0
  189. package/dist/solver-types/slates/held-out-slate.swe-rebench-v2.v2.json +19 -0
  190. package/dist/solver-types/slates/held-out-slate.swe-rebench-v2.v2.screening-report.json +628 -0
  191. package/dist/solver-types/solver-type.d.ts +8 -0
  192. package/dist/solver-types/swe-rebench-v2.d.ts +2 -0
  193. package/dist/solver-types/swe-rebench-v2.js +106 -2
  194. package/dist/solver-types/swe-rebench-v2.js.map +1 -1
  195. package/dist/solvernets/launched-record-dispatcher.d.ts +3 -0
  196. package/dist/solvernets/launched-record-dispatcher.js.map +1 -1
  197. package/dist/solvernets/registry-client-erc8004.js +29 -37
  198. package/dist/solvernets/registry-client-erc8004.js.map +1 -1
  199. package/dist/solvernets/registry-client.d.ts +6 -0
  200. package/dist/solvernets/store.js +7 -2
  201. package/dist/solvernets/store.js.map +1 -1
  202. package/dist/spend/ai-units-config.d.ts +10 -0
  203. package/dist/spend/ai-units-config.js +7 -1
  204. package/dist/spend/ai-units-config.js.map +1 -1
  205. package/dist/spend/ai-units.d.ts +51 -0
  206. package/dist/spend/ai-units.js +73 -0
  207. package/dist/spend/ai-units.js.map +1 -1
  208. package/dist/spend/record.js +12 -5
  209. package/dist/spend/record.js.map +1 -1
  210. package/dist/store/store.d.ts +91 -5
  211. package/dist/store/store.js +170 -7
  212. package/dist/store/store.js.map +1 -1
  213. package/dist/vendor/@jinn-network/sdk/dist/payloads/swe-rebench-v2.d.ts +108 -1
  214. package/dist/vendor/@jinn-network/sdk/dist/payloads/swe-rebench-v2.js +25 -1
  215. package/dist/vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2-held-out-slate.d.ts +65 -0
  216. package/dist/vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2-held-out-slate.js +123 -0
  217. package/dist/vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.d.ts +2 -2
  218. package/dist/vendor/@jinn-network/sdk/dist/solvernets/swe-rebench-v2.js +1 -1
  219. package/dist/vendor/@jinn-network/sdk/package.json +4 -0
  220. package/docker-compose.yml +3 -2
  221. package/package.json +22 -18
  222. package/plugins/learner/.claude-plugin/plugin.json +1 -1
  223. package/plugins/learner/.codex-plugin/plugin.json +1 -1
  224. package/plugins/learner/hooks/session-start +30 -1
  225. package/plugins/learner/skills/learn/consolidator-prompt.md +4 -0
  226. package/plugins/swe-rebench-v2-runtime/hooks/hooks.json +16 -0
  227. package/plugins/swe-rebench-v2-runtime/hooks/session-start +74 -0
  228. package/dist/dashboard/assets/index-CzKxvMcU.css +0 -32
  229. package/dist/dashboard/assets/index-yVemxHot.js +0 -351
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Fresh-volume vetted-pool recovery for the swe-rebench-v2 generator (#957).
3
+ *
4
+ * A freshly-deployed operator has an empty disk: no `validated-pool.json`, so
5
+ * the generator's admission gate hits `admission-required-no-data` and posts 0
6
+ * tasks. The vetted pool was previously published to IPFS; we recover it via the
7
+ * indexer (DR-2026-06-02 open-question #4, Option B — read the artifact ref via
8
+ * the DiscoveryAPI, not on-chain directly):
9
+ *
10
+ * 1. `discoveryApi.getMostRecentTaskCidDigest(manifestCid)` → a recent task's
11
+ * on-chain `taskCidDigest` (pure indexer/chain read; no IPFS in discovery).
12
+ * 2. Reconstruct the IPFS task CID (`f01551220 + digestHexWithout0x`, the same
13
+ * derivation as `adapters/mech/adapter.ts`) and fetch the task body.
14
+ * 3. Read the vetted-pool ref from the task's `eligibility.vettedPoolRef`
15
+ * (`VETTED_POOL_REF_ELIGIBILITY_KEY`).
16
+ * 4. Fetch + parse the artifact at `ref.artifactCid`.
17
+ * 5. SECURITY: verify `hashVettedPoolArtifact(artifact) === ref.artifactHash`.
18
+ * A mismatch is REJECTED — no file is written.
19
+ * 6. Write the scorable entries into `validated-pool.json` via
20
+ * `ValidatedPoolStore.record`. After a successful recovery the generator's
21
+ * existing `resolvePublishedVettedPool` / pool-load path picks up the now-
22
+ * present `validated-pool.json`.
23
+ *
24
+ * Never-overwrite: if a local validated pool already exists for the current
25
+ * `EVAL_SEMANTICS_VERSION`, recovery is a no-op (`local-pool-present`) — the same
26
+ * "don't clobber newer local data" stance as `resolvePublishedVettedPool`'s
27
+ * `validatedNewer` guard.
28
+ *
29
+ * Fail-closed: every external call is wrapped so any failure returns a structured
30
+ * `{ recovered: false, reason }`. Recovery NEVER throws — it must fall through to
31
+ * the existing `admission-required-no-data` path so a recovery failure can never
32
+ * crash the generator tick.
33
+ */
34
+ import { EVAL_SEMANTICS_VERSION, vettedPoolArtifactRefFromEligibility, parseVettedPoolArtifact, hashVettedPoolArtifact, } from './_swe-rebench-v2-validated-pool.js';
35
+ /**
36
+ * Whether a recovery outcome is TERMINAL — i.e. retrying cannot help, so the
37
+ * caller should latch permanently and never attempt recovery again:
38
+ *
39
+ * - `recovered === true` — done; the pool was written.
40
+ * - `local-pool-present` — a local pool already exists; nothing to recover.
41
+ *
42
+ * All other outcomes (`no-task`, `no-ref`, `error:*`, `hash-mismatch`) are
43
+ * TRANSIENT/retryable, so the caller must NOT latch on them (it retries under a
44
+ * bounded cap instead):
45
+ * - `no-task` — a fresh SolverNet may not have posted its first task yet.
46
+ * - `no-ref` — the task's eligibility may not yet carry a vettedPoolRef.
47
+ * - `error:*` — the indexer / IPFS gateway may be having a transient blip.
48
+ * - `hash-mismatch` — the fetched artifact's hash didn't match the ref. This is
49
+ * SECURITY-rejected at write time (the bad artifact is NEVER written), and it
50
+ * can be transient: a corrupted IPFS-gateway response may fetch cleanly on a
51
+ * later attempt, or the launcher may re-publish a corrected ref that a later
52
+ * task carries. Retrying re-resolves the most-recent task's ref AND re-fetches
53
+ * from IPFS, so a corrected pool is picked up without a restart; the bounded
54
+ * retry cap prevents looping on a genuinely-bad ref.
55
+ */
56
+ export function isTerminalRecoveryOutcome(result) {
57
+ if (result.recovered)
58
+ return true;
59
+ return result.reason === 'local-pool-present';
60
+ }
61
+ export async function recoverVettedPoolFromNetwork(deps) {
62
+ const { stateDir: _stateDir, manifestCid, discoveryApi, ipfsGatewayUrl, store, fetchFromIpfs } = deps;
63
+ void _stateDir; // reserved for symmetry with resolvePublishedVettedPool; store already binds stateDir
64
+ try {
65
+ // Never-overwrite guard: a local validated pool already present (operator
66
+ // validated locally, or a prior recovery ran) → do not clobber it.
67
+ const existing = await store.getScorableIds(EVAL_SEMANTICS_VERSION);
68
+ if (existing !== null) {
69
+ return { recovered: false, reason: 'local-pool-present' };
70
+ }
71
+ // 1) Most-recent task digest for the SolverNet (pure indexer/chain read).
72
+ const recent = await discoveryApi.getMostRecentTaskCidDigest(manifestCid);
73
+ if (!recent)
74
+ return { recovered: false, reason: 'no-task' };
75
+ // 2) Reconstruct the IPFS task CID and fetch the task body. Lowercase the
76
+ // digest defensively (NIT #7): the on-chain bytes32 is lowercase, but a
77
+ // future uppercase-emitting indexer must not produce a wrong CID.
78
+ const taskCid = `f01551220${recent.taskCidDigest.replace(/^0x/, '').toLowerCase()}`;
79
+ const taskDoc = await fetchFromIpfs(ipfsGatewayUrl, taskCid);
80
+ // 3) Read the vetted-pool ref from the task's eligibility.
81
+ const eligibility = taskDoc?.eligibility;
82
+ const ref = vettedPoolArtifactRefFromEligibility(eligibility);
83
+ if (ref === null)
84
+ return { recovered: false, reason: 'no-ref' };
85
+ // 4) Fetch + parse the artifact.
86
+ const rawArtifact = await fetchFromIpfs(ipfsGatewayUrl, ref.artifactCid);
87
+ const artifact = parseVettedPoolArtifact(rawArtifact);
88
+ // 5) SECURITY: verify the artifact hash against the ref before writing.
89
+ if (hashVettedPoolArtifact(artifact) !== ref.artifactHash) {
90
+ return { recovered: false, reason: 'hash-mismatch' };
91
+ }
92
+ // 6) Write the scorable entries into validated-pool.json. Record under the
93
+ // artifact's evalSemanticsVersion so the generator's read path picks them up
94
+ // for the matching version. Each artifact entry is scorable by construction.
95
+ for (const entry of artifact.entries) {
96
+ const poolEntry = {
97
+ scorable: true,
98
+ reason: entry.reason,
99
+ checkedAt: entry.checkedAt,
100
+ ...(entry.rowHash ? { rowHash: entry.rowHash } : {}),
101
+ ...(entry.imageName ? { imageName: entry.imageName } : {}),
102
+ ...(entry.imageDigest ? { imageDigest: entry.imageDigest } : {}),
103
+ ...(entry.upstreamEvalCommit ? { upstreamEvalCommit: entry.upstreamEvalCommit } : {}),
104
+ };
105
+ await store.record(entry.instance_id, poolEntry, artifact.evalSemanticsVersion);
106
+ }
107
+ return { recovered: true, entriesRecovered: artifact.entries.length };
108
+ }
109
+ catch (err) {
110
+ // Fail-closed: any failure (discovery, IPFS, parse) must NOT throw — return a
111
+ // structured result so the generator falls through to admission-required-no-data.
112
+ const message = err instanceof Error ? err.message : String(err);
113
+ return { recovered: false, reason: `error:${message}` };
114
+ }
115
+ }
116
+ //# sourceMappingURL=_swe-rebench-v2-pool-recovery.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"_swe-rebench-v2-pool-recovery.js","sourceRoot":"","sources":["../../src/solver-types/_swe-rebench-v2-pool-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAEL,sBAAsB,EACtB,oCAAoC,EACpC,uBAAuB,EACvB,sBAAsB,GAEvB,MAAM,qCAAqC,CAAC;AAkB7C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,yBAAyB,CAAC,MAA+B;IACvE,IAAI,MAAM,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAClC,OAAO,MAAM,CAAC,MAAM,KAAK,oBAAoB,CAAC;AAChD,CAAC;AAYD,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,IAA2B;IAE3B,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,GAAG,IAAI,CAAC;IACtG,KAAK,SAAS,CAAC,CAAC,sFAAsF;IAEtG,IAAI,CAAC;QACH,0EAA0E;QAC1E,mEAAmE;QACnE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,cAAc,CAAC,sBAAsB,CAAC,CAAC;QACpE,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACtB,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC;QAC5D,CAAC;QAED,0EAA0E;QAC1E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,0BAA0B,CAAC,WAAW,CAAC,CAAC;QAC1E,IAAI,CAAC,MAAM;YAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;QAE5D,0EAA0E;QAC1E,wEAAwE;QACxE,kEAAkE;QAClE,MAAM,OAAO,GAAG,YAAY,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;QACpF,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;QAE7D,2DAA2D;QAC3D,MAAM,WAAW,GAAI,OAA4D,EAAE,WAAW,CAAC;QAC/F,MAAM,GAAG,GAAG,oCAAoC,CAAC,WAAW,CAAC,CAAC;QAC9D,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;QAEhE,iCAAiC;QACjC,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACzE,MAAM,QAAQ,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAC;QAEtD,wEAAwE;QACxE,IAAI,sBAAsB,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,YAAY,EAAE,CAAC;YAC1D,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;QACvD,CAAC;QAED,2EAA2E;QAC3E,6EAA6E;QAC7E,6EAA6E;QAC7E,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACrC,MAAM,SAAS,GAAuB;gBACpC,QAAQ,EAAE,IAAI;gBACd,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACpD,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChE,GAAG,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,KAAK,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtF,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,EAAE,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAClF,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IACxE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,8EAA8E;QAC9E,kFAAkF;QAClF,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,OAAO,EAAE,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC"}
@@ -33,6 +33,15 @@ export declare class GeneratorStateStore {
33
33
  invalidate(): void;
34
34
  private save;
35
35
  getCounters(instance_id: string): Promise<TaskCounters>;
36
+ /**
37
+ * Instance ids this generator has posted at least once — i.e. that entered
38
+ * the SolverNet train stream. Used by held-out screening (#986) to keep the
39
+ * exam to the never-posted remainder (a posted task may have been trained on,
40
+ * so holding it out later would make a trained-checkpoint pass memorization,
41
+ * not generalization). This is THIS launcher's posting history; complete for a
42
+ * single-launcher SolverNet.
43
+ */
44
+ postedInstanceIds(): Promise<Set<string>>;
36
45
  recordPosted(instance_id: string, now?: number): Promise<void>;
37
46
  recordSuccess(instance_id: string): Promise<void>;
38
47
  recordLastTaskId(instance_id: string, taskId: string): Promise<void>;
@@ -47,6 +47,20 @@ export class GeneratorStateStore {
47
47
  const state = await this.load();
48
48
  return state.tasks[instance_id] ?? { posted: 0, successful: 0, last_posted_at: 0 };
49
49
  }
50
+ /**
51
+ * Instance ids this generator has posted at least once — i.e. that entered
52
+ * the SolverNet train stream. Used by held-out screening (#986) to keep the
53
+ * exam to the never-posted remainder (a posted task may have been trained on,
54
+ * so holding it out later would make a trained-checkpoint pass memorization,
55
+ * not generalization). This is THIS launcher's posting history; complete for a
56
+ * single-launcher SolverNet.
57
+ */
58
+ async postedInstanceIds() {
59
+ const state = await this.load();
60
+ return new Set(Object.entries(state.tasks)
61
+ .filter(([, c]) => c.posted > 0)
62
+ .map(([id]) => id));
63
+ }
50
64
  async recordPosted(instance_id, now = Date.now()) {
51
65
  const state = await this.load();
52
66
  const c = state.tasks[instance_id] ?? { posted: 0, successful: 0, last_posted_at: 0 };
@@ -1 +1 @@
1
- {"version":3,"file":"_swe-rebench-v2-state.js","sourceRoot":"","sources":["../../src/solver-types/_swe-rebench-v2-state.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiBjC,MAAM,OAAO,mBAAmB;IACtB,SAAS,CAAS;IAClB,KAAK,GAAqB,IAAI,CAAC;IAEvC,YAAY,IAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAC/D,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,GAAG,EAAE,aAAa,EAAE,mCAAmC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC,KAAM,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,WAAmB;QACnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,WAAmB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,cAAc,GAAG,GAAG,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;QAClB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,WAAmB,EAAE,MAAc;QACxD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CACF"}
1
+ {"version":3,"file":"_swe-rebench-v2-state.js","sourceRoot":"","sources":["../../src/solver-types/_swe-rebench-v2-state.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAiBjC,MAAM,OAAO,mBAAmB;IACtB,SAAS,CAAS;IAClB,KAAK,GAAqB,IAAI,CAAC;IAEvC,YAAY,IAA0B;QACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC,CAAC;IAC/D,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YACnD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,KAAK,GAAG,EAAE,aAAa,EAAE,mCAAmC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;QACjF,CAAC;QACD,OAAO,IAAI,CAAC,KAAM,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACH,UAAU;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,WAAmB;QACnC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,iBAAiB;QACrB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,OAAO,IAAI,GAAG,CACZ,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC;aACxB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CACrB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,WAAmB,EAAE,MAAc,IAAI,CAAC,GAAG,EAAE;QAC9D,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;QACd,CAAC,CAAC,cAAc,GAAG,GAAG,CAAC;QACvB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,WAAmB;QACrC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;QAClB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,WAAmB,EAAE,MAAc;QACxD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC,YAAY,GAAG,MAAM,CAAC;QACxB,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;IACpB,CAAC;CACF"}
@@ -0,0 +1,20 @@
1
+ {
2
+ "comment": "HELD-OUT EVAL SLATE (issue #817). These swe-rebench-v2 instance_ids are RESERVED out of the training pool: the generator excludes them so they never enter the train stream, and the eval orchestrator scores solvers against them out-of-sample. Each id is gold-patch-VALIDATED (scorable=true / gold-patch-resolves) at evalSemanticsVersion 3, per the validation gate `validatePoolInstances` / `filterToScorablePool` in _swe-rebench-v2-validated-pool.ts — the same standard the train-stream generator enforces. They are NOT raw-sampled from a candidate seed pool: a raw sample admits ungradeable instances (gold patch does not resolve in the eval env), making the exam denominator measure noise. The slate's gradeability is bound to evalSemanticsVersion 3; an eval-semantics bump invalidates these validations and requires re-validating against the new semantics and cutting a distinct slate version (v2, ...). Content-addressed: `hash` is sha256 over the canonical, instanceIds-sorted artifact (see _swe-rebench-v2-held-out-slate.ts). Scores are only comparable WITHIN this version; a slate change is a distinct version, never an in-place edit.",
3
+ "schemaVersion": "held-out-slate.v1",
4
+ "solverType": "swe-rebench-v2.v1",
5
+ "version": "v1",
6
+ "generatedAt": "2026-05-29T00:00:00.000Z",
7
+ "hash": "sha256:2b029de15e271d5d2de35fe6477af98aef9fdc46f357e59139179edab1a42b15",
8
+ "instanceIds": [
9
+ "ASPP__pelita-863",
10
+ "ASPP__pelita-875",
11
+ "AbsaOSS__generate-release-notes-207",
12
+ "All-Hands-AI__OpenHands-11914",
13
+ "BQSKit__bqskit-337",
14
+ "BerriAI__litellm-14715",
15
+ "BerriAI__litellm-15753",
16
+ "BrianPugh__cyclopts-609",
17
+ "carsdotcom__skelebot-280",
18
+ "pandas-dev__pandas-60736"
19
+ ]
20
+ }
@@ -0,0 +1,19 @@
1
+ {
2
+ "comment": "BASELINE-FAILURE REGRESSION BENCHMARK (issue #986). Screened: gradeable at the current evalSemanticsVersion AND base claude-code/Haiku frozen fails 0/R (R≥3) AND a stronger Codex/GPT-5.5 prover passes ≥1 (proven headroom). Baseline 0% by construction. Held out from the generator train stream via the active-slate-version union. Content-addressed; scores comparable WITHIN this version only.",
3
+ "schemaVersion": "held-out-slate.v1",
4
+ "solverType": "swe-rebench-v2.v1",
5
+ "version": "v2",
6
+ "generatedAt": "2026-06-04T23:03:42.179Z",
7
+ "instanceIds": [
8
+ "agronholm__anyio-1134",
9
+ "astropy__astropy-19438",
10
+ "con__nwb2bids-347_interface",
11
+ "deepset-ai__haystack-9322",
12
+ "gidgethub__gidgethub-225",
13
+ "iris-hep__func_adl-199",
14
+ "MatterMiners__tardis-361",
15
+ "modelcontextprotocol__python-sdk-222",
16
+ "twisted__towncrier-687"
17
+ ],
18
+ "hash": "sha256:34bb8bfa5e85933e38706551d4afaec3adf4a6ad1c1c0e4d2132f2c47d43d039"
19
+ }