agent-hitch 0.2.6 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (562) hide show
  1. package/README.md +30 -3
  2. package/README.zh-CN.md +14 -2
  3. package/dist/bin/hitch.js +25 -3
  4. package/dist/bin/hitch.js.map +1 -1
  5. package/dist/scripts/canary-benchmark.js +54 -0
  6. package/dist/scripts/canary-benchmark.js.map +1 -0
  7. package/dist/scripts/canary-buildkit-secrets.js +166 -0
  8. package/dist/scripts/canary-buildkit-secrets.js.map +1 -0
  9. package/dist/scripts/canary-collision-domains.js +77 -0
  10. package/dist/scripts/canary-collision-domains.js.map +1 -0
  11. package/dist/scripts/canary-docker-image-observation.js +181 -0
  12. package/dist/scripts/canary-docker-image-observation.js.map +1 -0
  13. package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
  14. package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
  15. package/dist/scripts/canary-gpu-hardware.js +95 -0
  16. package/dist/scripts/canary-gpu-hardware.js.map +1 -0
  17. package/dist/scripts/canary-harbor-load.js +365 -0
  18. package/dist/scripts/canary-harbor-load.js.map +1 -0
  19. package/dist/scripts/canary-packaged-harness.js +75 -0
  20. package/dist/scripts/canary-packaged-harness.js.map +1 -0
  21. package/dist/scripts/canary-resource-load.js +188 -0
  22. package/dist/scripts/canary-resource-load.js.map +1 -0
  23. package/dist/scripts/check-architecture.js +10 -5
  24. package/dist/scripts/check-architecture.js.map +1 -1
  25. package/dist/scripts/docker-reaper-crash-child.js +21 -0
  26. package/dist/scripts/docker-reaper-crash-child.js.map +1 -0
  27. package/dist/src/adapters/catalog.js +3 -1
  28. package/dist/src/adapters/catalog.js.map +1 -1
  29. package/dist/src/adapters/discovery.js +1 -1
  30. package/dist/src/adapters/discovery.js.map +1 -1
  31. package/dist/src/adapters/providers/claude.js +6 -0
  32. package/dist/src/adapters/providers/claude.js.map +1 -1
  33. package/dist/src/adapters/providers/codex-auth.js +29 -0
  34. package/dist/src/adapters/providers/codex-auth.js.map +1 -0
  35. package/dist/src/adapters/providers/codex.js +29 -6
  36. package/dist/src/adapters/providers/codex.js.map +1 -1
  37. package/dist/src/adapters/providers/deepseek.js +6 -0
  38. package/dist/src/adapters/providers/deepseek.js.map +1 -1
  39. package/dist/src/adapters/providers/model-call.js +26 -0
  40. package/dist/src/adapters/providers/model-call.js.map +1 -0
  41. package/dist/src/adapters/providers/opencode.js +6 -0
  42. package/dist/src/adapters/providers/opencode.js.map +1 -1
  43. package/dist/src/adapters/providers/pi.js +6 -0
  44. package/dist/src/adapters/providers/pi.js.map +1 -1
  45. package/dist/src/artifacts/index.js +1 -0
  46. package/dist/src/artifacts/index.js.map +1 -1
  47. package/dist/src/artifacts/preparer.js +7 -2
  48. package/dist/src/artifacts/preparer.js.map +1 -1
  49. package/dist/src/backends/harbor/agent-budget.js +31 -0
  50. package/dist/src/backends/harbor/agent-budget.js.map +1 -0
  51. package/dist/src/backends/harbor/backend.js +59 -107
  52. package/dist/src/backends/harbor/backend.js.map +1 -1
  53. package/dist/src/backends/harbor/environment-config.js +92 -0
  54. package/dist/src/backends/harbor/environment-config.js.map +1 -0
  55. package/dist/src/backends/harbor/index.js +6 -1
  56. package/dist/src/backends/harbor/index.js.map +1 -1
  57. package/dist/src/backends/harbor/model-proxy-config.js +23 -0
  58. package/dist/src/backends/harbor/model-proxy-config.js.map +1 -0
  59. package/dist/src/backends/harbor/process.js +177 -0
  60. package/dist/src/backends/harbor/process.js.map +1 -0
  61. package/dist/src/backends/harbor/regrade.js +77 -0
  62. package/dist/src/backends/harbor/regrade.js.map +1 -0
  63. package/dist/src/backends/harbor/runtime-toolchain.js +30 -0
  64. package/dist/src/backends/harbor/runtime-toolchain.js.map +1 -0
  65. package/dist/src/backends/harbor/supervisor.js +74 -0
  66. package/dist/src/backends/harbor/supervisor.js.map +1 -0
  67. package/dist/src/backends/harbor/task-resources.js +206 -0
  68. package/dist/src/backends/harbor/task-resources.js.map +1 -0
  69. package/dist/src/backends/harbor/tools.js +6 -29
  70. package/dist/src/backends/harbor/tools.js.map +1 -1
  71. package/dist/src/backends/index.js +5 -1
  72. package/dist/src/backends/index.js.map +1 -1
  73. package/dist/src/benchmarks/index.js +3 -0
  74. package/dist/src/benchmarks/index.js.map +1 -0
  75. package/dist/src/benchmarks/loader.js +271 -0
  76. package/dist/src/benchmarks/loader.js.map +1 -0
  77. package/dist/src/benchmarks/metrics.js +16 -0
  78. package/dist/src/benchmarks/metrics.js.map +1 -0
  79. package/dist/src/benchmarks/toml.js +13 -0
  80. package/dist/src/benchmarks/toml.js.map +1 -0
  81. package/dist/src/benchmarks/validation.js +242 -0
  82. package/dist/src/benchmarks/validation.js.map +1 -0
  83. package/dist/src/cli/arguments.js +6 -4
  84. package/dist/src/cli/arguments.js.map +1 -1
  85. package/dist/src/cli/commands/benchmark.js +22 -0
  86. package/dist/src/cli/commands/benchmark.js.map +1 -0
  87. package/dist/src/cli/commands/capabilities.js +18 -0
  88. package/dist/src/cli/commands/capabilities.js.map +1 -0
  89. package/dist/src/cli/commands/daemon.js +135 -8
  90. package/dist/src/cli/commands/daemon.js.map +1 -1
  91. package/dist/src/cli/commands/eval.js +228 -6
  92. package/dist/src/cli/commands/eval.js.map +1 -1
  93. package/dist/src/cli/commands/images.js +40 -0
  94. package/dist/src/cli/commands/images.js.map +1 -0
  95. package/dist/src/cli/commands/run.js +42 -17
  96. package/dist/src/cli/commands/run.js.map +1 -1
  97. package/dist/src/cli/commands/runs.js +25 -3
  98. package/dist/src/cli/commands/runs.js.map +1 -1
  99. package/dist/src/cli/commands/trajectory.js +96 -5
  100. package/dist/src/cli/commands/trajectory.js.map +1 -1
  101. package/dist/src/cli/commands/verifier.js +36 -0
  102. package/dist/src/cli/commands/verifier.js.map +1 -0
  103. package/dist/src/cli/commands/worker.js +79 -0
  104. package/dist/src/cli/commands/worker.js.map +1 -0
  105. package/dist/src/cli/main.js +10 -0
  106. package/dist/src/cli/main.js.map +1 -1
  107. package/dist/src/cli/output.js +84 -4
  108. package/dist/src/cli/output.js.map +1 -1
  109. package/dist/src/control-plane/build-admission.js +69 -0
  110. package/dist/src/control-plane/build-admission.js.map +1 -0
  111. package/dist/src/control-plane/build-records.js +6 -0
  112. package/dist/src/control-plane/build-records.js.map +1 -0
  113. package/dist/src/control-plane/collisions.js +40 -0
  114. package/dist/src/control-plane/collisions.js.map +1 -0
  115. package/dist/src/control-plane/eval-control-state.js +22 -0
  116. package/dist/src/control-plane/eval-control-state.js.map +1 -0
  117. package/dist/src/control-plane/eval-control-work.js +44 -0
  118. package/dist/src/control-plane/eval-control-work.js.map +1 -0
  119. package/dist/src/control-plane/eval-image-resolution.js +52 -0
  120. package/dist/src/control-plane/eval-image-resolution.js.map +1 -0
  121. package/dist/src/control-plane/eval-image-services.js +30 -0
  122. package/dist/src/control-plane/eval-image-services.js.map +1 -0
  123. package/dist/src/control-plane/eval-records.js +252 -0
  124. package/dist/src/control-plane/eval-records.js.map +1 -0
  125. package/dist/src/control-plane/eval-recovery.js +181 -0
  126. package/dist/src/control-plane/eval-recovery.js.map +1 -0
  127. package/dist/src/control-plane/eval-scheduler.js +450 -0
  128. package/dist/src/control-plane/eval-scheduler.js.map +1 -0
  129. package/dist/src/control-plane/index.js +20 -0
  130. package/dist/src/control-plane/index.js.map +1 -0
  131. package/dist/src/control-plane/local-worker.js +49 -0
  132. package/dist/src/control-plane/local-worker.js.map +1 -0
  133. package/dist/src/control-plane/model-capture-planning.js +12 -0
  134. package/dist/src/control-plane/model-capture-planning.js.map +1 -0
  135. package/dist/src/control-plane/remote-result-transport.js +175 -0
  136. package/dist/src/control-plane/remote-result-transport.js.map +1 -0
  137. package/dist/src/control-plane/remote-work-coordinator.js +333 -0
  138. package/dist/src/control-plane/remote-work-coordinator.js.map +1 -0
  139. package/dist/src/control-plane/remote-work-inputs.js +266 -0
  140. package/dist/src/control-plane/remote-work-inputs.js.map +1 -0
  141. package/dist/src/control-plane/remote-work-item.js +62 -0
  142. package/dist/src/control-plane/remote-work-item.js.map +1 -0
  143. package/dist/src/control-plane/remote-work-recovery.js +220 -0
  144. package/dist/src/control-plane/remote-work-recovery.js.map +1 -0
  145. package/dist/src/control-plane/remote-worker-artifacts.js +135 -0
  146. package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -0
  147. package/dist/src/control-plane/remote-worker-client.js +202 -0
  148. package/dist/src/control-plane/remote-worker-client.js.map +1 -0
  149. package/dist/src/control-plane/remote-worker-configuration.js +34 -0
  150. package/dist/src/control-plane/remote-worker-configuration.js.map +1 -0
  151. package/dist/src/control-plane/remote-worker-credentials.js +58 -0
  152. package/dist/src/control-plane/remote-worker-credentials.js.map +1 -0
  153. package/dist/src/control-plane/remote-worker-protocol.js +489 -0
  154. package/dist/src/control-plane/remote-worker-protocol.js.map +1 -0
  155. package/dist/src/control-plane/remote-worker-runner.js +214 -0
  156. package/dist/src/control-plane/remote-worker-runner.js.map +1 -0
  157. package/dist/src/control-plane/remote-workers.js +377 -0
  158. package/dist/src/control-plane/remote-workers.js.map +1 -0
  159. package/dist/src/control-plane/rerun-scheduler.js +448 -0
  160. package/dist/src/control-plane/rerun-scheduler.js.map +1 -0
  161. package/dist/src/control-plane/rerun-submission.js +62 -0
  162. package/dist/src/control-plane/rerun-submission.js.map +1 -0
  163. package/dist/src/control-plane/resources.js +182 -0
  164. package/dist/src/control-plane/resources.js.map +1 -0
  165. package/dist/src/control-plane/scheduler-eval-entry.js +32 -0
  166. package/dist/src/control-plane/scheduler-eval-entry.js.map +1 -0
  167. package/dist/src/control-plane/synthetic-result.js +21 -0
  168. package/dist/src/control-plane/synthetic-result.js.map +1 -0
  169. package/dist/src/control-plane/work-admission.js +23 -0
  170. package/dist/src/control-plane/work-admission.js.map +1 -0
  171. package/dist/src/control-plane/work-dispatcher.js +237 -0
  172. package/dist/src/control-plane/work-dispatcher.js.map +1 -0
  173. package/dist/src/controller-runtime/hash.js +19 -3
  174. package/dist/src/controller-runtime/hash.js.map +1 -1
  175. package/dist/src/controller-runtime/index.js +1 -1
  176. package/dist/src/controller-runtime/index.js.map +1 -1
  177. package/dist/src/controller-runtime/store.js +15 -6
  178. package/dist/src/controller-runtime/store.js.map +1 -1
  179. package/dist/src/daemon/auth.js +5 -0
  180. package/dist/src/daemon/auth.js.map +1 -1
  181. package/dist/src/daemon/client.js.map +1 -1
  182. package/dist/src/daemon/event-stream.js +46 -0
  183. package/dist/src/daemon/event-stream.js.map +1 -0
  184. package/dist/src/daemon/health.js +66 -0
  185. package/dist/src/daemon/health.js.map +1 -0
  186. package/dist/src/daemon/launcher.js +28 -1
  187. package/dist/src/daemon/launcher.js.map +1 -1
  188. package/dist/src/daemon/logging.js +23 -0
  189. package/dist/src/daemon/logging.js.map +1 -0
  190. package/dist/src/daemon/scheduler.js +26 -5
  191. package/dist/src/daemon/scheduler.js.map +1 -1
  192. package/dist/src/daemon/server.js +255 -98
  193. package/dist/src/daemon/server.js.map +1 -1
  194. package/dist/src/daemon/telemetry.js +163 -0
  195. package/dist/src/daemon/telemetry.js.map +1 -0
  196. package/dist/src/daemon/worker-routes.js +219 -0
  197. package/dist/src/daemon/worker-routes.js.map +1 -0
  198. package/dist/src/domain/benchmarks.js +2 -0
  199. package/dist/src/domain/benchmarks.js.map +1 -0
  200. package/dist/src/domain/bundles.js +2 -0
  201. package/dist/src/domain/bundles.js.map +1 -0
  202. package/dist/src/domain/control-plane.js +2 -0
  203. package/dist/src/domain/control-plane.js.map +1 -0
  204. package/dist/src/domain/execution-evidence.js +2 -0
  205. package/dist/src/domain/execution-evidence.js.map +1 -0
  206. package/dist/src/domain/execution-plan.js +2 -0
  207. package/dist/src/domain/execution-plan.js.map +1 -0
  208. package/dist/src/domain/images.js +2 -0
  209. package/dist/src/domain/images.js.map +1 -0
  210. package/dist/src/domain/index.js +13 -0
  211. package/dist/src/domain/index.js.map +1 -1
  212. package/dist/src/domain/interactions.js +2 -0
  213. package/dist/src/domain/interactions.js.map +1 -0
  214. package/dist/src/domain/providers.js +2 -0
  215. package/dist/src/domain/providers.js.map +1 -0
  216. package/dist/src/domain/resources.js +2 -0
  217. package/dist/src/domain/resources.js.map +1 -0
  218. package/dist/src/domain/runs.js.map +1 -1
  219. package/dist/src/domain/validation.js +11 -4
  220. package/dist/src/domain/validation.js.map +1 -1
  221. package/dist/src/domain/verifier-evidence-validation.js +214 -0
  222. package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
  223. package/dist/src/domain/verifier-evidence.js +3 -0
  224. package/dist/src/domain/verifier-evidence.js.map +1 -0
  225. package/dist/src/domain/verifier-score-contract.js +232 -0
  226. package/dist/src/domain/verifier-score-contract.js.map +1 -0
  227. package/dist/src/domain/workers.js +2 -0
  228. package/dist/src/domain/workers.js.map +1 -0
  229. package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
  230. package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
  231. package/dist/src/evals/benchmark-candidate.js +36 -0
  232. package/dist/src/evals/benchmark-candidate.js.map +1 -0
  233. package/dist/src/evals/benchmark-run.js +166 -0
  234. package/dist/src/evals/benchmark-run.js.map +1 -0
  235. package/dist/src/evals/collect-only-rerun.js +183 -0
  236. package/dist/src/evals/collect-only-rerun.js.map +1 -0
  237. package/dist/src/evals/directory.js +30 -0
  238. package/dist/src/evals/directory.js.map +1 -0
  239. package/dist/src/evals/docker-engine-stats.js +89 -0
  240. package/dist/src/evals/docker-engine-stats.js.map +1 -0
  241. package/dist/src/evals/docker-ownership.js +51 -0
  242. package/dist/src/evals/docker-ownership.js.map +1 -0
  243. package/dist/src/evals/docker-reaper.js +144 -0
  244. package/dist/src/evals/docker-reaper.js.map +1 -0
  245. package/dist/src/evals/docker-resource-observer.js +207 -0
  246. package/dist/src/evals/docker-resource-observer.js.map +1 -0
  247. package/dist/src/evals/duration-estimator.js +126 -0
  248. package/dist/src/evals/duration-estimator.js.map +1 -0
  249. package/dist/src/evals/environment-image-planning.js +164 -0
  250. package/dist/src/evals/environment-image-planning.js.map +1 -0
  251. package/dist/src/evals/eval-artifact-planning.js +56 -0
  252. package/dist/src/evals/eval-artifact-planning.js.map +1 -0
  253. package/dist/src/evals/eval-finalization.js +10 -0
  254. package/dist/src/evals/eval-finalization.js.map +1 -0
  255. package/dist/src/evals/eval-lifecycle-events.js +29 -0
  256. package/dist/src/evals/eval-lifecycle-events.js.map +1 -0
  257. package/dist/src/evals/eval-logical-plan.js +51 -0
  258. package/dist/src/evals/eval-logical-plan.js.map +1 -0
  259. package/dist/src/evals/eval-result-builder.js +81 -0
  260. package/dist/src/evals/eval-result-builder.js.map +1 -0
  261. package/dist/src/evals/events.js +53 -5
  262. package/dist/src/evals/events.js.map +1 -1
  263. package/dist/src/evals/evolution-baseline.js +191 -0
  264. package/dist/src/evals/evolution-baseline.js.map +1 -0
  265. package/dist/src/evals/execution-evidence.js +116 -0
  266. package/dist/src/evals/execution-evidence.js.map +1 -0
  267. package/dist/src/evals/execution-leases.js +342 -0
  268. package/dist/src/evals/execution-leases.js.map +1 -0
  269. package/dist/src/evals/execution-plan-artifacts.js +72 -0
  270. package/dist/src/evals/execution-plan-artifacts.js.map +1 -0
  271. package/dist/src/evals/execution-plan-images.js +69 -0
  272. package/dist/src/evals/execution-plan-images.js.map +1 -0
  273. package/dist/src/evals/execution-plan-resources.js +174 -0
  274. package/dist/src/evals/execution-plan-resources.js.map +1 -0
  275. package/dist/src/evals/execution-plan.js +451 -0
  276. package/dist/src/evals/execution-plan.js.map +1 -0
  277. package/dist/src/evals/failure-classifier.js +61 -0
  278. package/dist/src/evals/failure-classifier.js.map +1 -0
  279. package/dist/src/evals/harbor-artifact-builder-image.js +99 -0
  280. package/dist/src/evals/harbor-artifact-builder-image.js.map +1 -0
  281. package/dist/src/evals/harbor-artifact-builder.js +358 -0
  282. package/dist/src/evals/harbor-artifact-builder.js.map +1 -0
  283. package/dist/src/evals/harbor-bridge-error.js +58 -3
  284. package/dist/src/evals/harbor-bridge-error.js.map +1 -1
  285. package/dist/src/evals/harbor-node-runtime.js +146 -0
  286. package/dist/src/evals/harbor-node-runtime.js.map +1 -0
  287. package/dist/src/evals/harbor-phase-timings.js +60 -0
  288. package/dist/src/evals/harbor-phase-timings.js.map +1 -0
  289. package/dist/src/evals/index.js +40 -2
  290. package/dist/src/evals/index.js.map +1 -1
  291. package/dist/src/evals/infrastructure-retry.js +208 -84
  292. package/dist/src/evals/infrastructure-retry.js.map +1 -1
  293. package/dist/src/evals/interaction-capture-import.js +49 -0
  294. package/dist/src/evals/interaction-capture-import.js.map +1 -0
  295. package/dist/src/evals/local-docker-provider.js +372 -0
  296. package/dist/src/evals/local-docker-provider.js.map +1 -0
  297. package/dist/src/evals/local-eval-planning.js +80 -0
  298. package/dist/src/evals/local-eval-planning.js.map +1 -0
  299. package/dist/src/evals/model-capture-plan.js +22 -0
  300. package/dist/src/evals/model-capture-plan.js.map +1 -0
  301. package/dist/src/evals/model-capture-runtime.js +114 -0
  302. package/dist/src/evals/model-capture-runtime.js.map +1 -0
  303. package/dist/src/evals/model-proxy-runtime-state.js +53 -0
  304. package/dist/src/evals/model-proxy-runtime-state.js.map +1 -0
  305. package/dist/src/evals/native-phase-evidence.js +440 -0
  306. package/dist/src/evals/native-phase-evidence.js.map +1 -0
  307. package/dist/src/evals/physical-retry-work.js +31 -0
  308. package/dist/src/evals/physical-retry-work.js.map +1 -0
  309. package/dist/src/evals/planned-execution-support.js +66 -0
  310. package/dist/src/evals/planned-execution-support.js.map +1 -0
  311. package/dist/src/evals/planned-execution.js +469 -0
  312. package/dist/src/evals/planned-execution.js.map +1 -0
  313. package/dist/src/evals/planned-progress-publisher.js +62 -0
  314. package/dist/src/evals/planned-progress-publisher.js.map +1 -0
  315. package/dist/src/evals/planned-remote-execution.js +40 -0
  316. package/dist/src/evals/planned-remote-execution.js.map +1 -0
  317. package/dist/src/evals/planned-retry-execution.js +174 -0
  318. package/dist/src/evals/planned-retry-execution.js.map +1 -0
  319. package/dist/src/evals/planned-retry-lifecycle.js +188 -0
  320. package/dist/src/evals/planned-retry-lifecycle.js.map +1 -0
  321. package/dist/src/evals/preparation-rerun.js +225 -0
  322. package/dist/src/evals/preparation-rerun.js.map +1 -0
  323. package/dist/src/evals/prepared-harness.js +76 -0
  324. package/dist/src/evals/prepared-harness.js.map +1 -0
  325. package/dist/src/evals/progress.js +52 -12
  326. package/dist/src/evals/progress.js.map +1 -1
  327. package/dist/src/evals/records.js +20 -3
  328. package/dist/src/evals/records.js.map +1 -1
  329. package/dist/src/evals/recovery.js +242 -0
  330. package/dist/src/evals/recovery.js.map +1 -0
  331. package/dist/src/evals/regrade-evidence.js +77 -0
  332. package/dist/src/evals/regrade-evidence.js.map +1 -0
  333. package/dist/src/evals/remote-infrastructure-retry.js +165 -0
  334. package/dist/src/evals/remote-infrastructure-retry.js.map +1 -0
  335. package/dist/src/evals/request.js +21 -3
  336. package/dist/src/evals/request.js.map +1 -1
  337. package/dist/src/evals/rerun-inputs.js +108 -0
  338. package/dist/src/evals/rerun-inputs.js.map +1 -0
  339. package/dist/src/evals/rerun-slots.js +4 -3
  340. package/dist/src/evals/rerun-slots.js.map +1 -1
  341. package/dist/src/evals/rerun-types.js +45 -0
  342. package/dist/src/evals/rerun-types.js.map +1 -0
  343. package/dist/src/evals/rerun.js +133 -88
  344. package/dist/src/evals/rerun.js.map +1 -1
  345. package/dist/src/evals/result-helpers.js +78 -0
  346. package/dist/src/evals/result-helpers.js.map +1 -0
  347. package/dist/src/evals/resume-state.js +36 -0
  348. package/dist/src/evals/resume-state.js.map +1 -0
  349. package/dist/src/evals/retry-backoff.js +20 -0
  350. package/dist/src/evals/retry-backoff.js.map +1 -0
  351. package/dist/src/evals/retry-state.js +195 -0
  352. package/dist/src/evals/retry-state.js.map +1 -0
  353. package/dist/src/evals/scheduler-metrics.js +95 -0
  354. package/dist/src/evals/scheduler-metrics.js.map +1 -0
  355. package/dist/src/evals/scheduler-trace-replay.js +107 -0
  356. package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
  357. package/dist/src/evals/service-types.js +2 -0
  358. package/dist/src/evals/service-types.js.map +1 -0
  359. package/dist/src/evals/service.js +331 -347
  360. package/dist/src/evals/service.js.map +1 -1
  361. package/dist/src/evals/task-resources.js +209 -0
  362. package/dist/src/evals/task-resources.js.map +1 -0
  363. package/dist/src/evals/trial-environment-evidence.js +87 -0
  364. package/dist/src/evals/trial-environment-evidence.js.map +1 -0
  365. package/dist/src/evals/trial-execution-evidence.js +13 -0
  366. package/dist/src/evals/trial-execution-evidence.js.map +1 -0
  367. package/dist/src/evals/trial-import-identity.js +31 -0
  368. package/dist/src/evals/trial-import-identity.js.map +1 -0
  369. package/dist/src/evals/trial-import.js +183 -188
  370. package/dist/src/evals/trial-import.js.map +1 -1
  371. package/dist/src/evals/trial-publication-recovery.js +144 -0
  372. package/dist/src/evals/trial-publication-recovery.js.map +1 -0
  373. package/dist/src/evals/trial-publication.js +25 -0
  374. package/dist/src/evals/trial-publication.js.map +1 -0
  375. package/dist/src/evals/trial-reference-validation.js +49 -0
  376. package/dist/src/evals/trial-reference-validation.js.map +1 -0
  377. package/dist/src/evals/verifier-artifacts.js +317 -0
  378. package/dist/src/evals/verifier-artifacts.js.map +1 -0
  379. package/dist/src/evals/verifier-eligibility.js +25 -0
  380. package/dist/src/evals/verifier-eligibility.js.map +1 -0
  381. package/dist/src/evals/verifier-only-rerun.js +210 -0
  382. package/dist/src/evals/verifier-only-rerun.js.map +1 -0
  383. package/dist/src/evals/verifier-runtime.js +61 -0
  384. package/dist/src/evals/verifier-runtime.js.map +1 -0
  385. package/dist/src/evals/verifier-score-artifacts.js +119 -0
  386. package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
  387. package/dist/src/evals/work-item-artifacts.js +23 -0
  388. package/dist/src/evals/work-item-artifacts.js.map +1 -0
  389. package/dist/src/foundation/config.js +17 -0
  390. package/dist/src/foundation/config.js.map +1 -1
  391. package/dist/src/foundation/contained-file.js +85 -0
  392. package/dist/src/foundation/contained-file.js.map +1 -0
  393. package/dist/src/foundation/credential-redaction.js +162 -0
  394. package/dist/src/foundation/credential-redaction.js.map +1 -0
  395. package/dist/src/foundation/environment-image-references.js +59 -0
  396. package/dist/src/foundation/environment-image-references.js.map +1 -0
  397. package/dist/src/foundation/executable.js +45 -5
  398. package/dist/src/foundation/executable.js.map +1 -1
  399. package/dist/src/foundation/fs.js +2 -1
  400. package/dist/src/foundation/fs.js.map +1 -1
  401. package/dist/src/foundation/index.js +6 -2
  402. package/dist/src/foundation/index.js.map +1 -1
  403. package/dist/src/foundation/process-identity.js +69 -0
  404. package/dist/src/foundation/process-identity.js.map +1 -0
  405. package/dist/src/images/context.js +47 -0
  406. package/dist/src/images/context.js.map +1 -0
  407. package/dist/src/images/docker-buildkit.js +137 -0
  408. package/dist/src/images/docker-buildkit.js.map +1 -0
  409. package/dist/src/images/dockerfile.js +67 -0
  410. package/dist/src/images/dockerfile.js.map +1 -0
  411. package/dist/src/images/gc.js +261 -0
  412. package/dist/src/images/gc.js.map +1 -0
  413. package/dist/src/images/index.js +10 -0
  414. package/dist/src/images/index.js.map +1 -0
  415. package/dist/src/images/manifest.js +109 -0
  416. package/dist/src/images/manifest.js.map +1 -0
  417. package/dist/src/images/ownership.js +5 -0
  418. package/dist/src/images/ownership.js.map +1 -0
  419. package/dist/src/images/records.js +32 -0
  420. package/dist/src/images/records.js.map +1 -0
  421. package/dist/src/images/registry.js +154 -0
  422. package/dist/src/images/registry.js.map +1 -0
  423. package/dist/src/images/service.js +224 -0
  424. package/dist/src/images/service.js.map +1 -0
  425. package/dist/src/model-access/capture.js +255 -0
  426. package/dist/src/model-access/capture.js.map +1 -0
  427. package/dist/src/model-access/index.js +5 -0
  428. package/dist/src/model-access/index.js.map +1 -0
  429. package/dist/src/model-access/policy.js +49 -0
  430. package/dist/src/model-access/policy.js.map +1 -0
  431. package/dist/src/model-access/proxy.js +336 -0
  432. package/dist/src/model-access/proxy.js.map +1 -0
  433. package/dist/src/model-access/records.js +167 -0
  434. package/dist/src/model-access/records.js.map +1 -0
  435. package/dist/src/runs/adapter-process.js +12 -0
  436. package/dist/src/runs/adapter-process.js.map +1 -0
  437. package/dist/src/runs/bundle.js +450 -0
  438. package/dist/src/runs/bundle.js.map +1 -0
  439. package/dist/src/runs/events.js +12 -1
  440. package/dist/src/runs/events.js.map +1 -1
  441. package/dist/src/runs/executor.js +46 -45
  442. package/dist/src/runs/executor.js.map +1 -1
  443. package/dist/src/runs/finalizer.js +17 -0
  444. package/dist/src/runs/finalizer.js.map +1 -1
  445. package/dist/src/runs/index.js +8 -1
  446. package/dist/src/runs/index.js.map +1 -1
  447. package/dist/src/runs/manifest.js +5 -5
  448. package/dist/src/runs/manifest.js.map +1 -1
  449. package/dist/src/runs/phase-bundle.js +86 -0
  450. package/dist/src/runs/phase-bundle.js.map +1 -0
  451. package/dist/src/runs/phase-cancellation.js +107 -0
  452. package/dist/src/runs/phase-cancellation.js.map +1 -0
  453. package/dist/src/runs/phase-group.js +113 -0
  454. package/dist/src/runs/phase-group.js.map +1 -0
  455. package/dist/src/runs/query.js +1 -1
  456. package/dist/src/runs/query.js.map +1 -1
  457. package/dist/src/runs/records.js +3 -0
  458. package/dist/src/runs/records.js.map +1 -1
  459. package/dist/src/runs/request.js +10 -3
  460. package/dist/src/runs/request.js.map +1 -1
  461. package/dist/src/runs/training-candidate.js +161 -0
  462. package/dist/src/runs/training-candidate.js.map +1 -0
  463. package/dist/src/runs/verifier-evidence-redaction.js +61 -0
  464. package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
  465. package/dist/src/runs/verifier-evidence.js +459 -0
  466. package/dist/src/runs/verifier-evidence.js.map +1 -0
  467. package/dist/src/runs/verifier-structured-evidence.js +83 -0
  468. package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
  469. package/dist/src/trajectories/analysis.js +288 -0
  470. package/dist/src/trajectories/analysis.js.map +1 -0
  471. package/dist/src/trajectories/chunk-projection.js +148 -0
  472. package/dist/src/trajectories/chunk-projection.js.map +1 -0
  473. package/dist/src/trajectories/content-projection.js +389 -0
  474. package/dist/src/trajectories/content-projection.js.map +1 -0
  475. package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
  476. package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
  477. package/dist/src/trajectories/dsh-contract.js +284 -0
  478. package/dist/src/trajectories/dsh-contract.js.map +1 -0
  479. package/dist/src/trajectories/events-chunk-drill.js +95 -0
  480. package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
  481. package/dist/src/trajectories/events-page.js +367 -0
  482. package/dist/src/trajectories/events-page.js.map +1 -0
  483. package/dist/src/trajectories/index.js +3 -0
  484. package/dist/src/trajectories/index.js.map +1 -1
  485. package/dist/src/trajectories/provider-capture.js +15 -24
  486. package/dist/src/trajectories/provider-capture.js.map +1 -1
  487. package/dist/src/trajectories/providers/deepseek.js +3 -3
  488. package/dist/src/trajectories/providers/deepseek.js.map +1 -1
  489. package/dist/src/trajectories/request-attempt.js +47 -0
  490. package/dist/src/trajectories/request-attempt.js.map +1 -0
  491. package/dist/src/trajectories/stream-reader.js +311 -0
  492. package/dist/src/trajectories/stream-reader.js.map +1 -0
  493. package/dist/src/trajectories/surface-fold.js +126 -0
  494. package/dist/src/trajectories/surface-fold.js.map +1 -0
  495. package/dist/src/workers/index.js +3 -0
  496. package/dist/src/workers/index.js.map +1 -0
  497. package/dist/src/workers/remote-harbor-work-spec.js +97 -0
  498. package/dist/src/workers/remote-harbor-work-spec.js.map +1 -0
  499. package/dist/src/workers/remote-harbor-worker.js +223 -0
  500. package/dist/src/workers/remote-harbor-worker.js.map +1 -0
  501. package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
  502. package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
  503. package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
  504. package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
  505. package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
  506. package/docs/schemas/benchmark-package-v1.schema.json +167 -0
  507. package/docs/schemas/benchmark-phase-group.schema.json +41 -0
  508. package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
  509. package/docs/schemas/benchmark-task-v1.schema.json +384 -0
  510. package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
  511. package/docs/schemas/environment-build-index.schema.json +13 -0
  512. package/docs/schemas/environment-build-record.schema.json +25 -0
  513. package/docs/schemas/environment-image-manifest.schema.json +62 -0
  514. package/docs/schemas/error.schema.json +22 -0
  515. package/docs/schemas/eval-control.schema.json +36 -0
  516. package/docs/schemas/eval-execution-plan.schema.json +216 -0
  517. package/docs/schemas/eval-progress.schema.json +1 -15
  518. package/docs/schemas/eval-rerun-result.schema.json +58 -0
  519. package/docs/schemas/eval-rerun-state.schema.json +61 -0
  520. package/docs/schemas/eval-rerun-submission.schema.json +60 -0
  521. package/docs/schemas/eval-result.schema.json +32 -13
  522. package/docs/schemas/eval-submission-input.schema.json +14 -0
  523. package/docs/schemas/eval-submission.schema.json +68 -0
  524. package/docs/schemas/eval-trial-publication.schema.json +15 -0
  525. package/docs/schemas/eval-trial-reference.schema.json +129 -0
  526. package/docs/schemas/execution-evidence.schema.json +96 -0
  527. package/docs/schemas/execution-lease.schema.json +50 -0
  528. package/docs/schemas/execution-provider-status.schema.json +65 -0
  529. package/docs/schemas/execution-worker.schema.json +53 -0
  530. package/docs/schemas/harbor-process-exit.schema.json +14 -0
  531. package/docs/schemas/interaction-capture-ref.schema.json +45 -0
  532. package/docs/schemas/local-provider-execution.schema.json +33 -0
  533. package/docs/schemas/model-interaction.schema.json +41 -0
  534. package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
  535. package/docs/schemas/remote-result-envelope.schema.json +32 -0
  536. package/docs/schemas/remote-tree-envelope.schema.json +25 -0
  537. package/docs/schemas/remote-work-offer.schema.json +58 -0
  538. package/docs/schemas/remote-work-receipt.schema.json +50 -0
  539. package/docs/schemas/remote-worker-artifact.schema.json +17 -0
  540. package/docs/schemas/remote-worker-event.schema.json +18 -0
  541. package/docs/schemas/remote-worker-heartbeat.schema.json +37 -0
  542. package/docs/schemas/remote-worker-registration.schema.json +52 -0
  543. package/docs/schemas/result-bundle-index.schema.json +133 -0
  544. package/docs/schemas/run-context.schema.json +15 -0
  545. package/docs/schemas/run-environment-images.schema.json +39 -0
  546. package/docs/schemas/run-manifest.schema.json +4 -0
  547. package/docs/schemas/run-request.schema.json +4 -0
  548. package/docs/schemas/training-data-candidate.schema.json +46 -0
  549. package/docs/schemas/trajectory-analysis.schema.json +258 -0
  550. package/docs/schemas/trajectory-events-page.schema.json +40 -0
  551. package/docs/schemas/verifier-assessment.schema.json +77 -0
  552. package/docs/schemas/verifier-evidence.schema.json +276 -0
  553. package/integrations/harbor/hitch_benchmark.py +252 -0
  554. package/integrations/harbor/hitch_candidate_recycle.py +262 -0
  555. package/integrations/harbor/hitch_harbor_agent.py +698 -568
  556. package/integrations/harbor/hitch_harbor_environment.py +469 -0
  557. package/integrations/harbor/hitch_harbor_task_resources.py +275 -0
  558. package/integrations/harbor/hitch_harbor_verifier.py +83 -2
  559. package/integrations/harbor/hitch_phase_supervisor.py +452 -0
  560. package/integrations/harbor/hitch_tool_client.mjs +91 -0
  561. package/integrations/model-call/cli.js +57 -0
  562. package/package.json +30 -6
@@ -0,0 +1,271 @@
1
+ import { lstat, readdir, readFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import { atomicWriteJSON, invalidInput, sha256Bytes, sha256JSON } from "../foundation/index.js";
5
+ import { parseBenchmarkToml } from "./toml.js";
6
+ import { fields, object, parseManifest, parseProfile, parseTask, relativePath, unsupported } from "./validation.js";
7
+ const RESOLVER_VERSION = "1";
8
+ export async function loadBenchmark(directory) {
9
+ directory = path.resolve(directory);
10
+ const files = await inventory(directory);
11
+ const available = new Set(files.map((f) => f.path));
12
+ const required = (name) => {
13
+ relativePath(name);
14
+ if (!available.has(name))
15
+ throw invalidInput(`required package file missing: ${name}`);
16
+ return path.join(directory, name);
17
+ };
18
+ const read = async (name) => JSON.parse(await readFile(required(name), "utf8"));
19
+ const manifest = parseManifest(parseBenchmarkToml(await readFile(required("benchmark.toml"), "utf8")));
20
+ const profile = parseProfile(await read(manifest.default_profile));
21
+ if (profile.track !== manifest.publication.track)
22
+ throw invalidInput("profile and publication tracks differ");
23
+ const sourceManifest = await read("source-manifest.json");
24
+ const provenance = object(sourceManifest, "source manifest");
25
+ const transformations = [];
26
+ if (provenance.transformations !== undefined && !Array.isArray(provenance.transformations))
27
+ throw invalidInput("source transformations must be an array");
28
+ for (const transformation of (provenance.transformations ?? [])) {
29
+ const entry = fields(transformation, ["kind", "before_path", "after_path"], "source transformation");
30
+ if (typeof entry.kind !== "string")
31
+ throw invalidInput("invalid transformation kind");
32
+ required(relativePath(entry.before_path));
33
+ required(relativePath(entry.after_path));
34
+ transformations.push({ kind: entry.kind, before: files.find((f) => f.path === entry.before_path).digest, after: files.find((f) => f.path === entry.after_path).digest });
35
+ }
36
+ let sourceAdapter;
37
+ if (provenance.adapter !== undefined) {
38
+ const adapter = object(provenance.adapter, "source adapter provenance");
39
+ if (typeof adapter.id !== "string" || typeof adapter.version !== "string")
40
+ throw invalidInput("invalid source adapter provenance");
41
+ if (adapter.path !== undefined) {
42
+ const adapterPath = relativePath(adapter.path);
43
+ required(adapterPath);
44
+ sourceAdapter = { id: adapter.id, version: adapter.version, code_digest: files.find((f) => f.path === adapterPath).digest };
45
+ }
46
+ }
47
+ const treeDigest = (prefix) => {
48
+ const subset = files.filter((f) => f.path === prefix || f.path.startsWith(`${prefix}/`));
49
+ if (!subset.length)
50
+ throw invalidInput(`empty or missing package component: ${prefix}`);
51
+ return sha256JSON(subset.map((f) => ({ ...f, path: f.path.slice(prefix.length) })));
52
+ };
53
+ const components = manifest.runtime_components.map((c) => ({ id: c.id, protocol: c.protocol, content_digest: treeDigest(c.path) }));
54
+ const tasks = [];
55
+ const lockedTasks = [];
56
+ const sourceIds = new Set();
57
+ const actualTaskIds = (await readdir(path.join(directory, manifest.task_root), { withFileTypes: true })).filter((e) => e.isDirectory()).map((e) => e.name).sort();
58
+ if (JSON.stringify(actualTaskIds) !== JSON.stringify([...manifest.task_ids].sort()))
59
+ throw invalidInput("task directory membership differs from manifest task_ids");
60
+ for (const id of [...manifest.task_ids].sort()) {
61
+ const taskPath = `${manifest.task_root}/${id}`;
62
+ const config = parseTask(await read(`${taskPath}/task.hitch.json`), manifest);
63
+ if (sourceIds.has(config.source_task_id))
64
+ throw invalidInput("duplicate source_task_id");
65
+ sourceIds.add(config.source_task_id);
66
+ const harbor = parseBenchmarkToml(await readFile(required(`${taskPath}/task.toml`), "utf8"));
67
+ validateHarbor(harbor, config);
68
+ required(`${taskPath}/tests/test.sh`);
69
+ let tools = [];
70
+ if (config.driver.kind === "model-call") {
71
+ if (!config.driver.config.input.startsWith(`${taskPath}/environment/`))
72
+ throw invalidInput("model input must be in the candidate environment");
73
+ required(config.driver.config.input);
74
+ }
75
+ if (config.driver.kind === "tool-server") {
76
+ required(`${taskPath}/environment/docker-compose.yaml`);
77
+ required(`${taskPath}/tests/Dockerfile`);
78
+ const schema = await read(config.driver.config.schema);
79
+ validateTools(schema);
80
+ tools = schema;
81
+ }
82
+ required(`${taskPath}/instruction.md`);
83
+ for (const cap of config.requirements)
84
+ if (!profile.tool_policy.allowed.includes(cap))
85
+ throw invalidInput(`profile does not grant required capability: ${cap}`);
86
+ if (config.driver.kind === "tool-server" && config.driver.config.native_phases
87
+ && config.driver.config.native_phases.shutdown_timeout_ms > profile.budget.collection_timeout_ms)
88
+ throw invalidInput("native phase shutdown exceeds profile collection allowance");
89
+ if (config.driver.kind === "tool-server" && config.driver.config.native_phases?.protocol === "hitch-native-phase-control@2"
90
+ && config.driver.config.native_phases.finalization_timeout_ms > profile.budget.collection_timeout_ms)
91
+ throw invalidInput("native finalization exceeds profile collection allowance");
92
+ for (const [phase, hook] of Object.entries(config.lifecycle)) {
93
+ const limit = phase === "prepare" ? profile.budget.setup_timeout_ms : phase === "cleanup" ? profile.budget.cleanup_grace_ms : profile.budget.collection_timeout_ms;
94
+ if (hook.timeout_ms > limit)
95
+ throw invalidInput(`hook ${phase} timeout exceeds profile phase budget`);
96
+ }
97
+ const taskDigest = sha256JSON({ tree: treeDigest(taskPath), components, tools: sha256JSON(tools) });
98
+ const environments = [];
99
+ for (const area of ["environment", "tests"]) {
100
+ const baseImages = new Set();
101
+ const dockerfiles = files.filter((f) => f.path.startsWith(`${taskPath}/${area}/`) && /^Dockerfile(?:\.|$)/.test(path.basename(f.path)));
102
+ for (const dockerfile of dockerfiles) {
103
+ const stages = new Set();
104
+ let continued = false;
105
+ for (const line of (await readFile(required(dockerfile.path), "utf8")).split(/\r?\n/)) {
106
+ const continuation = continued;
107
+ if (line.trim() && !line.trimStart().startsWith("#"))
108
+ continued = line.trimEnd().endsWith("\\");
109
+ if (continuation)
110
+ continue;
111
+ const match = /^\s*FROM\s+(?:--platform=linux\/amd64\s+)?(\S+)(?:\s+AS\s+(\S+))?\s*$/i.exec(line);
112
+ if (!match) {
113
+ if (/^\s*FROM\b/i.test(line))
114
+ unsupported("unsupported Docker FROM syntax");
115
+ continue;
116
+ }
117
+ const image = match[1];
118
+ if (image !== "scratch" && !stages.has(image)) {
119
+ const digest = /@sha256:([a-f0-9]{64})$/.exec(image);
120
+ if (!digest)
121
+ throw invalidInput(`unresolved base image: ${image}`);
122
+ baseImages.add(`sha256:${digest[1]}`);
123
+ }
124
+ if (match[2])
125
+ stages.add(match[2]);
126
+ }
127
+ }
128
+ environments.push({ role: area, kind: "build", context_digest: treeDigest(`${taskPath}/${area}`), base_image_digests: [...baseImages].sort(), platform: "linux/amd64" });
129
+ }
130
+ lockedTasks.push({ task_id: id, source_task_id: config.source_task_id, path: taskPath, task_digest: taskDigest, input_digest: treeDigest(`${taskPath}/instruction.md`), grader_digest: sha256JSON({ tests: treeDigest(`${taskPath}/tests`), components }), environment_refs: environments });
131
+ tasks.push({ id, path: taskPath, config, harbor, tools });
132
+ }
133
+ // Values, modes and paths participate in identity; host directory and mtime do not.
134
+ const packageDigest = sha256JSON(files);
135
+ const resolverFiles = await Promise.all(["loader", "validation", "toml", "metrics"].map(async (name) => sha256Bytes(await readFile(new URL(`./${name}.js`, import.meta.url)))));
136
+ resolverFiles.push(sha256JSON(await inventory(path.dirname(fileURLToPath(import.meta.resolve("smol-toml"))))));
137
+ const lock = {
138
+ schema_version: "1", protocol: "hitch-benchmark@1", benchmark_id: manifest.id, release: manifest.release,
139
+ package_digest: packageDigest,
140
+ source: {
141
+ kind: manifest.source.kind,
142
+ uri: manifest.source.kind === "local" ? `cas:${packageDigest}` : manifest.source.uri,
143
+ resolved_revision: manifest.source.kind === "local" ? packageDigest : manifest.source.resolved_revision,
144
+ access: manifest.source.access ?? "public", manifest_digest: sha256JSON(sourceManifest),
145
+ },
146
+ resolver: { id: "hitch-benchmark", version: RESOLVER_VERSION, code_digest: sha256JSON(resolverFiles) },
147
+ ...(sourceAdapter ? { source_adapter: sourceAdapter } : {}),
148
+ components: components.map((c) => ({ role: c.id, uri: `cas:${c.content_digest}`, resolved_revision: c.content_digest, digest: c.content_digest })),
149
+ task_dialect: manifest.task_format, runtime_components: components, tasks: lockedTasks,
150
+ profile_digest: sha256JSON(profile), required_capabilities: [...new Set(tasks.flatMap((t) => t.config.requirements))].sort(),
151
+ metric_spec_digest: sha256JSON({ primary: manifest.primary_metric, metrics: manifest.metrics }), files, transformations,
152
+ };
153
+ return { directory, manifest, profile, tasks, lock };
154
+ }
155
+ export async function benchmarkTreeDigest(directory) {
156
+ return sha256JSON(await inventory(directory));
157
+ }
158
+ export async function validateBenchmark(directory) {
159
+ const loaded = await loadBenchmark(directory);
160
+ return { valid: true, protocol: loaded.lock.protocol, benchmark_id: loaded.manifest.id, package_digest: loaded.lock.package_digest, tasks: loaded.lock.tasks, required_capabilities: loaded.lock.required_capabilities };
161
+ }
162
+ export async function lockBenchmark(directory, output = path.join(directory, "benchmark.lock.json")) {
163
+ // A custom lock file outside the package avoids changing package identity.
164
+ const inside = path.relative(path.resolve(directory), path.resolve(output));
165
+ if (!inside.startsWith("..") && !path.isAbsolute(inside) && inside !== "benchmark.lock.json")
166
+ throw invalidInput("an in-package lock must be named benchmark.lock.json");
167
+ const loaded = await loadBenchmark(directory);
168
+ await atomicWriteJSON(output, loaded.lock);
169
+ return loaded.lock;
170
+ }
171
+ export async function loadBenchmarkLock(file, directory = path.dirname(file)) {
172
+ const lock = JSON.parse(await readFile(file, "utf8"));
173
+ const loaded = await loadBenchmark(directory);
174
+ if (sha256JSON(lock) !== sha256JSON(loaded.lock))
175
+ throw invalidInput("benchmark lock does not match the current package/resolver; create a new lock explicitly");
176
+ return loaded;
177
+ }
178
+ async function inventory(directory, prefix = "") {
179
+ const files = [];
180
+ const info = await lstat(directory);
181
+ if (!info.isDirectory() || info.isSymbolicLink())
182
+ throw invalidInput("package directory must be a real directory");
183
+ for (const entry of (await readdir(directory, { withFileTypes: true })).sort((a, b) => a.name < b.name ? -1 : 1)) {
184
+ const name = prefix ? `${prefix}/${entry.name}` : entry.name;
185
+ if (!prefix && entry.name === "benchmark.lock.json" && entry.isFile())
186
+ continue;
187
+ relativePath(name);
188
+ const absolute = path.join(directory, entry.name);
189
+ const stat = await lstat(absolute);
190
+ if (stat.isSymbolicLink() || (!stat.isDirectory() && !stat.isFile()))
191
+ unsupported(`package special files/symlinks are unsupported: ${name}`);
192
+ if (stat.isDirectory())
193
+ files.push(...await inventory(absolute, name));
194
+ else
195
+ files.push({ path: name, digest: sha256Bytes(await readFile(absolute)), bytes: stat.size, mode: stat.mode & 0o777 });
196
+ }
197
+ return files;
198
+ }
199
+ function validateTools(value) {
200
+ if (!Array.isArray(value) || !value.length)
201
+ throw invalidInput("tool schema must be a nonempty array");
202
+ const names = new Set();
203
+ for (const tool of value) {
204
+ const item = fields(tool, ["name", "description", "inputSchema"], "tool");
205
+ if (typeof item.name !== "string" || !/^[A-Za-z_][A-Za-z0-9_.-]*$/.test(item.name) || names.has(item.name))
206
+ throw invalidInput("invalid or duplicate tool name");
207
+ if (typeof item.description !== "string")
208
+ throw invalidInput("missing tool description");
209
+ const schema = object(item.inputSchema, "tool inputSchema");
210
+ if (schema.type !== "object")
211
+ unsupported("tool inputSchema must describe an object");
212
+ names.add(item.name);
213
+ }
214
+ }
215
+ function validateHarbor(h, task) {
216
+ if (task.driver.kind !== "tool-server") {
217
+ // Preserve the native Harbor task rather than imposing the sidecar dialect.
218
+ // The existing Harbor inspector validates the full resource/config model.
219
+ fields(h, ["schema_version", "version", "task", "metadata", "agent", "environment", "verifier", "artifacts", "solution", "source"], "Harbor task");
220
+ if (h.schema_version !== undefined && h.schema_version !== "1.4")
221
+ unsupported("unsupported Harbor task schema");
222
+ const agent = object(h.agent, "Harbor agent");
223
+ if (typeof agent.timeout_sec !== "number" || agent.timeout_sec <= 0)
224
+ throw invalidInput("task agent timeout must be positive");
225
+ const environment = object(h.environment, "Harbor environment");
226
+ if (environment.network_mode !== undefined && environment.network_mode !== "public")
227
+ unsupported("terminal profile requires public network");
228
+ const verifier = object(h.verifier, "Harbor verifier");
229
+ const mode = verifier.environment_mode ?? "shared";
230
+ if (!["shared", "separate"].includes(String(mode)))
231
+ unsupported("unsupported verifier mode");
232
+ if (!task.requirements.includes(`${mode}-verifier`))
233
+ throw invalidInput("verifier isolation must be declared in requirements");
234
+ if (task.submission.final_response && mode !== "separate")
235
+ throw invalidInput("canonical final response requires a separate verifier");
236
+ if (mode === "separate" && task.submission.kind !== "artifacts")
237
+ throw invalidInput("separate verifier requires explicit artifacts");
238
+ const artifacts = h.artifacts ?? [];
239
+ if (!Array.isArray(artifacts))
240
+ throw invalidInput("invalid Harbor artifacts");
241
+ const paths = artifacts.map(a => typeof a === "string" ? a : object(a, "artifact").source);
242
+ if (task.submission.kind === "artifacts" && JSON.stringify([...paths].sort()) !== JSON.stringify([...task.submission.paths].sort()))
243
+ throw invalidInput("submission paths and Harbor artifacts differ");
244
+ return;
245
+ }
246
+ const service = task.driver.config.service, paths = task.submission.paths;
247
+ fields(h, ["schema_version", "metadata", "agent", "environment", "verifier", "artifacts"], "Harbor task");
248
+ if (h.schema_version !== "1.4")
249
+ unsupported("unsupported Harbor task schema");
250
+ const agent = fields(h.agent, ["timeout_sec"], "Harbor agent");
251
+ if (typeof agent.timeout_sec !== "number" || agent.timeout_sec <= 0)
252
+ throw invalidInput("task agent timeout must be positive");
253
+ const environment = fields(h.environment, ["cpus", "memory_mb", "storage_mb", "build_timeout_sec", "workdir", "network_mode"], "Harbor environment");
254
+ if (environment.network_mode !== "public")
255
+ unsupported("MVP task network must be explicitly public");
256
+ const verifier = fields(h.verifier, ["timeout_sec", "environment_mode", "environment"], "Harbor verifier");
257
+ if (verifier.environment_mode !== "separate")
258
+ unsupported("tool-server tasks require a separate verifier");
259
+ fields(verifier.environment, ["cpus", "memory_mb", "storage_mb", "build_timeout_sec", "network_mode"], "Harbor verifier environment");
260
+ if (!Array.isArray(h.artifacts))
261
+ throw invalidInput("missing authoritative artifacts");
262
+ const sources = h.artifacts.map((a) => {
263
+ const artifact = fields(a, ["source", "service"], "artifact");
264
+ if (artifact.service !== service)
265
+ unsupported("tool-server snapshots must originate from the declared sidecar");
266
+ return artifact.source;
267
+ });
268
+ if (JSON.stringify([...sources].sort()) !== JSON.stringify([...paths].sort()))
269
+ throw invalidInput("submission paths and Harbor artifacts differ");
270
+ }
271
+ //# sourceMappingURL=loader.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"loader.js","sourceRoot":"","sources":["../../../src/benchmarks/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAChG,OAAO,EAAE,kBAAkB,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEpH,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,SAAiB;IACnD,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC;IACzC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAU,EAAE;QACxC,YAAY,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,MAAM,YAAY,CAAC,kCAAkC,IAAI,EAAE,CAAC,CAAC;QACvF,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,KAAK,EAAE,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAY,CAAC;IACnG,MAAM,QAAQ,GAAG,aAAa,CAAC,kBAAkB,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC;IACnE,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,WAAW,CAAC,KAAK;QAAE,MAAM,YAAY,CAAC,uCAAuC,CAAC,CAAC;IAC9G,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,sBAAsB,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,MAAM,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IAC7D,MAAM,eAAe,GAAuC,EAAE,CAAC;IAC/D,IAAI,UAAU,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,eAAe,CAAC;QAAE,MAAM,YAAY,CAAC,yCAAyC,CAAC,CAAC;IAC1J,KAAK,MAAM,cAAc,IAAI,CAAC,UAAU,CAAC,eAAe,IAAI,EAAE,CAAc,EAAE,CAAC;QAC7E,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,EAAE,uBAAuB,CAAC,CAAC;QACrG,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;YAAE,MAAM,YAAY,CAAC,6BAA6B,CAAC,CAAC;QACtF,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;QAAC,QAAQ,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;QACpF,eAAe,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,WAAW,CAAE,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC,UAAU,CAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7K,CAAC;IACD,IAAI,aAAgD,CAAC;IACrD,IAAI,UAAU,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QACrC,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,2BAA2B,CAAC,CAAC;QACxE,IAAI,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ,IAAI,OAAO,OAAO,CAAC,OAAO,KAAK,QAAQ;YAAE,MAAM,YAAY,CAAC,mCAAmC,CAAC,CAAC;QACnI,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YACtE,aAAa,GAAG,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAE,CAAC,MAAM,EAAE,CAAC;QAC/H,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,MAAc,EAAU,EAAE;QAC5C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,YAAY,CAAC,uCAAuC,MAAM,EAAE,CAAC,CAAC;QACxF,OAAO,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACpI,MAAM,KAAK,GAA+B,EAAE,CAAC;IAC7C,MAAM,WAAW,GAA6B,EAAE,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,MAAM,aAAa,GAAG,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAClK,IAAI,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,MAAM,YAAY,CAAC,0DAA0D,CAAC,CAAC;IACpK,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,QAAQ,GAAG,GAAG,QAAQ,CAAC,SAAS,IAAI,EAAE,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,IAAI,CAAC,GAAG,QAAQ,kBAAkB,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC9E,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC;YAAE,MAAM,YAAY,CAAC,0BAA0B,CAAC,CAAC;QACzF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,GAAG,QAAQ,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QAC7F,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC/B,QAAQ,CAAC,GAAG,QAAQ,gBAAgB,CAAC,CAAC;QACtC,IAAI,KAAK,GAAc,EAAE,CAAC;QAC1B,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACxC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,QAAQ,eAAe,CAAC;gBAAE,MAAM,YAAY,CAAC,kDAAkD,CAAC,CAAC;YAC/I,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACzC,QAAQ,CAAC,GAAG,QAAQ,kCAAkC,CAAC,CAAC;YACxD,QAAQ,CAAC,GAAG,QAAQ,mBAAmB,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACvD,aAAa,CAAC,MAAM,CAAC,CAAC;YAAC,KAAK,GAAG,MAAM,CAAC;QACxC,CAAC;QACD,QAAQ,CAAC,GAAG,QAAQ,iBAAiB,CAAC,CAAC;QACvC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,YAAY;YAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAAE,MAAM,YAAY,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;QAChK,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa;eACzE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,mBAAmB,GAAG,OAAO,CAAC,MAAM,CAAC,qBAAqB;YAAE,MAAM,YAAY,CAAC,4DAA4D,CAAC,CAAC;QACrL,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,QAAQ,KAAK,8BAA8B;eACtH,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,uBAAuB,GAAG,OAAO,CAAC,MAAM,CAAC,qBAAqB;YAAE,MAAM,YAAY,CAAC,0DAA0D,CAAC,CAAC;QACvL,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7D,MAAM,KAAK,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,qBAAqB,CAAC;YACnK,IAAI,IAAI,CAAC,UAAU,GAAG,KAAK;gBAAE,MAAM,YAAY,CAAC,QAAQ,KAAK,uCAAuC,CAAC,CAAC;QACxG,CAAC;QACD,MAAM,UAAU,GAAG,UAAU,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpG,MAAM,YAAY,GAAgC,EAAE,CAAC;QACrD,KAAK,MAAM,IAAI,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;YACrC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxI,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;gBACjC,IAAI,SAAS,GAAG,KAAK,CAAC;gBACtB,KAAK,MAAM,IAAI,IAAI,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;oBACtF,MAAM,YAAY,GAAG,SAAS,CAAC;oBAC/B,IAAI,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;wBAAE,SAAS,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAChG,IAAI,YAAY;wBAAE,SAAS;oBAC3B,MAAM,KAAK,GAAG,wEAAwE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAClG,IAAI,CAAC,KAAK,EAAE,CAAC;wBAAC,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC;4BAAE,WAAW,CAAC,gCAAgC,CAAC,CAAC;wBAAC,SAAS;oBAAC,CAAC;oBACtG,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;oBACxB,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;wBAC9C,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;wBACrD,IAAI,CAAC,MAAM;4BAAE,MAAM,YAAY,CAAC,0BAA0B,KAAK,EAAE,CAAC,CAAC;wBACnE,UAAU,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBACxC,CAAC;oBACD,IAAI,KAAK,CAAC,CAAC,CAAC;wBAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,UAAU,CAAC,GAAG,QAAQ,IAAI,IAAI,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAC;QAC3K,CAAC;QACD,WAAW,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,CAAC,GAAG,QAAQ,iBAAiB,CAAC,EAAE,aAAa,EAAE,UAAU,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,GAAG,QAAQ,QAAQ,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,gBAAgB,EAAE,YAAY,EAAE,CAAC,CAAC;QAC7R,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,oFAAoF;IACpF,MAAM,aAAa,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChL,aAAa,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,IAAI,GAAoB;QAC5B,cAAc,EAAE,GAAG,EAAE,QAAQ,EAAE,mBAAmB,EAAE,YAAY,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO;QACxG,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE;YACN,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI;YAC1B,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,aAAa,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAI;YACrF,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,iBAAkB;YACxG,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,IAAI,QAAQ,EAAE,eAAe,EAAE,UAAU,CAAC,cAAc,CAAC;SACxF;QACD,QAAQ,EAAE,EAAE,EAAE,EAAE,iBAAiB,EAAE,OAAO,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,CAAC,aAAa,CAAC,EAAE;QACtG,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,cAAc,EAAE,EAAE,iBAAiB,EAAE,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;QAClJ,YAAY,EAAE,QAAQ,CAAC,WAAW,EAAE,kBAAkB,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW;QACtF,cAAc,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,qBAAqB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;QAC5H,kBAAkB,EAAE,UAAU,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,cAAc,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,eAAe;KACxH,CAAC;IACF,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACvD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,SAAiB;IACzD,OAAO,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,SAAiB;IACvD,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAC9C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,qBAAqB,EAAE,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;AAC3N,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC;IACzG,2EAA2E;IAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,qBAAqB;QAAE,MAAM,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACzK,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAC9C,MAAM,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY,EAAE,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAClF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;IACjE,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,0FAA0F,CAAC,CAAC;IACjK,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAiB,EAAE,MAAM,GAAG,EAAE;IACrD,MAAM,KAAK,GAAsB,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,IAAI,CAAC,cAAc,EAAE;QAAE,MAAM,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACnH,KAAK,MAAM,KAAK,IAAI,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;QAC7D,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,qBAAqB,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,SAAS;QAChF,YAAY,CAAC,IAAI,CAAC,CAAC;QACnB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAAE,WAAW,CAAC,mDAAmD,IAAI,EAAE,CAAC,CAAC;QAC7I,IAAI,IAAI,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;;YAClE,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC;IAC5H,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,sCAAsC,CAAC,CAAC;IACvG,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,aAAa,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1E,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,YAAY,CAAC,gCAAgC,CAAC,CAAC;QACjK,IAAI,OAAO,IAAI,CAAC,WAAW,KAAK,QAAQ;YAAE,MAAM,YAAY,CAAC,0BAA0B,CAAC,CAAC;QACzF,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ;YAAE,WAAW,CAAC,0CAA0C,CAAC,CAAC;QACtF,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,CAA0B,EAAE,IAAqB;IACvE,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACvC,4EAA4E;QAC5E,0EAA0E;QAC1E,MAAM,CAAC,CAAC,EAAE,CAAC,gBAAgB,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE,aAAa,CAAC,CAAC;QACnJ,IAAI,CAAC,CAAC,cAAc,KAAK,SAAS,IAAI,CAAC,CAAC,cAAc,KAAK,KAAK;YAAE,WAAW,CAAC,gCAAgC,CAAC,CAAC;QAChH,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC;YAAE,MAAM,YAAY,CAAC,qCAAqC,CAAC,CAAC;QAC/H,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;QAChE,IAAI,WAAW,CAAC,YAAY,KAAK,SAAS,IAAI,WAAW,CAAC,YAAY,KAAK,QAAQ;YAAE,WAAW,CAAC,0CAA0C,CAAC,CAAC;QAC7I,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,gBAAgB,IAAI,QAAQ,CAAC;QACnD,IAAI,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAAE,WAAW,CAAC,2BAA2B,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,IAAI,WAAW,CAAC;YAAE,MAAM,YAAY,CAAC,qDAAqD,CAAC,CAAC;QAC/H,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,IAAI,IAAI,KAAK,UAAU;YAAE,MAAM,YAAY,CAAC,uDAAuD,CAAC,CAAC;QACvI,IAAI,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,WAAW;YAAE,MAAM,YAAY,CAAC,+CAA+C,CAAC,CAAC;QACrI,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YAAE,MAAM,YAAY,CAAC,0BAA0B,CAAC,CAAC;QAC9E,MAAM,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;QAC3F,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;YAAE,MAAM,YAAY,CAAC,8CAA8C,CAAC,CAAC;QACxM,OAAO;IACT,CAAC;IACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;IAC1E,MAAM,CAAC,CAAC,EAAE,CAAC,gBAAgB,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,WAAW,CAAC,EAAE,aAAa,CAAC,CAAC;IAC1G,IAAI,CAAC,CAAC,cAAc,KAAK,KAAK;QAAE,WAAW,CAAC,gCAAgC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;IAC/D,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,IAAI,KAAK,CAAC,WAAW,IAAI,CAAC;QAAE,MAAM,YAAY,CAAC,qCAAqC,CAAC,CAAC;IAC/H,MAAM,WAAW,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,mBAAmB,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACrJ,IAAI,WAAW,CAAC,YAAY,KAAK,QAAQ;QAAE,WAAW,CAAC,4CAA4C,CAAC,CAAC;IACrG,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,aAAa,EAAE,kBAAkB,EAAE,aAAa,CAAC,EAAE,iBAAiB,CAAC,CAAC;IAC3G,IAAI,QAAQ,CAAC,gBAAgB,KAAK,UAAU;QAAE,WAAW,CAAC,+CAA+C,CAAC,CAAC;IAC3G,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,6BAA6B,CAAC,CAAC;IACtI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;QAAE,MAAM,YAAY,CAAC,iCAAiC,CAAC,CAAC;IACvF,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,UAAU,CAAC,CAAC;QAC9D,IAAI,QAAQ,CAAC,OAAO,KAAK,OAAO;YAAE,WAAW,CAAC,gEAAgE,CAAC,CAAC;QAChH,OAAO,QAAQ,CAAC,MAAM,CAAC;IACzB,CAAC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,MAAM,YAAY,CAAC,8CAA8C,CAAC,CAAC;AACpJ,CAAC"}
@@ -0,0 +1,16 @@
1
+ import { HitchError } from "../foundation/index.js";
2
+ export function mapBenchmarkMetrics(raw, manifest, task) {
3
+ const metrics = {};
4
+ for (const [name, spec] of Object.entries(manifest.metrics)) {
5
+ const field = task.grading.metric_map[name];
6
+ const value = raw && typeof raw === "object" && !Array.isArray(raw) ? raw[field] : undefined;
7
+ if (value === undefined)
8
+ throw new HitchError(`missing required grader metric: ${field}`, { code: "metric_missing", exitCode: 12 });
9
+ if (typeof value !== "number" || !Number.isFinite(value) || value < spec.range[0] || value > spec.range[1] || (spec.type === "binary" && value !== 0 && value !== 1)) {
10
+ throw new HitchError(`invalid grader metric: ${field}`, { code: "metric_invalid", exitCode: 12 });
11
+ }
12
+ metrics[name] = value;
13
+ }
14
+ return { primary: metrics[manifest.primary_metric], metrics };
15
+ }
16
+ //# sourceMappingURL=metrics.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/benchmarks/metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,MAAM,UAAU,mBAAmB,CAAC,GAAY,EAAE,QAA6B,EAAE,IAAqB;IACpG,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,GAA+B,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1H,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;YACrK,MAAM,IAAI,UAAU,CAAC,0BAA0B,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACpG,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAE,EAAE,OAAO,EAAE,CAAC;AACjE,CAAC"}
@@ -0,0 +1,13 @@
1
+ import { parse } from "smol-toml";
2
+ import { invalidInput } from "../foundation/index.js";
3
+ /** Native Harbor tasks use multiline strings and nested array tables. Keep TOML
4
+ * parsing separate from protocol and capability validation. */
5
+ export function parseBenchmarkToml(text) {
6
+ try {
7
+ return parse(text);
8
+ }
9
+ catch (error) {
10
+ throw invalidInput(`invalid benchmark TOML: ${error.message}`);
11
+ }
12
+ }
13
+ //# sourceMappingURL=toml.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"toml.js","sourceRoot":"","sources":["../../../src/benchmarks/toml.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAEtD;+DAC+D;AAC/D,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,IAAI,CAAC;QAAC,OAAO,KAAK,CAAC,IAAI,CAA4B,CAAC;IAAC,CAAC;IACtD,OAAO,KAAK,EAAE,CAAC;QAAC,MAAM,YAAY,CAAC,2BAA4B,KAAe,CAAC,OAAO,EAAE,CAAC,CAAC;IAAC,CAAC;AAC9F,CAAC"}
@@ -0,0 +1,242 @@
1
+ import { HitchError, invalidInput } from "../foundation/index.js";
2
+ export const BENCHMARK_CAPABILITIES = new Set([
3
+ "shell", "artifact-export", "separate-verifier", "shared-verifier", "compose", "tool-server@1", "http-json-cli", "hitch-hook@1",
4
+ "model-call@1", "native-image-input", "no-tools", "tool-result-images@1", "native-phases@1",
5
+ ]);
6
+ export function unsupported(message) {
7
+ throw new HitchError(message, { code: "unsupported_feature", exitCode: 2 });
8
+ }
9
+ export function object(value, label) {
10
+ if (!value || typeof value !== "object" || Array.isArray(value))
11
+ throw invalidInput(`${label} must be an object`);
12
+ return value;
13
+ }
14
+ export function fields(value, allowed, label) {
15
+ const result = object(value, label);
16
+ for (const key of Object.keys(result))
17
+ if (!allowed.includes(key))
18
+ unsupported(`${label}.${key} is unsupported`);
19
+ return result;
20
+ }
21
+ export function nonempty(value, label) {
22
+ if (typeof value !== "string" || !value.trim() || /[\0\r\n]/.test(value))
23
+ throw invalidInput(`${label} must be a nonempty string`);
24
+ }
25
+ export function strings(value, label) {
26
+ if (!Array.isArray(value))
27
+ throw invalidInput(`${label} must be an array`);
28
+ value.forEach((item) => nonempty(item, label));
29
+ if (new Set(value).size !== value.length)
30
+ throw invalidInput(`${label} contains duplicates`);
31
+ }
32
+ export function positive(value, label) {
33
+ if (!Number.isSafeInteger(value) || Number(value) < 1)
34
+ throw invalidInput(`${label} must be a positive integer`);
35
+ }
36
+ export function relativePath(value) {
37
+ nonempty(value, "package path");
38
+ if (/[\x00-\x1f\x7f\\:]/.test(value) || value.startsWith("/") || value.split("/").some((part) => ["", ".", ".."].includes(part))) {
39
+ throw invalidInput(`invalid package-relative path: ${value}`);
40
+ }
41
+ return value;
42
+ }
43
+ export function parseManifest(value) {
44
+ const m = fields(value, ["schema_version", "protocol", "id", "release", "task_root", "task_ids", "default_profile", "primary_metric", "task_format", "source", "metrics", "publication", "runtime_components", "extensions"], "manifest");
45
+ if (m.schema_version !== "1" || m.protocol !== "hitch-benchmark@1")
46
+ unsupported("unsupported benchmark protocol");
47
+ for (const key of ["id", "release", "primary_metric"])
48
+ nonempty(m[key], key);
49
+ relativePath(m.task_root);
50
+ relativePath(m.default_profile);
51
+ strings(m.task_ids, "task_ids");
52
+ if (!m.task_ids.length || m.task_ids.some((id) => !/^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/.test(id)))
53
+ throw invalidInput("invalid or empty task membership");
54
+ const dialect = fields(m.task_format, ["name", "schema_version"], "task_format");
55
+ if (dialect.name !== "harbor" || dialect.schema_version !== "1.4")
56
+ unsupported("only Harbor schema 1.4 is supported");
57
+ const source = fields(m.source, ["kind", "path", "uri", "resolved_revision", "license", "access"], "source");
58
+ nonempty(source.license, "source.license");
59
+ if (source.access !== undefined && !["public", "private", "gated"].includes(String(source.access)))
60
+ throw invalidInput("invalid source access");
61
+ if (source.kind === "git") {
62
+ nonempty(source.uri, "source.uri");
63
+ let uri;
64
+ try {
65
+ uri = new URL(source.uri);
66
+ }
67
+ catch {
68
+ throw invalidInput("Git source requires a credential-free HTTPS URI");
69
+ }
70
+ if (uri.protocol !== "https:" || uri.username || uri.password || uri.search || uri.hash || source.path !== undefined)
71
+ throw invalidInput("Git source requires a credential-free HTTPS URI without a host path");
72
+ if (!/^[a-f0-9]{40}$/.test(String(source.resolved_revision)))
73
+ throw invalidInput("Git source requires a full resolved commit");
74
+ }
75
+ else if (source.kind !== "local")
76
+ unsupported("unsupported source kind");
77
+ else if (source.path !== "." || source.uri !== undefined || source.resolved_revision !== undefined)
78
+ throw invalidInput("local source must name package root '.'; its content identity is resolved by Hitch");
79
+ const metrics = object(m.metrics, "metrics");
80
+ if (!Object.hasOwn(metrics, m.primary_metric))
81
+ throw invalidInput("primary_metric must name a declared metric");
82
+ for (const [name, value] of Object.entries(metrics)) {
83
+ nonempty(name, "metric name");
84
+ const metric = fields(value, ["type", "direction", "range", "reducer"], `metrics.${name}`);
85
+ if (!["binary", "scalar"].includes(String(metric.type)) || !["maximize", "minimize"].includes(String(metric.direction)) || metric.reducer !== "task_macro_mean")
86
+ unsupported(`unsupported metric definition: ${name}`);
87
+ if (!Array.isArray(metric.range) || metric.range.length !== 2 || !metric.range.every((v) => typeof v === "number" && Number.isFinite(v)) || metric.range[0] > metric.range[1])
88
+ throw invalidInput(`invalid metric range: ${name}`);
89
+ if (metric.type === "binary" && JSON.stringify(metric.range) !== "[0,1]")
90
+ throw invalidInput("binary metrics require range [0,1]");
91
+ }
92
+ const publication = fields(m.publication, ["track", "training_eligible"], "publication");
93
+ if (!["custom", "public-subset"].includes(String(publication.track)) || publication.training_eligible !== false)
94
+ unsupported("MVP packages require custom/public-subset track and training_eligible=false");
95
+ m.runtime_components ??= [];
96
+ if (!Array.isArray(m.runtime_components))
97
+ throw invalidInput("runtime_components must be an array");
98
+ const ids = new Set();
99
+ for (const value of m.runtime_components) {
100
+ const component = fields(value, ["id", "protocol", "path"], "runtime component");
101
+ nonempty(component.id, "component.id");
102
+ nonempty(component.protocol, "component.protocol");
103
+ relativePath(component.path);
104
+ if (ids.has(component.id))
105
+ throw invalidInput("duplicate runtime component id");
106
+ ids.add(component.id);
107
+ }
108
+ return m;
109
+ }
110
+ export function parseProfile(value) {
111
+ const p = fields(value, ["schema_version", "id", "track", "input_mode", "tool_policy", "budget", "sampling", "grading", "extensions"], "profile");
112
+ if (p.schema_version !== "1" || p.input_mode !== "instruction")
113
+ unsupported("unsupported profile version/input mode");
114
+ nonempty(p.id, "profile.id");
115
+ if (!["custom", "public-subset"].includes(String(p.track)))
116
+ unsupported("unsupported profile track");
117
+ const policy = fields(p.tool_policy, ["id", "allowed", "network", "enforcement"], "tool_policy");
118
+ nonempty(policy.id, "tool_policy.id");
119
+ strings(policy.allowed, "tool_policy.allowed");
120
+ if (policy.network !== "open" || policy.enforcement !== "required")
121
+ unsupported("this backend profile currently supports explicit open network only");
122
+ if (policy.allowed.some((c) => !BENCHMARK_CAPABILITIES.has(c)))
123
+ unsupported("unknown allowed tool capability");
124
+ const budget = fields(p.budget, ["agent_timeout", "setup_timeout_ms", "collection_timeout_ms", "cleanup_grace_ms"], "budget");
125
+ const timeout = fields(budget.agent_timeout, ["source"], "agent_timeout");
126
+ if (timeout.source !== "task")
127
+ unsupported("agent timeout must inherit task");
128
+ for (const key of ["setup_timeout_ms", "collection_timeout_ms", "cleanup_grace_ms"])
129
+ positive(budget[key], key);
130
+ const sampling = fields(p.sampling, ["attempts_per_task", "seed"], "sampling");
131
+ positive(sampling.attempts_per_task, "attempts_per_task");
132
+ if (!Number.isSafeInteger(sampling.seed))
133
+ throw invalidInput("seed must be an integer");
134
+ const grading = fields(p.grading, ["on_agent_budget_exhausted", "on_missing_submission", "infrastructure_retries"], "grading");
135
+ if (grading.on_agent_budget_exhausted !== "grade_final_state" || grading.on_missing_submission !== "error")
136
+ unsupported("unsupported grading policy");
137
+ if (!Number.isSafeInteger(grading.infrastructure_retries) || Number(grading.infrastructure_retries) < 0)
138
+ throw invalidInput("invalid grading retries");
139
+ return p;
140
+ }
141
+ export function parseTask(value, manifest) {
142
+ const t = fields(value, ["schema_version", "source_task_id", "driver", "requirements", "lifecycle", "submission", "grading", "extensions"], "task");
143
+ if (t.schema_version !== "1")
144
+ unsupported("unsupported task extension version");
145
+ nonempty(t.source_task_id, "source_task_id");
146
+ strings(t.requirements, "requirements");
147
+ for (const cap of t.requirements)
148
+ if (!BENCHMARK_CAPABILITIES.has(cap))
149
+ unsupported(`required capability unavailable: ${cap}`);
150
+ const driver = fields(t.driver, ["kind", "protocol_version", "config"], "driver");
151
+ if (!["tool-server", "terminal", "model-call"].includes(String(driver.kind)) || driver.protocol_version !== "1")
152
+ unsupported("unsupported driver protocol");
153
+ const required = driver.kind === "model-call" ? ["model-call@1", "native-image-input", "no-tools", "artifact-export", "separate-verifier"] : driver.kind === "terminal" ? ["shell"] : ["shell", "tool-server@1", "http-json-cli", "hitch-hook@1", "separate-verifier", "compose", "artifact-export"];
154
+ for (const cap of required)
155
+ if (!t.requirements.includes(cap))
156
+ throw invalidInput(`${driver.kind} driver requires capability ${cap}`);
157
+ if (t.requirements.includes("tool-result-images@1") && (driver.kind !== "tool-server" || !t.requirements.includes("native-image-input")))
158
+ throw invalidInput("tool-result-images@1 requires a tool-server driver and native-image-input");
159
+ if (driver.kind !== "tool-server") {
160
+ const config = fields(driver.config, driver.kind === "model-call" ? ["input"] : [], "driver.config");
161
+ if (driver.kind === "model-call")
162
+ relativePath(config.input);
163
+ if (driver.kind !== "model-call" && t.requirements.includes("no-tools"))
164
+ unsupported("only the trusted model-call runner enforces no-tools");
165
+ fields(t.lifecycle, [], "native lifecycle (Harbor owns the lifecycle)");
166
+ }
167
+ else {
168
+ const config = fields(driver.config, ["transport", "endpoint", "schema", "service", "native_phases"], "driver.config");
169
+ if (config.transport !== "http-json-cli")
170
+ unsupported("unsupported tool-server transport");
171
+ nonempty(config.service, "tool service");
172
+ if (!/^[a-z][a-z0-9_-]*$/.test(config.service) || config.service === "main")
173
+ throw invalidInput("tool service must be a sidecar");
174
+ nonempty(config.endpoint, "tool endpoint");
175
+ const url = new URL(config.endpoint);
176
+ if (url.protocol !== "http:" || url.hostname !== config.service || url.username || url.password || url.search || url.hash || url.pathname !== "/")
177
+ throw invalidInput("tool endpoint must target the declared isolated Compose service root");
178
+ relativePath(config.schema);
179
+ if (config.native_phases !== undefined) {
180
+ const phases = fields(config.native_phases, ["protocol", "argv", "audit_path", "shutdown_timeout_ms", "finalization_timeout_ms"], "native_phases");
181
+ if (!["hitch-native-phase-control@1", "hitch-native-phase-control@2"].includes(String(phases.protocol)))
182
+ unsupported("unsupported native phase protocol");
183
+ if (phases.protocol === "hitch-native-phase-control@2")
184
+ positive(phases.finalization_timeout_ms, "native finalization timeout");
185
+ else if (phases.finalization_timeout_ms !== undefined)
186
+ throw invalidInput("native finalization requires control protocol v2");
187
+ if (!Array.isArray(phases.argv) || !phases.argv.length || phases.argv.length > 32)
188
+ throw invalidInput("invalid native phase controller argv");
189
+ phases.argv.forEach(arg => { nonempty(arg, "native phase argv"); if (arg.length > 8192)
190
+ throw invalidInput("native phase argv too long"); });
191
+ positive(phases.shutdown_timeout_ms, "phase shutdown timeout");
192
+ if (phases.shutdown_timeout_ms > 600000)
193
+ throw invalidInput("native phase shutdown allowance too large");
194
+ nonempty(phases.audit_path, "native phase audit path");
195
+ if (!phases.audit_path.startsWith("/") || phases.audit_path.slice(1).split("/").some(p => !p || p === "." || p === "..") || /[\\\x00-\x1f\x7f]/.test(phases.audit_path))
196
+ throw invalidInput("invalid native phase audit path");
197
+ for (const cap of ["native-phases@1", "native-image-input", "tool-result-images@1"])
198
+ if (!t.requirements.includes(cap))
199
+ throw invalidInput(`native phases require ${cap}`);
200
+ }
201
+ const lifecycle = fields(t.lifecycle, ["prepare", "quiesce", "snapshot", "cleanup"], "lifecycle");
202
+ for (const phase of ["prepare", "quiesce", "snapshot", "cleanup"]) {
203
+ const hook = fields(lifecycle[phase], ["protocol", "target", "argv", "timeout_ms"], `hook.${phase}`);
204
+ if (hook.protocol !== "hitch-hook@1" || hook.target !== `environment:${config.service}`)
205
+ unsupported(`unsupported hook target/protocol: ${phase}`);
206
+ // Repeated argv values are valid, unlike membership lists.
207
+ if (!Array.isArray(hook.argv) || !hook.argv.length)
208
+ throw invalidInput(`missing hook argv: ${phase}`);
209
+ hook.argv.forEach((arg) => nonempty(arg, "hook argv"));
210
+ positive(hook.timeout_ms, "hook timeout");
211
+ }
212
+ }
213
+ if (t.requirements.includes("native-phases@1") && (driver.kind !== "tool-server" || !driver.config.native_phases))
214
+ throw invalidInput("native-phases@1 requires a native phase controller");
215
+ const submission = fields(t.submission, ["kind", "paths", "max_bytes", "final_response"], "submission");
216
+ if (submission.kind !== "artifacts" && !(driver.kind === "terminal" && submission.kind === "environment"))
217
+ unsupported("unsupported submission");
218
+ strings(submission.paths, "submission.paths");
219
+ positive(submission.max_bytes, "submission.max_bytes");
220
+ if ((submission.kind === "artifacts" && !submission.paths.length) || submission.paths.some((p) => !p.startsWith("/") || p.split("/").includes("..")))
221
+ throw invalidInput("submission paths must be absolute safe container paths");
222
+ if (submission.kind === "environment" && submission.paths.length)
223
+ throw invalidInput("environment submission uses the upstream verifier's live workspace, not artifact paths");
224
+ if (driver.kind === "tool-server") {
225
+ const phases = driver.config.native_phases;
226
+ if (phases && !submission.paths.some(p => phases.audit_path === p || phases.audit_path.startsWith(p.replace(/\/$/, "") + "/")))
227
+ throw invalidInput("native audit must be in a declared controller snapshot");
228
+ }
229
+ if (submission.final_response !== undefined && (submission.final_response !== "/hitch-evidence/final-response.json" || !submission.paths.includes(submission.final_response) || driver.kind === "tool-server"))
230
+ throw invalidInput("invalid final response export");
231
+ if (driver.kind === "model-call" && !submission.final_response)
232
+ throw invalidInput("model-call requires canonical final response export");
233
+ const grading = fields(t.grading, ["kind", "entrypoint", "metric_map"], "grading");
234
+ if ((grading.kind !== "command" && !(driver.kind === "terminal" && grading.kind === "harbor")) || JSON.stringify(grading.entrypoint) !== '["bash","/tests/test.sh"]')
235
+ unsupported("unsupported grader command");
236
+ const mapping = object(grading.metric_map, "metric_map");
237
+ if (Object.keys(mapping).sort().join("\0") !== Object.keys(manifest.metrics).sort().join("\0"))
238
+ throw invalidInput("metric_map must cover exactly the declared metrics");
239
+ Object.values(mapping).forEach((v) => nonempty(v, "metric mapping"));
240
+ return t;
241
+ }
242
+ //# sourceMappingURL=validation.js.map