agent-hitch 0.2.8 → 0.2.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (414) hide show
  1. package/README.md +206 -137
  2. package/README.zh-CN.md +193 -119
  3. package/dist/scripts/canary-native-training.js +221 -0
  4. package/dist/scripts/canary-native-training.js.map +1 -0
  5. package/dist/scripts/canary-remote-training.js +246 -0
  6. package/dist/scripts/canary-remote-training.js.map +1 -0
  7. package/dist/scripts/canary-remote-verifier.js +124 -0
  8. package/dist/scripts/canary-remote-verifier.js.map +1 -0
  9. package/dist/scripts/canary-worker-host.js +113 -0
  10. package/dist/scripts/canary-worker-host.js.map +1 -0
  11. package/dist/scripts/canary-worker-peer.js +188 -0
  12. package/dist/scripts/canary-worker-peer.js.map +1 -0
  13. package/dist/scripts/check-architecture.js +4 -3
  14. package/dist/scripts/check-architecture.js.map +1 -1
  15. package/dist/src/adapters/catalog.js +2 -1
  16. package/dist/src/adapters/catalog.js.map +1 -1
  17. package/dist/src/adapters/providers/codex.js +65 -2
  18. package/dist/src/adapters/providers/codex.js.map +1 -1
  19. package/dist/src/adapters/providers/model-call.js +17 -1
  20. package/dist/src/adapters/providers/model-call.js.map +1 -1
  21. package/dist/src/adapters/providers/training-tool.js +29 -0
  22. package/dist/src/adapters/providers/training-tool.js.map +1 -0
  23. package/dist/src/artifacts/index.js +1 -1
  24. package/dist/src/artifacts/index.js.map +1 -1
  25. package/dist/src/artifacts/store.js +15 -10
  26. package/dist/src/artifacts/store.js.map +1 -1
  27. package/dist/src/backends/harbor/backend.js +24 -15
  28. package/dist/src/backends/harbor/backend.js.map +1 -1
  29. package/dist/src/backends/harbor/bridge-environment.js +12 -0
  30. package/dist/src/backends/harbor/bridge-environment.js.map +1 -0
  31. package/dist/src/backends/harbor/environment-config.js +2 -2
  32. package/dist/src/backends/harbor/environment-config.js.map +1 -1
  33. package/dist/src/backends/harbor/host-credential-helper.js +81 -0
  34. package/dist/src/backends/harbor/host-credential-helper.js.map +1 -0
  35. package/dist/src/backends/harbor/index.js +3 -0
  36. package/dist/src/backends/harbor/index.js.map +1 -1
  37. package/dist/src/backends/harbor/model-proxy-config.js +18 -1
  38. package/dist/src/backends/harbor/model-proxy-config.js.map +1 -1
  39. package/dist/src/backends/harbor/observation.js +61 -0
  40. package/dist/src/backends/harbor/observation.js.map +1 -0
  41. package/dist/src/backends/harbor/portable-regrade.js +150 -0
  42. package/dist/src/backends/harbor/portable-regrade.js.map +1 -0
  43. package/dist/src/backends/harbor/process.js +26 -5
  44. package/dist/src/backends/harbor/process.js.map +1 -1
  45. package/dist/src/backends/harbor/regrade.js +4 -1
  46. package/dist/src/backends/harbor/regrade.js.map +1 -1
  47. package/dist/src/backends/harbor/supervisor.js +22 -0
  48. package/dist/src/backends/harbor/supervisor.js.map +1 -1
  49. package/dist/src/backends/harbor/tools.js +2 -0
  50. package/dist/src/backends/harbor/tools.js.map +1 -1
  51. package/dist/src/backends/index.js +3 -0
  52. package/dist/src/backends/index.js.map +1 -1
  53. package/dist/src/cli/arguments.js +33 -0
  54. package/dist/src/cli/arguments.js.map +1 -1
  55. package/dist/src/cli/commands/capabilities.js +14 -1
  56. package/dist/src/cli/commands/capabilities.js.map +1 -1
  57. package/dist/src/cli/commands/daemon.js +19 -0
  58. package/dist/src/cli/commands/daemon.js.map +1 -1
  59. package/dist/src/cli/commands/eval.js +38 -10
  60. package/dist/src/cli/commands/eval.js.map +1 -1
  61. package/dist/src/cli/commands/local.js +130 -0
  62. package/dist/src/cli/commands/local.js.map +1 -0
  63. package/dist/src/cli/commands/model-node.js +26 -0
  64. package/dist/src/cli/commands/model-node.js.map +1 -0
  65. package/dist/src/cli/commands/models.js +72 -0
  66. package/dist/src/cli/commands/models.js.map +1 -0
  67. package/dist/src/cli/commands/run.js +36 -1
  68. package/dist/src/cli/commands/run.js.map +1 -1
  69. package/dist/src/cli/commands/training.js +32 -0
  70. package/dist/src/cli/commands/training.js.map +1 -0
  71. package/dist/src/cli/commands/worker.js +30 -2
  72. package/dist/src/cli/commands/worker.js.map +1 -1
  73. package/dist/src/cli/main.js +8 -0
  74. package/dist/src/cli/main.js.map +1 -1
  75. package/dist/src/cli/output.js +24 -2
  76. package/dist/src/cli/output.js.map +1 -1
  77. package/dist/src/control-plane/eval-records.js +2 -0
  78. package/dist/src/control-plane/eval-records.js.map +1 -1
  79. package/dist/src/control-plane/eval-recovery.js +7 -1
  80. package/dist/src/control-plane/eval-recovery.js.map +1 -1
  81. package/dist/src/control-plane/eval-scheduler.js +24 -50
  82. package/dist/src/control-plane/eval-scheduler.js.map +1 -1
  83. package/dist/src/control-plane/eval-submission-persist.js +30 -0
  84. package/dist/src/control-plane/eval-submission-persist.js.map +1 -0
  85. package/dist/src/control-plane/execution-observation-contract.js +60 -0
  86. package/dist/src/control-plane/execution-observation-contract.js.map +1 -0
  87. package/dist/src/control-plane/index.js +10 -0
  88. package/dist/src/control-plane/index.js.map +1 -1
  89. package/dist/src/control-plane/inference-manager.js +357 -0
  90. package/dist/src/control-plane/inference-manager.js.map +1 -0
  91. package/dist/src/control-plane/managed-gateway-receipt.js +26 -0
  92. package/dist/src/control-plane/managed-gateway-receipt.js.map +1 -0
  93. package/dist/src/control-plane/managed-service-recovery.js +158 -0
  94. package/dist/src/control-plane/managed-service-recovery.js.map +1 -0
  95. package/dist/src/control-plane/model-capture-planning.js +15 -1
  96. package/dist/src/control-plane/model-capture-planning.js.map +1 -1
  97. package/dist/src/control-plane/ordered-eval-control.js +221 -0
  98. package/dist/src/control-plane/ordered-eval-control.js.map +1 -0
  99. package/dist/src/control-plane/remote-backend-result.js +18 -0
  100. package/dist/src/control-plane/remote-backend-result.js.map +1 -0
  101. package/dist/src/control-plane/remote-execution-lease.js +15 -0
  102. package/dist/src/control-plane/remote-execution-lease.js.map +1 -0
  103. package/dist/src/control-plane/remote-execution-observation.js +108 -0
  104. package/dist/src/control-plane/remote-execution-observation.js.map +1 -0
  105. package/dist/src/control-plane/remote-model-routes.js +107 -0
  106. package/dist/src/control-plane/remote-model-routes.js.map +1 -0
  107. package/dist/src/control-plane/remote-observation-reporter.js +37 -0
  108. package/dist/src/control-plane/remote-observation-reporter.js.map +1 -0
  109. package/dist/src/control-plane/remote-offer-contract.js +48 -0
  110. package/dist/src/control-plane/remote-offer-contract.js.map +1 -0
  111. package/dist/src/control-plane/remote-rerun-journal.js +61 -0
  112. package/dist/src/control-plane/remote-rerun-journal.js.map +1 -0
  113. package/dist/src/control-plane/remote-result-transport.js +55 -9
  114. package/dist/src/control-plane/remote-result-transport.js.map +1 -1
  115. package/dist/src/control-plane/remote-verifier-import.js +52 -0
  116. package/dist/src/control-plane/remote-verifier-import.js.map +1 -0
  117. package/dist/src/control-plane/remote-verifier-result.js +61 -0
  118. package/dist/src/control-plane/remote-verifier-result.js.map +1 -0
  119. package/dist/src/control-plane/remote-verifier-source-transport.js +68 -0
  120. package/dist/src/control-plane/remote-verifier-source-transport.js.map +1 -0
  121. package/dist/src/control-plane/remote-work-coordinator.js +103 -47
  122. package/dist/src/control-plane/remote-work-coordinator.js.map +1 -1
  123. package/dist/src/control-plane/remote-work-inputs.js +26 -5
  124. package/dist/src/control-plane/remote-work-inputs.js.map +1 -1
  125. package/dist/src/control-plane/remote-work-recovery.js +123 -34
  126. package/dist/src/control-plane/remote-work-recovery.js.map +1 -1
  127. package/dist/src/control-plane/remote-worker-artifacts.js +10 -7
  128. package/dist/src/control-plane/remote-worker-artifacts.js.map +1 -1
  129. package/dist/src/control-plane/remote-worker-cleanup.js +104 -0
  130. package/dist/src/control-plane/remote-worker-cleanup.js.map +1 -0
  131. package/dist/src/control-plane/remote-worker-client.js +156 -14
  132. package/dist/src/control-plane/remote-worker-client.js.map +1 -1
  133. package/dist/src/control-plane/remote-worker-codecs.js +160 -0
  134. package/dist/src/control-plane/remote-worker-codecs.js.map +1 -0
  135. package/dist/src/control-plane/remote-worker-generation-recovery.js +67 -0
  136. package/dist/src/control-plane/remote-worker-generation-recovery.js.map +1 -0
  137. package/dist/src/control-plane/remote-worker-ownership.js +51 -0
  138. package/dist/src/control-plane/remote-worker-ownership.js.map +1 -0
  139. package/dist/src/control-plane/remote-worker-protocol.js +85 -119
  140. package/dist/src/control-plane/remote-worker-protocol.js.map +1 -1
  141. package/dist/src/control-plane/remote-worker-runner.js +95 -18
  142. package/dist/src/control-plane/remote-worker-runner.js.map +1 -1
  143. package/dist/src/control-plane/remote-workers.js +33 -5
  144. package/dist/src/control-plane/remote-workers.js.map +1 -1
  145. package/dist/src/control-plane/rerun-recovery.js +89 -0
  146. package/dist/src/control-plane/rerun-recovery.js.map +1 -0
  147. package/dist/src/control-plane/rerun-scheduler.js +70 -96
  148. package/dist/src/control-plane/rerun-scheduler.js.map +1 -1
  149. package/dist/src/control-plane/rerun-source.js +57 -0
  150. package/dist/src/control-plane/rerun-source.js.map +1 -0
  151. package/dist/src/control-plane/worker-model-relay.js +76 -0
  152. package/dist/src/control-plane/worker-model-relay.js.map +1 -0
  153. package/dist/src/controller-runtime/hash.js +3 -0
  154. package/dist/src/controller-runtime/hash.js.map +1 -1
  155. package/dist/src/controller-runtime/index.js +2 -0
  156. package/dist/src/controller-runtime/index.js.map +1 -1
  157. package/dist/src/controller-runtime/observation.js +29 -0
  158. package/dist/src/controller-runtime/observation.js.map +1 -0
  159. package/dist/src/controller-runtime/resident.js +22 -0
  160. package/dist/src/controller-runtime/resident.js.map +1 -0
  161. package/dist/src/daemon/auth.js +7 -2
  162. package/dist/src/daemon/auth.js.map +1 -1
  163. package/dist/src/daemon/client.js +34 -1
  164. package/dist/src/daemon/client.js.map +1 -1
  165. package/dist/src/daemon/execution-observation.js +38 -0
  166. package/dist/src/daemon/execution-observation.js.map +1 -0
  167. package/dist/src/daemon/http.js +57 -0
  168. package/dist/src/daemon/http.js.map +1 -0
  169. package/dist/src/daemon/inference-prepare.js +47 -0
  170. package/dist/src/daemon/inference-prepare.js.map +1 -0
  171. package/dist/src/daemon/scheduler.js +4 -1
  172. package/dist/src/daemon/scheduler.js.map +1 -1
  173. package/dist/src/daemon/server.js +80 -82
  174. package/dist/src/daemon/server.js.map +1 -1
  175. package/dist/src/daemon/worker-observation.js +34 -0
  176. package/dist/src/daemon/worker-observation.js.map +1 -0
  177. package/dist/src/daemon/worker-ownership.js +71 -0
  178. package/dist/src/daemon/worker-ownership.js.map +1 -0
  179. package/dist/src/daemon/worker-routes.js +83 -15
  180. package/dist/src/daemon/worker-routes.js.map +1 -1
  181. package/dist/src/domain/execution-observation.js +2 -0
  182. package/dist/src/domain/execution-observation.js.map +1 -0
  183. package/dist/src/domain/index.js +1 -0
  184. package/dist/src/domain/index.js.map +1 -1
  185. package/dist/src/domain/inference.js +2 -0
  186. package/dist/src/domain/inference.js.map +1 -0
  187. package/dist/src/domain/runs.js.map +1 -1
  188. package/dist/src/domain/training.js +2 -0
  189. package/dist/src/domain/training.js.map +1 -0
  190. package/dist/src/domain/validation.js +11 -0
  191. package/dist/src/domain/validation.js.map +1 -1
  192. package/dist/src/domain/verifier-source.js +2 -0
  193. package/dist/src/domain/verifier-source.js.map +1 -0
  194. package/dist/src/domain/worker-recovery.js +2 -0
  195. package/dist/src/domain/worker-recovery.js.map +1 -0
  196. package/dist/src/evals/eval-logical-plan.js +8 -1
  197. package/dist/src/evals/eval-logical-plan.js.map +1 -1
  198. package/dist/src/evals/execution-leases.js +15 -2
  199. package/dist/src/evals/execution-leases.js.map +1 -1
  200. package/dist/src/evals/execution-plan.js +11 -1
  201. package/dist/src/evals/execution-plan.js.map +1 -1
  202. package/dist/src/evals/harbor-bridge-error.js +9 -0
  203. package/dist/src/evals/harbor-bridge-error.js.map +1 -1
  204. package/dist/src/evals/index.js +19 -0
  205. package/dist/src/evals/index.js.map +1 -1
  206. package/dist/src/evals/model-capture-plan.js +12 -0
  207. package/dist/src/evals/model-capture-plan.js.map +1 -1
  208. package/dist/src/evals/model-capture-runtime.js +25 -0
  209. package/dist/src/evals/model-capture-runtime.js.map +1 -1
  210. package/dist/src/evals/physical-work-plan.js +57 -0
  211. package/dist/src/evals/physical-work-plan.js.map +1 -0
  212. package/dist/src/evals/preparation-rerun.js +6 -0
  213. package/dist/src/evals/preparation-rerun.js.map +1 -1
  214. package/dist/src/evals/recovery.js +8 -1
  215. package/dist/src/evals/recovery.js.map +1 -1
  216. package/dist/src/evals/regrade-evidence.js +4 -4
  217. package/dist/src/evals/regrade-evidence.js.map +1 -1
  218. package/dist/src/evals/remote-execution-ownership.js +50 -0
  219. package/dist/src/evals/remote-execution-ownership.js.map +1 -0
  220. package/dist/src/evals/remote-infrastructure-retry.js +2 -0
  221. package/dist/src/evals/remote-infrastructure-retry.js.map +1 -1
  222. package/dist/src/evals/remote-lease-release.js +58 -0
  223. package/dist/src/evals/remote-lease-release.js.map +1 -0
  224. package/dist/src/evals/remote-rerun-execution.js +85 -0
  225. package/dist/src/evals/remote-rerun-execution.js.map +1 -0
  226. package/dist/src/evals/remote-trial.js +6 -0
  227. package/dist/src/evals/remote-trial.js.map +1 -0
  228. package/dist/src/evals/remote-verifier-assessment.js +85 -0
  229. package/dist/src/evals/remote-verifier-assessment.js.map +1 -0
  230. package/dist/src/evals/remote-verifier-contract.js +43 -0
  231. package/dist/src/evals/remote-verifier-contract.js.map +1 -0
  232. package/dist/src/evals/remote-verifier-execution.js +105 -0
  233. package/dist/src/evals/remote-verifier-execution.js.map +1 -0
  234. package/dist/src/evals/remote-verifier-journal.js +21 -0
  235. package/dist/src/evals/remote-verifier-journal.js.map +1 -0
  236. package/dist/src/evals/remote-verifier-outcome.js +51 -0
  237. package/dist/src/evals/remote-verifier-outcome.js.map +1 -0
  238. package/dist/src/evals/remote-verifier-rerun.js +134 -0
  239. package/dist/src/evals/remote-verifier-rerun.js.map +1 -0
  240. package/dist/src/evals/remote-verifier-selection.js +61 -0
  241. package/dist/src/evals/remote-verifier-selection.js.map +1 -0
  242. package/dist/src/evals/remote-verifier-source.js +33 -0
  243. package/dist/src/evals/remote-verifier-source.js.map +1 -0
  244. package/dist/src/evals/remote-worker-cleanup.js +46 -0
  245. package/dist/src/evals/remote-worker-cleanup.js.map +1 -0
  246. package/dist/src/evals/request.js +58 -1
  247. package/dist/src/evals/request.js.map +1 -1
  248. package/dist/src/evals/rerun-completion.js +39 -0
  249. package/dist/src/evals/rerun-completion.js.map +1 -0
  250. package/dist/src/evals/rerun-inputs.js +54 -0
  251. package/dist/src/evals/rerun-inputs.js.map +1 -1
  252. package/dist/src/evals/rerun-resume.js +70 -0
  253. package/dist/src/evals/rerun-resume.js.map +1 -0
  254. package/dist/src/evals/rerun-state.js +25 -0
  255. package/dist/src/evals/rerun-state.js.map +1 -0
  256. package/dist/src/evals/rerun-types.js.map +1 -1
  257. package/dist/src/evals/rerun.js +94 -101
  258. package/dist/src/evals/rerun.js.map +1 -1
  259. package/dist/src/evals/service.js +59 -5
  260. package/dist/src/evals/service.js.map +1 -1
  261. package/dist/src/evals/verifier-only-rerun.js +2 -16
  262. package/dist/src/evals/verifier-only-rerun.js.map +1 -1
  263. package/dist/src/evals/verifier-rerun-result.js +24 -0
  264. package/dist/src/evals/verifier-rerun-result.js.map +1 -0
  265. package/dist/src/evals/verifier-source-import.js +48 -0
  266. package/dist/src/evals/verifier-source-import.js.map +1 -0
  267. package/dist/src/evals/verifier-source.js +139 -0
  268. package/dist/src/evals/verifier-source.js.map +1 -0
  269. package/dist/src/foundation/base64.js +19 -0
  270. package/dist/src/foundation/base64.js.map +1 -0
  271. package/dist/src/foundation/config.js +10 -0
  272. package/dist/src/foundation/config.js.map +1 -1
  273. package/dist/src/foundation/credential-redaction.js +22 -9
  274. package/dist/src/foundation/credential-redaction.js.map +1 -1
  275. package/dist/src/foundation/index.js +1 -0
  276. package/dist/src/foundation/index.js.map +1 -1
  277. package/dist/src/foundation/process-identity.js +13 -9
  278. package/dist/src/foundation/process-identity.js.map +1 -1
  279. package/dist/src/inference/daemon-capacity.js +41 -0
  280. package/dist/src/inference/daemon-capacity.js.map +1 -0
  281. package/dist/src/inference/device-reservation.js +58 -0
  282. package/dist/src/inference/device-reservation.js.map +1 -0
  283. package/dist/src/inference/doctor.js +95 -0
  284. package/dist/src/inference/doctor.js.map +1 -0
  285. package/dist/src/inference/generation-recovery.js +83 -0
  286. package/dist/src/inference/generation-recovery.js.map +1 -0
  287. package/dist/src/inference/index.js +24 -0
  288. package/dist/src/inference/index.js.map +1 -0
  289. package/dist/src/inference/lock.js +331 -0
  290. package/dist/src/inference/lock.js.map +1 -0
  291. package/dist/src/inference/managed-launcher.js +35 -0
  292. package/dist/src/inference/managed-launcher.js.map +1 -0
  293. package/dist/src/inference/manifest.js +262 -0
  294. package/dist/src/inference/manifest.js.map +1 -0
  295. package/dist/src/inference/materialize.js +66 -0
  296. package/dist/src/inference/materialize.js.map +1 -0
  297. package/dist/src/inference/model-location.js +34 -0
  298. package/dist/src/inference/model-location.js.map +1 -0
  299. package/dist/src/inference/model-store.js +439 -0
  300. package/dist/src/inference/model-store.js.map +1 -0
  301. package/dist/src/inference/node-client.js +167 -0
  302. package/dist/src/inference/node-client.js.map +1 -0
  303. package/dist/src/inference/node-registry.js +79 -0
  304. package/dist/src/inference/node-registry.js.map +1 -0
  305. package/dist/src/inference/observation.js +79 -0
  306. package/dist/src/inference/observation.js.map +1 -0
  307. package/dist/src/inference/preflight.js +132 -0
  308. package/dist/src/inference/preflight.js.map +1 -0
  309. package/dist/src/inference/process-sglang.js +169 -0
  310. package/dist/src/inference/process-sglang.js.map +1 -0
  311. package/dist/src/inference/protocol-probe.js +29 -0
  312. package/dist/src/inference/protocol-probe.js.map +1 -0
  313. package/dist/src/inference/remote-model-store.js +24 -0
  314. package/dist/src/inference/remote-model-store.js.map +1 -0
  315. package/dist/src/inference/runtime-catalog.js +41 -0
  316. package/dist/src/inference/runtime-catalog.js.map +1 -0
  317. package/dist/src/inference/runtime-store.js +93 -0
  318. package/dist/src/inference/runtime-store.js.map +1 -0
  319. package/dist/src/inference/service-attachment.js +46 -0
  320. package/dist/src/inference/service-attachment.js.map +1 -0
  321. package/dist/src/inference/service-inspection.js +38 -0
  322. package/dist/src/inference/service-inspection.js.map +1 -0
  323. package/dist/src/inference/sglang.js +279 -0
  324. package/dist/src/inference/sglang.js.map +1 -0
  325. package/dist/src/inference/supervisor.js +460 -0
  326. package/dist/src/inference/supervisor.js.map +1 -0
  327. package/dist/src/model-access/environment.js +28 -0
  328. package/dist/src/model-access/environment.js.map +1 -0
  329. package/dist/src/model-access/index.js +4 -0
  330. package/dist/src/model-access/index.js.map +1 -1
  331. package/dist/src/model-access/local-gateway-request.js +139 -0
  332. package/dist/src/model-access/local-gateway-request.js.map +1 -0
  333. package/dist/src/model-access/local-gateway.js +323 -0
  334. package/dist/src/model-access/local-gateway.js.map +1 -0
  335. package/dist/src/model-access/proxy.js +90 -5
  336. package/dist/src/model-access/proxy.js.map +1 -1
  337. package/dist/src/model-access/remote-model.js +58 -0
  338. package/dist/src/model-access/remote-model.js.map +1 -0
  339. package/dist/src/model-access/training.js +115 -0
  340. package/dist/src/model-access/training.js.map +1 -0
  341. package/dist/src/runs/adapter-process.js +2 -2
  342. package/dist/src/runs/adapter-process.js.map +1 -1
  343. package/dist/src/runs/executor-types.js +2 -0
  344. package/dist/src/runs/executor-types.js.map +1 -0
  345. package/dist/src/runs/executor.js +54 -34
  346. package/dist/src/runs/executor.js.map +1 -1
  347. package/dist/src/runs/index.js +1 -0
  348. package/dist/src/runs/index.js.map +1 -1
  349. package/dist/src/runs/local-inference-environment.js +59 -0
  350. package/dist/src/runs/local-inference-environment.js.map +1 -0
  351. package/dist/src/runs/local-inference-run.js +53 -0
  352. package/dist/src/runs/local-inference-run.js.map +1 -0
  353. package/dist/src/runs/outcome.js +22 -0
  354. package/dist/src/runs/outcome.js.map +1 -1
  355. package/dist/src/runs/records.js +9 -0
  356. package/dist/src/runs/records.js.map +1 -1
  357. package/dist/src/runs/request.js +58 -3
  358. package/dist/src/runs/request.js.map +1 -1
  359. package/dist/src/runs/training-evidence.js +22 -0
  360. package/dist/src/runs/training-evidence.js.map +1 -0
  361. package/dist/src/runs/training-runtime.js +20 -0
  362. package/dist/src/runs/training-runtime.js.map +1 -0
  363. package/dist/src/workers/execution-observation.js +32 -0
  364. package/dist/src/workers/execution-observation.js.map +1 -0
  365. package/dist/src/workers/index.js +2 -0
  366. package/dist/src/workers/index.js.map +1 -1
  367. package/dist/src/workers/remote-harbor-ownership.js +266 -0
  368. package/dist/src/workers/remote-harbor-ownership.js.map +1 -0
  369. package/dist/src/workers/remote-harbor-verifier.js +76 -0
  370. package/dist/src/workers/remote-harbor-verifier.js.map +1 -0
  371. package/dist/src/workers/remote-harbor-work-spec.js +66 -10
  372. package/dist/src/workers/remote-harbor-work-spec.js.map +1 -1
  373. package/dist/src/workers/remote-harbor-worker.js +56 -49
  374. package/dist/src/workers/remote-harbor-worker.js.map +1 -1
  375. package/dist/src/workers/remote-worker-host.js +33 -0
  376. package/dist/src/workers/remote-worker-host.js.map +1 -0
  377. package/docs/local-model-inference-spec.zh-CN.md +666 -0
  378. package/docs/schemas/controller-runtime-observation.schema.json +21 -0
  379. package/docs/schemas/eval-execution-plan.schema.json +1 -0
  380. package/docs/schemas/eval-request.schema.json +3 -1
  381. package/docs/schemas/execution-lease.schema.json +95 -1
  382. package/docs/schemas/execution-provider-status.schema.json +162 -28
  383. package/docs/schemas/inference-execution-evidence.schema.json +487 -0
  384. package/docs/schemas/inference-lock.schema.json +487 -0
  385. package/docs/schemas/inference-runtime-manifest.schema.json +156 -0
  386. package/docs/schemas/inference-service-record.schema.json +227 -0
  387. package/docs/schemas/local-inference-selection.schema.json +98 -0
  388. package/docs/schemas/local-model-manifest.schema.json +43 -0
  389. package/docs/schemas/model-node-binding.schema.json +34 -0
  390. package/docs/schemas/model-node-generation-release.schema.json +46 -0
  391. package/docs/schemas/ordered-eval-control.schema.json +39 -0
  392. package/docs/schemas/remote-execution-lease-grant.schema.json +45 -0
  393. package/docs/schemas/remote-harbor-work-spec.schema.json +325 -0
  394. package/docs/schemas/remote-model-binding.schema.json +21 -0
  395. package/docs/schemas/remote-physical-execution.schema.json +108 -0
  396. package/docs/schemas/remote-result-envelope.schema.json +9 -3
  397. package/docs/schemas/remote-tree-envelope.schema.json +17 -3
  398. package/docs/schemas/remote-verifier-result.schema.json +203 -0
  399. package/docs/schemas/remote-verifier-source.schema.json +52 -0
  400. package/docs/schemas/remote-verifier-work.schema.json +125 -0
  401. package/docs/schemas/remote-work-offer.schema.json +170 -34
  402. package/docs/schemas/remote-worker-execution-admission.schema.json +214 -0
  403. package/docs/schemas/remote-worker-generation-cleanup.schema.json +199 -0
  404. package/docs/schemas/remote-worker-registration.schema.json +226 -25
  405. package/docs/schemas/run-manifest.schema.json +211 -44
  406. package/docs/schemas/run-request.schema.json +2 -1
  407. package/docs/schemas/training-external-binding.schema.json +26 -0
  408. package/integrations/harbor/hitch_harbor_agent.py +366 -38
  409. package/integrations/harbor/hitch_harbor_environment.py +22 -1
  410. package/integrations/harbor/hitch_host_credentials.py +354 -0
  411. package/integrations/harbor/hitch_private_exec.py +150 -0
  412. package/integrations/model-call/cli.js +13 -5
  413. package/integrations/training-tool/cli.js +72 -0
  414. package/package.json +8 -2
@@ -0,0 +1 @@
1
+ {"version":3,"file":"canary-native-training.js","sourceRoot":"","sources":["../../scripts/canary-native-training.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC/G,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAEjG,2FAA2F;AAC3F,+FAA+F;AAC/F,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;AACpD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AACtD,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,EAAE,yFAAyF,CAAC,CAAC;AACtH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,+BAA+B,CAAC,CAAC,CAAC;AACjF,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC;AACtD,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;AACjE,MAAM,CAAC,EAAE,CAAC,aAAa,EAAE,oCAAoC,CAAC,CAAC;AAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAA4D,CAAC;AAC/H,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,8BAA8B,CAAC,CAAC;AAC7D,MAAM,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;AACrD,MAAM,CAAC,EAAE,CAAC,YAAY,IAAI,MAAM,EAAE,gEAAgE,CAAC,CAAC;AACpG,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAA+D,CAAC;AAChI,MAAM,OAAO,GAAG,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;AACvD,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,CAAC;AACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AAC5C,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACvF,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;AAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;AAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;AACrF,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACnG,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACpF,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,CAAC;AACpF,OAAO,GAAG,CAAC,gCAAgC,CAAC;AAC5C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AAC3H,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAqD,CAAC;AACnG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;AACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,QAAQ,CAAC,CAAC;AAC/C,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;AACzE,MAAM,OAAO,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,wEAAwE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACvH,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAChE,MAAM,iBAAiB,GAAG,MAAM,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,iBAAiB,EAAE,MAAM,IAAI,WAAW,CAAC;AACxD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC;IAC/E,gBAAgB,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,kBAAkB,EAAE,QAAQ;IAC/E,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CAAC;AACvG,IAAI,MAA4C,CAAC;AACjD,IAAI,UAA8C,CAAC;AACnD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;AACnE,IAAI,MAAqD,CAAC;AAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,MAAM,OAAO,GAA4B,EAAE,IAAI,EAAE,6CAA6C,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,oBAAoB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;AACrQ,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AACrF,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,QAAQ,EAAE,eAAe;QACtG,mBAAmB,EAAE,iBAAiB,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC/G,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,yBAAyB,EAAE,GAAG,EAAE;QACjI,eAAe,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,CAAC;IAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;IACxI,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,eAAe,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACtH,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/I,MAAM,OAAO,GAAG,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QACpH,oBAAoB,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5H,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QAC5N,mBAAmB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,uBAAuB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACjN,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAAC,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/F,UAAU,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAO,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,KAAK,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACxF,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,SAAS,EAAE,EAAE,gBAAgB,EAAE,OAAO;gBACzG,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE;YAC7G,SAAS,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE;gBAChG,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;SACjF,CAAC,EAAE,CAAC,CAAC;IACN,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;IACzC,IAAI,OAAO,GAA4B,EAAE,EAAE,QAAQ,GAAG,EAAE,CAAC;IACzD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YAAC,QAAQ,GAAG,KAAK,CAAC;YAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAC/H,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM;QAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3H,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAA4G,CAAC;IACpI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACnG,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAC5C,MAAM,CAAC,EAAE,CAAC,UAAU,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,EAAE,qDAAqD,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAqH,CAAC;IACvK,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IACvD,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,iBAAiB,CAAC,cAAc,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC7F,MAAM,cAAc,GAAG,cAAc,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;IAC7D,MAAM,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC;IACvG,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4E,CAAC,CAAC;IACtJ,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,gBAAgB,CAAC,CAAC,MAAM,CAAC;IAChG,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,aAAa,KAAK,mBAAmB,CAAC,CAAC;IAC5F,MAAM,CAAC,EAAE,CAAC,SAAS,IAAI,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,qEAAqE,CAAC,CAAC;IAC1H,MAAM,CAAC,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAK,CAAC,MAAO,CAAC,UAAW,CAAC,CAAC;IAC3E,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IAC1D,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,EAAE,UAAU,CAAC,CAAC;IACnE,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IAE1E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,oBAAoB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxH,MAAM,YAAY,EAAE,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5G,KAAK,MAAM,IAAI,IAAI,MAAM,YAAY,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,gCAAgC,IAAI,EAAE,CAAC,CAAC;IACtK,CAAC;IACD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,UAAU,KAAK,CAAC;QAClG,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,CAAC;IACtM,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5G,MAAM,KAAK,CAAC;AACd,CAAC;QAAS,CAAC;IACT,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,KAAK,EAAE,OAAO,CAAC,aAAa,MAAM,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC;QAAC,MAAM,YAAY,EAAE,IAAI,EAAE,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,CAAC,iBAAiB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;IACzI,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,UAAU;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxH,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,UAAU,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IACxG,SAAS,CAAC,GAAG,EAAE,CAAC;IAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAChC,wBAAwB,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACvG,CAAC,CAAC,wBAAwB,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,IAAI,YAAY,IAAI,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,uBAAuB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,OAAO,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IAC3I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAChE,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACvC,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,8CAA8C,CAAC,CAAC;AAE3F,KAAK,UAAU,aAAa;IAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACxG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,SAAS,CAAC,WAAW,CAAC,CAAC;IAC1E,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IACrF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,gFAAgF,SAAS,CAAC,EAAE,yDAAyD,CAAC,CAAC;IACrM,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,oPAAoP,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzT,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,4BAA4B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,mCAAmC,CAAC,EAAE,MAAM,QAAQ,CAAC,mCAAmC,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,IAAI,EAAE,+BAA+B,CAAC,CAAC;QAAE,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3O,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClH,OAAO,qBAAqB,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,246 @@
1
+ import assert from "node:assert/strict";
2
+ import { remoteCanaryWorker } from "./canary-worker-host.js";
3
+ import { randomBytes } from "node:crypto";
4
+ import { spawn } from "node:child_process";
5
+ import http from "node:http";
6
+ import { once } from "node:events";
7
+ import { createWriteStream } from "node:fs";
8
+ import { mkdir, mkdtemp, readFile, readdir, writeFile } from "node:fs/promises";
9
+ import { tmpdir } from "node:os";
10
+ import path from "node:path";
11
+ import { pathToFileURL } from "node:url";
12
+ import { DaemonServer, daemonClient } from "../src/daemon/index.js";
13
+ import { readExecutionLeases, reapOwnedDockerResources, runEval, validateEvalId } from "../src/evals/index.js";
14
+ import { parseTrainingBinding } from "../src/model-access/index.js";
15
+ import { atomicWriteJSON, runCommand, sha256JSON, statePaths } from "../src/foundation/index.js";
16
+ // Real Harbor/Docker and public worker CLI, with a deterministic model fixture.
17
+ // This cannot certify native token facts, Slime training, or a GPU runtime.
18
+ const python = process.env.HITCH_HARBOR_TEST_PYTHON;
19
+ const image = process.env.HITCH_TRAINING_CANARY_IMAGE;
20
+ assert.ok(python && image, "Set HITCH_HARBOR_TEST_PYTHON and HITCH_TRAINING_CANARY_IMAGE (local bash-capable image)");
21
+ const harbor = path.join(path.dirname(python), "harbor");
22
+ const cli = path.resolve("dist/bin/hitch.js");
23
+ const root = await mkdtemp(path.join(tmpdir(), "hitch-real-remote-training-"));
24
+ const controllerRoot = path.join(root, "controller"), workerRoot = path.join(root, "worker");
25
+ const privateRoot = path.join(root, "private"), dataset = path.join(root, "dataset");
26
+ await mkdir(privateRoot, { recursive: true, mode: 0o700 });
27
+ const resource = { cpu_millis: 1000, memory_bytes: 1024 ** 3, container_slots: 1, build_slots: 0 };
28
+ const zero = { cpu_millis: 0, memory_bytes: 0, container_slots: 0, build_slots: 0 };
29
+ const env = { ...process.env, HITCH_HARBOR_PYTHON_PATH: python };
30
+ delete env.HITCH_TEST_HOST_ARTIFACT_BUILDER;
31
+ const imageInfo = await runCommand("docker", ["image", "inspect", image, "--format", "{{json .}}"], { timeoutMs: 10_000 });
32
+ const inspected = JSON.parse(imageInfo.stdout);
33
+ assert.match(inspected.Id, /^sha256:[a-f0-9]{64}$/);
34
+ assert.equal(`${inspected.Os}/${inspected.Architecture}`, "linux/amd64");
35
+ const version = (await runCommand(python, ["-c", 'import importlib.metadata; print(importlib.metadata.version("harbor"))'], { timeoutMs: 10_000 })).stdout.trim();
36
+ assert.equal(version, "0.21.0");
37
+ const localEngine = (await runCommand("docker", ["info", "--format", "{{.ID}}"], { timeoutMs: 10_000 })).stdout.trim();
38
+ const remoteWorker = await remoteCanaryWorker(root, workerRoot);
39
+ const workerObservation = await remoteWorker?.observe(inspected.Id, localEngine);
40
+ const engine = workerObservation?.engine ?? localEngine;
41
+ const hash = sha256JSON({ canary: "remote-training", image: inspected.Id });
42
+ const secret = randomBytes(32).toString("hex");
43
+ const binding = parseTrainingBinding({ kind: "training-external", bindingId: "canary_training", trainingRunId: "train_canary",
44
+ policyLeaseRef: { uri: `cas:${hash}`, digest: hash, mediaType: "application/json" }, expectedPolicyVersion: "canary/update-0",
45
+ fencingToken: randomBytes(16).toString("hex"), expiresAt: new Date(Date.now() + 10 * 60_000).toISOString(),
46
+ endpointRef: "hitch-training:canary_training", credentialRef: "hitch-training:canary_training",
47
+ generationContractDigest: hash, requiredCapture: "exact-policy-tokens-v1", api: "chat-completions", maxOutputTokens: 64, maxEpisodeSteps: 4 });
48
+ let boundRun, generations = 0;
49
+ const protocolErrors = [];
50
+ const upstream = http.createServer((request, response) => {
51
+ const chunks = [];
52
+ request.on("data", chunk => chunks.push(Buffer.from(chunk)));
53
+ request.on("end", () => {
54
+ try {
55
+ assert.ok(request.headers.authorization === `Bearer ${secret}`, "missing private model credential");
56
+ response.setHeader("content-type", "application/json");
57
+ if (request.url === "/v1/lease") {
58
+ response.end(JSON.stringify({ schemaVersion: 1, trainingRunId: binding.trainingRunId, policyVersion: binding.expectedPolicyVersion,
59
+ fencingToken: binding.fencingToken, state: "serving", generationContractDigest: hash, capture: binding.requiredCapture }));
60
+ return;
61
+ }
62
+ const body = JSON.parse(Buffer.concat(chunks).toString("utf8"));
63
+ if (request.url === "/v1/hitch/run") {
64
+ assert.ok(!boundRun || boundRun === body.runId, "binding acquired another canonical run");
65
+ boundRun = body.runId;
66
+ response.end(JSON.stringify({ runId: boundRun, policyVersion: binding.expectedPolicyVersion }));
67
+ return;
68
+ }
69
+ assert.equal(request.url, "/v1/chat/completions");
70
+ assert.ok(boundRun);
71
+ assert.equal(request.headers["idempotency-key"], `${boundRun}-${generations}`);
72
+ assert.equal(body.model, binding.expectedPolicyVersion);
73
+ if (generations === 1)
74
+ assert.match(body.messages.at(-1).content, /first-tool/);
75
+ if (generations === 2)
76
+ assert.match(body.messages.at(-1).content, /first-toolsecond-tool/);
77
+ generations++;
78
+ assert.ok(generations <= 3, "unexpected extra model generation");
79
+ const command = generations === 1
80
+ ? "printf first-tool > /tmp/gear-canary.txt; cat /tmp/gear-canary.txt"
81
+ : "printf second-tool >> /tmp/gear-canary.txt; cat /tmp/gear-canary.txt";
82
+ response.setHeader("x-gear-receipt-id", `fixture-receipt-${generations}`);
83
+ response.end(JSON.stringify({ model: binding.expectedPolicyVersion, choices: [{ index: 0,
84
+ finish_reason: generations < 3 ? "tool_calls" : "stop", message: generations < 3
85
+ ? { role: "assistant", content: null, tool_calls: [{ id: `call-${generations}`, type: "function", function: { name: "bash", arguments: JSON.stringify({ command }) } }] }
86
+ : { role: "assistant", content: "done" } }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }));
87
+ }
88
+ catch (error) {
89
+ protocolErrors.push(error instanceof Error ? error.message : String(error));
90
+ response.statusCode = 500;
91
+ response.end(JSON.stringify({ error: "canary model protocol mismatch" }));
92
+ }
93
+ });
94
+ });
95
+ upstream.listen(0, "127.0.0.1");
96
+ await once(upstream, "listening");
97
+ const upstreamUrl = `http://127.0.0.1:${upstream.address().port}`;
98
+ const server = new DaemonServer({ root: controllerRoot, port: 0, maxConcurrent: 1,
99
+ resourceCapacity: { ...resource, build_slots: 1 }, evalTrialResources: resource,
100
+ evalExecutor: options => runEval({ ...options, harborExecutable: harbor, env }), logger: () => { } });
101
+ let worker;
102
+ let workerDone;
103
+ const workerLog = createWriteStream(path.join(root, "worker.log"));
104
+ let evalId;
105
+ let passed = false;
106
+ const summary = { kind: "real-harbor-training-transport-canary", validated_gpu: false, worker_host: remoteWorker ? "ssh" : "local-process", root, harbor: version, image_id: inspected.Id, docker_engine_digest: sha256JSON(engine) };
107
+ console.log(JSON.stringify({ event: "canary-start", root, image_id: inspected.Id }));
108
+ try {
109
+ const harness = await prepareInputs();
110
+ await server.start();
111
+ const registration = { schema_version: "1", worker_id: "worker_real_training", provider: "remote-docker",
112
+ collision_domain_id: `docker-engine:${engine}`, platforms: ["linux-x64"], backends: [{ id: "harbor", version }],
113
+ features: { docker: true, buildkit: true, model_proxy: true, isolated_same_task_attempts: false, training_external_binding: "2" },
114
+ task_membership: ["known"], capacity: { total: resource, reserved_for_system: zero, allocatable: resource } };
115
+ const registrationFile = path.join(privateRoot, "registration.json"), credentialFile = path.join(privateRoot, "worker-credential.json");
116
+ const bindingFile = path.join(privateRoot, "binding.json");
117
+ await atomicWriteJSON(registrationFile, registration);
118
+ await atomicWriteJSON(bindingFile, { schema_version: "1", binding, base_url: `${upstreamUrl}/v1`, credential: secret });
119
+ await runCommand(process.execPath, [cli, "--root", controllerRoot, "training", "register", "--file", bindingFile], { env, timeoutMs: 10_000 });
120
+ const baseUrl = `http://127.0.0.1:${server.port}`;
121
+ await runCommand(process.execPath, [cli, "worker", "register", "--server", baseUrl, "--registration", registrationFile,
122
+ "--admin-token-file", statePaths(controllerRoot).token, "--credential-file", credentialFile], { env, timeoutMs: 10_000 });
123
+ worker = remoteWorker ? await remoteWorker.start(server.port, registrationFile, credentialFile) : spawn(process.execPath, [cli, "--root", workerRoot, "worker", "run", "--server", baseUrl, "--registration", registrationFile,
124
+ "--credential-file", credentialFile, "--harbor", harbor, "--docker", "/usr/local/bin/docker", "--once", "--poll-interval", "100ms", "--heartbeat-interval", "1s"], { env, stdio: ["ignore", "pipe", "pipe"] });
125
+ worker.stdout.pipe(workerLog, { end: false });
126
+ worker.stderr.pipe(workerLog, { end: false });
127
+ workerDone = new Promise((resolve, reject) => { worker.once("error", reject); worker.once("exit", code => resolve(code)); });
128
+ void workerDone.catch(() => { });
129
+ const admin = await daemonClient(controllerRoot);
130
+ const submitted = await admin.request("/v1/evals", { method: "POST", body: JSON.stringify({
131
+ request: { dataset, harness_ref: harness, model: `training/${binding.bindingId}`, training_binding: binding,
132
+ max_concurrent: 1, attempts: 1, infrastructure_retries: 0, timeout_ms: 120_000, setup_timeout_ms: 120_000 },
133
+ execution: { provider: "remote-docker", max_parallelism: 1, resources: { default_trial: resource },
134
+ build: { mode: "backend" }, model_capture: { mode: "proxy", required: true } },
135
+ }) });
136
+ evalId = validateEvalId(String(submitted.eval_id));
137
+ console.log(JSON.stringify({ event: "eval-submitted", eval_id: evalId }));
138
+ const deadline = Date.now() + 5 * 60_000;
139
+ let current = {}, previous = "";
140
+ while (Date.now() < deadline) {
141
+ current = await admin.request(`/v1/evals/${evalId}`);
142
+ const phase = JSON.stringify(current.control);
143
+ if (phase !== previous) {
144
+ previous = phase;
145
+ console.log(JSON.stringify({ event: "eval-control", control: current.control }));
146
+ }
147
+ if (current.result)
148
+ break;
149
+ if (worker.exitCode !== null && worker.exitCode !== 0)
150
+ throw new Error(`worker exited before result (${worker.exitCode})`);
151
+ await new Promise(resolve => setTimeout(resolve, 1000));
152
+ }
153
+ await atomicWriteJSON(path.join(root, "eval-inspection.json"), current);
154
+ assert.ok(current.result, "real Harbor candidate timed out");
155
+ const result = current.result;
156
+ assert.equal(result.status, "succeeded", JSON.stringify(result));
157
+ assert.equal(result.trials.length, 1);
158
+ assert.equal(result.trials[0].observation_status, "valid");
159
+ assert.equal(result.trials[0].reward, 1);
160
+ assert.equal(result.trials[0].run_id, boundRun);
161
+ assert.equal(generations, 3);
162
+ assert.deepEqual(protocolErrors, []);
163
+ const evidence = await runCommand(process.execPath, [cli, "--root", controllerRoot, "training", "evidence", boundRun, "--json"], { env, timeoutMs: 10_000 });
164
+ await writeFile(path.join(root, "training-evidence.json"), evidence.stdout);
165
+ const leases = await readExecutionLeases(path.join(controllerRoot, "evals", evalId));
166
+ assert.equal(leases.length, 1);
167
+ assert.equal(leases[0].state, "released");
168
+ assert.ok(leases[0].release_confirmation);
169
+ assert.equal(await Promise.race([workerDone, new Promise(resolve => setTimeout(() => resolve("timeout"), 10_000))]), 0);
170
+ await remoteWorker?.collect();
171
+ for (const directory of [path.join(controllerRoot, "runs"), path.join(controllerRoot, "evals"), workerRoot]) {
172
+ for (const file of await regularFiles(directory))
173
+ assert.equal((await readFile(file)).includes(Buffer.from(secret)), false, `model credential leaked into ${file}`);
174
+ }
175
+ Object.assign(summary, { status: "passed", eval_id: evalId, run_id: boundRun, model_calls: generations, tool_calls: 2, lease_id: leases[0].lease_id, private_model_credential_in_public_files: false });
176
+ passed = true;
177
+ }
178
+ catch (error) {
179
+ Object.assign(summary, { status: "failed", error: error instanceof Error ? error.message : String(error), protocol_errors: protocolErrors });
180
+ throw error;
181
+ }
182
+ finally {
183
+ if (evalId && !passed) {
184
+ const admin = await daemonClient(controllerRoot).catch(() => undefined);
185
+ await admin?.request(`/v1/evals/${evalId}/cancel`, { method: "POST", body: "{}" }).catch(() => { });
186
+ }
187
+ try {
188
+ await remoteWorker?.stop();
189
+ }
190
+ catch (error) {
191
+ summary.remote_stop_error = error instanceof Error ? error.message : String(error);
192
+ }
193
+ if (worker && worker.exitCode === null)
194
+ worker.kill("SIGTERM");
195
+ if (workerDone)
196
+ await Promise.race([workerDone.catch(() => null), new Promise(resolve => setTimeout(resolve, 10_000))]);
197
+ if (worker && worker.exitCode === null) {
198
+ worker.kill("SIGKILL");
199
+ await workerDone?.catch(() => null);
200
+ }
201
+ workerLog.end();
202
+ await server.close();
203
+ upstream.closeAllConnections();
204
+ await new Promise(resolve => upstream.close(() => resolve()));
205
+ const leases = evalId ? await readExecutionLeases(path.join(controllerRoot, "evals", evalId)).catch(() => []) : [];
206
+ const leaseIds = leases.map(lease => lease.lease_id);
207
+ const cleanup = await Promise.all([
208
+ reapOwnedDockerResources({ root: controllerRoot, leaseIds, env }),
209
+ remoteWorker ? remoteWorker.cleanup(leaseIds).catch(error => ({ issues: [String(error)], retained: [] }))
210
+ : reapOwnedDockerResources({ root: workerRoot, leaseIds, env }),
211
+ ]);
212
+ if (remoteWorker && !passed)
213
+ await remoteWorker.collect().catch(error => { summary.remote_collection_error = String(error); });
214
+ summary.cleanup_proven = !summary.remote_stop_error && cleanup.every(report => report.issues.length === 0 && report.retained.length === 0);
215
+ await atomicWriteJSON(path.join(root, "summary.json"), summary);
216
+ console.log(JSON.stringify(summary));
217
+ }
218
+ assert.equal(summary.cleanup_proven, true, "owned Docker resources were not all released");
219
+ async function prepareInputs() {
220
+ const task = path.join(dataset, "one"), source = path.join(root, "training-source");
221
+ await mkdir(path.join(task, "environment"), { recursive: true });
222
+ await mkdir(path.join(task, "tests"));
223
+ await writeFile(path.join(task, "instruction.md"), "Use two bash tool calls to write first-toolsecond-tool into /tmp/gear-canary.txt.\n");
224
+ await writeFile(path.join(task, "environment/Dockerfile"), `FROM ${inspected.Id}\n`);
225
+ await writeFile(path.join(task, "task.toml"), `schema_version="1.4"\n[agent]\ntimeout_sec=120\n[environment]\ndocker_image="${inspected.Id}"\ncpus=1\nmemory_mb=1024\n[verifier]\ntimeout_sec=30\n`);
226
+ await writeFile(path.join(task, "tests/test.sh"), '#!/bin/bash\nset -eu\nmkdir -p /logs/verifier\nif test -f /tmp/gear-canary.txt && test "$(cat /tmp/gear-canary.txt)" = first-toolsecond-tool; then\n printf "1" > /logs/verifier/reward.txt\nelse\n printf "0" > /logs/verifier/reward.txt\nfi\n', { mode: 0o755 });
227
+ await mkdir(path.join(source, "integrations/training-tool"), { recursive: true });
228
+ await writeFile(path.join(source, "integrations/training-tool/cli.js"), await readFile("integrations/training-tool/cli.js"), { mode: 0o755 });
229
+ await writeFile(path.join(source, "package.json"), '{"type":"module"}');
230
+ for (const args of [["init"], ["add", "."], ["-c", "user.name=Hitch Canary", "-c", "user.email=canary@hitch.invalid", "commit", "-m", "fixed training harness canary"]])
231
+ await runCommand("git", args, { cwd: source, timeoutMs: 10_000 });
232
+ const commit = (await runCommand("git", ["rev-parse", "HEAD"], { cwd: source, timeoutMs: 10_000 })).stdout.trim();
233
+ return `training-tool@git+${pathToFileURL(source).href}#${commit}`;
234
+ }
235
+ async function regularFiles(directory) {
236
+ const files = [];
237
+ for (const entry of await readdir(directory, { withFileTypes: true })) {
238
+ const file = path.join(directory, entry.name);
239
+ if (entry.isDirectory())
240
+ files.push(...await regularFiles(file));
241
+ else if (entry.isFile())
242
+ files.push(file);
243
+ }
244
+ return files;
245
+ }
246
+ //# sourceMappingURL=canary-remote-training.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"canary-remote-training.js","sourceRoot":"","sources":["../../scripts/canary-remote-training.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC5C,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC/G,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAEjG,gFAAgF;AAChF,4EAA4E;AAC5E,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;AACpD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;AACtD,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,KAAK,EAAE,yFAAyF,CAAC,CAAC;AACtH,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,CAAC;AACzD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;AAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,6BAA6B,CAAC,CAAC,CAAC;AAC/E,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAC7F,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;AACrF,MAAM,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAC3D,MAAM,QAAQ,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACnG,MAAM,IAAI,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AACpF,MAAM,GAAG,GAAsB,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,CAAC;AACpF,OAAO,GAAG,CAAC,gCAAgC,CAAC;AAC5C,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AAC3H,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAqD,CAAC;AACnG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,EAAE,uBAAuB,CAAC,CAAC;AACpD,MAAM,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC,EAAE,IAAI,SAAS,CAAC,YAAY,EAAE,EAAE,aAAa,CAAC,CAAC;AACzE,MAAM,OAAO,GAAG,CAAC,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,wEAAwE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AAClK,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;AAChC,MAAM,WAAW,GAAG,CAAC,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACvH,MAAM,YAAY,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAChE,MAAM,iBAAiB,GAAG,MAAM,YAAY,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;AACjF,MAAM,MAAM,GAAG,iBAAiB,EAAE,MAAM,IAAI,WAAW,CAAC;AACxD,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5E,MAAM,MAAM,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC/C,MAAM,OAAO,GAAG,oBAAoB,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,SAAS,EAAE,iBAAiB,EAAE,aAAa,EAAE,cAAc;IAC3H,cAAc,EAAE,EAAE,GAAG,EAAE,OAAO,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,kBAAkB,EAAE,EAAE,qBAAqB,EAAE,iBAAiB;IAC7H,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,CAAC,WAAW,EAAE;IAC1G,WAAW,EAAE,gCAAgC,EAAE,aAAa,EAAE,gCAAgC;IAC9F,wBAAwB,EAAE,IAAI,EAAE,eAAe,EAAE,wBAAwB,EAAE,GAAG,EAAE,kBAAkB,EAAE,eAAe,EAAE,EAAE,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC,CAAC;AACjJ,IAAI,QAA4B,EAAE,WAAW,GAAG,CAAC,CAAC;AAClD,MAAM,cAAc,GAAa,EAAE,CAAC;AACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE;IACvD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;QACrB,IAAI,CAAC;YACH,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,KAAK,UAAU,MAAM,EAAE,EAAE,kCAAkC,CAAC,CAAC;YACpG,QAAQ,CAAC,SAAS,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;YACvD,IAAI,OAAO,CAAC,GAAG,KAAK,WAAW,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,aAAa,EAAE,OAAO,CAAC,qBAAqB;oBAChI,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,KAAK,EAAE,SAAS,EAAE,wBAAwB,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;gBAAC,OAAO;YACvI,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAChE,IAAI,OAAO,CAAC,GAAG,KAAK,eAAe,EAAE,CAAC;gBACpC,MAAM,CAAC,EAAE,CAAC,CAAC,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,KAAK,EAAE,wCAAwC,CAAC,CAAC;gBAC1F,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC;gBAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC;gBAAC,OAAO;YACjI,CAAC;YACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,sBAAsB,CAAC,CAAC;YAAC,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE,GAAG,QAAQ,IAAI,WAAW,EAAE,CAAC,CAAC;YAC/E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC;YACxD,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;YAChF,IAAI,WAAW,KAAK,CAAC;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,uBAAuB,CAAC,CAAC;YAC3F,WAAW,EAAE,CAAC;YACd,MAAM,CAAC,EAAE,CAAC,WAAW,IAAI,CAAC,EAAE,mCAAmC,CAAC,CAAC;YACjE,MAAM,OAAO,GAAG,WAAW,KAAK,CAAC;gBAC/B,CAAC,CAAC,oEAAoE;gBACtE,CAAC,CAAC,sEAAsE,CAAC;YAC3E,QAAQ,CAAC,SAAS,CAAC,mBAAmB,EAAE,mBAAmB,WAAW,EAAE,CAAC,CAAC;YAC1E,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,qBAAqB,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;wBACtF,aAAa,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,WAAW,GAAG,CAAC;4BAC9E,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE;4BACzK,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1H,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5E,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC;YAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gCAAgC,EAAE,CAAC,CAAC,CAAC;QACvG,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AACH,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;AAAC,MAAM,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACnE,MAAM,WAAW,GAAG,oBAAqB,QAAQ,CAAC,OAAO,EAAqC,CAAC,IAAI,EAAE,CAAC;AACtG,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC;IAC/E,gBAAgB,EAAE,EAAE,GAAG,QAAQ,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE,kBAAkB,EAAE,QAAQ;IAC/E,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC,CAAC;AACvG,IAAI,MAA4C,CAAC;AACjD,IAAI,UAA8C,CAAC;AACnD,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAAC;AACnE,IAAI,MAAqD,CAAC;AAC1D,IAAI,MAAM,GAAG,KAAK,CAAC;AACnB,MAAM,OAAO,GAA4B,EAAE,IAAI,EAAE,uCAAuC,EAAE,aAAa,EAAE,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,oBAAoB,EAAE,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/P,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AACrF,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,MAAM,YAAY,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,SAAS,EAAE,sBAAsB,EAAE,QAAQ,EAAE,eAAe;QACtG,mBAAmB,EAAE,iBAAiB,MAAM,EAAE,EAAE,SAAS,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC/G,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,yBAAyB,EAAE,GAAG,EAAE;QACjI,eAAe,EAAE,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE,CAAC;IAChH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,mBAAmB,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC,CAAC;IACxI,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC3D,MAAM,eAAe,CAAC,gBAAgB,EAAE,YAAY,CAAC,CAAC;IACtD,MAAM,eAAe,CAAC,WAAW,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,WAAW,KAAK,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IACxH,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC/I,MAAM,OAAO,GAAG,oBAAoB,MAAM,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QACpH,oBAAoB,EAAE,UAAU,CAAC,cAAc,CAAC,CAAC,KAAK,EAAE,mBAAmB,EAAE,cAAc,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5H,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB;QAC5N,mBAAmB,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,uBAAuB,EAAE,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IACjN,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAAC,MAAM,CAAC,MAAO,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IAC/F,UAAU,GAAG,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,MAAO,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,MAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,KAAK,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACxF,OAAO,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,YAAY,OAAO,CAAC,SAAS,EAAE,EAAE,gBAAgB,EAAE,OAAO;gBACzG,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,EAAE,OAAO,EAAE;YAC7G,SAAS,EAAE,EAAE,QAAQ,EAAE,eAAe,EAAE,eAAe,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,QAAQ,EAAE;gBAChG,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE;SACjF,CAAC,EAAE,CAAC,CAAC;IACN,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;IACzC,IAAI,OAAO,GAA4B,EAAE,EAAE,QAAQ,GAAG,EAAE,CAAC;IACzD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,OAAO,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAa,MAAM,EAAE,CAAC,CAAC;QACrD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,KAAK,KAAK,QAAQ,EAAE,CAAC;YAAC,QAAQ,GAAG,KAAK,CAAC;YAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAAC,CAAC;QAC/H,IAAI,OAAO,CAAC,MAAM;YAAE,MAAM;QAC1B,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QAC3H,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,sBAAsB,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,iCAAiC,CAAC,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAA4G,CAAC;IACpI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;IACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACjD,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,EAAE,QAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9J,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IACrF,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAC3E,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,oBAAoB,CAAC,CAAC;IAC3C,MAAM,CAAC,KAAK,CAAC,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACxH,MAAM,YAAY,EAAE,OAAO,EAAE,CAAC;IAC9B,KAAK,MAAM,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC;QAC5G,KAAK,MAAM,IAAI,IAAI,MAAM,YAAY,CAAC,SAAS,CAAC;YAAE,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,gCAAgC,IAAI,EAAE,CAAC,CAAC;IACtK,CAAC;IACD,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,EAAE,wCAAwC,EAAE,KAAK,EAAE,CAAC,CAAC;IACzM,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,CAAC;IAC7I,MAAM,KAAK,CAAC;AACd,CAAC;QAAS,CAAC;IACT,IAAI,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxE,MAAM,KAAK,EAAE,OAAO,CAAC,aAAa,MAAM,SAAS,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrG,CAAC;IACD,IAAI,CAAC;QAAC,MAAM,YAAY,EAAE,IAAI,EAAE,CAAC;IAAC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAAC,OAAO,CAAC,iBAAiB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAAC,CAAC;IACzI,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,UAAU;QAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACxH,IAAI,MAAM,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAAC,MAAM,UAAU,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAAC,CAAC;IACxG,SAAS,CAAC,GAAG,EAAE,CAAC;IAChB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IACrB,QAAQ,CAAC,mBAAmB,EAAE,CAAC;IAAC,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IACpG,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnH,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAChC,wBAAwB,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;QACjE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;YACvG,CAAC,CAAC,wBAAwB,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC;KAClE,CAAC,CAAC;IACH,IAAI,YAAY,IAAI,CAAC,MAAM;QAAE,MAAM,YAAY,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,GAAG,OAAO,CAAC,uBAAuB,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/H,OAAO,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;IAC3I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AACvC,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,IAAI,EAAE,8CAA8C,CAAC,CAAC;AAE3F,KAAK,UAAU,aAAa;IAC1B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IACpF,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IACxG,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,gBAAgB,CAAC,EAAE,qFAAqF,CAAC,CAAC;IAC1I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,wBAAwB,CAAC,EAAE,QAAQ,SAAS,CAAC,EAAE,IAAI,CAAC,CAAC;IACrF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,gFAAgF,SAAS,CAAC,EAAE,yDAAyD,CAAC,CAAC;IACrM,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,CAAC,EAAE,oPAAoP,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzT,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,4BAA4B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAClF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,mCAAmC,CAAC,EAAE,MAAM,QAAQ,CAAC,mCAAmC,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAC9I,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE,mBAAmB,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,iCAAiC,EAAE,QAAQ,EAAE,IAAI,EAAE,+BAA+B,CAAC,CAAC;QAAE,MAAM,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IAC3O,MAAM,MAAM,GAAG,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;IAClH,OAAO,qBAAqB,aAAa,CAAC,MAAM,CAAC,CAAC,IAAI,IAAI,MAAM,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,SAAiB;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9G,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,124 @@
1
+ import assert from "node:assert/strict";
2
+ import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
3
+ import path from "node:path";
4
+ import { tmpdir } from "node:os";
5
+ import { buildEvalExecutionPlan, captureRemoteVerifierSource, createExecutionLease, portableRemoteTrial, reapOwnedDockerResources, runRemoteVerifierWork, validateEvalId } from "../src/evals/index.js";
6
+ import { ensureControllerRuntime } from "../src/controller-runtime/index.js";
7
+ import { atomicWriteJSON, readJSON, runCommand, sha256JSON } from "../src/foundation/index.js";
8
+ import { writeResultBundleIndex } from "../src/runs/index.js";
9
+ // Real Harbor/Docker CPU canary. Requires a locally available image with bash; never pulls or provisions GPUs.
10
+ const python = process.env.HITCH_HARBOR_TEST_PYTHON, image = process.env.HITCH_VERIFIER_IMAGE;
11
+ if (!python || !image)
12
+ throw Error("Set HITCH_HARBOR_TEST_PYTHON and HITCH_VERIFIER_IMAGE to the Harbor 0.21.0 Python and a local bash-capable Docker image");
13
+ const inspected = await runCommand("docker", ["image", "inspect", image, "--format", "{{.Id}}"], { timeoutMs: 10_000 });
14
+ const imageId = inspected.stdout.trim();
15
+ if (!/^sha256:[a-f0-9]{64}$/.test(imageId))
16
+ throw Error("A local immutable Docker image ID is required");
17
+ const root = await mkdtemp(path.join(tmpdir(), "hitch-real-verifier-"));
18
+ const evalId = validateEvalId(`eval_${"b".repeat(32)}`), runId = `run_${"a".repeat(32)}`;
19
+ const evalDirectory = path.join(root, "evals", evalId), taskDirectory = path.join(root, "one"), trialDirectory = path.join(root, "original");
20
+ const reservation = { cpu_millis: 1000, memory_bytes: 512 * 1024 ** 2, container_slots: 1, build_slots: 0 };
21
+ const lease = await createExecutionLease({ evalId, evalDirectory, workId: `work_${"c".repeat(32)}`,
22
+ worker: { workerId: "verifier-canary", provider: "remote-docker", collisionDomainId: "verifier-canary-docker" }, reservation, ttlMs: 45_000 });
23
+ const controller = new AbortController();
24
+ const heartbeat = setInterval(() => { void lease.heartbeat().catch(error => controller.abort(error)); }, 10_000);
25
+ heartbeat.unref();
26
+ let cleanupProven = false, passed = false;
27
+ let summary = {};
28
+ try {
29
+ const runtime = await ensureControllerRuntime({ root });
30
+ await mkdir(path.join(taskDirectory, "tests"), { recursive: true });
31
+ await mkdir(path.join(taskDirectory, "environment"));
32
+ await writeFile(path.join(taskDirectory, "instruction.md"), "The candidate has already produced candidate.txt. Regrade its recorded output.\n");
33
+ await writeFile(path.join(taskDirectory, "environment/Dockerfile"), `FROM ${imageId}\n`);
34
+ await writeFile(path.join(taskDirectory, "task.toml"), `schema_version="1.4"
35
+ [agent]
36
+ timeout_sec=10
37
+ [environment]
38
+ docker_image="${imageId}"
39
+ cpus=1
40
+ memory_mb=512
41
+ [verifier]
42
+ timeout_sec=30
43
+ environment_mode="separate"
44
+ [verifier.environment]
45
+ docker_image="${imageId}"
46
+ cpus=1
47
+ memory_mb=512
48
+ `);
49
+ await writeFile(path.join(taskDirectory, "tests/test.sh"), '#!/bin/bash\nset -eu\ntest "$(cat /logs/artifacts/candidate.txt)" = "recorded candidate"\nmkdir -p /logs/verifier\nprintf "1" > /logs/verifier/reward.txt\n');
50
+ await mkdir(path.join(trialDirectory, "artifacts/logs/artifacts"), { recursive: true });
51
+ await writeFile(path.join(trialDirectory, "artifacts/logs/artifacts/candidate.txt"), "recorded candidate");
52
+ await atomicWriteJSON(path.join(trialDirectory, "artifacts/manifest.json"), [{ source: "/logs/artifacts", destination: "artifacts/logs/artifacts", type: "directory", status: "ok", service: null }]);
53
+ await atomicWriteJSON(path.join(trialDirectory, "benchmark-lifecycle.json"), { schema_version: "1", failure: null, phases: {} });
54
+ await atomicWriteJSON(path.join(trialDirectory, "config.json"), { task: { path: taskDirectory }, trial_name: "one__original", trials_dir: root,
55
+ agent: { import_path: "hitch_harbor_agent:HitchHarborAgent", kwargs: { private_canary_marker: "never-transfer-agent-config" } },
56
+ environment: { type: "docker", import_path: "hitch_harbor_environment:HitchHarborDockerEnvironment", delete: false,
57
+ cpu_enforcement_policy: "limit", memory_enforcement_policy: "limit", override_cpus: 1, override_memory_mb: 512 },
58
+ verifier: { import_path: "hitch_harbor_verifier:HitchRetryingVerifier", override_timeout_sec: 30,
59
+ kwargs: { infrastructure_retries: 0, infrastructure_retry_backoff_ms: 0 } } });
60
+ await runCommand(python, ["-c", `import json,sys
61
+ from pathlib import Path
62
+ from harbor.models.trial.config import TrialConfig
63
+ from harbor.models.trial.result import TrialResult
64
+ from harbor.models.task.task import Task
65
+ p=Path(sys.argv[1]); c=TrialConfig.model_validate_json((p/'config.json').read_text()); t=Task(task_dir=c.task.path)
66
+ (p/'config.json').write_text(c.model_dump_json())
67
+ r=TrialResult(task_name=t.name,trial_name=c.trial_name,trial_uri=p.as_uri(),task_id={'path':c.task.path},task_checksum=t.checksum,config=c,
68
+ agent_info={'name':'HitchHarborAgent','version':'canary'},agent_result={'metadata':{'hitch_run_id':sys.argv[2],'controller_runtime_id':sys.argv[3]}})
69
+ (p/'result.json').write_text(r.model_dump_json())
70
+ `, trialDirectory, runId, runtime.runtime_id], { timeoutMs: 10_000 });
71
+ const trial = await readJSON(path.join(trialDirectory, "result.json"));
72
+ const candidateResult = { run_id: runId, status: "succeeded", output: "recorded candidate" }, bundleDirectory = path.join(root, "bundle");
73
+ await atomicWriteJSON(path.join(bundleDirectory, "manifest.json"), { schema_version: "1", run_id: runId, sealed: true });
74
+ await atomicWriteJSON(path.join(bundleDirectory, "result.json"), candidateResult);
75
+ await writeResultBundleIndex(bundleDirectory);
76
+ const capture = await captureRemoteVerifierSource({ taskId: "one", runId, runtimeId: runtime.runtime_id, taskDirectory, trialDirectory, trial,
77
+ bundleDirectory, destination: path.join(root, "snapshot") });
78
+ assert.equal(capture.manifest.status, "available");
79
+ await lease.markRunning();
80
+ const directory = path.join(root, "regrade");
81
+ const plan = buildEvalExecutionPlan({ evalId, tasks: ["one"], maxParallelism: 1, provider: lease.current().provider, trialResources: reservation,
82
+ candidate: { revisionIdentity: sha256JSON("recorded-canary-revision"), artifactId: sha256JSON("recorded-canary-artifact") },
83
+ request: { schema_version: "1", backend: "harbor", dataset: taskDirectory, harness_ref: "canary@version:1", model: "canary/unused",
84
+ attempts: 1, max_concurrent: 1, infrastructure_retries: 0, infrastructure_retry_backoff_ms: 0, timeout_ms: 10_000, setup_timeout_ms: 10_000,
85
+ agent_args: [], pass_env: [], benchmark_id: "verifier-canary", benchmark_revision: sha256JSON("canary") } });
86
+ const outcome = await runRemoteVerifierWork({ root, directory, sourceSnapshotDirectory: capture.directory, taskDirectory,
87
+ sourceManifest: capture.manifest, sourceTrial: portableRemoteTrial(trial), candidateResult, candidateResultDigest: sha256JSON(candidateResult),
88
+ runtimeDirectory: runtime.directory, verifierRuntimeDirectory: runtime.directory, verifierRuntimeId: runtime.runtime_id,
89
+ trialName: "one__regrade", lease: lease.current(), plan,
90
+ env: process.env, harborExecutable: path.join(path.dirname(python), "harbor"), signal: controller.signal });
91
+ assert.equal(outcome.backend.process_exit_code, 0);
92
+ assert.equal(outcome.trial.agent_setup, null);
93
+ assert.equal(outcome.trial.agent_execution, null);
94
+ assert.deepEqual(outcome.trial.agent_result, trial.agent_result);
95
+ assert.equal(outcome.trial.verifier_result?.rewards?.reward, 1);
96
+ assert.equal((await readFile(path.join(directory, "regrade.json"), "utf8")).includes("never-transfer-agent-config"), false);
97
+ passed = true;
98
+ summary = { status: "passed", harbor: outcome.backend.version, image_id: imageId, candidate_executes: false,
99
+ reward: 1, agent_result_preserved: true, private_config_transferred: false };
100
+ }
101
+ finally {
102
+ clearInterval(heartbeat);
103
+ await lease.release();
104
+ const report = await reapOwnedDockerResources({ root, leaseIds: [lease.leaseId], env: process.env });
105
+ cleanupProven = report.issues.length === 0 && report.retained.length === 0;
106
+ if (cleanupProven && passed) {
107
+ await writable(root);
108
+ await rm(root, { recursive: true, force: true });
109
+ }
110
+ else
111
+ console.error(`Verifier canary retained evidence: ${root}; cleanup_proven=${cleanupProven}`);
112
+ }
113
+ assert.equal(cleanupProven, true);
114
+ console.log(JSON.stringify({ ...summary, cleanup_proven: true }));
115
+ async function writable(directory) {
116
+ const info = await lstat(directory);
117
+ if (info.isSymbolicLink())
118
+ return;
119
+ await chmod(directory, info.isDirectory() ? 0o700 : 0o600);
120
+ if (info.isDirectory())
121
+ for (const entry of await readdir(directory))
122
+ await writable(path.join(directory, entry));
123
+ }
124
+ //# sourceMappingURL=canary-remote-verifier.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"canary-remote-verifier.js","sourceRoot":"","sources":["../../scripts/canary-remote-verifier.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClG,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,sBAAsB,EAAE,2BAA2B,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACxM,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AAC/F,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,+GAA+G;AAC/G,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,wBAAwB,EAAE,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC;AAC9F,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK;IAAE,MAAM,KAAK,CAAC,yHAAyH,CAAC,CAAC;AAC9J,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;AACxH,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;AACxC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC;IAAE,MAAM,KAAK,CAAC,+CAA+C,CAAC,CAAC;AACzG,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,sBAAsB,CAAC,CAAC,CAAC;AACxE,MAAM,MAAM,GAAG,cAAc,CAAC,QAAQ,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;AACzF,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;AAC7I,MAAM,WAAW,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,GAAG,IAAI,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;AAC5G,MAAM,KAAK,GAAG,MAAM,oBAAoB,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE;IAChG,MAAM,EAAE,EAAE,QAAQ,EAAE,iBAAiB,EAAE,QAAQ,EAAE,eAAe,EAAE,iBAAiB,EAAE,wBAAwB,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACjJ,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;AACzC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,KAAK,KAAK,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AACjH,SAAS,CAAC,KAAK,EAAE,CAAC;AAClB,IAAI,aAAa,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;AAC1C,IAAI,OAAO,GAA4B,EAAE,CAAC;AAC1C,IAAI,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1H,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,gBAAgB,CAAC,EAAE,kFAAkF,CAAC,CAAC;IAChJ,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAAE,QAAQ,OAAO,IAAI,CAAC,CAAC;IACzF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,EAAE;;;;gBAIzC,OAAO;;;;;;;gBAOP,OAAO;;;CAGtB,CAAC,CAAC;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,eAAe,CAAC,EAAE,6JAA6J,CAAC,CAAC;IAC1N,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,0BAA0B,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxF,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,wCAAwC,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC3G,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,yBAAyB,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,EAAE,0BAA0B,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtM,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,0BAA0B,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;IACjI,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,EAAE,UAAU,EAAE,eAAe,EAAE,UAAU,EAAE,IAAI;QAC5I,KAAK,EAAE,EAAE,WAAW,EAAE,qCAAqC,EAAE,MAAM,EAAE,EAAE,qBAAqB,EAAE,6BAA6B,EAAE,EAAE;QAC/H,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uDAAuD,EAAE,MAAM,EAAE,KAAK;YAChH,sBAAsB,EAAE,OAAO,EAAE,yBAAyB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,EAAE,kBAAkB,EAAE,GAAG,EAAE;QAClH,QAAQ,EAAE,EAAE,WAAW,EAAE,6CAA6C,EAAE,oBAAoB,EAAE,EAAE;YAC9F,MAAM,EAAE,EAAE,sBAAsB,EAAE,CAAC,EAAE,+BAA+B,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACnF,MAAM,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE;;;;;;;;;;CAUjC,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,MAAM,QAAQ,CAA0B,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC;IAChG,MAAM,eAAe,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,oBAAoB,EAAE,EAAE,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC1I,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,eAAe,CAAC,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACzH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,aAAa,CAAC,EAAE,eAAe,CAAC,CAAC;IAAC,MAAM,sBAAsB,CAAC,eAAe,CAAC,CAAC;IACjI,MAAM,OAAO,GAAG,MAAM,2BAA2B,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,aAAa,EAAE,cAAc,EAAE,KAAK;QAC3I,eAAe,EAAE,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC;IAC/D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,MAAM,KAAK,CAAC,WAAW,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,sBAAsB,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE,cAAc,EAAE,WAAW;QAC9I,SAAS,EAAE,EAAE,gBAAgB,EAAE,UAAU,CAAC,0BAA0B,CAAC,EAAE,UAAU,EAAE,UAAU,CAAC,0BAA0B,CAAC,EAAE;QAC3H,OAAO,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,kBAAkB,EAAE,KAAK,EAAE,eAAe;YAChI,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,sBAAsB,EAAE,CAAC,EAAE,+BAA+B,EAAE,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM;YAC3I,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IACjH,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,uBAAuB,EAAE,OAAO,CAAC,SAAU,EAAE,aAAa;QACvH,cAAc,EAAE,OAAO,CAAC,QAAQ,EAAE,WAAW,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE,eAAe,EAAE,qBAAqB,EAAE,UAAU,CAAC,eAAe,CAAC;QAC9I,gBAAgB,EAAE,OAAO,CAAC,SAAS,EAAE,wBAAwB,EAAE,OAAO,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,CAAC,UAAU;QACvH,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,EAAE,IAAI;QACvD,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9G,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;IACnD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACjE,MAAM,CAAC,KAAK,CAAE,OAAO,CAAC,KAAK,CAAC,eAAqD,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACvG,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,6BAA6B,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5H,MAAM,GAAG,IAAI,CAAC;IACd,OAAO,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,kBAAkB,EAAE,KAAK;QACzG,MAAM,EAAE,CAAC,EAAE,sBAAsB,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,CAAC;AACjF,CAAC;QAAS,CAAC;IACT,aAAa,CAAC,SAAS,CAAC,CAAC;IACzB,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;IACtB,MAAM,MAAM,GAAG,MAAM,wBAAwB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACrG,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC;IAC3E,IAAI,aAAa,IAAI,MAAM,EAAE,CAAC;QAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAAC,CAAC;;QACnG,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,oBAAoB,aAAa,EAAE,CAAC,CAAC;AACpG,CAAC;AACD,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC;AAClC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AAElE,KAAK,UAAU,QAAQ,CAAC,SAAiB;IACvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;IACpC,IAAI,IAAI,CAAC,cAAc,EAAE;QAAE,OAAO;IAClC,MAAM,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC3D,IAAI,IAAI,CAAC,WAAW,EAAE;QAAE,KAAK,MAAM,KAAK,IAAI,MAAM,OAAO,CAAC,SAAS,CAAC;YAAE,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC;AACpH,CAAC"}
@@ -0,0 +1,113 @@
1
+ import assert from "node:assert/strict";
2
+ import { randomBytes } from "node:crypto";
3
+ import { spawn } from "node:child_process";
4
+ import { mkdir, readFile } from "node:fs/promises";
5
+ import path from "node:path";
6
+ import { observeControllerRuntime } from "../src/controller-runtime/index.js";
7
+ import { atomicWriteJSON, hitchRootId, runCommand } from "../src/foundation/index.js";
8
+ import { parseCanaryWorkerConfig } from "./canary-worker-peer.js";
9
+ export function shellQuote(value) { return `'${value.replaceAll("'", `'"'"'`)}'`; }
10
+ export class RemoteCanaryWorker {
11
+ config;
12
+ localRoot;
13
+ localWorkerRoot;
14
+ id = `hitch-canary-${randomBytes(16).toString("hex")}`;
15
+ prepared = false;
16
+ constructor(config, localRoot, localWorkerRoot) {
17
+ this.config = config;
18
+ this.localRoot = localRoot;
19
+ this.localWorkerRoot = localWorkerRoot;
20
+ }
21
+ get workerRoot() { return path.posix.join(this.config.runs_directory, this.id, "worker"); }
22
+ sshOptions() {
23
+ return ["-F", this.config.ssh_config, "-o", "BatchMode=yes", "-o", "ConnectTimeout=10",
24
+ "-o", "ExitOnForwardFailure=yes", "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=3"];
25
+ }
26
+ launch(operation, details = {}, controllerPort) {
27
+ const forward = controllerPort === undefined ? [] : ["-R", `127.0.0.1:${this.config.remote_port}:127.0.0.1:${controllerPort}`];
28
+ const command = [this.config.node, path.posix.join(this.config.hitch, "dist/scripts/canary-worker-peer.js")].map(shellQuote).join(" ");
29
+ const child = spawn("ssh", [...this.sshOptions(), ...forward, this.config.ssh_host, command], { stdio: ["pipe", "pipe", "pipe"] });
30
+ child.stdin.on("error", () => { });
31
+ child.stdin.end(JSON.stringify({ operation, config: this.config, id: this.id, ...details }));
32
+ return child;
33
+ }
34
+ async request(operation, details = {}) {
35
+ const child = this.launch(operation, details);
36
+ let stdout = "", stderr = "", oversized = false, timedOut = false;
37
+ child.stdout.on("data", chunk => {
38
+ stdout += chunk.toString();
39
+ if (stdout.length > 1024 * 1024) {
40
+ oversized = true;
41
+ child.kill("SIGTERM");
42
+ }
43
+ });
44
+ child.stderr.on("data", chunk => { stderr = (stderr + chunk.toString()).slice(-32_768); });
45
+ const timer = setTimeout(() => { timedOut = true; child.kill("SIGTERM"); }, 60_000);
46
+ try {
47
+ const code = await new Promise((resolve, reject) => { child.once("error", reject); child.once("close", resolve); });
48
+ assert.equal(timedOut, false, "SSH worker observation timed out; remote state remains unconfirmed");
49
+ assert.equal(oversized, false, "SSH worker response exceeded the diagnostic bound");
50
+ assert.equal(code, 0, `SSH canary ${operation} failed: ${stderr}`);
51
+ return JSON.parse(stdout);
52
+ }
53
+ finally {
54
+ clearTimeout(timer);
55
+ }
56
+ }
57
+ async observe(imageId, localEngine) {
58
+ await atomicWriteJSON(path.join(this.localRoot, "remote-worker-location.json"), {
59
+ diagnosticId: this.id, sshHost: this.config.ssh_host, workerRoot: this.workerRoot,
60
+ });
61
+ const observed = await this.request("observe", { imageId });
62
+ assert.equal(observed.imageId, imageId);
63
+ assert.equal(observed.harbor, "0.21.0");
64
+ assert.ok(typeof observed.engine === "string" && observed.engine.length > 0);
65
+ assert.notEqual(observed.engine, localEngine, "cross-host canary requires a different Docker engine");
66
+ assert.equal(typeof observed.bootId, "string");
67
+ assert.match(String(observed.bootId), /^[a-f0-9-]{36}$/);
68
+ assert.deepEqual(observed.runtime, await observeControllerRuntime(), "remote worker runtime differs from controller");
69
+ await atomicWriteJSON(path.join(this.localRoot, "remote-worker-observation.json"), observed);
70
+ return { engine: observed.engine, harbor: "0.21.0" };
71
+ }
72
+ async start(controllerPort, registrationFile, credentialFile) {
73
+ assert.ok(Number.isSafeInteger(controllerPort) && controllerPort > 0 && controllerPort <= 65535);
74
+ this.prepared = true; // An uncertain prepare must still be followed by a remote stop.
75
+ const prepared = await this.request("prepare", {
76
+ registration: JSON.parse(await readFile(registrationFile, "utf8")),
77
+ credential: JSON.parse(await readFile(credentialFile, "utf8")),
78
+ });
79
+ assert.equal(prepared.prepared, true, "remote worker did not confirm preparation");
80
+ return this.launch("run", {}, controllerPort);
81
+ }
82
+ async stop() {
83
+ if (!this.prepared)
84
+ return;
85
+ const result = await this.request("stop");
86
+ assert.equal(result.workerProcessTerminal, true);
87
+ await atomicWriteJSON(path.join(this.localRoot, "remote-worker-process-stop.json"), result);
88
+ }
89
+ async collect() {
90
+ if (!this.prepared)
91
+ return;
92
+ await mkdir(this.localWorkerRoot, { recursive: true, mode: 0o700 });
93
+ const sshCommand = ["ssh", ...this.sshOptions()].map(shellQuote).join(" ");
94
+ // Copy only this canary's worker state. Registration credentials are in a
95
+ // sibling private directory and never included. Do not follow symlinks.
96
+ await runCommand("rsync", ["-r", "--safe-links", "--checksum", "--partial", "--timeout=60", "-e", sshCommand,
97
+ `${this.config.ssh_host}:${shellQuote(`${this.workerRoot}/`)}`, `${this.localWorkerRoot}/`], { timeoutMs: 90_000 });
98
+ }
99
+ async cleanup(leaseIds) {
100
+ if (!this.prepared)
101
+ return { issues: [], retained: [] };
102
+ const result = await this.request("cleanup", { leaseIds });
103
+ assert.equal(result.root_id, hitchRootId(this.workerRoot));
104
+ assert.ok(Array.isArray(result.issues) && Array.isArray(result.retained));
105
+ await atomicWriteJSON(path.join(this.localRoot, "remote-worker-docker-cleanup.json"), result);
106
+ return { issues: result.issues, retained: result.retained };
107
+ }
108
+ }
109
+ export async function remoteCanaryWorker(localRoot, localWorkerRoot) {
110
+ const file = process.env.HITCH_CANARY_WORKER_SSH_CONFIG;
111
+ return file ? new RemoteCanaryWorker(parseCanaryWorkerConfig(JSON.parse(await readFile(file, "utf8"))), localRoot, localWorkerRoot) : undefined;
112
+ }
113
+ //# sourceMappingURL=canary-worker-host.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"canary-worker-host.js","sourceRoot":"","sources":["../../scripts/canary-worker-host.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,oBAAoB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAqB,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,MAAM,oCAAoC,CAAC;AAC9E,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACtF,OAAO,EAAE,uBAAuB,EAA2B,MAAM,yBAAyB,CAAC;AAE3F,MAAM,UAAU,UAAU,CAAC,KAAa,IAAY,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;AAEnG,MAAM,OAAO,kBAAkB;IAGR;IAAqC;IAA4B;IAF7E,EAAE,GAAG,gBAAgB,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IACxD,QAAQ,GAAG,KAAK,CAAC;IACzB,YAAqB,MAA0B,EAAW,SAAiB,EAAW,eAAuB;QAAxF,WAAM,GAAN,MAAM,CAAoB;QAAW,cAAS,GAAT,SAAS,CAAQ;QAAW,oBAAe,GAAf,eAAe,CAAQ;IAAG,CAAC;IAEjH,IAAI,UAAU,KAAa,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC3F,UAAU;QAChB,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,IAAI,EAAE,mBAAmB;YACpF,IAAI,EAAE,0BAA0B,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,EAAE,uBAAuB,CAAC,CAAC;IACrG,CAAC;IACO,MAAM,CAAC,SAAiB,EAAE,UAAmC,EAAE,EAAE,cAAuB;QAC9F,MAAM,OAAO,GAAG,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,aAAa,IAAI,CAAC,MAAM,CAAC,WAAW,cAAc,cAAc,EAAE,CAAC,CAAC;QAC/H,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,oCAAoC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvI,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QACnI,KAAK,CAAC,KAAM,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACnC,KAAK,CAAC,KAAM,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC,CAAC;QAC9F,OAAO,KAAK,CAAC;IACf,CAAC;IACO,KAAK,CAAC,OAAO,CAAC,SAAiB,EAAE,UAAmC,EAAE;QAC5E,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,EAAE,EAAE,SAAS,GAAG,KAAK,EAAE,QAAQ,GAAG,KAAK,CAAC;QAClE,KAAK,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YAC/B,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3B,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;gBAAC,SAAS,GAAG,IAAI,CAAC;gBAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QACH,KAAK,CAAC,MAAO,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5F,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;QACpF,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,IAAI,OAAO,CAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnI,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,EAAE,oEAAoE,CAAC,CAAC;YACpG,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,mDAAmD,CAAC,CAAC;YACpF,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,cAAc,SAAS,YAAY,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;gBAAS,CAAC;YAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAe,EAAE,WAAmB;QAChD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,6BAA6B,CAAC,EAAE;YAC9E,YAAY,EAAE,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU;SAClF,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxC,MAAM,CAAC,EAAE,CAAC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC7E,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW,EAAE,sDAAsD,CAAC,CAAC;QACtG,MAAM,CAAC,KAAK,CAAC,OAAO,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,iBAAiB,CAAC,CAAC;QACzD,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,wBAAwB,EAAE,EAAE,+CAA+C,CAAC,CAAC;QACtH,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC7F,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,cAAsB,EAAE,gBAAwB,EAAE,cAAsB;QAClF,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,IAAI,cAAc,IAAI,KAAK,CAAC,CAAC;QACjG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,gEAAgE;QACtF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE;YAC7C,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;YAClE,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;SAC/D,CAAC,CAAC;QACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,EAAE,2CAA2C,CAAC,CAAC;QACnF,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,EAAE,cAAc,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;QACjD,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,iCAAiC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,MAAM,KAAK,CAAC,IAAI,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACpE,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3E,0EAA0E;QAC1E,wEAAwE;QACxE,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,IAAI,EAAE,UAAU;YAC1G,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;IACxH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAkB;QAC9B,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QACxD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAC3D,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,MAAM,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,mCAAmC,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9F,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC9D,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,eAAuB;IACjF,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC;IACxD,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,kBAAkB,CAAC,uBAAuB,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClJ,CAAC"}