@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -0,0 +1,2823 @@
1
+ // src/sandbox-snapshot-diagnostics.ts
2
+ import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
3
+ function safeSnapshotError(error, origin = "sandbox-resume") {
4
+ const fields = {
5
+ errorClass: "SnapshotOperationError",
6
+ errorCode: "snapshot_operation_failed",
7
+ origin
8
+ };
9
+ try {
10
+ if (error && typeof error === "object") {
11
+ const candidate = error;
12
+ if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
13
+ fields.causeName = candidate.name;
14
+ }
15
+ if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
16
+ fields.integrityCode = candidate.code;
17
+ }
18
+ const status = Number(candidate.status ?? candidate.statusCode);
19
+ if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
20
+ }
21
+ } catch {
22
+ }
23
+ const read = (value, key) => {
24
+ try {
25
+ return value && typeof value === "object" ? Reflect.get(value, key) : void 0;
26
+ } catch {
27
+ return void 0;
28
+ }
29
+ };
30
+ const details = read(error, "details");
31
+ const providerName = read(details, "errorName");
32
+ if (typeof providerName === "string" && [
33
+ "ClientError",
34
+ "TimeoutError",
35
+ "ConnectionError",
36
+ "AuthError",
37
+ "NotFoundError",
38
+ "InvalidError",
39
+ "RemoteError",
40
+ "AbortError"
41
+ ].includes(providerName))
42
+ fields.providerErrorName = providerName;
43
+ const grpcCode = read(details, "errorCode");
44
+ if (providerName === "ClientError" && typeof grpcCode === "number" && Number.isInteger(grpcCode) && grpcCode >= 0 && grpcCode <= 16)
45
+ fields.providerGrpcCode = grpcCode;
46
+ for (const key of ["status", "httpStatus", "responseStatus"]) {
47
+ const status = read(details, key);
48
+ if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) {
49
+ fields.providerHttpStatus = status;
50
+ break;
51
+ }
52
+ }
53
+ const retryable = read(error, "retryable");
54
+ if (typeof retryable === "boolean") fields.providerRetryable = retryable;
55
+ return fields;
56
+ }
57
+ function warnDrainSnapshotFailure(observability, message, error, lease) {
58
+ try {
59
+ observability.warn(message, {
60
+ ...safeSnapshotError(error, "worker-lifecycle"),
61
+ ...lease.workspaceId ? { sandboxLeaseKey: sandboxLeaseTelemetryKey(lease.workspaceId, lease.sandboxGroupId) } : {},
62
+ leaseEpoch: lease.leaseEpoch
63
+ });
64
+ } catch {
65
+ }
66
+ }
67
+
68
+ // src/session-event-fanout.ts
69
+ async function publishDurableSessionEvents(bus, workspaceId, events, onError = () => void 0) {
70
+ if (!bus || !events?.length) return;
71
+ for (const sessionId of new Set(events.map((event) => event.sessionId))) {
72
+ await bus.publish(
73
+ workspaceId,
74
+ sessionId,
75
+ events.filter((event) => event.sessionId === sessionId)
76
+ ).catch(onError);
77
+ }
78
+ }
79
+
80
+ // src/sandbox-archive-storage.ts
81
+ import { uploadWorkspaceArchiveSpool } from "@opengeni/storage";
82
+ import { randomUUID } from "crypto";
83
+ import {
84
+ parseWorkspaceArchiveObjectRef,
85
+ workspaceArchiveObjectKey
86
+ } from "@opengeni/contracts";
87
+ var WorkspaceArchiveObjectStorageRequiredError = class extends Error {
88
+ code = "workspace_archive_object_storage_required";
89
+ constructor(backend) {
90
+ super(`${backend} workspace archives require object storage`);
91
+ this.name = "WorkspaceArchiveObjectStorageRequiredError";
92
+ }
93
+ };
94
+ async function putTarWorkspaceArchiveObject(input) {
95
+ const descriptor = structuredClone(input.archive.descriptor);
96
+ if (descriptor.version !== 1) {
97
+ throw new Error("Object-storage workspace archives are portable tar only");
98
+ }
99
+ const key = workspaceArchiveObjectKey({
100
+ accountId: input.accountId,
101
+ workspaceId: input.workspaceId,
102
+ sandboxGroupId: input.sandboxGroupId,
103
+ revision: descriptor.revision,
104
+ uploadId: randomUUID()
105
+ });
106
+ const archive = input.archive;
107
+ if ("spool" in archive) {
108
+ if (archive.spool.byteSize !== descriptor.archiveBytes || archive.spool.sha256 !== descriptor.archiveSha256) {
109
+ throw new Error("Workspace archive spool does not match its verified descriptor");
110
+ }
111
+ }
112
+ const source = "spool" in archive ? archive.spool : {
113
+ path: "",
114
+ byteSize: archive.bytes.byteLength,
115
+ sha256: descriptor.archiveSha256,
116
+ async *open() {
117
+ for (let offset = 0; offset < archive.bytes.length; offset += 1024 * 1024)
118
+ yield archive.bytes.subarray(offset, offset + 1024 * 1024);
119
+ },
120
+ async dispose() {
121
+ }
122
+ };
123
+ try {
124
+ await uploadWorkspaceArchiveSpool(input.objectStorage, key, {
125
+ ...source,
126
+ byteSize: descriptor.archiveBytes,
127
+ sha256: descriptor.archiveSha256,
128
+ open: source.open.bind(source)
129
+ });
130
+ } catch (error) {
131
+ await input.objectStorage.deleteObject(key).catch(() => void 0);
132
+ throw error;
133
+ }
134
+ return {
135
+ schema: "sandbox_archive_object_v1",
136
+ key,
137
+ sha256: descriptor.archiveSha256,
138
+ bytes: descriptor.archiveBytes,
139
+ backend: input.objectStorage.backend
140
+ };
141
+ }
142
+ async function persistWorkspaceArchiveCandidate(input) {
143
+ const objectStorage = input.objectStorage;
144
+ const ref = input.ref ? { ...input.ref } : void 0;
145
+ const result = await input.persist();
146
+ if (objectStorage && ref && result.candidateDisposition === "unused") {
147
+ await deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref);
148
+ }
149
+ return result;
150
+ }
151
+ async function putVersion1TarArchiveOrInline(input) {
152
+ if (input.archive.descriptor.version !== 1) {
153
+ if ("spool" in input.archive) throw new Error("Native snapshots cannot use a portable spool");
154
+ return { workspaceArchive: input.archive.base64 };
155
+ }
156
+ if (input.backend === "opensandbox" && !input.objectStorage) {
157
+ throw new WorkspaceArchiveObjectStorageRequiredError("opensandbox");
158
+ }
159
+ if (!input.objectStorage) {
160
+ if ("spool" in input.archive)
161
+ throw new WorkspaceArchiveObjectStorageRequiredError(input.backend);
162
+ return { workspaceArchive: input.archive.base64 };
163
+ }
164
+ try {
165
+ const workspaceArchiveRef = await putTarWorkspaceArchiveObject({
166
+ objectStorage: input.objectStorage,
167
+ accountId: input.accountId,
168
+ workspaceId: input.workspaceId,
169
+ sandboxGroupId: input.sandboxGroupId,
170
+ archive: input.archive
171
+ });
172
+ try {
173
+ input.metrics?.onWorkspaceArchiveObject?.({
174
+ outcome: "put",
175
+ backend: input.objectStorage.backend
176
+ });
177
+ } catch {
178
+ }
179
+ console.info("workspace archive object put", {
180
+ key: workspaceArchiveRef.key,
181
+ bytes: workspaceArchiveRef.bytes,
182
+ sha256: workspaceArchiveRef.sha256
183
+ });
184
+ return { workspaceArchiveRef };
185
+ } catch (error) {
186
+ try {
187
+ input.metrics?.onWorkspaceArchiveObject?.({
188
+ outcome: "put_failed",
189
+ backend: input.objectStorage.backend
190
+ });
191
+ } catch {
192
+ }
193
+ throw error;
194
+ }
195
+ }
196
+ async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metrics) {
197
+ if (!objectStorage || !ref) return;
198
+ try {
199
+ await objectStorage.deleteObject(ref.key);
200
+ try {
201
+ metrics?.onWorkspaceArchiveObject?.({
202
+ outcome: "deleted_unpublished",
203
+ backend: objectStorage.backend
204
+ });
205
+ } catch {
206
+ }
207
+ } catch {
208
+ console.error("unpublished workspace archive object delete failed", { key: ref.key });
209
+ }
210
+ }
211
+
212
+ // src/sandbox-resume.ts
213
+ import {
214
+ effectiveSandboxLifecycle,
215
+ sandboxArchiveCaptureTimeoutMs,
216
+ sandboxLifecycleTransitionWaitMs,
217
+ sandboxWarmRateMicrosPerSecond
218
+ } from "@opengeni/config";
219
+ import { randomUUID as randomUUID2 } from "crypto";
220
+
221
+ // src/warm-capture-holder.ts
222
+ function retainsHolderForWarmCapture(lease, expectedEpoch, now = Date.now()) {
223
+ return Boolean(
224
+ lease?.liveness === "warm" && lease.leaseEpoch === expectedEpoch && lease.archiveCapture && lease.archiveCapture.deadlineAt.getTime() > now
225
+ );
226
+ }
227
+
228
+ // src/sandbox-resume.ts
229
+ import {
230
+ acquireLease,
231
+ adoptLegacyModalCheckpointArtifact,
232
+ beginSandboxRematerialization,
233
+ sessionHoldsFreshWorkspaceRecovery,
234
+ claimWorkspaceArchiveCapture,
235
+ commitWarmingToWarm,
236
+ failSandboxRematerialization,
237
+ failWarmingToCold,
238
+ beginModalProviderCreate,
239
+ getSandboxSessionEnvelope,
240
+ heartbeatLeaseHolderStatus,
241
+ markSandboxRestoreVerifying,
242
+ markWarmLeaseInstanceLost,
243
+ markSandboxCheckpointArtifactDeletePending,
244
+ persistWarmSnapshot,
245
+ readLease,
246
+ registerSandboxCheckpointArtifact,
247
+ recordWarmingSandboxCreated,
248
+ reinstateTurnLeaseHolder,
249
+ leaseMayHaveUntrackedWriters,
250
+ releaseWorkspaceArchiveCapture,
251
+ releaseLeaseHolder,
252
+ touchLeaseHolder,
253
+ SandboxLeaseRecoveryBlockedError,
254
+ SandboxLeaseSupersededError,
255
+ SandboxLeaseTransitionError
256
+ } from "@opengeni/db";
257
+ import {
258
+ captureWorkspaceArchiveForStorage,
259
+ disposeWorkspaceArchive,
260
+ describeLegacyNativeSnapshotArchive,
261
+ inlineWorkspaceArchiveForRestore,
262
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
263
+ SandboxExecReadinessError,
264
+ WorkspaceArchiveIntegrityError,
265
+ establishSandboxSessionFromEnvelope,
266
+ isProviderSandboxNotFoundError,
267
+ parseWorkspaceArchiveDescriptor,
268
+ providerWorkspaceCapturePolicy,
269
+ SandboxProviderContinuityUnavailableError,
270
+ requirePersistableReplacementSandboxEnvelope,
271
+ renewSandboxProviderExpiration,
272
+ modalSessionMatchesCheckpointProviderBinding,
273
+ resolveModalCheckpointProviderBindingForSession,
274
+ serializeReplacementSandboxEnvelope,
275
+ tagModalSandbox,
276
+ terminateUnpublishedSandboxSession,
277
+ verifySandboxExecReadiness,
278
+ withoutSandboxProviderIdentity,
279
+ workspaceArchiveDownloadTemporaryDirectory
280
+ } from "@opengeni/runtime";
281
+ import {
282
+ downloadWorkspaceArchiveSpool,
283
+ WorkspaceArchiveStorageError
284
+ } from "@opengeni/storage";
285
+ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni/observability";
286
+ import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
287
+ function sandboxLeaseHolderIdForAttempt(attemptId) {
288
+ const normalized = attemptId.trim();
289
+ if (!normalized) {
290
+ throw new Error("Sandbox lease holder requires a turn attempt id");
291
+ }
292
+ return `turn-attempt:${normalized}`;
293
+ }
294
+ function isRetryableDegradedRestore(restore) {
295
+ return restore.status === "degraded" && restore.retryable === true;
296
+ }
297
+ var SandboxProvisionStageError = class extends Error {
298
+ constructor(stage, source) {
299
+ super(source instanceof Error ? source.message : String(source), { cause: source });
300
+ this.stage = stage;
301
+ this.source = source;
302
+ }
303
+ name = "SandboxProvisionStageError";
304
+ };
305
+ var sandboxProvisionFailureStages = /* @__PURE__ */ new WeakMap();
306
+ function sandboxProvisionFailureStage(error) {
307
+ if (!error || typeof error !== "object") return null;
308
+ return sandboxProvisionFailureStages.get(error) ?? (error instanceof SandboxProvisionStageError ? error.stage : null);
309
+ }
310
+ function sandboxProvisionStageError(stage, error) {
311
+ if (error && typeof error === "object") {
312
+ sandboxProvisionFailureStages.set(error, stage);
313
+ return error;
314
+ }
315
+ return new SandboxProvisionStageError(stage, error);
316
+ }
317
+ var SandboxWarmingTimeoutError = class extends Error {
318
+ constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
319
+ const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
320
+ const identity = [
321
+ sandboxGroupId ? `group ${sandboxGroupId}` : null,
322
+ instanceId ? `sandbox ${instanceId}` : null
323
+ ].filter((part) => part !== null).join(", ");
324
+ const target = identity ? ` (${identity})` : "";
325
+ super(
326
+ stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
327
+ );
328
+ this.backend = backend;
329
+ this.timeoutMs = timeoutMs;
330
+ this.stage = stage;
331
+ this.sandboxGroupId = sandboxGroupId;
332
+ this.instanceId = instanceId;
333
+ this.name = "SandboxWarmingTimeoutError";
334
+ }
335
+ };
336
+ var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
337
+ code = "sandbox_exec_readiness_timeout";
338
+ constructor(backend, timeoutMs, identity = {}) {
339
+ super(
340
+ backend,
341
+ timeoutMs,
342
+ "exec_readiness",
343
+ identity.sandboxGroupId ?? null,
344
+ identity.instanceId ?? null
345
+ );
346
+ this.name = "SandboxExecReadinessTimeoutError";
347
+ }
348
+ };
349
+ var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
350
+ code = "sandbox_sibling_warming_timeout";
351
+ constructor(backend, timeoutMs, identity) {
352
+ super(
353
+ backend,
354
+ timeoutMs,
355
+ "sibling_warming",
356
+ identity.sandboxGroupId,
357
+ identity.instanceId ?? null
358
+ );
359
+ this.name = "SandboxSiblingWarmingTimeoutError";
360
+ }
361
+ };
362
+ var disposedUnreadyFreshSandboxErrors = /* @__PURE__ */ new WeakSet();
363
+ function isReplaceableFreshSandboxReadinessTimeout(error) {
364
+ return error instanceof SandboxExecReadinessTimeoutError && disposedUnreadyFreshSandboxErrors.has(error);
365
+ }
366
+ var FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2e3;
367
+ var FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8e3;
368
+ function freshSandboxReadinessReplacementDelayMs(random = Math.random) {
369
+ const sample = random();
370
+ const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
371
+ return FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS + Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS);
372
+ }
373
+ var FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
374
+ function createFreshSandboxReadinessReplacementBudget() {
375
+ return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
376
+ }
377
+ function recordFreshSandboxReadinessReplacement(metrics, backend, outcome) {
378
+ try {
379
+ metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
380
+ } catch {
381
+ }
382
+ }
383
+ async function sleepUnlessCancelled(ms, signal) {
384
+ if (signal?.aborted) return false;
385
+ if (ms <= 0) return true;
386
+ return await new Promise((resolve) => {
387
+ const onAbort = () => {
388
+ clearTimeout(timer);
389
+ resolve(false);
390
+ };
391
+ const timer = setTimeout(() => {
392
+ signal?.removeEventListener("abort", onAbort);
393
+ resolve(true);
394
+ }, ms);
395
+ signal?.addEventListener("abort", onAbort, { once: true });
396
+ });
397
+ }
398
+ var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
399
+ constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
400
+ super(sandboxGroupId, leaseEpoch);
401
+ this.lostInstanceId = lostInstanceId;
402
+ this.name = "SandboxLeaseInstanceLostError";
403
+ }
404
+ };
405
+ var WARMING_POLL_INTERVAL_MS = 250;
406
+ async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}, signal) {
407
+ try {
408
+ await verifySandboxExecReadiness(established, timeoutMs, signal);
409
+ } catch (error) {
410
+ if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
411
+ throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
412
+ sandboxGroupId: identity.sandboxGroupId ?? null,
413
+ instanceId: established.instanceId
414
+ });
415
+ }
416
+ throw error;
417
+ }
418
+ }
419
+ var SnapshotTimeoutError = class extends Error {
420
+ constructor(timeoutMs) {
421
+ super(`workspace snapshot timed out after ${timeoutMs}ms`);
422
+ this.name = "SnapshotTimeoutError";
423
+ }
424
+ };
425
+ async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
426
+ if (signal?.aborted) return false;
427
+ let timeout;
428
+ let cancelListener;
429
+ try {
430
+ await Promise.race([
431
+ snapshot,
432
+ ...signal ? [
433
+ new Promise((_resolve, reject) => {
434
+ cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
435
+ signal.addEventListener("abort", cancelListener, { once: true });
436
+ if (signal.aborted) cancelListener();
437
+ })
438
+ ] : [],
439
+ new Promise((_, reject) => {
440
+ timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
441
+ if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
442
+ timeout.unref();
443
+ }
444
+ })
445
+ ]);
446
+ return true;
447
+ } catch (error) {
448
+ if (signal?.aborted) return false;
449
+ if (error instanceof SnapshotTimeoutError) {
450
+ console.error(
451
+ "mid-session workspace snapshot wait timed out (turn unaffected)",
452
+ safeSnapshotError(error)
453
+ );
454
+ return false;
455
+ }
456
+ return true;
457
+ } finally {
458
+ if (timeout) {
459
+ clearTimeout(timeout);
460
+ }
461
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
462
+ }
463
+ }
464
+ async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
465
+ if (signal?.aborted) {
466
+ throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
467
+ }
468
+ const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
469
+ let timeout;
470
+ let cancelListener;
471
+ try {
472
+ return await Promise.race([
473
+ pending,
474
+ ...signal ? [
475
+ new Promise((_resolve, reject) => {
476
+ cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
477
+ signal.addEventListener("abort", cancelListener, { once: true });
478
+ if (signal.aborted) cancelListener();
479
+ })
480
+ ] : [],
481
+ new Promise((_resolve, reject) => {
482
+ timeout = setTimeout(
483
+ () => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
484
+ settings.sandboxSnapshotTimeoutMs
485
+ );
486
+ timeout.unref?.();
487
+ })
488
+ ]);
489
+ } finally {
490
+ if (timeout) clearTimeout(timeout);
491
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
492
+ }
493
+ }
494
+ async function terminateEstablishedSandbox(established) {
495
+ if (!established) return true;
496
+ try {
497
+ await terminateUnpublishedSandboxSession(established);
498
+ return true;
499
+ } catch (error) {
500
+ if (isProviderSandboxNotFoundError(established.backendId, error)) {
501
+ return true;
502
+ }
503
+ return false;
504
+ }
505
+ }
506
+ function recordSandboxWarmingTimeout(metrics, error) {
507
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
508
+ return;
509
+ }
510
+ try {
511
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
512
+ } catch {
513
+ }
514
+ }
515
+ function workspaceArchiveFieldsFromEnvelope(envelope) {
516
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
517
+ const archive = sessionState?.workspaceArchive;
518
+ const archiveRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchiveRef);
519
+ const hasInline = typeof archive === "string" && archive.length > 0;
520
+ if (!hasInline && !archiveRef) {
521
+ return null;
522
+ }
523
+ const previous = sessionState?.workspaceArchivePrev;
524
+ const previousRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchivePrevRef);
525
+ const metadata = sessionState?.workspaceArchiveMeta;
526
+ const previousMetadata = sessionState?.workspaceArchivePrevMeta;
527
+ const capturedAt = sessionState?.workspaceArchiveAt;
528
+ return {
529
+ ...hasInline && !archiveRef ? { workspaceArchive: archive } : {},
530
+ ...archiveRef ? { workspaceArchiveRef: archiveRef } : {},
531
+ ...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
532
+ ...typeof previous === "string" && previous.length > 0 && !previousRef ? { workspaceArchivePrev: previous } : {},
533
+ ...previousRef ? { workspaceArchivePrevRef: previousRef } : {},
534
+ ...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
535
+ ...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
536
+ };
537
+ }
538
+ async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
539
+ if (!envelope || typeof envelope !== "object") return envelope;
540
+ const record = envelope;
541
+ const sessionState = record.sessionState && typeof record.sessionState === "object" && !Array.isArray(record.sessionState) ? record.sessionState : null;
542
+ if (!sessionState || !parseWorkspaceArchiveObjectRef2(sessionState.workspaceArchiveRef)) {
543
+ return envelope;
544
+ }
545
+ if (!objectStorage) {
546
+ throw new WorkspaceArchiveIntegrityError(
547
+ "archive_storage_unavailable",
548
+ "workspace archive object storage is not configured",
549
+ { retryable: true }
550
+ );
551
+ }
552
+ const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
553
+ if (process.platform === "linux" && descriptor?.version === 1 && descriptor.workspace.projection === "sdk_local_archive_v1") {
554
+ return envelope;
555
+ }
556
+ return {
557
+ ...record,
558
+ sessionState: await inlineWorkspaceArchiveForRestore(
559
+ sessionState,
560
+ (key) => objectStorage.getObjectBytes(key)
561
+ )
562
+ };
563
+ }
564
+ function legacyNativeArchiveFromEnvelope(envelope) {
565
+ const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
566
+ if (!sessionState) return null;
567
+ const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
568
+ if (existing?.version === 2) return null;
569
+ return describeLegacyNativeSnapshotArchive(
570
+ sessionState.workspaceArchive,
571
+ existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
572
+ );
573
+ }
574
+ function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
575
+ let continuationRegistered = false;
576
+ let resolvePhysicalSettlement = () => void 0;
577
+ const physicalSettlement = new Promise((resolve) => {
578
+ resolvePhysicalSettlement = resolve;
579
+ });
580
+ const result = persistWarmWorkspaceSnapshot(
581
+ services,
582
+ ids,
583
+ session,
584
+ leaseEpoch,
585
+ signal,
586
+ force,
587
+ (continuation) => {
588
+ continuationRegistered = true;
589
+ void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
590
+ }
591
+ );
592
+ void result.then(
593
+ () => {
594
+ if (!continuationRegistered) resolvePhysicalSettlement();
595
+ },
596
+ () => {
597
+ if (!continuationRegistered) resolvePhysicalSettlement();
598
+ }
599
+ );
600
+ return Object.assign(result, { settled: physicalSettlement });
601
+ }
602
+ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force, registerOwnedContinuation) {
603
+ const { db, settings } = services;
604
+ const warnSnapshotFailure = (message, error) => {
605
+ const fields = {
606
+ ...safeSnapshotError(error),
607
+ sandboxLeaseKey: sandboxLeaseTelemetryKey2(ids.workspaceId, ids.sandboxGroupId),
608
+ leaseEpoch
609
+ };
610
+ try {
611
+ if (services.observability) services.observability.warn(message, fields);
612
+ else console.warn(message, fields);
613
+ } catch {
614
+ }
615
+ };
616
+ const reportSkipped = (backend, reason) => {
617
+ try {
618
+ services.sandboxMetrics?.onWorkspaceCaptureSkipped?.({
619
+ backend: typeof backend === "string" ? backend : "unknown",
620
+ reason
621
+ });
622
+ } catch {
623
+ }
624
+ };
625
+ const intervalMs = settings.sandboxSnapshotIntervalMs;
626
+ if (intervalMs <= 0 && !force) {
627
+ return false;
628
+ }
629
+ const persistable = session;
630
+ if (typeof persistable.persistWorkspace !== "function" && typeof persistable.persistWorkspaceTar !== "function") {
631
+ console.error("mid-session workspace snapshot skipped (no persist primitive)", {
632
+ backendId: typeof persistable.backendId === "string" ? persistable.backendId : null
633
+ });
634
+ reportSkipped(persistable.backendId, "no_persist_primitive");
635
+ return false;
636
+ }
637
+ const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
638
+ if (signal?.aborted) {
639
+ return false;
640
+ }
641
+ try {
642
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
643
+ const canWarmCapture = lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
644
+ const canForceDrainingCapture = force === true && lease?.liveness === "draining" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
645
+ const turnOwner = ids.idleCheckpoint === true ? null : ids;
646
+ if (!lease || !canWarmCapture && (!canForceDrainingCapture || !turnOwner)) {
647
+ console.error("mid-session workspace snapshot skipped (lease not warm)", {
648
+ sandboxGroupId: ids.sandboxGroupId,
649
+ expectedEpoch: leaseEpoch,
650
+ liveness: lease?.liveness ?? null,
651
+ leaseEpoch: lease?.leaseEpoch ?? null,
652
+ hasInstance: lease?.instanceId !== null && lease?.instanceId !== void 0
653
+ });
654
+ reportSkipped(lease?.backend ?? persistable.backendId, "lease_not_warm");
655
+ return false;
656
+ }
657
+ if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) {
658
+ return false;
659
+ }
660
+ if (lease.instanceId === null) {
661
+ return false;
662
+ }
663
+ const instanceId = lease.instanceId;
664
+ const captureLiveness = canWarmCapture ? "warm" : "draining";
665
+ const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
666
+ if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
667
+ console.error("mid-session workspace snapshot skipped (capture policy)", {
668
+ sandboxGroupId: ids.sandboxGroupId,
669
+ backend: lease.backend,
670
+ liveInstance: capturePolicy?.liveInstance ?? null
671
+ });
672
+ reportSkipped(lease.backend, "capture_policy");
673
+ return false;
674
+ }
675
+ const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
676
+ if (!turnOwner && !nativeModalPersistence) {
677
+ reportSkipped(lease.backend, "capture_policy");
678
+ return false;
679
+ }
680
+ const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
681
+ if (signal?.aborted) return false;
682
+ const captureId = randomUUID2();
683
+ const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
684
+ const claimed = await claimWorkspaceArchiveCapture(db, {
685
+ accountId: ids.accountId,
686
+ workspaceId: ids.workspaceId,
687
+ sandboxGroupId: ids.sandboxGroupId,
688
+ captureId,
689
+ expectedEpoch: leaseEpoch,
690
+ expectedInstanceId: instanceId,
691
+ liveness: captureLiveness,
692
+ captureTimeoutMs,
693
+ minIntervalMs: force ? 0 : intervalMs,
694
+ providerReplaySafe: capturePolicy.takeover === "same_request",
695
+ takeoverSafe: capturePolicy.takeover !== "exclusive",
696
+ // Modal pauses the box for a native snapshot, so the image is one
697
+ // instant even while background commands run.
698
+ pointInTimeCapture: nativeModalPersistence,
699
+ ...captureLiveness !== "warm" ? {} : turnOwner ? {
700
+ warmAttempt: {
701
+ sessionId: turnOwner.sessionId,
702
+ turnId: turnOwner.turnId,
703
+ attemptId: turnOwner.attemptId,
704
+ holderId: sandboxLeaseHolderIdForAttempt(turnOwner.attemptId)
705
+ }
706
+ } : { idleCheckpoint: true }
707
+ });
708
+ if (claimed.status !== "claimed") {
709
+ if (claimed.status === "throttled" || claimed.status === "clean") return false;
710
+ console.error("mid-session workspace snapshot skipped (capture claim)", {
711
+ sandboxGroupId: ids.sandboxGroupId,
712
+ status: claimed.status
713
+ });
714
+ reportSkipped(lease.backend, claimed.status);
715
+ return false;
716
+ }
717
+ let timeout;
718
+ const capturedAtMs = Date.now();
719
+ const registerCandidate = async (archive) => {
720
+ if (archive.kind === "host_spool" || archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
721
+ return null;
722
+ }
723
+ if (!checkpointBinding) {
724
+ throw new Error("Modal native snapshot has no exact session provider identity");
725
+ }
726
+ return await registerSandboxCheckpointArtifact(db, {
727
+ accountId: ids.accountId,
728
+ workspaceId: ids.workspaceId,
729
+ sandboxGroupId: ids.sandboxGroupId,
730
+ sourceLeaseId: claimed.claim.leaseId,
731
+ sourceLeaseEpoch: leaseEpoch,
732
+ sourceInstanceId: instanceId,
733
+ sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
734
+ providerBindingKey: checkpointBinding.key,
735
+ providerBinding: checkpointBinding.binding,
736
+ workspaceArchive: archive.base64,
737
+ workspaceArchiveMeta: archive.descriptor
738
+ });
739
+ };
740
+ const abandonCandidate = async (artifactId, reason) => {
741
+ await markSandboxCheckpointArtifactDeletePending(db, {
742
+ accountId: ids.accountId,
743
+ workspaceId: ids.workspaceId,
744
+ artifactId,
745
+ reason
746
+ }).catch(() => void 0);
747
+ };
748
+ const captureAbort = new AbortController();
749
+ const captureAbortTimer = setTimeout(
750
+ () => captureAbort.abort(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
751
+ settings.sandboxSnapshotTimeoutMs
752
+ );
753
+ captureAbortTimer.unref?.();
754
+ const captureAndPublish = (async () => {
755
+ const captureStarted = performance.now();
756
+ let captureOutcome = "failed";
757
+ let archive;
758
+ let candidate = null;
759
+ let publicationAttempted = false;
760
+ try {
761
+ archive = await captureWorkspaceArchiveForStorage(
762
+ session,
763
+ capturedAtMs,
764
+ {
765
+ requestId: claimed.claim.providerRequestId,
766
+ strategy: capturePolicy.strategy,
767
+ signal: captureAbort.signal
768
+ },
769
+ Boolean(services.objectStorage)
770
+ );
771
+ clearTimeout(captureAbortTimer);
772
+ candidate = await registerCandidate(archive);
773
+ const archiveDescriptor = archive.descriptor;
774
+ const published = await putVersion1TarArchiveOrInline({
775
+ backend: lease.backend,
776
+ objectStorage: services.objectStorage,
777
+ accountId: ids.accountId,
778
+ workspaceId: ids.workspaceId,
779
+ sandboxGroupId: ids.sandboxGroupId,
780
+ archive,
781
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
782
+ });
783
+ publicationAttempted = true;
784
+ const { wrote } = await persistWorkspaceArchiveCandidate({
785
+ ...services.objectStorage ? { objectStorage: services.objectStorage } : {},
786
+ ...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
787
+ persist: () => persistWarmSnapshot(db, {
788
+ accountId: ids.accountId,
789
+ workspaceId: ids.workspaceId,
790
+ ...turnOwner ? {
791
+ sessionId: turnOwner.sessionId,
792
+ turnId: turnOwner.turnId,
793
+ attemptId: turnOwner.attemptId
794
+ } : { idleCheckpoint: true },
795
+ sandboxGroupId: ids.sandboxGroupId,
796
+ expectedEpoch: leaseEpoch,
797
+ expectedInstanceId: instanceId,
798
+ expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
799
+ captureId,
800
+ workspaceArchiveMeta: archiveDescriptor,
801
+ ...published,
802
+ checkpointArtifactId: candidate?.id ?? null,
803
+ minIntervalMs: force ? 0 : intervalMs,
804
+ capturedAtMs
805
+ })
806
+ });
807
+ if (!wrote && candidate) {
808
+ await abandonCandidate(candidate.id, "snapshot_publication_fenced");
809
+ }
810
+ if (wrote) captureOutcome = "completed";
811
+ return wrote;
812
+ } catch (error) {
813
+ if (candidate && !publicationAttempted)
814
+ await abandonCandidate(candidate.id, "snapshot_capture_failed");
815
+ throw error;
816
+ } finally {
817
+ clearTimeout(captureAbortTimer);
818
+ await disposeWorkspaceArchive(archive).catch(() => {
819
+ console.warn("workspace archive spool cleanup failed");
820
+ });
821
+ await releaseWorkspaceArchiveCapture(db, {
822
+ accountId: ids.accountId,
823
+ workspaceId: ids.workspaceId,
824
+ sandboxGroupId: ids.sandboxGroupId,
825
+ captureId,
826
+ expectedEpoch: leaseEpoch,
827
+ expectedInstanceId: instanceId
828
+ }).catch((error) => {
829
+ captureOutcome = "failed";
830
+ warnSnapshotFailure("mid-session workspace capture gate release failed", error);
831
+ });
832
+ try {
833
+ services.sandboxMetrics?.onWorkspaceCapture?.({
834
+ backend: lease.backend,
835
+ outcome: captureOutcome,
836
+ durationSeconds: Math.max(0, performance.now() - captureStarted) / 1e3
837
+ });
838
+ } catch {
839
+ }
840
+ }
841
+ })();
842
+ const settled = captureAndPublish.then(
843
+ (persisted) => ({ kind: "settled", persisted }),
844
+ (error) => {
845
+ warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
846
+ return { kind: "settled", persisted: false };
847
+ }
848
+ );
849
+ registerOwnedContinuation(settled.then(() => void 0));
850
+ let cancelListener;
851
+ const outcome = await Promise.race([
852
+ settled,
853
+ ...signal ? [
854
+ new Promise((resolve) => {
855
+ cancelListener = () => resolve({ kind: "cancelled" });
856
+ signal.addEventListener("abort", cancelListener, { once: true });
857
+ if (signal.aborted) cancelListener();
858
+ })
859
+ ] : [],
860
+ new Promise((resolve) => {
861
+ timeout = setTimeout(
862
+ () => resolve({ kind: "timed_out" }),
863
+ settings.sandboxSnapshotTimeoutMs
864
+ );
865
+ })
866
+ ]).finally(() => {
867
+ if (timeout) clearTimeout(timeout);
868
+ if (cancelListener) signal?.removeEventListener("abort", cancelListener);
869
+ });
870
+ return outcome.kind === "settled" ? outcome.persisted : false;
871
+ } catch (error) {
872
+ warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
873
+ return false;
874
+ }
875
+ }
876
+ async function resumeBoxForTurn(services, ids, kind, holderId) {
877
+ try {
878
+ return await resumeBoxForTurnOnce(services, ids, kind, holderId);
879
+ } catch (error) {
880
+ if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
881
+ const readiness = error;
882
+ const warn = (message, fields) => {
883
+ if (services.observability) services.observability.warn(message, fields);
884
+ else console.warn(message, fields);
885
+ };
886
+ const identity = {
887
+ workspaceId: ids.workspaceId,
888
+ sessionId: ids.sessionId,
889
+ sandboxGroupId: ids.sandboxGroupId,
890
+ backend: readiness.backend,
891
+ instanceId: readiness.instanceId,
892
+ readinessTimeoutMs: readiness.timeoutMs
893
+ };
894
+ const budget = services.freshSandboxReadinessReplacementBudget ?? createFreshSandboxReadinessReplacementBudget();
895
+ if (budget.remaining <= 0) {
896
+ recordFreshSandboxReadinessReplacement(
897
+ services.sandboxMetrics,
898
+ readiness.backend,
899
+ "budget_spent"
900
+ );
901
+ warn(
902
+ "sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
903
+ identity
904
+ );
905
+ throw error;
906
+ }
907
+ budget.remaining -= 1;
908
+ const delayMs = await (services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs)();
909
+ warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
910
+ ...identity,
911
+ replacementDelayMs: delayMs
912
+ });
913
+ if (!await sleepUnlessCancelled(delayMs, services.cancellationSignal)) {
914
+ recordFreshSandboxReadinessReplacement(
915
+ services.sandboxMetrics,
916
+ readiness.backend,
917
+ "cancelled"
918
+ );
919
+ throw error;
920
+ }
921
+ try {
922
+ const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
923
+ recordFreshSandboxReadinessReplacement(
924
+ services.sandboxMetrics,
925
+ readiness.backend,
926
+ "replaced"
927
+ );
928
+ return replaced;
929
+ } catch (replacementError) {
930
+ recordFreshSandboxReadinessReplacement(
931
+ services.sandboxMetrics,
932
+ readiness.backend,
933
+ services.cancellationSignal?.aborted ? "cancelled" : replacementError instanceof SandboxExecReadinessTimeoutError ? "failed_again" : "replacement_failed"
934
+ );
935
+ throw replacementError;
936
+ }
937
+ }
938
+ }
939
+ async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
940
+ const { db, settings } = services;
941
+ const os = ids.os ?? "linux";
942
+ const leaseTtlMs = settings.sandboxLeaseTtlMs;
943
+ const cancellationSignal = services.cancellationSignal;
944
+ let releaseStarted = false;
945
+ let holderRelease = null;
946
+ let quiescedRelease = null;
947
+ let holderLivenessTimer;
948
+ let cancellationListener;
949
+ const release = (options) => {
950
+ const writersQuiesced = options?.workspaceWritersQuiesced === true;
951
+ if (!releaseStarted) {
952
+ releaseStarted = true;
953
+ if (cancellationListener) {
954
+ cancellationSignal?.removeEventListener("abort", cancellationListener);
955
+ cancellationListener = void 0;
956
+ }
957
+ if (holderLivenessTimer) {
958
+ clearInterval(holderLivenessTimer);
959
+ holderLivenessTimer = void 0;
960
+ }
961
+ }
962
+ const persistRelease = async (workspaceWritersQuiesced) => {
963
+ await releaseLeaseHolder(db, {
964
+ accountId: ids.accountId,
965
+ workspaceId: ids.workspaceId,
966
+ sandboxGroupId: ids.sandboxGroupId,
967
+ kind,
968
+ holderId,
969
+ idleGraceMs: settings.sandboxIdleGraceMs,
970
+ ...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
971
+ });
972
+ };
973
+ if (writersQuiesced) {
974
+ if (!quiescedRelease) {
975
+ const predecessor = holderRelease;
976
+ const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
977
+ async () => await persistRelease(true)
978
+ );
979
+ const tracked = attempt.catch((error) => {
980
+ if (quiescedRelease === tracked) quiescedRelease = null;
981
+ throw error;
982
+ });
983
+ quiescedRelease = tracked;
984
+ holderRelease = tracked;
985
+ }
986
+ return quiescedRelease;
987
+ }
988
+ if (!holderRelease) {
989
+ const attempt = persistRelease(false);
990
+ const tracked = attempt.catch((error) => {
991
+ if (holderRelease === tracked) holderRelease = null;
992
+ throw error;
993
+ });
994
+ holderRelease = tracked;
995
+ }
996
+ return holderRelease;
997
+ };
998
+ const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
999
+ const throwIfReleasedOrCancelled = () => {
1000
+ if (releaseStarted || cancellationSignal?.aborted) {
1001
+ throw cancellationError();
1002
+ }
1003
+ };
1004
+ const acquired = await acquireLease(db, {
1005
+ accountId: ids.accountId,
1006
+ workspaceId: ids.workspaceId,
1007
+ sandboxGroupId: ids.sandboxGroupId,
1008
+ kind,
1009
+ holderId,
1010
+ subjectId: ids.sessionId,
1011
+ backend: ids.backend,
1012
+ warmBilling: {
1013
+ mode: settings.sandboxWarmBillingMode,
1014
+ rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend)
1015
+ },
1016
+ os,
1017
+ // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
1018
+ // conflicts on a live box already running a different required image. A
1019
+ // SandboxImageConflictError propagates while another holder is active; a
1020
+ // solo change requests a capture-and-drain rotation and this attempt retries
1021
+ // after the cold successor can safely stamp the new image.
1022
+ ...ids.image ? { image: ids.image } : {},
1023
+ ...ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {},
1024
+ // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
1025
+ // + conflicts on a live box under a different rig. A SandboxRigConflictError
1026
+ // propagates while another holder is active; a solo change uses the same
1027
+ // durable capture-and-drain rotation as an image change.
1028
+ ...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
1029
+ leaseTtlMs,
1030
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
1031
+ captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
1032
+ ...cancellationSignal ? { waitSignal: cancellationSignal } : {}
1033
+ });
1034
+ if (cancellationSignal) {
1035
+ cancellationListener = () => {
1036
+ void release().catch(() => void 0);
1037
+ };
1038
+ cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
1039
+ }
1040
+ if (cancellationSignal?.aborted) {
1041
+ await release();
1042
+ throw cancellationError();
1043
+ }
1044
+ let holderLeaseHeartbeat = acquired.role === "spawner" ? {
1045
+ expectedEpoch: acquired.lease.leaseEpoch,
1046
+ leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
1047
+ } : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
1048
+ let providerRenewalTarget = acquired.lease.liveness === "warm" && acquired.lease.instanceId ? {
1049
+ backend: ids.backend,
1050
+ instanceId: acquired.lease.instanceId
1051
+ } : null;
1052
+ let providerRenewedAtMs = Date.now();
1053
+ let providerRenewalInFlight = null;
1054
+ const maybeRenewProviderExpiration = async (force = false) => {
1055
+ const target = providerRenewalTarget;
1056
+ if (!target || providerRenewalInFlight) return;
1057
+ const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
1058
+ if (ttlSeconds === null) return;
1059
+ const intervalMs = Math.max(1e4, Math.floor(ttlSeconds * 1e3 / 3));
1060
+ if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
1061
+ providerRenewalInFlight = renewSandboxProviderExpiration({
1062
+ backend: target.backend,
1063
+ settings,
1064
+ instanceId: target.instanceId,
1065
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
1066
+ }).then(() => {
1067
+ providerRenewedAtMs = Date.now();
1068
+ }).finally(() => {
1069
+ providerRenewalInFlight = null;
1070
+ });
1071
+ await providerRenewalInFlight;
1072
+ };
1073
+ const releaseDeadHolder = async (reason) => {
1074
+ if (reason === "holder_gone" && kind === "turn" && holderLeaseHeartbeat) {
1075
+ const expectedEpoch = holderLeaseHeartbeat.expectedEpoch;
1076
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1077
+ if (retainsHolderForWarmCapture(lease, expectedEpoch)) return;
1078
+ }
1079
+ const message = "sandbox resume holder released: holder no longer live";
1080
+ const fields = {
1081
+ workspaceId: ids.workspaceId,
1082
+ sessionId: ids.sessionId,
1083
+ sandboxGroupId: ids.sandboxGroupId,
1084
+ holderKind: kind,
1085
+ reason
1086
+ };
1087
+ if (services.observability) services.observability.warn(message, fields);
1088
+ else console.warn(message, fields);
1089
+ void release().catch(() => void 0);
1090
+ };
1091
+ holderLivenessTimer = setInterval(() => {
1092
+ const heartbeat = holderLeaseHeartbeat;
1093
+ if (heartbeat) {
1094
+ void heartbeatLeaseHolderStatus(db, {
1095
+ accountId: ids.accountId,
1096
+ workspaceId: ids.workspaceId,
1097
+ sandboxGroupId: ids.sandboxGroupId,
1098
+ kind,
1099
+ holderId,
1100
+ expectedEpoch: heartbeat.expectedEpoch,
1101
+ leaseTtlMs: heartbeat.leaseTtlMs,
1102
+ billingMode: services.settings.sandboxWarmBillingMode
1103
+ }).then(async (status) => {
1104
+ if (heartbeat !== holderLeaseHeartbeat) return;
1105
+ if (!status.holderAlive) {
1106
+ await releaseDeadHolder(status.fence);
1107
+ return;
1108
+ }
1109
+ if (status.fence === "epoch") {
1110
+ await releaseDeadHolder(status.fence);
1111
+ return;
1112
+ }
1113
+ if (status.leaseExtended || status.fence === "rotation_requested") {
1114
+ void maybeRenewProviderExpiration().catch(() => void 0);
1115
+ }
1116
+ }).catch(() => void 0);
1117
+ return;
1118
+ }
1119
+ void touchLeaseHolder(db, {
1120
+ accountId: ids.accountId,
1121
+ workspaceId: ids.workspaceId,
1122
+ sandboxGroupId: ids.sandboxGroupId,
1123
+ kind,
1124
+ holderId
1125
+ }).then(async (touched) => {
1126
+ if (heartbeat !== holderLeaseHeartbeat) return;
1127
+ if (!touched) {
1128
+ await releaseDeadHolder("holder_gone");
1129
+ return;
1130
+ }
1131
+ void maybeRenewProviderExpiration().catch(() => void 0);
1132
+ }).catch(() => void 0);
1133
+ }, services.holderLivenessIntervalMs ?? 1e4);
1134
+ if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
1135
+ holderLivenessTimer.unref();
1136
+ }
1137
+ if (acquired.role === "blocked") {
1138
+ await release();
1139
+ throw new SandboxLeaseRecoveryBlockedError(
1140
+ ids.sandboxGroupId,
1141
+ acquired.lease.leaseEpoch,
1142
+ acquired.code,
1143
+ acquired.lease.recovery
1144
+ );
1145
+ }
1146
+ if (acquired.role === "fenced") {
1147
+ await release();
1148
+ if (acquired.reason !== "superseded") {
1149
+ throw new SandboxLeaseTransitionError(
1150
+ ids.sandboxGroupId,
1151
+ acquired.lease.leaseEpoch,
1152
+ acquired.reason,
1153
+ acquired.lease.backend,
1154
+ acquired.lease.instanceId,
1155
+ acquired.lease.liveness
1156
+ );
1157
+ }
1158
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
1159
+ backend: acquired.lease.backend,
1160
+ instanceId: acquired.lease.instanceId,
1161
+ liveness: acquired.lease.liveness
1162
+ });
1163
+ }
1164
+ if (acquired.role === "spawner") {
1165
+ const expectedEpoch = acquired.lease.leaseEpoch;
1166
+ let createdEstablished = null;
1167
+ let published = false;
1168
+ let providerCreateOperationId;
1169
+ let providerCreateBindingKey;
1170
+ let rematerialization = null;
1171
+ try {
1172
+ const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
1173
+ const envelope = freshWorkspaceRecoveryId ? null : await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
1174
+ const legacyFallbackRefused = !freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null && await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId);
1175
+ const fallbackArchiveEnvelope = !freshWorkspaceRecoveryId && !legacyFallbackRefused && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
1176
+ let spawnEnvelope = freshWorkspaceRecoveryId ? null : legacyFallbackRefused ? acquired.lease.resumeState ?? null : fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
1177
+ const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
1178
+ const continuityRecovery = acquired.lease.recovery.continuity;
1179
+ if (!freshWorkspaceRecoveryId && (acquired.lease.recovery.archive.status === "available" && (acquired.lease.archiveComplete || acquired.lease.historicalRecoveryAuthorized === true) || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)) {
1180
+ const rematerializationId = crypto.randomUUID();
1181
+ const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
1182
+ const begun = await beginSandboxRematerialization(db, {
1183
+ accountId: ids.accountId,
1184
+ workspaceId: ids.workspaceId,
1185
+ sandboxGroupId: ids.sandboxGroupId,
1186
+ expectedEpoch,
1187
+ rematerializationId,
1188
+ archiveSource,
1189
+ legacyNativeArchive
1190
+ });
1191
+ if (begun.status !== "started") {
1192
+ if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
1193
+ throw new SandboxLeaseSupersededError(
1194
+ ids.sandboxGroupId,
1195
+ begun.lease?.leaseEpoch ?? expectedEpoch
1196
+ );
1197
+ }
1198
+ throw new SandboxLeaseRecoveryBlockedError(
1199
+ ids.sandboxGroupId,
1200
+ begun.lease?.leaseEpoch ?? expectedEpoch,
1201
+ begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
1202
+ begun.lease?.recovery ?? acquired.lease.recovery
1203
+ );
1204
+ }
1205
+ spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
1206
+ const selectedRevision = begun.lease.recovery.restore.selectedRevision;
1207
+ if (!selectedRevision) {
1208
+ throw new WorkspaceArchiveIntegrityError(
1209
+ "archive_metadata_invalid",
1210
+ "sandbox rematerialization selected no durable archive revision"
1211
+ );
1212
+ }
1213
+ rematerialization = {
1214
+ id: rematerializationId,
1215
+ selectedRevision,
1216
+ workspaceGeneration: begun.lease.workspaceGeneration,
1217
+ providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
1218
+ legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
1219
+ legacyProviderBinding: null
1220
+ };
1221
+ } else if (!freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
1222
+ throw new SandboxLeaseRecoveryBlockedError(
1223
+ ids.sandboxGroupId,
1224
+ expectedEpoch,
1225
+ "restore_degraded",
1226
+ acquired.lease.recovery
1227
+ );
1228
+ }
1229
+ const providerCreateStartedAt = /* @__PURE__ */ new Date();
1230
+ const persistArchiveSource = spawnEnvelope && typeof spawnEnvelope === "object" ? spawnEnvelope : null;
1231
+ const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
1232
+ persistArchiveSource,
1233
+ services.objectStorage
1234
+ );
1235
+ const hydrateEnvelope = hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object" ? hydrateEnvelopeRaw : null;
1236
+ const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
1237
+ sessionId: ids.sessionId,
1238
+ recovery: "create-or-restore",
1239
+ ...services.objectStorage ? {
1240
+ loadHostWorkspaceArchive: async (ref) => {
1241
+ try {
1242
+ return await downloadWorkspaceArchiveSpool(
1243
+ services.objectStorage,
1244
+ ref.key,
1245
+ { bytes: ref.bytes, sha256: ref.sha256 },
1246
+ { temporaryDirectory: workspaceArchiveDownloadTemporaryDirectory }
1247
+ );
1248
+ } catch (error) {
1249
+ if (error instanceof WorkspaceArchiveStorageError) {
1250
+ throw new WorkspaceArchiveIntegrityError(error.code, error.message, {
1251
+ retryable: error.retryable,
1252
+ cause: error
1253
+ });
1254
+ }
1255
+ throw error;
1256
+ }
1257
+ }
1258
+ } : {},
1259
+ backendOverride: ids.backend,
1260
+ ...ids.environment ? { environment: ids.environment } : {},
1261
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
1262
+ ...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
1263
+ onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
1264
+ if (ids.backend !== "modal") return;
1265
+ if (!intent || !providerContext)
1266
+ throw new Error("Modal create requires the provider dispatch boundary");
1267
+ throwIfReleasedOrCancelled();
1268
+ const binding = await resolveModalCheckpointProviderBindingForSession(
1269
+ createSettings,
1270
+ providerContext
1271
+ );
1272
+ const operationId = intent.operationId;
1273
+ await beginModalProviderCreate(db, {
1274
+ accountId: ids.accountId,
1275
+ workspaceId: ids.workspaceId,
1276
+ sandboxGroupId: ids.sandboxGroupId,
1277
+ expectedEpoch,
1278
+ operationId,
1279
+ providerBindingKey: binding.key,
1280
+ rematerializationId: rematerialization?.id ?? null,
1281
+ selectedRevision: rematerialization?.selectedRevision ?? null,
1282
+ imageId: intent.imageId,
1283
+ imageRef: createSettings.modalImageRef ?? null,
1284
+ appId: intent.appId,
1285
+ providerName: intent.name,
1286
+ requestSha256: intent.requestSha256
1287
+ });
1288
+ providerCreateOperationId = operationId;
1289
+ providerCreateBindingKey = binding.key;
1290
+ },
1291
+ onSandboxCreated: async (created) => {
1292
+ createdEstablished = created;
1293
+ providerRenewalTarget = {
1294
+ backend: ids.backend,
1295
+ instanceId: created.instanceId
1296
+ };
1297
+ providerRenewedAtMs = Date.now();
1298
+ if (providerCreateBindingKey && !await modalSessionMatchesCheckpointProviderBinding(
1299
+ settings,
1300
+ created.session,
1301
+ providerCreateBindingKey
1302
+ )) {
1303
+ throw new Error("Modal creation receipt crossed the fenced provider namespace");
1304
+ }
1305
+ if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
1306
+ if (created.backendId !== "modal") {
1307
+ throw new WorkspaceArchiveIntegrityError(
1308
+ "native_snapshot_reference_invalid",
1309
+ "Modal checkpoint restore resolved a non-Modal sandbox backend"
1310
+ );
1311
+ }
1312
+ if (rematerialization.providerBindingKey) {
1313
+ if (!await modalSessionMatchesCheckpointProviderBinding(
1314
+ settings,
1315
+ created.session,
1316
+ rematerialization.providerBindingKey
1317
+ )) {
1318
+ throw new WorkspaceArchiveIntegrityError(
1319
+ "native_snapshot_reference_invalid",
1320
+ "Modal checkpoint restore refused because the authenticated provider workspace changed"
1321
+ );
1322
+ }
1323
+ } else {
1324
+ const resolved = await resolveModalCheckpointProviderBindingForSession(
1325
+ settings,
1326
+ created.session
1327
+ );
1328
+ if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
1329
+ throw new WorkspaceArchiveIntegrityError(
1330
+ "native_snapshot_reference_invalid",
1331
+ "Legacy Modal checkpoint restore crossed authenticated provider workspaces"
1332
+ );
1333
+ }
1334
+ rematerialization.legacyProviderBinding = resolved;
1335
+ }
1336
+ }
1337
+ const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
1338
+ await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
1339
+ created.backendId
1340
+ );
1341
+ const recorded = await recordWarmingSandboxCreated(db, {
1342
+ accountId: ids.accountId,
1343
+ workspaceId: ids.workspaceId,
1344
+ sandboxGroupId: ids.sandboxGroupId,
1345
+ expectedEpoch,
1346
+ rematerializationId: rematerialization?.id ?? null,
1347
+ ...providerCreateOperationId ? { providerCreateOperationId } : {},
1348
+ ...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
1349
+ instanceId: created.instanceId,
1350
+ resumeBackendId: created.backendId,
1351
+ resumeState: resumeEnvelope2,
1352
+ ...created.backendId === "modal" ? {
1353
+ providerCreatedAt: providerCreateStartedAt,
1354
+ providerDeadlineAt: new Date(
1355
+ providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
1356
+ )
1357
+ } : {},
1358
+ leaseTtlMs,
1359
+ // Keep the warming budget after create(): manifest setup and
1360
+ // commitWarmingToWarm still run, and can exceed the 90s turn TTL.
1361
+ warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
1362
+ });
1363
+ if (!recorded.recorded) {
1364
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1365
+ }
1366
+ if (created.backendId === "modal") {
1367
+ await tagModalSandbox(settings, created.instanceId, {
1368
+ leaseId: acquired.lease.id,
1369
+ workspaceId: ids.workspaceId,
1370
+ sandboxGroupId: ids.sandboxGroupId
1371
+ }).catch(() => void 0);
1372
+ }
1373
+ throwIfReleasedOrCancelled();
1374
+ },
1375
+ onWorkspaceRestoreVerifying: async (descriptor) => {
1376
+ if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
1377
+ throw new WorkspaceArchiveIntegrityError(
1378
+ "archive_metadata_invalid",
1379
+ `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
1380
+ );
1381
+ }
1382
+ if (rematerialization.legacyCheckpoint) {
1383
+ const binding = rematerialization.legacyProviderBinding;
1384
+ if (!binding) {
1385
+ throw new WorkspaceArchiveIntegrityError(
1386
+ "native_snapshot_reference_invalid",
1387
+ "Legacy Modal checkpoint restore produced no authenticated provider identity"
1388
+ );
1389
+ }
1390
+ const adopted = await adoptLegacyModalCheckpointArtifact(db, {
1391
+ accountId: ids.accountId,
1392
+ workspaceId: ids.workspaceId,
1393
+ sandboxGroupId: ids.sandboxGroupId,
1394
+ leaseId: acquired.lease.id,
1395
+ leaseEpoch: expectedEpoch,
1396
+ workspaceGeneration: rematerialization.workspaceGeneration,
1397
+ slot: "current",
1398
+ archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
1399
+ descriptor: rematerialization.legacyCheckpoint.descriptor,
1400
+ providerBindingKey: binding.key,
1401
+ providerBinding: binding.binding,
1402
+ rematerializationId: rematerialization.id
1403
+ });
1404
+ if (!adopted) {
1405
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1406
+ }
1407
+ rematerialization.providerBindingKey = binding.key;
1408
+ }
1409
+ const verifying = await markSandboxRestoreVerifying(db, {
1410
+ accountId: ids.accountId,
1411
+ workspaceId: ids.workspaceId,
1412
+ sandboxGroupId: ids.sandboxGroupId,
1413
+ expectedEpoch,
1414
+ rematerializationId: rematerialization.id
1415
+ });
1416
+ if (!verifying.wrote) {
1417
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1418
+ }
1419
+ }
1420
+ });
1421
+ createdEstablished = established;
1422
+ throwIfReleasedOrCancelled();
1423
+ if (services.verifySpawnedSandboxReadiness) {
1424
+ await services.verifySpawnedSandboxReadiness(established, {
1425
+ sandboxGroupId: ids.sandboxGroupId
1426
+ });
1427
+ } else {
1428
+ await waitForSandboxExecReadiness(
1429
+ established,
1430
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
1431
+ {
1432
+ sandboxGroupId: ids.sandboxGroupId
1433
+ },
1434
+ cancellationSignal
1435
+ );
1436
+ }
1437
+ await maybeRenewProviderExpiration(true);
1438
+ throwIfReleasedOrCancelled();
1439
+ const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
1440
+ await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
1441
+ established.backendId
1442
+ );
1443
+ throwIfReleasedOrCancelled();
1444
+ if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
1445
+ throw new WorkspaceArchiveIntegrityError(
1446
+ "workspace_fingerprint_mismatch",
1447
+ "sandbox restore completed without the exact selected durable archive revision"
1448
+ );
1449
+ }
1450
+ const committed = await commitWarmingToWarm(db, {
1451
+ accountId: ids.accountId,
1452
+ workspaceId: ids.workspaceId,
1453
+ sandboxGroupId: ids.sandboxGroupId,
1454
+ expectedEpoch,
1455
+ instanceId: established.instanceId,
1456
+ // Viewer attach resolves and records the tunnel URL lazily. A headless
1457
+ // turn must not touch the optional desktop data plane.
1458
+ dataPlaneUrl: null,
1459
+ resumeBackendId: established.backendId,
1460
+ resumeState: resumeEnvelope,
1461
+ ...freshWorkspaceRecoveryId ? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } } : established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
1462
+ rematerialization: {
1463
+ id: rematerialization.id,
1464
+ verifiedRevision: rematerialization.selectedRevision
1465
+ }
1466
+ } : {},
1467
+ leaseTtlMs
1468
+ });
1469
+ if (!committed.committed || !committed.lease) {
1470
+ const terminated = await terminateEstablishedSandbox(established);
1471
+ if (terminated && rematerialization && !established.providerContinuity) {
1472
+ await failSandboxRematerialization(db, {
1473
+ accountId: ids.accountId,
1474
+ workspaceId: ids.workspaceId,
1475
+ sandboxGroupId: ids.sandboxGroupId,
1476
+ expectedEpoch,
1477
+ rematerializationId: rematerialization.id,
1478
+ failureCode: committed.reason ?? "warm_commit_rejected",
1479
+ retryable: false
1480
+ });
1481
+ } else if (terminated) {
1482
+ await failWarmingToCold(db, {
1483
+ accountId: ids.accountId,
1484
+ workspaceId: ids.workspaceId,
1485
+ sandboxGroupId: ids.sandboxGroupId,
1486
+ expectedEpoch
1487
+ });
1488
+ }
1489
+ await release();
1490
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1491
+ }
1492
+ published = true;
1493
+ holderLeaseHeartbeat = {
1494
+ expectedEpoch: committed.lease.leaseEpoch,
1495
+ leaseTtlMs
1496
+ };
1497
+ await services.onSpawnedSandboxPublished?.();
1498
+ throwIfReleasedOrCancelled();
1499
+ return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1500
+ } catch (error) {
1501
+ if (published) {
1502
+ await release();
1503
+ throw error;
1504
+ }
1505
+ if (error instanceof SandboxLeaseSupersededError) {
1506
+ await terminateEstablishedSandbox(createdEstablished);
1507
+ await release();
1508
+ throw error;
1509
+ }
1510
+ const terminated = await terminateEstablishedSandbox(createdEstablished);
1511
+ if (terminated) {
1512
+ const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
1513
+ if (rematerialization && !createdEstablished?.providerContinuity) {
1514
+ await failSandboxRematerialization(db, {
1515
+ accountId: ids.accountId,
1516
+ workspaceId: ids.workspaceId,
1517
+ sandboxGroupId: ids.sandboxGroupId,
1518
+ expectedEpoch,
1519
+ rematerializationId: rematerialization.id,
1520
+ failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
1521
+ retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
1522
+ ...continuityUnavailable ? { discardContinuity: true } : {}
1523
+ });
1524
+ } else {
1525
+ await failWarmingToCold(db, {
1526
+ accountId: ids.accountId,
1527
+ workspaceId: ids.workspaceId,
1528
+ sandboxGroupId: ids.sandboxGroupId,
1529
+ expectedEpoch,
1530
+ ...continuityUnavailable ? { discardContinuity: true } : {}
1531
+ });
1532
+ }
1533
+ }
1534
+ await release();
1535
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1536
+ if (terminated && createdEstablished && error instanceof SandboxExecReadinessTimeoutError && error.instanceId === createdEstablished.instanceId && (createdEstablished.origin === "created" || createdEstablished.origin === "restored") && !createdEstablished.providerContinuity) {
1537
+ disposedUnreadyFreshSandboxErrors.add(error);
1538
+ }
1539
+ throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
1540
+ }
1541
+ }
1542
+ try {
1543
+ let leaseEpoch = acquired.lease.leaseEpoch;
1544
+ if (acquired.lease.liveness === "warming") {
1545
+ leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
1546
+ holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
1547
+ }
1548
+ throwIfReleasedOrCancelled();
1549
+ const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1550
+ if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
1551
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
1552
+ }
1553
+ let established;
1554
+ try {
1555
+ const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
1556
+ established = await establish(settings, live.resumeState, {
1557
+ sessionId: ids.sessionId,
1558
+ recovery: "resume-only",
1559
+ backendOverride: ids.backend,
1560
+ ...ids.environment ? { environment: ids.environment } : {},
1561
+ ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
1562
+ });
1563
+ throwIfReleasedOrCancelled();
1564
+ if (services.verifyAttachedSandboxReadiness) {
1565
+ await services.verifyAttachedSandboxReadiness(established);
1566
+ } else {
1567
+ await waitForSandboxExecReadiness(
1568
+ established,
1569
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
1570
+ {
1571
+ sandboxGroupId: ids.sandboxGroupId
1572
+ },
1573
+ cancellationSignal
1574
+ );
1575
+ }
1576
+ providerRenewalTarget = {
1577
+ backend: ids.backend,
1578
+ instanceId: established.instanceId
1579
+ };
1580
+ await maybeRenewProviderExpiration(true);
1581
+ throwIfReleasedOrCancelled();
1582
+ } catch (error) {
1583
+ if (!isProviderSandboxNotFoundError(ids.backend, error)) {
1584
+ throw sandboxProvisionStageError("resume", error);
1585
+ }
1586
+ const marked = await markWarmLeaseInstanceLost(db, {
1587
+ accountId: ids.accountId,
1588
+ workspaceId: ids.workspaceId,
1589
+ sandboxGroupId: ids.sandboxGroupId,
1590
+ expectedEpoch: leaseEpoch,
1591
+ expectedInstanceId: live.instanceId,
1592
+ expectedBackend: ids.backend
1593
+ });
1594
+ if (marked.status === "marked") {
1595
+ await publishDurableSessionEvents(
1596
+ services.bus,
1597
+ ids.workspaceId,
1598
+ marked.backgroundCommandEvents
1599
+ );
1600
+ await services.onSandboxLost?.({
1601
+ sandboxGroupId: ids.sandboxGroupId,
1602
+ instanceId: live.instanceId,
1603
+ leaseEpoch: marked.lease.leaseEpoch
1604
+ });
1605
+ throw new SandboxLeaseInstanceLostError(
1606
+ ids.sandboxGroupId,
1607
+ marked.lease.leaseEpoch,
1608
+ live.instanceId
1609
+ );
1610
+ }
1611
+ throw new SandboxLeaseSupersededError(
1612
+ ids.sandboxGroupId,
1613
+ marked.lease?.leaseEpoch ?? leaseEpoch
1614
+ );
1615
+ }
1616
+ throwIfReleasedOrCancelled();
1617
+ return { established, leaseEpoch, release };
1618
+ } catch (error) {
1619
+ await release();
1620
+ recordSandboxWarmingTimeout(services.sandboxMetrics, error);
1621
+ throw error;
1622
+ }
1623
+ }
1624
+ async function waitForWarm(services, ids) {
1625
+ const { db, settings } = services;
1626
+ const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1627
+ let instanceId = null;
1628
+ while (Date.now() < deadline) {
1629
+ if (!await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal)) {
1630
+ throw services.cancellationSignal?.reason instanceof Error ? services.cancellationSignal.reason : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
1631
+ }
1632
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1633
+ if (!lease) {
1634
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
1635
+ }
1636
+ instanceId = lease.instanceId;
1637
+ if (lease.liveness === "warm") {
1638
+ return { leaseEpoch: lease.leaseEpoch };
1639
+ }
1640
+ if (lease.liveness === "draining") {
1641
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1642
+ }
1643
+ if (lease.liveness === "cold") {
1644
+ if (isRetryableDegradedRestore(lease.recovery.restore)) {
1645
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1646
+ }
1647
+ if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
1648
+ throw new SandboxLeaseRecoveryBlockedError(
1649
+ ids.sandboxGroupId,
1650
+ lease.leaseEpoch,
1651
+ lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
1652
+ lease.recovery
1653
+ );
1654
+ }
1655
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
1656
+ }
1657
+ }
1658
+ throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
1659
+ sandboxGroupId: ids.sandboxGroupId,
1660
+ instanceId
1661
+ });
1662
+ }
1663
+
1664
+ // src/sandbox-routing.ts
1665
+ import { isDeepStrictEqual } from "util";
1666
+ import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
1667
+ import {
1668
+ createProviderCommandRetainer,
1669
+ supervisedCommandProtocolReady
1670
+ } from "@opengeni/db/retained-provider-commands";
1671
+ import {
1672
+ adoptConnectedMachineSessionBackgroundCommand,
1673
+ adoptManagedSessionBackgroundCommand,
1674
+ advanceWorkspaceGenerationForRetainedProcess,
1675
+ retainedProviderCommandPersistence,
1676
+ advanceWorkspaceGeneration,
1677
+ getRetainedProcess,
1678
+ retainWorkspaceMutationProcess,
1679
+ retainedProcessSettlementIdentity,
1680
+ settleConnectedMachineSessionBackgroundCommand,
1681
+ settleRetainedProcess,
1682
+ verifyRetainedProcessMutationSettlement,
1683
+ verifyWorkspaceMutationSettlement,
1684
+ getSandbox,
1685
+ getLiveEnrollmentConnection,
1686
+ markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
1687
+ readLease as readLease2,
1688
+ readActiveSandbox,
1689
+ resolvePersonalMachineConnectionForAttempt,
1690
+ SandboxRetainedProcessPromotionFencedError,
1691
+ SandboxRetainedProcessTerminalError,
1692
+ SandboxWorkspaceMutationOutputRejectedError
1693
+ } from "@opengeni/db";
1694
+ import {
1695
+ observeSessionBackgroundCommandCompletion,
1696
+ recordConnectedCommandOutputConsumption
1697
+ } from "@opengeni/db/session-background-commands";
1698
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
1699
+ import {
1700
+ buildSelfhostedBackendSession,
1701
+ ActiveBackendUnresolvableError,
1702
+ establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
1703
+ isProviderSandboxGoneDuringRoutedOperation,
1704
+ isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
1705
+ makeActiveBackendResolver,
1706
+ NatsControlRpc,
1707
+ NatsOpStreamTransport,
1708
+ RoutingBackendRecoveryRequiredError,
1709
+ RoutingSandboxSession,
1710
+ RoutingMutationOutputRejectedError,
1711
+ resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
1712
+ resolveConnectedMachineWorkspaceRoot,
1713
+ sandboxProviderInstanceIdFromEnvelope,
1714
+ sandboxBackendForSdkBackendId,
1715
+ selectBackend,
1716
+ verifySandboxExecReadiness as verifySandboxExecReadiness2
1717
+ } from "@opengeni/runtime";
1718
+ var retainWorkspaceProviderCommand = createProviderCommandRetainer(
1719
+ retainWorkspaceMutationProcess,
1720
+ (error) => error instanceof SandboxRetainedProcessPromotionFencedError ? error.process : null
1721
+ );
1722
+ function admittedCommandHandle(value) {
1723
+ if (!value || typeof value !== "object") return void 0;
1724
+ return value.admission?.workspaceGeneration;
1725
+ }
1726
+ function relayConfigFromSettings(settings) {
1727
+ const raw = settings.selfhostedRelayUrl?.trim();
1728
+ if (!raw) {
1729
+ return { host: "relay.opengeni.local", port: 443, tls: true };
1730
+ }
1731
+ try {
1732
+ const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
1733
+ const tls = url.protocol === "wss:" || url.protocol === "https:";
1734
+ const port = url.port ? Number(url.port) : tls ? 443 : 80;
1735
+ return { host: url.hostname, port, tls };
1736
+ } catch {
1737
+ return { host: raw, port: 443, tls: true };
1738
+ }
1739
+ }
1740
+ function selfhostedTimeoutsFromSettings(settings) {
1741
+ return {
1742
+ timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
1743
+ execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
1744
+ };
1745
+ }
1746
+ function selfhostedResolverTimeouts(settings) {
1747
+ const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
1748
+ return {
1749
+ selfhostedTimeoutMs: timeoutMs,
1750
+ selfhostedExecTimeoutMs: execTimeoutMs
1751
+ };
1752
+ }
1753
+ function homeRouteRecoveryDisposition(lease) {
1754
+ if (!lease) return "unrecoverable";
1755
+ if (lease.liveness === "warming") return "pending";
1756
+ if (lease.liveness === "draining") return "superseded";
1757
+ if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
1758
+ return "pending";
1759
+ }
1760
+ if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
1761
+ return "degraded";
1762
+ }
1763
+ return "unrecoverable";
1764
+ }
1765
+ function homeRouteRecoveryError(lease, fallbackEpoch) {
1766
+ return new RoutingBackendRecoveryRequiredError(
1767
+ "resolve_home_backend",
1768
+ lease?.leaseEpoch ?? fallbackEpoch,
1769
+ homeRouteRecoveryDisposition(lease)
1770
+ );
1771
+ }
1772
+ function providerIdentityFromResumeState(resumeState) {
1773
+ return sandboxProviderInstanceIdFromEnvelope(resumeState);
1774
+ }
1775
+ async function resolveCurrentHomeBackend(services, ids, established) {
1776
+ const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
1777
+ const fallbackEpoch = lease?.leaseEpoch ?? 0;
1778
+ if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
1779
+ throw homeRouteRecoveryError(lease, fallbackEpoch);
1780
+ }
1781
+ if (lease.instanceId === established.instanceId) {
1782
+ await services.onHomeSandboxRebound?.({
1783
+ established,
1784
+ leaseEpoch: lease.leaseEpoch
1785
+ });
1786
+ return {
1787
+ session: established.session,
1788
+ sandboxId: null,
1789
+ kind: established.backendId,
1790
+ leaseEpoch: lease.leaseEpoch,
1791
+ providerInstanceId: lease.instanceId
1792
+ };
1793
+ }
1794
+ const resumeState = lease.resumeState;
1795
+ const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
1796
+ const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
1797
+ if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
1798
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
1799
+ }
1800
+ let rebound;
1801
+ try {
1802
+ rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
1803
+ sessionId: ids.sessionId,
1804
+ recovery: "resume-only",
1805
+ backendOverride: resumeBackend
1806
+ });
1807
+ await verifySandboxExecReadiness2(rebound);
1808
+ } catch (error) {
1809
+ if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
1810
+ throw new RoutingBackendRecoveryRequiredError(
1811
+ "resolve_home_backend",
1812
+ lease.leaseEpoch,
1813
+ "pending"
1814
+ );
1815
+ }
1816
+ const marked = await markWarmLeaseInstanceLost2(services.db, {
1817
+ accountId: ids.accountId,
1818
+ workspaceId: ids.workspaceId,
1819
+ sandboxGroupId: ids.sandboxGroupId,
1820
+ expectedEpoch: lease.leaseEpoch,
1821
+ expectedInstanceId: lease.instanceId,
1822
+ expectedBackend: lease.backend,
1823
+ diagnostic: "provider_not_found_during_home_route_rebind"
1824
+ });
1825
+ if (marked.status === "marked") {
1826
+ await publishDurableSessionEvents(
1827
+ services.bus,
1828
+ ids.workspaceId,
1829
+ marked.backgroundCommandEvents
1830
+ );
1831
+ await services.onHomeSandboxLost?.({
1832
+ sandboxGroupId: ids.sandboxGroupId,
1833
+ instanceId: lease.instanceId,
1834
+ leaseEpoch: marked.lease.leaseEpoch
1835
+ });
1836
+ }
1837
+ throw new RoutingBackendRecoveryRequiredError(
1838
+ "resolve_home_backend",
1839
+ marked.lease?.leaseEpoch ?? lease.leaseEpoch,
1840
+ marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
1841
+ );
1842
+ }
1843
+ const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
1844
+ if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
1845
+ throw homeRouteRecoveryError(lease, lease.leaseEpoch);
1846
+ }
1847
+ await services.onHomeSandboxRebound?.({
1848
+ established: rebound,
1849
+ leaseEpoch: lease.leaseEpoch
1850
+ });
1851
+ Object.assign(established, rebound);
1852
+ return {
1853
+ session: rebound.session,
1854
+ sandboxId: null,
1855
+ kind: rebound.backendId,
1856
+ leaseEpoch: lease.leaseEpoch,
1857
+ providerInstanceId: lease.instanceId
1858
+ };
1859
+ }
1860
+ function beforePersistableHomeMutation(services, ids, home) {
1861
+ const fence = ids.workspaceMutationFence;
1862
+ if (!home || !fence) return void 0;
1863
+ return async ({ op, backend, onCaptureWait }) => {
1864
+ if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
1865
+ return null;
1866
+ }
1867
+ const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession2(services.settings, backend.session) : null;
1868
+ const admission = await advanceWorkspaceGeneration(services.db, {
1869
+ accountId: fence.accountId,
1870
+ workspaceId: ids.workspaceId,
1871
+ sessionId: ids.sessionId,
1872
+ turnId: fence.turnId,
1873
+ executionGeneration: fence.executionGeneration,
1874
+ attemptId: fence.attemptId,
1875
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
1876
+ sandboxGroupId: home.sandboxGroupId,
1877
+ expectedEpoch: backend.leaseEpoch,
1878
+ expectedInstanceId: backend.providerInstanceId,
1879
+ operation: op,
1880
+ captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
1881
+ ...services.waitSignal ? { waitSignal: services.waitSignal } : {},
1882
+ ...onCaptureWait ? { onCaptureWait } : {}
1883
+ });
1884
+ return { admission, providerBinding };
1885
+ };
1886
+ }
1887
+ function afterPersistableHomeMutation(services, ids, home) {
1888
+ const fence = ids.workspaceMutationFence;
1889
+ if (!home || !fence) return void 0;
1890
+ return async ({ op, backend, admission, outcome, retainedProcess }) => {
1891
+ if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
1892
+ return;
1893
+ }
1894
+ const boundAdmission = admission;
1895
+ const exactAdmission = boundAdmission.admission;
1896
+ if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
1897
+ throw new Error("Persistable home mutation settlement lacked its bound admission");
1898
+ }
1899
+ if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
1900
+ try {
1901
+ await retainWorkspaceProviderCommand(services.db, {
1902
+ accountId: fence.accountId,
1903
+ workspaceId: ids.workspaceId,
1904
+ sessionId: ids.sessionId,
1905
+ processId: retainedProcess.id,
1906
+ providerSessionId: retainedProcess.providerSessionId,
1907
+ ...retainedProcess.providerCommand ? { providerCommand: retainedProcess.providerCommand } : {},
1908
+ admissionId: exactAdmission.id,
1909
+ admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
1910
+ operation: op,
1911
+ providerBinding: boundAdmission.providerBinding ?? null,
1912
+ owner: {
1913
+ kind: "turn",
1914
+ turnId: fence.turnId,
1915
+ executionGeneration: fence.executionGeneration,
1916
+ attemptId: fence.attemptId,
1917
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
1918
+ sandboxGroupId: home.sandboxGroupId,
1919
+ expectedEpoch: backend.leaseEpoch,
1920
+ expectedInstanceId: backend.providerInstanceId,
1921
+ routeKind: exactAdmission.routeKind,
1922
+ routeTargetId: exactAdmission.routeTargetId,
1923
+ routeEpoch: exactAdmission.routeEpoch
1924
+ }
1925
+ });
1926
+ } catch (error) {
1927
+ if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
1928
+ const durable = error.process;
1929
+ if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
1930
+ throw new Error("Durable retained-process promotion returned a mismatched identity", {
1931
+ cause: error
1932
+ });
1933
+ }
1934
+ return {
1935
+ status: "retained_process_durable_output_rejected",
1936
+ retainedProcess: {
1937
+ id: durable.id,
1938
+ providerSessionId: durable.providerSessionId
1939
+ }
1940
+ };
1941
+ }
1942
+ return;
1943
+ }
1944
+ if (outcome === "outcome_unknown")
1945
+ throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
1946
+ try {
1947
+ await verifyWorkspaceMutationSettlement(services.db, {
1948
+ accountId: fence.accountId,
1949
+ workspaceId: ids.workspaceId,
1950
+ sessionId: ids.sessionId,
1951
+ turnId: fence.turnId,
1952
+ executionGeneration: fence.executionGeneration,
1953
+ attemptId: fence.attemptId,
1954
+ holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
1955
+ sandboxGroupId: home.sandboxGroupId,
1956
+ expectedEpoch: backend.leaseEpoch,
1957
+ expectedInstanceId: backend.providerInstanceId,
1958
+ admission: exactAdmission,
1959
+ operation: op,
1960
+ outcome
1961
+ });
1962
+ } catch (error) {
1963
+ if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
1964
+ accountId: fence.accountId,
1965
+ workspaceId: ids.workspaceId,
1966
+ admission: exactAdmission,
1967
+ operation: op,
1968
+ outcome
1969
+ })) {
1970
+ throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
1971
+ }
1972
+ throw error;
1973
+ }
1974
+ };
1975
+ }
1976
+ function beforeRetainedProcessMutation(services, ids) {
1977
+ const fence = ids.workspaceMutationFence;
1978
+ if (!fence) return void 0;
1979
+ return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
1980
+ accountId: fence.accountId,
1981
+ workspaceId: ids.workspaceId,
1982
+ sessionId: ids.sessionId,
1983
+ processId: process2.id,
1984
+ operation: op,
1985
+ captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
1986
+ ...services.waitSignal ? { waitSignal: services.waitSignal } : {}
1987
+ });
1988
+ }
1989
+ function afterRetainedProcessMutation(services, ids) {
1990
+ const fence = ids.workspaceMutationFence;
1991
+ if (!fence) return void 0;
1992
+ return async ({ op, process: process2, admission, outcome }) => {
1993
+ if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
1994
+ throw new Error("Retained-process mutation settlement lacked its exact admission");
1995
+ }
1996
+ try {
1997
+ await verifyRetainedProcessMutationSettlement(services.db, {
1998
+ accountId: fence.accountId,
1999
+ workspaceId: ids.workspaceId,
2000
+ sessionId: ids.sessionId,
2001
+ processId: process2.id,
2002
+ admission,
2003
+ operation: op,
2004
+ outcome
2005
+ });
2006
+ } catch (error) {
2007
+ if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
2008
+ accountId: fence.accountId,
2009
+ workspaceId: ids.workspaceId,
2010
+ admission,
2011
+ operation: op,
2012
+ outcome
2013
+ })) {
2014
+ if (error.retainedProcessTerminal) {
2015
+ throw new SandboxRetainedProcessTerminalError(
2016
+ error.retainedProcessTerminal.state,
2017
+ error.retainedProcessTerminal.exitCode
2018
+ );
2019
+ }
2020
+ throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
2021
+ }
2022
+ throw error;
2023
+ }
2024
+ };
2025
+ }
2026
+ function captureConnectedCommandOutput(db, ids, bus) {
2027
+ return async (commandId, frames) => {
2028
+ for (const frame of frames) {
2029
+ const events = await appendSessionCommandOutput(db, {
2030
+ ...ids,
2031
+ commandId,
2032
+ chunkId: `op-frame:${frame.sequence}`,
2033
+ stream: frame.stream,
2034
+ chunk: frame.chunk
2035
+ });
2036
+ if (events.length && bus)
2037
+ await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => void 0);
2038
+ }
2039
+ };
2040
+ }
2041
+ function captureRetainedProcessOutputForTurn(services, ids) {
2042
+ const accountId = ids.workspaceMutationFence?.accountId;
2043
+ if (!accountId) return void 0;
2044
+ return async ({
2045
+ process: process2,
2046
+ chunkId,
2047
+ chunk,
2048
+ stream,
2049
+ streamFidelity
2050
+ }) => {
2051
+ const events = await appendSessionCommandOutput(services.db, {
2052
+ accountId,
2053
+ workspaceId: ids.workspaceId,
2054
+ sessionId: ids.sessionId,
2055
+ commandId: process2.id,
2056
+ chunkId,
2057
+ chunk,
2058
+ stream,
2059
+ streamFidelity
2060
+ });
2061
+ if (events.length && services.bus)
2062
+ await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => void 0);
2063
+ };
2064
+ }
2065
+ function observeRetainedProcessTerminalForTurn(services, ids) {
2066
+ const fence = ids.workspaceMutationFence;
2067
+ if (!fence) return void 0;
2068
+ return async ({ process: process2 }) => {
2069
+ await observeSessionBackgroundCommandCompletion(services.db, {
2070
+ accountId: fence.accountId,
2071
+ workspaceId: ids.workspaceId,
2072
+ sessionId: ids.sessionId,
2073
+ commandId: process2.id
2074
+ });
2075
+ };
2076
+ }
2077
+ function settleRetainedProcessForTurn(services, ids) {
2078
+ const fence = ids.workspaceMutationFence;
2079
+ if (!fence) return void 0;
2080
+ return async ({ backend, process: process2, proof }) => {
2081
+ const durable = await getRetainedProcess(services.db, {
2082
+ workspaceId: ids.workspaceId,
2083
+ sessionId: ids.sessionId,
2084
+ processId: process2.id
2085
+ });
2086
+ if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
2087
+ throw new Error("Retained-process settlement lost its exact durable backend identity");
2088
+ }
2089
+ const settlement = await settleRetainedProcess(services.db, {
2090
+ accountId: fence.accountId,
2091
+ workspaceId: ids.workspaceId,
2092
+ sessionId: ids.sessionId,
2093
+ processId: process2.id,
2094
+ expected: retainedProcessSettlementIdentity(durable),
2095
+ outcome: proof.outcome,
2096
+ exitCode: proof.exitCode,
2097
+ reason: proof.reason,
2098
+ idleGraceMs: services.settings.sandboxIdleGraceMs
2099
+ });
2100
+ if (settlement.process.state === "active") {
2101
+ throw new Error("Retained-process settlement returned an active durable process");
2102
+ }
2103
+ if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
2104
+ await services.bus.publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents).catch(() => void 0);
2105
+ }
2106
+ };
2107
+ }
2108
+ function isRetainedProcessSettledForTurn(services, ids) {
2109
+ const fence = ids.workspaceMutationFence;
2110
+ if (!fence) return void 0;
2111
+ return async ({
2112
+ backend,
2113
+ process: process2
2114
+ }) => {
2115
+ const durable = await getRetainedProcess(services.db, {
2116
+ workspaceId: ids.workspaceId,
2117
+ sessionId: ids.sessionId,
2118
+ processId: process2.id
2119
+ });
2120
+ if (!durable || durable.state === "active" || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch)
2121
+ return false;
2122
+ if (process2.providerCommand) {
2123
+ const command = await retainedProviderCommandPersistence(services.db, {
2124
+ accountId: fence.accountId,
2125
+ workspaceId: ids.workspaceId,
2126
+ sessionId: ids.sessionId,
2127
+ processId: process2.id
2128
+ }).load();
2129
+ if (!command || command.kind !== process2.providerCommand.kind || command.execId !== process2.providerCommand.execId || command.taskId !== process2.providerCommand.taskId || command.sandboxId !== process2.providerCommand.sandboxId || Boolean(command.pty) !== Boolean(process2.providerCommand.pty) || command.kind === "modal-router-v1" && process2.providerCommand.kind === "modal-router-v1" && !isDeepStrictEqual(command.supervision, process2.providerCommand.supervision))
2130
+ return false;
2131
+ }
2132
+ return true;
2133
+ };
2134
+ }
2135
+ function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
2136
+ const fence = ids.workspaceMutationFence;
2137
+ if (!fence) return void 0;
2138
+ return async ({ backend, process: process2, command }) => {
2139
+ const durable = await getRetainedProcess(services.db, {
2140
+ workspaceId: ids.workspaceId,
2141
+ sessionId: ids.sessionId,
2142
+ processId: process2.id
2143
+ });
2144
+ if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
2145
+ throw new Error("Background command adoption lost its exact retained-process identity");
2146
+ }
2147
+ await adoptManagedSessionBackgroundCommand(services.db, {
2148
+ accountId: fence.accountId,
2149
+ workspaceId: ids.workspaceId,
2150
+ sessionId: ids.sessionId,
2151
+ turnId: fence.turnId,
2152
+ executionGeneration: fence.executionGeneration,
2153
+ attemptId: fence.attemptId,
2154
+ processId: process2.id,
2155
+ expected: retainedProcessSettlementIdentity(durable),
2156
+ command: command ?? "Background command"
2157
+ });
2158
+ };
2159
+ }
2160
+ function controlRpcFactory(bus) {
2161
+ return () => new NatsControlRpc(async () => {
2162
+ if (!bus) {
2163
+ return null;
2164
+ }
2165
+ return bus.getRequestConnection();
2166
+ });
2167
+ }
2168
+ function wrapTurnBoxWithRouting(services, ids, established) {
2169
+ const { db, settings, bus, onOp } = services;
2170
+ const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
2171
+ const backgroundCommandAttempt = ids.workspaceMutationFence;
2172
+ const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
2173
+ const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
2174
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
2175
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
2176
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
2177
+ const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
2178
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
2179
+ services,
2180
+ ids
2181
+ );
2182
+ const resolver = makeActiveBackendResolver({
2183
+ workspaceId: ids.workspaceId,
2184
+ defaultBackend: established.session,
2185
+ defaultKind: established.backendId,
2186
+ getSandbox: async (sandboxId) => {
2187
+ const sandbox = await getSandbox(
2188
+ db,
2189
+ ids.resourceAccountId && ids.resourceSubjectId ? {
2190
+ accountId: ids.resourceAccountId,
2191
+ workspaceId: ids.workspaceId,
2192
+ subjectId: ids.resourceSubjectId
2193
+ } : ids.workspaceId,
2194
+ sandboxId
2195
+ );
2196
+ return sandbox ? {
2197
+ id: sandbox.id,
2198
+ workspaceId: sandbox.workspaceId,
2199
+ kind: sandbox.kind,
2200
+ name: sandbox.name,
2201
+ ...sandbox.scope ? { scope: sandbox.scope } : {},
2202
+ enrollmentId: sandbox.enrollmentId
2203
+ } : null;
2204
+ },
2205
+ controlRpcFactory: controlRpcFactory(bus),
2206
+ resolveSelfhostedConnection: async (sandbox) => {
2207
+ if (!sandbox.enrollmentId) return null;
2208
+ const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
2209
+ return connectionBindingFor(services, enrollment);
2210
+ },
2211
+ relay: relayConfigFromSettings(settings),
2212
+ // A selfhosted swap target runs real commands too, so give it the same split
2213
+ // deadlines the machine-primary establish path uses (short control, long exec).
2214
+ ...selfhostedResolverTimeouts(settings),
2215
+ ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
2216
+ ...backgroundCommandAccountId && backgroundCommandAttempt ? {
2217
+ selfhostedAdoptBackgroundCommand: async (command) => {
2218
+ const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
2219
+ accountId: backgroundCommandAccountId,
2220
+ workspaceId: ids.workspaceId,
2221
+ sessionId: ids.sessionId,
2222
+ turnId: backgroundCommandAttempt.turnId,
2223
+ executionGeneration: backgroundCommandAttempt.executionGeneration,
2224
+ attemptId: backgroundCommandAttempt.attemptId,
2225
+ commandId: crypto.randomUUID(),
2226
+ controlWorkspaceId: command.controlWorkspaceId,
2227
+ enrollmentId: command.enrollmentId,
2228
+ connectionInstanceId: command.connectionInstanceId,
2229
+ opId: command.opId,
2230
+ command: command.command
2231
+ });
2232
+ return { commandId: adopted.id };
2233
+ },
2234
+ selfhostedSettleBackgroundCommand: async (command) => {
2235
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
2236
+ accountId: backgroundCommandAccountId,
2237
+ workspaceId: ids.workspaceId,
2238
+ sessionId: ids.sessionId,
2239
+ commandId: command.commandId,
2240
+ controlWorkspaceId: command.controlWorkspaceId,
2241
+ enrollmentId: command.enrollmentId,
2242
+ connectionInstanceId: command.connectionInstanceId,
2243
+ opId: command.opId,
2244
+ outcome: command.outcome,
2245
+ exitCode: command.exitCode,
2246
+ reason: command.reason,
2247
+ ...command.failure ? { failure: command.failure } : {}
2248
+ });
2249
+ if (settlement && bus) {
2250
+ await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
2251
+ }
2252
+ },
2253
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
2254
+ db,
2255
+ {
2256
+ accountId: backgroundCommandAccountId,
2257
+ workspaceId: ids.workspaceId,
2258
+ sessionId: ids.sessionId
2259
+ },
2260
+ bus
2261
+ )
2262
+ } : {},
2263
+ // The turn's declared environment → a selfhosted swap target's manifest, so the
2264
+ // SDK's per-turn manifest-env delta is empty (no "cannot change manifest
2265
+ // environment variables" throw when the turn pins to a vm). Mirrors the group
2266
+ // box, which is created WITH this same environment (resumeBoxForTurn).
2267
+ ...ids.environment !== void 0 ? { environment: ids.environment } : {},
2268
+ ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {},
2269
+ // Stage D machine-primary: pin THIS established SelfhostedSession for the machine
2270
+ // pointer so the resolver returns the SAME instance (no two-instance manifest
2271
+ // divergence). `established.session` is the SelfhostedSession the establish branch
2272
+ // bound to (sandboxId, epoch).
2273
+ ...ids.pinnedSelfhosted ? {
2274
+ pinnedSelfhosted: {
2275
+ sandboxId: ids.pinnedSelfhosted.sandboxId,
2276
+ epoch: ids.pinnedSelfhosted.epoch,
2277
+ session: established.session
2278
+ }
2279
+ } : {},
2280
+ // A modal swap target in the turn path would need its own lease resume-by-id;
2281
+ // that is a future cross-group-box concern. Until then a modal swap target is
2282
+ // unresolvable (the swap tool validates liveness, so this only triggers if a
2283
+ // session points at a sibling modal box the turn cannot resume here) and the
2284
+ // op surfaces unresolvable — never a silent wrong-box landing.
2285
+ //
2286
+ // For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
2287
+ // group box established this turn), a mid-turn clear-to-null must NOT fall back to
2288
+ // the pinned machine — passing defaultIsHome:false makes the null branch throw typed
2289
+ // `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
2290
+ ...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
2291
+ ...ids.homeLease ? {
2292
+ resolveDefaultBackend: () => resolveCurrentHomeBackend(
2293
+ services,
2294
+ {
2295
+ workspaceId: ids.workspaceId,
2296
+ sessionId: ids.sessionId,
2297
+ accountId: ids.homeLease.accountId,
2298
+ sandboxGroupId: ids.homeLease.sandboxGroupId,
2299
+ backend: ids.homeLease.backend
2300
+ },
2301
+ established
2302
+ )
2303
+ } : {}
2304
+ });
2305
+ const proxy = new RoutingSandboxSession({
2306
+ // Seed the DEFAULT backend (the established group box) at construction so
2307
+ // `session.state` is the real backend's state object BEFORE the first op. The
2308
+ // SDK reads `session.state.manifest` at turn START (and writes it back); an
2309
+ // empty `{}` there crashes serializeManifestEnvironment /
2310
+ // validateProvidedSessionManifestUpdate. This is byte-identical to what the
2311
+ // resolver returns for the default pointer (`activeSandboxId === null`).
2312
+ defaultResolved: {
2313
+ session: established.session,
2314
+ sandboxId: null,
2315
+ kind: established.backendId,
2316
+ ...ids.homeLease ? {
2317
+ leaseEpoch: ids.homeLease.leaseEpoch,
2318
+ providerInstanceId: ids.homeLease.instanceId
2319
+ } : {}
2320
+ },
2321
+ readPointer: async () => {
2322
+ if (!routingEnabled(settings)) {
2323
+ return { activeSandboxId: null, activeEpoch: 0 };
2324
+ }
2325
+ const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
2326
+ return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
2327
+ },
2328
+ resolveActiveBackend: resolver,
2329
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
2330
+ ...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
2331
+ providerCommandHandle: admittedCommandHandle,
2332
+ providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
2333
+ ...ids.workspaceMutationFence ? {
2334
+ providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
2335
+ db,
2336
+ {
2337
+ accountId: ids.workspaceMutationFence.accountId,
2338
+ workspaceId: ids.workspaceId,
2339
+ sessionId: ids.sessionId,
2340
+ processId: process2.id
2341
+ },
2342
+ services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
2343
+ )
2344
+ } : {},
2345
+ ...beforeMutation ? { beforeMutation } : {},
2346
+ ...afterMutation ? { afterMutation } : {},
2347
+ ...beforeProcessMutation ? { beforeProcessMutation } : {},
2348
+ ...afterProcessMutation ? { afterProcessMutation } : {},
2349
+ ...settleProcess ? { settleProcess } : {},
2350
+ ...isProcessSettled ? { isProcessSettled } : {},
2351
+ ...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
2352
+ ...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
2353
+ ...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
2354
+ ...ids.homeLease ? {
2355
+ onDefaultBackendError: async ({
2356
+ error,
2357
+ backend
2358
+ }) => {
2359
+ const home = ids.homeLease;
2360
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
2361
+ const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
2362
+ const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
2363
+ const marked = await markWarmLeaseInstanceLost2(db, {
2364
+ accountId: home.accountId,
2365
+ workspaceId: ids.workspaceId,
2366
+ sandboxGroupId: home.sandboxGroupId,
2367
+ expectedEpoch,
2368
+ expectedInstanceId,
2369
+ expectedBackend: home.backend,
2370
+ diagnostic: "provider_not_found_during_routed_operation"
2371
+ });
2372
+ if (marked.status === "marked") {
2373
+ await publishDurableSessionEvents(
2374
+ services.bus,
2375
+ ids.workspaceId,
2376
+ marked.backgroundCommandEvents
2377
+ );
2378
+ await services.onHomeSandboxLost?.({
2379
+ sandboxGroupId: home.sandboxGroupId,
2380
+ instanceId: expectedInstanceId,
2381
+ leaseEpoch: marked.lease.leaseEpoch
2382
+ });
2383
+ }
2384
+ const lease = marked.lease;
2385
+ const restore = lease?.recovery.restore.status;
2386
+ return {
2387
+ leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
2388
+ recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
2389
+ };
2390
+ }
2391
+ } : {}
2392
+ });
2393
+ return { ...established, session: proxy };
2394
+ }
2395
+ function wrapLazyTurnBoxWithRouting(services, ids, args) {
2396
+ const { db, settings, bus, onOp } = services;
2397
+ const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
2398
+ const backgroundCommandAttempt = ids.workspaceMutationFence;
2399
+ const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
2400
+ const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
2401
+ const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
2402
+ const afterProcessMutation = afterRetainedProcessMutation(services, ids);
2403
+ const settleProcess = settleRetainedProcessForTurn(services, ids);
2404
+ const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
2405
+ const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
2406
+ services,
2407
+ ids
2408
+ );
2409
+ const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
2410
+ const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
2411
+ const syntheticSession = {
2412
+ state: { manifest: args.agentDefaultManifest },
2413
+ // The SDK binds shell tools synchronously before lazy provisioning. Preserve
2414
+ // the authoritative descriptor's PTY shape so a Modal home exposes
2415
+ // write_stdin on its first turn operation; the async cancellation transport
2416
+ // below still resolves the exact live route before executing a command.
2417
+ supportsPty: () => defaultSupportsPty
2418
+ };
2419
+ const routedResolver = makeActiveBackendResolver({
2420
+ workspaceId: ids.workspaceId,
2421
+ defaultBackend: syntheticSession,
2422
+ defaultKind: "unprovisioned",
2423
+ getSandbox: async (sandboxId) => {
2424
+ const sandbox = await getSandbox(
2425
+ db,
2426
+ ids.resourceAccountId && ids.resourceSubjectId ? {
2427
+ accountId: ids.resourceAccountId,
2428
+ workspaceId: ids.workspaceId,
2429
+ subjectId: ids.resourceSubjectId
2430
+ } : ids.workspaceId,
2431
+ sandboxId
2432
+ );
2433
+ return sandbox ? {
2434
+ id: sandbox.id,
2435
+ workspaceId: sandbox.workspaceId,
2436
+ kind: sandbox.kind,
2437
+ name: sandbox.name,
2438
+ ...sandbox.scope ? { scope: sandbox.scope } : {},
2439
+ enrollmentId: sandbox.enrollmentId
2440
+ } : null;
2441
+ },
2442
+ controlRpcFactory: controlRpcFactory(bus),
2443
+ resolveSelfhostedConnection: async (sandbox) => {
2444
+ if (!sandbox.enrollmentId) return null;
2445
+ const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
2446
+ return connectionBindingFor(services, enrollment);
2447
+ },
2448
+ relay: relayConfigFromSettings(settings),
2449
+ ...selfhostedResolverTimeouts(settings),
2450
+ ...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
2451
+ ...backgroundCommandAccountId && backgroundCommandAttempt ? {
2452
+ selfhostedAdoptBackgroundCommand: async (command) => {
2453
+ const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
2454
+ accountId: backgroundCommandAccountId,
2455
+ workspaceId: ids.workspaceId,
2456
+ sessionId: ids.sessionId,
2457
+ turnId: backgroundCommandAttempt.turnId,
2458
+ executionGeneration: backgroundCommandAttempt.executionGeneration,
2459
+ attemptId: backgroundCommandAttempt.attemptId,
2460
+ commandId: crypto.randomUUID(),
2461
+ controlWorkspaceId: command.controlWorkspaceId,
2462
+ enrollmentId: command.enrollmentId,
2463
+ connectionInstanceId: command.connectionInstanceId,
2464
+ opId: command.opId,
2465
+ command: command.command
2466
+ });
2467
+ return { commandId: adopted.id };
2468
+ },
2469
+ selfhostedSettleBackgroundCommand: async (command) => {
2470
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
2471
+ accountId: backgroundCommandAccountId,
2472
+ workspaceId: ids.workspaceId,
2473
+ sessionId: ids.sessionId,
2474
+ commandId: command.commandId,
2475
+ controlWorkspaceId: command.controlWorkspaceId,
2476
+ enrollmentId: command.enrollmentId,
2477
+ connectionInstanceId: command.connectionInstanceId,
2478
+ opId: command.opId,
2479
+ outcome: command.outcome,
2480
+ exitCode: command.exitCode,
2481
+ reason: command.reason,
2482
+ ...command.failure ? { failure: command.failure } : {}
2483
+ });
2484
+ if (settlement && bus) {
2485
+ await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
2486
+ }
2487
+ },
2488
+ selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
2489
+ db,
2490
+ {
2491
+ accountId: backgroundCommandAccountId,
2492
+ workspaceId: ids.workspaceId,
2493
+ sessionId: ids.sessionId
2494
+ },
2495
+ bus
2496
+ )
2497
+ } : {},
2498
+ ...ids.environment !== void 0 ? { environment: ids.environment } : {},
2499
+ ...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {}
2500
+ });
2501
+ const proxy = new RoutingSandboxSession({
2502
+ // Before the first op the SDK reads `state.manifest`; the synthetic backend
2503
+ // points at agent.defaultManifest BY REFERENCE so the provided-session delta is
2504
+ // empty. The first default-pointer op resolves the real box through the
2505
+ // provisioner and `state` switches to that real backend by reference.
2506
+ defaultResolved: {
2507
+ session: syntheticSession,
2508
+ sandboxId: null,
2509
+ kind: "unprovisioned"
2510
+ },
2511
+ readPointer: async () => {
2512
+ const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
2513
+ return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
2514
+ },
2515
+ resolveActiveBackend: async (pointer) => {
2516
+ if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
2517
+ const provisioned = await args.provisioner.get();
2518
+ if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
2519
+ return resolveCurrentHomeBackend(
2520
+ services,
2521
+ {
2522
+ workspaceId: ids.workspaceId,
2523
+ sessionId: ids.sessionId,
2524
+ accountId: args.homeLeaseIdentity.accountId,
2525
+ sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
2526
+ backend: args.homeLeaseIdentity.backend
2527
+ },
2528
+ provisioned.established
2529
+ );
2530
+ }
2531
+ return {
2532
+ session: provisioned.established.session,
2533
+ sandboxId: null,
2534
+ kind: provisioned.established.backendId
2535
+ };
2536
+ }
2537
+ return routedResolver(pointer);
2538
+ },
2539
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
2540
+ ...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
2541
+ ...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
2542
+ providerCommandHandle: admittedCommandHandle,
2543
+ providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
2544
+ ...ids.workspaceMutationFence ? {
2545
+ providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
2546
+ db,
2547
+ {
2548
+ accountId: ids.workspaceMutationFence.accountId,
2549
+ workspaceId: ids.workspaceId,
2550
+ sessionId: ids.sessionId,
2551
+ processId: process2.id
2552
+ },
2553
+ services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
2554
+ )
2555
+ } : {},
2556
+ ...beforeMutation ? { beforeMutation } : {},
2557
+ ...afterMutation ? { afterMutation } : {},
2558
+ ...beforeProcessMutation ? { beforeProcessMutation } : {},
2559
+ ...afterProcessMutation ? { afterProcessMutation } : {},
2560
+ ...settleProcess ? { settleProcess } : {},
2561
+ ...isProcessSettled ? { isProcessSettled } : {},
2562
+ ...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
2563
+ ...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
2564
+ ...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
2565
+ ...args.homeLeaseIdentity ? {
2566
+ onDefaultBackendError: async ({
2567
+ error,
2568
+ backend
2569
+ }) => {
2570
+ const home = args.homeLeaseIdentity;
2571
+ if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
2572
+ if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
2573
+ return null;
2574
+ }
2575
+ const marked = await markWarmLeaseInstanceLost2(db, {
2576
+ accountId: home.accountId,
2577
+ workspaceId: ids.workspaceId,
2578
+ sandboxGroupId: home.sandboxGroupId,
2579
+ expectedEpoch: backend.leaseEpoch,
2580
+ expectedInstanceId: backend.providerInstanceId,
2581
+ expectedBackend: home.backend,
2582
+ diagnostic: "provider_not_found_during_routed_operation"
2583
+ });
2584
+ if (marked.status === "marked") {
2585
+ await publishDurableSessionEvents(
2586
+ services.bus,
2587
+ ids.workspaceId,
2588
+ marked.backgroundCommandEvents
2589
+ );
2590
+ await services.onHomeSandboxLost?.({
2591
+ sandboxGroupId: home.sandboxGroupId,
2592
+ instanceId: backend.providerInstanceId,
2593
+ leaseEpoch: marked.lease.leaseEpoch
2594
+ });
2595
+ }
2596
+ const lease = marked.lease;
2597
+ const restore = lease?.recovery.restore.status;
2598
+ return {
2599
+ leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
2600
+ recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
2601
+ };
2602
+ }
2603
+ } : {}
2604
+ });
2605
+ return {
2606
+ client: args.client,
2607
+ session: proxy,
2608
+ sessionState: void 0,
2609
+ instanceId: "unprovisioned",
2610
+ backendId: args.backendId
2611
+ };
2612
+ }
2613
+ function opStreamDepsFor(services, machineAdvertisesOpStream) {
2614
+ const { settings, bus, opJournal } = services;
2615
+ if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
2616
+ return void 0;
2617
+ }
2618
+ return {
2619
+ transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
2620
+ ...opJournal !== void 0 ? { journal: opJournal } : {}
2621
+ };
2622
+ }
2623
+ function connectionBindingFor(services, enrollment) {
2624
+ if (!enrollment?.connectionInstanceId) return null;
2625
+ const opStream = opStreamDepsFor(services, enrollment.opStream === true);
2626
+ return {
2627
+ workspaceId: enrollment.workspaceId,
2628
+ connectionInstanceId: enrollment.connectionInstanceId,
2629
+ workspaceRoot: enrollment.workspaceRoot,
2630
+ ...opStream !== void 0 ? { opStream } : {},
2631
+ operationResourcePolicy: enrollment.operationPolicy,
2632
+ operationResourcePolicySupported: enrollment.agentCapabilities.operationResourcePolicy === true,
2633
+ operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true,
2634
+ transactionalFsWriteSupported: enrollment.agentCapabilities.transactionalFsWrite === true
2635
+ };
2636
+ }
2637
+ async function resolveRoutedMachineConnection(db, ids, sandbox) {
2638
+ if (!sandbox.enrollmentId) return null;
2639
+ const current = await getLiveEnrollmentConnection(db, sandbox.workspaceId, sandbox.enrollmentId);
2640
+ if (!current) return null;
2641
+ if (current.scope === "user") {
2642
+ const authority = ids.personalMachineAttempt;
2643
+ if (!authority) return null;
2644
+ return await resolvePersonalMachineConnectionForAttempt(db, {
2645
+ accountId: authority.accountId,
2646
+ workspaceId: ids.workspaceId,
2647
+ subjectId: authority.subjectId,
2648
+ sessionId: ids.sessionId,
2649
+ turnId: authority.turnId,
2650
+ attemptId: authority.attemptId,
2651
+ executionGeneration: authority.executionGeneration,
2652
+ enrollmentId: sandbox.enrollmentId,
2653
+ requireActiveSandbox: true
2654
+ });
2655
+ }
2656
+ if (!ids.resourceAccountId || !ids.resourceSubjectId) return current;
2657
+ return await getLiveEnrollmentConnection(
2658
+ db,
2659
+ {
2660
+ accountId: ids.resourceAccountId,
2661
+ workspaceId: ids.workspaceId,
2662
+ subjectId: ids.resourceSubjectId
2663
+ },
2664
+ sandbox.enrollmentId
2665
+ );
2666
+ }
2667
+ async function establishSelfhostedTurnSession(services, args) {
2668
+ const { db, settings, bus, onOp } = services;
2669
+ const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
2670
+ const opStream = opStreamDepsFor(services, args.opStream);
2671
+ const { client, session } = await buildSelfhostedBackendSession({
2672
+ workspaceId: args.workspaceId,
2673
+ ...args.controlWorkspaceId !== void 0 ? { controlWorkspaceId: args.controlWorkspaceId } : {},
2674
+ agentId: args.agentId,
2675
+ connectionInstanceId: args.connectionInstanceId,
2676
+ workspaceRoot: args.workspaceRoot,
2677
+ relay: relayConfigFromSettings(settings),
2678
+ controlRpcFactory: controlRpcFactory(bus),
2679
+ epoch: args.epoch,
2680
+ environment: args.environment,
2681
+ ...args.transientExecEnvironment !== void 0 ? { transientExecEnvironment: args.transientExecEnvironment } : {},
2682
+ // Give this turn's exec ops the long deadline (control ops stay short) so a real
2683
+ // command is not killed at the control wall.
2684
+ timeoutMs,
2685
+ execTimeoutMs,
2686
+ operationResourcePolicy: args.operationResourcePolicy,
2687
+ operationResourcePolicySupported: args.operationResourcePolicySupported,
2688
+ operationCpuQuotaSupported: args.operationCpuQuotaSupported,
2689
+ transactionalFsWriteSupported: args.transactionalFsWriteSupported === true,
2690
+ resolveOperationAdmission: async () => {
2691
+ const current = await getLiveEnrollmentConnection(
2692
+ db,
2693
+ args.controlWorkspaceId ?? args.workspaceId,
2694
+ args.agentId
2695
+ );
2696
+ const enrollment = current?.scope !== "user" ? current : args.personalMachineAttempt ? await resolvePersonalMachineConnectionForAttempt(db, {
2697
+ accountId: args.personalMachineAttempt.accountId,
2698
+ workspaceId: args.workspaceId,
2699
+ subjectId: args.personalMachineAttempt.subjectId,
2700
+ sessionId: args.personalMachineAttempt.sessionId,
2701
+ turnId: args.personalMachineAttempt.turnId,
2702
+ attemptId: args.personalMachineAttempt.attemptId,
2703
+ executionGeneration: args.personalMachineAttempt.executionGeneration,
2704
+ enrollmentId: args.agentId,
2705
+ requireActiveSandbox: true
2706
+ }) : null;
2707
+ const binding = connectionBindingFor(services, enrollment);
2708
+ if (!binding) return null;
2709
+ if (!binding.workspaceRoot) {
2710
+ throw new ActiveBackendUnresolvableError(
2711
+ "workspace_root_unavailable",
2712
+ "Connected Machine no longer has a reported workspace root"
2713
+ );
2714
+ }
2715
+ return {
2716
+ ...binding,
2717
+ workspaceRoot: resolveConnectedMachineWorkspaceRoot(binding.workspaceRoot, args.workingDir)
2718
+ };
2719
+ },
2720
+ ...args.backgroundCommandAttempt ? {
2721
+ adoptBackgroundCommand: async (command) => {
2722
+ const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
2723
+ accountId: args.accountId,
2724
+ workspaceId: args.workspaceId,
2725
+ sessionId: args.sessionId,
2726
+ turnId: args.backgroundCommandAttempt.turnId,
2727
+ executionGeneration: args.backgroundCommandAttempt.executionGeneration,
2728
+ attemptId: args.backgroundCommandAttempt.attemptId,
2729
+ commandId: crypto.randomUUID(),
2730
+ controlWorkspaceId: command.controlWorkspaceId,
2731
+ enrollmentId: command.enrollmentId,
2732
+ connectionInstanceId: command.connectionInstanceId,
2733
+ opId: command.opId,
2734
+ command: command.command
2735
+ });
2736
+ return { commandId: adopted.id };
2737
+ }
2738
+ } : {},
2739
+ captureBackgroundCommandOutput: captureConnectedCommandOutput(
2740
+ db,
2741
+ {
2742
+ accountId: args.accountId,
2743
+ workspaceId: args.workspaceId,
2744
+ sessionId: args.sessionId
2745
+ },
2746
+ bus
2747
+ ),
2748
+ settleBackgroundCommand: async (command) => {
2749
+ const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
2750
+ accountId: args.accountId,
2751
+ workspaceId: args.workspaceId,
2752
+ sessionId: args.sessionId,
2753
+ commandId: command.commandId,
2754
+ controlWorkspaceId: command.controlWorkspaceId,
2755
+ enrollmentId: command.enrollmentId,
2756
+ connectionInstanceId: command.connectionInstanceId,
2757
+ opId: command.opId,
2758
+ outcome: command.outcome,
2759
+ exitCode: command.exitCode,
2760
+ reason: command.reason,
2761
+ ...command.failure ? { failure: command.failure } : {}
2762
+ });
2763
+ if (command.outputReceipt && command.outcome === "exited" && command.exitCode !== null) {
2764
+ await recordConnectedCommandOutputConsumption(db, {
2765
+ accountId: args.accountId,
2766
+ workspaceId: args.workspaceId,
2767
+ sessionId: args.sessionId,
2768
+ commandId: command.commandId,
2769
+ controlWorkspaceId: command.controlWorkspaceId,
2770
+ enrollmentId: command.enrollmentId,
2771
+ connectionInstanceId: command.connectionInstanceId,
2772
+ opId: command.opId,
2773
+ receipt: command.outputReceipt,
2774
+ exitCode: command.exitCode
2775
+ });
2776
+ }
2777
+ if (settlement && bus) {
2778
+ await bus.publish(args.workspaceId, args.sessionId, settlement.events).catch(() => void 0);
2779
+ }
2780
+ },
2781
+ // Meter every control op (out-of-band telemetry) — no-op when unwired.
2782
+ ...onOp !== void 0 ? { onOp } : {},
2783
+ // The streaming exec transport — present iff the machine advertised the
2784
+ // capability AND the server flag is on. Connected Machine exec fails closed
2785
+ // when it is absent; there is no request/reply downgrade.
2786
+ ...opStream !== void 0 ? { opStream } : {}
2787
+ });
2788
+ return {
2789
+ client,
2790
+ session,
2791
+ sessionState: { agentId: args.agentId },
2792
+ instanceId: args.agentId,
2793
+ backendId: "selfhosted"
2794
+ };
2795
+ }
2796
+ function routingEnabled(settings) {
2797
+ return settings.sandboxSelfhostedEnabled === true;
2798
+ }
2799
+ function lazyProvisionEnabled(settings) {
2800
+ return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
2801
+ }
2802
+
2803
+ export {
2804
+ warnDrainSnapshotFailure,
2805
+ publishDurableSessionEvents,
2806
+ persistWorkspaceArchiveCandidate,
2807
+ putVersion1TarArchiveOrInline,
2808
+ sandboxLeaseHolderIdForAttempt,
2809
+ SandboxProvisionStageError,
2810
+ sandboxProvisionFailureStage,
2811
+ SandboxWarmingTimeoutError,
2812
+ createFreshSandboxReadinessReplacementBudget,
2813
+ waitForWarmSnapshot,
2814
+ maybePersistWarmWorkspaceSnapshot,
2815
+ resumeBoxForTurn,
2816
+ captureConnectedCommandOutput,
2817
+ wrapTurnBoxWithRouting,
2818
+ wrapLazyTurnBoxWithRouting,
2819
+ establishSelfhostedTurnSession,
2820
+ routingEnabled,
2821
+ lazyProvisionEnabled
2822
+ };
2823
+ //# sourceMappingURL=chunk-NQDVJ6RI.js.map