@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38686408778001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +172 -28
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +3 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +30 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-NTXZHYET.js} +1552 -2876
  123. package/dist/activities-control-NTXZHYET.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-RKRI5GY4.js} +9238 -4392
  126. package/dist/activities-turn-RKRI5GY4.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +104 -35
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/{chunk-M3NVMF2V.js → chunk-7BIHFUGB.js} +2293 -3956
  140. package/dist/chunk-7BIHFUGB.js.map +1 -0
  141. package/dist/chunk-MVESOYFC.js +3746 -0
  142. package/dist/chunk-MVESOYFC.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1134 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +26 -4
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +94 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +599 -110
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +162 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +116 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +758 -141
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +79 -38
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -0,0 +1,3746 @@
1
+ import {
2
+ captureConnectedCommandOutput,
3
+ maybePersistWarmWorkspaceSnapshot,
4
+ persistWorkspaceArchiveCandidate,
5
+ publishDurableSessionEvents,
6
+ putVersion1TarArchiveOrInline,
7
+ warnDrainSnapshotFailure
8
+ } from "./chunk-NQDVJ6RI.js";
9
+ import {
10
+ CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
11
+ SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
12
+ SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
13
+ assertSandboxDrainInputTiming,
14
+ sandboxDrainTimeoutClass,
15
+ sandboxDrainTiming,
16
+ sandboxIdleCheckpointWorkflowId
17
+ } from "./chunk-24GP3BX5.js";
18
+ import {
19
+ inspectOpenSandboxKubernetesInventory,
20
+ recordActiveUserGauges,
21
+ recordCreditBalanceGauges,
22
+ recordCreditGrantGauges,
23
+ recordCreditMicros,
24
+ recordExpiredDrainingSandboxLeaseGauges,
25
+ recordInteractionOnlyLeaseGauges,
26
+ recordManagedAuthNewSignupsSwitchGauge,
27
+ recordModalSandboxInventoryGauges,
28
+ recordOpenSandboxKubernetesInventoryGauges,
29
+ recordRetainedProcessInventoryGauges,
30
+ recordRetainedProcessReconciliation,
31
+ recordSandboxCheckpointArtifactGauges,
32
+ recordSandboxCheckpointArtifactOutcome,
33
+ recordSandboxCheckpointStalenessGauges,
34
+ recordSandboxCommandContainment,
35
+ recordSandboxDeadlineRotationsRequested,
36
+ recordSandboxDockerWorkspaceRelease,
37
+ recordSandboxIdleInteractionRelease,
38
+ recordSandboxInventoryProjectionFailure,
39
+ recordSandboxInventoryProjectionSuccess,
40
+ recordSandboxLeaseGauges,
41
+ recordSandboxOrphansTerminated,
42
+ recordSandboxProviderMissingBeforeCapture,
43
+ recordSandboxRecoveryObservationGauges,
44
+ recordSandboxRotationBacklogGauges,
45
+ recordTurnsQueuedGauge,
46
+ recordVerifiedSignupTrialDeploymentFlagGauge,
47
+ recordVerifiedSignupTrialSwitchGauge,
48
+ runtimeMetricsHooksForObservability
49
+ } from "./chunk-5NNS4327.js";
50
+ import {
51
+ reconcilePendingParentSystemUpdates
52
+ } from "./chunk-W2QSM5HR.js";
53
+
54
+ // src/retained-process-diagnostics.ts
55
+ import {
56
+ databaseFailureCode,
57
+ nestedPostgresSqlState,
58
+ safeDatabaseErrorFacts,
59
+ SandboxWorkspaceMutationFencedError
60
+ } from "@opengeni/db";
61
+ function warnRetainedProcessProofFailure(observability, error, process) {
62
+ let correlationId;
63
+ try {
64
+ const sqlState = nestedPostgresSqlState(error);
65
+ const constraint = safeDatabaseErrorFacts(error).constraint;
66
+ correlationId = observability.recordFailureDiagnostic({
67
+ code: error instanceof SandboxWorkspaceMutationFencedError ? "retained_process_fenced" : sqlState ? databaseFailureCode(sqlState) : "retained_process_proof_failed",
68
+ stage: "sandbox_retained_processes.proof",
69
+ error,
70
+ processId: process.id,
71
+ sessionId: process.sessionId,
72
+ ...process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {},
73
+ ...process.ownerTurnId ? { turnId: process.ownerTurnId } : {},
74
+ attempts: process.reconcileAttempts,
75
+ sqlState,
76
+ ...constraint ? { constraint } : {}
77
+ });
78
+ } catch {
79
+ }
80
+ try {
81
+ observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
82
+ ...correlationId ? { correlationId } : {}
83
+ });
84
+ } catch {
85
+ }
86
+ }
87
+
88
+ // src/retained-process-retry.ts
89
+ function retainedProcessDeadlineRetryMs(process, lease, settings, retryAfterMs, now = Date.now()) {
90
+ if (process.providerBackend !== "modal" || process.routeTargetId !== null || !lease || lease.id !== process.leaseId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.providerDeadlineAt)
91
+ return retryAfterMs;
92
+ const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;
93
+ if (!Number.isFinite(untilRotation)) return retryAfterMs;
94
+ return Math.min(
95
+ retryAfterMs,
96
+ Math.max(1e3, settings.sandboxLeaseReaperPeriodMs, untilRotation)
97
+ );
98
+ }
99
+
100
+ // src/activities/sandbox-lease.ts
101
+ import { createHash, randomUUID } from "crypto";
102
+ import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
103
+ import { Context as Context2 } from "@temporalio/activity";
104
+ import { OpLostReason, OpState } from "@opengeni/agent-proto";
105
+ import {
106
+ accrueWarmSeconds,
107
+ retainedProviderCommandPersistence,
108
+ adoptLegacyModalCheckpointArtifact,
109
+ confirmDrainCold,
110
+ markWarmLeaseInstanceLost,
111
+ appendSessionEventToSandboxGroup,
112
+ bindRetainedProcessProviderIdentity,
113
+ claimWorkspaceArchiveCapture,
114
+ leaseMayHaveUntrackedWriters,
115
+ releaseWorkspaceArchiveCapture,
116
+ listIdleCheckpointCandidates,
117
+ recordIdleCheckpointAttempt,
118
+ claimSandboxCheckpointArtifactsForGc,
119
+ claimTerminalRetainedProcesses,
120
+ getEnrollment,
121
+ countActiveRetainedProcessesByOwnerState,
122
+ countSandboxCheckpointArtifactsByState,
123
+ countExpiredDrainingSandboxLeases,
124
+ countQueuedTurns,
125
+ countSandboxLeasesByLiveness,
126
+ deferRetainedProcessReconciliation,
127
+ forceDrainOverLimitViewerOnlyBoxes,
128
+ listCreditBalancesByAccount,
129
+ countActiveUsers,
130
+ readCreditGrantTotals,
131
+ readManagedAuthNewSignupsSwitch,
132
+ readVerifiedSignupTrialSwitch,
133
+ listLegacyModalCheckpointSlots,
134
+ listLiveModalSandboxLeaseAttributions,
135
+ listPendingModalProviderCreates,
136
+ claimModalProviderCreateRecovery,
137
+ recordRecoveredModalProviderCreate,
138
+ markWarmBillingStopCutoff,
139
+ listMeterableWarmLeases,
140
+ countInteractionOnlyWarmLeasesByIdle,
141
+ listSandboxViewerForceDrainWorkspaceIds,
142
+ markSandboxCheckpointArtifactDeletePending,
143
+ persistDrainSnapshot,
144
+ pruneDeletedSandboxCheckpointArtifacts,
145
+ registerSandboxCheckpointArtifact,
146
+ recordRetainedProcessReconciliationProof,
147
+ replaceWorkspaceArchiveCaptureAfterProof,
148
+ listEndedEpochWorkspaceBlockers,
149
+ listEndedEpochSandboxSetupUncertainties,
150
+ reconcileCompletedSandboxSetup,
151
+ readLease as readLease2,
152
+ settleEndedEpochWorkspaceBlockers,
153
+ readWorkspaceArchiveCapturePreflight,
154
+ reapExpiredSessionListSnapshots,
155
+ reapStaleLeaseHoldersGlobal,
156
+ requestDueSandboxRotationsGlobal,
157
+ readSandboxRotationBacklog,
158
+ readSandboxCheckpointStaleness,
159
+ readRecentSandboxRecoveryObservations,
160
+ workspaceArchiveCaptureDeadlineElapsed,
161
+ retainedProcessReconciliationProof,
162
+ retainedProcessSettlementIdentity,
163
+ recoverManagedSessionBackgroundCommand,
164
+ settleClaimedConnectedMachineBackgroundCommand,
165
+ settleSandboxCheckpointArtifactGc,
166
+ rlsContextForWorkspace,
167
+ settleRetainedProcess
168
+ } from "@opengeni/db";
169
+ import {
170
+ claimConnectedCommandOutputReleases,
171
+ claimConnectedMachineSessionBackgroundCommands,
172
+ deferConnectedMachineBackgroundCommandReconciliation,
173
+ recordConnectedMachineBackgroundCommandProof,
174
+ recordConnectedCommandOutputConsumption,
175
+ settleConnectedCommandOutputReleaseClaim
176
+ } from "@opengeni/db/session-background-commands";
177
+ import {
178
+ SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
179
+ sandboxArchiveCaptureTimeoutMs,
180
+ sandboxDeadlineMandatoryCaptureLeadMs,
181
+ sandboxWarmRateMicrosPerSecond
182
+ } from "@opengeni/config";
183
+ import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
184
+ import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
185
+ import {
186
+ captureWorkspaceArchiveForStorage,
187
+ disposeWorkspaceArchive,
188
+ cancelSelfhostedOp,
189
+ assertConsistentSandboxProviderIdentity,
190
+ createSandboxClientForBackend as createSandboxClientForBackend2,
191
+ deleteModalCheckpointSnapshot,
192
+ deserializeSandboxSessionStateEnvelope as deserializeSandboxSessionStateEnvelope2,
193
+ establishSandboxSessionFromEnvelope,
194
+ inspectModalSandboxLifecycle,
195
+ isExecSessionLostBanner,
196
+ isProviderSandboxNotFoundError,
197
+ isProviderSandboxGoneDuringRoutedOperation,
198
+ NatsControlRpc,
199
+ NatsOpStreamTransport,
200
+ OpStreamExecClient,
201
+ defaultSelfhostedRetryClock,
202
+ stripExecBanner,
203
+ parseExecBannerExitCode,
204
+ prepareProviderForTeardownAfterCapture,
205
+ providerWorkspaceCapturePolicy,
206
+ providerWorkspaceCaptureIsPointInTime,
207
+ resolveModalCheckpointProviderBindingForLiveSandbox,
208
+ resolveModalCheckpointProviderBinding,
209
+ findModalProviderCreateReceipt,
210
+ resolveModalCheckpointProviderBindingForSession,
211
+ querySelfhostedOp,
212
+ resumeExactSandboxSession as resumeExactSandboxSession2,
213
+ sandboxBackendForSdkBackendId,
214
+ sandboxProviderInstanceIdFromEnvelope,
215
+ sandboxCommandExitCode,
216
+ sandboxCommandStillRunning,
217
+ SelfhostedControlError,
218
+ subjectFor,
219
+ modalAppHasRunningSandboxes,
220
+ sweepModalOrphanSandboxes,
221
+ terminateManagedSandboxSession,
222
+ terminateModalSandboxById,
223
+ verifySandboxExecReadiness
224
+ } from "@opengeni/runtime/sandbox";
225
+
226
+ // src/activities/browser-deadline-checkpoint.ts
227
+ import { Context } from "@temporalio/activity";
228
+ import {
229
+ environmentsEncryptionKeyBytes,
230
+ resolveFirstPartyDelegationSecret
231
+ } from "@opengeni/config";
232
+ import { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from "@opengeni/contracts";
233
+ import {
234
+ browserDeadlineCheckpoint,
235
+ clearSuspendedBrowserSessionController,
236
+ commitBrowserSessionSuspension,
237
+ dispatchBrowserSessionOperation,
238
+ failBrowserSessionSuspension,
239
+ listBrowserDeadlineCheckpoints,
240
+ readLease,
241
+ releaseLeaseHolder,
242
+ withRlsContext
243
+ } from "@opengeni/db";
244
+ import { deriveBrowserControllerAdminToken } from "@opengeni/interaction/browser-controller-authority";
245
+ import {
246
+ browserStateArtifactAad,
247
+ browserStateManifestDigest,
248
+ browserStateObjectKey,
249
+ deriveBrowserStateDataKey,
250
+ wrapBrowserStateDataKey
251
+ } from "@opengeni/interaction/browser-state-authority";
252
+ import {
253
+ BrowserControlClient,
254
+ BrowserControlRequestError,
255
+ createSandboxClientForBackend,
256
+ deserializeSandboxSessionStateEnvelope,
257
+ resumeExactSandboxSession
258
+ } from "@opengeni/runtime";
259
+ var UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 6e4;
260
+ function heartbeat() {
261
+ try {
262
+ Context.current().heartbeat();
263
+ } catch {
264
+ }
265
+ }
266
+ async function withActivityHeartbeat(run) {
267
+ heartbeat();
268
+ const timer = setInterval(heartbeat, 5e3);
269
+ try {
270
+ return await run();
271
+ } finally {
272
+ clearInterval(timer);
273
+ }
274
+ }
275
+ async function resumeCheckpointController({
276
+ target,
277
+ lease,
278
+ settings,
279
+ controllerRoot
280
+ }) {
281
+ const provider = createSandboxClientForBackend("modal", settings);
282
+ const envelope = lease.resumeState;
283
+ const state = await deserializeSandboxSessionStateEnvelope(
284
+ provider,
285
+ envelope.sessionState ?? lease.resumeState,
286
+ target.instanceId
287
+ );
288
+ const resumed = await resumeExactSandboxSession(provider, "modal", state, target.instanceId);
289
+ return new BrowserControlClient(resumed.session, {
290
+ adminToken: deriveBrowserControllerAdminToken({
291
+ rootSecret: controllerRoot,
292
+ accountId: target.accountId,
293
+ workspaceId: target.workspaceId,
294
+ placement: { kind: "sandbox_group", sandboxGroupId: target.sandboxGroupId },
295
+ placementInstanceId: target.instanceId
296
+ })
297
+ });
298
+ }
299
+ function browserCheckpointPrerequisites(settings, objectStorage) {
300
+ if (!settings.sandboxOwnershipEnabled || !objectStorage || !resolveFirstPartyDelegationSecret(settings))
301
+ return false;
302
+ const root = environmentsEncryptionKeyBytes(settings);
303
+ if (!root) return false;
304
+ root.fill(0);
305
+ return true;
306
+ }
307
+ function createBrowserDeadlineCheckpointActivities(services, connectController = resumeCheckpointController) {
308
+ return {
309
+ async listDueBrowserCheckpoints() {
310
+ return await withActivityHeartbeat(async () => {
311
+ const { db, settings, objectStorage } = await services();
312
+ if (!browserCheckpointPrerequisites(settings, objectStorage)) return [];
313
+ return await listBrowserDeadlineCheckpoints(db);
314
+ });
315
+ },
316
+ async checkpointBrowserBeforeDeadline(target) {
317
+ return await withActivityHeartbeat(async () => {
318
+ const { db, settings, objectStorage } = await services();
319
+ const rootKey = environmentsEncryptionKeyBytes(settings);
320
+ const controllerRoot = resolveFirstPartyDelegationSecret(settings);
321
+ if (!rootKey || !objectStorage || !controllerRoot) {
322
+ rootKey?.fill(0);
323
+ throw new Error("browser deadline checkpoint storage or encryption is unavailable");
324
+ }
325
+ let dataKey = null;
326
+ let aad = null;
327
+ let timer = null;
328
+ try {
329
+ const claim = await browserDeadlineCheckpoint(db, target, {
330
+ prepare: target.reason !== "idle",
331
+ touch: true
332
+ });
333
+ if (!claim) return { status: "skipped" };
334
+ let pulseRunning = false;
335
+ let authorityLost = false;
336
+ const pulse = async () => {
337
+ if (pulseRunning) return;
338
+ pulseRunning = true;
339
+ try {
340
+ if (!await browserDeadlineCheckpoint(db, target, { touch: true }))
341
+ authorityLost = true;
342
+ } finally {
343
+ pulseRunning = false;
344
+ }
345
+ };
346
+ timer = setInterval(() => {
347
+ void pulse().catch(() => void 0);
348
+ }, 5e3);
349
+ const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
350
+ if (!lease || lease.instanceId !== target.instanceId || lease.leaseEpoch !== target.leaseEpoch || !lease.resumeState) {
351
+ throw new Error("browser deadline checkpoint placement changed");
352
+ }
353
+ const placement = {
354
+ kind: "sandbox_group",
355
+ sandboxGroupId: target.sandboxGroupId
356
+ };
357
+ const client = await connectController({ target, lease, settings, controllerRoot });
358
+ if (authorityLost || !await browserDeadlineCheckpoint(db, target, { touch: true })) {
359
+ throw new Error("browser deadline checkpoint authority changed");
360
+ }
361
+ if (claim.state !== "completed") {
362
+ const scope = { ...target, operationId: claim.operationId };
363
+ const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);
364
+ dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });
365
+ aad = browserStateArtifactAad({ ...scope, objectKey });
366
+ await dispatchBrowserSessionOperation(db, {
367
+ ...scope,
368
+ deadlineTarget: target,
369
+ stateUpload: {
370
+ objectKey,
371
+ cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS)
372
+ }
373
+ });
374
+ const signed = await objectStorage.createPutUrl({
375
+ key: objectKey,
376
+ contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,
377
+ audience: "sandbox"
378
+ });
379
+ const receipt = await client.captureState({
380
+ ...scope,
381
+ objectKey,
382
+ afterCapture: "stop",
383
+ dataKey,
384
+ aad,
385
+ upload: {
386
+ url: signed.url,
387
+ requiredHeaders: signed.requiredHeaders,
388
+ expiresAt: signed.expiresAt.toISOString()
389
+ }
390
+ }).catch(async (error) => {
391
+ if (error instanceof BrowserControlRequestError && (error.error.code === "outcome_unknown" || !error.error.retryable)) {
392
+ await failBrowserSessionSuspension(db, { ...scope, error: error.error });
393
+ }
394
+ throw error;
395
+ });
396
+ if (authorityLost) throw new Error("browser deadline checkpoint authority changed");
397
+ const portable = receipt.manifest.profileCrypto !== "platform_bound";
398
+ await commitBrowserSessionSuspension(db, {
399
+ ...scope,
400
+ deadlineTarget: target,
401
+ artifact: {
402
+ kind: "chromium_profile",
403
+ format: receipt.format,
404
+ objectKey,
405
+ artifactDigest: receipt.artifactDigest,
406
+ contentDigest: receipt.contentDigest,
407
+ manifestDigest: browserStateManifestDigest(receipt.manifest),
408
+ sizeBytes: receipt.sizeBytes,
409
+ encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {
410
+ ...scope,
411
+ objectKey,
412
+ artifactDigest: receipt.artifactDigest,
413
+ contentDigest: receipt.contentDigest
414
+ }),
415
+ materialization: {
416
+ portability: portable ? "portable" : "placement_bound",
417
+ reason: portable ? null : "Profile encryption depends on the source operating-system credential store.",
418
+ platform: receipt.manifest.platform,
419
+ architecture: receipt.manifest.architecture,
420
+ engine: receipt.manifest.engine,
421
+ engineVersion: receipt.manifest.engineVersion,
422
+ driverId: receipt.manifest.driverId,
423
+ driverSchemaVersion: receipt.manifest.driverSchemaVersion,
424
+ profileCrypto: receipt.manifest.profileCrypto,
425
+ providerId: null,
426
+ placement: portable ? null : placement
427
+ }
428
+ }
429
+ });
430
+ }
431
+ await client.endSession(target, { removeState: true }).catch((error) => {
432
+ if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;
433
+ });
434
+ await withRlsContext(
435
+ db,
436
+ target,
437
+ async (tx) => {
438
+ if ((await browserDeadlineCheckpoint(tx, target))?.state !== "completed") {
439
+ throw new Error("browser deadline checkpoint cleanup authority changed");
440
+ }
441
+ await clearSuspendedBrowserSessionController(tx, {
442
+ ...target,
443
+ expectedControllerGeneration: target.controllerGeneration
444
+ });
445
+ await releaseLeaseHolder(tx, {
446
+ ...target,
447
+ kind: "interaction",
448
+ holderId: `browser-session:${target.browserSessionId}`,
449
+ // The idle window already elapsed before this save began, so
450
+ // a box the saved browser was last to hold drains now.
451
+ idleGraceMs: target.reason === "idle" ? 0 : settings.sandboxIdleGraceMs,
452
+ preserveHoldersChangedAt: target.reason === "idle"
453
+ });
454
+ },
455
+ void 0,
456
+ "none"
457
+ );
458
+ return { status: "suspended" };
459
+ } finally {
460
+ if (timer) clearInterval(timer);
461
+ dataKey?.fill(0);
462
+ aad?.fill(0);
463
+ rootKey.fill(0);
464
+ }
465
+ });
466
+ }
467
+ };
468
+ }
469
+
470
+ // src/activities/sandbox-lease.ts
471
+ import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni/observability";
472
+ var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
473
+ var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
474
+ var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
475
+ var RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
476
+ var RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 1e4;
477
+ var RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 6e4;
478
+ var RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
479
+ var RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
480
+ var RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 6e4;
481
+ var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
482
+ var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
483
+ var SandboxProviderCaptureTimeoutError = class extends Error {
484
+ constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
485
+ super(
486
+ `sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} workspace capture exceeded ${timeoutMs}ms; the exact durable claim remains fenced while its provider response settles`
487
+ );
488
+ this.sandboxGroupId = sandboxGroupId;
489
+ this.backend = backend;
490
+ this.timeoutMs = timeoutMs;
491
+ this.leaseEpoch = leaseEpoch;
492
+ this.instanceId = instanceId;
493
+ }
494
+ name = "SandboxProviderCaptureTimeoutError";
495
+ };
496
+ async function awaitProviderCaptureWithLatePublication(input) {
497
+ let timeout;
498
+ const timeoutError = input.timeoutError;
499
+ try {
500
+ return await Promise.race([
501
+ input.capture,
502
+ new Promise((_, reject) => {
503
+ timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);
504
+ })
505
+ ]);
506
+ } catch (error) {
507
+ if (error !== timeoutError) throw error;
508
+ const settled = () => {
509
+ try {
510
+ input.onLateSettled?.();
511
+ } catch {
512
+ }
513
+ };
514
+ void input.capture.then(input.publishLate, input.observeLateFailure).catch(input.observeLatePublicationFailure).then(settled, settled);
515
+ throw error;
516
+ } finally {
517
+ if (timeout) clearTimeout(timeout);
518
+ }
519
+ }
520
+ function startSandboxReaperHeartbeat(details) {
521
+ let context;
522
+ try {
523
+ context = Context2.current();
524
+ } catch {
525
+ return () => void 0;
526
+ }
527
+ const heartbeat2 = () => {
528
+ try {
529
+ context.heartbeat({ ...details, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() });
530
+ } catch {
531
+ }
532
+ };
533
+ heartbeat2();
534
+ const timer = setInterval(heartbeat2, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
535
+ if ("unref" in timer && typeof timer.unref === "function") timer.unref();
536
+ return () => clearInterval(timer);
537
+ }
538
+ var SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(
539
+ 1,
540
+ Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2)
541
+ );
542
+ var SandboxDrainRecoveryDeferredError = class extends Error {
543
+ name = "SandboxDrainRecoveryDeferredError";
544
+ };
545
+ var settlingDrainCaptures = /* @__PURE__ */ new Map();
546
+ function drainCaptureKey(lease) {
547
+ return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);
548
+ }
549
+ function sandboxDrainCaptureSettlingInProcess(lease, nowMs = Date.now()) {
550
+ if (!lease.instanceId) return false;
551
+ const entry = settlingDrainCaptures.get(
552
+ drainCaptureKey({ ...lease, instanceId: lease.instanceId })
553
+ );
554
+ return entry !== void 0 && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;
555
+ }
556
+ function trackSettlingDrainCapture(lease) {
557
+ if (!lease.instanceId) return () => void 0;
558
+ const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });
559
+ const entry = { startedAtMs: Date.now() };
560
+ settlingDrainCaptures.set(key, entry);
561
+ return () => {
562
+ if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);
563
+ };
564
+ }
565
+ function sandboxDrainCaptureId(operationId, attempt) {
566
+ if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {
567
+ throw new Error("Sandbox drain operation id is invalid");
568
+ }
569
+ if (!Number.isSafeInteger(attempt) || attempt <= 0) {
570
+ throw new Error("Sandbox drain activity attempt is invalid");
571
+ }
572
+ if (attempt === 1) return operationId.toLowerCase();
573
+ const bytes = createHash("sha256").update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`).digest().subarray(0, 16);
574
+ bytes[6] = bytes[6] & 15 | 80;
575
+ bytes[8] = bytes[8] & 63 | 128;
576
+ const hex = bytes.toString("hex");
577
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
578
+ }
579
+ function sandboxDrainActivityAttempt() {
580
+ try {
581
+ const attempt = Context2.current().info.attempt;
582
+ return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
583
+ } catch {
584
+ return 1;
585
+ }
586
+ }
587
+ function createSandboxLeaseActivities(services, options = {}) {
588
+ const terminateBox = (options.terminateBox ? async (...args) => {
589
+ await args[10]?.();
590
+ return options.terminateBox(...args);
591
+ } : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop, beforeWorkspaceCapture) => await terminateProviderBox(
592
+ settings,
593
+ lease,
594
+ observability,
595
+ persistArchive,
596
+ options.createClientForBackend ?? createSandboxClientForBackend2,
597
+ terminateModalSandboxById,
598
+ providerCaptureRequestId,
599
+ captureDisposition,
600
+ capturePolicy,
601
+ diskBackedArchives,
602
+ releaseFailedCapture,
603
+ workspaceId,
604
+ beforeProviderStop,
605
+ beforeWorkspaceCapture
606
+ ));
607
+ const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
608
+ const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
609
+ const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
610
+ const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
611
+ const resumeIdleCheckpointSession = options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;
612
+ const embeddedIdleCheckpoints = /* @__PURE__ */ new Set();
613
+ const endedEpochProbeBackoff = /* @__PURE__ */ new Map();
614
+ async function prepareSandboxLeaseSweep() {
615
+ const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
616
+ try {
617
+ const { db, settings, observability, objectStorage } = await services();
618
+ const timing = sandboxDrainTiming(settings);
619
+ const commandContainment = {
620
+ // Browsers and desktops follow the general idle grace: unused for it,
621
+ // they stop holding the box (saved browsers resume on demand).
622
+ // Never under a minute, so a visible live view's 30 s heartbeat
623
+ // always lands inside the window.
624
+ idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 6e4),
625
+ idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),
626
+ onIdleInteractionRelease: (outcome) => recordSandboxIdleInteractionRelease(observability, outcome),
627
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
628
+ deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
629
+ onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
630
+ onCommandContainmentError: (error) => {
631
+ observability.warn("sandbox reaper: retained command containment inspection failed", {
632
+ error: error instanceof Error ? error.message : String(error)
633
+ });
634
+ }
635
+ };
636
+ if (!settings.sandboxOwnershipEnabled) {
637
+ const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
638
+ ...commandContainment,
639
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
640
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
641
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
642
+ idleGraceMs: settings.sandboxIdleGraceMs
643
+ });
644
+ const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
645
+ return {
646
+ drainable: drainable2,
647
+ timeoutClass: timing.timeoutClass,
648
+ snapshotTimeoutMs: timing.snapshotTimeoutMs,
649
+ captureTimeoutMs: timing.captureTimeoutMs,
650
+ rotationsRequested: 0
651
+ };
652
+ }
653
+ const rotationsRequested = await requestDueSandboxRotationsGlobal(
654
+ db,
655
+ settings.sandboxRotationLeadMs,
656
+ settings.sandboxRotationBatchSize
657
+ ).catch((error) => {
658
+ observability.warn("sandbox reaper: provider-deadline rotation request failed", {
659
+ error: error instanceof Error ? error.message : String(error)
660
+ });
661
+ return 0;
662
+ });
663
+ recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
664
+ const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
665
+ ...commandContainment,
666
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
667
+ // Dead-worker turn holders: a live holder is touched every 10s from the
668
+ // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
669
+ // the whole warmup — waitForWarm/establish/display-stack — and the turn
670
+ // heartbeat covers the run), so NO live path is ever silent for more than
671
+ // one tick. The ordinary warm lease TTL is therefore already a generous
672
+ // dead-worker horizon. Canonical holders whose turn is durably closed are
673
+ // removed immediately by the DB function; no path-duration guess belongs
674
+ // in this lifecycle contract.
675
+ turnHolderTtlMs: settings.sandboxLeaseTtlMs,
676
+ interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
677
+ idleGraceMs: settings.sandboxIdleGraceMs
678
+ });
679
+ const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
680
+ if (drainableInventory.length > drainable.length) {
681
+ observability.info("sandbox reaper: drain inventory continues next sweep", {
682
+ admitted: drainable.length,
683
+ deferred: drainableInventory.length - drainable.length
684
+ });
685
+ }
686
+ return {
687
+ drainable,
688
+ timeoutClass: timing.timeoutClass,
689
+ snapshotTimeoutMs: timing.snapshotTimeoutMs,
690
+ captureTimeoutMs: timing.captureTimeoutMs,
691
+ rotationsRequested
692
+ };
693
+ } finally {
694
+ stopHeartbeat();
695
+ }
696
+ }
697
+ async function listIdleSandboxCheckpoints() {
698
+ const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "idle_checkpoint_inventory" });
699
+ try {
700
+ const { db, settings } = await services();
701
+ const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));
702
+ if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
703
+ return { targets: [], timeoutClass };
704
+ }
705
+ const rows = await listIdleCheckpointCandidates(db, {
706
+ limit: SANDBOX_IDLE_CHECKPOINT_BATCH,
707
+ intervalMs: settings.sandboxSnapshotIntervalMs
708
+ });
709
+ return {
710
+ targets: rows.map(({ accountId: _accountId, ...target }) => target),
711
+ timeoutClass
712
+ };
713
+ } finally {
714
+ stopHeartbeat();
715
+ }
716
+ }
717
+ async function checkpointIdleSandboxLease(input) {
718
+ const target = input.target;
719
+ const stopHeartbeat = startSandboxReaperHeartbeat({
720
+ phase: "idle_checkpoint",
721
+ workspaceId: target.workspaceId,
722
+ sandboxGroupId: target.sandboxGroupId,
723
+ leaseEpoch: target.leaseEpoch,
724
+ instanceId: target.instanceId
725
+ });
726
+ try {
727
+ const { db, settings, observability, objectStorage } = await services();
728
+ const fields = {
729
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
730
+ leaseEpoch: target.leaseEpoch
731
+ };
732
+ const skipped = (reason) => {
733
+ observability.info("sandbox reaper: idle checkpoint skipped", { ...fields, reason });
734
+ return { status: "skipped", reason };
735
+ };
736
+ if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
737
+ return skipped("disabled");
738
+ }
739
+ const lease = await readLease2(db, target.workspaceId, target.sandboxGroupId);
740
+ if (!lease || lease.liveness !== "warm" || lease.leaseEpoch !== target.leaseEpoch || lease.instanceId !== target.instanceId) {
741
+ return skipped("lease_fenced");
742
+ }
743
+ if (lease.turnHolders > 0) return skipped("turn_held");
744
+ if (lease.refcount === 0) return skipped("drain_owned");
745
+ const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);
746
+ const stamped = await recordIdleCheckpointAttempt(db, {
747
+ accountId,
748
+ workspaceId: target.workspaceId,
749
+ sandboxGroupId: target.sandboxGroupId,
750
+ expectedEpoch: target.leaseEpoch,
751
+ expectedInstanceId: target.instanceId
752
+ });
753
+ if (!stamped) return skipped("lease_fenced");
754
+ if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {
755
+ return skipped("capture_policy");
756
+ }
757
+ if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped("clean");
758
+ if (lease.archiveCapture) {
759
+ const tookOver = await releaseWorkspaceArchiveCapture(db, {
760
+ accountId,
761
+ workspaceId: target.workspaceId,
762
+ sandboxGroupId: target.sandboxGroupId,
763
+ captureId: lease.archiveCapture.id,
764
+ expectedEpoch: target.leaseEpoch,
765
+ expectedInstanceId: target.instanceId,
766
+ onlyExpiredWarmTakeover: true
767
+ });
768
+ if (!tookOver) return skipped("capture_in_progress");
769
+ observability.warn("sandbox reaper: released an expired warm capture claim", {
770
+ ...fields,
771
+ captureId: lease.archiveCapture.id
772
+ });
773
+ }
774
+ let session;
775
+ try {
776
+ session = await resumeIdleCheckpointSession(settings, lease);
777
+ } catch (error) {
778
+ observability.warn("sandbox reaper: idle checkpoint resume failed", {
779
+ ...fields,
780
+ error: error instanceof Error ? error.message : String(error)
781
+ });
782
+ return skipped("provider_error");
783
+ }
784
+ if (session === null) return skipped("provider_missing");
785
+ const capture = maybePersistWarmWorkspaceSnapshot(
786
+ { db, settings, objectStorage, observability },
787
+ {
788
+ accountId,
789
+ workspaceId: target.workspaceId,
790
+ sandboxGroupId: target.sandboxGroupId,
791
+ idleCheckpoint: true
792
+ },
793
+ session,
794
+ target.leaseEpoch
795
+ );
796
+ const persisted = await capture;
797
+ await capture.settled;
798
+ if (!persisted) return skipped("not_captured");
799
+ observability.info("sandbox reaper: idle checkpoint published", fields);
800
+ return { status: "checkpointed" };
801
+ } finally {
802
+ stopHeartbeat();
803
+ }
804
+ }
805
+ async function drainSandboxLease(input) {
806
+ const stopHeartbeat = startSandboxReaperHeartbeat({
807
+ phase: "drain",
808
+ workspaceId: input.target.workspaceId,
809
+ sandboxGroupId: input.target.sandboxGroupId,
810
+ leaseEpoch: input.target.leaseEpoch,
811
+ instanceId: input.target.instanceId
812
+ });
813
+ try {
814
+ const { db, settings, observability, objectStorage, bus } = await services();
815
+ assertSandboxDrainInputTiming(input);
816
+ const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
817
+ const row = input.target;
818
+ const activityAttempt = sandboxDrainActivityAttempt();
819
+ const captureAttempt = {
820
+ operationId: input.operationId,
821
+ captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),
822
+ attempt: activityAttempt,
823
+ captureTimeoutMs: input.captureTimeoutMs
824
+ };
825
+ try {
826
+ const drainedCold = await terminateDrainableBox(
827
+ db,
828
+ drainSettings,
829
+ row,
830
+ observability,
831
+ terminateBox,
832
+ probeDrainableProvider,
833
+ captureAttempt,
834
+ objectStorage,
835
+ bus
836
+ );
837
+ return { status: drainedCold ? "terminated" : "skipped" };
838
+ } catch (error) {
839
+ const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
840
+ const details = {
841
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
842
+ workspaceId: row.workspaceId,
843
+ sandboxGroupId: row.sandboxGroupId,
844
+ leaseEpoch: row.leaseEpoch,
845
+ backend: lease?.backend ?? "unknown",
846
+ instanceId: lease?.instanceId ?? row.instanceId,
847
+ captureId: lease?.archiveCapture?.id ?? null,
848
+ captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,
849
+ captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,
850
+ timeoutClass: input.timeoutClass,
851
+ error: error instanceof Error ? error.message : String(error)
852
+ };
853
+ if (error instanceof SandboxDrainRecoveryDeferredError) {
854
+ observability.info("sandbox reaper: per-sandbox recovery deferred safely", details);
855
+ } else {
856
+ observability.warn("sandbox reaper: per-sandbox drain failed", details);
857
+ }
858
+ throw error;
859
+ }
860
+ } finally {
861
+ stopHeartbeat();
862
+ }
863
+ }
864
+ async function maintainSandboxLeaseSweep(input) {
865
+ const stopHeartbeat = startSandboxReaperHeartbeat({
866
+ phase: "maintain",
867
+ examined: input.examined,
868
+ started: input.started
869
+ });
870
+ try {
871
+ const service = await services();
872
+ const { db, settings, observability } = service;
873
+ const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(
874
+ (error) => {
875
+ observability.warn("system-update outbox reconciliation failed", {
876
+ error: error instanceof Error ? error.message : String(error)
877
+ });
878
+ return { claimed: 0, delivered: 0, failed: 1 };
879
+ }
880
+ );
881
+ const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
882
+ (error) => {
883
+ observability.warn("session-list snapshot reconciliation failed", {
884
+ error: error instanceof Error ? error.message : String(error)
885
+ });
886
+ return 0;
887
+ }
888
+ );
889
+ if (expiredSessionListSnapshots > 0) {
890
+ observability.info("expired session-list snapshots reaped", {
891
+ deleted: expiredSessionListSnapshots
892
+ });
893
+ }
894
+ if (parentUpdates.claimed > 0) {
895
+ observability.info("system-update outbox reconciled", parentUpdates);
896
+ }
897
+ let metered = 0;
898
+ let forceDrained = 0;
899
+ const rotationsRequested = input.rotationsRequested;
900
+ let modalOrphansTerminated = 0;
901
+ await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
902
+ await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
903
+ try {
904
+ await reconcileModalProviderCreates(
905
+ db,
906
+ settings,
907
+ observability,
908
+ options.findModalProviderCreateReceipt
909
+ );
910
+ } catch (error) {
911
+ observability.warn("sandbox reaper: Modal create recovery inventory failed", {
912
+ error: error instanceof Error ? error.message : String(error)
913
+ });
914
+ }
915
+ if (!settings.sandboxOwnershipEnabled) {
916
+ await refreshQueueLeaseAndCreditGauges(
917
+ db,
918
+ settings,
919
+ observability,
920
+ inspectOpenSandboxInventory
921
+ );
922
+ return {
923
+ metered,
924
+ forceDrained,
925
+ rotationsRequested,
926
+ modalOrphansTerminated
927
+ };
928
+ }
929
+ const meterResult = await accrueWarmTick(db, settings, observability);
930
+ metered = meterResult.accrued;
931
+ const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
932
+ if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
933
+ for (const workspaceId of meterResult.workspaceIds) {
934
+ forceDrainWorkspaceIds.add(workspaceId);
935
+ }
936
+ }
937
+ try {
938
+ for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
939
+ forceDrainWorkspaceIds.add(workspaceId);
940
+ }
941
+ } catch (error) {
942
+ observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
943
+ error: error instanceof Error ? error.message : String(error)
944
+ });
945
+ }
946
+ forceDrained = await forceDrainOverLimitWorkspaces(
947
+ db,
948
+ settings,
949
+ forceDrainWorkspaceIds,
950
+ observability
951
+ );
952
+ await reconcileTerminalRetainedProcesses(
953
+ db,
954
+ settings,
955
+ observability,
956
+ probeRetainedProcess,
957
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
958
+ service.bus
959
+ );
960
+ await settleEndedEpochBlockers(
961
+ db,
962
+ settings,
963
+ observability,
964
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
965
+ endedEpochProbeBackoff
966
+ );
967
+ try {
968
+ modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
969
+ recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
970
+ } catch (error) {
971
+ observability.warn("sandbox reaper: Modal orphan sweep failed", {
972
+ error: error instanceof Error ? error.message : String(error)
973
+ });
974
+ }
975
+ const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
976
+ (error) => {
977
+ observability.warn("sandbox reaper: checkpoint artifact GC failed", {
978
+ error: error instanceof Error ? error.message : String(error)
979
+ });
980
+ return { claimed: 0, deleted: 0, failed: 0 };
981
+ }
982
+ );
983
+ await refreshQueueLeaseAndCreditGauges(
984
+ db,
985
+ settings,
986
+ observability,
987
+ inspectOpenSandboxInventory
988
+ );
989
+ if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
990
+ observability.info("sandbox reaper dispatched", {
991
+ drainable: input.examined,
992
+ childrenStarted: input.started,
993
+ childrenAlreadyRunning: input.alreadyRunning,
994
+ childrenStartFailed: input.startFailed,
995
+ metered,
996
+ forceDrained,
997
+ modalOrphansTerminated,
998
+ rotationsRequested,
999
+ checkpointArtifactsClaimed: checkpointGc.claimed,
1000
+ checkpointArtifactsDeleted: checkpointGc.deleted,
1001
+ checkpointArtifactsFailed: checkpointGc.failed
1002
+ });
1003
+ }
1004
+ return {
1005
+ metered,
1006
+ forceDrained,
1007
+ rotationsRequested,
1008
+ modalOrphansTerminated
1009
+ };
1010
+ } finally {
1011
+ stopHeartbeat();
1012
+ }
1013
+ }
1014
+ async function reapSandboxLeases() {
1015
+ const service = await services();
1016
+ if (service.startSandboxReaperWorkflow) {
1017
+ const dispatch = await service.startSandboxReaperWorkflow();
1018
+ service.observability.info("sandbox reaper routed to versioned workflow", {
1019
+ dispatch
1020
+ });
1021
+ return {
1022
+ examined: 0,
1023
+ terminated: 0,
1024
+ skipped: 0,
1025
+ metered: 0,
1026
+ forceDrained: 0,
1027
+ modalOrphansTerminated: 0
1028
+ };
1029
+ }
1030
+ const plan = await prepareSandboxLeaseSweep();
1031
+ const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);
1032
+ const outcomes = await Promise.allSettled(
1033
+ admitted.map(
1034
+ (target) => drainSandboxLease({
1035
+ target,
1036
+ timeoutClass: plan.timeoutClass,
1037
+ snapshotTimeoutMs: plan.snapshotTimeoutMs,
1038
+ captureTimeoutMs: plan.captureTimeoutMs,
1039
+ operationId: randomUUID()
1040
+ })
1041
+ )
1042
+ );
1043
+ try {
1044
+ const { targets, timeoutClass } = await listIdleSandboxCheckpoints();
1045
+ for (const target of targets) {
1046
+ const key = sandboxIdleCheckpointWorkflowId(target);
1047
+ if (embeddedIdleCheckpoints.has(key)) continue;
1048
+ embeddedIdleCheckpoints.add(key);
1049
+ void checkpointIdleSandboxLease({ target, timeoutClass }).catch((error) => {
1050
+ service.observability.warn("sandbox reaper: idle checkpoint failed", {
1051
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
1052
+ error: error instanceof Error ? error.message : String(error)
1053
+ });
1054
+ }).finally(() => embeddedIdleCheckpoints.delete(key));
1055
+ }
1056
+ } catch (error) {
1057
+ service.observability.warn("sandbox reaper: idle checkpoint inventory failed", {
1058
+ error: error instanceof Error ? error.message : String(error)
1059
+ });
1060
+ }
1061
+ const terminated = outcomes.filter(
1062
+ (outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated"
1063
+ ).length;
1064
+ const skipped = outcomes.length - terminated;
1065
+ const maintenance = await maintainSandboxLeaseSweep({
1066
+ examined: admitted.length,
1067
+ started: admitted.length,
1068
+ alreadyRunning: 0,
1069
+ startFailed: 0,
1070
+ rotationsRequested: plan.rotationsRequested
1071
+ });
1072
+ return {
1073
+ examined: admitted.length,
1074
+ terminated,
1075
+ skipped,
1076
+ metered: maintenance.metered,
1077
+ forceDrained: maintenance.forceDrained,
1078
+ modalOrphansTerminated: maintenance.modalOrphansTerminated
1079
+ };
1080
+ }
1081
+ return {
1082
+ prepareSandboxLeaseSweep,
1083
+ drainSandboxLease,
1084
+ listIdleSandboxCheckpoints,
1085
+ checkpointIdleSandboxLease,
1086
+ maintainSandboxLeaseSweep,
1087
+ reapSandboxLeases
1088
+ };
1089
+ }
1090
+ async function resumeIdleCheckpointSessionAtProvider(settings, lease) {
1091
+ const durableBackendId = lease.resumeBackendId ?? lease.backend;
1092
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
1093
+ if (backend !== "modal" || !lease.instanceId || !lease.resumeState) {
1094
+ throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);
1095
+ }
1096
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {
1097
+ throw new Error("idle checkpoint lease envelope does not name its provider instance");
1098
+ }
1099
+ const client = createSandboxClientForBackend2(backend, settings);
1100
+ if (!client?.resume) {
1101
+ throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);
1102
+ }
1103
+ try {
1104
+ const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
1105
+ const resumedState = await deserializeSandboxSessionStateEnvelope2(
1106
+ client,
1107
+ envelopeSessionState,
1108
+ lease.instanceId
1109
+ );
1110
+ if (resumedState === void 0) {
1111
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
1112
+ }
1113
+ const resumed = await withRetainedProcessProbeTimeout(
1114
+ resumeExactSandboxSession2(client, backend, resumedState, lease.instanceId)
1115
+ );
1116
+ return resumed.session;
1117
+ } catch (error) {
1118
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
1119
+ throw new Error("idle checkpoint provider resume timed out", { cause: error });
1120
+ }
1121
+ if (isProviderSandboxNotFoundError(client.backendId, error)) return null;
1122
+ throw error;
1123
+ }
1124
+ }
1125
+ var CHECKPOINT_GC_LIMIT = 50;
1126
+ var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
1127
+ var SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;
1128
+ var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
1129
+ var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
1130
+ var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
1131
+ function recordConnectedCommandReconciliation(observability, outcome) {
1132
+ observability.incrementCounter({
1133
+ name: "opengeni_connected_background_command_reconciliation_total",
1134
+ help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
1135
+ labels: { outcome }
1136
+ });
1137
+ }
1138
+ var CONNECTED_RUNNER_STOPPED_REASONS = /* @__PURE__ */ new Set([
1139
+ "GOING_OFFLINE_REASON_USER_STOP",
1140
+ "GOING_OFFLINE_REASON_UPDATE",
1141
+ "GOING_OFFLINE_REASON_HOST_SHUTDOWN"
1142
+ ]);
1143
+ async function connectedCommandRunnerRetirement(db, claim, error, lookups) {
1144
+ if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {
1145
+ return null;
1146
+ }
1147
+ const key = JSON.stringify([
1148
+ claim.controlWorkspaceId,
1149
+ claim.enrollmentId,
1150
+ claim.connectionInstanceId
1151
+ ]);
1152
+ let lookup = lookups.get(key);
1153
+ if (!lookup) {
1154
+ lookup = (async () => {
1155
+ const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);
1156
+ if (!enrollment) return null;
1157
+ if (enrollment.status === "revoked") return "enrollment_revoked";
1158
+ if (enrollment.connectionInstanceId) {
1159
+ return enrollment.connectionInstanceId === claim.connectionInstanceId ? null : "connection_replaced";
1160
+ }
1161
+ return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? "") ? "connection_stopped" : null;
1162
+ })().catch(() => null);
1163
+ lookups.set(key, lookup);
1164
+ }
1165
+ return await lookup;
1166
+ }
1167
+ async function forEachWithConcurrency(values, concurrency, visit) {
1168
+ if (values.length === 0) return;
1169
+ const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
1170
+ let nextIndex = 0;
1171
+ await Promise.all(
1172
+ Array.from({ length: workerCount }, async () => {
1173
+ while (nextIndex < values.length) {
1174
+ const index = nextIndex;
1175
+ nextIndex += 1;
1176
+ await visit(values[index], index);
1177
+ }
1178
+ })
1179
+ );
1180
+ }
1181
+ async function gcSandboxCheckpointArtifacts(db, settings, observability) {
1182
+ const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(
1183
+ (error) => {
1184
+ observability.warn("sandbox reaper: legacy Modal checkpoint discovery failed", {
1185
+ error: error instanceof Error ? error.message : String(error)
1186
+ });
1187
+ return 0;
1188
+ }
1189
+ );
1190
+ if (adopted > 0) {
1191
+ recordSandboxCheckpointArtifactOutcome(observability, "legacy_adopted", adopted);
1192
+ observability.info("sandbox reaper: adopted legacy Modal checkpoint receipts", { adopted });
1193
+ }
1194
+ const claimId = crypto.randomUUID();
1195
+ const claims = await claimSandboxCheckpointArtifactsForGc(db, {
1196
+ claimId,
1197
+ limit: CHECKPOINT_GC_LIMIT,
1198
+ claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS
1199
+ });
1200
+ recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
1201
+ let deleted = 0;
1202
+ let failed = 0;
1203
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1204
+ try {
1205
+ if (claim.providerBackend !== "modal") {
1206
+ throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
1207
+ }
1208
+ await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);
1209
+ const settled = await settleSandboxCheckpointArtifactGc(db, {
1210
+ artifactId: claim.id,
1211
+ claimId,
1212
+ deleted: true,
1213
+ retryAfterMs: 1e3
1214
+ });
1215
+ if (settled) {
1216
+ deleted += 1;
1217
+ recordSandboxCheckpointArtifactOutcome(observability, "deleted");
1218
+ }
1219
+ } catch (error) {
1220
+ failed += 1;
1221
+ recordSandboxCheckpointArtifactOutcome(observability, "delete_failed");
1222
+ const retryAfterMs = Math.min(
1223
+ 6 * 60 * 6e4,
1224
+ 5e3 * 2 ** Math.min(claim.deleteAttempts, 17)
1225
+ );
1226
+ await settleSandboxCheckpointArtifactGc(db, {
1227
+ artifactId: claim.id,
1228
+ claimId,
1229
+ deleted: false,
1230
+ error: error instanceof Error ? error.message : String(error),
1231
+ retryAfterMs
1232
+ }).catch(() => false);
1233
+ observability.warn("sandbox reaper: checkpoint artifact delete failed", {
1234
+ artifactId: claim.id,
1235
+ providerBackend: claim.providerBackend,
1236
+ objectKind: claim.objectKind,
1237
+ attempt: claim.deleteAttempts,
1238
+ error: error instanceof Error ? error.message : String(error)
1239
+ });
1240
+ }
1241
+ });
1242
+ const pruned = await pruneDeletedSandboxCheckpointArtifacts(
1243
+ db,
1244
+ CHECKPOINT_TOMBSTONE_RETENTION_MS,
1245
+ CHECKPOINT_TOMBSTONE_PRUNE_LIMIT
1246
+ );
1247
+ recordSandboxCheckpointArtifactOutcome(observability, "tombstone_pruned", pruned);
1248
+ return { claimed: claims.length, deleted, failed };
1249
+ }
1250
+ async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
1251
+ const slots = await listLegacyModalCheckpointSlots(db, 25);
1252
+ if (slots.length === 0) return 0;
1253
+ const bindings = /* @__PURE__ */ new Map();
1254
+ let adopted = 0;
1255
+ await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {
1256
+ try {
1257
+ let binding = bindings.get(slot.instanceId);
1258
+ if (!binding) {
1259
+ binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);
1260
+ bindings.set(slot.instanceId, binding);
1261
+ }
1262
+ const resolvedBinding = await binding;
1263
+ if (await adoptLegacyModalCheckpointArtifact(db, {
1264
+ accountId: slot.accountId,
1265
+ workspaceId: slot.workspaceId,
1266
+ sandboxGroupId: slot.sandboxGroupId,
1267
+ leaseId: slot.leaseId,
1268
+ leaseEpoch: slot.leaseEpoch,
1269
+ workspaceGeneration: slot.workspaceGeneration,
1270
+ slot: slot.slot,
1271
+ archiveBase64: slot.archiveBase64,
1272
+ descriptor: slot.descriptor,
1273
+ providerBindingKey: resolvedBinding.key,
1274
+ providerBinding: resolvedBinding.binding
1275
+ })) {
1276
+ adopted += 1;
1277
+ }
1278
+ } catch (error) {
1279
+ observability.warn("sandbox reaper: legacy Modal checkpoint adoption failed", {
1280
+ workspaceId: slot.workspaceId,
1281
+ sandboxGroupId: slot.sandboxGroupId,
1282
+ slot: slot.slot,
1283
+ error: error instanceof Error ? error.message : String(error)
1284
+ });
1285
+ }
1286
+ });
1287
+ return adopted;
1288
+ }
1289
+ async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
1290
+ const dueBefore = /* @__PURE__ */ new Date();
1291
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
1292
+ const offlineConnections = /* @__PURE__ */ new Set();
1293
+ const firstConnectionProbes = /* @__PURE__ */ new Map();
1294
+ do {
1295
+ const claimId = crypto.randomUUID();
1296
+ let claims;
1297
+ try {
1298
+ claims = await claimConnectedMachineSessionBackgroundCommands(db, {
1299
+ claimId,
1300
+ dueBefore,
1301
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1302
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
1303
+ });
1304
+ } catch (error) {
1305
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1306
+ observability.warn("sandbox reaper: connected-command claim failed", {
1307
+ error: error instanceof Error ? error.message : String(error)
1308
+ });
1309
+ return;
1310
+ }
1311
+ const controlRpc = controlRpcOverride ?? new NatsControlRpc(
1312
+ async () => bus.getRequestConnection()
1313
+ );
1314
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1315
+ let proof = claim.proof;
1316
+ let outputReplay = null;
1317
+ const connectionKey = JSON.stringify([
1318
+ claim.controlWorkspaceId,
1319
+ claim.enrollmentId,
1320
+ claim.connectionInstanceId
1321
+ ]);
1322
+ if (!proof) {
1323
+ const firstProbe = firstConnectionProbes.get(connectionKey);
1324
+ if (firstProbe) await firstProbe;
1325
+ if (offlineConnections.has(connectionKey)) {
1326
+ await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
1327
+ return;
1328
+ }
1329
+ let finishFirstProbe;
1330
+ if (!firstProbe) {
1331
+ firstConnectionProbes.set(
1332
+ connectionKey,
1333
+ new Promise((resolve) => {
1334
+ finishFirstProbe = resolve;
1335
+ })
1336
+ );
1337
+ }
1338
+ try {
1339
+ proof = await probeConnectedMachineBackgroundCommand(
1340
+ claim,
1341
+ controlRpc,
1342
+ Math.min(
1343
+ settings.sandboxSelfhostedControlTimeoutMs,
1344
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
1345
+ )
1346
+ );
1347
+ if (proof?.outcome !== "lost") {
1348
+ if (!bus.getOpStreamConnection)
1349
+ throw new Error("Connected command output transport unavailable");
1350
+ const client = new OpStreamExecClient({
1351
+ workspaceId: claim.controlWorkspaceId,
1352
+ agentId: claim.enrollmentId,
1353
+ connectionInstanceId: claim.connectionInstanceId,
1354
+ epoch: 0,
1355
+ controlRpc,
1356
+ rpcSubject: subjectFor(
1357
+ claim.controlWorkspaceId,
1358
+ claim.enrollmentId,
1359
+ claim.connectionInstanceId
1360
+ ),
1361
+ transport: new NatsOpStreamTransport(
1362
+ async () => bus.getOpStreamConnection?.() ?? null
1363
+ ),
1364
+ controlTimeoutMs: Math.min(
1365
+ settings.sandboxSelfhostedControlTimeoutMs,
1366
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
1367
+ ),
1368
+ retryClock: defaultSelfhostedRetryClock,
1369
+ journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
1370
+ } }
1371
+ });
1372
+ outputReplay = await replayConnectedCommandOutput(
1373
+ client,
1374
+ claim.opId,
1375
+ Boolean(proof),
1376
+ async (frames) => {
1377
+ await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
1378
+ }
1379
+ );
1380
+ }
1381
+ if (!proof) {
1382
+ await deferConnectedCommandClaim(
1383
+ db,
1384
+ settings,
1385
+ observability,
1386
+ claim,
1387
+ "provider_running"
1388
+ );
1389
+ return;
1390
+ }
1391
+ } catch (error) {
1392
+ const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
1393
+ if (offline) offlineConnections.add(connectionKey);
1394
+ if (!offline) {
1395
+ observability.warn("sandbox reaper: connected-command provider probe failed", {
1396
+ commandId: claim.commandId,
1397
+ enrollmentId: claim.enrollmentId,
1398
+ connectionInstanceId: claim.connectionInstanceId,
1399
+ error: error instanceof Error ? error.message : String(error)
1400
+ });
1401
+ }
1402
+ await deferConnectedCommandClaim(
1403
+ db,
1404
+ settings,
1405
+ observability,
1406
+ claim,
1407
+ offline ? "provider_offline" : "provider_error"
1408
+ );
1409
+ return;
1410
+ } finally {
1411
+ finishFirstProbe?.();
1412
+ }
1413
+ try {
1414
+ await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
1415
+ recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
1416
+ } catch (error) {
1417
+ recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
1418
+ observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
1419
+ commandId: claim.commandId,
1420
+ error: error instanceof Error ? error.message : String(error)
1421
+ });
1422
+ return;
1423
+ }
1424
+ }
1425
+ try {
1426
+ const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
1427
+ if (!settlement.settled) {
1428
+ throw new Error("Connected command settlement lost its exact claim or proof");
1429
+ }
1430
+ const receipt = outputReplay?.outputReceipt ?? (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : void 0);
1431
+ const terminal = outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
1432
+ if (proof.outcome === "exited" && receipt && terminal) {
1433
+ await recordConnectedCommandOutputConsumption(db, {
1434
+ ...claim,
1435
+ receipt,
1436
+ exitCode: terminal.outcome.response.exitCode
1437
+ });
1438
+ }
1439
+ if (settlement.events.length > 0) {
1440
+ try {
1441
+ await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
1442
+ } catch (error) {
1443
+ observability.warn("sandbox reaper: connected-command event fanout failed", {
1444
+ commandId: claim.commandId,
1445
+ error: error instanceof Error ? error.message : String(error)
1446
+ });
1447
+ }
1448
+ }
1449
+ recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
1450
+ } catch (error) {
1451
+ recordConnectedCommandReconciliation(observability, "settlement_failed");
1452
+ observability.warn("sandbox reaper: connected-command settlement failed", {
1453
+ commandId: claim.commandId,
1454
+ error: error instanceof Error ? error.message : String(error)
1455
+ });
1456
+ await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
1457
+ }
1458
+ });
1459
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
1460
+ } while (Date.now() < deadline);
1461
+ }
1462
+ async function replayConnectedCommandOutput(client, opId, terminalKnown, capture) {
1463
+ let capturedThrough = 0n;
1464
+ while (true) {
1465
+ const before = capturedThrough;
1466
+ const replay = await client.readExisting(opId, terminalKnown ? 5e3 : 250, capture);
1467
+ if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
1468
+ capturedThrough = BigInt(replay.replaySequence);
1469
+ if (capturedThrough === before) {
1470
+ throw new Error("Connected command terminal output replay has not reached its exit frontier");
1471
+ }
1472
+ }
1473
+ }
1474
+ async function reconcileConnectedCommandOutputReleases(db, settings, observability, bus, controlRpcOverride) {
1475
+ const dueBefore = /* @__PURE__ */ new Date();
1476
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
1477
+ const controlRpc = controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
1478
+ const runnerLookups = /* @__PURE__ */ new Map();
1479
+ do {
1480
+ let claims;
1481
+ try {
1482
+ claims = await claimConnectedCommandOutputReleases(db, {
1483
+ claimId: crypto.randomUUID(),
1484
+ dueBefore,
1485
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1486
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
1487
+ });
1488
+ } catch (error) {
1489
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1490
+ observability.warn("sandbox reaper: connected-command output claim failed", {
1491
+ error: error instanceof Error ? error.message : String(error)
1492
+ });
1493
+ return;
1494
+ }
1495
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1496
+ let receipt = claim.receipt;
1497
+ try {
1498
+ if (!bus.getOpStreamConnection)
1499
+ throw new Error("Connected command output transport unavailable");
1500
+ const rpcSubject = subjectFor(
1501
+ claim.controlWorkspaceId,
1502
+ claim.enrollmentId,
1503
+ claim.connectionInstanceId
1504
+ );
1505
+ const controlTimeoutMs = Math.min(
1506
+ settings.sandboxSelfhostedControlTimeoutMs,
1507
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
1508
+ );
1509
+ const client = new OpStreamExecClient({
1510
+ workspaceId: claim.controlWorkspaceId,
1511
+ agentId: claim.enrollmentId,
1512
+ connectionInstanceId: claim.connectionInstanceId,
1513
+ epoch: 0,
1514
+ controlRpc,
1515
+ rpcSubject,
1516
+ transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
1517
+ controlTimeoutMs,
1518
+ retryClock: defaultSelfhostedRetryClock,
1519
+ journal: {
1520
+ attachGeneration: () => String(Date.now()),
1521
+ persistSettled: () => {
1522
+ throw new Error("Background output release requires its database receipt");
1523
+ }
1524
+ }
1525
+ });
1526
+ if (!receipt) {
1527
+ const status = await querySelfhostedOp({
1528
+ controlRpc,
1529
+ rpcSubject,
1530
+ opId: claim.opId,
1531
+ controlTimeoutMs
1532
+ });
1533
+ if (status.state === OpState.OP_STATE_LOST) {
1534
+ if (!await settleConnectedCommandOutputReleaseClaim(db, {
1535
+ claim,
1536
+ outcome: "unavailable",
1537
+ retryAfterMs: 0
1538
+ }))
1539
+ throw new Error("Connected command output loss no longer owns its exact claim");
1540
+ recordConnectedCommandReconciliation(observability, "output_unavailable");
1541
+ return;
1542
+ }
1543
+ if (status.state !== OpState.OP_STATE_COMPLETE)
1544
+ throw new Error("Terminal command output returned a live operation");
1545
+ const replay = await replayConnectedCommandOutput(
1546
+ client,
1547
+ claim.opId,
1548
+ true,
1549
+ async (frames) => captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames)
1550
+ );
1551
+ const terminal = replay.status === "completed" ? replay : replay.terminal;
1552
+ receipt = replay.outputReceipt ?? (replay.status === "running" ? replay.terminal?.outputReceipt : void 0) ?? null;
1553
+ if (!terminal || !receipt)
1554
+ throw new Error("Connected command output replay has no complete custody receipt");
1555
+ await recordConnectedCommandOutputConsumption(db, {
1556
+ ...claim,
1557
+ receipt,
1558
+ exitCode: terminal.outcome.response.exitCode
1559
+ });
1560
+ }
1561
+ const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
1562
+ if (!await settleConnectedCommandOutputReleaseClaim(db, {
1563
+ claim,
1564
+ outcome,
1565
+ retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0
1566
+ }))
1567
+ throw new Error("Connected command output release no longer owns its exact claim");
1568
+ recordConnectedCommandReconciliation(
1569
+ observability,
1570
+ outcome === "published" ? "output_ack_published" : "output_released"
1571
+ );
1572
+ } catch (error) {
1573
+ const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);
1574
+ if (retired && await settleConnectedCommandOutputReleaseClaim(db, {
1575
+ claim,
1576
+ outcome: receipt ? "not_retained" : "unavailable",
1577
+ retryAfterMs: 0
1578
+ }).catch(() => false)) {
1579
+ recordConnectedCommandReconciliation(
1580
+ observability,
1581
+ receipt ? "output_released" : "output_unavailable"
1582
+ );
1583
+ return;
1584
+ }
1585
+ await settleConnectedCommandOutputReleaseClaim(db, {
1586
+ claim,
1587
+ outcome: "retry",
1588
+ retryAfterMs: Math.min(
1589
+ 5 * 6e4,
1590
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** Math.min(4, claim.reconcileAttempts - 1)
1591
+ )
1592
+ }).catch(() => false);
1593
+ observability.warn("sandbox reaper: connected-command output release deferred", {
1594
+ commandId: claim.commandId,
1595
+ error: error instanceof Error ? error.message : String(error)
1596
+ });
1597
+ }
1598
+ });
1599
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
1600
+ } while (Date.now() < deadline);
1601
+ }
1602
+ async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
1603
+ const input = {
1604
+ controlRpc,
1605
+ rpcSubject: subjectFor(
1606
+ claim.controlWorkspaceId,
1607
+ claim.enrollmentId,
1608
+ claim.connectionInstanceId
1609
+ ),
1610
+ opId: claim.opId,
1611
+ controlTimeoutMs
1612
+ };
1613
+ const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
1614
+ return connectedCommandProofFromStatus(status, observedAt);
1615
+ }
1616
+ function connectedCommandProofFromStatus(status, observedAt) {
1617
+ switch (status.state) {
1618
+ case OpState.OP_STATE_ACCEPTED:
1619
+ case OpState.OP_STATE_RUNNING:
1620
+ return null;
1621
+ case OpState.OP_STATE_COMPLETE: {
1622
+ const exit = status.exit;
1623
+ if (!exit) {
1624
+ throw new Error("Connected command completed without an exit record");
1625
+ }
1626
+ const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
1627
+ const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
1628
+ return {
1629
+ outcome: "exited",
1630
+ exitCode: exit.exitCode,
1631
+ reason,
1632
+ ...failureCode ? {
1633
+ failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
1634
+ } : {},
1635
+ observedAt
1636
+ };
1637
+ }
1638
+ case OpState.OP_STATE_LOST:
1639
+ return {
1640
+ outcome: "lost",
1641
+ exitCode: null,
1642
+ reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
1643
+ observedAt
1644
+ };
1645
+ default:
1646
+ throw new Error(`Connected command returned invalid op state ${status.state}`);
1647
+ }
1648
+ }
1649
+ async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
1650
+ const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
1651
+ const retryAfterMs = Math.min(
1652
+ 5 * 6e4,
1653
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
1654
+ );
1655
+ try {
1656
+ if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
1657
+ claim,
1658
+ outcome,
1659
+ retryAfterMs
1660
+ })) {
1661
+ throw new Error("Connected command defer lost its exact claim");
1662
+ }
1663
+ recordConnectedCommandReconciliation(observability, outcome);
1664
+ } catch (error) {
1665
+ recordConnectedCommandReconciliation(observability, "defer_failed");
1666
+ observability.warn("sandbox reaper: connected-command defer failed", {
1667
+ commandId: claim.commandId,
1668
+ error: error instanceof Error ? error.message : String(error)
1669
+ });
1670
+ }
1671
+ }
1672
+ var ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;
1673
+ var ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 6e4;
1674
+ var ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 6e4;
1675
+ var ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 6e4;
1676
+ async function settleEndedEpochBlockers(db, settings, observability, inspectHistoricalModalSandbox, backoff) {
1677
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
1678
+ return;
1679
+ }
1680
+ const startedAtMs = Date.now();
1681
+ const observed = /* @__PURE__ */ new Map();
1682
+ const kept = {};
1683
+ async function observe(instanceId, details) {
1684
+ const known = observed.get(instanceId);
1685
+ if (known !== void 0) return known;
1686
+ const nowMs = Date.now();
1687
+ if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;
1688
+ if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;
1689
+ let status;
1690
+ try {
1691
+ status = (await withRetainedProcessProbeTimeout(
1692
+ inspectHistoricalModalSandbox(settings, instanceId, null)
1693
+ )).status;
1694
+ } catch (error) {
1695
+ status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "error";
1696
+ observability.warn("sandbox reaper: ended-epoch box probe failed", {
1697
+ ...details,
1698
+ error: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : error instanceof Error ? error.message : String(error)
1699
+ });
1700
+ }
1701
+ observed.set(instanceId, status);
1702
+ if (status === "terminated") {
1703
+ backoff.delete(instanceId);
1704
+ } else {
1705
+ deferEndedEpochProbe(backoff, instanceId);
1706
+ kept[status] = (kept[status] ?? 0) + 1;
1707
+ }
1708
+ return status;
1709
+ }
1710
+ let tuples;
1711
+ try {
1712
+ tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });
1713
+ } catch (error) {
1714
+ observability.warn("sandbox reaper: ended-epoch blocker inventory failed", {
1715
+ error: error instanceof Error ? error.message : String(error)
1716
+ });
1717
+ tuples = [];
1718
+ }
1719
+ for (const tuple of tuples) {
1720
+ const instanceId = tuple.lostInstanceId;
1721
+ const details = {
1722
+ workspaceId: tuple.workspaceId,
1723
+ sandboxGroupId: tuple.sandboxGroupId,
1724
+ lostEpoch: tuple.lostEpoch,
1725
+ instanceId
1726
+ };
1727
+ try {
1728
+ const status = await observe(instanceId, details);
1729
+ if (status !== "terminated") continue;
1730
+ const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);
1731
+ if (settled) {
1732
+ observability.info("sandbox reaper: settled blockers of an ended lease epoch", {
1733
+ ...details,
1734
+ ...settled
1735
+ });
1736
+ }
1737
+ } catch (error) {
1738
+ observability.warn("sandbox reaper: ended-epoch blocker settlement failed", {
1739
+ ...details,
1740
+ error: error instanceof Error ? error.message : String(error)
1741
+ });
1742
+ }
1743
+ }
1744
+ let setups;
1745
+ try {
1746
+ setups = await listEndedEpochSandboxSetupUncertainties(db, {
1747
+ limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT
1748
+ });
1749
+ } catch (error) {
1750
+ observability.warn("sandbox reaper: ended-epoch setup inventory failed", {
1751
+ error: error instanceof Error ? error.message : String(error)
1752
+ });
1753
+ setups = [];
1754
+ }
1755
+ for (const setup of setups) {
1756
+ const instanceId = setup.box.providerInstanceId;
1757
+ const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;
1758
+ if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;
1759
+ const details = {
1760
+ workspaceId: setup.workspaceId,
1761
+ sessionId: setup.sessionId,
1762
+ turnId: setup.turnId,
1763
+ sandboxGroupId: setup.sandboxGroupId,
1764
+ lostEpoch: setup.box.leaseEpoch,
1765
+ instanceId
1766
+ };
1767
+ try {
1768
+ const status = await observe(instanceId, details);
1769
+ if (status !== "terminated") continue;
1770
+ const { reconciled } = await reconcileCompletedSandboxSetup(db, {
1771
+ accountId: setup.accountId,
1772
+ workspaceId: setup.workspaceId,
1773
+ sessionId: setup.sessionId,
1774
+ turnId: setup.turnId,
1775
+ attemptId: setup.attemptId,
1776
+ terminatedBox: setup.box
1777
+ });
1778
+ if (reconciled) {
1779
+ backoff.delete(turnKey);
1780
+ observability.info(
1781
+ "sandbox reaper: resumed a turn whose unknown setup ran on a terminated box",
1782
+ details
1783
+ );
1784
+ } else {
1785
+ deferEndedEpochProbe(backoff, turnKey);
1786
+ }
1787
+ } catch (error) {
1788
+ deferEndedEpochProbe(backoff, turnKey);
1789
+ observability.warn("sandbox reaper: ended-epoch setup reconciliation failed", {
1790
+ ...details,
1791
+ error: error instanceof Error ? error.message : String(error)
1792
+ });
1793
+ }
1794
+ }
1795
+ if (Object.keys(kept).length) {
1796
+ observability.info("sandbox reaper: ended-epoch blockers kept without terminal proof", kept);
1797
+ }
1798
+ }
1799
+ function deferEndedEpochProbe(backoff, key) {
1800
+ const failures = (backoff.get(key)?.failures ?? 0) + 1;
1801
+ backoff.set(key, {
1802
+ failures,
1803
+ nextProbeAtMs: Date.now() + Math.min(
1804
+ ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,
1805
+ ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16)
1806
+ )
1807
+ });
1808
+ }
1809
+ async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox, bus) {
1810
+ const claimId = crypto.randomUUID();
1811
+ let claims;
1812
+ try {
1813
+ claims = await claimTerminalRetainedProcesses(db, {
1814
+ claimId,
1815
+ limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
1816
+ claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
1817
+ });
1818
+ } catch (error) {
1819
+ recordRetainedProcessReconciliation(observability, "claim_failed");
1820
+ observability.warn("sandbox reaper: retained-process claim failed", {
1821
+ error: error instanceof Error ? error.message : String(error)
1822
+ });
1823
+ return;
1824
+ }
1825
+ const drainBudget = {
1826
+ until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS
1827
+ };
1828
+ const lostBoxes = /* @__PURE__ */ new Set();
1829
+ for (const claim of claims) {
1830
+ let process = claim.process;
1831
+ if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))
1832
+ continue;
1833
+ const expected = retainedProcessSettlementIdentity(process);
1834
+ let proof = retainedProcessReconciliationProof(process);
1835
+ let wholeBoxGone = false;
1836
+ const processScope = {
1837
+ accountId: process.accountId,
1838
+ workspaceId: process.workspaceId,
1839
+ sessionId: process.sessionId,
1840
+ processId: process.id
1841
+ };
1842
+ if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
1843
+ try {
1844
+ const command = await recoverManagedSessionBackgroundCommand(db, {
1845
+ ...processScope,
1846
+ expected,
1847
+ reconciliationClaimId: claim.claimId
1848
+ });
1849
+ if (command) {
1850
+ claim.ownerState = `background_${command.state}`;
1851
+ recordRetainedProcessReconciliation(observability, "background_recovered");
1852
+ }
1853
+ } catch (error) {
1854
+ observability.warn("sandbox reaper: retained-command background recovery failed", {
1855
+ processId: process.id,
1856
+ error: error instanceof Error ? error.message : String(error)
1857
+ });
1858
+ }
1859
+ }
1860
+ const storedCommandPersistence = retainedProviderCommandPersistence(
1861
+ db,
1862
+ processScope,
1863
+ bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : void 0
1864
+ );
1865
+ const commandPersistence = {
1866
+ ...storedCommandPersistence,
1867
+ requestCancellation: async (reason) => {
1868
+ if (reason === "provider_deadline") {
1869
+ await requestRetainedProcessDeadlineCancellation(db, processScope);
1870
+ if (!await storedCommandPersistence.cancellationRequested())
1871
+ throw new Error(
1872
+ "Supervised deadline cancellation no longer owns its original rotating lease"
1873
+ );
1874
+ } else await storedCommandPersistence.requestCancellation(reason);
1875
+ }
1876
+ };
1877
+ if (proof && process.providerBackend === "modal") {
1878
+ try {
1879
+ const command = await commandPersistence.load();
1880
+ if (command?.kind === "modal-router-v1" && command.supervision && (!await commandPersistence.loadSupervisionReceipt() || ![command.streams.stdout, command.streams.stderr].every(
1881
+ (stream) => stream.eof && stream.exitCode === 0
1882
+ )))
1883
+ proof = null;
1884
+ } catch {
1885
+ proof = null;
1886
+ }
1887
+ }
1888
+ if (!proof) {
1889
+ let observation = null;
1890
+ if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
1891
+ try {
1892
+ const lifecycle = await withRetainedProcessProbeTimeout(
1893
+ inspectHistoricalModalSandbox(settings, process.providerInstanceId, null)
1894
+ );
1895
+ if (lifecycle.status === "not_found") {
1896
+ observation = {
1897
+ status: "deferred",
1898
+ reason: "provider_binding_missing"
1899
+ };
1900
+ } else {
1901
+ process = await bindRetainedProcessProviderIdentity(db, {
1902
+ accountId: process.accountId,
1903
+ workspaceId: process.workspaceId,
1904
+ sessionId: process.sessionId,
1905
+ processId: process.id,
1906
+ expected,
1907
+ claimId: claim.claimId,
1908
+ providerBindingKey: lifecycle.providerBindingKey,
1909
+ providerBinding: lifecycle.providerBinding
1910
+ });
1911
+ recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
1912
+ if (lifecycle.status === "terminated") {
1913
+ observation = {
1914
+ status: "proved",
1915
+ proof: {
1916
+ outcome: "lost",
1917
+ exitCode: null,
1918
+ reason: "provider_instance_terminated"
1919
+ }
1920
+ };
1921
+ }
1922
+ }
1923
+ } catch (error) {
1924
+ observation = {
1925
+ status: "deferred",
1926
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
1927
+ };
1928
+ }
1929
+ }
1930
+ if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
1931
+ try {
1932
+ await requestRetainedProcessDeadlineCancellation(db, processScope);
1933
+ } catch (error) {
1934
+ observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
1935
+ processId: process.id,
1936
+ error: error instanceof Error ? error.message : String(error)
1937
+ });
1938
+ }
1939
+ }
1940
+ if (observation === null) {
1941
+ const lease = await readLease2(db, process.workspaceId, process.sandboxGroupId).catch(
1942
+ () => null
1943
+ );
1944
+ const currentLeaseIdentityMatches = Boolean(lease) && lease.id === process.leaseId && lease.leaseEpoch === process.leaseEpoch && lease.backend === process.providerBackend && lease.instanceId === process.providerInstanceId && process.routeTargetId === null;
1945
+ if (!currentLeaseIdentityMatches) {
1946
+ if (process.providerBackend !== "modal" || process.routeTargetId !== null) {
1947
+ observation = { status: "deferred", reason: "identity_mismatch" };
1948
+ } else {
1949
+ try {
1950
+ const lifecycle = await withRetainedProcessProbeTimeout(
1951
+ inspectHistoricalModalSandbox(
1952
+ settings,
1953
+ process.providerInstanceId,
1954
+ process.providerBindingKey
1955
+ )
1956
+ );
1957
+ observation = lifecycle.status === "not_found" ? {
1958
+ status: "proved",
1959
+ proof: {
1960
+ outcome: "lost",
1961
+ exitCode: null,
1962
+ reason: "provider_instance_not_found"
1963
+ }
1964
+ } : lifecycle.status === "terminated" ? {
1965
+ status: "proved",
1966
+ proof: {
1967
+ outcome: "lost",
1968
+ exitCode: null,
1969
+ reason: "provider_instance_terminated"
1970
+ }
1971
+ } : { status: "deferred", reason: "identity_mismatch" };
1972
+ } catch (error) {
1973
+ observation = {
1974
+ status: "deferred",
1975
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
1976
+ };
1977
+ }
1978
+ }
1979
+ } else {
1980
+ try {
1981
+ const command = await commandPersistence.load();
1982
+ const supervised = command?.kind === "modal-router-v1" && Boolean(command.supervision);
1983
+ const supervisionMetric = (outcome) => observability.incrementCounter({
1984
+ name: "opengeni_command_supervision_total",
1985
+ help: "Native retained command supervision reconciliation outcomes.",
1986
+ labels: { outcome }
1987
+ });
1988
+ if (supervised && (lease.rotationReason === "provider_deadline" || claim.ownerState === "background_stopping")) {
1989
+ await commandPersistence.requestCancellation(
1990
+ lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
1991
+ );
1992
+ supervisionMetric("cancellation_intent");
1993
+ }
1994
+ const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
1995
+ const probeOutcome = await probe(
1996
+ settings,
1997
+ lease,
1998
+ process,
1999
+ claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
2000
+ async (result, chunkId, stream, streamFidelity) => {
2001
+ if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
2002
+ return;
2003
+ const events = await appendSessionCommandOutput(db, {
2004
+ accountId: process.accountId,
2005
+ workspaceId: process.workspaceId,
2006
+ sessionId: process.sessionId,
2007
+ commandId: process.id,
2008
+ chunkId,
2009
+ stream: stream ?? "stdout",
2010
+ streamFidelity: streamFidelity ?? (stream ? "separate" : "merged"),
2011
+ chunk: stream ? result : stripExecBanner(result)
2012
+ });
2013
+ if (events.length && bus)
2014
+ await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
2015
+ },
2016
+ commandPersistence,
2017
+ drainBudget
2018
+ ).then(
2019
+ (value) => ({ ok: true, value }),
2020
+ (error) => ({ ok: false, error })
2021
+ );
2022
+ if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
2023
+ throw new Error("Deadline cancellation no longer owns its rotating lease");
2024
+ if (!probeOutcome.ok) throw probeOutcome.error;
2025
+ observation = probeOutcome.value;
2026
+ wholeBoxGone = observation.status === "proved" && observation.proof.outcome === "lost" && observation.proof.reason === "provider_instance_not_found";
2027
+ if (supervised) {
2028
+ supervisionMetric(
2029
+ await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
2030
+ );
2031
+ if (observation.status === "deferred")
2032
+ supervisionMetric(
2033
+ observation.reason === "provider_running" ? "checkpoint_blocked" : "provider_failure"
2034
+ );
2035
+ }
2036
+ } catch (error) {
2037
+ observability.warn("sandbox reaper: retained-process provider probe failed", {
2038
+ processId: process.id,
2039
+ providerBackend: process.providerBackend,
2040
+ error: error instanceof Error ? error.message : String(error)
2041
+ });
2042
+ observation = { status: "deferred", reason: "provider_error" };
2043
+ }
2044
+ }
2045
+ }
2046
+ if (observation.status === "binding") {
2047
+ try {
2048
+ process = await bindRetainedProcessProviderIdentity(db, {
2049
+ accountId: process.accountId,
2050
+ workspaceId: process.workspaceId,
2051
+ sessionId: process.sessionId,
2052
+ processId: process.id,
2053
+ expected,
2054
+ claimId: claim.claimId,
2055
+ providerBindingKey: observation.providerBindingKey,
2056
+ providerBinding: observation.providerBinding
2057
+ });
2058
+ recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
2059
+ observation = { status: "deferred", reason: "provider_running" };
2060
+ } catch (error) {
2061
+ observability.warn("sandbox reaper: retained-process binding adoption failed", {
2062
+ processId: process.id,
2063
+ error: error instanceof Error ? error.message : String(error)
2064
+ });
2065
+ observation = { status: "deferred", reason: "provider_error" };
2066
+ }
2067
+ }
2068
+ if (observation.status === "deferred") {
2069
+ await deferRetainedProcessClaim(
2070
+ db,
2071
+ settings,
2072
+ observability,
2073
+ process,
2074
+ expected,
2075
+ claim.claimId,
2076
+ observation.reason,
2077
+ observation.outputBacklog === true
2078
+ );
2079
+ continue;
2080
+ }
2081
+ try {
2082
+ process = await recordRetainedProcessReconciliationProof(db, {
2083
+ accountId: process.accountId,
2084
+ workspaceId: process.workspaceId,
2085
+ sessionId: process.sessionId,
2086
+ processId: process.id,
2087
+ expected,
2088
+ claimId: claim.claimId,
2089
+ proof: observation.proof
2090
+ });
2091
+ proof = observation.proof;
2092
+ recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
2093
+ } catch (error) {
2094
+ recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
2095
+ warnRetainedProcessProofFailure(observability, error, process);
2096
+ continue;
2097
+ }
2098
+ }
2099
+ if (wholeBoxGone) {
2100
+ const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;
2101
+ for (const other of claims) {
2102
+ const sibling = other.process;
2103
+ if (other === claim || `${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey || sibling.ownerActorKind !== "turn" || sibling.providerBackend !== "modal" || other.ownerState.startsWith("background_") || retainedProcessReconciliationProof(sibling))
2104
+ continue;
2105
+ try {
2106
+ await recoverManagedSessionBackgroundCommand(db, {
2107
+ accountId: sibling.accountId,
2108
+ workspaceId: sibling.workspaceId,
2109
+ sessionId: sibling.sessionId,
2110
+ processId: sibling.id,
2111
+ expected: retainedProcessSettlementIdentity(sibling),
2112
+ reconciliationClaimId: other.claimId
2113
+ });
2114
+ } catch (error) {
2115
+ observability.warn("sandbox reaper: retained-command background recovery failed", {
2116
+ processId: sibling.id,
2117
+ error: error instanceof Error ? error.message : String(error)
2118
+ });
2119
+ }
2120
+ }
2121
+ try {
2122
+ const marked = await markWarmLeaseInstanceLost(db, {
2123
+ accountId: process.accountId,
2124
+ workspaceId: process.workspaceId,
2125
+ sandboxGroupId: process.sandboxGroupId,
2126
+ expectedEpoch: process.leaseEpoch,
2127
+ expectedInstanceId: process.providerInstanceId,
2128
+ expectedBackend: process.providerBackend
2129
+ });
2130
+ if (marked.status === "marked") {
2131
+ lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);
2132
+ recordRetainedProcessReconciliation(observability, "provider_lost_whole_box");
2133
+ if (marked.backgroundCommandEvents?.length) {
2134
+ await publishDurableSessionEvents(
2135
+ bus,
2136
+ process.workspaceId,
2137
+ marked.backgroundCommandEvents,
2138
+ (error) => {
2139
+ observability.warn("sandbox reaper: lost-box command event fanout failed", {
2140
+ sandboxGroupId: process.sandboxGroupId,
2141
+ error: error instanceof Error ? error.message : String(error)
2142
+ });
2143
+ }
2144
+ );
2145
+ }
2146
+ continue;
2147
+ }
2148
+ } catch (error) {
2149
+ observability.warn("sandbox reaper: whole-box loss settlement failed", {
2150
+ processId: process.id,
2151
+ error: error instanceof Error ? error.message : String(error)
2152
+ });
2153
+ }
2154
+ }
2155
+ try {
2156
+ const settlement = await settleRetainedProcess(db, {
2157
+ accountId: process.accountId,
2158
+ workspaceId: process.workspaceId,
2159
+ sessionId: process.sessionId,
2160
+ processId: process.id,
2161
+ expected,
2162
+ reconciliationClaimId: claim.claimId,
2163
+ outcome: proof.outcome,
2164
+ exitCode: proof.exitCode,
2165
+ reason: proof.reason,
2166
+ idleGraceMs: settings.sandboxIdleGraceMs
2167
+ });
2168
+ if (settlement.process.state === "active") {
2169
+ throw new Error("Retained-process reconciliation returned an active durable process");
2170
+ }
2171
+ if (settlement.backgroundCommandEvents.length > 0) {
2172
+ try {
2173
+ await bus.publish(
2174
+ process.workspaceId,
2175
+ process.sessionId,
2176
+ settlement.backgroundCommandEvents
2177
+ );
2178
+ } catch (error) {
2179
+ observability.warn("sandbox reaper: retained-command event fanout failed", {
2180
+ processId: process.id,
2181
+ error: error instanceof Error ? error.message : String(error)
2182
+ });
2183
+ }
2184
+ }
2185
+ recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
2186
+ } catch (error) {
2187
+ recordRetainedProcessReconciliation(observability, "settlement_failed");
2188
+ observability.warn("sandbox reaper: retained-process settlement failed", {
2189
+ processId: process.id,
2190
+ error: error instanceof Error ? error.message : String(error)
2191
+ });
2192
+ await deferRetainedProcessClaim(
2193
+ db,
2194
+ settings,
2195
+ observability,
2196
+ process,
2197
+ expected,
2198
+ claim.claimId,
2199
+ "settlement_failed"
2200
+ );
2201
+ }
2202
+ }
2203
+ }
2204
+ async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome, outputBacklog = false) {
2205
+ const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
2206
+ try {
2207
+ const lease = process.providerBackend === "modal" && process.routeTargetId === null ? await readLease2(db, process.workspaceId, process.sandboxGroupId) : null;
2208
+ await deferRetainedProcessReconciliation(db, {
2209
+ accountId: process.accountId,
2210
+ workspaceId: process.workspaceId,
2211
+ sessionId: process.sessionId,
2212
+ processId: process.id,
2213
+ expected,
2214
+ claimId,
2215
+ outcome: deferral.durableOutcome,
2216
+ retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs)
2217
+ });
2218
+ recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
2219
+ } catch (error) {
2220
+ recordRetainedProcessReconciliation(observability, "defer_failed");
2221
+ observability.warn("sandbox reaper: retained-process defer failed", {
2222
+ processId: process.id,
2223
+ error: error instanceof Error ? error.message : String(error)
2224
+ });
2225
+ }
2226
+ }
2227
+ function retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog = false) {
2228
+ const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch" || outcome === "process_observation_unavailable");
2229
+ if (bindingQuarantine) {
2230
+ return {
2231
+ durableOutcome: `quarantined_${outcome}`,
2232
+ metricOutcome: outcome === "provider_binding_missing" ? "quarantined_binding_missing" : outcome === "provider_binding_mismatch" ? "quarantined_binding_mismatch" : "quarantined_process_observation_unavailable",
2233
+ retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
2234
+ quarantined: true
2235
+ };
2236
+ }
2237
+ const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4);
2238
+ const exponent = outcome === "provider_running" && outputBacklog ? 0 : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
2239
+ const retryAfterMs = Math.min(5 * 6e4, baseRetryMs * 2 ** exponent);
2240
+ return {
2241
+ durableOutcome: outcome,
2242
+ metricOutcome: outcome,
2243
+ retryAfterMs,
2244
+ quarantined: false
2245
+ };
2246
+ }
2247
+ var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
2248
+ async function drainRetainedCommandBacklog(session, providerSessionId, first, capturePage, budget) {
2249
+ let backlog = retainedBacklogRemains(session, first);
2250
+ await capturePage(first);
2251
+ let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
2252
+ if (typeof session.writeStdin !== "function") return observation;
2253
+ const until = Math.min(
2254
+ Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
2255
+ budget?.until ?? Number.POSITIVE_INFINITY
2256
+ );
2257
+ for (let reads = 0; backlog && observation.status === "deferred" && observation.reason === "provider_running" && reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS && Date.now() < until; reads++) {
2258
+ let next;
2259
+ try {
2260
+ next = await withRetainedProcessProbeTimeout(
2261
+ session.writeStdin({
2262
+ sessionId: providerSessionId,
2263
+ chars: "",
2264
+ yieldTimeMs: 1e3,
2265
+ maxOutputTokens: 2e3
2266
+ })
2267
+ );
2268
+ } catch {
2269
+ break;
2270
+ }
2271
+ backlog = retainedBacklogRemains(session, next);
2272
+ await capturePage(next);
2273
+ observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
2274
+ }
2275
+ return backlog && observation.status === "deferred" && observation.reason === "provider_running" ? { ...observation, outputBacklog: true } : observation;
2276
+ }
2277
+ function retainedBacklogRemains(session, result) {
2278
+ const page = session.getProviderCommandOutput?.(result);
2279
+ if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
2280
+ if (page.command.supervision || page.exitCode !== null) return false;
2281
+ if (page.providerExited) return true;
2282
+ const { command, expected } = page;
2283
+ return ["stdout", "stderr"].some(
2284
+ (stream) => command.streams[stream].byteOffset - expected.streams[stream].byteOffset >= RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES
2285
+ );
2286
+ }
2287
+ async function withRetainedProcessProbeTimeout(promise) {
2288
+ let timer;
2289
+ try {
2290
+ return await Promise.race([
2291
+ promise,
2292
+ new Promise((_resolve, reject) => {
2293
+ timer = setTimeout(
2294
+ () => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
2295
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
2296
+ );
2297
+ timer.unref?.();
2298
+ })
2299
+ ]);
2300
+ } finally {
2301
+ if (timer) clearTimeout(timer);
2302
+ }
2303
+ }
2304
+ var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
2305
+ async function captureRetainedProbeOutput(processId, result, capture) {
2306
+ const pending = pendingRetainedProbeOutput.get(processId) ?? {
2307
+ result,
2308
+ chunkId: crypto.randomUUID()
2309
+ };
2310
+ pendingRetainedProbeOutput.set(processId, pending);
2311
+ await capture(pending.result, pending.chunkId);
2312
+ pendingRetainedProbeOutput.delete(processId);
2313
+ }
2314
+ async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence, drainBudget) {
2315
+ if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.resumeBackendId || sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend || process.routeTargetId !== null) {
2316
+ return { status: "deferred", reason: "identity_mismatch" };
2317
+ }
2318
+ if (!lease.resumeState) {
2319
+ return { status: "deferred", reason: "resume_state_missing" };
2320
+ }
2321
+ const pending = pendingRetainedProbeOutput.get(process.id);
2322
+ if (pending) {
2323
+ if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
2324
+ await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
2325
+ const pendingObservation = classifyRetainedProcessPollResult(
2326
+ pending.result,
2327
+ process.providerSessionId
2328
+ );
2329
+ if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
2330
+ }
2331
+ const envelopeBackend = lease.resumeState.backendId;
2332
+ if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
2333
+ return { status: "deferred", reason: "identity_mismatch" };
2334
+ }
2335
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
2336
+ return { status: "deferred", reason: "identity_mismatch" };
2337
+ }
2338
+ let configuredModalBinding = null;
2339
+ if (process.providerBackend === "modal" && process.providerBindingKey) {
2340
+ try {
2341
+ configuredModalBinding = await withRetainedProcessProbeTimeout(
2342
+ resolveModalCheckpointProviderBinding(settings)
2343
+ );
2344
+ } catch (error) {
2345
+ return {
2346
+ status: "deferred",
2347
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
2348
+ };
2349
+ }
2350
+ if (configuredModalBinding.key !== process.providerBindingKey) {
2351
+ return { status: "deferred", reason: "provider_binding_mismatch" };
2352
+ }
2353
+ }
2354
+ let client;
2355
+ try {
2356
+ client = createSandboxClientForBackend2(process.providerBackend, settings);
2357
+ } catch {
2358
+ return { status: "deferred", reason: "provider_error" };
2359
+ }
2360
+ if (!client || sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend || !client.resume) {
2361
+ return { status: "deferred", reason: "backend_unsupported" };
2362
+ }
2363
+ const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
2364
+ let resumedState;
2365
+ try {
2366
+ resumedState = await deserializeSandboxSessionStateEnvelope2(
2367
+ client,
2368
+ envelopeSessionState,
2369
+ process.providerInstanceId
2370
+ );
2371
+ } catch {
2372
+ return { status: "deferred", reason: "provider_error" };
2373
+ }
2374
+ if (resumedState === void 0) {
2375
+ return { status: "deferred", reason: "resume_state_missing" };
2376
+ }
2377
+ let session;
2378
+ try {
2379
+ const resumed = await withRetainedProcessProbeTimeout(
2380
+ resumeExactSandboxSession2(
2381
+ client,
2382
+ process.providerBackend,
2383
+ resumedState,
2384
+ process.providerInstanceId
2385
+ )
2386
+ );
2387
+ session = resumed.session;
2388
+ } catch (error) {
2389
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
2390
+ return { status: "deferred", reason: "provider_timeout" };
2391
+ }
2392
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
2393
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
2394
+ return { status: "deferred", reason: "provider_binding_missing" };
2395
+ }
2396
+ return {
2397
+ status: "proved",
2398
+ proof: {
2399
+ outcome: "lost",
2400
+ exitCode: null,
2401
+ reason: "provider_instance_not_found"
2402
+ }
2403
+ };
2404
+ }
2405
+ return { status: "deferred", reason: "provider_error" };
2406
+ }
2407
+ if (typeof session.writeStdin !== "function") {
2408
+ return { status: "deferred", reason: "backend_unsupported" };
2409
+ }
2410
+ let legacyPty = false;
2411
+ if (process.providerBackend === "modal") {
2412
+ let liveBinding;
2413
+ try {
2414
+ liveBinding = await withRetainedProcessProbeTimeout(
2415
+ resolveModalCheckpointProviderBindingForSession(settings, session)
2416
+ );
2417
+ } catch (error) {
2418
+ return {
2419
+ status: "deferred",
2420
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
2421
+ };
2422
+ }
2423
+ if (!process.providerBindingKey) {
2424
+ return {
2425
+ status: "binding",
2426
+ providerBindingKey: liveBinding.key,
2427
+ providerBinding: liveBinding.binding
2428
+ };
2429
+ }
2430
+ if (liveBinding.key !== process.providerBindingKey || configuredModalBinding?.key !== process.providerBindingKey) {
2431
+ return { status: "deferred", reason: "provider_binding_mismatch" };
2432
+ }
2433
+ const command = await providerPersistence?.load();
2434
+ if (!command || !providerPersistence || !session.bindProviderCommand)
2435
+ return { status: "deferred", reason: "process_observation_unavailable" };
2436
+ session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
2437
+ legacyPty = command.pty === true;
2438
+ }
2439
+ const capturePage = async (value) => {
2440
+ if (!captureOutput) {
2441
+ if (session.getProviderCommandOutput?.(value))
2442
+ throw new Error("Provider command output requires durable capture before settlement");
2443
+ return;
2444
+ }
2445
+ const page = session.getProviderCommandOutput?.(value);
2446
+ if (page?.command.kind === "modal-router-v1" && typeof value === "string") {
2447
+ if (!session.captureCommandOutput || !await session.captureCommandOutput(value))
2448
+ throw new Error("Byte-offset output requires atomic capture before settlement");
2449
+ return;
2450
+ }
2451
+ if (page) {
2452
+ for (const chunk of page.chunks)
2453
+ if (chunk.text)
2454
+ await captureOutput(chunk.text, chunk.chunkId, chunk.stream, page.streamFidelity);
2455
+ } else {
2456
+ await captureRetainedProbeOutput(process.id, value, captureOutput);
2457
+ }
2458
+ if (typeof value === "string") await session.acknowledgeCommandOutput?.(value);
2459
+ };
2460
+ let result;
2461
+ try {
2462
+ const supervisedCancelled = mode === "cancel" || lease.rotationReason === "provider_deadline" ? await session.cancelSupervisedCommand?.(
2463
+ process.providerSessionId,
2464
+ lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
2465
+ ) ?? false : false;
2466
+ result = await withRetainedProcessProbeTimeout(
2467
+ session.writeStdin({
2468
+ sessionId: process.providerSessionId,
2469
+ // Ctrl-C is a signal only through a PTY. For a non-PTY command it is
2470
+ // ordinary stdin data and can corrupt the command's input.
2471
+ chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
2472
+ yieldTimeMs: 1e3,
2473
+ maxOutputTokens: 2e3
2474
+ })
2475
+ );
2476
+ } catch (error) {
2477
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
2478
+ return { status: "deferred", reason: "provider_timeout" };
2479
+ }
2480
+ if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
2481
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
2482
+ return { status: "deferred", reason: "provider_binding_missing" };
2483
+ }
2484
+ return {
2485
+ status: "proved",
2486
+ proof: {
2487
+ outcome: "lost",
2488
+ exitCode: null,
2489
+ reason: "provider_instance_not_found"
2490
+ }
2491
+ };
2492
+ }
2493
+ return { status: "deferred", reason: "provider_error" };
2494
+ }
2495
+ const observation = await drainRetainedCommandBacklog(
2496
+ session,
2497
+ process.providerSessionId,
2498
+ result,
2499
+ capturePage,
2500
+ drainBudget
2501
+ );
2502
+ if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
2503
+ try {
2504
+ const interrupted = await withRetainedProcessProbeTimeout(
2505
+ session.writeStdin({
2506
+ sessionId: process.providerSessionId,
2507
+ chars: "",
2508
+ yieldTimeMs: 1e3,
2509
+ maxOutputTokens: 2e3
2510
+ })
2511
+ );
2512
+ await capturePage(interrupted);
2513
+ return classifyRetainedProcessPollResult(interrupted, process.providerSessionId, session);
2514
+ } catch (error) {
2515
+ if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
2516
+ if (process.providerBackend === "modal" && !process.providerBindingKey) {
2517
+ return { status: "deferred", reason: "provider_binding_missing" };
2518
+ }
2519
+ return {
2520
+ status: "proved",
2521
+ proof: {
2522
+ outcome: "lost",
2523
+ exitCode: null,
2524
+ reason: "provider_instance_not_found"
2525
+ }
2526
+ };
2527
+ }
2528
+ return {
2529
+ status: "deferred",
2530
+ reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
2531
+ };
2532
+ }
2533
+ }
2534
+ return observation;
2535
+ }
2536
+ function classifyRetainedProcessPollResult(result, providerSessionId, source) {
2537
+ if (typeof result !== "string") {
2538
+ return { status: "deferred", reason: "provider_unknown" };
2539
+ }
2540
+ if (isExecSessionLostBanner(result, providerSessionId, source)) {
2541
+ return {
2542
+ status: "proved",
2543
+ proof: {
2544
+ outcome: "lost",
2545
+ exitCode: null,
2546
+ reason: "provider_session_lost_banner"
2547
+ }
2548
+ };
2549
+ }
2550
+ const exitCode = parseExecBannerExitCode(result);
2551
+ if (exitCode !== null) {
2552
+ return {
2553
+ status: "proved",
2554
+ proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" }
2555
+ };
2556
+ }
2557
+ return {
2558
+ status: "deferred",
2559
+ reason: result.includes("Process running with session ID") ? "provider_running" : "provider_unknown"
2560
+ };
2561
+ }
2562
+ async function accrueWarmTick(db, settings, observability) {
2563
+ const workspaceIds = /* @__PURE__ */ new Set();
2564
+ let accrued = 0;
2565
+ let leases = [];
2566
+ try {
2567
+ leases = await listMeterableWarmLeases(db);
2568
+ } catch (error) {
2569
+ observability.warn("sandbox reaper: warm-lease read failed", {
2570
+ error: error instanceof Error ? error.message : String(error)
2571
+ });
2572
+ return { accrued, workspaceIds };
2573
+ }
2574
+ for (const lease of leases) {
2575
+ workspaceIds.add(lease.workspaceId);
2576
+ }
2577
+ await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
2578
+ try {
2579
+ const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
2580
+ const result = await accrueWarmSeconds(db, {
2581
+ accountId: lease.accountId,
2582
+ workspaceId: lease.workspaceId,
2583
+ sandboxGroupId: lease.sandboxGroupId,
2584
+ expectedEpoch: lease.leaseEpoch,
2585
+ warmRateMicrosPerSecond: rate,
2586
+ billingMode: settings.sandboxWarmBillingMode,
2587
+ subjectId: lease.sandboxGroupId
2588
+ });
2589
+ if (result.accrued) {
2590
+ accrued += 1;
2591
+ }
2592
+ if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
2593
+ recordCreditMicros(observability, "usage", result.costMicros);
2594
+ }
2595
+ } catch (error) {
2596
+ observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
2597
+ workspaceId: lease.workspaceId,
2598
+ sandboxGroupId: lease.sandboxGroupId,
2599
+ error: error instanceof Error ? error.message : String(error)
2600
+ });
2601
+ }
2602
+ });
2603
+ return { accrued, workspaceIds };
2604
+ }
2605
+ async function refreshIdleSandboxReaperProjections(input) {
2606
+ await refreshQueueLeaseAndCreditGauges(
2607
+ input.db,
2608
+ input.settings,
2609
+ input.observability,
2610
+ input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory
2611
+ );
2612
+ }
2613
+ async function modalOrphanSweepMayHaveWork(input) {
2614
+ const { settings, observability } = input;
2615
+ if (!modalOrphanSweepConfigured(settings)) return false;
2616
+ if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;
2617
+ recordModalSandboxInventoryGauges(observability, {
2618
+ running: 0,
2619
+ unleased: 0,
2620
+ liveLeaseInstancesMissing: 0
2621
+ });
2622
+ recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
2623
+ return false;
2624
+ }
2625
+ function modalOrphanSweepConfigured(settings) {
2626
+ return settings.sandboxBackend === "modal" || !!settings.modalTokenId || !!settings.modalTokenSecret;
2627
+ }
2628
+ async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
2629
+ await Promise.all([
2630
+ (async () => {
2631
+ try {
2632
+ recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
2633
+ } catch (error) {
2634
+ observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
2635
+ error: error instanceof Error ? error.message : String(error)
2636
+ });
2637
+ }
2638
+ })(),
2639
+ refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
2640
+ recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
2641
+ }),
2642
+ refreshSandboxInventoryGauge(
2643
+ observability,
2644
+ "checkpoint_artifacts",
2645
+ "checkpoint-artifact",
2646
+ async () => {
2647
+ recordSandboxCheckpointArtifactGauges(
2648
+ observability,
2649
+ await countSandboxCheckpointArtifactsByState(db)
2650
+ );
2651
+ }
2652
+ ),
2653
+ refreshSandboxInventoryGauge(
2654
+ observability,
2655
+ "recovery_observations",
2656
+ "recovery-observations",
2657
+ async () => {
2658
+ recordSandboxRecoveryObservationGauges(
2659
+ observability,
2660
+ await readRecentSandboxRecoveryObservations(db)
2661
+ );
2662
+ }
2663
+ ),
2664
+ refreshSandboxInventoryGauge(
2665
+ observability,
2666
+ "rotation_backlog",
2667
+ "rotation-backlog",
2668
+ async () => {
2669
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
2670
+ }
2671
+ ),
2672
+ refreshSandboxInventoryGauge(
2673
+ observability,
2674
+ "checkpoint_staleness",
2675
+ "checkpoint-staleness",
2676
+ async () => {
2677
+ recordSandboxCheckpointStalenessGauges(
2678
+ observability,
2679
+ await readSandboxCheckpointStaleness(db)
2680
+ );
2681
+ }
2682
+ ),
2683
+ refreshSandboxInventoryGauge(
2684
+ observability,
2685
+ "retained_processes",
2686
+ "retained-process",
2687
+ async () => {
2688
+ recordRetainedProcessInventoryGauges(
2689
+ observability,
2690
+ await countActiveRetainedProcessesByOwnerState(db)
2691
+ );
2692
+ }
2693
+ ),
2694
+ refreshSandboxInventoryGauge(
2695
+ observability,
2696
+ "interaction_idle",
2697
+ "interaction-only-lease",
2698
+ async () => {
2699
+ recordInteractionOnlyLeaseGauges(
2700
+ observability,
2701
+ await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db))
2702
+ );
2703
+ }
2704
+ ),
2705
+ refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
2706
+ recordExpiredDrainingSandboxLeaseGauges(
2707
+ observability,
2708
+ await countExpiredDrainingSandboxLeases(db)
2709
+ );
2710
+ }),
2711
+ ...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
2712
+ refreshSandboxInventoryGauge(
2713
+ observability,
2714
+ "opensandbox_kubernetes",
2715
+ "OpenSandbox Kubernetes",
2716
+ async () => {
2717
+ recordOpenSandboxKubernetesInventoryGauges(
2718
+ observability,
2719
+ await inspectOpenSandboxInventory({
2720
+ namespace: settings.openSandboxKubernetesInventoryNamespace
2721
+ })
2722
+ );
2723
+ }
2724
+ )
2725
+ ] : [],
2726
+ (async () => {
2727
+ try {
2728
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
2729
+ } catch (error) {
2730
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
2731
+ error: error instanceof Error ? error.message : String(error)
2732
+ });
2733
+ }
2734
+ })(),
2735
+ (async () => {
2736
+ try {
2737
+ recordActiveUserGauges(observability, await countActiveUsers(db));
2738
+ } catch (error) {
2739
+ observability.warn("sandbox reaper: active-user gauge refresh failed", {
2740
+ error: error instanceof Error ? error.message : String(error)
2741
+ });
2742
+ }
2743
+ })(),
2744
+ (async () => {
2745
+ try {
2746
+ recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
2747
+ } catch (error) {
2748
+ observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
2749
+ error: error instanceof Error ? error.message : String(error)
2750
+ });
2751
+ }
2752
+ })(),
2753
+ (async () => {
2754
+ recordVerifiedSignupTrialDeploymentFlagGauge(
2755
+ observability,
2756
+ settings.verifiedSignupTrialCreditsEnabled
2757
+ );
2758
+ try {
2759
+ const trialSwitch = await readVerifiedSignupTrialSwitch(db);
2760
+ recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
2761
+ } catch (error) {
2762
+ observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
2763
+ error: error instanceof Error ? error.message : String(error)
2764
+ });
2765
+ }
2766
+ })(),
2767
+ (async () => {
2768
+ try {
2769
+ const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
2770
+ recordManagedAuthNewSignupsSwitchGauge(
2771
+ observability,
2772
+ signupsSwitch?.signupsEnabled !== false
2773
+ );
2774
+ } catch (error) {
2775
+ observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
2776
+ error: error instanceof Error ? error.message : String(error)
2777
+ });
2778
+ }
2779
+ })()
2780
+ ]);
2781
+ }
2782
+ async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
2783
+ try {
2784
+ await refresh();
2785
+ recordSandboxInventoryProjectionSuccess(observability, domain);
2786
+ } catch (error) {
2787
+ recordSandboxInventoryProjectionFailure(observability, domain);
2788
+ observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
2789
+ error: error instanceof Error ? error.message : String(error)
2790
+ });
2791
+ }
2792
+ }
2793
+ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
2794
+ const enforceBalance = false;
2795
+ const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
2796
+ let forceDrained = 0;
2797
+ await forEachWithConcurrency(
2798
+ [...workspaceIds],
2799
+ SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
2800
+ async (workspaceId) => {
2801
+ try {
2802
+ const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
2803
+ workspaceId,
2804
+ enforceBalance,
2805
+ maxWarmSecondsPerWorkspace: cap,
2806
+ idleGraceMs: settings.sandboxIdleGraceMs
2807
+ });
2808
+ if (result.overLimit && result.drained.length > 0) {
2809
+ forceDrained += result.drained.length;
2810
+ observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
2811
+ workspaceId,
2812
+ reason: result.reason,
2813
+ drained: result.drained.length
2814
+ });
2815
+ }
2816
+ } catch (error) {
2817
+ observability.warn("sandbox reaper: force-drain check failed for workspace", {
2818
+ workspaceId,
2819
+ error: error instanceof Error ? error.message : String(error)
2820
+ });
2821
+ }
2822
+ }
2823
+ );
2824
+ return forceDrained;
2825
+ }
2826
+ async function reconcileModalProviderCreates(db, settings, observability, discover = findModalProviderCreateReceipt) {
2827
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
2828
+ return;
2829
+ const candidates = await listPendingModalProviderCreates(db);
2830
+ await forEachWithConcurrency(candidates, 2, async (candidate) => {
2831
+ try {
2832
+ const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
2833
+ const identity = { ...candidate, accountId };
2834
+ const attempt = await claimModalProviderCreateRecovery(db, identity);
2835
+ if (!attempt) return;
2836
+ const instanceId = await discover(settings, attempt);
2837
+ if (!instanceId) return;
2838
+ if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
2839
+ observability.info("sandbox reaper: recovered Modal create receipt", {
2840
+ ...candidate,
2841
+ operationId: attempt.operationId,
2842
+ instanceId
2843
+ });
2844
+ }
2845
+ } catch (error) {
2846
+ observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
2847
+ ...candidate,
2848
+ error: error instanceof Error ? error.message : String(error)
2849
+ });
2850
+ }
2851
+ });
2852
+ }
2853
+ function modalOrphanTerminationStillEligible(latest, candidate) {
2854
+ if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
2855
+ return false;
2856
+ }
2857
+ return modalOrphanCandidateStillUnowned(latest, candidate);
2858
+ }
2859
+ function countWarmModalLeasesMissingInstance(latest, missingInstanceIds) {
2860
+ return missingInstanceIds.filter(
2861
+ (instanceId) => latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId)
2862
+ ).length;
2863
+ }
2864
+ function modalOrphanCandidateStillUnowned(latest, candidate) {
2865
+ if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
2866
+ return false;
2867
+ }
2868
+ const leaseId = candidate.tags.opengeni_lease_id;
2869
+ const workspaceId = candidate.tags.opengeni_workspace_id;
2870
+ const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
2871
+ if (!leaseId || !workspaceId || !sandboxGroupId) {
2872
+ return true;
2873
+ }
2874
+ const activeAttribution = latest.find(
2875
+ (lease) => lease.leaseId === leaseId && lease.workspaceId === workspaceId && lease.sandboxGroupId === sandboxGroupId
2876
+ );
2877
+ return Boolean(
2878
+ !activeAttribution || activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId
2879
+ );
2880
+ }
2881
+ async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
2882
+ if (!modalOrphanSweepConfigured(settings)) {
2883
+ return 0;
2884
+ }
2885
+ let liveLeases;
2886
+ try {
2887
+ liveLeases = await listLiveModalSandboxLeaseAttributions(db);
2888
+ } catch (error) {
2889
+ observability.warn("sandbox reaper: live Modal lease attribution read failed", {
2890
+ error: error instanceof Error ? error.message : String(error)
2891
+ });
2892
+ return 0;
2893
+ }
2894
+ const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
2895
+ revalidateTermination: async (candidate) => {
2896
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
2897
+ return modalOrphanTerminationStillEligible(latest, candidate);
2898
+ }
2899
+ });
2900
+ if (result.inventory.complete) {
2901
+ try {
2902
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
2903
+ recordModalSandboxInventoryGauges(observability, {
2904
+ running: result.inventory.running,
2905
+ unleased: result.inventory.unterminated.filter(
2906
+ (candidate) => modalOrphanCandidateStillUnowned(latest, candidate)
2907
+ ).length,
2908
+ liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
2909
+ latest,
2910
+ result.inventory.missingLiveLeaseInstanceIds
2911
+ )
2912
+ });
2913
+ recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
2914
+ } catch (error) {
2915
+ recordSandboxInventoryProjectionFailure(observability, "modal_provider");
2916
+ observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
2917
+ error: error instanceof Error ? error.message : String(error)
2918
+ });
2919
+ }
2920
+ }
2921
+ for (const terminated of result.terminated) {
2922
+ observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
2923
+ sandboxId: terminated.sandboxId,
2924
+ reason: terminated.reason,
2925
+ tags: JSON.stringify(terminated.tags)
2926
+ });
2927
+ }
2928
+ if (result.examined > 0) {
2929
+ observability.info("sandbox reaper: Modal orphan sweep completed", {
2930
+ examined: result.examined,
2931
+ terminated: result.terminated.length,
2932
+ skipped: result.skipped,
2933
+ appName: settings.modalAppName
2934
+ });
2935
+ }
2936
+ return result.terminated.length;
2937
+ }
2938
+ async function probeDrainableProviderReadiness(settings, lease) {
2939
+ if (!lease.instanceId || !lease.resumeState) {
2940
+ throw new Error("Expired workspace capture has no resumable provider identity");
2941
+ }
2942
+ const durableBackendId = lease.resumeBackendId ?? lease.backend;
2943
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
2944
+ let established;
2945
+ try {
2946
+ established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
2947
+ sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,
2948
+ recovery: "resume-only",
2949
+ backendOverride: backend
2950
+ });
2951
+ } catch (error) {
2952
+ if (isProviderSandboxNotFoundError(backend, error)) return "missing";
2953
+ throw error;
2954
+ }
2955
+ if (established.instanceId !== lease.instanceId) {
2956
+ throw new Error("Expired workspace capture resumed a different provider instance");
2957
+ }
2958
+ try {
2959
+ if (established.backendId === "modal") {
2960
+ await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);
2961
+ return "ready";
2962
+ }
2963
+ const session = established.session;
2964
+ const run = session.exec ?? session.execCommand;
2965
+ if (!run) throw new Error("Expired workspace capture provider has no readiness command");
2966
+ let timer;
2967
+ const result = await Promise.race([
2968
+ run.call(session, {
2969
+ cmd: "true",
2970
+ yieldTimeMs: 1e3,
2971
+ maxOutputTokens: 1e3
2972
+ }),
2973
+ new Promise((_, reject) => {
2974
+ timer = setTimeout(
2975
+ () => reject(new Error("Expired workspace capture readiness probe timed out")),
2976
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
2977
+ );
2978
+ })
2979
+ ]).finally(() => {
2980
+ if (timer) clearTimeout(timer);
2981
+ });
2982
+ if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {
2983
+ throw new Error("Expired workspace capture provider is not command-ready");
2984
+ }
2985
+ return "ready";
2986
+ } catch (error) {
2987
+ if (isProviderSandboxNotFoundError(backend, error)) return "missing";
2988
+ throw error;
2989
+ }
2990
+ }
2991
+ function sandboxDrainReason(lease) {
2992
+ if (lease.rotationRequestedAt === null) return "idle";
2993
+ return lease.rotationReason ?? "rotation_requested";
2994
+ }
2995
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
2996
+ const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
2997
+ const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
2998
+ if (!lease) {
2999
+ return false;
3000
+ }
3001
+ if (lease.liveness !== "draining" || lease.refcount !== 0 && !lease.unobservableCommandDrainIds?.length || lease.leaseEpoch !== row.leaseEpoch) {
3002
+ return false;
3003
+ }
3004
+ const durableBackendId = lease.resumeBackendId ?? lease.backend;
3005
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
3006
+ const managedProvider = backend !== "none" && backend !== "selfhosted";
3007
+ const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);
3008
+ if (managedProvider && !capturePolicy) {
3009
+ throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
3010
+ }
3011
+ if (capturePolicy?.liveInstance === "replaced") {
3012
+ throw new Error(
3013
+ `sandbox backend ${backend} workspace capture would replace the fenced live instance`
3014
+ );
3015
+ }
3016
+ const captureTimeoutMs = attempt.captureTimeoutMs;
3017
+ if (managedProvider && !lease.archiveCapture?.publishedAt && sandboxDrainCaptureSettlingInProcess(lease)) {
3018
+ throw new SandboxDrainRecoveryDeferredError(
3019
+ `sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`
3020
+ );
3021
+ }
3022
+ let captureClaim = null;
3023
+ let providerMissingBeforeCapture = false;
3024
+ let captureDisposition = "capture_required";
3025
+ if (lease.instanceId && managedProvider) {
3026
+ if (lease.archiveCapture) {
3027
+ const priorCapture = lease.archiveCapture;
3028
+ const instanceId = lease.instanceId;
3029
+ if (priorCapture.publishedAt) {
3030
+ captureClaim = {
3031
+ ...priorCapture,
3032
+ leaseId: lease.id,
3033
+ leaseEpoch: lease.leaseEpoch,
3034
+ instanceId,
3035
+ archiveGeneration: lease.archiveGeneration,
3036
+ archiveComplete: lease.archiveComplete
3037
+ };
3038
+ captureDisposition = "archive_published";
3039
+ } else {
3040
+ if (priorCapture.operationId === attempt.operationId && priorCapture.attempt >= attempt.attempt) {
3041
+ throw new SandboxDrainRecoveryDeferredError(
3042
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`
3043
+ );
3044
+ }
3045
+ const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {
3046
+ accountId,
3047
+ workspaceId: row.workspaceId,
3048
+ sandboxGroupId: row.sandboxGroupId,
3049
+ captureId: priorCapture.id,
3050
+ expectedEpoch: row.leaseEpoch,
3051
+ expectedInstanceId: instanceId
3052
+ });
3053
+ const unpublishedWarmingProvider = backend === "modal" && !sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) && lease.recovery.provider.status === "creating" && lease.recovery.provider.instanceId === instanceId && lease.recovery.workspace.status === "not_ready";
3054
+ const replace = async () => await replaceWorkspaceArchiveCaptureAfterProof(db, {
3055
+ accountId,
3056
+ workspaceId: row.workspaceId,
3057
+ sandboxGroupId: row.sandboxGroupId,
3058
+ priorCaptureId: priorCapture.id,
3059
+ captureId: attempt.captureId,
3060
+ operationId: attempt.operationId,
3061
+ attempt: attempt.attempt,
3062
+ expectedEpoch: row.leaseEpoch,
3063
+ expectedInstanceId: instanceId,
3064
+ captureTimeoutMs
3065
+ });
3066
+ const currentTakeoverSafe = capturePolicy !== null && capturePolicy.takeover !== "exclusive";
3067
+ if (!lease.resumeState) {
3068
+ if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {
3069
+ const replacement = await replace();
3070
+ if (!replacement) {
3071
+ throw new SandboxDrainRecoveryDeferredError(
3072
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`
3073
+ );
3074
+ }
3075
+ captureClaim = replacement;
3076
+ } else {
3077
+ if (!deadlineExpired) {
3078
+ throw new SandboxDrainRecoveryDeferredError(
3079
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`
3080
+ );
3081
+ }
3082
+ if (!unpublishedWarmingProvider) {
3083
+ throw new SandboxDrainRecoveryDeferredError(
3084
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`
3085
+ );
3086
+ }
3087
+ const replacement = await replace();
3088
+ if (!replacement) {
3089
+ throw new SandboxDrainRecoveryDeferredError(
3090
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`
3091
+ );
3092
+ }
3093
+ captureClaim = replacement;
3094
+ }
3095
+ } else if (priorCapture.takeoverSafe && currentTakeoverSafe) {
3096
+ const replacement = await replace();
3097
+ if (!replacement) {
3098
+ throw new SandboxDrainRecoveryDeferredError(
3099
+ `sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`
3100
+ );
3101
+ }
3102
+ captureClaim = replacement;
3103
+ } else {
3104
+ if (!deadlineExpired) {
3105
+ throw new SandboxDrainRecoveryDeferredError(
3106
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`
3107
+ );
3108
+ }
3109
+ const providerState = await probeDrainableProvider(settings, lease);
3110
+ if (providerState === "missing") {
3111
+ const replacement = await replace();
3112
+ if (!replacement) {
3113
+ throw new SandboxDrainRecoveryDeferredError(
3114
+ `sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`
3115
+ );
3116
+ }
3117
+ providerMissingBeforeCapture = true;
3118
+ captureClaim = replacement;
3119
+ } else {
3120
+ throw new SandboxDrainRecoveryDeferredError(
3121
+ `sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`
3122
+ );
3123
+ }
3124
+ }
3125
+ }
3126
+ } else {
3127
+ const claimed = await claimWorkspaceArchiveCapture(db, {
3128
+ accountId,
3129
+ workspaceId: row.workspaceId,
3130
+ sandboxGroupId: row.sandboxGroupId,
3131
+ captureId: attempt.captureId,
3132
+ operationId: attempt.operationId,
3133
+ attempt: attempt.attempt,
3134
+ expectedEpoch: row.leaseEpoch,
3135
+ expectedInstanceId: lease.instanceId,
3136
+ liveness: "draining",
3137
+ captureTimeoutMs,
3138
+ minIntervalMs: 0,
3139
+ pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),
3140
+ deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
3141
+ providerReplaySafe: capturePolicy?.takeover === "same_request",
3142
+ takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive"
3143
+ });
3144
+ if (claimed.status === "claimed") {
3145
+ captureClaim = claimed.claim;
3146
+ } else if (claimed.status === "mutation_in_progress") {
3147
+ const providerState = await probeDrainableProvider(settings, lease);
3148
+ if (providerState !== "missing") return false;
3149
+ providerMissingBeforeCapture = true;
3150
+ } else if (claimed.status === "capture_in_progress") {
3151
+ throw new SandboxDrainRecoveryDeferredError(
3152
+ `sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`
3153
+ );
3154
+ } else {
3155
+ return false;
3156
+ }
3157
+ }
3158
+ }
3159
+ let persisted = captureDisposition === "archive_published";
3160
+ let archiveRevision = captureDisposition === "archive_published" ? lease.recovery.archive.current?.revision ?? null : null;
3161
+ const persistArchive = async (archiveBase64, archiveMetadata, _providerSession, providerBinding) => {
3162
+ if (!lease.instanceId || !captureClaim) {
3163
+ return { wrote: false, archiveRevision: null };
3164
+ }
3165
+ const archiveMetrics = runtimeMetricsHooksForObservability(observability);
3166
+ let checkpointArtifactId = null;
3167
+ if (typeof archiveBase64 === "string" && archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
3168
+ if (!providerBinding) {
3169
+ throw new Error("Modal native snapshot has no exact session provider identity");
3170
+ }
3171
+ const candidate = await registerSandboxCheckpointArtifact(db, {
3172
+ accountId,
3173
+ workspaceId: row.workspaceId,
3174
+ sandboxGroupId: row.sandboxGroupId,
3175
+ sourceLeaseId: lease.id,
3176
+ sourceLeaseEpoch: row.leaseEpoch,
3177
+ sourceInstanceId: lease.instanceId,
3178
+ sourceWorkspaceGeneration: captureClaim.workspaceGeneration,
3179
+ providerBindingKey: providerBinding.key,
3180
+ providerBinding: providerBinding.binding,
3181
+ workspaceArchive: archiveBase64,
3182
+ workspaceArchiveMeta: archiveMetadata
3183
+ });
3184
+ checkpointArtifactId = candidate.id;
3185
+ }
3186
+ const baseInput = {
3187
+ accountId,
3188
+ workspaceId: row.workspaceId,
3189
+ sandboxGroupId: row.sandboxGroupId,
3190
+ expectedLeaseId: lease.id,
3191
+ expectedEpoch: row.leaseEpoch,
3192
+ expectedInstanceId: lease.instanceId,
3193
+ expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
3194
+ captureId: captureClaim.id,
3195
+ providerRequestId: captureClaim.providerRequestId
3196
+ };
3197
+ let result;
3198
+ if (archiveBase64 === null) {
3199
+ result = await persistDrainSnapshot(db, {
3200
+ ...baseInput,
3201
+ workspaceArchive: null
3202
+ });
3203
+ } else {
3204
+ if (!archiveMetadata) {
3205
+ throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
3206
+ }
3207
+ const published = await putVersion1TarArchiveOrInline({
3208
+ backend: lease.backend,
3209
+ objectStorage,
3210
+ accountId,
3211
+ workspaceId: row.workspaceId,
3212
+ sandboxGroupId: row.sandboxGroupId,
3213
+ archive: typeof archiveBase64 === "object" ? archiveBase64 : {
3214
+ bytes: Buffer.from(archiveBase64, "base64"),
3215
+ descriptor: archiveMetadata,
3216
+ base64: archiveBase64
3217
+ },
3218
+ metrics: archiveMetrics
3219
+ });
3220
+ result = await persistWorkspaceArchiveCandidate({
3221
+ objectStorage,
3222
+ ...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
3223
+ persist: () => persistDrainSnapshot(db, {
3224
+ ...baseInput,
3225
+ workspaceArchiveMeta: archiveMetadata,
3226
+ ...published,
3227
+ ...checkpointArtifactId ? { checkpointArtifactId } : {}
3228
+ })
3229
+ });
3230
+ }
3231
+ if (!result.wrote && checkpointArtifactId) {
3232
+ await markSandboxCheckpointArtifactDeletePending(db, {
3233
+ accountId,
3234
+ workspaceId: row.workspaceId,
3235
+ artifactId: checkpointArtifactId,
3236
+ reason: "drain_snapshot_publication_fenced"
3237
+ }).catch(() => void 0);
3238
+ }
3239
+ if (result.wrote && archiveBase64 !== null) {
3240
+ persisted = true;
3241
+ archiveRevision = result.archiveRevision;
3242
+ }
3243
+ return result;
3244
+ };
3245
+ if (!providerMissingBeforeCapture && backend === "modal" && lease.instanceId && !await readWorkspaceArchiveCapturePreflight(db, {
3246
+ accountId,
3247
+ workspaceId: row.workspaceId,
3248
+ sandboxGroupId: row.sandboxGroupId,
3249
+ expectedEpoch: row.leaseEpoch,
3250
+ expectedInstanceId: lease.instanceId,
3251
+ liveness: "draining"
3252
+ })) {
3253
+ return false;
3254
+ }
3255
+ let stopCutoffMarked = false;
3256
+ const beforeProviderStop = async () => {
3257
+ const cutoff = await markWarmBillingStopCutoff(db, {
3258
+ accountId,
3259
+ workspaceId: row.workspaceId,
3260
+ sandboxGroupId: row.sandboxGroupId,
3261
+ expectedEpoch: row.leaseEpoch
3262
+ });
3263
+ if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
3264
+ stopCutoffMarked = true;
3265
+ };
3266
+ const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
3267
+ settings,
3268
+ lease,
3269
+ observability,
3270
+ persistArchive,
3271
+ captureClaim?.providerRequestId ?? attempt.operationId,
3272
+ captureDisposition,
3273
+ capturePolicy,
3274
+ Boolean(objectStorage),
3275
+ async () => {
3276
+ if (!captureClaim || !lease.instanceId) return;
3277
+ await releaseWorkspaceArchiveCapture(db, {
3278
+ accountId,
3279
+ workspaceId: row.workspaceId,
3280
+ sandboxGroupId: row.sandboxGroupId,
3281
+ captureId: captureClaim.id,
3282
+ expectedEpoch: row.leaseEpoch,
3283
+ expectedInstanceId: lease.instanceId
3284
+ });
3285
+ },
3286
+ row.workspaceId,
3287
+ beforeProviderStop,
3288
+ backend === "docker" ? async () => {
3289
+ if (!captureClaim || !lease.instanceId)
3290
+ throw new Error("Docker drain has no current claimed capture");
3291
+ return await verifyDockerDrainCaptureFence(db, {
3292
+ accountId,
3293
+ workspaceId: row.workspaceId,
3294
+ sandboxGroupId: row.sandboxGroupId,
3295
+ leaseId: lease.id,
3296
+ leaseEpoch: row.leaseEpoch,
3297
+ instanceId: lease.instanceId,
3298
+ workspaceGeneration: captureClaim.workspaceGeneration,
3299
+ captureId: captureClaim.id,
3300
+ providerRequestId: captureClaim.providerRequestId
3301
+ });
3302
+ } : void 0
3303
+ );
3304
+ const terminated = typeof termination === "boolean" ? termination : termination.terminated;
3305
+ if (!terminated) {
3306
+ return false;
3307
+ }
3308
+ const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
3309
+ if (!stopCutoffMarked && !providerMissingDuringStop) {
3310
+ if (managedProvider && lease.instanceId) {
3311
+ throw new Error("managed sandbox stopped without a warm usage cutoff");
3312
+ }
3313
+ await beforeProviderStop();
3314
+ }
3315
+ const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
3316
+ if (!providerMissingDuringStop) {
3317
+ const finalTick = await accrueWarmSeconds(db, {
3318
+ accountId,
3319
+ workspaceId: row.workspaceId,
3320
+ sandboxGroupId: row.sandboxGroupId,
3321
+ expectedEpoch: row.leaseEpoch,
3322
+ warmRateMicrosPerSecond: finalRate,
3323
+ billingMode: settings.sandboxWarmBillingMode,
3324
+ finalDrain: true
3325
+ });
3326
+ if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
3327
+ recordCreditMicros(observability, "usage", finalTick.costMicros);
3328
+ }
3329
+ }
3330
+ const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
3331
+ const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
3332
+ db,
3333
+ {
3334
+ accountId,
3335
+ workspaceId: row.workspaceId,
3336
+ sandboxGroupId: row.sandboxGroupId,
3337
+ expectedEpoch: row.leaseEpoch,
3338
+ ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
3339
+ providerMissingBeforeCapture: providerMissing,
3340
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
3341
+ // Mirrors terminateProviderBox: a selfhosted machine is never stopped.
3342
+ providerStopped: managedProvider && lease.backend !== "selfhosted" && lease.resumeBackendId !== "selfhosted"
3343
+ }
3344
+ );
3345
+ await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
3346
+ observability.warn("sandbox reaper: contained command event fanout failed", {
3347
+ sandboxGroupId: row.sandboxGroupId,
3348
+ error: error instanceof Error ? error.message : String(error)
3349
+ });
3350
+ });
3351
+ if (wentCold) {
3352
+ if (providerMissing && !unpublishedProviderLost) {
3353
+ recordSandboxProviderMissingBeforeCapture(observability, backend);
3354
+ }
3355
+ if (lease.unobservableCommandDrainIds?.length) {
3356
+ recordSandboxCommandContainment(
3357
+ observability,
3358
+ providerMissing ? "provider_missing" : "contained"
3359
+ );
3360
+ }
3361
+ try {
3362
+ await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
3363
+ type: "sandbox.box.terminated",
3364
+ payload: {
3365
+ actor: "reaper",
3366
+ persisted,
3367
+ archiveRevision,
3368
+ instanceId: lease.instanceId,
3369
+ providerMissingBeforeCapture: providerMissing,
3370
+ // Why the box drained: the lease's bounded rotation reason when one
3371
+ // was requested (provider deadline, operator/image conflict, teardown
3372
+ // claim), otherwise the ordinary idle drain. Additive diagnostic
3373
+ // fact; no identifiers beyond the ones already carried above.
3374
+ drainReason: sandboxDrainReason(lease)
3375
+ }
3376
+ });
3377
+ } catch (eventError) {
3378
+ observability.warn(
3379
+ "sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
3380
+ {
3381
+ sandboxGroupId: row.sandboxGroupId,
3382
+ leaseEpoch: row.leaseEpoch,
3383
+ backend: lease.backend,
3384
+ instanceId: lease.instanceId,
3385
+ error: eventError instanceof Error ? eventError.message : String(eventError)
3386
+ }
3387
+ );
3388
+ }
3389
+ }
3390
+ return wentCold;
3391
+ }
3392
+ function providerDrainLogIdentity(lease, backend) {
3393
+ return {
3394
+ leaseId: lease.id,
3395
+ sandboxGroupId: lease.sandboxGroupId,
3396
+ leaseEpoch: lease.leaseEpoch,
3397
+ backend,
3398
+ instanceId: lease.instanceId
3399
+ };
3400
+ }
3401
+ async function verifyDockerDrainCaptureFence(db, input, readers = { readLease: readLease2, readWorkspaceArchiveCapturePreflight }) {
3402
+ const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);
3403
+ if (!current || current.id !== input.leaseId || current.liveness !== "draining" || current.leaseEpoch !== input.leaseEpoch || current.instanceId !== input.instanceId || current.backend !== "docker" || current.workspaceGeneration !== input.workspaceGeneration || current.archiveCapture?.id !== input.captureId || current.archiveCapture.providerRequestId !== input.providerRequestId || current.archiveCapture.workspaceGeneration !== input.workspaceGeneration)
3404
+ throw new Error("Docker drain capture lease/epoch/claim changed");
3405
+ const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {
3406
+ accountId: input.accountId,
3407
+ workspaceId: input.workspaceId,
3408
+ sandboxGroupId: input.sandboxGroupId,
3409
+ expectedEpoch: input.leaseEpoch,
3410
+ expectedInstanceId: input.instanceId,
3411
+ liveness: "draining"
3412
+ });
3413
+ if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)
3414
+ throw new Error("Docker drain capture has an active writer or changed generation");
3415
+ return {
3416
+ archivePublished: current.archiveCapture.publishedAt instanceof Date && Number.isFinite(current.archiveCapture.publishedAt.getTime()) && current.archiveComplete === true
3417
+ };
3418
+ }
3419
+ function observeDockerWorkspaceRelease(observability, session, logIdentity) {
3420
+ const release = session.hostWorkspaceRelease;
3421
+ if (!release) return;
3422
+ recordSandboxDockerWorkspaceRelease(observability, release.status);
3423
+ if (release.status === "retained") {
3424
+ observability.warn(
3425
+ "sandbox reaper: Docker drain kept its host workspace directory after teardown",
3426
+ { ...logIdentity, reason: release.reason }
3427
+ );
3428
+ }
3429
+ }
3430
+ async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend2, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop, beforeWorkspaceCapture) {
3431
+ const durableBackendId = lease.resumeBackendId ?? lease.backend;
3432
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
3433
+ const logIdentity = providerDrainLogIdentity(lease, backend);
3434
+ const warnCaptureFailure = (message, error) => warnDrainSnapshotFailure(observability, message, error, {
3435
+ sandboxGroupId: lease.sandboxGroupId,
3436
+ leaseEpoch: lease.leaseEpoch,
3437
+ ...workspaceId ? { workspaceId } : {}
3438
+ });
3439
+ const releaseAfterCaptureFailure = async () => {
3440
+ try {
3441
+ await releaseFailedCapture?.();
3442
+ } catch (error) {
3443
+ warnCaptureFailure("sandbox reaper: failed capture claim release failed", error);
3444
+ }
3445
+ };
3446
+ if (!backend || backend === "none") {
3447
+ return { terminated: true, providerMissingBeforeCapture: false };
3448
+ }
3449
+ if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
3450
+ observability.info(
3451
+ "sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped \u2014 it is the user's machine)",
3452
+ {
3453
+ ...logIdentity
3454
+ }
3455
+ );
3456
+ return { terminated: true, providerMissingBeforeCapture: false };
3457
+ }
3458
+ const durableCapturePolicy = claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);
3459
+ if (!durableCapturePolicy) {
3460
+ throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
3461
+ }
3462
+ if (durableCapturePolicy.liveInstance !== "preserved") {
3463
+ throw new Error(
3464
+ `sandbox backend ${backend} workspace capture would replace the fenced live instance`
3465
+ );
3466
+ }
3467
+ const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
3468
+ if (!lease.instanceId) {
3469
+ if (lease.providerCreateAttempt?.instanceId === null) {
3470
+ throw new Error(
3471
+ "provider_create_outcome_unknown: cannot infer termination from missing instance ID"
3472
+ );
3473
+ }
3474
+ if (persistedInstanceId) {
3475
+ throw new Error(
3476
+ `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`
3477
+ );
3478
+ }
3479
+ return { terminated: true, providerMissingBeforeCapture: false };
3480
+ }
3481
+ if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {
3482
+ throw new Error(
3483
+ `sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`
3484
+ );
3485
+ }
3486
+ const isUnpublishedWarmingProvider = backend === "modal" && !persistedInstanceId && lease.recovery.provider.status === "creating" && lease.recovery.provider.instanceId === lease.instanceId && lease.recovery.workspace.status === "not_ready";
3487
+ if (isUnpublishedWarmingProvider) {
3488
+ const { wrote } = await persistArchive(null);
3489
+ if (!wrote) {
3490
+ observability.info(
3491
+ "sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING",
3492
+ {
3493
+ ...logIdentity
3494
+ }
3495
+ );
3496
+ return { terminated: false, providerMissingBeforeCapture: false };
3497
+ }
3498
+ try {
3499
+ await beforeProviderStop?.();
3500
+ await terminateModalById(
3501
+ settings,
3502
+ lease.instanceId,
3503
+ void 0,
3504
+ lease.providerCreateAttempt?.providerBindingKey
3505
+ );
3506
+ } catch (error) {
3507
+ if (!isProviderSandboxNotFoundError("modal", error)) {
3508
+ throw error;
3509
+ }
3510
+ observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
3511
+ ...logIdentity
3512
+ });
3513
+ }
3514
+ return { terminated: true, providerMissingBeforeCapture: false };
3515
+ }
3516
+ if (!persistedInstanceId || !lease.resumeState) {
3517
+ throw new Error(
3518
+ `sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`
3519
+ );
3520
+ }
3521
+ const client = createClientForBackend(backend, settings);
3522
+ if (!client) {
3523
+ return { terminated: true, providerMissingBeforeCapture: false };
3524
+ }
3525
+ let session;
3526
+ let sessionState;
3527
+ try {
3528
+ if (!client.deserializeSessionState || backend !== "docker" && !client.resume) {
3529
+ throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
3530
+ }
3531
+ const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
3532
+ const resumedState = await deserializeSandboxSessionStateEnvelope2(
3533
+ client,
3534
+ envelopeSessionState,
3535
+ lease.instanceId
3536
+ );
3537
+ if (resumedState === void 0) {
3538
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
3539
+ }
3540
+ if (backend === "docker") {
3541
+ if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {
3542
+ throw new Error(
3543
+ "Docker drain requires a creation-free attachment and current capture fence"
3544
+ );
3545
+ }
3546
+ await beforeWorkspaceCapture();
3547
+ session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);
3548
+ sessionState = resumedState;
3549
+ } else {
3550
+ const resumed = await resumeExactSandboxSession2(
3551
+ client,
3552
+ backend,
3553
+ resumedState,
3554
+ lease.instanceId
3555
+ );
3556
+ session = resumed.session;
3557
+ sessionState = resumed.sessionState;
3558
+ }
3559
+ } catch (error) {
3560
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
3561
+ observability.info("sandbox reaper: drainable box already gone before workspace capture", {
3562
+ ...logIdentity
3563
+ });
3564
+ return {
3565
+ terminated: true,
3566
+ providerMissingBeforeCapture: captureDisposition !== "archive_published"
3567
+ };
3568
+ }
3569
+ throw error;
3570
+ }
3571
+ const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);
3572
+ if (!liveCapturePolicy || liveCapturePolicy.takeover !== durableCapturePolicy.takeover || liveCapturePolicy.strategy !== durableCapturePolicy.strategy || liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance) {
3573
+ throw new Error(
3574
+ `sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`
3575
+ );
3576
+ }
3577
+ let verifiedArchive;
3578
+ const sessionWorkspacePersistence = session?.state?.workspacePersistence;
3579
+ const checkpointBinding = captureDisposition === "capture_required" && backend === "modal" && (sessionWorkspacePersistence === "snapshot_filesystem" || sessionWorkspacePersistence === "snapshot_directory") ? await withRetainedProcessProbeTimeout(
3580
+ resolveModalCheckpointProviderBindingForSession(settings, session)
3581
+ ) : null;
3582
+ let releaseSettlingCapture = () => void 0;
3583
+ try {
3584
+ if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
3585
+ releaseSettlingCapture = trackSettlingDrainCapture(lease);
3586
+ const capture = captureWorkspaceArchiveForStorage(
3587
+ session,
3588
+ Date.now(),
3589
+ {
3590
+ requestId: providerCaptureRequestId ?? randomUUID(),
3591
+ strategy: liveCapturePolicy.strategy
3592
+ },
3593
+ diskBackedArchives
3594
+ );
3595
+ verifiedArchive = await awaitProviderCaptureWithLatePublication({
3596
+ capture,
3597
+ timeoutMs: settings.sandboxSnapshotTimeoutMs,
3598
+ timeoutError: new SandboxProviderCaptureTimeoutError(
3599
+ lease.sandboxGroupId,
3600
+ backend,
3601
+ settings.sandboxSnapshotTimeoutMs,
3602
+ lease.leaseEpoch,
3603
+ lease.instanceId
3604
+ ),
3605
+ publishLate: async (archive) => {
3606
+ try {
3607
+ const result = await persistArchive(
3608
+ archive.kind === "host_spool" ? archive : archive.base64,
3609
+ archive.descriptor,
3610
+ session,
3611
+ checkpointBinding
3612
+ );
3613
+ observability.info(
3614
+ result.wrote ? "sandbox reaper: late workspace capture published under durable claim" : "sandbox reaper: late workspace capture fenced by successor",
3615
+ {
3616
+ ...logIdentity
3617
+ }
3618
+ );
3619
+ } catch (error) {
3620
+ warnCaptureFailure("sandbox reaper: late workspace capture publication failed", error);
3621
+ } finally {
3622
+ await disposeWorkspaceArchive(archive);
3623
+ }
3624
+ },
3625
+ observeLatePublicationFailure: (error) => {
3626
+ warnCaptureFailure("sandbox reaper: late capture publication cleanup failed", error);
3627
+ },
3628
+ observeLateFailure: (error) => {
3629
+ void releaseAfterCaptureFailure();
3630
+ warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
3631
+ },
3632
+ onLateSettled: () => releaseSettlingCapture()
3633
+ });
3634
+ }
3635
+ } catch (error) {
3636
+ if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();
3637
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
3638
+ observability.info("sandbox reaper: box gone during workspace capture", {
3639
+ ...logIdentity
3640
+ });
3641
+ return { terminated: true, providerMissingBeforeCapture: true };
3642
+ }
3643
+ warnCaptureFailure(
3644
+ "sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
3645
+ error
3646
+ );
3647
+ if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
3648
+ await releaseAfterCaptureFailure();
3649
+ }
3650
+ throw error;
3651
+ }
3652
+ if (captureDisposition === "archive_published") {
3653
+ observability.info("sandbox reaper: resuming teardown from durable archive publication", {
3654
+ ...logIdentity
3655
+ });
3656
+ } else if (verifiedArchive) {
3657
+ let wrote;
3658
+ try {
3659
+ ({ wrote } = await persistArchive(
3660
+ verifiedArchive.kind === "host_spool" ? verifiedArchive : verifiedArchive.base64,
3661
+ verifiedArchive.descriptor,
3662
+ session,
3663
+ checkpointBinding
3664
+ ));
3665
+ } finally {
3666
+ try {
3667
+ await disposeWorkspaceArchive(verifiedArchive);
3668
+ } finally {
3669
+ releaseSettlingCapture();
3670
+ }
3671
+ }
3672
+ if (!wrote) {
3673
+ observability.info(
3674
+ "sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)",
3675
+ {
3676
+ ...logIdentity
3677
+ }
3678
+ );
3679
+ return { terminated: false, providerMissingBeforeCapture: false };
3680
+ }
3681
+ } else {
3682
+ throw new Error(
3683
+ `sandbox backend ${backend} produced no verified workspace archive during drain`
3684
+ );
3685
+ }
3686
+ try {
3687
+ if (!session || !sessionState) {
3688
+ throw new Error(`sandbox backend ${backend} returned no terminable session state`);
3689
+ }
3690
+ prepareProviderForTeardownAfterCapture(backend, session);
3691
+ await beforeProviderStop?.();
3692
+ await terminateManagedSandboxSession(client, sessionState, session);
3693
+ if (backend === "docker") observeDockerWorkspaceRelease(observability, session, logIdentity);
3694
+ return { terminated: true, providerMissingBeforeCapture: false };
3695
+ } catch (error) {
3696
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
3697
+ observability.info("sandbox reaper: provider already gone during terminate", {
3698
+ ...logIdentity
3699
+ });
3700
+ return { terminated: true, providerMissingBeforeCapture: false };
3701
+ }
3702
+ throw error;
3703
+ }
3704
+ }
3705
+
3706
+ export {
3707
+ createBrowserDeadlineCheckpointActivities,
3708
+ RETAINED_PROCESS_RECONCILIATION_LIMIT,
3709
+ RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,
3710
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
3711
+ RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS,
3712
+ RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
3713
+ RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,
3714
+ RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,
3715
+ RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS,
3716
+ RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
3717
+ CONNECTED_COMMAND_RECONCILIATION_LIMIT,
3718
+ CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
3719
+ SandboxProviderCaptureTimeoutError,
3720
+ awaitProviderCaptureWithLatePublication,
3721
+ sandboxDrainCaptureId,
3722
+ createSandboxLeaseActivities,
3723
+ resumeIdleCheckpointSessionAtProvider,
3724
+ SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS,
3725
+ reconcileConnectedMachineBackgroundCommands,
3726
+ replayConnectedCommandOutput,
3727
+ reconcileConnectedCommandOutputReleases,
3728
+ probeConnectedMachineBackgroundCommand,
3729
+ connectedCommandProofFromStatus,
3730
+ retainedProcessReconciliationDeferral,
3731
+ drainRetainedCommandBacklog,
3732
+ captureRetainedProbeOutput,
3733
+ probeRetainedProcessAtProvider,
3734
+ classifyRetainedProcessPollResult,
3735
+ refreshIdleSandboxReaperProjections,
3736
+ modalOrphanSweepMayHaveWork,
3737
+ reconcileModalProviderCreates,
3738
+ modalOrphanTerminationStillEligible,
3739
+ countWarmModalLeasesMissingInstance,
3740
+ modalOrphanCandidateStillUnowned,
3741
+ sandboxDrainReason,
3742
+ verifyDockerDrainCaptureFence,
3743
+ terminateProviderBox,
3744
+ sandboxLeaseTelemetryKey2 as sandboxLeaseTelemetryKey
3745
+ };
3746
+ //# sourceMappingURL=chunk-MVESOYFC.js.map