@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -14,7 +14,10 @@
14
14
  // timer, viewer activity, owner task queue, or provider-specific lifecycle path
15
15
  // in the normal drain state machine.
16
16
  import { warnDrainSnapshotFailure } from "../sandbox-snapshot-diagnostics";
17
+ import { maybePersistWarmWorkspaceSnapshot } from "../sandbox-resume";
18
+ import { publishDurableSessionEvents } from "../session-event-fanout";
17
19
  import { warnRetainedProcessProofFailure } from "../retained-process-diagnostics";
20
+ import { retainedProcessDeadlineRetryMs } from "../retained-process-retry";
18
21
 
19
22
  import { createHash, randomUUID } from "node:crypto";
20
23
  import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
@@ -26,12 +29,17 @@ import {
26
29
  retainedProviderCommandPersistence,
27
30
  adoptLegacyModalCheckpointArtifact,
28
31
  confirmDrainCold,
32
+ markWarmLeaseInstanceLost,
29
33
  appendSessionEventToSandboxGroup,
30
34
  bindRetainedProcessProviderIdentity,
31
35
  claimWorkspaceArchiveCapture,
36
+ leaseMayHaveUntrackedWriters,
32
37
  releaseWorkspaceArchiveCapture,
38
+ listIdleCheckpointCandidates,
39
+ recordIdleCheckpointAttempt,
33
40
  claimSandboxCheckpointArtifactsForGc,
34
41
  claimTerminalRetainedProcesses,
42
+ getEnrollment,
35
43
  countActiveRetainedProcessesByOwnerState,
36
44
  countSandboxCheckpointArtifactsByState,
37
45
  countExpiredDrainingSandboxLeases,
@@ -39,11 +47,19 @@ import {
39
47
  countSandboxLeasesByLiveness,
40
48
  deferRetainedProcessReconciliation,
41
49
  forceDrainOverLimitViewerOnlyBoxes,
42
- getBillingBalance,
43
50
  listCreditBalancesByAccount,
51
+ countActiveUsers,
52
+ readCreditGrantTotals,
53
+ readManagedAuthNewSignupsSwitch,
54
+ readVerifiedSignupTrialSwitch,
44
55
  listLegacyModalCheckpointSlots,
45
56
  listLiveModalSandboxLeaseAttributions,
57
+ listPendingModalProviderCreates,
58
+ claimModalProviderCreateRecovery,
59
+ recordRecoveredModalProviderCreate,
60
+ markWarmBillingStopCutoff,
46
61
  listMeterableWarmLeases,
62
+ countInteractionOnlyWarmLeasesByIdle,
47
63
  listSandboxViewerForceDrainWorkspaceIds,
48
64
  markSandboxCheckpointArtifactDeletePending,
49
65
  persistDrainSnapshot,
@@ -51,15 +67,22 @@ import {
51
67
  registerSandboxCheckpointArtifact,
52
68
  recordRetainedProcessReconciliationProof,
53
69
  replaceWorkspaceArchiveCaptureAfterProof,
70
+ listEndedEpochWorkspaceBlockers,
71
+ listEndedEpochSandboxSetupUncertainties,
72
+ reconcileCompletedSandboxSetup,
54
73
  readLease,
74
+ settleEndedEpochWorkspaceBlockers,
55
75
  readWorkspaceArchiveCapturePreflight,
56
76
  reapExpiredSessionListSnapshots,
57
77
  reapStaleLeaseHoldersGlobal,
58
78
  requestDueSandboxRotationsGlobal,
59
79
  readSandboxRotationBacklog,
80
+ readSandboxCheckpointStaleness,
81
+ readRecentSandboxRecoveryObservations,
60
82
  workspaceArchiveCaptureDeadlineElapsed,
61
83
  retainedProcessReconciliationProof,
62
84
  retainedProcessSettlementIdentity,
85
+ recoverManagedSessionBackgroundCommand,
63
86
  settleClaimedConnectedMachineBackgroundCommand,
64
87
  settleSandboxCheckpointArtifactGc,
65
88
  rlsContextForWorkspace,
@@ -71,13 +94,21 @@ import {
71
94
  type LeaseSnapshot,
72
95
  } from "@opengeni/db";
73
96
  import {
97
+ claimConnectedCommandOutputReleases,
74
98
  claimConnectedMachineSessionBackgroundCommands,
75
99
  deferConnectedMachineBackgroundCommandReconciliation,
76
100
  recordConnectedMachineBackgroundCommandProof,
101
+ recordConnectedCommandOutputConsumption,
102
+ settleConnectedCommandOutputReleaseClaim,
77
103
  type ConnectedMachineBackgroundCommandClaim,
78
104
  type ConnectedMachineBackgroundCommandProof,
79
105
  } from "@opengeni/db/session-background-commands";
80
- import { sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
106
+ import {
107
+ SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
108
+ sandboxArchiveCaptureTimeoutMs,
109
+ sandboxDeadlineMandatoryCaptureLeadMs,
110
+ sandboxWarmRateMicrosPerSecond,
111
+ } from "@opengeni/config";
81
112
  import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
82
113
  import { captureConnectedCommandOutput } from "../sandbox-routing";
83
114
  import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
@@ -109,22 +140,25 @@ import {
109
140
  parseExecBannerExitCode,
110
141
  prepareProviderForTeardownAfterCapture,
111
142
  providerWorkspaceCapturePolicy,
143
+ providerWorkspaceCaptureIsPointInTime,
112
144
  resolveModalCheckpointProviderBindingForLiveSandbox,
113
145
  resolveModalCheckpointProviderBinding,
146
+ findModalProviderCreateReceipt,
114
147
  resolveModalCheckpointProviderBindingForSession,
115
148
  querySelfhostedOp,
116
149
  resumeExactSandboxSession,
117
- sandboxProviderContinuityForState,
118
150
  sandboxBackendForSdkBackendId,
119
151
  sandboxProviderInstanceIdFromEnvelope,
120
152
  sandboxCommandExitCode,
121
153
  sandboxCommandStillRunning,
122
154
  SelfhostedControlError,
123
155
  subjectFor,
156
+ modalAppHasRunningSandboxes,
124
157
  sweepModalOrphanSandboxes,
125
158
  terminateManagedSandboxSession,
126
159
  terminateModalSandboxById,
127
160
  verifySandboxExecReadiness,
161
+ type DockerHostWorkspaceRelease,
128
162
  type ModalOrphanSweepTermination,
129
163
  type ModalCheckpointProviderBinding,
130
164
  type ProviderWorkspaceCapturePolicy,
@@ -137,13 +171,20 @@ import {
137
171
  SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
138
172
  type SandboxDrainActivityInput,
139
173
  type SandboxLeaseSweepMaintenanceInput,
174
+ sandboxDrainTimeoutClass,
175
+ sandboxIdleCheckpointWorkflowId,
176
+ type SandboxIdleCheckpointActivityInput,
177
+ type SandboxIdleCheckpointPlan,
140
178
  } from "../sandbox-reaper-contract";
141
179
  import { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from "../concurrency";
180
+ import { browserCheckpointPrerequisites } from "./browser-deadline-checkpoint";
142
181
  import { assertSandboxDrainInputTiming, sandboxDrainTiming } from "../sandbox-reaper-timeout";
143
182
  import type { ControlActivityServices as ActivityServices } from "./types";
144
183
  import { reconcilePendingParentSystemUpdates } from "./parent-wake";
145
184
  import {
185
+ recordActiveUserGauges,
146
186
  recordCreditBalanceGauges,
187
+ recordCreditGrantGauges,
147
188
  recordCreditMicros,
148
189
  recordExpiredDrainingSandboxLeaseGauges,
149
190
  recordOpenSandboxKubernetesInventoryGauges,
@@ -157,9 +198,20 @@ import {
157
198
  type RetainedProcessReconciliationOutcome,
158
199
  type SandboxInventoryProjectionDomain,
159
200
  recordSandboxLeaseGauges,
201
+ recordModalSandboxInventoryGauges,
202
+ recordInteractionOnlyLeaseGauges,
160
203
  recordSandboxOrphansTerminated,
204
+ recordSandboxCommandContainment,
205
+ recordSandboxIdleInteractionRelease,
206
+ recordSandboxProviderMissingBeforeCapture,
207
+ recordSandboxDockerWorkspaceRelease,
208
+ recordSandboxRecoveryObservationGauges,
161
209
  recordSandboxRotationBacklogGauges,
210
+ recordSandboxCheckpointStalenessGauges,
162
211
  recordTurnsQueuedGauge,
212
+ recordVerifiedSignupTrialDeploymentFlagGauge,
213
+ recordManagedAuthNewSignupsSwitchGauge,
214
+ recordVerifiedSignupTrialSwitchGauge,
163
215
  runtimeMetricsHooksForObservability,
164
216
  } from "../observability-metrics";
165
217
  import {
@@ -186,8 +238,7 @@ export type ReapSandboxLeasesResult = {
186
238
  skipped: number;
187
239
  /** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
188
240
  metered: number;
189
- /** Viewer-only boxes force-drained because their workspace is over a limit
190
- * (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
241
+ /** Holderless boxes accelerated to draining by the optional warm cap. */
191
242
  forceDrained: number;
192
243
  /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
193
244
  modalOrphansTerminated: number;
@@ -200,6 +251,10 @@ type DrainSandboxClient = {
200
251
  resume?: (state: unknown) => Promise<unknown>;
201
252
  deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
202
253
  delete?: (state: unknown) => Promise<void>;
254
+ attachWorkspaceForDrain?: (
255
+ state: unknown,
256
+ assertCurrentCapture: () => Promise<{ archivePublished: boolean }>,
257
+ ) => Promise<PersistableSession>;
203
258
  };
204
259
 
205
260
  /**
@@ -256,6 +311,8 @@ export type TerminateBoxFn = (
256
311
  diskBackedArchives?: boolean,
257
312
  releaseFailedCapture?: () => Promise<void>,
258
313
  workspaceId?: string,
314
+ beforeProviderStop?: () => Promise<void>,
315
+ beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,
259
316
  ) => Promise<boolean | ProviderTerminationOutcome>;
260
317
 
261
318
  export type SweepModalOrphansFn = (
@@ -275,6 +332,8 @@ export type SandboxLeaseActivityOptions = {
275
332
  /** Override the provider-side Modal orphan sweep (tests spy this; defaults to
276
333
  * Modal list+tag comparison when Modal is configured). */
277
334
  sweepModalOrphans?: SweepModalOrphansFn;
335
+ /** Only provider discovery is injected; durable recovery remains real. */
336
+ findModalProviderCreateReceipt?: typeof findModalProviderCreateReceipt;
278
337
  /** Override the read-only bounded OpenSandbox Kubernetes projection. */
279
338
  inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;
280
339
  /** Override only the read-only provider process probe. The canonical DB
@@ -286,8 +345,20 @@ export type SandboxLeaseActivityOptions = {
286
345
  /** Override the read-only exact-instance readiness probe used before the
287
346
  * reaper settles blockers for a provider that has definitively vanished. */
288
347
  probeDrainableProvider?: DrainableProviderProbeFn;
348
+ /** Override only the resume-by-id of the box an idle checkpoint captures.
349
+ * Returns the live session, or null when the provider no longer has it. The
350
+ * capture claim, publication and release remain real. */
351
+ resumeIdleCheckpointSession?: ResumeIdleCheckpointSessionFn;
352
+ /** Override only the provider client factory of the default drain. The
353
+ * claim, capture, publication and teardown ordering remain real. */
354
+ createClientForBackend?: CreateSandboxClientForBackendFn;
289
355
  };
290
356
 
357
+ export type ResumeIdleCheckpointSessionFn = (
358
+ settings: ActivityServices["settings"],
359
+ lease: LeaseSnapshot,
360
+ ) => Promise<unknown | null>;
361
+
291
362
  export type RetainedProcessProbeResult =
292
363
  | { status: "proved"; proof: RetainedProcessProviderProof }
293
364
  | {
@@ -308,6 +379,8 @@ export type RetainedProcessProbeResult =
308
379
  | "provider_binding_missing"
309
380
  | "provider_binding_mismatch"
310
381
  | "process_observation_unavailable";
382
+ /** A running command whose last read still returned a large backlog. */
383
+ outputBacklog?: boolean;
311
384
  };
312
385
 
313
386
  export type RetainedProcessProbeFn = (
@@ -322,8 +395,12 @@ export type RetainedProcessProbeFn = (
322
395
  streamFidelity?: "separate" | "merged",
323
396
  ) => Promise<void>,
324
397
  providerPersistence?: ProviderCommandPersistence,
398
+ drainBudget?: RetainedProcessDrainBudget,
325
399
  ) => Promise<RetainedProcessProbeResult>;
326
400
 
401
+ /** Shared wall-clock bound on output draining within one reconciliation sweep. */
402
+ export type RetainedProcessDrainBudget = { until: number };
403
+
327
404
  export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
328
405
 
329
406
  export type DrainableProviderProbeFn = (
@@ -334,6 +411,16 @@ export type DrainableProviderProbeFn = (
334
411
  export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
335
412
  export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
336
413
  export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
414
+ /** Bounds on reading a command's output backlog: per claim, and across one
415
+ * reconciliation sweep so the reaper activity stays well inside its timeout.
416
+ * Each read returns up to one provider page per stream. */
417
+ export const RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
418
+ export const RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 10_000;
419
+ export const RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 60_000;
420
+ /** A live process whose read returned at least this much output left a backlog
421
+ * behind it. A server or watcher that only trickles output stays on ordinary
422
+ * backoff; a process the provider reports exited is always read to EOF. */
423
+ export const RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
337
424
  export const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
338
425
  export const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;
339
426
  export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
@@ -364,6 +451,9 @@ export async function awaitProviderCaptureWithLatePublication<T>(input: {
364
451
  publishLate: (value: T) => Promise<void>;
365
452
  observeLateFailure: (error: unknown) => void;
366
453
  observeLatePublicationFailure: (error: unknown) => void;
454
+ /** Called once the detached late path has fully settled (after publication
455
+ * or failure handling), never on the in-budget path. */
456
+ onLateSettled?: () => void;
367
457
  }): Promise<T> {
368
458
  let timeout: ReturnType<typeof setTimeout> | undefined;
369
459
  const timeoutError = input.timeoutError;
@@ -381,9 +471,17 @@ export async function awaitProviderCaptureWithLatePublication<T>(input: {
381
471
  // durable archive publication: never provider teardown or a cold commit from
382
472
  // an activity Temporal has already rejected. A replaced claim fences this
383
473
  // callback; Modal candidates then flow to artifact-ledger GC.
474
+ const settled = () => {
475
+ try {
476
+ input.onLateSettled?.();
477
+ } catch {
478
+ // Settlement bookkeeping cannot surface as an unhandled rejection.
479
+ }
480
+ };
384
481
  void input.capture
385
482
  .then(input.publishLate, input.observeLateFailure)
386
- .catch(input.observeLatePublicationFailure);
483
+ .catch(input.observeLatePublicationFailure)
484
+ .then(settled, settled);
387
485
  throw error;
388
486
  } finally {
389
487
  if (timeout) clearTimeout(timeout);
@@ -425,6 +523,20 @@ export type SandboxLeaseSweepPlan = {
425
523
  rotationsRequested: number;
426
524
  };
427
525
 
526
+ export type SandboxIdleCheckpointActivityResult = {
527
+ status: "checkpointed" | "skipped";
528
+ reason?: string;
529
+ };
530
+
531
+ /** Children per sweep; the next sweep continues with the least recently tried
532
+ * boxes (each box is tried at most once per snapshot interval). Half the
533
+ * lifecycle worker's activity slots, so a burst of captures never queues the
534
+ * drains that stop idle boxes behind it. */
535
+ const SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(
536
+ 1,
537
+ Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2),
538
+ );
539
+
428
540
  export type SandboxLeaseSweepMaintenanceResult = {
429
541
  metered: number;
430
542
  forceDrained: number;
@@ -447,6 +559,55 @@ class SandboxDrainRecoveryDeferredError extends Error {
447
559
  readonly name = "SandboxDrainRecoveryDeferredError";
448
560
  }
449
561
 
562
+ /**
563
+ * Drain captures still physically running in this worker, keyed by exact lease
564
+ * epoch and provider instance. A capture that outlives its attempt's budget
565
+ * keeps running and may publish late through its claim's provider-request
566
+ * lineage. A parallel-read provider (Docker, for one) also lets the next
567
+ * attempt replace that unpublished claim at once and start its own full read,
568
+ * so a capture slower than the retry interval used to gain one more concurrent
569
+ * read and host spool per retry. This keeps one physical capture per lease
570
+ * epoch and instance in each worker; the durable claim still decides which
571
+ * archive publishes, and a capture started by another worker (or one that died
572
+ * with its process) is handled by the ordinary claim replacement. An entry
573
+ * older than the longest durable capture window no longer defers, so a
574
+ * provider call that never settles cannot pin the lease to this worker forever.
575
+ */
576
+ const settlingDrainCaptures = new Map<string, { startedAtMs: number }>();
577
+
578
+ type DrainCaptureIdentity = {
579
+ sandboxGroupId: string;
580
+ leaseEpoch: number;
581
+ instanceId: string | null;
582
+ };
583
+
584
+ function drainCaptureKey(lease: DrainCaptureIdentity & { instanceId: string }): string {
585
+ return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);
586
+ }
587
+
588
+ /** True while this worker still runs (or publishes, or removes the spool of)
589
+ * a drain capture of this exact lease epoch and instance. */
590
+ function sandboxDrainCaptureSettlingInProcess(
591
+ lease: DrainCaptureIdentity,
592
+ nowMs = Date.now(),
593
+ ): boolean {
594
+ if (!lease.instanceId) return false;
595
+ const entry = settlingDrainCaptures.get(
596
+ drainCaptureKey({ ...lease, instanceId: lease.instanceId }),
597
+ );
598
+ return entry !== undefined && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;
599
+ }
600
+
601
+ function trackSettlingDrainCapture(lease: DrainCaptureIdentity): () => void {
602
+ if (!lease.instanceId) return () => undefined;
603
+ const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });
604
+ const entry = { startedAtMs: Date.now() };
605
+ settlingDrainCaptures.set(key, entry);
606
+ return () => {
607
+ if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);
608
+ };
609
+ }
610
+
450
611
  /** Stable UUID per logical operation + Temporal attempt. Attempt one uses the
451
612
  * operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.
452
613
  * This makes the DB receipt idempotent even if an activity delivery is replayed. */
@@ -482,7 +643,14 @@ export function createSandboxLeaseActivities(
482
643
  options: SandboxLeaseActivityOptions = {},
483
644
  ) {
484
645
  const terminateBox: TerminateBoxFn =
485
- options.terminateBox ??
646
+ (options.terminateBox
647
+ ? async (...args: Parameters<TerminateBoxFn>) => {
648
+ // The injected test seam represents the stop itself; the real
649
+ // implementation invokes this only after capture has succeeded.
650
+ await args[10]?.();
651
+ return options.terminateBox!(...args);
652
+ }
653
+ : null) ??
486
654
  (async (
487
655
  settings,
488
656
  lease,
@@ -494,13 +662,15 @@ export function createSandboxLeaseActivities(
494
662
  diskBackedArchives,
495
663
  releaseFailedCapture,
496
664
  workspaceId,
665
+ beforeProviderStop,
666
+ beforeWorkspaceCapture,
497
667
  ) =>
498
668
  await terminateProviderBox(
499
669
  settings,
500
670
  lease,
501
671
  observability,
502
672
  persistArchive,
503
- createSandboxClientForBackend,
673
+ options.createClientForBackend ?? createSandboxClientForBackend,
504
674
  terminateModalSandboxById,
505
675
  providerCaptureRequestId,
506
676
  captureDisposition,
@@ -508,6 +678,8 @@ export function createSandboxLeaseActivities(
508
678
  diskBackedArchives,
509
679
  releaseFailedCapture,
510
680
  workspaceId,
681
+ beforeProviderStop,
682
+ beforeWorkspaceCapture,
511
683
  ));
512
684
  const sweepModalOrphans: SweepModalOrphansFn =
513
685
  options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
@@ -515,15 +687,35 @@ export function createSandboxLeaseActivities(
515
687
  options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
516
688
  const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
517
689
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
690
+ const resumeIdleCheckpointSession =
691
+ options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;
692
+ /** Embedded idle checkpoints still running, by exact lease epoch. */
693
+ const embeddedIdleCheckpoints = new Set<string>();
694
+ const endedEpochProbeBackoff: EndedEpochProbeBackoff = new Map();
518
695
  async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
519
696
  const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
520
697
  try {
521
- const { db, settings, observability } = await services();
698
+ const { db, settings, observability, objectStorage } = await services();
522
699
  const timing = sandboxDrainTiming(settings);
523
- const onUnobservableCommandDrainError = (error: unknown) => {
524
- observability.warn("sandbox reaper: unobservable command drain inspection failed", {
525
- error: error instanceof Error ? error.message : String(error),
526
- });
700
+ const commandContainment = {
701
+ // Browsers and desktops follow the general idle grace: unused for it,
702
+ // they stop holding the box (saved browsers resume on demand).
703
+ // Never under a minute, so a visible live view's 30 s heartbeat
704
+ // always lands inside the window.
705
+ idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 60_000),
706
+ idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),
707
+ onIdleInteractionRelease: (
708
+ outcome: Parameters<typeof recordSandboxIdleInteractionRelease>[1],
709
+ ) => recordSandboxIdleInteractionRelease(observability, outcome),
710
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
711
+ deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
712
+ onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>
713
+ recordSandboxCommandContainment(observability, outcome),
714
+ onCommandContainmentError: (error: unknown) => {
715
+ observability.warn("sandbox reaper: retained command containment inspection failed", {
716
+ error: error instanceof Error ? error.message : String(error),
717
+ });
718
+ },
527
719
  };
528
720
  if (!settings.sandboxOwnershipEnabled) {
529
721
  // Turns skip leases when the flag is off, but Computer/Browser attach
@@ -531,7 +723,7 @@ export function createSandboxLeaseActivities(
531
723
  // behind rotation_in_progress forever. Inventory and drain those rows;
532
724
  // do not request NEW deadline rotations or meter warm time.
533
725
  const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
534
- onUnobservableCommandDrainError,
726
+ ...commandContainment,
535
727
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
536
728
  turnHolderTtlMs: settings.sandboxLeaseTtlMs,
537
729
  interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -568,7 +760,7 @@ export function createSandboxLeaseActivities(
568
760
  // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
569
761
  // run only after every drainable box has its own durable child.
570
762
  const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
571
- onUnobservableCommandDrainError,
763
+ ...commandContainment,
572
764
  viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
573
765
  // Dead-worker turn holders: a live holder is touched every 10s from the
574
766
  // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -601,6 +793,141 @@ export function createSandboxLeaseActivities(
601
793
  }
602
794
  }
603
795
 
796
+ /** Inventory for the idle checkpoint sweep. DB only: warm boxes no turn
797
+ * holds, kept warm by viewers, interactions or running background commands,
798
+ * with writes no capture may cover yet and neither a capture attempt nor an
799
+ * idle checkpoint attempt within the snapshot interval.
800
+ * Least recently tried first; the next sweep continues past the batch. */
801
+ async function listIdleSandboxCheckpoints(): Promise<SandboxIdleCheckpointPlan> {
802
+ const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "idle_checkpoint_inventory" });
803
+ try {
804
+ const { db, settings } = await services();
805
+ // The idle capture claims the warm checkpoint budget, not the drain's;
806
+ // freeze the matching activity class with each child.
807
+ const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));
808
+ if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
809
+ return { targets: [], timeoutClass };
810
+ }
811
+ const rows = await listIdleCheckpointCandidates(db, {
812
+ limit: SANDBOX_IDLE_CHECKPOINT_BATCH,
813
+ intervalMs: settings.sandboxSnapshotIntervalMs,
814
+ });
815
+ return {
816
+ targets: rows.map(({ accountId: _accountId, ...target }) => target),
817
+ timeoutClass,
818
+ };
819
+ } finally {
820
+ stopHeartbeat();
821
+ }
822
+ }
823
+
824
+ /** Checkpoint one warm box between turns. Reuses the turn's warm checkpoint
825
+ * path with an idle owner: the exact capture claim requires that only
826
+ * viewers, interactions and active background commands hold the
827
+ * box and that no other request is open, then the native Modal snapshot runs
828
+ * around them and is published one generation behind the workspace. */
829
+ async function checkpointIdleSandboxLease(
830
+ input: SandboxIdleCheckpointActivityInput,
831
+ ): Promise<SandboxIdleCheckpointActivityResult> {
832
+ const target = input.target;
833
+ const stopHeartbeat = startSandboxReaperHeartbeat({
834
+ phase: "idle_checkpoint",
835
+ workspaceId: target.workspaceId,
836
+ sandboxGroupId: target.sandboxGroupId,
837
+ leaseEpoch: target.leaseEpoch,
838
+ instanceId: target.instanceId,
839
+ });
840
+ try {
841
+ const { db, settings, observability, objectStorage } = await services();
842
+ const fields = {
843
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
844
+ leaseEpoch: target.leaseEpoch,
845
+ };
846
+ const skipped = (reason: string): SandboxIdleCheckpointActivityResult => {
847
+ observability.info("sandbox reaper: idle checkpoint skipped", { ...fields, reason });
848
+ return { status: "skipped", reason };
849
+ };
850
+ if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
851
+ return skipped("disabled");
852
+ }
853
+ const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
854
+ if (
855
+ !lease ||
856
+ lease.liveness !== "warm" ||
857
+ lease.leaseEpoch !== target.leaseEpoch ||
858
+ lease.instanceId !== target.instanceId
859
+ ) {
860
+ return skipped("lease_fenced");
861
+ }
862
+ // A turn's own checkpoints cover a box it holds; a box nothing holds
863
+ // belongs to the idle drain.
864
+ if (lease.turnHolders > 0) return skipped("turn_held");
865
+ if (lease.refcount === 0) return skipped("drain_owned");
866
+ const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);
867
+ // Counted as tried whatever happens next, so a refused or failing box
868
+ // waits a full interval instead of taking a batch slot every sweep.
869
+ const stamped = await recordIdleCheckpointAttempt(db, {
870
+ accountId,
871
+ workspaceId: target.workspaceId,
872
+ sandboxGroupId: target.sandboxGroupId,
873
+ expectedEpoch: target.leaseEpoch,
874
+ expectedInstanceId: target.instanceId,
875
+ });
876
+ if (!stamped) return skipped("lease_fenced");
877
+ if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {
878
+ return skipped("capture_policy");
879
+ }
880
+ if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped("clean");
881
+ if (lease.archiveCapture) {
882
+ const tookOver = await releaseWorkspaceArchiveCapture(db, {
883
+ accountId,
884
+ workspaceId: target.workspaceId,
885
+ sandboxGroupId: target.sandboxGroupId,
886
+ captureId: lease.archiveCapture.id,
887
+ expectedEpoch: target.leaseEpoch,
888
+ expectedInstanceId: target.instanceId,
889
+ onlyExpiredWarmTakeover: true,
890
+ });
891
+ if (!tookOver) return skipped("capture_in_progress");
892
+ observability.warn("sandbox reaper: released an expired warm capture claim", {
893
+ ...fields,
894
+ captureId: lease.archiveCapture.id,
895
+ });
896
+ }
897
+ let session: unknown;
898
+ try {
899
+ session = await resumeIdleCheckpointSession(settings, lease);
900
+ } catch (error) {
901
+ observability.warn("sandbox reaper: idle checkpoint resume failed", {
902
+ ...fields,
903
+ error: error instanceof Error ? error.message : String(error),
904
+ });
905
+ return skipped("provider_error");
906
+ }
907
+ if (session === null) return skipped("provider_missing");
908
+ const capture = maybePersistWarmWorkspaceSnapshot(
909
+ { db, settings, objectStorage, observability },
910
+ {
911
+ accountId,
912
+ workspaceId: target.workspaceId,
913
+ sandboxGroupId: target.sandboxGroupId,
914
+ idleCheckpoint: true,
915
+ },
916
+ session,
917
+ target.leaseEpoch,
918
+ );
919
+ const persisted = await capture;
920
+ // Return only after the provider capture settled and its exact claim
921
+ // was released, so the activity never outlives its own gate unobserved.
922
+ await capture.settled;
923
+ if (!persisted) return skipped("not_captured");
924
+ observability.info("sandbox reaper: idle checkpoint published", fields);
925
+ return { status: "checkpointed" };
926
+ } finally {
927
+ stopHeartbeat();
928
+ }
929
+ }
930
+
604
931
  async function drainSandboxLease(
605
932
  input: SandboxDrainActivityInput,
606
933
  ): Promise<SandboxDrainActivityResult> {
@@ -612,7 +939,7 @@ export function createSandboxLeaseActivities(
612
939
  instanceId: input.target.instanceId,
613
940
  });
614
941
  try {
615
- const { db, settings, observability, objectStorage } = await services();
942
+ const { db, settings, observability, objectStorage, bus } = await services();
616
943
  assertSandboxDrainInputTiming(input);
617
944
  const drainSettings =
618
945
  settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
@@ -636,6 +963,7 @@ export function createSandboxLeaseActivities(
636
963
  probeDrainableProvider,
637
964
  captureAttempt,
638
965
  objectStorage,
966
+ bus,
639
967
  );
640
968
  return { status: drainedCold ? "terminated" : "skipped" };
641
969
  } catch (error) {
@@ -710,6 +1038,23 @@ export function createSandboxLeaseActivities(
710
1038
  // immutable launch locators must reconcile even when managed ownership is
711
1039
  // disabled, otherwise an exact runner exit/loss proof can remain stranded.
712
1040
  await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
1041
+ await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
1042
+
1043
+ // Disabling new lease ownership cannot strand a previously dispatched
1044
+ // operation. This only attributes a positive receipt; normal draining
1045
+ // already runs in both ownership modes.
1046
+ try {
1047
+ await reconcileModalProviderCreates(
1048
+ db,
1049
+ settings,
1050
+ observability,
1051
+ options.findModalProviderCreateReceipt,
1052
+ );
1053
+ } catch (error) {
1054
+ observability.warn("sandbox reaper: Modal create recovery inventory failed", {
1055
+ error: error instanceof Error ? error.message : String(error),
1056
+ });
1057
+ }
713
1058
 
714
1059
  if (!settings.sandboxOwnershipEnabled) {
715
1060
  // Inventory remains useful while provider ownership is intentionally
@@ -735,8 +1080,7 @@ export function createSandboxLeaseActivities(
735
1080
  metered = meterResult.accrued;
736
1081
  const forceDrainWorkspaceIds = new Set<string>();
737
1082
  if (
738
- settings.billingMode === "stripe" ||
739
- settings.usageLimitsMode === "managed" ||
1083
+ settings.sandboxWarmBillingMode === "credits" ||
740
1084
  settings.sandboxMaxWarmSecondsPerWorkspace > 0
741
1085
  ) {
742
1086
  for (const workspaceId of meterResult.workspaceIds) {
@@ -771,6 +1115,16 @@ export function createSandboxLeaseActivities(
771
1115
  service.bus,
772
1116
  );
773
1117
 
1118
+ // A request or PTY left on a Modal box whose lease epoch has ended can
1119
+ // never progress once that box is gone; settle it on exact proof.
1120
+ await settleEndedEpochBlockers(
1121
+ db,
1122
+ settings,
1123
+ observability,
1124
+ options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
1125
+ endedEpochProbeBackoff,
1126
+ );
1127
+
774
1128
  try {
775
1129
  modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
776
1130
  recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
@@ -865,6 +1219,30 @@ export function createSandboxLeaseActivities(
865
1219
  }),
866
1220
  ),
867
1221
  );
1222
+ // The embedded path has no idle checkpoint sweep workflow; run the same
1223
+ // inventory and per-box activity. Captures run detached so a provider
1224
+ // snapshot never holds up this sweep's drains; a box whose capture is
1225
+ // still running is not started twice. Failure defers to the next sweep.
1226
+ try {
1227
+ const { targets, timeoutClass } = await listIdleSandboxCheckpoints();
1228
+ for (const target of targets) {
1229
+ const key = sandboxIdleCheckpointWorkflowId(target);
1230
+ if (embeddedIdleCheckpoints.has(key)) continue;
1231
+ embeddedIdleCheckpoints.add(key);
1232
+ void checkpointIdleSandboxLease({ target, timeoutClass })
1233
+ .catch((error: unknown) => {
1234
+ service.observability.warn("sandbox reaper: idle checkpoint failed", {
1235
+ sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
1236
+ error: error instanceof Error ? error.message : String(error),
1237
+ });
1238
+ })
1239
+ .finally(() => embeddedIdleCheckpoints.delete(key));
1240
+ }
1241
+ } catch (error) {
1242
+ service.observability.warn("sandbox reaper: idle checkpoint inventory failed", {
1243
+ error: error instanceof Error ? error.message : String(error),
1244
+ });
1245
+ }
868
1246
  const terminated = outcomes.filter(
869
1247
  (outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated",
870
1248
  ).length;
@@ -889,13 +1267,63 @@ export function createSandboxLeaseActivities(
889
1267
  return {
890
1268
  prepareSandboxLeaseSweep,
891
1269
  drainSandboxLease,
1270
+ listIdleSandboxCheckpoints,
1271
+ checkpointIdleSandboxLease,
892
1272
  maintainSandboxLeaseSweep,
893
1273
  reapSandboxLeases,
894
1274
  };
895
1275
  }
896
1276
 
1277
+ /** Resume the exact warm box by id for an idle checkpoint, with no cold
1278
+ * restore and no replacement. Returns null when the provider no longer has
1279
+ * it; the ordinary loss paths own that box. The session is not closed: the
1280
+ * box keeps running its commands. */
1281
+ export async function resumeIdleCheckpointSessionAtProvider(
1282
+ settings: ActivityServices["settings"],
1283
+ lease: LeaseSnapshot,
1284
+ ): Promise<unknown | null> {
1285
+ const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
1286
+ const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
1287
+ if (backend !== "modal" || !lease.instanceId || !lease.resumeState) {
1288
+ throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);
1289
+ }
1290
+ if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {
1291
+ throw new Error("idle checkpoint lease envelope does not name its provider instance");
1292
+ }
1293
+ const client = createSandboxClientForBackend(backend as never, settings) as
1294
+ | RetainedProcessProbeClient
1295
+ | undefined;
1296
+ if (!client?.resume) {
1297
+ throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);
1298
+ }
1299
+ try {
1300
+ const envelopeSessionState =
1301
+ (lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
1302
+ const resumedState = await deserializeSandboxSessionStateEnvelope(
1303
+ client as never,
1304
+ envelopeSessionState,
1305
+ lease.instanceId,
1306
+ );
1307
+ if (resumedState === undefined) {
1308
+ throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
1309
+ }
1310
+ const resumed = await withRetainedProcessProbeTimeout(
1311
+ resumeExactSandboxSession(client, backend, resumedState, lease.instanceId),
1312
+ );
1313
+ return resumed.session;
1314
+ } catch (error) {
1315
+ if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
1316
+ throw new Error("idle checkpoint provider resume timed out", { cause: error });
1317
+ }
1318
+ if (isProviderSandboxNotFoundError(client.backendId, error)) return null;
1319
+ throw error;
1320
+ }
1321
+ }
1322
+
897
1323
  const CHECKPOINT_GC_LIMIT = 50;
898
1324
  const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
1325
+ /** The checkpoint artifact GC claim TTL, which the reaper's work check previews. */
1326
+ export const SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;
899
1327
  const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
900
1328
  const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
901
1329
  const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
@@ -910,6 +1338,9 @@ type ConnectedCommandReconciliationOutcome =
910
1338
  | "proof_checkpoint_failed"
911
1339
  | "settled_exited"
912
1340
  | "settled_lost"
1341
+ | "output_unavailable"
1342
+ | "output_ack_published"
1343
+ | "output_released"
913
1344
  | "settlement_failed"
914
1345
  | "defer_failed";
915
1346
 
@@ -924,6 +1355,65 @@ function recordConnectedCommandReconciliation(
924
1355
  });
925
1356
  }
926
1357
 
1358
+ /** Why a connected command's runner can never answer for it again. */
1359
+ type ConnectedRunnerRetirement =
1360
+ | "enrollment_revoked"
1361
+ | "connection_replaced"
1362
+ | "connection_stopped";
1363
+ type ConnectedRunnerLookups = Map<string, Promise<ConnectedRunnerRetirement | null>>;
1364
+
1365
+ /** Offline reasons after which the runner process has exited for good. */
1366
+ const CONNECTED_RUNNER_STOPPED_REASONS = new Set([
1367
+ "GOING_OFFLINE_REASON_USER_STOP",
1368
+ "GOING_OFFLINE_REASON_UPDATE",
1369
+ "GOING_OFFLINE_REASON_HOST_SHUTDOWN",
1370
+ ]);
1371
+
1372
+ /**
1373
+ * A connected command's operation lives in the memory of the runner process
1374
+ * that launched it and is addressed through that process's connection
1375
+ * instance. No responder on that exact address is not proof by itself: the
1376
+ * machine may be offline or reconnecting. It is proof once the enrollment
1377
+ * also retires that process, by the same rules the running-command claim
1378
+ * applies: the enrollment was revoked, a different runner process now holds
1379
+ * the connection, or the runner went offline by stopping, updating or shutting
1380
+ * down. Anything else, including a failed read, keeps the existing retry. One
1381
+ * read per runner address per sweep.
1382
+ */
1383
+ async function connectedCommandRunnerRetirement(
1384
+ db: ActivityServices["db"],
1385
+ claim: { controlWorkspaceId: string; enrollmentId: string; connectionInstanceId: string },
1386
+ error: unknown,
1387
+ lookups: ConnectedRunnerLookups,
1388
+ ): Promise<ConnectedRunnerRetirement | null> {
1389
+ if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {
1390
+ return null;
1391
+ }
1392
+ const key = JSON.stringify([
1393
+ claim.controlWorkspaceId,
1394
+ claim.enrollmentId,
1395
+ claim.connectionInstanceId,
1396
+ ]);
1397
+ let lookup = lookups.get(key);
1398
+ if (!lookup) {
1399
+ lookup = (async (): Promise<ConnectedRunnerRetirement | null> => {
1400
+ const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);
1401
+ if (!enrollment) return null;
1402
+ if (enrollment.status === "revoked") return "enrollment_revoked";
1403
+ if (enrollment.connectionInstanceId) {
1404
+ return enrollment.connectionInstanceId === claim.connectionInstanceId
1405
+ ? null
1406
+ : "connection_replaced";
1407
+ }
1408
+ return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? "")
1409
+ ? "connection_stopped"
1410
+ : null;
1411
+ })().catch(() => null);
1412
+ lookups.set(key, lookup);
1413
+ }
1414
+ return await lookup;
1415
+ }
1416
+
927
1417
  async function forEachWithConcurrency<T>(
928
1418
  values: readonly T[],
929
1419
  concurrency: number,
@@ -1110,6 +1600,7 @@ export async function reconcileConnectedMachineBackgroundCommands(
1110
1600
  );
1111
1601
  await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1112
1602
  let proof = claim.proof;
1603
+ let outputReplay: Awaited<ReturnType<OpStreamExecClient["readExisting"]>> | null = null;
1113
1604
  const connectionKey = JSON.stringify([
1114
1605
  claim.controlWorkspaceId,
1115
1606
  claim.enrollmentId,
@@ -1167,7 +1658,7 @@ export async function reconcileConnectedMachineBackgroundCommands(
1167
1658
  retryClock: defaultSelfhostedRetryClock,
1168
1659
  journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
1169
1660
  });
1170
- await replayConnectedCommandOutput(
1661
+ outputReplay = await replayConnectedCommandOutput(
1171
1662
  client,
1172
1663
  claim.opId,
1173
1664
  Boolean(proof),
@@ -1229,6 +1720,18 @@ export async function reconcileConnectedMachineBackgroundCommands(
1229
1720
  if (!settlement.settled) {
1230
1721
  throw new Error("Connected command settlement lost its exact claim or proof");
1231
1722
  }
1723
+ const receipt =
1724
+ outputReplay?.outputReceipt ??
1725
+ (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : undefined);
1726
+ const terminal =
1727
+ outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
1728
+ if (proof.outcome === "exited" && receipt && terminal) {
1729
+ await recordConnectedCommandOutputConsumption(db, {
1730
+ ...claim,
1731
+ receipt,
1732
+ exitCode: terminal.outcome.response.exitCode,
1733
+ });
1734
+ }
1232
1735
  if (settlement.events.length > 0) {
1233
1736
  try {
1234
1737
  await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
@@ -1258,21 +1761,19 @@ export async function replayConnectedCommandOutput(
1258
1761
  opId: string,
1259
1762
  terminalKnown: boolean,
1260
1763
  capture: Parameters<OpStreamExecClient["readExisting"]>[2],
1261
- ): Promise<void> {
1764
+ ): Promise<Awaited<ReturnType<OpStreamExecClient["readExisting"]>>> {
1262
1765
  // A completed operation can need several bounded reads to replay its retained
1263
1766
  // frames. Keep the reader's integrity checkpoint until the exit is verified;
1264
1767
  // replacing it after each partial read would restart forever at frame one.
1265
1768
  let capturedThrough = 0n;
1266
1769
  while (true) {
1267
1770
  const before = capturedThrough;
1268
- const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, async (frames) => {
1269
- await capture(frames);
1270
- for (const frame of frames) {
1271
- const sequence = BigInt(frame.sequence);
1272
- if (sequence > capturedThrough) capturedThrough = sequence;
1273
- }
1274
- });
1275
- if (!terminalKnown || replay.status === "completed" || replay.terminal) return;
1771
+ const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, capture);
1772
+ if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
1773
+ // Quiet jobs can retain many heartbeat frames (or an incomplete UTF-8
1774
+ // chunk). These are real replay progress even when capture receives no
1775
+ // stdout/stderr. Use the reader's verified, contiguous protocol frontier.
1776
+ capturedThrough = BigInt(replay.replaySequence);
1276
1777
  // This is a finite terminal drain, not a poller for a running command.
1277
1778
  // Failed persistence, transport/integrity errors, and no-progress reads
1278
1779
  // return control to normal reconciliation without licensing settlement.
@@ -1282,6 +1783,159 @@ export async function replayConnectedCommandOutput(
1282
1783
  }
1283
1784
  }
1284
1785
 
1786
+ /** Drain the durable terminal-output obligation independently of the launching
1787
+ * attempt or model completion notice. Existing terminal rows receive no inferred
1788
+ * receipt: they must replay and capture all retained bytes before any final ACK. */
1789
+ export async function reconcileConnectedCommandOutputReleases(
1790
+ db: ActivityServices["db"],
1791
+ settings: ActivityServices["settings"],
1792
+ observability: ActivityServices["observability"],
1793
+ bus: ActivityServices["bus"],
1794
+ controlRpcOverride?: ControlRpc,
1795
+ ): Promise<void> {
1796
+ const dueBefore = new Date();
1797
+ const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
1798
+ const controlRpc =
1799
+ controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
1800
+ const runnerLookups: ConnectedRunnerLookups = new Map();
1801
+ do {
1802
+ let claims: Awaited<ReturnType<typeof claimConnectedCommandOutputReleases>>;
1803
+ try {
1804
+ claims = await claimConnectedCommandOutputReleases(db, {
1805
+ claimId: crypto.randomUUID(),
1806
+ dueBefore,
1807
+ limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
1808
+ claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
1809
+ });
1810
+ } catch (error) {
1811
+ recordConnectedCommandReconciliation(observability, "claim_failed");
1812
+ observability.warn("sandbox reaper: connected-command output claim failed", {
1813
+ error: error instanceof Error ? error.message : String(error),
1814
+ });
1815
+ return;
1816
+ }
1817
+ await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
1818
+ let receipt = claim.receipt;
1819
+ try {
1820
+ if (!bus.getOpStreamConnection)
1821
+ throw new Error("Connected command output transport unavailable");
1822
+ const rpcSubject = subjectFor(
1823
+ claim.controlWorkspaceId,
1824
+ claim.enrollmentId,
1825
+ claim.connectionInstanceId,
1826
+ );
1827
+ const controlTimeoutMs = Math.min(
1828
+ settings.sandboxSelfhostedControlTimeoutMs,
1829
+ RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
1830
+ );
1831
+ const client = new OpStreamExecClient({
1832
+ workspaceId: claim.controlWorkspaceId,
1833
+ agentId: claim.enrollmentId,
1834
+ connectionInstanceId: claim.connectionInstanceId,
1835
+ epoch: 0,
1836
+ controlRpc,
1837
+ rpcSubject,
1838
+ transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
1839
+ controlTimeoutMs,
1840
+ retryClock: defaultSelfhostedRetryClock,
1841
+ journal: {
1842
+ attachGeneration: () => String(Date.now()),
1843
+ persistSettled: () => {
1844
+ throw new Error("Background output release requires its database receipt");
1845
+ },
1846
+ },
1847
+ });
1848
+ if (!receipt) {
1849
+ const status = await querySelfhostedOp({
1850
+ controlRpc,
1851
+ rpcSubject,
1852
+ opId: claim.opId,
1853
+ controlTimeoutMs,
1854
+ });
1855
+ if (status.state === OpState.OP_STATE_LOST) {
1856
+ if (
1857
+ !(await settleConnectedCommandOutputReleaseClaim(db, {
1858
+ claim,
1859
+ outcome: "unavailable",
1860
+ retryAfterMs: 0,
1861
+ }))
1862
+ )
1863
+ throw new Error("Connected command output loss no longer owns its exact claim");
1864
+ recordConnectedCommandReconciliation(observability, "output_unavailable");
1865
+ return;
1866
+ }
1867
+ if (status.state !== OpState.OP_STATE_COMPLETE)
1868
+ throw new Error("Terminal command output returned a live operation");
1869
+ const replay = await replayConnectedCommandOutput(
1870
+ client,
1871
+ claim.opId,
1872
+ true,
1873
+ async (frames) =>
1874
+ captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames),
1875
+ );
1876
+ const terminal = replay.status === "completed" ? replay : replay.terminal;
1877
+ receipt =
1878
+ replay.outputReceipt ??
1879
+ (replay.status === "running" ? replay.terminal?.outputReceipt : undefined) ??
1880
+ null;
1881
+ if (!terminal || !receipt)
1882
+ throw new Error("Connected command output replay has no complete custody receipt");
1883
+ await recordConnectedCommandOutputConsumption(db, {
1884
+ ...claim,
1885
+ receipt,
1886
+ exitCode: terminal.outcome.response.exitCode,
1887
+ });
1888
+ }
1889
+ const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
1890
+ if (
1891
+ !(await settleConnectedCommandOutputReleaseClaim(db, {
1892
+ claim,
1893
+ outcome,
1894
+ retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0,
1895
+ }))
1896
+ )
1897
+ throw new Error("Connected command output release no longer owns its exact claim");
1898
+ recordConnectedCommandReconciliation(
1899
+ observability,
1900
+ outcome === "published" ? "output_ack_published" : "output_released",
1901
+ );
1902
+ } catch (error) {
1903
+ const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);
1904
+ // A retired runner retains nothing: captured output is released, and
1905
+ // output it never handed over is recorded as missing.
1906
+ if (
1907
+ retired &&
1908
+ (await settleConnectedCommandOutputReleaseClaim(db, {
1909
+ claim,
1910
+ outcome: receipt ? "not_retained" : "unavailable",
1911
+ retryAfterMs: 0,
1912
+ }).catch(() => false))
1913
+ ) {
1914
+ recordConnectedCommandReconciliation(
1915
+ observability,
1916
+ receipt ? "output_released" : "output_unavailable",
1917
+ );
1918
+ return;
1919
+ }
1920
+ await settleConnectedCommandOutputReleaseClaim(db, {
1921
+ claim,
1922
+ outcome: "retry",
1923
+ retryAfterMs: Math.min(
1924
+ 5 * 60_000,
1925
+ Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) *
1926
+ 2 ** Math.min(4, claim.reconcileAttempts - 1),
1927
+ ),
1928
+ }).catch(() => false);
1929
+ observability.warn("sandbox reaper: connected-command output release deferred", {
1930
+ commandId: claim.commandId,
1931
+ error: error instanceof Error ? error.message : String(error),
1932
+ });
1933
+ }
1934
+ });
1935
+ if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
1936
+ } while (Date.now() < deadline);
1937
+ }
1938
+
1285
1939
  export async function probeConnectedMachineBackgroundCommand(
1286
1940
  claim: ConnectedMachineBackgroundCommandClaim,
1287
1941
  controlRpc: ControlRpc,
@@ -1385,6 +2039,195 @@ async function deferConnectedCommandClaim(
1385
2039
  }
1386
2040
  }
1387
2041
 
2042
+ /** Tuples listed per sweep (random order, the next sweep lists again) and the
2043
+ * share of the reaper tick the probes may use, so the pass never crowds out
2044
+ * the orphan sweep and GC after it. A box the provider cannot yet prove gone is
2045
+ * re-probed on an exponential backoff instead of on every tick. */
2046
+ const ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;
2047
+ const ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 60_000;
2048
+ const ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 60_000;
2049
+ const ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 60_000;
2050
+
2051
+ /** Per worker process: when each historical box may be probed again. */
2052
+ export type EndedEpochProbeBackoff = Map<string, { failures: number; nextProbeAtMs: number }>;
2053
+
2054
+ /**
2055
+ * Settle open requests still recorded on a Modal box whose lease
2056
+ * epoch has ended (a legacy loss, or a cold commit that left them). Lease
2057
+ * succession alone proves nothing about the old box, so each exact tuple is
2058
+ * settled only after its historical sandbox is observed terminated. NotFound
2059
+ * is ambiguous here (deletion or a rotated Modal credential workspace) and,
2060
+ * like a running or unreachable box, keeps the rows for a later probe (with
2061
+ * backoff) or an operator.
2062
+ *
2063
+ * The same observation also ends a turn's wait on sandbox setup whose start
2064
+ * outcome is unknown: once the exact box all its setup ran on is terminated,
2065
+ * nothing it may have started can still run, so the turn's setup reruns its
2066
+ * idempotent preparation (`reconcileCompletedSandboxSetup`). A start that
2067
+ * failed before any process was recorded otherwise had nothing to exit and
2068
+ * stayed recovering forever.
2069
+ */
2070
+ async function settleEndedEpochBlockers(
2071
+ db: ActivityServices["db"],
2072
+ settings: ActivityServices["settings"],
2073
+ observability: ActivityServices["observability"],
2074
+ inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
2075
+ backoff: EndedEpochProbeBackoff,
2076
+ ): Promise<void> {
2077
+ // Same scope as the Modal orphan sweep: no Modal API calls from workers of
2078
+ // a deployment that never configured Modal.
2079
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
2080
+ return;
2081
+ }
2082
+ const startedAtMs = Date.now();
2083
+ // One observation per box: several tuples and turns can name the same box.
2084
+ const observed = new Map<string, string>();
2085
+ const kept: Record<string, number> = {};
2086
+ /** The box's status, or null when it is backing off or the budget is spent. */
2087
+ async function observe(
2088
+ instanceId: string,
2089
+ details: Record<string, unknown>,
2090
+ ): Promise<string | null> {
2091
+ const known = observed.get(instanceId);
2092
+ if (known !== undefined) return known;
2093
+ const nowMs = Date.now();
2094
+ if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;
2095
+ if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;
2096
+ let status: string;
2097
+ try {
2098
+ status = (
2099
+ await withRetainedProcessProbeTimeout(
2100
+ inspectHistoricalModalSandbox(settings, instanceId, null),
2101
+ )
2102
+ ).status;
2103
+ } catch (error) {
2104
+ status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "error";
2105
+ observability.warn("sandbox reaper: ended-epoch box probe failed", {
2106
+ ...details,
2107
+ error:
2108
+ error === RETAINED_PROCESS_PROBE_TIMEOUT
2109
+ ? "provider_timeout"
2110
+ : error instanceof Error
2111
+ ? error.message
2112
+ : String(error),
2113
+ });
2114
+ }
2115
+ observed.set(instanceId, status);
2116
+ if (status === "terminated") {
2117
+ backoff.delete(instanceId);
2118
+ } else {
2119
+ deferEndedEpochProbe(backoff, instanceId);
2120
+ kept[status] = (kept[status] ?? 0) + 1;
2121
+ }
2122
+ return status;
2123
+ }
2124
+
2125
+ let tuples: Awaited<ReturnType<typeof listEndedEpochWorkspaceBlockers>>;
2126
+ try {
2127
+ tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });
2128
+ } catch (error) {
2129
+ observability.warn("sandbox reaper: ended-epoch blocker inventory failed", {
2130
+ error: error instanceof Error ? error.message : String(error),
2131
+ });
2132
+ tuples = [];
2133
+ }
2134
+ for (const tuple of tuples) {
2135
+ const instanceId = tuple.lostInstanceId;
2136
+ const details = {
2137
+ workspaceId: tuple.workspaceId,
2138
+ sandboxGroupId: tuple.sandboxGroupId,
2139
+ lostEpoch: tuple.lostEpoch,
2140
+ instanceId,
2141
+ };
2142
+ try {
2143
+ const status = await observe(instanceId, details);
2144
+ if (status !== "terminated") continue;
2145
+ const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);
2146
+ if (settled) {
2147
+ observability.info("sandbox reaper: settled blockers of an ended lease epoch", {
2148
+ ...details,
2149
+ ...settled,
2150
+ });
2151
+ }
2152
+ } catch (error) {
2153
+ observability.warn("sandbox reaper: ended-epoch blocker settlement failed", {
2154
+ ...details,
2155
+ error: error instanceof Error ? error.message : String(error),
2156
+ });
2157
+ }
2158
+ }
2159
+
2160
+ let setups: Awaited<ReturnType<typeof listEndedEpochSandboxSetupUncertainties>>;
2161
+ try {
2162
+ setups = await listEndedEpochSandboxSetupUncertainties(db, {
2163
+ limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT,
2164
+ });
2165
+ } catch (error) {
2166
+ observability.warn("sandbox reaper: ended-epoch setup inventory failed", {
2167
+ error: error instanceof Error ? error.message : String(error),
2168
+ });
2169
+ setups = [];
2170
+ }
2171
+ for (const setup of setups) {
2172
+ const instanceId = setup.box.providerInstanceId;
2173
+ // A turn whose reconciliation is refused on a terminated box (for example
2174
+ // an ancestor is paused) backs off by turn, so it never re-probes per tick.
2175
+ const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;
2176
+ if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;
2177
+ const details = {
2178
+ workspaceId: setup.workspaceId,
2179
+ sessionId: setup.sessionId,
2180
+ turnId: setup.turnId,
2181
+ sandboxGroupId: setup.sandboxGroupId,
2182
+ lostEpoch: setup.box.leaseEpoch,
2183
+ instanceId,
2184
+ };
2185
+ try {
2186
+ const status = await observe(instanceId, details);
2187
+ if (status !== "terminated") continue;
2188
+ const { reconciled } = await reconcileCompletedSandboxSetup(db, {
2189
+ accountId: setup.accountId,
2190
+ workspaceId: setup.workspaceId,
2191
+ sessionId: setup.sessionId,
2192
+ turnId: setup.turnId,
2193
+ attemptId: setup.attemptId,
2194
+ terminatedBox: setup.box,
2195
+ });
2196
+ if (reconciled) {
2197
+ backoff.delete(turnKey);
2198
+ observability.info(
2199
+ "sandbox reaper: resumed a turn whose unknown setup ran on a terminated box",
2200
+ details,
2201
+ );
2202
+ } else {
2203
+ deferEndedEpochProbe(backoff, turnKey);
2204
+ }
2205
+ } catch (error) {
2206
+ deferEndedEpochProbe(backoff, turnKey);
2207
+ observability.warn("sandbox reaper: ended-epoch setup reconciliation failed", {
2208
+ ...details,
2209
+ error: error instanceof Error ? error.message : String(error),
2210
+ });
2211
+ }
2212
+ }
2213
+ if (Object.keys(kept).length) {
2214
+ observability.info("sandbox reaper: ended-epoch blockers kept without terminal proof", kept);
2215
+ }
2216
+ }
2217
+
2218
+ function deferEndedEpochProbe(backoff: EndedEpochProbeBackoff, key: string): void {
2219
+ const failures = (backoff.get(key)?.failures ?? 0) + 1;
2220
+ backoff.set(key, {
2221
+ failures,
2222
+ nextProbeAtMs:
2223
+ Date.now() +
2224
+ Math.min(
2225
+ ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,
2226
+ ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16),
2227
+ ),
2228
+ });
2229
+ }
2230
+
1388
2231
  async function reconcileTerminalRetainedProcesses(
1389
2232
  db: ActivityServices["db"],
1390
2233
  settings: ActivityServices["settings"],
@@ -1409,16 +2252,53 @@ async function reconcileTerminalRetainedProcesses(
1409
2252
  return;
1410
2253
  }
1411
2254
 
2255
+ const drainBudget: RetainedProcessDrainBudget = {
2256
+ until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,
2257
+ };
2258
+ // Boxes already retired as lost in this batch: their other commands were
2259
+ // settled by that same transaction and need no probe of their own.
2260
+ const lostBoxes = new Set<string>();
1412
2261
  for (const claim of claims) {
1413
2262
  let process = claim.process;
2263
+ if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))
2264
+ continue;
1414
2265
  const expected = retainedProcessSettlementIdentity(process);
1415
2266
  let proof = retainedProcessReconciliationProof(process);
2267
+ // The exact current box itself answered NotFound to this probe: that is
2268
+ // loss proof for the whole box, not just this command.
2269
+ let wholeBoxGone = false;
1416
2270
  const processScope = {
1417
2271
  accountId: process.accountId,
1418
2272
  workspaceId: process.workspaceId,
1419
2273
  sessionId: process.sessionId,
1420
2274
  processId: process.id,
1421
2275
  };
2276
+ // Recover a yield whose observation failed before normal receipt adoption.
2277
+ // Do this before provider I/O: an unavailable observer must not fence every
2278
+ // subsequent turn. Physical retention and unknown outcome stay unchanged.
2279
+ if (
2280
+ process.ownerActorKind === "turn" &&
2281
+ process.providerBackend === "modal" &&
2282
+ !proof &&
2283
+ !claim.ownerState.startsWith("background_")
2284
+ ) {
2285
+ try {
2286
+ const command = await recoverManagedSessionBackgroundCommand(db, {
2287
+ ...processScope,
2288
+ expected,
2289
+ reconciliationClaimId: claim.claimId,
2290
+ });
2291
+ if (command) {
2292
+ claim.ownerState = `background_${command.state}`;
2293
+ recordRetainedProcessReconciliation(observability, "background_recovered");
2294
+ }
2295
+ } catch (error) {
2296
+ observability.warn("sandbox reaper: retained-command background recovery failed", {
2297
+ processId: process.id,
2298
+ error: error instanceof Error ? error.message : String(error),
2299
+ });
2300
+ }
2301
+ }
1422
2302
  const storedCommandPersistence = retainedProviderCommandPersistence(
1423
2303
  db,
1424
2304
  processScope,
@@ -1507,6 +2387,25 @@ async function reconcileTerminalRetainedProcesses(
1507
2387
  }
1508
2388
  }
1509
2389
 
2390
+ if (
2391
+ observation?.status === "deferred" &&
2392
+ process.providerBackend === "modal" &&
2393
+ !process.providerBindingKey &&
2394
+ !process.deadlineCancellationRequestedAt
2395
+ ) {
2396
+ // An unbound historical command cannot be signalled safely. Still
2397
+ // record deadline intent when its lease matches: a failed observer
2398
+ // must not block capture until the provider expires.
2399
+ try {
2400
+ await requestRetainedProcessDeadlineCancellation(db, processScope);
2401
+ } catch (error) {
2402
+ observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
2403
+ processId: process.id,
2404
+ error: error instanceof Error ? error.message : String(error),
2405
+ });
2406
+ }
2407
+ }
2408
+
1510
2409
  if (observation === null) {
1511
2410
  const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
1512
2411
  () => null,
@@ -1583,11 +2482,19 @@ async function reconcileTerminalRetainedProcesses(
1583
2482
  );
1584
2483
  supervisionMetric("cancellation_intent");
1585
2484
  }
1586
- observation = await probe(
2485
+ const legacyDeadlineStop =
2486
+ lease!.rotationReason === "provider_deadline" &&
2487
+ !supervised &&
2488
+ !process.deadlineCancellationRequestedAt;
2489
+ const probeOutcome = await probe(
1587
2490
  settings,
1588
2491
  lease!,
1589
2492
  process,
1590
- claim.ownerState === "background_stopping" ? "cancel" : "observe",
2493
+ (claim.ownerState === "background_stopping" &&
2494
+ !process.deadlineCancellationRequestedAt) ||
2495
+ legacyDeadlineStop
2496
+ ? "cancel"
2497
+ : "observe",
1591
2498
  async (result, chunkId, stream, streamFidelity) => {
1592
2499
  if (
1593
2500
  typeof result !== "string" ||
@@ -1610,7 +2517,25 @@ async function reconcileTerminalRetainedProcesses(
1610
2517
  .catch(() => undefined);
1611
2518
  },
1612
2519
  commandPersistence,
2520
+ drainBudget,
2521
+ ).then(
2522
+ (value) => ({ ok: true as const, value }),
2523
+ (error: unknown) => ({ ok: false as const, error }),
1613
2524
  );
2525
+ // Legacy PTY input reserves Ctrl-C before the cancellation fence.
2526
+ // Non-PTY has no command signal in the Modal SDK: probe it without
2527
+ // injecting Ctrl-C as data, then fence and save after the grace.
2528
+ if (
2529
+ legacyDeadlineStop &&
2530
+ !(await requestRetainedProcessDeadlineCancellation(db, processScope))
2531
+ )
2532
+ throw new Error("Deadline cancellation no longer owns its rotating lease");
2533
+ if (!probeOutcome.ok) throw probeOutcome.error;
2534
+ observation = probeOutcome.value;
2535
+ wholeBoxGone =
2536
+ observation.status === "proved" &&
2537
+ observation.proof.outcome === "lost" &&
2538
+ observation.proof.reason === "provider_instance_not_found";
1614
2539
  if (supervised) {
1615
2540
  supervisionMetric(
1616
2541
  (await commandPersistence.loadSupervisionReceipt())
@@ -1665,6 +2590,7 @@ async function reconcileTerminalRetainedProcesses(
1665
2590
  expected,
1666
2591
  claim.claimId,
1667
2592
  observation.reason,
2593
+ observation.outputBacklog === true,
1668
2594
  );
1669
2595
  continue;
1670
2596
  }
@@ -1687,6 +2613,79 @@ async function reconcileTerminalRetainedProcesses(
1687
2613
  }
1688
2614
  }
1689
2615
 
2616
+ if (wholeBoxGone) {
2617
+ // Settle every blocker of the exact lost box in one transaction: all its
2618
+ // commands, open requests, PTYs and process holders, then cold the lease
2619
+ // and wake waiters. Settling one command per probe took a 20-per-sweep
2620
+ // queue 11 minutes for 35 commands in staging session 5040c525 and kept
2621
+ // the session waiting on a box that was already gone.
2622
+ // Commands of this box that a closed turn never adopted get their
2623
+ // session background record first, exactly as their own claim would
2624
+ // have, so the agent hears about them when the box settles below.
2625
+ const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;
2626
+ for (const other of claims) {
2627
+ const sibling = other.process;
2628
+ if (
2629
+ other === claim ||
2630
+ `${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey ||
2631
+ sibling.ownerActorKind !== "turn" ||
2632
+ sibling.providerBackend !== "modal" ||
2633
+ other.ownerState.startsWith("background_") ||
2634
+ retainedProcessReconciliationProof(sibling)
2635
+ )
2636
+ continue;
2637
+ try {
2638
+ await recoverManagedSessionBackgroundCommand(db, {
2639
+ accountId: sibling.accountId,
2640
+ workspaceId: sibling.workspaceId,
2641
+ sessionId: sibling.sessionId,
2642
+ processId: sibling.id,
2643
+ expected: retainedProcessSettlementIdentity(sibling),
2644
+ reconciliationClaimId: other.claimId,
2645
+ });
2646
+ } catch (error) {
2647
+ observability.warn("sandbox reaper: retained-command background recovery failed", {
2648
+ processId: sibling.id,
2649
+ error: error instanceof Error ? error.message : String(error),
2650
+ });
2651
+ }
2652
+ }
2653
+ try {
2654
+ const marked = await markWarmLeaseInstanceLost(db, {
2655
+ accountId: process.accountId,
2656
+ workspaceId: process.workspaceId,
2657
+ sandboxGroupId: process.sandboxGroupId,
2658
+ expectedEpoch: process.leaseEpoch,
2659
+ expectedInstanceId: process.providerInstanceId,
2660
+ expectedBackend: process.providerBackend,
2661
+ });
2662
+ if (marked.status === "marked") {
2663
+ lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);
2664
+ recordRetainedProcessReconciliation(observability, "provider_lost_whole_box");
2665
+ if (marked.backgroundCommandEvents?.length) {
2666
+ await publishDurableSessionEvents(
2667
+ bus,
2668
+ process.workspaceId,
2669
+ marked.backgroundCommandEvents,
2670
+ (error) => {
2671
+ observability.warn("sandbox reaper: lost-box command event fanout failed", {
2672
+ sandboxGroupId: process.sandboxGroupId,
2673
+ error: error instanceof Error ? error.message : String(error),
2674
+ });
2675
+ },
2676
+ );
2677
+ }
2678
+ continue;
2679
+ }
2680
+ } catch (error) {
2681
+ observability.warn("sandbox reaper: whole-box loss settlement failed", {
2682
+ processId: process.id,
2683
+ error: error instanceof Error ? error.message : String(error),
2684
+ });
2685
+ }
2686
+ // Stale (a draining drain or another observer owns the box): fall back
2687
+ // to this command's own exact settlement below.
2688
+ }
1690
2689
  try {
1691
2690
  const settlement = await settleRetainedProcess(db, {
1692
2691
  accountId: process.accountId,
@@ -1745,9 +2744,17 @@ async function deferRetainedProcessClaim(
1745
2744
  expected: ReturnType<typeof retainedProcessSettlementIdentity>,
1746
2745
  claimId: string,
1747
2746
  outcome: RetainedProcessDeferralOutcome,
2747
+ outputBacklog = false,
1748
2748
  ): Promise<void> {
1749
- const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
2749
+ const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
1750
2750
  try {
2751
+ // A healthy long-lived command can already be on a five-minute backoff
2752
+ // when rotation becomes due. Wake it at the lead boundary, then at the
2753
+ // reaper cadence, so observation does not consume the capture window.
2754
+ const lease =
2755
+ process.providerBackend === "modal" && process.routeTargetId === null
2756
+ ? await readLease(db, process.workspaceId, process.sandboxGroupId)
2757
+ : null;
1751
2758
  await deferRetainedProcessReconciliation(db, {
1752
2759
  accountId: process.accountId,
1753
2760
  workspaceId: process.workspaceId,
@@ -1756,7 +2763,7 @@ async function deferRetainedProcessClaim(
1756
2763
  expected,
1757
2764
  claimId,
1758
2765
  outcome: deferral.durableOutcome,
1759
- retryAfterMs: deferral.retryAfterMs,
2766
+ retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs),
1760
2767
  });
1761
2768
  recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
1762
2769
  } catch (error) {
@@ -1776,6 +2783,7 @@ export function retainedProcessReconciliationDeferral(
1776
2783
  settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">,
1777
2784
  process: Pick<SandboxRetainedProcess, "reconcileAttempts">,
1778
2785
  outcome: RetainedProcessDeferralOutcome,
2786
+ outputBacklog = false,
1779
2787
  ): {
1780
2788
  durableOutcome: string;
1781
2789
  metricOutcome: RetainedProcessReconciliationOutcome;
@@ -1800,11 +2808,14 @@ export function retainedProcessReconciliationDeferral(
1800
2808
  quarantined: true,
1801
2809
  };
1802
2810
  }
1803
- const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
1804
- const retryAfterMs = Math.min(
1805
- 5 * 60_000,
1806
- Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
1807
- );
2811
+ const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000);
2812
+ // A backlog still being drained is progress, not an unhealthy provider:
2813
+ // come back at the reaper cadence, without backoff.
2814
+ const exponent =
2815
+ outcome === "provider_running" && outputBacklog
2816
+ ? 0
2817
+ : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
2818
+ const retryAfterMs = Math.min(5 * 60_000, baseRetryMs * 2 ** exponent);
1808
2819
  return {
1809
2820
  durableOutcome: outcome,
1810
2821
  metricOutcome: outcome,
@@ -1831,6 +2842,77 @@ type RetainedProcessProbeSession = ProviderCommandSession & {
1831
2842
 
1832
2843
  const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
1833
2844
 
2845
+ /** A command's exit is reported only after both output streams reach EOF, so a
2846
+ * finished command with a large backlog looks running until it is read. While a
2847
+ * backlog remains, keep reading within the per-claim and per-sweep budgets, so
2848
+ * it settles in this claim instead of one page per backoff step. Every page,
2849
+ * including `first`, is captured here. */
2850
+ export async function drainRetainedCommandBacklog(
2851
+ session: RetainedProcessProbeSession,
2852
+ providerSessionId: number,
2853
+ first: unknown,
2854
+ capturePage: (value: unknown) => Promise<void>,
2855
+ budget?: RetainedProcessDrainBudget,
2856
+ ): Promise<RetainedProcessProbeResult> {
2857
+ // Measure before capture: capturing a byte-offset page consumes its receipt.
2858
+ let backlog = retainedBacklogRemains(session, first);
2859
+ await capturePage(first);
2860
+ let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
2861
+ if (typeof session.writeStdin !== "function") return observation;
2862
+ const until = Math.min(
2863
+ Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
2864
+ budget?.until ?? Number.POSITIVE_INFINITY,
2865
+ );
2866
+ for (
2867
+ let reads = 0;
2868
+ backlog &&
2869
+ observation.status === "deferred" &&
2870
+ observation.reason === "provider_running" &&
2871
+ reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS &&
2872
+ Date.now() < until;
2873
+ reads++
2874
+ ) {
2875
+ let next: unknown;
2876
+ try {
2877
+ next = await withRetainedProcessProbeTimeout(
2878
+ session.writeStdin({
2879
+ sessionId: providerSessionId,
2880
+ chars: "",
2881
+ yieldTimeMs: 1_000,
2882
+ maxOutputTokens: 2_000,
2883
+ }),
2884
+ );
2885
+ } catch {
2886
+ // The observation already made stays authoritative; the next sweep
2887
+ // classifies any provider failure through the ordinary path.
2888
+ break;
2889
+ }
2890
+ backlog = retainedBacklogRemains(session, next);
2891
+ await capturePage(next);
2892
+ observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
2893
+ }
2894
+ return backlog && observation.status === "deferred" && observation.reason === "provider_running"
2895
+ ? { ...observation, outputBacklog: true }
2896
+ : observation;
2897
+ }
2898
+
2899
+ /** Whether a byte-offset page leaves a backlog worth reading now: the provider
2900
+ * reports the process exited but its output is not yet read to EOF, or a live
2901
+ * process produced a large page. Must run before the page is captured, which
2902
+ * consumes its receipt. Supervised commands are left to their proof path. */
2903
+ function retainedBacklogRemains(session: RetainedProcessProbeSession, result: unknown): boolean {
2904
+ const page = session.getProviderCommandOutput?.(result);
2905
+ if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
2906
+ if (page.command.supervision || page.exitCode !== null) return false;
2907
+ if (page.providerExited) return true;
2908
+ const { command, expected } = page;
2909
+ return (["stdout", "stderr"] as const).some(
2910
+ (stream) =>
2911
+ command.streams[stream].byteOffset - expected.streams[stream].byteOffset >=
2912
+ RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,
2913
+ );
2914
+ }
2915
+
1834
2916
  async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
1835
2917
  let timer: ReturnType<typeof setTimeout> | undefined;
1836
2918
  try {
@@ -1879,6 +2961,7 @@ export async function probeRetainedProcessAtProvider(
1879
2961
  streamFidelity?: "separate" | "merged",
1880
2962
  ) => Promise<void>,
1881
2963
  providerPersistence?: ProviderCommandPersistence,
2964
+ drainBudget?: RetainedProcessDrainBudget,
1882
2965
  ): Promise<RetainedProcessProbeResult> {
1883
2966
  if (
1884
2967
  lease.id !== process.leaseId ||
@@ -1899,7 +2982,13 @@ export async function probeRetainedProcessAtProvider(
1899
2982
  if (pending) {
1900
2983
  if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
1901
2984
  await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
1902
- return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
2985
+ const pendingObservation = classifyRetainedProcessPollResult(
2986
+ pending.result,
2987
+ process.providerSessionId,
2988
+ );
2989
+ // Capture of a previous page must not consume the only deadline stop
2990
+ // attempt. Terminal proof needs no further provider call.
2991
+ if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
1903
2992
  }
1904
2993
  const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
1905
2994
  if (
@@ -1996,6 +3085,7 @@ export async function probeRetainedProcessAtProvider(
1996
3085
  if (typeof session.writeStdin !== "function") {
1997
3086
  return { status: "deferred", reason: "backend_unsupported" };
1998
3087
  }
3088
+ let legacyPty = false;
1999
3089
  if (process.providerBackend === "modal") {
2000
3090
  let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;
2001
3091
  try {
@@ -2027,6 +3117,7 @@ export async function probeRetainedProcessAtProvider(
2027
3117
  if (!command || !providerPersistence || !session.bindProviderCommand)
2028
3118
  return { status: "deferred", reason: "process_observation_unavailable" };
2029
3119
  session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
3120
+ legacyPty = command.pty === true;
2030
3121
  }
2031
3122
  const capturePage = async (value: unknown): Promise<void> => {
2032
3123
  if (!captureOutput) {
@@ -2062,7 +3153,14 @@ export async function probeRetainedProcessAtProvider(
2062
3153
  result = await withRetainedProcessProbeTimeout(
2063
3154
  session.writeStdin({
2064
3155
  sessionId: process.providerSessionId,
2065
- chars: mode === "cancel" && !supervisedCancelled ? "\u0003" : "",
3156
+ // Ctrl-C is a signal only through a PTY. For a non-PTY command it is
3157
+ // ordinary stdin data and can corrupt the command's input.
3158
+ chars:
3159
+ mode === "cancel" &&
3160
+ !supervisedCancelled &&
3161
+ (process.providerBackend !== "modal" || legacyPty)
3162
+ ? "\u0003"
3163
+ : "",
2066
3164
  yieldTimeMs: 1_000,
2067
3165
  maxOutputTokens: 2_000,
2068
3166
  }),
@@ -2086,12 +3184,22 @@ export async function probeRetainedProcessAtProvider(
2086
3184
  }
2087
3185
  return { status: "deferred", reason: "provider_error" };
2088
3186
  }
2089
- await capturePage(result);
2090
- const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
3187
+ // Cancellation keeps draining with empty reads too: a stopped command's exit
3188
+ // is also reported only after its unread output reaches EOF.
3189
+ const observation = await drainRetainedCommandBacklog(
3190
+ session,
3191
+ process.providerSessionId,
3192
+ result,
3193
+ capturePage,
3194
+ drainBudget,
3195
+ );
2091
3196
  if (
2092
3197
  observation.status === "deferred" &&
2093
3198
  observation.reason === "provider_running" &&
2094
3199
  lease.rotationRequestedAt !== null &&
3200
+ mode !== "cancel" &&
3201
+ !process.cancellationRequestedAt &&
3202
+ legacyPty &&
2095
3203
  !(await providerPersistence
2096
3204
  ?.load()
2097
3205
  .then((command) => command?.kind === "modal-router-v1" && command.supervision))
@@ -2197,19 +3305,25 @@ async function accrueWarmTick(
2197
3305
  }
2198
3306
  await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
2199
3307
  try {
2200
- const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
3308
+ const rate =
3309
+ settings.sandboxWarmBillingMode === "usage_only"
3310
+ ? 0
3311
+ : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
2201
3312
  const result = await accrueWarmSeconds(db, {
2202
3313
  accountId: lease.accountId,
2203
3314
  workspaceId: lease.workspaceId,
2204
3315
  sandboxGroupId: lease.sandboxGroupId,
2205
3316
  expectedEpoch: lease.leaseEpoch,
2206
3317
  warmRateMicrosPerSecond: rate,
3318
+ billingMode: settings.sandboxWarmBillingMode,
2207
3319
  subjectId: lease.sandboxGroupId,
2208
3320
  });
2209
3321
  if (result.accrued) {
2210
3322
  accrued += 1;
2211
3323
  }
2212
- recordCreditMicros(observability, "usage", result.costMicros);
3324
+ if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
3325
+ recordCreditMicros(observability, "usage", result.costMicros);
3326
+ }
2213
3327
  } catch (error) {
2214
3328
  observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
2215
3329
  workspaceId: lease.workspaceId,
@@ -2221,6 +3335,54 @@ async function accrueWarmTick(
2221
3335
  return { accrued, workspaceIds };
2222
3336
  }
2223
3337
 
3338
+ /**
3339
+ * The reaper maintenance pass's read-only projections, for a reaper tick that
3340
+ * started no workflow because there was no work. Same gauges, same process:
3341
+ * the control worker both runs the cadence and polls the lifecycle queue.
3342
+ */
3343
+ export async function refreshIdleSandboxReaperProjections(input: {
3344
+ db: ActivityServices["db"];
3345
+ settings: ActivityServices["settings"];
3346
+ observability: ActivityServices["observability"];
3347
+ inspectOpenSandboxInventory?: InspectOpenSandboxKubernetesInventoryFn;
3348
+ }): Promise<void> {
3349
+ await refreshQueueLeaseAndCreditGauges(
3350
+ input.db,
3351
+ input.settings,
3352
+ input.observability,
3353
+ input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory,
3354
+ );
3355
+ }
3356
+
3357
+ /**
3358
+ * Whether the Modal orphan sweep could act: the deployment uses Modal and its
3359
+ * app has an unfinished sandbox. Called only once the database checks found no
3360
+ * warm lease, so an empty listing is the sweep's complete, empty inventory and
3361
+ * is projected exactly as the sweep would project it.
3362
+ */
3363
+ export async function modalOrphanSweepMayHaveWork(input: {
3364
+ settings: ActivityServices["settings"];
3365
+ observability: ActivityServices["observability"];
3366
+ hasRunningSandboxes?: (settings: ActivityServices["settings"]) => Promise<boolean>;
3367
+ }): Promise<boolean> {
3368
+ const { settings, observability } = input;
3369
+ if (!modalOrphanSweepConfigured(settings)) return false;
3370
+ if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;
3371
+ recordModalSandboxInventoryGauges(observability, {
3372
+ running: 0,
3373
+ unleased: 0,
3374
+ liveLeaseInstancesMissing: 0,
3375
+ });
3376
+ recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
3377
+ return false;
3378
+ }
3379
+
3380
+ function modalOrphanSweepConfigured(settings: ActivityServices["settings"]): boolean {
3381
+ return (
3382
+ settings.sandboxBackend === "modal" || !!settings.modalTokenId || !!settings.modalTokenSecret
3383
+ );
3384
+ }
3385
+
2224
3386
  async function refreshQueueLeaseAndCreditGauges(
2225
3387
  db: ActivityServices["db"],
2226
3388
  settings: ActivityServices["settings"],
@@ -2251,6 +3413,17 @@ async function refreshQueueLeaseAndCreditGauges(
2251
3413
  );
2252
3414
  },
2253
3415
  ),
3416
+ refreshSandboxInventoryGauge(
3417
+ observability,
3418
+ "recovery_observations",
3419
+ "recovery-observations",
3420
+ async () => {
3421
+ recordSandboxRecoveryObservationGauges(
3422
+ observability,
3423
+ await readRecentSandboxRecoveryObservations(db),
3424
+ );
3425
+ },
3426
+ ),
2254
3427
  refreshSandboxInventoryGauge(
2255
3428
  observability,
2256
3429
  "rotation_backlog",
@@ -2259,6 +3432,17 @@ async function refreshQueueLeaseAndCreditGauges(
2259
3432
  recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
2260
3433
  },
2261
3434
  ),
3435
+ refreshSandboxInventoryGauge(
3436
+ observability,
3437
+ "checkpoint_staleness",
3438
+ "checkpoint-staleness",
3439
+ async () => {
3440
+ recordSandboxCheckpointStalenessGauges(
3441
+ observability,
3442
+ await readSandboxCheckpointStaleness(db),
3443
+ );
3444
+ },
3445
+ ),
2262
3446
  refreshSandboxInventoryGauge(
2263
3447
  observability,
2264
3448
  "retained_processes",
@@ -2270,6 +3454,17 @@ async function refreshQueueLeaseAndCreditGauges(
2270
3454
  );
2271
3455
  },
2272
3456
  ),
3457
+ refreshSandboxInventoryGauge(
3458
+ observability,
3459
+ "interaction_idle",
3460
+ "interaction-only-lease",
3461
+ async () => {
3462
+ recordInteractionOnlyLeaseGauges(
3463
+ observability,
3464
+ await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db)),
3465
+ );
3466
+ },
3467
+ ),
2273
3468
  refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
2274
3469
  recordExpiredDrainingSandboxLeaseGauges(
2275
3470
  observability,
@@ -2303,6 +3498,51 @@ async function refreshQueueLeaseAndCreditGauges(
2303
3498
  });
2304
3499
  }
2305
3500
  })(),
3501
+ (async () => {
3502
+ try {
3503
+ recordActiveUserGauges(observability, await countActiveUsers(db));
3504
+ } catch (error) {
3505
+ observability.warn("sandbox reaper: active-user gauge refresh failed", {
3506
+ error: error instanceof Error ? error.message : String(error),
3507
+ });
3508
+ }
3509
+ })(),
3510
+ (async () => {
3511
+ try {
3512
+ recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
3513
+ } catch (error) {
3514
+ observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
3515
+ error: error instanceof Error ? error.message : String(error),
3516
+ });
3517
+ }
3518
+ })(),
3519
+ (async () => {
3520
+ recordVerifiedSignupTrialDeploymentFlagGauge(
3521
+ observability,
3522
+ settings.verifiedSignupTrialCreditsEnabled,
3523
+ );
3524
+ try {
3525
+ const trialSwitch = await readVerifiedSignupTrialSwitch(db);
3526
+ recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
3527
+ } catch (error) {
3528
+ observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
3529
+ error: error instanceof Error ? error.message : String(error),
3530
+ });
3531
+ }
3532
+ })(),
3533
+ (async () => {
3534
+ try {
3535
+ const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
3536
+ recordManagedAuthNewSignupsSwitchGauge(
3537
+ observability,
3538
+ signupsSwitch?.signupsEnabled !== false,
3539
+ );
3540
+ } catch (error) {
3541
+ observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
3542
+ error: error instanceof Error ? error.message : String(error),
3543
+ });
3544
+ }
3545
+ })(),
2306
3546
  ]);
2307
3547
  }
2308
3548
 
@@ -2325,8 +3565,8 @@ async function refreshSandboxInventoryGauge(
2325
3565
 
2326
3566
  /**
2327
3567
  * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
2328
- * live warm box, under the usage lock: if it is at 0 balance (when a billing /
2329
- * managed mode is on) or over its warm cap, force-drain its VIEWER-ONLY boxes
3568
+ * live warm box, under the usage lock: if it is over its optional warm cap,
3569
+ * accelerate only holderless boxes to draining
2330
3570
  * (guarded turn_holders=0 — a paying turn is never killed). Returns the count of
2331
3571
  * viewer-only boxes force-drained. Per-workspace best-effort.
2332
3572
  */
@@ -2336,8 +3576,7 @@ async function forceDrainOverLimitWorkspaces(
2336
3576
  workspaceIds: Set<string>,
2337
3577
  observability: ActivityServices["observability"],
2338
3578
  ): Promise<number> {
2339
- const enforceBalance =
2340
- settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
3579
+ const enforceBalance = false;
2341
3580
  const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
2342
3581
  let forceDrained = 0;
2343
3582
  await forEachWithConcurrency(
@@ -2345,13 +3584,8 @@ async function forceDrainOverLimitWorkspaces(
2345
3584
  SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
2346
3585
  async (workspaceId) => {
2347
3586
  try {
2348
- const { accountId } = await rlsContextForWorkspace(db, workspaceId);
2349
- const balance = enforceBalance
2350
- ? await getBillingBalance(db, accountId)
2351
- : ({ balanceMicros: 1 } as { balanceMicros: number });
2352
3587
  const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
2353
3588
  workspaceId,
2354
- balanceMicros: balance.balanceMicros,
2355
3589
  enforceBalance,
2356
3590
  maxWarmSecondsPerWorkspace: cap,
2357
3591
  idleGraceMs: settings.sandboxIdleGraceMs,
@@ -2375,9 +3609,72 @@ async function forceDrainOverLimitWorkspaces(
2375
3609
  return forceDrained;
2376
3610
  }
2377
3611
 
3612
+ export async function reconcileModalProviderCreates(
3613
+ db: ActivityServices["db"],
3614
+ settings: ActivityServices["settings"],
3615
+ observability: ActivityServices["observability"],
3616
+ discover: typeof findModalProviderCreateReceipt = findModalProviderCreateReceipt,
3617
+ ): Promise<void> {
3618
+ if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
3619
+ return;
3620
+ const candidates = await listPendingModalProviderCreates(db);
3621
+ await forEachWithConcurrency(candidates, 2, async (candidate) => {
3622
+ try {
3623
+ const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
3624
+ const identity = { ...candidate, accountId };
3625
+ const attempt = await claimModalProviderCreateRecovery(db, identity);
3626
+ if (!attempt) return;
3627
+ const instanceId = await discover(settings, attempt);
3628
+ if (!instanceId) return; // Absence never settles an unknown dispatch.
3629
+ if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
3630
+ observability.info("sandbox reaper: recovered Modal create receipt", {
3631
+ ...candidate,
3632
+ operationId: attempt.operationId,
3633
+ instanceId,
3634
+ });
3635
+ }
3636
+ } catch (error) {
3637
+ observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
3638
+ ...candidate,
3639
+ error: error instanceof Error ? error.message : String(error),
3640
+ });
3641
+ }
3642
+ });
3643
+ }
3644
+
2378
3645
  export function modalOrphanTerminationStillEligible(
2379
3646
  latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
2380
3647
  candidate: ModalOrphanSweepTermination,
3648
+ ): boolean {
3649
+ // The provider may have accepted a create whose reply has not arrived.
3650
+ // Before attribution, neither its ID nor its separately-applied tags are
3651
+ // available here. Postpone orphan deletion while any such owner exists;
3652
+ // age/absence cannot distinguish its provider from an abandoned one.
3653
+ if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
3654
+ return false;
3655
+ }
3656
+ return modalOrphanCandidateStillUnowned(latest, candidate);
3657
+ }
3658
+
3659
+ /** Live leases still WARM on an instance the complete listing did not show.
3660
+ * A draining lease legitimately loses its box mid-drain (capture, terminate,
3661
+ * then cold), and warming has not published a box yet, so only a warm lease
3662
+ * whose exact box is gone is a zombie. */
3663
+ export function countWarmModalLeasesMissingInstance(
3664
+ latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
3665
+ missingInstanceIds: string[],
3666
+ ): number {
3667
+ return missingInstanceIds.filter((instanceId) =>
3668
+ latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId),
3669
+ ).length;
3670
+ }
3671
+
3672
+ /** Whether no live lease owns the candidate box by exact instance or by its
3673
+ * attribution tags. Unlike termination eligibility this ignores the pending
3674
+ * create postponement: a postponed orphan is still running without a lease. */
3675
+ export function modalOrphanCandidateStillUnowned(
3676
+ latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
3677
+ candidate: ModalOrphanSweepTermination,
2381
3678
  ): boolean {
2382
3679
  if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
2383
3680
  return false;
@@ -2409,7 +3706,7 @@ async function sweepModalOrphansForConfiguredBackend(
2409
3706
  // Keep the provider-side sweep tightly scoped to deployments that have a Modal
2410
3707
  // app path configured. Local/docker-only workers should not attempt Modal API
2411
3708
  // calls just because modalAppName has a default.
2412
- if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
3709
+ if (!modalOrphanSweepConfigured(settings)) {
2413
3710
  return 0;
2414
3711
  }
2415
3712
  let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
@@ -2428,6 +3725,31 @@ async function sweepModalOrphansForConfiguredBackend(
2428
3725
  return modalOrphanTerminationStillEligible(latest, candidate);
2429
3726
  },
2430
3727
  });
3728
+ // Only a complete listing is an inventory; a pass cut short by the
3729
+ // termination budget leaves the previous projection to age out as stale.
3730
+ // Re-read durable ownership after the listing so a lease that started or
3731
+ // finished while the app was being listed is not reported as a leak.
3732
+ if (result.inventory.complete) {
3733
+ try {
3734
+ const latest = await listLiveModalSandboxLeaseAttributions(db);
3735
+ recordModalSandboxInventoryGauges(observability, {
3736
+ running: result.inventory.running,
3737
+ unleased: result.inventory.unterminated.filter((candidate) =>
3738
+ modalOrphanCandidateStillUnowned(latest, candidate),
3739
+ ).length,
3740
+ liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
3741
+ latest,
3742
+ result.inventory.missingLiveLeaseInstanceIds,
3743
+ ),
3744
+ });
3745
+ recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
3746
+ } catch (error) {
3747
+ recordSandboxInventoryProjectionFailure(observability, "modal_provider");
3748
+ observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
3749
+ error: error instanceof Error ? error.message : String(error),
3750
+ });
3751
+ }
3752
+ }
2431
3753
  for (const terminated of result.terminated) {
2432
3754
  observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
2433
3755
  sandboxId: terminated.sandboxId,
@@ -2566,6 +3888,7 @@ async function terminateDrainableBox(
2566
3888
  probeDrainableProvider: DrainableProviderProbeFn,
2567
3889
  attempt: SandboxDrainCaptureAttempt,
2568
3890
  objectStorage: ObjectStorage | null,
3891
+ bus: ActivityServices["bus"] | null = null,
2569
3892
  ): Promise<boolean> {
2570
3893
  // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
2571
3894
  // returns no account_id; the workspace->account map is the bootstrap read).
@@ -2602,6 +3925,20 @@ async function terminateDrainableBox(
2602
3925
  }
2603
3926
 
2604
3927
  const captureTimeoutMs = attempt.captureTimeoutMs;
3928
+ if (
3929
+ managedProvider &&
3930
+ !lease.archiveCapture?.publishedAt &&
3931
+ sandboxDrainCaptureSettlingInProcess(lease)
3932
+ ) {
3933
+ // An earlier attempt's capture is still reading, publishing late, or
3934
+ // removing its spool in this worker. Leave its claim untouched and do not
3935
+ // start a second physical capture of the same workspace; a later retry
3936
+ // tears down from its published archive or, if it failed, captures anew.
3937
+ // An already published archive still resumes teardown immediately.
3938
+ throw new SandboxDrainRecoveryDeferredError(
3939
+ `sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`,
3940
+ );
3941
+ }
2605
3942
  let captureClaim:
2606
3943
  | NonNullable<
2607
3944
  Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
@@ -2766,6 +4103,8 @@ async function terminateDrainableBox(
2766
4103
  liveness: "draining",
2767
4104
  captureTimeoutMs,
2768
4105
  minIntervalMs: 0,
4106
+ pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),
4107
+ deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
2769
4108
  providerReplaySafe: capturePolicy?.takeover === "same_request",
2770
4109
  takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive",
2771
4110
  });
@@ -2939,6 +4278,20 @@ async function terminateDrainableBox(
2939
4278
  ) {
2940
4279
  return false;
2941
4280
  }
4281
+ // Stamp the charge horizon after verified capture and immediately before
4282
+ // provider stop. Capture can fail and release its claim, allowing re-arm.
4283
+ // A retry after stop success cannot charge post-termination wall time.
4284
+ let stopCutoffMarked = false;
4285
+ const beforeProviderStop = async (): Promise<void> => {
4286
+ const cutoff = await markWarmBillingStopCutoff(db, {
4287
+ accountId,
4288
+ workspaceId: row.workspaceId,
4289
+ sandboxGroupId: row.sandboxGroupId,
4290
+ expectedEpoch: row.leaseEpoch,
4291
+ });
4292
+ if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
4293
+ stopCutoffMarked = true;
4294
+ };
2942
4295
  const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
2943
4296
  ? { terminated: true, providerMissingBeforeCapture: true }
2944
4297
  : await terminateBox(
@@ -2962,12 +4315,66 @@ async function terminateDrainableBox(
2962
4315
  });
2963
4316
  },
2964
4317
  row.workspaceId,
4318
+ beforeProviderStop,
4319
+ backend === "docker"
4320
+ ? async () => {
4321
+ if (!captureClaim || !lease.instanceId)
4322
+ throw new Error("Docker drain has no current claimed capture");
4323
+ return await verifyDockerDrainCaptureFence(db, {
4324
+ accountId,
4325
+ workspaceId: row.workspaceId,
4326
+ sandboxGroupId: row.sandboxGroupId,
4327
+ leaseId: lease.id,
4328
+ leaseEpoch: row.leaseEpoch,
4329
+ instanceId: lease.instanceId,
4330
+ workspaceGeneration: captureClaim.workspaceGeneration,
4331
+ captureId: captureClaim.id,
4332
+ providerRequestId: captureClaim.providerRequestId,
4333
+ });
4334
+ }
4335
+ : undefined,
2965
4336
  );
2966
4337
  const terminated = typeof termination === "boolean" ? termination : termination.terminated;
2967
4338
  if (!terminated) {
2968
4339
  return false;
2969
4340
  }
2970
4341
 
4342
+ // Self-hosted/none and an unassigned lease have no provider stop callback;
4343
+ // finish their usage at logical detach without touching customer compute.
4344
+ // A managed instance that was stopped without its callback must fail closed.
4345
+ const providerMissingDuringStop =
4346
+ providerMissingBeforeCapture ||
4347
+ (typeof termination !== "boolean" && termination.providerMissingBeforeCapture);
4348
+ if (!stopCutoffMarked && !providerMissingDuringStop) {
4349
+ if (managedProvider && lease.instanceId) {
4350
+ throw new Error("managed sandbox stopped without a warm usage cutoff");
4351
+ }
4352
+ await beforeProviderStop();
4353
+ }
4354
+
4355
+ // Stop was confirmed but the lease is not cold yet. This final epoch-fenced
4356
+ // tick settles the interval since the last heartbeat/reaper sweep, including
4357
+ // a box drained before its first periodic tick. If settlement fails, keep
4358
+ // the draining receipt for retry instead of silently losing the charge.
4359
+ const finalRate =
4360
+ settings.sandboxWarmBillingMode === "usage_only"
4361
+ ? 0
4362
+ : sandboxWarmRateMicrosPerSecond(settings, backend);
4363
+ if (!providerMissingDuringStop) {
4364
+ const finalTick = await accrueWarmSeconds(db, {
4365
+ accountId,
4366
+ workspaceId: row.workspaceId,
4367
+ sandboxGroupId: row.sandboxGroupId,
4368
+ expectedEpoch: row.leaseEpoch,
4369
+ warmRateMicrosPerSecond: finalRate,
4370
+ billingMode: settings.sandboxWarmBillingMode,
4371
+ finalDrain: true,
4372
+ });
4373
+ if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
4374
+ recordCreditMicros(observability, "usage", finalTick.costMicros);
4375
+ }
4376
+ }
4377
+
2971
4378
  const providerMissing =
2972
4379
  providerMissingBeforeCapture ||
2973
4380
  (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
@@ -2975,15 +4382,44 @@ async function terminateDrainableBox(
2975
4382
  // with draining->cold. Until this succeeds, arrivals remain fenced by that
2976
4383
  // exact claim; a timestamp or a failed provider call can never reopen a box
2977
4384
  // while termination may still be in flight.
2978
- const { wentCold } = await confirmDrainCold(db, {
2979
- accountId,
2980
- workspaceId: row.workspaceId,
2981
- sandboxGroupId: row.sandboxGroupId,
2982
- expectedEpoch: row.leaseEpoch,
2983
- ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
2984
- providerMissingBeforeCapture: providerMissing,
4385
+ const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
4386
+ db,
4387
+ {
4388
+ accountId,
4389
+ workspaceId: row.workspaceId,
4390
+ sandboxGroupId: row.sandboxGroupId,
4391
+ expectedEpoch: row.leaseEpoch,
4392
+ ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
4393
+ providerMissingBeforeCapture: providerMissing,
4394
+ idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
4395
+ // Mirrors terminateProviderBox: a selfhosted machine is never stopped.
4396
+ providerStopped:
4397
+ managedProvider && lease.backend !== "selfhosted" && lease.resumeBackendId !== "selfhosted",
4398
+ },
4399
+ );
4400
+ // The command terminal events and agent inputs are already durable in the
4401
+ // cold commit; this is only best-effort live fanout.
4402
+ await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
4403
+ observability.warn("sandbox reaper: contained command event fanout failed", {
4404
+ sandboxGroupId: row.sandboxGroupId,
4405
+ error: error instanceof Error ? error.message : String(error),
4406
+ });
2985
4407
  });
2986
4408
  if (wentCold) {
4409
+ // Only the exact successful cold commit counts provider loss. A missing
4410
+ // probe, a stale capture, a failed commit, or a retried child is not another
4411
+ // observed loss. Keep this outside the best-effort session event writer.
4412
+ // An unpublished warming replacement (its creator died before warm
4413
+ // publication) held no workspace; its absence is not a capture loss.
4414
+ if (providerMissing && !unpublishedProviderLost) {
4415
+ recordSandboxProviderMissingBeforeCapture(observability, backend);
4416
+ }
4417
+ if (lease.unobservableCommandDrainIds?.length) {
4418
+ recordSandboxCommandContainment(
4419
+ observability,
4420
+ providerMissing ? "provider_missing" : "contained",
4421
+ );
4422
+ }
2987
4423
  // Durable termination record (sandbox-file-persistence observability): who
2988
4424
  // ended this box and whether its /workspace was captured first, appended to
2989
4425
  // every session sharing the group's box. Best-effort: attribution must
@@ -3084,6 +4520,73 @@ function providerDrainLogIdentity(
3084
4520
  * draining until its adapter can prove termination. A truly instance-less or
3085
4521
  * `none`-backed row is a no-op.
3086
4522
  */
4523
+ /** Internal server pre-read fence; not a grant to arbitrary callers. The
4524
+ * owning drain already derived account/workspace/group authority and claimed
4525
+ * capture through the canonical DB lifecycle. No claim is fabricated here. */
4526
+ export async function verifyDockerDrainCaptureFence(
4527
+ db: ActivityServices["db"],
4528
+ input: {
4529
+ accountId: string;
4530
+ workspaceId: string;
4531
+ sandboxGroupId: string;
4532
+ leaseId: string;
4533
+ leaseEpoch: number;
4534
+ instanceId: string;
4535
+ workspaceGeneration: number;
4536
+ captureId: string;
4537
+ providerRequestId: string;
4538
+ },
4539
+ readers = { readLease, readWorkspaceArchiveCapturePreflight },
4540
+ ): Promise<{ archivePublished: boolean }> {
4541
+ const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);
4542
+ if (
4543
+ !current ||
4544
+ current.id !== input.leaseId ||
4545
+ current.liveness !== "draining" ||
4546
+ current.leaseEpoch !== input.leaseEpoch ||
4547
+ current.instanceId !== input.instanceId ||
4548
+ current.backend !== "docker" ||
4549
+ current.workspaceGeneration !== input.workspaceGeneration ||
4550
+ current.archiveCapture?.id !== input.captureId ||
4551
+ current.archiveCapture.providerRequestId !== input.providerRequestId ||
4552
+ current.archiveCapture.workspaceGeneration !== input.workspaceGeneration
4553
+ )
4554
+ throw new Error("Docker drain capture lease/epoch/claim changed");
4555
+ const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {
4556
+ accountId: input.accountId,
4557
+ workspaceId: input.workspaceId,
4558
+ sandboxGroupId: input.sandboxGroupId,
4559
+ expectedEpoch: input.leaseEpoch,
4560
+ expectedInstanceId: input.instanceId,
4561
+ liveness: "draining",
4562
+ });
4563
+ if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)
4564
+ throw new Error("Docker drain capture has an active writer or changed generation");
4565
+ return {
4566
+ archivePublished:
4567
+ current.archiveCapture.publishedAt instanceof Date &&
4568
+ Number.isFinite(current.archiveCapture.publishedAt.getTime()) &&
4569
+ current.archiveComplete === true,
4570
+ };
4571
+ }
4572
+
4573
+ function observeDockerWorkspaceRelease(
4574
+ observability: ActivityServices["observability"],
4575
+ session: unknown,
4576
+ logIdentity: ReturnType<typeof providerDrainLogIdentity>,
4577
+ ): void {
4578
+ const release = (session as { hostWorkspaceRelease?: DockerHostWorkspaceRelease | null })
4579
+ .hostWorkspaceRelease;
4580
+ if (!release) return;
4581
+ recordSandboxDockerWorkspaceRelease(observability, release.status);
4582
+ if (release.status === "retained") {
4583
+ observability.warn(
4584
+ "sandbox reaper: Docker drain kept its host workspace directory after teardown",
4585
+ { ...logIdentity, reason: release.reason },
4586
+ );
4587
+ }
4588
+ }
4589
+
3087
4590
  export async function terminateProviderBox(
3088
4591
  settings: ActivityServices["settings"],
3089
4592
  lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
@@ -3097,6 +4600,8 @@ export async function terminateProviderBox(
3097
4600
  diskBackedArchives = false,
3098
4601
  releaseFailedCapture?: () => Promise<void>,
3099
4602
  workspaceId?: string,
4603
+ beforeProviderStop?: () => Promise<void>,
4604
+ beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,
3100
4605
  ): Promise<ProviderTerminationOutcome> {
3101
4606
  const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
3102
4607
  const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
@@ -3161,6 +4666,11 @@ export async function terminateProviderBox(
3161
4666
  // fail closed instead of silently leaving a live provider behind while the
3162
4667
  // caller commits the lease cold.
3163
4668
  if (!lease.instanceId) {
4669
+ if (lease.providerCreateAttempt?.instanceId === null) {
4670
+ throw new Error(
4671
+ "provider_create_outcome_unknown: cannot infer termination from missing instance ID",
4672
+ );
4673
+ }
3164
4674
  if (persistedInstanceId) {
3165
4675
  throw new Error(
3166
4676
  `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,
@@ -3196,7 +4706,13 @@ export async function terminateProviderBox(
3196
4706
  return { terminated: false, providerMissingBeforeCapture: false };
3197
4707
  }
3198
4708
  try {
3199
- await terminateModalById(settings, lease.instanceId);
4709
+ await beforeProviderStop?.();
4710
+ await terminateModalById(
4711
+ settings,
4712
+ lease.instanceId,
4713
+ undefined,
4714
+ lease.providerCreateAttempt?.providerBindingKey,
4715
+ );
3200
4716
  } catch (error) {
3201
4717
  if (!isProviderSandboxNotFoundError("modal", error)) {
3202
4718
  throw error;
@@ -3234,7 +4750,7 @@ export async function terminateProviderBox(
3234
4750
  let session: PersistableSession | undefined;
3235
4751
  let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;
3236
4752
  try {
3237
- if (!client.resume || !client.deserializeSessionState) {
4753
+ if (!client.deserializeSessionState || (backend !== "docker" && !client.resume)) {
3238
4754
  // A cloud backend that cannot prove provider state is not safely
3239
4755
  // terminable: treating this as success would cold the lease while the
3240
4756
  // provider may still be live. Leave it draining for a later retry.
@@ -3261,16 +4777,25 @@ export async function terminateProviderBox(
3261
4777
  if (resumedState === undefined) {
3262
4778
  throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
3263
4779
  }
3264
- const continuity = sandboxProviderContinuityForState(backend, resumedState, lease.instanceId);
3265
- const resumed = await resumeExactSandboxSession(
3266
- client,
3267
- backend,
3268
- resumedState,
3269
- lease.instanceId,
3270
- continuity ? { continuity } : undefined,
3271
- );
3272
- session = resumed.session as PersistableSession;
3273
- sessionState = resumed.sessionState;
4780
+ if (backend === "docker") {
4781
+ if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {
4782
+ throw new Error(
4783
+ "Docker drain requires a creation-free attachment and current capture fence",
4784
+ );
4785
+ }
4786
+ await beforeWorkspaceCapture();
4787
+ session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);
4788
+ sessionState = resumedState;
4789
+ } else {
4790
+ const resumed = await resumeExactSandboxSession(
4791
+ client,
4792
+ backend,
4793
+ resumedState,
4794
+ lease.instanceId,
4795
+ );
4796
+ session = resumed.session as PersistableSession;
4797
+ sessionState = resumed.sessionState;
4798
+ }
3274
4799
  } catch (error) {
3275
4800
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
3276
4801
  observability.info("sandbox reaper: drainable box already gone before workspace capture", {
@@ -3315,8 +4840,15 @@ export async function terminateProviderBox(
3315
4840
  resolveModalCheckpointProviderBindingForSession(settings, session),
3316
4841
  )
3317
4842
  : null;
4843
+ // Physical capture bookkeeping for this worker: released once the capture
4844
+ // and its spool are settled, or by the late path that inherits them.
4845
+ let releaseSettlingCapture: () => void = () => undefined;
3318
4846
  try {
3319
4847
  if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
4848
+ releaseSettlingCapture = trackSettlingDrainCapture(lease);
4849
+ // The budget bounds only this attempt's wait, never the capture: a slow
4850
+ // but finite capture keeps reading and publishes late, and the retry
4851
+ // that finds it still settling here defers instead of starting another.
3320
4852
  const capture = captureWorkspaceArchiveForStorage(
3321
4853
  session,
3322
4854
  Date.now(),
@@ -3368,9 +4900,13 @@ export async function terminateProviderBox(
3368
4900
  void releaseAfterCaptureFailure();
3369
4901
  warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
3370
4902
  },
4903
+ onLateSettled: () => releaseSettlingCapture(),
3371
4904
  });
3372
4905
  }
3373
4906
  } catch (error) {
4907
+ // Every rejection except the caller's own timeout means the capture has
4908
+ // settled; a timed-out capture is released by its late path instead.
4909
+ if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();
3374
4910
  if (isProviderSandboxNotFoundError(client.backendId, error)) {
3375
4911
  observability.info("sandbox reaper: box gone during workspace capture", {
3376
4912
  ...logIdentity,
@@ -3412,7 +4948,11 @@ export async function terminateProviderBox(
3412
4948
  checkpointBinding,
3413
4949
  ));
3414
4950
  } finally {
3415
- await disposeWorkspaceArchive(verifiedArchive);
4951
+ try {
4952
+ await disposeWorkspaceArchive(verifiedArchive);
4953
+ } finally {
4954
+ releaseSettlingCapture();
4955
+ }
3416
4956
  }
3417
4957
  if (!wrote) {
3418
4958
  observability.info(
@@ -3439,7 +4979,9 @@ export async function terminateProviderBox(
3439
4979
  throw new Error(`sandbox backend ${backend} returned no terminable session state`);
3440
4980
  }
3441
4981
  prepareProviderForTeardownAfterCapture(backend, session);
4982
+ await beforeProviderStop?.();
3442
4983
  await terminateManagedSandboxSession(client, sessionState, session);
4984
+ if (backend === "docker") observeDockerWorkspaceRelease(observability, session, logIdentity);
3443
4985
  return { terminated: true, providerMissingBeforeCapture: false };
3444
4986
  } catch (error) {
3445
4987
  if (isProviderSandboxNotFoundError(client.backendId, error)) {