@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (379) hide show
  1. package/dist/activities/agent-run-admission.d.ts +1 -12
  2. package/dist/activities/agent-turn/admission.d.ts +88 -10
  3. package/dist/activities/agent-turn/agent-build.d.ts +14 -3
  4. package/dist/activities/agent-turn/claim.d.ts +4 -1
  5. package/dist/activities/agent-turn/claude-core.d.ts +195 -0
  6. package/dist/activities/agent-turn/code-search.d.ts +60 -0
  7. package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
  8. package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
  9. package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
  10. package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
  11. package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
  12. package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
  13. package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
  14. package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
  15. package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
  16. package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
  17. package/dist/activities/agent-turn/codex.d.ts +2 -19
  18. package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
  19. package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
  20. package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
  21. package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
  22. package/dist/activities/agent-turn/errors.d.ts +185 -26
  23. package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
  24. package/dist/activities/agent-turn/final-reply.d.ts +18 -0
  25. package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
  26. package/dist/activities/agent-turn/finalization.d.ts +2 -0
  27. package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
  28. package/dist/activities/agent-turn/governance-model.d.ts +13 -1
  29. package/dist/activities/agent-turn/history-sink.d.ts +4 -1
  30. package/dist/activities/agent-turn/history.d.ts +2 -0
  31. package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
  32. package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
  33. package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
  34. package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
  35. package/dist/activities/agent-turn/model-usage.d.ts +56 -14
  36. package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
  37. package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
  38. package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
  39. package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
  40. package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
  41. package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
  42. package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
  43. package/dist/activities/agent-turn/quiescence.d.ts +6 -7
  44. package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
  45. package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
  46. package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
  47. package/dist/activities/agent-turn/run.d.ts +27 -2
  48. package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
  49. package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
  50. package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
  51. package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
  52. package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
  53. package/dist/activities/agent-turn/session-title.d.ts +60 -3
  54. package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
  55. package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
  56. package/dist/activities/agent-turn/skill-read.d.ts +70 -2
  57. package/dist/activities/agent-turn/skill-search.d.ts +2 -0
  58. package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
  59. package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
  60. package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
  61. package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
  62. package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
  63. package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
  64. package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
  65. package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
  66. package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
  67. package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
  68. package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
  69. package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
  70. package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
  71. package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
  72. package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
  73. package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
  74. package/dist/activities/agent-turn/turn-context.d.ts +104 -5
  75. package/dist/activities/agent-turn/web-search.d.ts +57 -0
  76. package/dist/activities/agent-turn/xai-core.d.ts +96 -0
  77. package/dist/activities/automations.d.ts +3 -1
  78. package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
  79. package/dist/activities/capabilities.d.ts +8 -20
  80. package/dist/activities/codemode-dispatcher.d.ts +20 -3
  81. package/dist/activities/codex-capacity.d.ts +1 -56
  82. package/dist/activities/codex-image-generation.d.ts +49 -3
  83. package/dist/activities/common.d.ts +3 -1
  84. package/dist/activities/context-compaction.d.ts +41 -2
  85. package/dist/activities/environment.d.ts +1 -19
  86. package/dist/activities/gmail-files.d.ts +50 -0
  87. package/dist/activities/goals.d.ts +1 -9
  88. package/dist/activities/incident-telemetry-authority.d.ts +5 -2
  89. package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
  90. package/dist/activities/knowledge-indexing.d.ts +19 -0
  91. package/dist/activities/mcp-account-routes.d.ts +2 -21
  92. package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
  93. package/dist/activities/open-suffix-resume.d.ts +15 -0
  94. package/dist/activities/parent-wake.d.ts +17 -12
  95. package/dist/activities/programmatic-approvals.d.ts +10 -0
  96. package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
  97. package/dist/activities/retained-screenshots.d.ts +7 -2
  98. package/dist/activities/run-credential-renewal.d.ts +1 -0
  99. package/dist/activities/run-credentials.d.ts +16 -4
  100. package/dist/activities/run-input.d.ts +17 -0
  101. package/dist/activities/sandbox-lease.d.ts +132 -12
  102. package/dist/activities/scheduled-tasks.d.ts +4 -35
  103. package/dist/activities/session-archive.d.ts +58 -0
  104. package/dist/activities/session-state.d.ts +15 -3
  105. package/dist/activities/session-storage.d.ts +44 -0
  106. package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
  107. package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
  108. package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
  109. package/dist/activities/subscription-core-image-operation.d.ts +29 -0
  110. package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
  111. package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
  112. package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
  113. package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
  114. package/dist/activities/types.d.ts +68 -5
  115. package/dist/activities/video-generation-admission.d.ts +2 -8
  116. package/dist/activities/video-generation-credential.d.ts +15 -12
  117. package/dist/activities/video-generation-subscription-core.d.ts +70 -0
  118. package/dist/activities/workspace-capture.d.ts +2 -0
  119. package/dist/activities/workspace-credential-provider.d.ts +47 -0
  120. package/dist/activities/xai-auth.d.ts +6 -18
  121. package/dist/activities/xai-image-generation.d.ts +22 -6
  122. package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
  123. package/dist/activities-control-5IIVRXLA.js.map +1 -0
  124. package/dist/activities-control.d.ts +64 -4
  125. package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
  126. package/dist/activities-turn-FLYJOM54.js.map +1 -0
  127. package/dist/activities.d.ts +64 -4
  128. package/dist/artifact-materializer-entry.js +107 -36
  129. package/dist/artifact-materializer-entry.js.map +1 -1
  130. package/dist/artifact-outbox-entry.js +132 -61
  131. package/dist/artifact-outbox-entry.js.map +1 -1
  132. package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
  133. package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
  134. package/dist/chunk-24GP3BX5.js.map +1 -0
  135. package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
  136. package/dist/chunk-4ER3YXOT.js.map +1 -0
  137. package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
  138. package/dist/chunk-5NNS4327.js.map +1 -0
  139. package/dist/chunk-MVESOYFC.js +3746 -0
  140. package/dist/chunk-MVESOYFC.js.map +1 -0
  141. package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
  142. package/dist/chunk-N57B3PJ5.js.map +1 -0
  143. package/dist/chunk-NQDVJ6RI.js +2823 -0
  144. package/dist/chunk-NQDVJ6RI.js.map +1 -0
  145. package/dist/chunk-O7UPBWXK.js +79 -0
  146. package/dist/chunk-O7UPBWXK.js.map +1 -0
  147. package/dist/chunk-W2QSM5HR.js +414 -0
  148. package/dist/chunk-W2QSM5HR.js.map +1 -0
  149. package/dist/chunk-ZN3XJR33.js +227 -0
  150. package/dist/chunk-ZN3XJR33.js.map +1 -0
  151. package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
  152. package/dist/editable-artifact-outbox-service.d.ts +3 -1
  153. package/dist/host-export-pump.d.ts +3 -1
  154. package/dist/host-export.d.ts +2 -2
  155. package/dist/host-export.js +1 -1
  156. package/dist/http.d.ts +1 -1
  157. package/dist/index.d.ts +49 -48
  158. package/dist/index.js +835 -256
  159. package/dist/index.js.map +1 -1
  160. package/dist/internal-maintenance.d.ts +63 -0
  161. package/dist/maintenance-cadence.d.ts +43 -0
  162. package/dist/observability-metrics.d.ts +293 -24
  163. package/dist/onepassword-vault-provider.d.ts +7 -0
  164. package/dist/parent-wake-PCLH7A3Y.js +24 -0
  165. package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
  166. package/dist/personal-github-git-credentials.d.ts +1 -1
  167. package/dist/readiness-checks.d.ts +39 -0
  168. package/dist/retained-process-retry.d.ts +7 -0
  169. package/dist/sandbox-lease-BJYU72N3.js +84 -0
  170. package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
  171. package/dist/sandbox-reaper-contract.d.ts +25 -0
  172. package/dist/sandbox-resume.d.ts +72 -4
  173. package/dist/sandbox-routing.d.ts +4 -2
  174. package/dist/session-event-fanout.d.ts +6 -0
  175. package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
  176. package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
  177. package/dist/telemetry-scheduler.d.ts +14 -0
  178. package/dist/temporal-runtime-logger.d.ts +22 -0
  179. package/dist/turn-task-queue-reader.d.ts +18 -0
  180. package/dist/turn-worker-activity-telemetry.d.ts +47 -0
  181. package/dist/worker-service-lifecycle.d.ts +12 -0
  182. package/dist/workflow-bundle.js +468 -71
  183. package/dist/workflows/activities.d.ts +1 -1
  184. package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
  185. package/dist/workflows/sandbox-reaper.d.ts +10 -1
  186. package/dist/workflows/scheduled-tasks.d.ts +4 -0
  187. package/dist/workflows/session-storage.d.ts +2 -0
  188. package/dist/workflows/session.d.ts +36 -0
  189. package/dist/workflows/site-auth-maintenance.d.ts +2 -1
  190. package/dist/workflows/workflow-wake.d.ts +14 -3
  191. package/dist/workflows.d.ts +3 -1
  192. package/package.json +26 -20
  193. package/src/activities/agent-run-admission.ts +1 -82
  194. package/src/activities/agent-turn/admission.ts +175 -24
  195. package/src/activities/agent-turn/agent-build.ts +226 -145
  196. package/src/activities/agent-turn/claim.ts +363 -64
  197. package/src/activities/agent-turn/claude-core.ts +587 -0
  198. package/src/activities/agent-turn/code-search.ts +275 -0
  199. package/src/activities/agent-turn/codex-capacity.ts +30 -757
  200. package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
  201. package/src/activities/agent-turn/codex-core-claim.ts +83 -0
  202. package/src/activities/agent-turn/codex-core-errors.ts +308 -0
  203. package/src/activities/agent-turn/codex-core-failover.ts +73 -0
  204. package/src/activities/agent-turn/codex-core-requests.ts +86 -0
  205. package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
  206. package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
  207. package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
  208. package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
  209. package/src/activities/agent-turn/codex.ts +8 -76
  210. package/src/activities/agent-turn/compaction-interruption.ts +56 -0
  211. package/src/activities/agent-turn/compaction-prep.ts +249 -36
  212. package/src/activities/agent-turn/credential-leases.ts +269 -214
  213. package/src/activities/agent-turn/credential-restriction.ts +14 -0
  214. package/src/activities/agent-turn/errors.ts +1152 -82
  215. package/src/activities/agent-turn/failure-settlement.ts +879 -886
  216. package/src/activities/agent-turn/final-reply.ts +46 -0
  217. package/src/activities/agent-turn/finalization-monitor.ts +17 -5
  218. package/src/activities/agent-turn/finalization.ts +107 -88
  219. package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
  220. package/src/activities/agent-turn/governance-model.ts +90 -22
  221. package/src/activities/agent-turn/history-sink.ts +42 -12
  222. package/src/activities/agent-turn/history.ts +25 -5
  223. package/src/activities/agent-turn/input-wait-reply.ts +139 -0
  224. package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
  225. package/src/activities/agent-turn/meaningful-output.ts +44 -0
  226. package/src/activities/agent-turn/media-artifacts.ts +3 -1
  227. package/src/activities/agent-turn/model-call-admission.ts +114 -0
  228. package/src/activities/agent-turn/model-usage.ts +299 -233
  229. package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
  230. package/src/activities/agent-turn/optional-repositories.ts +224 -0
  231. package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
  232. package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
  233. package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
  234. package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
  235. package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
  236. package/src/activities/agent-turn/quiescence.ts +7 -7
  237. package/src/activities/agent-turn/recovery-warning.ts +6 -8
  238. package/src/activities/agent-turn/refresh-credentials.ts +44 -0
  239. package/src/activities/agent-turn/run-credentials.ts +124 -76
  240. package/src/activities/agent-turn/run.ts +501 -112
  241. package/src/activities/agent-turn/sandbox-establish.ts +152 -75
  242. package/src/activities/agent-turn/sandbox-provision.ts +54 -1
  243. package/src/activities/agent-turn/sandbox-route.ts +4 -1
  244. package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
  245. package/src/activities/agent-turn/secret-resolver.ts +155 -0
  246. package/src/activities/agent-turn/session-title.ts +177 -12
  247. package/src/activities/agent-turn/session-usage-custody.ts +170 -0
  248. package/src/activities/agent-turn/skill-checkout.ts +111 -24
  249. package/src/activities/agent-turn/skill-install.ts +1 -1
  250. package/src/activities/agent-turn/skill-read.ts +374 -28
  251. package/src/activities/agent-turn/skill-save.ts +35 -7
  252. package/src/activities/agent-turn/skill-search.ts +9 -0
  253. package/src/activities/agent-turn/skill-selection.ts +4 -0
  254. package/src/activities/agent-turn/skill-tools.ts +208 -21
  255. package/src/activities/agent-turn/skill-transfer.ts +68 -23
  256. package/src/activities/agent-turn/stream-attempt.ts +634 -111
  257. package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
  258. package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
  259. package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
  260. package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
  261. package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
  262. package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
  263. package/src/activities/agent-turn/subscription-lease.ts +140 -0
  264. package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
  265. package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
  266. package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
  267. package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
  268. package/src/activities/agent-turn/tool-environment.ts +384 -292
  269. package/src/activities/agent-turn/tool-policy.ts +104 -4
  270. package/src/activities/agent-turn/tool-result-spill.ts +1 -0
  271. package/src/activities/agent-turn/turn-context.ts +124 -8
  272. package/src/activities/agent-turn/web-search.ts +431 -0
  273. package/src/activities/agent-turn/xai-core.ts +301 -0
  274. package/src/activities/automations.ts +81 -6
  275. package/src/activities/browser-deadline-checkpoint.ts +297 -0
  276. package/src/activities/capabilities.ts +53 -213
  277. package/src/activities/codemode-dispatcher.ts +236 -14
  278. package/src/activities/codex-capacity.ts +27 -380
  279. package/src/activities/codex-image-generation.ts +252 -42
  280. package/src/activities/common.ts +12 -0
  281. package/src/activities/connector-attachments.ts +3 -1
  282. package/src/activities/context-compaction.ts +182 -21
  283. package/src/activities/environment.ts +18 -106
  284. package/src/activities/gmail-files.ts +108 -0
  285. package/src/activities/goals.ts +120 -204
  286. package/src/activities/image-generation-operation.ts +1 -1
  287. package/src/activities/incident-telemetry-authority.ts +80 -50
  288. package/src/activities/incident-telemetry-preflight.ts +27 -15
  289. package/src/activities/knowledge-indexing.ts +287 -17
  290. package/src/activities/knowledge-source-sync.ts +10 -0
  291. package/src/activities/mcp-account-routes.ts +2 -124
  292. package/src/activities/model-call-fact-reconciler.ts +112 -0
  293. package/src/activities/open-suffix-resume.ts +122 -88
  294. package/src/activities/parent-wake.ts +89 -21
  295. package/src/activities/programmatic-approvals.ts +26 -0
  296. package/src/activities/requester-unavailable-receipt.ts +14 -0
  297. package/src/activities/retained-screenshots.ts +135 -24
  298. package/src/activities/run-credential-renewal.ts +13 -1
  299. package/src/activities/run-credentials.ts +61 -19
  300. package/src/activities/run-input.ts +124 -9
  301. package/src/activities/sandbox-lease.ts +1615 -73
  302. package/src/activities/scheduled-tasks.ts +780 -147
  303. package/src/activities/session-archive.ts +382 -0
  304. package/src/activities/session-state.ts +219 -10
  305. package/src/activities/session-storage.ts +124 -0
  306. package/src/activities/site-auth-maintenance.ts +5 -2
  307. package/src/activities/subscription-core-claude-waits.ts +49 -0
  308. package/src/activities/subscription-core-codex-quota.ts +46 -0
  309. package/src/activities/subscription-core-codex-waits.ts +488 -0
  310. package/src/activities/subscription-core-image-operation.ts +126 -0
  311. package/src/activities/subscription-core-usage-recheck.ts +146 -0
  312. package/src/activities/subscription-core-wait-bindings.ts +12 -0
  313. package/src/activities/subscription-core-xai-quota.ts +85 -0
  314. package/src/activities/subscription-core-xai-waits.ts +48 -0
  315. package/src/activities/types.ts +85 -9
  316. package/src/activities/video-generation-admission.ts +1 -36
  317. package/src/activities/video-generation-credential.ts +38 -56
  318. package/src/activities/video-generation-reconciliation.ts +150 -131
  319. package/src/activities/video-generation-subscription-core.ts +180 -0
  320. package/src/activities/workspace-capture.ts +13 -2
  321. package/src/activities/workspace-credential-provider.ts +302 -0
  322. package/src/activities/xai-auth.ts +34 -112
  323. package/src/activities/xai-image-generation.ts +84 -36
  324. package/src/activities-control.ts +8 -0
  325. package/src/activities-turn.ts +5 -0
  326. package/src/activities.ts +17 -0
  327. package/src/activity-services.ts +29 -4
  328. package/src/browser-deadline-checkpoint-contract.ts +5 -0
  329. package/src/editable-artifact-hint-broker.ts +3 -0
  330. package/src/editable-artifact-materializer-service.ts +85 -39
  331. package/src/editable-artifact-materializer-subprocess.ts +47 -2
  332. package/src/editable-artifact-outbox-dispatcher.ts +107 -46
  333. package/src/editable-artifact-outbox-service.ts +55 -25
  334. package/src/github-rest-mcp.ts +1 -1
  335. package/src/host-export-pump.ts +26 -4
  336. package/src/host-export.ts +4 -1
  337. package/src/http.ts +4 -2
  338. package/src/index.ts +222 -282
  339. package/src/internal-maintenance.ts +347 -0
  340. package/src/maintenance-cadence.ts +156 -0
  341. package/src/observability-metrics.ts +1173 -73
  342. package/src/onepassword-vault-provider.ts +11 -0
  343. package/src/personal-github-git-credentials.ts +3 -1
  344. package/src/pr-review-credentials.ts +1 -1
  345. package/src/readiness-checks.ts +113 -0
  346. package/src/retained-process-retry.ts +36 -0
  347. package/src/sandbox-reaper-contract.ts +33 -0
  348. package/src/sandbox-resume.ts +444 -73
  349. package/src/sandbox-routing.ts +230 -75
  350. package/src/session-event-fanout.ts +23 -0
  351. package/src/telemetry-scheduler.ts +53 -0
  352. package/src/temporal-runtime-logger.ts +44 -0
  353. package/src/turn-task-queue-reader.ts +110 -0
  354. package/src/turn-worker-activity-telemetry.ts +209 -0
  355. package/src/worker-service-lifecycle.ts +54 -1
  356. package/src/workflows/activities.ts +16 -1
  357. package/src/workflows/browser-deadline-checkpoint.ts +55 -0
  358. package/src/workflows/file-upload-reaper.ts +2 -0
  359. package/src/workflows/sandbox-reaper.ts +88 -0
  360. package/src/workflows/scheduled-tasks.ts +26 -13
  361. package/src/workflows/session-storage.ts +19 -0
  362. package/src/workflows/session.ts +238 -33
  363. package/src/workflows/site-auth-maintenance.ts +2 -1
  364. package/src/workflows/workflow-wake.ts +26 -4
  365. package/src/workflows.ts +7 -0
  366. package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
  367. package/dist/activities/codex-fleet-shadow.d.ts +0 -74
  368. package/dist/activities/codex-rotation.d.ts +0 -200
  369. package/dist/activities/xai-quota.d.ts +0 -3
  370. package/dist/activities-control-QA6OLRQY.js.map +0 -1
  371. package/dist/activities-turn-YVVKCGAK.js.map +0 -1
  372. package/dist/chunk-GDCLDKSH.js.map +0 -1
  373. package/dist/chunk-I7HKKMUJ.js.map +0 -1
  374. package/dist/chunk-M3NVMF2V.js.map +0 -1
  375. package/dist/chunk-RAMNYSAW.js.map +0 -1
  376. package/src/activities/agent-turn/xai-capacity.ts +0 -257
  377. package/src/activities/codex-fleet-shadow.ts +0 -306
  378. package/src/activities/codex-rotation.ts +0 -685
  379. package/src/activities/xai-quota.ts +0 -35
@@ -20,10 +20,13 @@
20
20
  //
21
21
  // Liveness between turns is the lease refcount; there is no keepalive loop.
22
22
 
23
+ import type { EventBus } from "@opengeni/events";
24
+ import { publishDurableSessionEvents } from "./session-event-fanout";
23
25
  import {
24
26
  effectiveSandboxLifecycle,
25
27
  sandboxArchiveCaptureTimeoutMs,
26
28
  sandboxLifecycleTransitionWaitMs,
29
+ sandboxWarmRateMicrosPerSecond,
27
30
  type Settings,
28
31
  } from "@opengeni/config";
29
32
  import { randomUUID } from "node:crypto";
@@ -32,10 +35,12 @@ import {
32
35
  acquireLease,
33
36
  adoptLegacyModalCheckpointArtifact,
34
37
  beginSandboxRematerialization,
38
+ sessionHoldsFreshWorkspaceRecovery,
35
39
  claimWorkspaceArchiveCapture,
36
40
  commitWarmingToWarm,
37
41
  failSandboxRematerialization,
38
42
  failWarmingToCold,
43
+ beginModalProviderCreate,
39
44
  getSandboxSessionEnvelope,
40
45
  heartbeatLeaseHolderStatus,
41
46
  markSandboxRestoreVerifying,
@@ -46,6 +51,7 @@ import {
46
51
  registerSandboxCheckpointArtifact,
47
52
  recordWarmingSandboxCreated,
48
53
  reinstateTurnLeaseHolder,
54
+ leaseMayHaveUntrackedWriters,
49
55
  releaseWorkspaceArchiveCapture,
50
56
  releaseLeaseHolder,
51
57
  touchLeaseHolder,
@@ -78,8 +84,11 @@ import {
78
84
  terminateUnpublishedSandboxSession,
79
85
  verifySandboxExecReadiness,
80
86
  withoutSandboxProviderIdentity,
87
+ workspaceArchiveDownloadTemporaryDirectory,
81
88
  type EstablishedSandboxSession,
82
89
  type RuntimeMetricsHooks,
90
+ type WorkspaceCaptureSkipReason,
91
+ type SandboxReadinessReplacementOutcome,
83
92
  type WorkspaceArchiveDescriptor,
84
93
  } from "@opengeni/runtime";
85
94
  import {
@@ -149,6 +158,29 @@ export type SandboxResumeServices = {
149
158
  /** Test seam for the bounded command-readiness proof performed before an
150
159
  * attached provider box is handed to the agent. */
151
160
  verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
161
+ /** Test seam for the bounded command-readiness proof the elected spawner runs
162
+ * on its freshly created box before publishing the lease warm. Production
163
+ * uses {@link waitForSandboxExecReadiness} with the shared 60 s budget. */
164
+ verifySpawnedSandboxReadiness?: (
165
+ established: EstablishedSandboxSession,
166
+ identity: { sandboxGroupId: string },
167
+ ) => Promise<void>;
168
+ /** Test seam for the jittered pause before the single fresh-box readiness
169
+ * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
170
+ * It may be async so a test can observe the rolled-back lease in between. */
171
+ freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
172
+ /** Test seam: runs immediately after the elected spawner published its box
173
+ * warm, before the final cancellation check. */
174
+ onSpawnedSandboxPublished?: () => void | Promise<void>;
175
+ /**
176
+ * The turn attempt's fresh-box readiness replacement budget. The lazy
177
+ * provisioner may call resumeBoxForTurn again after a typed lease
178
+ * supersession, so the turn creates the budget once and shares it with every
179
+ * call. Absent, the call gets its own single replacement.
180
+ */
181
+ freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
182
+ /** Live fanout for background commands a provider loss settled. */
183
+ bus?: EventBus | null;
152
184
  /** Called only by the observer that wins the exact warm->cold loss CAS. */
153
185
  onSandboxLost?: (input: {
154
186
  sandboxGroupId: string;
@@ -188,6 +220,9 @@ export type ResumeBoxIds = {
188
220
  * never conflicts).
189
221
  */
190
222
  image?: string;
223
+ /** Deployment/workspace image pins apply only to new creates; an existing
224
+ * group keeps its image. Omission preserves explicit image matching (B3). */
225
+ imagePolicy?: "require_match" | "new_creates_only";
191
226
  /**
192
227
  * RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
193
228
  * acquireLease, which stamps it on the cold-create and conflicts on a live box
@@ -317,6 +352,80 @@ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutErro
317
352
  }
318
353
  }
319
354
 
355
+ /**
356
+ * Out-of-band proof attached to one exact {@link SandboxExecReadinessTimeoutError}
357
+ * thrown by the elected spawner: the box that missed its readiness budget was
358
+ * freshly created by this caller (not an attached, resumed, or provider-continuity
359
+ * box), it was never published warm, the provider terminated it, and the holder
360
+ * was released. Only such an error may be replaced by re-entering admission.
361
+ */
362
+ const disposedUnreadyFreshSandboxErrors = new WeakSet<object>();
363
+
364
+ /** True only for a spawner readiness timeout whose unready fresh box is proven gone. */
365
+ export function isReplaceableFreshSandboxReadinessTimeout(error: unknown): boolean {
366
+ return (
367
+ error instanceof SandboxExecReadinessTimeoutError &&
368
+ disposedUnreadyFreshSandboxErrors.has(error)
369
+ );
370
+ }
371
+
372
+ /**
373
+ * A burst of simultaneous turn starts can leave many freshly created provider
374
+ * boxes command-unready at once. The single replacement re-enters admission
375
+ * after a jittered pause so the replacement creates do not re-synchronize into
376
+ * the same burst. Bounded to [2 s, 10 s).
377
+ */
378
+ export const FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2_000;
379
+ export const FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8_000;
380
+
381
+ export function freshSandboxReadinessReplacementDelayMs(
382
+ random: () => number = Math.random,
383
+ ): number {
384
+ const sample = random();
385
+ const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
386
+ return (
387
+ FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS +
388
+ Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS)
389
+ );
390
+ }
391
+
392
+ /** Replacements allowed per turn attempt, across every resumeBoxForTurn call. */
393
+ export const FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
394
+
395
+ export type FreshSandboxReadinessReplacementBudget = { remaining: number };
396
+
397
+ export function createFreshSandboxReadinessReplacementBudget(): FreshSandboxReadinessReplacementBudget {
398
+ return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
399
+ }
400
+
401
+ function recordFreshSandboxReadinessReplacement(
402
+ metrics: RuntimeMetricsHooks | undefined,
403
+ backend: string,
404
+ outcome: SandboxReadinessReplacementOutcome,
405
+ ): void {
406
+ try {
407
+ metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
408
+ } catch {
409
+ // Metrics emission must never affect sandbox recovery or error propagation.
410
+ }
411
+ }
412
+
413
+ async function sleepUnlessCancelled(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
414
+ if (signal?.aborted) return false;
415
+ if (ms <= 0) return true;
416
+ return await new Promise<boolean>((resolve) => {
417
+ const onAbort = () => {
418
+ clearTimeout(timer);
419
+ resolve(false);
420
+ };
421
+ const timer = setTimeout(() => {
422
+ signal?.removeEventListener("abort", onAbort);
423
+ resolve(true);
424
+ }, ms);
425
+ signal?.addEventListener("abort", onAbort, { once: true });
426
+ });
427
+ }
428
+
320
429
  /** The exact attached caller that won warm->cold after proving the provider
321
430
  * instance gone. It is a lease supersession (the same logical turn recovers),
322
431
  * plus the lost id needed for one durable observability event. */
@@ -335,10 +444,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
335
444
  // user-facing and separate from the lease TTL heartbeat/reaper horizon.
336
445
  const WARMING_POLL_INTERVAL_MS = 250;
337
446
 
338
- async function sleep(ms: number): Promise<void> {
339
- await new Promise<void>((resolve) => setTimeout(resolve, ms));
340
- }
341
-
342
447
  /**
343
448
  * A remote provider may return a sandbox handle before its command router
344
449
  * accepts the first exec. The upstream session's yieldTimeMs starts only after
@@ -349,9 +454,10 @@ export async function waitForSandboxExecReadiness(
349
454
  established: EstablishedSandboxSession,
350
455
  timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
351
456
  identity: { sandboxGroupId?: string | null } = {},
457
+ signal?: AbortSignal,
352
458
  ): Promise<void> {
353
459
  try {
354
- await verifySandboxExecReadiness(established, timeoutMs);
460
+ await verifySandboxExecReadiness(established, timeoutMs, signal);
355
461
  } catch (error) {
356
462
  if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
357
463
  throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
@@ -535,8 +641,9 @@ async function materializeSpawnEnvelopeArchive(
535
641
  }
536
642
  if (!objectStorage) {
537
643
  throw new WorkspaceArchiveIntegrityError(
538
- "archive_base64_invalid",
644
+ "archive_storage_unavailable",
539
645
  "workspace archive object storage is not configured",
646
+ { retryable: true },
540
647
  );
541
648
  }
542
649
  const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
@@ -680,16 +787,23 @@ export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
680
787
  readonly settled: Promise<void>;
681
788
  };
682
789
 
790
+ /** Who owns a warm checkpoint: the exact turn attempt holding the box, or
791
+ * (`idleCheckpoint`) the reaper checkpointing a box that no turn holds and
792
+ * viewers, interactions or running background commands keep warm. The idle
793
+ * owner is fenced by its exact capture claim alone and is limited to
794
+ * point-in-time Modal captures. */
795
+ export type WarmWorkspaceSnapshotOwner = {
796
+ accountId: string;
797
+ workspaceId: string;
798
+ sandboxGroupId: string;
799
+ } & (
800
+ | { sessionId: string; turnId: string; attemptId: string; idleCheckpoint?: never }
801
+ | { idleCheckpoint: true; sessionId?: never; turnId?: never; attemptId?: never }
802
+ );
803
+
683
804
  export function maybePersistWarmWorkspaceSnapshot(
684
805
  services: SandboxResumeServices,
685
- ids: {
686
- accountId: string;
687
- workspaceId: string;
688
- sessionId: string;
689
- turnId: string;
690
- attemptId: string;
691
- sandboxGroupId: string;
692
- },
806
+ ids: WarmWorkspaceSnapshotOwner,
693
807
  session: unknown,
694
808
  leaseEpoch: number,
695
809
  signal?: AbortSignal,
@@ -725,14 +839,7 @@ export function maybePersistWarmWorkspaceSnapshot(
725
839
 
726
840
  async function persistWarmWorkspaceSnapshot(
727
841
  services: SandboxResumeServices,
728
- ids: {
729
- accountId: string;
730
- workspaceId: string;
731
- sessionId: string;
732
- turnId: string;
733
- attemptId: string;
734
- sandboxGroupId: string;
735
- },
842
+ ids: WarmWorkspaceSnapshotOwner,
736
843
  session: unknown,
737
844
  leaseEpoch: number,
738
845
  signal: AbortSignal | undefined,
@@ -753,6 +860,16 @@ async function persistWarmWorkspaceSnapshot(
753
860
  // Diagnostics cannot change provider settlement or capture-gate cleanup.
754
861
  }
755
862
  };
863
+ const reportSkipped = (backend: unknown, reason: WorkspaceCaptureSkipReason): void => {
864
+ try {
865
+ services.sandboxMetrics?.onWorkspaceCaptureSkipped?.({
866
+ backend: typeof backend === "string" ? backend : "unknown",
867
+ reason,
868
+ });
869
+ } catch {
870
+ // Telemetry never changes whether a checkpoint is attempted.
871
+ }
872
+ };
756
873
  const intervalMs = settings.sandboxSnapshotIntervalMs;
757
874
  if (intervalMs <= 0 && !force) {
758
875
  return false;
@@ -773,12 +890,13 @@ async function persistWarmWorkspaceSnapshot(
773
890
  console.error("mid-session workspace snapshot skipped (no persist primitive)", {
774
891
  backendId: typeof persistable.backendId === "string" ? persistable.backendId : null,
775
892
  });
893
+ reportSkipped(persistable.backendId, "no_persist_primitive");
776
894
  return false;
777
895
  }
778
896
  // Filesystem and directory snapshots create retained Images without
779
897
  // terminating the source Sandbox. (Modal's termination warning applies to
780
898
  // memory snapshots.) They are therefore the preferred warm-checkpoint path:
781
- // the durable capture gate pauses OpenGeni commands while the provider reads
899
+ // the durable capture gate pauses Opengeni commands while the provider reads
782
900
  // the filesystem, then the same live instance continues serving the turn.
783
901
  const workspacePersistence =
784
902
  persistable.state?.workspacePersistence ??
@@ -798,7 +916,8 @@ async function persistWarmWorkspaceSnapshot(
798
916
  lease?.liveness === "draining" &&
799
917
  lease.instanceId !== null &&
800
918
  lease.leaseEpoch === leaseEpoch;
801
- if (!lease || (!canWarmCapture && !canForceDrainingCapture)) {
919
+ const turnOwner = ids.idleCheckpoint === true ? null : ids;
920
+ if (!lease || (!canWarmCapture && (!canForceDrainingCapture || !turnOwner))) {
802
921
  console.error("mid-session workspace snapshot skipped (lease not warm)", {
803
922
  sandboxGroupId: ids.sandboxGroupId,
804
923
  expectedEpoch: leaseEpoch,
@@ -806,12 +925,15 @@ async function persistWarmWorkspaceSnapshot(
806
925
  leaseEpoch: lease?.leaseEpoch ?? null,
807
926
  hasInstance: lease?.instanceId !== null && lease?.instanceId !== undefined,
808
927
  });
928
+ reportSkipped(lease?.backend ?? persistable.backendId, "lease_not_warm");
809
929
  return false;
810
930
  }
811
931
  // A checkpoint of this exact mutation generation already protects every
812
932
  // settled operation admitted so far. Capturing it again cannot improve the
813
- // recovery point, even when the wall-clock interval has elapsed.
814
- if (lease.archiveComplete) {
933
+ // recovery point, even when the wall-clock interval has elapsed. An
934
+ // attached viewer or interaction can write without advancing the
935
+ // generation; the claim then decides whether the box is really clean.
936
+ if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) {
815
937
  return false;
816
938
  }
817
939
  if (lease.instanceId === null) {
@@ -826,11 +948,18 @@ async function persistWarmWorkspaceSnapshot(
826
948
  backend: lease.backend,
827
949
  liveInstance: capturePolicy?.liveInstance ?? null,
828
950
  });
951
+ reportSkipped(lease.backend, "capture_policy");
829
952
  return false;
830
953
  }
831
954
  const nativeModalPersistence =
832
955
  workspacePersistence === "snapshot_filesystem" ||
833
956
  workspacePersistence === "snapshot_directory";
957
+ if (!turnOwner && !nativeModalPersistence) {
958
+ // Only a paused-box image may run around the commands keeping an idle
959
+ // box warm; a file-by-file read of the running box could tear.
960
+ reportSkipped(lease.backend, "capture_policy");
961
+ return false;
962
+ }
834
963
  const checkpointBinding = nativeModalPersistence
835
964
  ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
836
965
  : null;
@@ -850,25 +979,31 @@ async function persistWarmWorkspaceSnapshot(
850
979
  minIntervalMs: force ? 0 : intervalMs,
851
980
  providerReplaySafe: capturePolicy.takeover === "same_request",
852
981
  takeoverSafe: capturePolicy.takeover !== "exclusive",
853
- ...(captureLiveness === "warm"
854
- ? {
855
- warmAttempt: {
856
- sessionId: ids.sessionId,
857
- turnId: ids.turnId,
858
- attemptId: ids.attemptId,
859
- holderId: sandboxLeaseHolderIdForAttempt(ids.attemptId),
860
- },
861
- }
862
- : {}),
982
+ // Modal pauses the box for a native snapshot, so the image is one
983
+ // instant even while background commands run.
984
+ pointInTimeCapture: nativeModalPersistence,
985
+ ...(captureLiveness !== "warm"
986
+ ? {}
987
+ : turnOwner
988
+ ? {
989
+ warmAttempt: {
990
+ sessionId: turnOwner.sessionId,
991
+ turnId: turnOwner.turnId,
992
+ attemptId: turnOwner.attemptId,
993
+ holderId: sandboxLeaseHolderIdForAttempt(turnOwner.attemptId),
994
+ },
995
+ }
996
+ : { idleCheckpoint: true }),
863
997
  });
864
998
  if (claimed.status !== "claimed") {
865
999
  // A scheduled interval not being due is normal, including after a failed
866
1000
  // attempt. Do not turn every ten-second heartbeat into an error log.
867
- if (claimed.status === "throttled") return false;
1001
+ if (claimed.status === "throttled" || claimed.status === "clean") return false;
868
1002
  console.error("mid-session workspace snapshot skipped (capture claim)", {
869
1003
  sandboxGroupId: ids.sandboxGroupId,
870
1004
  status: claimed.status,
871
1005
  });
1006
+ reportSkipped(lease.backend, claimed.status);
872
1007
  return false;
873
1008
  }
874
1009
 
@@ -915,10 +1050,20 @@ async function persistWarmWorkspaceSnapshot(
915
1050
  reason,
916
1051
  }).catch(() => undefined);
917
1052
  };
918
- // The SDK capture itself is not cancellable. Keep one owned continuation
1053
+ // A provider-native capture is not cancellable. Keep one owned continuation
919
1054
  // alive through provider settlement even if this caller's bounded wait or
920
1055
  // turn signal resolves first. Its finally block is the only normal release
921
1056
  // of the exact admission gate; a late callback cannot release a successor.
1057
+ // A host-backed spool capture reads the workspace locally, so it is
1058
+ // stopped at the snapshot timeout instead: its rejection is the physical
1059
+ // end of the reads, and holding the write fence for an unbounded local
1060
+ // read would turn a slow host into failed workspace writes.
1061
+ const captureAbort = new AbortController();
1062
+ const captureAbortTimer = setTimeout(
1063
+ () => captureAbort.abort(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
1064
+ settings.sandboxSnapshotTimeoutMs,
1065
+ );
1066
+ captureAbortTimer.unref?.();
922
1067
  const captureAndPublish = (async (): Promise<boolean> => {
923
1068
  const captureStarted = performance.now();
924
1069
  let captureOutcome: "completed" | "failed" = "failed";
@@ -932,9 +1077,11 @@ async function persistWarmWorkspaceSnapshot(
932
1077
  {
933
1078
  requestId: claimed.claim.providerRequestId,
934
1079
  strategy: capturePolicy.strategy,
1080
+ signal: captureAbort.signal,
935
1081
  },
936
1082
  Boolean(services.objectStorage),
937
1083
  );
1084
+ clearTimeout(captureAbortTimer);
938
1085
  candidate = await registerCandidate(archive);
939
1086
  const archiveDescriptor = archive.descriptor;
940
1087
  const published = await putVersion1TarArchiveOrInline({
@@ -954,9 +1101,13 @@ async function persistWarmWorkspaceSnapshot(
954
1101
  persistWarmSnapshot(db, {
955
1102
  accountId: ids.accountId,
956
1103
  workspaceId: ids.workspaceId,
957
- sessionId: ids.sessionId,
958
- turnId: ids.turnId,
959
- attemptId: ids.attemptId,
1104
+ ...(turnOwner
1105
+ ? {
1106
+ sessionId: turnOwner.sessionId,
1107
+ turnId: turnOwner.turnId,
1108
+ attemptId: turnOwner.attemptId,
1109
+ }
1110
+ : { idleCheckpoint: true as const }),
960
1111
  sandboxGroupId: ids.sandboxGroupId,
961
1112
  expectedEpoch: leaseEpoch,
962
1113
  expectedInstanceId: instanceId,
@@ -979,6 +1130,7 @@ async function persistWarmWorkspaceSnapshot(
979
1130
  await abandonCandidate(candidate.id, "snapshot_capture_failed");
980
1131
  throw error;
981
1132
  } finally {
1133
+ clearTimeout(captureAbortTimer);
982
1134
  await disposeWorkspaceArchive(archive).catch(() => {
983
1135
  console.warn("workspace archive spool cleanup failed");
984
1136
  });
@@ -1056,12 +1208,103 @@ async function persistWarmWorkspaceSnapshot(
1056
1208
  * holderId is the globally unique durable turn-attempt id. It must not be a
1057
1209
  * Temporal activity id, because activity ids are only workflow-local and
1058
1210
  * collide when sibling sessions share one sandbox group.
1211
+ *
1212
+ * A freshly created box that misses its bounded command-readiness budget is
1213
+ * replaced at most once per turn attempt (the budget is shared by every call
1214
+ * the turn's provisioner makes). The first establish has already terminated that exact
1215
+ * unpublished box, rolled its exact warming epoch back to cold (advancing the
1216
+ * epoch), and released its holder, so the replacement is an ordinary new
1217
+ * admission: it re-runs the cold->warming CAS, persists its own provider
1218
+ * instance before readiness, and may instead attach to a sibling that won the
1219
+ * CAS first. Nothing model- or tool-visible ran on the discarded box, so no
1220
+ * side effect is replayed. A second readiness timeout fails the turn exactly
1221
+ * as before; attached/resumed boxes and unconfirmed terminations never
1222
+ * replace.
1059
1223
  */
1060
1224
  export async function resumeBoxForTurn(
1061
1225
  services: SandboxResumeServices,
1062
1226
  ids: ResumeBoxIds,
1063
1227
  kind: "turn",
1064
1228
  holderId: TurnSandboxLeaseHolderId,
1229
+ ): Promise<ResumedTurnSandbox> {
1230
+ try {
1231
+ return await resumeBoxForTurnOnce(services, ids, kind, holderId);
1232
+ } catch (error) {
1233
+ if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
1234
+ const readiness = error as SandboxExecReadinessTimeoutError;
1235
+ const warn = (message: string, fields: Parameters<Observability["warn"]>[1]) => {
1236
+ if (services.observability) services.observability.warn(message, fields);
1237
+ else console.warn(message, fields);
1238
+ };
1239
+ const identity = {
1240
+ workspaceId: ids.workspaceId,
1241
+ sessionId: ids.sessionId,
1242
+ sandboxGroupId: ids.sandboxGroupId,
1243
+ backend: readiness.backend,
1244
+ instanceId: readiness.instanceId,
1245
+ readinessTimeoutMs: readiness.timeoutMs,
1246
+ };
1247
+ const budget =
1248
+ services.freshSandboxReadinessReplacementBudget ??
1249
+ createFreshSandboxReadinessReplacementBudget();
1250
+ if (budget.remaining <= 0) {
1251
+ recordFreshSandboxReadinessReplacement(
1252
+ services.sandboxMetrics,
1253
+ readiness.backend,
1254
+ "budget_spent",
1255
+ );
1256
+ warn(
1257
+ "sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
1258
+ identity,
1259
+ );
1260
+ throw error;
1261
+ }
1262
+ budget.remaining -= 1;
1263
+ const delayMs = await (
1264
+ services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs
1265
+ )();
1266
+ warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
1267
+ ...identity,
1268
+ replacementDelayMs: delayMs,
1269
+ });
1270
+ if (!(await sleepUnlessCancelled(delayMs, services.cancellationSignal))) {
1271
+ // Cancellation owns the turn boundary; surface the original typed
1272
+ // readiness failure rather than starting a replacement.
1273
+ recordFreshSandboxReadinessReplacement(
1274
+ services.sandboxMetrics,
1275
+ readiness.backend,
1276
+ "cancelled",
1277
+ );
1278
+ throw error;
1279
+ }
1280
+ try {
1281
+ const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
1282
+ recordFreshSandboxReadinessReplacement(
1283
+ services.sandboxMetrics,
1284
+ readiness.backend,
1285
+ "replaced",
1286
+ );
1287
+ return replaced;
1288
+ } catch (replacementError) {
1289
+ recordFreshSandboxReadinessReplacement(
1290
+ services.sandboxMetrics,
1291
+ readiness.backend,
1292
+ services.cancellationSignal?.aborted
1293
+ ? "cancelled"
1294
+ : replacementError instanceof SandboxExecReadinessTimeoutError
1295
+ ? "failed_again"
1296
+ : "replacement_failed",
1297
+ );
1298
+ throw replacementError;
1299
+ }
1300
+ }
1301
+ }
1302
+
1303
+ async function resumeBoxForTurnOnce(
1304
+ services: SandboxResumeServices,
1305
+ ids: ResumeBoxIds,
1306
+ kind: "turn",
1307
+ holderId: TurnSandboxLeaseHolderId,
1065
1308
  ): Promise<ResumedTurnSandbox> {
1066
1309
  const { db, settings } = services;
1067
1310
  const os = ids.os ?? "linux";
@@ -1153,13 +1396,18 @@ export async function resumeBoxForTurn(
1153
1396
  holderId,
1154
1397
  subjectId: ids.sessionId,
1155
1398
  backend: ids.backend,
1399
+ warmBilling: {
1400
+ mode: settings.sandboxWarmBillingMode,
1401
+ rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend),
1402
+ },
1156
1403
  os,
1157
1404
  // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
1158
- // conflicts on a live box already running a different image. A
1405
+ // conflicts on a live box already running a different required image. A
1159
1406
  // SandboxImageConflictError propagates while another holder is active; a
1160
1407
  // solo change requests a capture-and-drain rotation and this attempt retries
1161
1408
  // after the cold successor can safely stamp the new image.
1162
1409
  ...(ids.image ? { image: ids.image } : {}),
1410
+ ...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
1163
1411
  // RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
1164
1412
  // + conflicts on a live box under a different rig. A SandboxRigConflictError
1165
1413
  // propagates while another holder is active; a solo change uses the same
@@ -1283,6 +1531,7 @@ export async function resumeBoxForTurn(
1283
1531
  holderId,
1284
1532
  expectedEpoch: heartbeat.expectedEpoch,
1285
1533
  leaseTtlMs: heartbeat.leaseTtlMs,
1534
+ billingMode: services.settings.sandboxWarmBillingMode,
1286
1535
  })
1287
1536
  .then(async (status) => {
1288
1537
  if (heartbeat !== holderLeaseHeartbeat) return;
@@ -1374,6 +1623,14 @@ export async function resumeBoxForTurn(
1374
1623
  if (acquired.role === "spawner") {
1375
1624
  const expectedEpoch = acquired.lease.leaseEpoch;
1376
1625
  let createdEstablished: EstablishedSandboxSession | null = null;
1626
+ // Set once commitWarmingToWarm publishes this box. From then on the box is
1627
+ // the group's shared, attachable workspace: a later cancellation (Pause,
1628
+ // Steer, worker shutdown) only drops this attempt's holder. Terminating it
1629
+ // here would hand the next attempt a warm lease naming a dead box, whose
1630
+ // exact-id resume then records a lost workspace that never lost anything.
1631
+ let published = false;
1632
+ let providerCreateOperationId: string | undefined;
1633
+ let providerCreateBindingKey: string | undefined;
1377
1634
  let rematerialization: {
1378
1635
  id: string;
1379
1636
  selectedRevision: string;
@@ -1385,30 +1642,53 @@ export async function resumeBoxForTurn(
1385
1642
  > | null;
1386
1643
  } | null = null;
1387
1644
  try {
1388
- const envelope = await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
1645
+ // An audited decision to continue a definitively lost workspace on a new
1646
+ // EMPTY box: hydrate nothing (not even a per-session legacy archive) and
1647
+ // resume no prior provider identity. Every group member was already told
1648
+ // the workspace is empty; the DB accepts only this exact publication.
1649
+ const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
1650
+ const envelope = freshWorkspaceRecoveryId
1651
+ ? null
1652
+ : await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
1389
1653
  // The lease is authoritative. A legacy per-session fallback archive may
1390
1654
  // only be used after beginSandboxRematerialization imports its archive
1391
1655
  // fields under the warming-row lock and records one selected revision.
1392
1656
  // Select the fallback only when the lease has no archive truth at all. A
1393
1657
  // lease-carried unverified/invalid archive must fail closed rather than
1394
1658
  // silently substituting another revision.
1659
+ // Backstop: a session already told its workspace is empty never gets a
1660
+ // legacy per-session archive back, even if the lease marker was lost.
1661
+ const legacyFallbackRefused =
1662
+ !freshWorkspaceRecoveryId &&
1663
+ acquired.lease.recovery.archive.status === "none" &&
1664
+ workspaceArchiveFieldsFromEnvelope(envelope) !== null &&
1665
+ (await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId));
1395
1666
  const fallbackArchiveEnvelope =
1667
+ !freshWorkspaceRecoveryId &&
1668
+ !legacyFallbackRefused &&
1396
1669
  acquired.lease.recovery.archive.status === "none" &&
1397
1670
  workspaceArchiveFieldsFromEnvelope(envelope) !== null
1398
1671
  ? withoutSandboxProviderIdentity(envelope)
1399
1672
  : null;
1400
- let spawnEnvelope = fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
1673
+ let spawnEnvelope = freshWorkspaceRecoveryId
1674
+ ? null
1675
+ : legacyFallbackRefused
1676
+ ? (acquired.lease.resumeState ?? null)
1677
+ : (fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope);
1401
1678
  const archiveSource =
1402
1679
  acquired.lease.recovery.archive.status === "none"
1403
1680
  ? fallbackArchiveEnvelope
1404
1681
  : acquired.lease.resumeState;
1405
1682
  const continuityRecovery = acquired.lease.recovery.continuity;
1683
+ // A fresh-workspace spawn is deliberately archive-free; commitWarmingToWarm
1684
+ // verifies the exact decision before publication.
1406
1685
  if (
1407
- (acquired.lease.recovery.archive.status === "available" &&
1686
+ !freshWorkspaceRecoveryId &&
1687
+ ((acquired.lease.recovery.archive.status === "available" &&
1408
1688
  (acquired.lease.archiveComplete ||
1409
1689
  acquired.lease.historicalRecoveryAuthorized === true)) ||
1410
- (acquired.lease.recovery.archive.status === "none" &&
1411
- workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
1690
+ (acquired.lease.recovery.archive.status === "none" &&
1691
+ workspaceArchiveFieldsFromEnvelope(archiveSource) !== null))
1412
1692
  ) {
1413
1693
  const rematerializationId = crypto.randomUUID();
1414
1694
  const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -1451,7 +1731,11 @@ export async function resumeBoxForTurn(
1451
1731
  legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
1452
1732
  legacyProviderBinding: null,
1453
1733
  };
1454
- } else if (acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
1734
+ } else if (
1735
+ !freshWorkspaceRecoveryId &&
1736
+ acquired.lease.recovery.archive.status !== "none" &&
1737
+ !continuityRecovery
1738
+ ) {
1455
1739
  throw new SandboxLeaseRecoveryBlockedError(
1456
1740
  ids.sandboxGroupId,
1457
1741
  expectedEpoch,
@@ -1488,10 +1772,12 @@ export async function resumeBoxForTurn(
1488
1772
  ? {
1489
1773
  loadHostWorkspaceArchive: async (ref) => {
1490
1774
  try {
1491
- return await downloadWorkspaceArchiveSpool(services.objectStorage!, ref.key, {
1492
- bytes: ref.bytes,
1493
- sha256: ref.sha256,
1494
- });
1775
+ return await downloadWorkspaceArchiveSpool(
1776
+ services.objectStorage!,
1777
+ ref.key,
1778
+ { bytes: ref.bytes, sha256: ref.sha256 },
1779
+ { temporaryDirectory: workspaceArchiveDownloadTemporaryDirectory },
1780
+ );
1495
1781
  } catch (error) {
1496
1782
  if (error instanceof WorkspaceArchiveStorageError) {
1497
1783
  throw new WorkspaceArchiveIntegrityError(error.code, error.message, {
@@ -1510,6 +1796,34 @@ export async function resumeBoxForTurn(
1510
1796
  ...(services.logicalFallbackSettings
1511
1797
  ? { logicalFallbackSettings: services.logicalFallbackSettings }
1512
1798
  : {}),
1799
+ onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
1800
+ if (ids.backend !== "modal") return;
1801
+ if (!intent || !providerContext)
1802
+ throw new Error("Modal create requires the provider dispatch boundary");
1803
+ throwIfReleasedOrCancelled();
1804
+ const binding = await resolveModalCheckpointProviderBindingForSession(
1805
+ createSettings,
1806
+ providerContext,
1807
+ );
1808
+ const operationId = intent.operationId;
1809
+ await beginModalProviderCreate(db, {
1810
+ accountId: ids.accountId,
1811
+ workspaceId: ids.workspaceId,
1812
+ sandboxGroupId: ids.sandboxGroupId,
1813
+ expectedEpoch,
1814
+ operationId,
1815
+ providerBindingKey: binding.key,
1816
+ rematerializationId: rematerialization?.id ?? null,
1817
+ selectedRevision: rematerialization?.selectedRevision ?? null,
1818
+ imageId: intent.imageId,
1819
+ imageRef: createSettings.modalImageRef ?? null,
1820
+ appId: intent.appId,
1821
+ providerName: intent.name,
1822
+ requestSha256: intent.requestSha256,
1823
+ });
1824
+ providerCreateOperationId = operationId;
1825
+ providerCreateBindingKey = binding.key;
1826
+ },
1513
1827
  onSandboxCreated: async (created) => {
1514
1828
  createdEstablished = created;
1515
1829
  providerRenewalTarget = {
@@ -1517,7 +1831,16 @@ export async function resumeBoxForTurn(
1517
1831
  instanceId: created.instanceId,
1518
1832
  };
1519
1833
  providerRenewedAtMs = Date.now();
1520
- throwIfReleasedOrCancelled();
1834
+ if (
1835
+ providerCreateBindingKey &&
1836
+ !(await modalSessionMatchesCheckpointProviderBinding(
1837
+ settings,
1838
+ created.session,
1839
+ providerCreateBindingKey,
1840
+ ))
1841
+ ) {
1842
+ throw new Error("Modal creation receipt crossed the fenced provider namespace");
1843
+ }
1521
1844
  if (
1522
1845
  rematerialization &&
1523
1846
  (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)
@@ -1568,6 +1891,7 @@ export async function resumeBoxForTurn(
1568
1891
  sandboxGroupId: ids.sandboxGroupId,
1569
1892
  expectedEpoch,
1570
1893
  rematerializationId: rematerialization?.id ?? null,
1894
+ ...(providerCreateOperationId ? { providerCreateOperationId } : {}),
1571
1895
  ...(created.providerContinuity
1572
1896
  ? { continuityRecovery: created.providerContinuity }
1573
1897
  : {}),
@@ -1650,11 +1974,22 @@ export async function resumeBoxForTurn(
1650
1974
  // A sandbox handle is not sufficient evidence that an asynchronous
1651
1975
  // provider's command router is live. Do not publish a warm lease until
1652
1976
  // one bounded no-op exec works.
1653
- // On timeout the catch below terminates the box and rolls warming -> cold,
1654
- // so the next turn cold-creates instead of hanging forever on first use.
1655
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1656
- sandboxGroupId: ids.sandboxGroupId,
1657
- });
1977
+ // On timeout the catch below terminates the box and rolls warming -> cold;
1978
+ // resumeBoxForTurn may then replace a proven-terminated fresh box once.
1979
+ if (services.verifySpawnedSandboxReadiness) {
1980
+ await services.verifySpawnedSandboxReadiness(established, {
1981
+ sandboxGroupId: ids.sandboxGroupId,
1982
+ });
1983
+ } else {
1984
+ await waitForSandboxExecReadiness(
1985
+ established,
1986
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
1987
+ {
1988
+ sandboxGroupId: ids.sandboxGroupId,
1989
+ },
1990
+ cancellationSignal,
1991
+ );
1992
+ }
1658
1993
  await maybeRenewProviderExpiration(true);
1659
1994
  throwIfReleasedOrCancelled();
1660
1995
  // Fold the LIVE box into a re-resumable envelope and persist it as the
@@ -1700,16 +2035,18 @@ export async function resumeBoxForTurn(
1700
2035
  dataPlaneUrl: null,
1701
2036
  resumeBackendId: established.backendId,
1702
2037
  resumeState: resumeEnvelope,
1703
- ...(established.providerContinuity
1704
- ? { continuityRecovery: established.providerContinuity }
1705
- : rematerialization
1706
- ? {
1707
- rematerialization: {
1708
- id: rematerialization.id,
1709
- verifiedRevision: rematerialization.selectedRevision,
1710
- },
1711
- }
1712
- : {}),
2038
+ ...(freshWorkspaceRecoveryId
2039
+ ? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } }
2040
+ : established.providerContinuity
2041
+ ? { continuityRecovery: established.providerContinuity }
2042
+ : rematerialization
2043
+ ? {
2044
+ rematerialization: {
2045
+ id: rematerialization.id,
2046
+ verifiedRevision: rematerialization.selectedRevision,
2047
+ },
2048
+ }
2049
+ : {}),
1713
2050
  leaseTtlMs,
1714
2051
  });
1715
2052
  if (!committed.committed || !committed.lease) {
@@ -1739,13 +2076,21 @@ export async function resumeBoxForTurn(
1739
2076
  await release();
1740
2077
  throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1741
2078
  }
2079
+ published = true;
1742
2080
  holderLeaseHeartbeat = {
1743
2081
  expectedEpoch: committed.lease.leaseEpoch,
1744
2082
  leaseTtlMs,
1745
2083
  };
2084
+ await services.onSpawnedSandboxPublished?.();
1746
2085
  throwIfReleasedOrCancelled();
1747
2086
  return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1748
2087
  } catch (error) {
2088
+ if (published) {
2089
+ // The published warm box stays for the replacement attempt to resume
2090
+ // by exact provider id; ordinary idle drain owns its capture/teardown.
2091
+ await release();
2092
+ throw error;
2093
+ }
1749
2094
  if (error instanceof SandboxLeaseSupersededError) {
1750
2095
  await terminateEstablishedSandbox(createdEstablished);
1751
2096
  await release();
@@ -1790,6 +2135,16 @@ export async function resumeBoxForTurn(
1790
2135
  }
1791
2136
  await release();
1792
2137
  recordSandboxWarmingTimeout(services.sandboxMetrics, error);
2138
+ if (
2139
+ terminated &&
2140
+ createdEstablished &&
2141
+ error instanceof SandboxExecReadinessTimeoutError &&
2142
+ error.instanceId === createdEstablished.instanceId &&
2143
+ (createdEstablished.origin === "created" || createdEstablished.origin === "restored") &&
2144
+ !createdEstablished.providerContinuity
2145
+ ) {
2146
+ disposedUnreadyFreshSandboxErrors.add(error);
2147
+ }
1793
2148
  throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
1794
2149
  }
1795
2150
  }
@@ -1832,16 +2187,21 @@ export async function resumeBoxForTurn(
1832
2187
  });
1833
2188
  throwIfReleasedOrCancelled();
1834
2189
  // A durable `warm` row is an ownership assertion, not provider liveness.
1835
- // A provider may have ended the exact box while OpenGeni was idle. Prove
2190
+ // A provider may have ended the exact box while Opengeni was idle. Prove
1836
2191
  // the command router before handing the session to the
1837
2192
  // agent so terminal evidence enters the atomic warm->cold recovery path
1838
2193
  // below instead of surfacing inside a model-visible tool call.
1839
2194
  if (services.verifyAttachedSandboxReadiness) {
1840
2195
  await services.verifyAttachedSandboxReadiness(established);
1841
2196
  } else {
1842
- await waitForSandboxExecReadiness(established, MODAL_EXEC_READINESS_TIMEOUT_MS, {
1843
- sandboxGroupId: ids.sandboxGroupId,
1844
- });
2197
+ await waitForSandboxExecReadiness(
2198
+ established,
2199
+ MODAL_EXEC_READINESS_TIMEOUT_MS,
2200
+ {
2201
+ sandboxGroupId: ids.sandboxGroupId,
2202
+ },
2203
+ cancellationSignal,
2204
+ );
1845
2205
  }
1846
2206
  providerRenewalTarget = {
1847
2207
  backend: ids.backend,
@@ -1862,6 +2222,11 @@ export async function resumeBoxForTurn(
1862
2222
  expectedBackend: ids.backend,
1863
2223
  });
1864
2224
  if (marked.status === "marked") {
2225
+ await publishDurableSessionEvents(
2226
+ services.bus,
2227
+ ids.workspaceId,
2228
+ marked.backgroundCommandEvents,
2229
+ );
1865
2230
  await services.onSandboxLost?.({
1866
2231
  sandboxGroupId: ids.sandboxGroupId,
1867
2232
  instanceId: live.instanceId,
@@ -1901,7 +2266,13 @@ async function waitForWarm(
1901
2266
  const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
1902
2267
  let instanceId: string | null = null;
1903
2268
  while (Date.now() < deadline) {
1904
- await sleep(WARMING_POLL_INTERVAL_MS);
2269
+ // A cancelled waiter owns no box and must not keep polling for up to the
2270
+ // warming budget: its activity finalizer joins this exact promise.
2271
+ if (!(await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal))) {
2272
+ throw services.cancellationSignal?.reason instanceof Error
2273
+ ? services.cancellationSignal.reason
2274
+ : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
2275
+ }
1905
2276
  const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1906
2277
  if (!lease) {
1907
2278
  // Lease vanished (cold-reaped). Re-dispatch from scratch.