@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -14,7 +14,10 @@
|
|
|
14
14
|
// timer, viewer activity, owner task queue, or provider-specific lifecycle path
|
|
15
15
|
// in the normal drain state machine.
|
|
16
16
|
import { warnDrainSnapshotFailure } from "../sandbox-snapshot-diagnostics";
|
|
17
|
+
import { maybePersistWarmWorkspaceSnapshot } from "../sandbox-resume";
|
|
18
|
+
import { publishDurableSessionEvents } from "../session-event-fanout";
|
|
17
19
|
import { warnRetainedProcessProofFailure } from "../retained-process-diagnostics";
|
|
20
|
+
import { retainedProcessDeadlineRetryMs } from "../retained-process-retry";
|
|
18
21
|
|
|
19
22
|
import { createHash, randomUUID } from "node:crypto";
|
|
20
23
|
import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
|
|
@@ -26,12 +29,17 @@ import {
|
|
|
26
29
|
retainedProviderCommandPersistence,
|
|
27
30
|
adoptLegacyModalCheckpointArtifact,
|
|
28
31
|
confirmDrainCold,
|
|
32
|
+
markWarmLeaseInstanceLost,
|
|
29
33
|
appendSessionEventToSandboxGroup,
|
|
30
34
|
bindRetainedProcessProviderIdentity,
|
|
31
35
|
claimWorkspaceArchiveCapture,
|
|
36
|
+
leaseMayHaveUntrackedWriters,
|
|
32
37
|
releaseWorkspaceArchiveCapture,
|
|
38
|
+
listIdleCheckpointCandidates,
|
|
39
|
+
recordIdleCheckpointAttempt,
|
|
33
40
|
claimSandboxCheckpointArtifactsForGc,
|
|
34
41
|
claimTerminalRetainedProcesses,
|
|
42
|
+
getEnrollment,
|
|
35
43
|
countActiveRetainedProcessesByOwnerState,
|
|
36
44
|
countSandboxCheckpointArtifactsByState,
|
|
37
45
|
countExpiredDrainingSandboxLeases,
|
|
@@ -39,11 +47,19 @@ import {
|
|
|
39
47
|
countSandboxLeasesByLiveness,
|
|
40
48
|
deferRetainedProcessReconciliation,
|
|
41
49
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
42
|
-
getBillingBalance,
|
|
43
50
|
listCreditBalancesByAccount,
|
|
51
|
+
countActiveUsers,
|
|
52
|
+
readCreditGrantTotals,
|
|
53
|
+
readManagedAuthNewSignupsSwitch,
|
|
54
|
+
readVerifiedSignupTrialSwitch,
|
|
44
55
|
listLegacyModalCheckpointSlots,
|
|
45
56
|
listLiveModalSandboxLeaseAttributions,
|
|
57
|
+
listPendingModalProviderCreates,
|
|
58
|
+
claimModalProviderCreateRecovery,
|
|
59
|
+
recordRecoveredModalProviderCreate,
|
|
60
|
+
markWarmBillingStopCutoff,
|
|
46
61
|
listMeterableWarmLeases,
|
|
62
|
+
countInteractionOnlyWarmLeasesByIdle,
|
|
47
63
|
listSandboxViewerForceDrainWorkspaceIds,
|
|
48
64
|
markSandboxCheckpointArtifactDeletePending,
|
|
49
65
|
persistDrainSnapshot,
|
|
@@ -51,15 +67,22 @@ import {
|
|
|
51
67
|
registerSandboxCheckpointArtifact,
|
|
52
68
|
recordRetainedProcessReconciliationProof,
|
|
53
69
|
replaceWorkspaceArchiveCaptureAfterProof,
|
|
70
|
+
listEndedEpochWorkspaceBlockers,
|
|
71
|
+
listEndedEpochSandboxSetupUncertainties,
|
|
72
|
+
reconcileCompletedSandboxSetup,
|
|
54
73
|
readLease,
|
|
74
|
+
settleEndedEpochWorkspaceBlockers,
|
|
55
75
|
readWorkspaceArchiveCapturePreflight,
|
|
56
76
|
reapExpiredSessionListSnapshots,
|
|
57
77
|
reapStaleLeaseHoldersGlobal,
|
|
58
78
|
requestDueSandboxRotationsGlobal,
|
|
59
79
|
readSandboxRotationBacklog,
|
|
80
|
+
readSandboxCheckpointStaleness,
|
|
81
|
+
readRecentSandboxRecoveryObservations,
|
|
60
82
|
workspaceArchiveCaptureDeadlineElapsed,
|
|
61
83
|
retainedProcessReconciliationProof,
|
|
62
84
|
retainedProcessSettlementIdentity,
|
|
85
|
+
recoverManagedSessionBackgroundCommand,
|
|
63
86
|
settleClaimedConnectedMachineBackgroundCommand,
|
|
64
87
|
settleSandboxCheckpointArtifactGc,
|
|
65
88
|
rlsContextForWorkspace,
|
|
@@ -71,13 +94,21 @@ import {
|
|
|
71
94
|
type LeaseSnapshot,
|
|
72
95
|
} from "@opengeni/db";
|
|
73
96
|
import {
|
|
97
|
+
claimConnectedCommandOutputReleases,
|
|
74
98
|
claimConnectedMachineSessionBackgroundCommands,
|
|
75
99
|
deferConnectedMachineBackgroundCommandReconciliation,
|
|
76
100
|
recordConnectedMachineBackgroundCommandProof,
|
|
101
|
+
recordConnectedCommandOutputConsumption,
|
|
102
|
+
settleConnectedCommandOutputReleaseClaim,
|
|
77
103
|
type ConnectedMachineBackgroundCommandClaim,
|
|
78
104
|
type ConnectedMachineBackgroundCommandProof,
|
|
79
105
|
} from "@opengeni/db/session-background-commands";
|
|
80
|
-
import {
|
|
106
|
+
import {
|
|
107
|
+
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
108
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
109
|
+
sandboxDeadlineMandatoryCaptureLeadMs,
|
|
110
|
+
sandboxWarmRateMicrosPerSecond,
|
|
111
|
+
} from "@opengeni/config";
|
|
81
112
|
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
82
113
|
import { captureConnectedCommandOutput } from "../sandbox-routing";
|
|
83
114
|
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
@@ -109,22 +140,25 @@ import {
|
|
|
109
140
|
parseExecBannerExitCode,
|
|
110
141
|
prepareProviderForTeardownAfterCapture,
|
|
111
142
|
providerWorkspaceCapturePolicy,
|
|
143
|
+
providerWorkspaceCaptureIsPointInTime,
|
|
112
144
|
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
113
145
|
resolveModalCheckpointProviderBinding,
|
|
146
|
+
findModalProviderCreateReceipt,
|
|
114
147
|
resolveModalCheckpointProviderBindingForSession,
|
|
115
148
|
querySelfhostedOp,
|
|
116
149
|
resumeExactSandboxSession,
|
|
117
|
-
sandboxProviderContinuityForState,
|
|
118
150
|
sandboxBackendForSdkBackendId,
|
|
119
151
|
sandboxProviderInstanceIdFromEnvelope,
|
|
120
152
|
sandboxCommandExitCode,
|
|
121
153
|
sandboxCommandStillRunning,
|
|
122
154
|
SelfhostedControlError,
|
|
123
155
|
subjectFor,
|
|
156
|
+
modalAppHasRunningSandboxes,
|
|
124
157
|
sweepModalOrphanSandboxes,
|
|
125
158
|
terminateManagedSandboxSession,
|
|
126
159
|
terminateModalSandboxById,
|
|
127
160
|
verifySandboxExecReadiness,
|
|
161
|
+
type DockerHostWorkspaceRelease,
|
|
128
162
|
type ModalOrphanSweepTermination,
|
|
129
163
|
type ModalCheckpointProviderBinding,
|
|
130
164
|
type ProviderWorkspaceCapturePolicy,
|
|
@@ -137,13 +171,20 @@ import {
|
|
|
137
171
|
SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
|
|
138
172
|
type SandboxDrainActivityInput,
|
|
139
173
|
type SandboxLeaseSweepMaintenanceInput,
|
|
174
|
+
sandboxDrainTimeoutClass,
|
|
175
|
+
sandboxIdleCheckpointWorkflowId,
|
|
176
|
+
type SandboxIdleCheckpointActivityInput,
|
|
177
|
+
type SandboxIdleCheckpointPlan,
|
|
140
178
|
} from "../sandbox-reaper-contract";
|
|
141
179
|
import { CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES } from "../concurrency";
|
|
180
|
+
import { browserCheckpointPrerequisites } from "./browser-deadline-checkpoint";
|
|
142
181
|
import { assertSandboxDrainInputTiming, sandboxDrainTiming } from "../sandbox-reaper-timeout";
|
|
143
182
|
import type { ControlActivityServices as ActivityServices } from "./types";
|
|
144
183
|
import { reconcilePendingParentSystemUpdates } from "./parent-wake";
|
|
145
184
|
import {
|
|
185
|
+
recordActiveUserGauges,
|
|
146
186
|
recordCreditBalanceGauges,
|
|
187
|
+
recordCreditGrantGauges,
|
|
147
188
|
recordCreditMicros,
|
|
148
189
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
149
190
|
recordOpenSandboxKubernetesInventoryGauges,
|
|
@@ -157,9 +198,20 @@ import {
|
|
|
157
198
|
type RetainedProcessReconciliationOutcome,
|
|
158
199
|
type SandboxInventoryProjectionDomain,
|
|
159
200
|
recordSandboxLeaseGauges,
|
|
201
|
+
recordModalSandboxInventoryGauges,
|
|
202
|
+
recordInteractionOnlyLeaseGauges,
|
|
160
203
|
recordSandboxOrphansTerminated,
|
|
204
|
+
recordSandboxCommandContainment,
|
|
205
|
+
recordSandboxIdleInteractionRelease,
|
|
206
|
+
recordSandboxProviderMissingBeforeCapture,
|
|
207
|
+
recordSandboxDockerWorkspaceRelease,
|
|
208
|
+
recordSandboxRecoveryObservationGauges,
|
|
161
209
|
recordSandboxRotationBacklogGauges,
|
|
210
|
+
recordSandboxCheckpointStalenessGauges,
|
|
162
211
|
recordTurnsQueuedGauge,
|
|
212
|
+
recordVerifiedSignupTrialDeploymentFlagGauge,
|
|
213
|
+
recordManagedAuthNewSignupsSwitchGauge,
|
|
214
|
+
recordVerifiedSignupTrialSwitchGauge,
|
|
163
215
|
runtimeMetricsHooksForObservability,
|
|
164
216
|
} from "../observability-metrics";
|
|
165
217
|
import {
|
|
@@ -186,8 +238,7 @@ export type ReapSandboxLeasesResult = {
|
|
|
186
238
|
skipped: number;
|
|
187
239
|
/** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
|
|
188
240
|
metered: number;
|
|
189
|
-
/**
|
|
190
|
-
* (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
|
|
241
|
+
/** Holderless boxes accelerated to draining by the optional warm cap. */
|
|
191
242
|
forceDrained: number;
|
|
192
243
|
/** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
|
|
193
244
|
modalOrphansTerminated: number;
|
|
@@ -200,6 +251,10 @@ type DrainSandboxClient = {
|
|
|
200
251
|
resume?: (state: unknown) => Promise<unknown>;
|
|
201
252
|
deserializeSessionState?: (state: Record<string, unknown>) => Promise<unknown>;
|
|
202
253
|
delete?: (state: unknown) => Promise<void>;
|
|
254
|
+
attachWorkspaceForDrain?: (
|
|
255
|
+
state: unknown,
|
|
256
|
+
assertCurrentCapture: () => Promise<{ archivePublished: boolean }>,
|
|
257
|
+
) => Promise<PersistableSession>;
|
|
203
258
|
};
|
|
204
259
|
|
|
205
260
|
/**
|
|
@@ -256,6 +311,8 @@ export type TerminateBoxFn = (
|
|
|
256
311
|
diskBackedArchives?: boolean,
|
|
257
312
|
releaseFailedCapture?: () => Promise<void>,
|
|
258
313
|
workspaceId?: string,
|
|
314
|
+
beforeProviderStop?: () => Promise<void>,
|
|
315
|
+
beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,
|
|
259
316
|
) => Promise<boolean | ProviderTerminationOutcome>;
|
|
260
317
|
|
|
261
318
|
export type SweepModalOrphansFn = (
|
|
@@ -275,6 +332,8 @@ export type SandboxLeaseActivityOptions = {
|
|
|
275
332
|
/** Override the provider-side Modal orphan sweep (tests spy this; defaults to
|
|
276
333
|
* Modal list+tag comparison when Modal is configured). */
|
|
277
334
|
sweepModalOrphans?: SweepModalOrphansFn;
|
|
335
|
+
/** Only provider discovery is injected; durable recovery remains real. */
|
|
336
|
+
findModalProviderCreateReceipt?: typeof findModalProviderCreateReceipt;
|
|
278
337
|
/** Override the read-only bounded OpenSandbox Kubernetes projection. */
|
|
279
338
|
inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;
|
|
280
339
|
/** Override only the read-only provider process probe. The canonical DB
|
|
@@ -286,8 +345,20 @@ export type SandboxLeaseActivityOptions = {
|
|
|
286
345
|
/** Override the read-only exact-instance readiness probe used before the
|
|
287
346
|
* reaper settles blockers for a provider that has definitively vanished. */
|
|
288
347
|
probeDrainableProvider?: DrainableProviderProbeFn;
|
|
348
|
+
/** Override only the resume-by-id of the box an idle checkpoint captures.
|
|
349
|
+
* Returns the live session, or null when the provider no longer has it. The
|
|
350
|
+
* capture claim, publication and release remain real. */
|
|
351
|
+
resumeIdleCheckpointSession?: ResumeIdleCheckpointSessionFn;
|
|
352
|
+
/** Override only the provider client factory of the default drain. The
|
|
353
|
+
* claim, capture, publication and teardown ordering remain real. */
|
|
354
|
+
createClientForBackend?: CreateSandboxClientForBackendFn;
|
|
289
355
|
};
|
|
290
356
|
|
|
357
|
+
export type ResumeIdleCheckpointSessionFn = (
|
|
358
|
+
settings: ActivityServices["settings"],
|
|
359
|
+
lease: LeaseSnapshot,
|
|
360
|
+
) => Promise<unknown | null>;
|
|
361
|
+
|
|
291
362
|
export type RetainedProcessProbeResult =
|
|
292
363
|
| { status: "proved"; proof: RetainedProcessProviderProof }
|
|
293
364
|
| {
|
|
@@ -308,6 +379,8 @@ export type RetainedProcessProbeResult =
|
|
|
308
379
|
| "provider_binding_missing"
|
|
309
380
|
| "provider_binding_mismatch"
|
|
310
381
|
| "process_observation_unavailable";
|
|
382
|
+
/** A running command whose last read still returned a large backlog. */
|
|
383
|
+
outputBacklog?: boolean;
|
|
311
384
|
};
|
|
312
385
|
|
|
313
386
|
export type RetainedProcessProbeFn = (
|
|
@@ -322,8 +395,12 @@ export type RetainedProcessProbeFn = (
|
|
|
322
395
|
streamFidelity?: "separate" | "merged",
|
|
323
396
|
) => Promise<void>,
|
|
324
397
|
providerPersistence?: ProviderCommandPersistence,
|
|
398
|
+
drainBudget?: RetainedProcessDrainBudget,
|
|
325
399
|
) => Promise<RetainedProcessProbeResult>;
|
|
326
400
|
|
|
401
|
+
/** Shared wall-clock bound on output draining within one reconciliation sweep. */
|
|
402
|
+
export type RetainedProcessDrainBudget = { until: number };
|
|
403
|
+
|
|
327
404
|
export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
|
|
328
405
|
|
|
329
406
|
export type DrainableProviderProbeFn = (
|
|
@@ -334,6 +411,16 @@ export type DrainableProviderProbeFn = (
|
|
|
334
411
|
export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
335
412
|
export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
|
|
336
413
|
export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
|
|
414
|
+
/** Bounds on reading a command's output backlog: per claim, and across one
|
|
415
|
+
* reconciliation sweep so the reaper activity stays well inside its timeout.
|
|
416
|
+
* Each read returns up to one provider page per stream. */
|
|
417
|
+
export const RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
|
|
418
|
+
export const RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 10_000;
|
|
419
|
+
export const RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 60_000;
|
|
420
|
+
/** A live process whose read returned at least this much output left a backlog
|
|
421
|
+
* behind it. A server or watcher that only trickles output stays on ordinary
|
|
422
|
+
* backoff; a process the provider reports exited is always read to EOF. */
|
|
423
|
+
export const RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
|
|
337
424
|
export const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
338
425
|
export const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;
|
|
339
426
|
export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
@@ -364,6 +451,9 @@ export async function awaitProviderCaptureWithLatePublication<T>(input: {
|
|
|
364
451
|
publishLate: (value: T) => Promise<void>;
|
|
365
452
|
observeLateFailure: (error: unknown) => void;
|
|
366
453
|
observeLatePublicationFailure: (error: unknown) => void;
|
|
454
|
+
/** Called once the detached late path has fully settled (after publication
|
|
455
|
+
* or failure handling), never on the in-budget path. */
|
|
456
|
+
onLateSettled?: () => void;
|
|
367
457
|
}): Promise<T> {
|
|
368
458
|
let timeout: ReturnType<typeof setTimeout> | undefined;
|
|
369
459
|
const timeoutError = input.timeoutError;
|
|
@@ -381,9 +471,17 @@ export async function awaitProviderCaptureWithLatePublication<T>(input: {
|
|
|
381
471
|
// durable archive publication: never provider teardown or a cold commit from
|
|
382
472
|
// an activity Temporal has already rejected. A replaced claim fences this
|
|
383
473
|
// callback; Modal candidates then flow to artifact-ledger GC.
|
|
474
|
+
const settled = () => {
|
|
475
|
+
try {
|
|
476
|
+
input.onLateSettled?.();
|
|
477
|
+
} catch {
|
|
478
|
+
// Settlement bookkeeping cannot surface as an unhandled rejection.
|
|
479
|
+
}
|
|
480
|
+
};
|
|
384
481
|
void input.capture
|
|
385
482
|
.then(input.publishLate, input.observeLateFailure)
|
|
386
|
-
.catch(input.observeLatePublicationFailure)
|
|
483
|
+
.catch(input.observeLatePublicationFailure)
|
|
484
|
+
.then(settled, settled);
|
|
387
485
|
throw error;
|
|
388
486
|
} finally {
|
|
389
487
|
if (timeout) clearTimeout(timeout);
|
|
@@ -425,6 +523,20 @@ export type SandboxLeaseSweepPlan = {
|
|
|
425
523
|
rotationsRequested: number;
|
|
426
524
|
};
|
|
427
525
|
|
|
526
|
+
export type SandboxIdleCheckpointActivityResult = {
|
|
527
|
+
status: "checkpointed" | "skipped";
|
|
528
|
+
reason?: string;
|
|
529
|
+
};
|
|
530
|
+
|
|
531
|
+
/** Children per sweep; the next sweep continues with the least recently tried
|
|
532
|
+
* boxes (each box is tried at most once per snapshot interval). Half the
|
|
533
|
+
* lifecycle worker's activity slots, so a burst of captures never queues the
|
|
534
|
+
* drains that stop idle boxes behind it. */
|
|
535
|
+
const SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(
|
|
536
|
+
1,
|
|
537
|
+
Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2),
|
|
538
|
+
);
|
|
539
|
+
|
|
428
540
|
export type SandboxLeaseSweepMaintenanceResult = {
|
|
429
541
|
metered: number;
|
|
430
542
|
forceDrained: number;
|
|
@@ -447,6 +559,55 @@ class SandboxDrainRecoveryDeferredError extends Error {
|
|
|
447
559
|
readonly name = "SandboxDrainRecoveryDeferredError";
|
|
448
560
|
}
|
|
449
561
|
|
|
562
|
+
/**
|
|
563
|
+
* Drain captures still physically running in this worker, keyed by exact lease
|
|
564
|
+
* epoch and provider instance. A capture that outlives its attempt's budget
|
|
565
|
+
* keeps running and may publish late through its claim's provider-request
|
|
566
|
+
* lineage. A parallel-read provider (Docker, for one) also lets the next
|
|
567
|
+
* attempt replace that unpublished claim at once and start its own full read,
|
|
568
|
+
* so a capture slower than the retry interval used to gain one more concurrent
|
|
569
|
+
* read and host spool per retry. This keeps one physical capture per lease
|
|
570
|
+
* epoch and instance in each worker; the durable claim still decides which
|
|
571
|
+
* archive publishes, and a capture started by another worker (or one that died
|
|
572
|
+
* with its process) is handled by the ordinary claim replacement. An entry
|
|
573
|
+
* older than the longest durable capture window no longer defers, so a
|
|
574
|
+
* provider call that never settles cannot pin the lease to this worker forever.
|
|
575
|
+
*/
|
|
576
|
+
const settlingDrainCaptures = new Map<string, { startedAtMs: number }>();
|
|
577
|
+
|
|
578
|
+
type DrainCaptureIdentity = {
|
|
579
|
+
sandboxGroupId: string;
|
|
580
|
+
leaseEpoch: number;
|
|
581
|
+
instanceId: string | null;
|
|
582
|
+
};
|
|
583
|
+
|
|
584
|
+
function drainCaptureKey(lease: DrainCaptureIdentity & { instanceId: string }): string {
|
|
585
|
+
return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
/** True while this worker still runs (or publishes, or removes the spool of)
|
|
589
|
+
* a drain capture of this exact lease epoch and instance. */
|
|
590
|
+
function sandboxDrainCaptureSettlingInProcess(
|
|
591
|
+
lease: DrainCaptureIdentity,
|
|
592
|
+
nowMs = Date.now(),
|
|
593
|
+
): boolean {
|
|
594
|
+
if (!lease.instanceId) return false;
|
|
595
|
+
const entry = settlingDrainCaptures.get(
|
|
596
|
+
drainCaptureKey({ ...lease, instanceId: lease.instanceId }),
|
|
597
|
+
);
|
|
598
|
+
return entry !== undefined && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
function trackSettlingDrainCapture(lease: DrainCaptureIdentity): () => void {
|
|
602
|
+
if (!lease.instanceId) return () => undefined;
|
|
603
|
+
const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });
|
|
604
|
+
const entry = { startedAtMs: Date.now() };
|
|
605
|
+
settlingDrainCaptures.set(key, entry);
|
|
606
|
+
return () => {
|
|
607
|
+
if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);
|
|
608
|
+
};
|
|
609
|
+
}
|
|
610
|
+
|
|
450
611
|
/** Stable UUID per logical operation + Temporal attempt. Attempt one uses the
|
|
451
612
|
* operation id itself; later attempts are SHA-256-derived UUIDv5-shaped values.
|
|
452
613
|
* This makes the DB receipt idempotent even if an activity delivery is replayed. */
|
|
@@ -482,7 +643,14 @@ export function createSandboxLeaseActivities(
|
|
|
482
643
|
options: SandboxLeaseActivityOptions = {},
|
|
483
644
|
) {
|
|
484
645
|
const terminateBox: TerminateBoxFn =
|
|
485
|
-
options.terminateBox
|
|
646
|
+
(options.terminateBox
|
|
647
|
+
? async (...args: Parameters<TerminateBoxFn>) => {
|
|
648
|
+
// The injected test seam represents the stop itself; the real
|
|
649
|
+
// implementation invokes this only after capture has succeeded.
|
|
650
|
+
await args[10]?.();
|
|
651
|
+
return options.terminateBox!(...args);
|
|
652
|
+
}
|
|
653
|
+
: null) ??
|
|
486
654
|
(async (
|
|
487
655
|
settings,
|
|
488
656
|
lease,
|
|
@@ -494,13 +662,15 @@ export function createSandboxLeaseActivities(
|
|
|
494
662
|
diskBackedArchives,
|
|
495
663
|
releaseFailedCapture,
|
|
496
664
|
workspaceId,
|
|
665
|
+
beforeProviderStop,
|
|
666
|
+
beforeWorkspaceCapture,
|
|
497
667
|
) =>
|
|
498
668
|
await terminateProviderBox(
|
|
499
669
|
settings,
|
|
500
670
|
lease,
|
|
501
671
|
observability,
|
|
502
672
|
persistArchive,
|
|
503
|
-
createSandboxClientForBackend,
|
|
673
|
+
options.createClientForBackend ?? createSandboxClientForBackend,
|
|
504
674
|
terminateModalSandboxById,
|
|
505
675
|
providerCaptureRequestId,
|
|
506
676
|
captureDisposition,
|
|
@@ -508,6 +678,8 @@ export function createSandboxLeaseActivities(
|
|
|
508
678
|
diskBackedArchives,
|
|
509
679
|
releaseFailedCapture,
|
|
510
680
|
workspaceId,
|
|
681
|
+
beforeProviderStop,
|
|
682
|
+
beforeWorkspaceCapture,
|
|
511
683
|
));
|
|
512
684
|
const sweepModalOrphans: SweepModalOrphansFn =
|
|
513
685
|
options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
@@ -515,15 +687,35 @@ export function createSandboxLeaseActivities(
|
|
|
515
687
|
options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
516
688
|
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
517
689
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
690
|
+
const resumeIdleCheckpointSession =
|
|
691
|
+
options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;
|
|
692
|
+
/** Embedded idle checkpoints still running, by exact lease epoch. */
|
|
693
|
+
const embeddedIdleCheckpoints = new Set<string>();
|
|
694
|
+
const endedEpochProbeBackoff: EndedEpochProbeBackoff = new Map();
|
|
518
695
|
async function prepareSandboxLeaseSweep(): Promise<SandboxLeaseSweepPlan> {
|
|
519
696
|
const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
|
|
520
697
|
try {
|
|
521
|
-
const { db, settings, observability } = await services();
|
|
698
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
522
699
|
const timing = sandboxDrainTiming(settings);
|
|
523
|
-
const
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
700
|
+
const commandContainment = {
|
|
701
|
+
// Browsers and desktops follow the general idle grace: unused for it,
|
|
702
|
+
// they stop holding the box (saved browsers resume on demand).
|
|
703
|
+
// Never under a minute, so a visible live view's 30 s heartbeat
|
|
704
|
+
// always lands inside the window.
|
|
705
|
+
idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 60_000),
|
|
706
|
+
idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),
|
|
707
|
+
onIdleInteractionRelease: (
|
|
708
|
+
outcome: Parameters<typeof recordSandboxIdleInteractionRelease>[1],
|
|
709
|
+
) => recordSandboxIdleInteractionRelease(observability, outcome),
|
|
710
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
|
|
711
|
+
deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
|
|
712
|
+
onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>
|
|
713
|
+
recordSandboxCommandContainment(observability, outcome),
|
|
714
|
+
onCommandContainmentError: (error: unknown) => {
|
|
715
|
+
observability.warn("sandbox reaper: retained command containment inspection failed", {
|
|
716
|
+
error: error instanceof Error ? error.message : String(error),
|
|
717
|
+
});
|
|
718
|
+
},
|
|
527
719
|
};
|
|
528
720
|
if (!settings.sandboxOwnershipEnabled) {
|
|
529
721
|
// Turns skip leases when the flag is off, but Computer/Browser attach
|
|
@@ -531,7 +723,7 @@ export function createSandboxLeaseActivities(
|
|
|
531
723
|
// behind rotation_in_progress forever. Inventory and drain those rows;
|
|
532
724
|
// do not request NEW deadline rotations or meter warm time.
|
|
533
725
|
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
534
|
-
|
|
726
|
+
...commandContainment,
|
|
535
727
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
536
728
|
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
537
729
|
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
@@ -568,7 +760,7 @@ export function createSandboxLeaseActivities(
|
|
|
568
760
|
// Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
|
|
569
761
|
// run only after every drainable box has its own durable child.
|
|
570
762
|
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
571
|
-
|
|
763
|
+
...commandContainment,
|
|
572
764
|
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
573
765
|
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
574
766
|
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
@@ -601,6 +793,141 @@ export function createSandboxLeaseActivities(
|
|
|
601
793
|
}
|
|
602
794
|
}
|
|
603
795
|
|
|
796
|
+
/** Inventory for the idle checkpoint sweep. DB only: warm boxes no turn
|
|
797
|
+
* holds, kept warm by viewers, interactions or running background commands,
|
|
798
|
+
* with writes no capture may cover yet and neither a capture attempt nor an
|
|
799
|
+
* idle checkpoint attempt within the snapshot interval.
|
|
800
|
+
* Least recently tried first; the next sweep continues past the batch. */
|
|
801
|
+
async function listIdleSandboxCheckpoints(): Promise<SandboxIdleCheckpointPlan> {
|
|
802
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "idle_checkpoint_inventory" });
|
|
803
|
+
try {
|
|
804
|
+
const { db, settings } = await services();
|
|
805
|
+
// The idle capture claims the warm checkpoint budget, not the drain's;
|
|
806
|
+
// freeze the matching activity class with each child.
|
|
807
|
+
const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));
|
|
808
|
+
if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
|
|
809
|
+
return { targets: [], timeoutClass };
|
|
810
|
+
}
|
|
811
|
+
const rows = await listIdleCheckpointCandidates(db, {
|
|
812
|
+
limit: SANDBOX_IDLE_CHECKPOINT_BATCH,
|
|
813
|
+
intervalMs: settings.sandboxSnapshotIntervalMs,
|
|
814
|
+
});
|
|
815
|
+
return {
|
|
816
|
+
targets: rows.map(({ accountId: _accountId, ...target }) => target),
|
|
817
|
+
timeoutClass,
|
|
818
|
+
};
|
|
819
|
+
} finally {
|
|
820
|
+
stopHeartbeat();
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/** Checkpoint one warm box between turns. Reuses the turn's warm checkpoint
|
|
825
|
+
* path with an idle owner: the exact capture claim requires that only
|
|
826
|
+
* viewers, interactions and active background commands hold the
|
|
827
|
+
* box and that no other request is open, then the native Modal snapshot runs
|
|
828
|
+
* around them and is published one generation behind the workspace. */
|
|
829
|
+
async function checkpointIdleSandboxLease(
|
|
830
|
+
input: SandboxIdleCheckpointActivityInput,
|
|
831
|
+
): Promise<SandboxIdleCheckpointActivityResult> {
|
|
832
|
+
const target = input.target;
|
|
833
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
834
|
+
phase: "idle_checkpoint",
|
|
835
|
+
workspaceId: target.workspaceId,
|
|
836
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
837
|
+
leaseEpoch: target.leaseEpoch,
|
|
838
|
+
instanceId: target.instanceId,
|
|
839
|
+
});
|
|
840
|
+
try {
|
|
841
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
842
|
+
const fields = {
|
|
843
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
|
|
844
|
+
leaseEpoch: target.leaseEpoch,
|
|
845
|
+
};
|
|
846
|
+
const skipped = (reason: string): SandboxIdleCheckpointActivityResult => {
|
|
847
|
+
observability.info("sandbox reaper: idle checkpoint skipped", { ...fields, reason });
|
|
848
|
+
return { status: "skipped", reason };
|
|
849
|
+
};
|
|
850
|
+
if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
|
|
851
|
+
return skipped("disabled");
|
|
852
|
+
}
|
|
853
|
+
const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
|
|
854
|
+
if (
|
|
855
|
+
!lease ||
|
|
856
|
+
lease.liveness !== "warm" ||
|
|
857
|
+
lease.leaseEpoch !== target.leaseEpoch ||
|
|
858
|
+
lease.instanceId !== target.instanceId
|
|
859
|
+
) {
|
|
860
|
+
return skipped("lease_fenced");
|
|
861
|
+
}
|
|
862
|
+
// A turn's own checkpoints cover a box it holds; a box nothing holds
|
|
863
|
+
// belongs to the idle drain.
|
|
864
|
+
if (lease.turnHolders > 0) return skipped("turn_held");
|
|
865
|
+
if (lease.refcount === 0) return skipped("drain_owned");
|
|
866
|
+
const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);
|
|
867
|
+
// Counted as tried whatever happens next, so a refused or failing box
|
|
868
|
+
// waits a full interval instead of taking a batch slot every sweep.
|
|
869
|
+
const stamped = await recordIdleCheckpointAttempt(db, {
|
|
870
|
+
accountId,
|
|
871
|
+
workspaceId: target.workspaceId,
|
|
872
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
873
|
+
expectedEpoch: target.leaseEpoch,
|
|
874
|
+
expectedInstanceId: target.instanceId,
|
|
875
|
+
});
|
|
876
|
+
if (!stamped) return skipped("lease_fenced");
|
|
877
|
+
if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {
|
|
878
|
+
return skipped("capture_policy");
|
|
879
|
+
}
|
|
880
|
+
if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped("clean");
|
|
881
|
+
if (lease.archiveCapture) {
|
|
882
|
+
const tookOver = await releaseWorkspaceArchiveCapture(db, {
|
|
883
|
+
accountId,
|
|
884
|
+
workspaceId: target.workspaceId,
|
|
885
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
886
|
+
captureId: lease.archiveCapture.id,
|
|
887
|
+
expectedEpoch: target.leaseEpoch,
|
|
888
|
+
expectedInstanceId: target.instanceId,
|
|
889
|
+
onlyExpiredWarmTakeover: true,
|
|
890
|
+
});
|
|
891
|
+
if (!tookOver) return skipped("capture_in_progress");
|
|
892
|
+
observability.warn("sandbox reaper: released an expired warm capture claim", {
|
|
893
|
+
...fields,
|
|
894
|
+
captureId: lease.archiveCapture.id,
|
|
895
|
+
});
|
|
896
|
+
}
|
|
897
|
+
let session: unknown;
|
|
898
|
+
try {
|
|
899
|
+
session = await resumeIdleCheckpointSession(settings, lease);
|
|
900
|
+
} catch (error) {
|
|
901
|
+
observability.warn("sandbox reaper: idle checkpoint resume failed", {
|
|
902
|
+
...fields,
|
|
903
|
+
error: error instanceof Error ? error.message : String(error),
|
|
904
|
+
});
|
|
905
|
+
return skipped("provider_error");
|
|
906
|
+
}
|
|
907
|
+
if (session === null) return skipped("provider_missing");
|
|
908
|
+
const capture = maybePersistWarmWorkspaceSnapshot(
|
|
909
|
+
{ db, settings, objectStorage, observability },
|
|
910
|
+
{
|
|
911
|
+
accountId,
|
|
912
|
+
workspaceId: target.workspaceId,
|
|
913
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
914
|
+
idleCheckpoint: true,
|
|
915
|
+
},
|
|
916
|
+
session,
|
|
917
|
+
target.leaseEpoch,
|
|
918
|
+
);
|
|
919
|
+
const persisted = await capture;
|
|
920
|
+
// Return only after the provider capture settled and its exact claim
|
|
921
|
+
// was released, so the activity never outlives its own gate unobserved.
|
|
922
|
+
await capture.settled;
|
|
923
|
+
if (!persisted) return skipped("not_captured");
|
|
924
|
+
observability.info("sandbox reaper: idle checkpoint published", fields);
|
|
925
|
+
return { status: "checkpointed" };
|
|
926
|
+
} finally {
|
|
927
|
+
stopHeartbeat();
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
|
|
604
931
|
async function drainSandboxLease(
|
|
605
932
|
input: SandboxDrainActivityInput,
|
|
606
933
|
): Promise<SandboxDrainActivityResult> {
|
|
@@ -612,7 +939,7 @@ export function createSandboxLeaseActivities(
|
|
|
612
939
|
instanceId: input.target.instanceId,
|
|
613
940
|
});
|
|
614
941
|
try {
|
|
615
|
-
const { db, settings, observability, objectStorage } = await services();
|
|
942
|
+
const { db, settings, observability, objectStorage, bus } = await services();
|
|
616
943
|
assertSandboxDrainInputTiming(input);
|
|
617
944
|
const drainSettings =
|
|
618
945
|
settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
|
|
@@ -636,6 +963,7 @@ export function createSandboxLeaseActivities(
|
|
|
636
963
|
probeDrainableProvider,
|
|
637
964
|
captureAttempt,
|
|
638
965
|
objectStorage,
|
|
966
|
+
bus,
|
|
639
967
|
);
|
|
640
968
|
return { status: drainedCold ? "terminated" : "skipped" };
|
|
641
969
|
} catch (error) {
|
|
@@ -710,6 +1038,23 @@ export function createSandboxLeaseActivities(
|
|
|
710
1038
|
// immutable launch locators must reconcile even when managed ownership is
|
|
711
1039
|
// disabled, otherwise an exact runner exit/loss proof can remain stranded.
|
|
712
1040
|
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
1041
|
+
await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
|
|
1042
|
+
|
|
1043
|
+
// Disabling new lease ownership cannot strand a previously dispatched
|
|
1044
|
+
// operation. This only attributes a positive receipt; normal draining
|
|
1045
|
+
// already runs in both ownership modes.
|
|
1046
|
+
try {
|
|
1047
|
+
await reconcileModalProviderCreates(
|
|
1048
|
+
db,
|
|
1049
|
+
settings,
|
|
1050
|
+
observability,
|
|
1051
|
+
options.findModalProviderCreateReceipt,
|
|
1052
|
+
);
|
|
1053
|
+
} catch (error) {
|
|
1054
|
+
observability.warn("sandbox reaper: Modal create recovery inventory failed", {
|
|
1055
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1056
|
+
});
|
|
1057
|
+
}
|
|
713
1058
|
|
|
714
1059
|
if (!settings.sandboxOwnershipEnabled) {
|
|
715
1060
|
// Inventory remains useful while provider ownership is intentionally
|
|
@@ -735,8 +1080,7 @@ export function createSandboxLeaseActivities(
|
|
|
735
1080
|
metered = meterResult.accrued;
|
|
736
1081
|
const forceDrainWorkspaceIds = new Set<string>();
|
|
737
1082
|
if (
|
|
738
|
-
settings.
|
|
739
|
-
settings.usageLimitsMode === "managed" ||
|
|
1083
|
+
settings.sandboxWarmBillingMode === "credits" ||
|
|
740
1084
|
settings.sandboxMaxWarmSecondsPerWorkspace > 0
|
|
741
1085
|
) {
|
|
742
1086
|
for (const workspaceId of meterResult.workspaceIds) {
|
|
@@ -771,6 +1115,16 @@ export function createSandboxLeaseActivities(
|
|
|
771
1115
|
service.bus,
|
|
772
1116
|
);
|
|
773
1117
|
|
|
1118
|
+
// A request or PTY left on a Modal box whose lease epoch has ended can
|
|
1119
|
+
// never progress once that box is gone; settle it on exact proof.
|
|
1120
|
+
await settleEndedEpochBlockers(
|
|
1121
|
+
db,
|
|
1122
|
+
settings,
|
|
1123
|
+
observability,
|
|
1124
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
1125
|
+
endedEpochProbeBackoff,
|
|
1126
|
+
);
|
|
1127
|
+
|
|
774
1128
|
try {
|
|
775
1129
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
776
1130
|
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
@@ -865,6 +1219,30 @@ export function createSandboxLeaseActivities(
|
|
|
865
1219
|
}),
|
|
866
1220
|
),
|
|
867
1221
|
);
|
|
1222
|
+
// The embedded path has no idle checkpoint sweep workflow; run the same
|
|
1223
|
+
// inventory and per-box activity. Captures run detached so a provider
|
|
1224
|
+
// snapshot never holds up this sweep's drains; a box whose capture is
|
|
1225
|
+
// still running is not started twice. Failure defers to the next sweep.
|
|
1226
|
+
try {
|
|
1227
|
+
const { targets, timeoutClass } = await listIdleSandboxCheckpoints();
|
|
1228
|
+
for (const target of targets) {
|
|
1229
|
+
const key = sandboxIdleCheckpointWorkflowId(target);
|
|
1230
|
+
if (embeddedIdleCheckpoints.has(key)) continue;
|
|
1231
|
+
embeddedIdleCheckpoints.add(key);
|
|
1232
|
+
void checkpointIdleSandboxLease({ target, timeoutClass })
|
|
1233
|
+
.catch((error: unknown) => {
|
|
1234
|
+
service.observability.warn("sandbox reaper: idle checkpoint failed", {
|
|
1235
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
|
|
1236
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1237
|
+
});
|
|
1238
|
+
})
|
|
1239
|
+
.finally(() => embeddedIdleCheckpoints.delete(key));
|
|
1240
|
+
}
|
|
1241
|
+
} catch (error) {
|
|
1242
|
+
service.observability.warn("sandbox reaper: idle checkpoint inventory failed", {
|
|
1243
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
868
1246
|
const terminated = outcomes.filter(
|
|
869
1247
|
(outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated",
|
|
870
1248
|
).length;
|
|
@@ -889,13 +1267,63 @@ export function createSandboxLeaseActivities(
|
|
|
889
1267
|
return {
|
|
890
1268
|
prepareSandboxLeaseSweep,
|
|
891
1269
|
drainSandboxLease,
|
|
1270
|
+
listIdleSandboxCheckpoints,
|
|
1271
|
+
checkpointIdleSandboxLease,
|
|
892
1272
|
maintainSandboxLeaseSweep,
|
|
893
1273
|
reapSandboxLeases,
|
|
894
1274
|
};
|
|
895
1275
|
}
|
|
896
1276
|
|
|
1277
|
+
/** Resume the exact warm box by id for an idle checkpoint, with no cold
|
|
1278
|
+
* restore and no replacement. Returns null when the provider no longer has
|
|
1279
|
+
* it; the ordinary loss paths own that box. The session is not closed: the
|
|
1280
|
+
* box keeps running its commands. */
|
|
1281
|
+
export async function resumeIdleCheckpointSessionAtProvider(
|
|
1282
|
+
settings: ActivityServices["settings"],
|
|
1283
|
+
lease: LeaseSnapshot,
|
|
1284
|
+
): Promise<unknown | null> {
|
|
1285
|
+
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
1286
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
1287
|
+
if (backend !== "modal" || !lease.instanceId || !lease.resumeState) {
|
|
1288
|
+
throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);
|
|
1289
|
+
}
|
|
1290
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {
|
|
1291
|
+
throw new Error("idle checkpoint lease envelope does not name its provider instance");
|
|
1292
|
+
}
|
|
1293
|
+
const client = createSandboxClientForBackend(backend as never, settings) as
|
|
1294
|
+
| RetainedProcessProbeClient
|
|
1295
|
+
| undefined;
|
|
1296
|
+
if (!client?.resume) {
|
|
1297
|
+
throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);
|
|
1298
|
+
}
|
|
1299
|
+
try {
|
|
1300
|
+
const envelopeSessionState =
|
|
1301
|
+
(lease.resumeState as { sessionState?: unknown }).sessionState ?? lease.resumeState;
|
|
1302
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope(
|
|
1303
|
+
client as never,
|
|
1304
|
+
envelopeSessionState,
|
|
1305
|
+
lease.instanceId,
|
|
1306
|
+
);
|
|
1307
|
+
if (resumedState === undefined) {
|
|
1308
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
1309
|
+
}
|
|
1310
|
+
const resumed = await withRetainedProcessProbeTimeout(
|
|
1311
|
+
resumeExactSandboxSession(client, backend, resumedState, lease.instanceId),
|
|
1312
|
+
);
|
|
1313
|
+
return resumed.session;
|
|
1314
|
+
} catch (error) {
|
|
1315
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
1316
|
+
throw new Error("idle checkpoint provider resume timed out", { cause: error });
|
|
1317
|
+
}
|
|
1318
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) return null;
|
|
1319
|
+
throw error;
|
|
1320
|
+
}
|
|
1321
|
+
}
|
|
1322
|
+
|
|
897
1323
|
const CHECKPOINT_GC_LIMIT = 50;
|
|
898
1324
|
const CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 60_000;
|
|
1325
|
+
/** The checkpoint artifact GC claim TTL, which the reaper's work check previews. */
|
|
1326
|
+
export const SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;
|
|
899
1327
|
const CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
900
1328
|
const CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
901
1329
|
const SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
@@ -910,6 +1338,9 @@ type ConnectedCommandReconciliationOutcome =
|
|
|
910
1338
|
| "proof_checkpoint_failed"
|
|
911
1339
|
| "settled_exited"
|
|
912
1340
|
| "settled_lost"
|
|
1341
|
+
| "output_unavailable"
|
|
1342
|
+
| "output_ack_published"
|
|
1343
|
+
| "output_released"
|
|
913
1344
|
| "settlement_failed"
|
|
914
1345
|
| "defer_failed";
|
|
915
1346
|
|
|
@@ -924,6 +1355,65 @@ function recordConnectedCommandReconciliation(
|
|
|
924
1355
|
});
|
|
925
1356
|
}
|
|
926
1357
|
|
|
1358
|
+
/** Why a connected command's runner can never answer for it again. */
|
|
1359
|
+
type ConnectedRunnerRetirement =
|
|
1360
|
+
| "enrollment_revoked"
|
|
1361
|
+
| "connection_replaced"
|
|
1362
|
+
| "connection_stopped";
|
|
1363
|
+
type ConnectedRunnerLookups = Map<string, Promise<ConnectedRunnerRetirement | null>>;
|
|
1364
|
+
|
|
1365
|
+
/** Offline reasons after which the runner process has exited for good. */
|
|
1366
|
+
const CONNECTED_RUNNER_STOPPED_REASONS = new Set([
|
|
1367
|
+
"GOING_OFFLINE_REASON_USER_STOP",
|
|
1368
|
+
"GOING_OFFLINE_REASON_UPDATE",
|
|
1369
|
+
"GOING_OFFLINE_REASON_HOST_SHUTDOWN",
|
|
1370
|
+
]);
|
|
1371
|
+
|
|
1372
|
+
/**
|
|
1373
|
+
* A connected command's operation lives in the memory of the runner process
|
|
1374
|
+
* that launched it and is addressed through that process's connection
|
|
1375
|
+
* instance. No responder on that exact address is not proof by itself: the
|
|
1376
|
+
* machine may be offline or reconnecting. It is proof once the enrollment
|
|
1377
|
+
* also retires that process, by the same rules the running-command claim
|
|
1378
|
+
* applies: the enrollment was revoked, a different runner process now holds
|
|
1379
|
+
* the connection, or the runner went offline by stopping, updating or shutting
|
|
1380
|
+
* down. Anything else, including a failed read, keeps the existing retry. One
|
|
1381
|
+
* read per runner address per sweep.
|
|
1382
|
+
*/
|
|
1383
|
+
async function connectedCommandRunnerRetirement(
|
|
1384
|
+
db: ActivityServices["db"],
|
|
1385
|
+
claim: { controlWorkspaceId: string; enrollmentId: string; connectionInstanceId: string },
|
|
1386
|
+
error: unknown,
|
|
1387
|
+
lookups: ConnectedRunnerLookups,
|
|
1388
|
+
): Promise<ConnectedRunnerRetirement | null> {
|
|
1389
|
+
if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {
|
|
1390
|
+
return null;
|
|
1391
|
+
}
|
|
1392
|
+
const key = JSON.stringify([
|
|
1393
|
+
claim.controlWorkspaceId,
|
|
1394
|
+
claim.enrollmentId,
|
|
1395
|
+
claim.connectionInstanceId,
|
|
1396
|
+
]);
|
|
1397
|
+
let lookup = lookups.get(key);
|
|
1398
|
+
if (!lookup) {
|
|
1399
|
+
lookup = (async (): Promise<ConnectedRunnerRetirement | null> => {
|
|
1400
|
+
const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);
|
|
1401
|
+
if (!enrollment) return null;
|
|
1402
|
+
if (enrollment.status === "revoked") return "enrollment_revoked";
|
|
1403
|
+
if (enrollment.connectionInstanceId) {
|
|
1404
|
+
return enrollment.connectionInstanceId === claim.connectionInstanceId
|
|
1405
|
+
? null
|
|
1406
|
+
: "connection_replaced";
|
|
1407
|
+
}
|
|
1408
|
+
return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? "")
|
|
1409
|
+
? "connection_stopped"
|
|
1410
|
+
: null;
|
|
1411
|
+
})().catch(() => null);
|
|
1412
|
+
lookups.set(key, lookup);
|
|
1413
|
+
}
|
|
1414
|
+
return await lookup;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
927
1417
|
async function forEachWithConcurrency<T>(
|
|
928
1418
|
values: readonly T[],
|
|
929
1419
|
concurrency: number,
|
|
@@ -1110,6 +1600,7 @@ export async function reconcileConnectedMachineBackgroundCommands(
|
|
|
1110
1600
|
);
|
|
1111
1601
|
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1112
1602
|
let proof = claim.proof;
|
|
1603
|
+
let outputReplay: Awaited<ReturnType<OpStreamExecClient["readExisting"]>> | null = null;
|
|
1113
1604
|
const connectionKey = JSON.stringify([
|
|
1114
1605
|
claim.controlWorkspaceId,
|
|
1115
1606
|
claim.enrollmentId,
|
|
@@ -1167,7 +1658,7 @@ export async function reconcileConnectedMachineBackgroundCommands(
|
|
|
1167
1658
|
retryClock: defaultSelfhostedRetryClock,
|
|
1168
1659
|
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
|
|
1169
1660
|
});
|
|
1170
|
-
await replayConnectedCommandOutput(
|
|
1661
|
+
outputReplay = await replayConnectedCommandOutput(
|
|
1171
1662
|
client,
|
|
1172
1663
|
claim.opId,
|
|
1173
1664
|
Boolean(proof),
|
|
@@ -1229,6 +1720,18 @@ export async function reconcileConnectedMachineBackgroundCommands(
|
|
|
1229
1720
|
if (!settlement.settled) {
|
|
1230
1721
|
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1231
1722
|
}
|
|
1723
|
+
const receipt =
|
|
1724
|
+
outputReplay?.outputReceipt ??
|
|
1725
|
+
(outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : undefined);
|
|
1726
|
+
const terminal =
|
|
1727
|
+
outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
|
|
1728
|
+
if (proof.outcome === "exited" && receipt && terminal) {
|
|
1729
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
1730
|
+
...claim,
|
|
1731
|
+
receipt,
|
|
1732
|
+
exitCode: terminal.outcome.response.exitCode,
|
|
1733
|
+
});
|
|
1734
|
+
}
|
|
1232
1735
|
if (settlement.events.length > 0) {
|
|
1233
1736
|
try {
|
|
1234
1737
|
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
@@ -1258,21 +1761,19 @@ export async function replayConnectedCommandOutput(
|
|
|
1258
1761
|
opId: string,
|
|
1259
1762
|
terminalKnown: boolean,
|
|
1260
1763
|
capture: Parameters<OpStreamExecClient["readExisting"]>[2],
|
|
1261
|
-
): Promise<
|
|
1764
|
+
): Promise<Awaited<ReturnType<OpStreamExecClient["readExisting"]>>> {
|
|
1262
1765
|
// A completed operation can need several bounded reads to replay its retained
|
|
1263
1766
|
// frames. Keep the reader's integrity checkpoint until the exit is verified;
|
|
1264
1767
|
// replacing it after each partial read would restart forever at frame one.
|
|
1265
1768
|
let capturedThrough = 0n;
|
|
1266
1769
|
while (true) {
|
|
1267
1770
|
const before = capturedThrough;
|
|
1268
|
-
const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250,
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
});
|
|
1275
|
-
if (!terminalKnown || replay.status === "completed" || replay.terminal) return;
|
|
1771
|
+
const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, capture);
|
|
1772
|
+
if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
|
|
1773
|
+
// Quiet jobs can retain many heartbeat frames (or an incomplete UTF-8
|
|
1774
|
+
// chunk). These are real replay progress even when capture receives no
|
|
1775
|
+
// stdout/stderr. Use the reader's verified, contiguous protocol frontier.
|
|
1776
|
+
capturedThrough = BigInt(replay.replaySequence);
|
|
1276
1777
|
// This is a finite terminal drain, not a poller for a running command.
|
|
1277
1778
|
// Failed persistence, transport/integrity errors, and no-progress reads
|
|
1278
1779
|
// return control to normal reconciliation without licensing settlement.
|
|
@@ -1282,6 +1783,159 @@ export async function replayConnectedCommandOutput(
|
|
|
1282
1783
|
}
|
|
1283
1784
|
}
|
|
1284
1785
|
|
|
1786
|
+
/** Drain the durable terminal-output obligation independently of the launching
|
|
1787
|
+
* attempt or model completion notice. Existing terminal rows receive no inferred
|
|
1788
|
+
* receipt: they must replay and capture all retained bytes before any final ACK. */
|
|
1789
|
+
export async function reconcileConnectedCommandOutputReleases(
|
|
1790
|
+
db: ActivityServices["db"],
|
|
1791
|
+
settings: ActivityServices["settings"],
|
|
1792
|
+
observability: ActivityServices["observability"],
|
|
1793
|
+
bus: ActivityServices["bus"],
|
|
1794
|
+
controlRpcOverride?: ControlRpc,
|
|
1795
|
+
): Promise<void> {
|
|
1796
|
+
const dueBefore = new Date();
|
|
1797
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
1798
|
+
const controlRpc =
|
|
1799
|
+
controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
|
|
1800
|
+
const runnerLookups: ConnectedRunnerLookups = new Map();
|
|
1801
|
+
do {
|
|
1802
|
+
let claims: Awaited<ReturnType<typeof claimConnectedCommandOutputReleases>>;
|
|
1803
|
+
try {
|
|
1804
|
+
claims = await claimConnectedCommandOutputReleases(db, {
|
|
1805
|
+
claimId: crypto.randomUUID(),
|
|
1806
|
+
dueBefore,
|
|
1807
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1808
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
|
|
1809
|
+
});
|
|
1810
|
+
} catch (error) {
|
|
1811
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1812
|
+
observability.warn("sandbox reaper: connected-command output claim failed", {
|
|
1813
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1814
|
+
});
|
|
1815
|
+
return;
|
|
1816
|
+
}
|
|
1817
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1818
|
+
let receipt = claim.receipt;
|
|
1819
|
+
try {
|
|
1820
|
+
if (!bus.getOpStreamConnection)
|
|
1821
|
+
throw new Error("Connected command output transport unavailable");
|
|
1822
|
+
const rpcSubject = subjectFor(
|
|
1823
|
+
claim.controlWorkspaceId,
|
|
1824
|
+
claim.enrollmentId,
|
|
1825
|
+
claim.connectionInstanceId,
|
|
1826
|
+
);
|
|
1827
|
+
const controlTimeoutMs = Math.min(
|
|
1828
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1829
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
1830
|
+
);
|
|
1831
|
+
const client = new OpStreamExecClient({
|
|
1832
|
+
workspaceId: claim.controlWorkspaceId,
|
|
1833
|
+
agentId: claim.enrollmentId,
|
|
1834
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1835
|
+
epoch: 0,
|
|
1836
|
+
controlRpc,
|
|
1837
|
+
rpcSubject,
|
|
1838
|
+
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
1839
|
+
controlTimeoutMs,
|
|
1840
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
1841
|
+
journal: {
|
|
1842
|
+
attachGeneration: () => String(Date.now()),
|
|
1843
|
+
persistSettled: () => {
|
|
1844
|
+
throw new Error("Background output release requires its database receipt");
|
|
1845
|
+
},
|
|
1846
|
+
},
|
|
1847
|
+
});
|
|
1848
|
+
if (!receipt) {
|
|
1849
|
+
const status = await querySelfhostedOp({
|
|
1850
|
+
controlRpc,
|
|
1851
|
+
rpcSubject,
|
|
1852
|
+
opId: claim.opId,
|
|
1853
|
+
controlTimeoutMs,
|
|
1854
|
+
});
|
|
1855
|
+
if (status.state === OpState.OP_STATE_LOST) {
|
|
1856
|
+
if (
|
|
1857
|
+
!(await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1858
|
+
claim,
|
|
1859
|
+
outcome: "unavailable",
|
|
1860
|
+
retryAfterMs: 0,
|
|
1861
|
+
}))
|
|
1862
|
+
)
|
|
1863
|
+
throw new Error("Connected command output loss no longer owns its exact claim");
|
|
1864
|
+
recordConnectedCommandReconciliation(observability, "output_unavailable");
|
|
1865
|
+
return;
|
|
1866
|
+
}
|
|
1867
|
+
if (status.state !== OpState.OP_STATE_COMPLETE)
|
|
1868
|
+
throw new Error("Terminal command output returned a live operation");
|
|
1869
|
+
const replay = await replayConnectedCommandOutput(
|
|
1870
|
+
client,
|
|
1871
|
+
claim.opId,
|
|
1872
|
+
true,
|
|
1873
|
+
async (frames) =>
|
|
1874
|
+
captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames),
|
|
1875
|
+
);
|
|
1876
|
+
const terminal = replay.status === "completed" ? replay : replay.terminal;
|
|
1877
|
+
receipt =
|
|
1878
|
+
replay.outputReceipt ??
|
|
1879
|
+
(replay.status === "running" ? replay.terminal?.outputReceipt : undefined) ??
|
|
1880
|
+
null;
|
|
1881
|
+
if (!terminal || !receipt)
|
|
1882
|
+
throw new Error("Connected command output replay has no complete custody receipt");
|
|
1883
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
1884
|
+
...claim,
|
|
1885
|
+
receipt,
|
|
1886
|
+
exitCode: terminal.outcome.response.exitCode,
|
|
1887
|
+
});
|
|
1888
|
+
}
|
|
1889
|
+
const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
|
|
1890
|
+
if (
|
|
1891
|
+
!(await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1892
|
+
claim,
|
|
1893
|
+
outcome,
|
|
1894
|
+
retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0,
|
|
1895
|
+
}))
|
|
1896
|
+
)
|
|
1897
|
+
throw new Error("Connected command output release no longer owns its exact claim");
|
|
1898
|
+
recordConnectedCommandReconciliation(
|
|
1899
|
+
observability,
|
|
1900
|
+
outcome === "published" ? "output_ack_published" : "output_released",
|
|
1901
|
+
);
|
|
1902
|
+
} catch (error) {
|
|
1903
|
+
const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);
|
|
1904
|
+
// A retired runner retains nothing: captured output is released, and
|
|
1905
|
+
// output it never handed over is recorded as missing.
|
|
1906
|
+
if (
|
|
1907
|
+
retired &&
|
|
1908
|
+
(await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1909
|
+
claim,
|
|
1910
|
+
outcome: receipt ? "not_retained" : "unavailable",
|
|
1911
|
+
retryAfterMs: 0,
|
|
1912
|
+
}).catch(() => false))
|
|
1913
|
+
) {
|
|
1914
|
+
recordConnectedCommandReconciliation(
|
|
1915
|
+
observability,
|
|
1916
|
+
receipt ? "output_released" : "output_unavailable",
|
|
1917
|
+
);
|
|
1918
|
+
return;
|
|
1919
|
+
}
|
|
1920
|
+
await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1921
|
+
claim,
|
|
1922
|
+
outcome: "retry",
|
|
1923
|
+
retryAfterMs: Math.min(
|
|
1924
|
+
5 * 60_000,
|
|
1925
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) *
|
|
1926
|
+
2 ** Math.min(4, claim.reconcileAttempts - 1),
|
|
1927
|
+
),
|
|
1928
|
+
}).catch(() => false);
|
|
1929
|
+
observability.warn("sandbox reaper: connected-command output release deferred", {
|
|
1930
|
+
commandId: claim.commandId,
|
|
1931
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
});
|
|
1935
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
1936
|
+
} while (Date.now() < deadline);
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1285
1939
|
export async function probeConnectedMachineBackgroundCommand(
|
|
1286
1940
|
claim: ConnectedMachineBackgroundCommandClaim,
|
|
1287
1941
|
controlRpc: ControlRpc,
|
|
@@ -1385,6 +2039,195 @@ async function deferConnectedCommandClaim(
|
|
|
1385
2039
|
}
|
|
1386
2040
|
}
|
|
1387
2041
|
|
|
2042
|
+
/** Tuples listed per sweep (random order, the next sweep lists again) and the
|
|
2043
|
+
* share of the reaper tick the probes may use, so the pass never crowds out
|
|
2044
|
+
* the orphan sweep and GC after it. A box the provider cannot yet prove gone is
|
|
2045
|
+
* re-probed on an exponential backoff instead of on every tick. */
|
|
2046
|
+
const ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;
|
|
2047
|
+
const ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 60_000;
|
|
2048
|
+
const ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 60_000;
|
|
2049
|
+
const ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 60_000;
|
|
2050
|
+
|
|
2051
|
+
/** Per worker process: when each historical box may be probed again. */
|
|
2052
|
+
export type EndedEpochProbeBackoff = Map<string, { failures: number; nextProbeAtMs: number }>;
|
|
2053
|
+
|
|
2054
|
+
/**
|
|
2055
|
+
* Settle open requests still recorded on a Modal box whose lease
|
|
2056
|
+
* epoch has ended (a legacy loss, or a cold commit that left them). Lease
|
|
2057
|
+
* succession alone proves nothing about the old box, so each exact tuple is
|
|
2058
|
+
* settled only after its historical sandbox is observed terminated. NotFound
|
|
2059
|
+
* is ambiguous here (deletion or a rotated Modal credential workspace) and,
|
|
2060
|
+
* like a running or unreachable box, keeps the rows for a later probe (with
|
|
2061
|
+
* backoff) or an operator.
|
|
2062
|
+
*
|
|
2063
|
+
* The same observation also ends a turn's wait on sandbox setup whose start
|
|
2064
|
+
* outcome is unknown: once the exact box all its setup ran on is terminated,
|
|
2065
|
+
* nothing it may have started can still run, so the turn's setup reruns its
|
|
2066
|
+
* idempotent preparation (`reconcileCompletedSandboxSetup`). A start that
|
|
2067
|
+
* failed before any process was recorded otherwise had nothing to exit and
|
|
2068
|
+
* stayed recovering forever.
|
|
2069
|
+
*/
|
|
2070
|
+
async function settleEndedEpochBlockers(
|
|
2071
|
+
db: ActivityServices["db"],
|
|
2072
|
+
settings: ActivityServices["settings"],
|
|
2073
|
+
observability: ActivityServices["observability"],
|
|
2074
|
+
inspectHistoricalModalSandbox: HistoricalModalSandboxLifecycleProbeFn,
|
|
2075
|
+
backoff: EndedEpochProbeBackoff,
|
|
2076
|
+
): Promise<void> {
|
|
2077
|
+
// Same scope as the Modal orphan sweep: no Modal API calls from workers of
|
|
2078
|
+
// a deployment that never configured Modal.
|
|
2079
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
2080
|
+
return;
|
|
2081
|
+
}
|
|
2082
|
+
const startedAtMs = Date.now();
|
|
2083
|
+
// One observation per box: several tuples and turns can name the same box.
|
|
2084
|
+
const observed = new Map<string, string>();
|
|
2085
|
+
const kept: Record<string, number> = {};
|
|
2086
|
+
/** The box's status, or null when it is backing off or the budget is spent. */
|
|
2087
|
+
async function observe(
|
|
2088
|
+
instanceId: string,
|
|
2089
|
+
details: Record<string, unknown>,
|
|
2090
|
+
): Promise<string | null> {
|
|
2091
|
+
const known = observed.get(instanceId);
|
|
2092
|
+
if (known !== undefined) return known;
|
|
2093
|
+
const nowMs = Date.now();
|
|
2094
|
+
if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;
|
|
2095
|
+
if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;
|
|
2096
|
+
let status: string;
|
|
2097
|
+
try {
|
|
2098
|
+
status = (
|
|
2099
|
+
await withRetainedProcessProbeTimeout(
|
|
2100
|
+
inspectHistoricalModalSandbox(settings, instanceId, null),
|
|
2101
|
+
)
|
|
2102
|
+
).status;
|
|
2103
|
+
} catch (error) {
|
|
2104
|
+
status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "error";
|
|
2105
|
+
observability.warn("sandbox reaper: ended-epoch box probe failed", {
|
|
2106
|
+
...details,
|
|
2107
|
+
error:
|
|
2108
|
+
error === RETAINED_PROCESS_PROBE_TIMEOUT
|
|
2109
|
+
? "provider_timeout"
|
|
2110
|
+
: error instanceof Error
|
|
2111
|
+
? error.message
|
|
2112
|
+
: String(error),
|
|
2113
|
+
});
|
|
2114
|
+
}
|
|
2115
|
+
observed.set(instanceId, status);
|
|
2116
|
+
if (status === "terminated") {
|
|
2117
|
+
backoff.delete(instanceId);
|
|
2118
|
+
} else {
|
|
2119
|
+
deferEndedEpochProbe(backoff, instanceId);
|
|
2120
|
+
kept[status] = (kept[status] ?? 0) + 1;
|
|
2121
|
+
}
|
|
2122
|
+
return status;
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
let tuples: Awaited<ReturnType<typeof listEndedEpochWorkspaceBlockers>>;
|
|
2126
|
+
try {
|
|
2127
|
+
tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });
|
|
2128
|
+
} catch (error) {
|
|
2129
|
+
observability.warn("sandbox reaper: ended-epoch blocker inventory failed", {
|
|
2130
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2131
|
+
});
|
|
2132
|
+
tuples = [];
|
|
2133
|
+
}
|
|
2134
|
+
for (const tuple of tuples) {
|
|
2135
|
+
const instanceId = tuple.lostInstanceId;
|
|
2136
|
+
const details = {
|
|
2137
|
+
workspaceId: tuple.workspaceId,
|
|
2138
|
+
sandboxGroupId: tuple.sandboxGroupId,
|
|
2139
|
+
lostEpoch: tuple.lostEpoch,
|
|
2140
|
+
instanceId,
|
|
2141
|
+
};
|
|
2142
|
+
try {
|
|
2143
|
+
const status = await observe(instanceId, details);
|
|
2144
|
+
if (status !== "terminated") continue;
|
|
2145
|
+
const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);
|
|
2146
|
+
if (settled) {
|
|
2147
|
+
observability.info("sandbox reaper: settled blockers of an ended lease epoch", {
|
|
2148
|
+
...details,
|
|
2149
|
+
...settled,
|
|
2150
|
+
});
|
|
2151
|
+
}
|
|
2152
|
+
} catch (error) {
|
|
2153
|
+
observability.warn("sandbox reaper: ended-epoch blocker settlement failed", {
|
|
2154
|
+
...details,
|
|
2155
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2156
|
+
});
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
let setups: Awaited<ReturnType<typeof listEndedEpochSandboxSetupUncertainties>>;
|
|
2161
|
+
try {
|
|
2162
|
+
setups = await listEndedEpochSandboxSetupUncertainties(db, {
|
|
2163
|
+
limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT,
|
|
2164
|
+
});
|
|
2165
|
+
} catch (error) {
|
|
2166
|
+
observability.warn("sandbox reaper: ended-epoch setup inventory failed", {
|
|
2167
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2168
|
+
});
|
|
2169
|
+
setups = [];
|
|
2170
|
+
}
|
|
2171
|
+
for (const setup of setups) {
|
|
2172
|
+
const instanceId = setup.box.providerInstanceId;
|
|
2173
|
+
// A turn whose reconciliation is refused on a terminated box (for example
|
|
2174
|
+
// an ancestor is paused) backs off by turn, so it never re-probes per tick.
|
|
2175
|
+
const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;
|
|
2176
|
+
if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;
|
|
2177
|
+
const details = {
|
|
2178
|
+
workspaceId: setup.workspaceId,
|
|
2179
|
+
sessionId: setup.sessionId,
|
|
2180
|
+
turnId: setup.turnId,
|
|
2181
|
+
sandboxGroupId: setup.sandboxGroupId,
|
|
2182
|
+
lostEpoch: setup.box.leaseEpoch,
|
|
2183
|
+
instanceId,
|
|
2184
|
+
};
|
|
2185
|
+
try {
|
|
2186
|
+
const status = await observe(instanceId, details);
|
|
2187
|
+
if (status !== "terminated") continue;
|
|
2188
|
+
const { reconciled } = await reconcileCompletedSandboxSetup(db, {
|
|
2189
|
+
accountId: setup.accountId,
|
|
2190
|
+
workspaceId: setup.workspaceId,
|
|
2191
|
+
sessionId: setup.sessionId,
|
|
2192
|
+
turnId: setup.turnId,
|
|
2193
|
+
attemptId: setup.attemptId,
|
|
2194
|
+
terminatedBox: setup.box,
|
|
2195
|
+
});
|
|
2196
|
+
if (reconciled) {
|
|
2197
|
+
backoff.delete(turnKey);
|
|
2198
|
+
observability.info(
|
|
2199
|
+
"sandbox reaper: resumed a turn whose unknown setup ran on a terminated box",
|
|
2200
|
+
details,
|
|
2201
|
+
);
|
|
2202
|
+
} else {
|
|
2203
|
+
deferEndedEpochProbe(backoff, turnKey);
|
|
2204
|
+
}
|
|
2205
|
+
} catch (error) {
|
|
2206
|
+
deferEndedEpochProbe(backoff, turnKey);
|
|
2207
|
+
observability.warn("sandbox reaper: ended-epoch setup reconciliation failed", {
|
|
2208
|
+
...details,
|
|
2209
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2210
|
+
});
|
|
2211
|
+
}
|
|
2212
|
+
}
|
|
2213
|
+
if (Object.keys(kept).length) {
|
|
2214
|
+
observability.info("sandbox reaper: ended-epoch blockers kept without terminal proof", kept);
|
|
2215
|
+
}
|
|
2216
|
+
}
|
|
2217
|
+
|
|
2218
|
+
function deferEndedEpochProbe(backoff: EndedEpochProbeBackoff, key: string): void {
|
|
2219
|
+
const failures = (backoff.get(key)?.failures ?? 0) + 1;
|
|
2220
|
+
backoff.set(key, {
|
|
2221
|
+
failures,
|
|
2222
|
+
nextProbeAtMs:
|
|
2223
|
+
Date.now() +
|
|
2224
|
+
Math.min(
|
|
2225
|
+
ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,
|
|
2226
|
+
ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16),
|
|
2227
|
+
),
|
|
2228
|
+
});
|
|
2229
|
+
}
|
|
2230
|
+
|
|
1388
2231
|
async function reconcileTerminalRetainedProcesses(
|
|
1389
2232
|
db: ActivityServices["db"],
|
|
1390
2233
|
settings: ActivityServices["settings"],
|
|
@@ -1409,16 +2252,53 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1409
2252
|
return;
|
|
1410
2253
|
}
|
|
1411
2254
|
|
|
2255
|
+
const drainBudget: RetainedProcessDrainBudget = {
|
|
2256
|
+
until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,
|
|
2257
|
+
};
|
|
2258
|
+
// Boxes already retired as lost in this batch: their other commands were
|
|
2259
|
+
// settled by that same transaction and need no probe of their own.
|
|
2260
|
+
const lostBoxes = new Set<string>();
|
|
1412
2261
|
for (const claim of claims) {
|
|
1413
2262
|
let process = claim.process;
|
|
2263
|
+
if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))
|
|
2264
|
+
continue;
|
|
1414
2265
|
const expected = retainedProcessSettlementIdentity(process);
|
|
1415
2266
|
let proof = retainedProcessReconciliationProof(process);
|
|
2267
|
+
// The exact current box itself answered NotFound to this probe: that is
|
|
2268
|
+
// loss proof for the whole box, not just this command.
|
|
2269
|
+
let wholeBoxGone = false;
|
|
1416
2270
|
const processScope = {
|
|
1417
2271
|
accountId: process.accountId,
|
|
1418
2272
|
workspaceId: process.workspaceId,
|
|
1419
2273
|
sessionId: process.sessionId,
|
|
1420
2274
|
processId: process.id,
|
|
1421
2275
|
};
|
|
2276
|
+
// Recover a yield whose observation failed before normal receipt adoption.
|
|
2277
|
+
// Do this before provider I/O: an unavailable observer must not fence every
|
|
2278
|
+
// subsequent turn. Physical retention and unknown outcome stay unchanged.
|
|
2279
|
+
if (
|
|
2280
|
+
process.ownerActorKind === "turn" &&
|
|
2281
|
+
process.providerBackend === "modal" &&
|
|
2282
|
+
!proof &&
|
|
2283
|
+
!claim.ownerState.startsWith("background_")
|
|
2284
|
+
) {
|
|
2285
|
+
try {
|
|
2286
|
+
const command = await recoverManagedSessionBackgroundCommand(db, {
|
|
2287
|
+
...processScope,
|
|
2288
|
+
expected,
|
|
2289
|
+
reconciliationClaimId: claim.claimId,
|
|
2290
|
+
});
|
|
2291
|
+
if (command) {
|
|
2292
|
+
claim.ownerState = `background_${command.state}`;
|
|
2293
|
+
recordRetainedProcessReconciliation(observability, "background_recovered");
|
|
2294
|
+
}
|
|
2295
|
+
} catch (error) {
|
|
2296
|
+
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
2297
|
+
processId: process.id,
|
|
2298
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2299
|
+
});
|
|
2300
|
+
}
|
|
2301
|
+
}
|
|
1422
2302
|
const storedCommandPersistence = retainedProviderCommandPersistence(
|
|
1423
2303
|
db,
|
|
1424
2304
|
processScope,
|
|
@@ -1507,6 +2387,25 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1507
2387
|
}
|
|
1508
2388
|
}
|
|
1509
2389
|
|
|
2390
|
+
if (
|
|
2391
|
+
observation?.status === "deferred" &&
|
|
2392
|
+
process.providerBackend === "modal" &&
|
|
2393
|
+
!process.providerBindingKey &&
|
|
2394
|
+
!process.deadlineCancellationRequestedAt
|
|
2395
|
+
) {
|
|
2396
|
+
// An unbound historical command cannot be signalled safely. Still
|
|
2397
|
+
// record deadline intent when its lease matches: a failed observer
|
|
2398
|
+
// must not block capture until the provider expires.
|
|
2399
|
+
try {
|
|
2400
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
2401
|
+
} catch (error) {
|
|
2402
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
2403
|
+
processId: process.id,
|
|
2404
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2405
|
+
});
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
|
|
1510
2409
|
if (observation === null) {
|
|
1511
2410
|
const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
1512
2411
|
() => null,
|
|
@@ -1583,11 +2482,19 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1583
2482
|
);
|
|
1584
2483
|
supervisionMetric("cancellation_intent");
|
|
1585
2484
|
}
|
|
1586
|
-
|
|
2485
|
+
const legacyDeadlineStop =
|
|
2486
|
+
lease!.rotationReason === "provider_deadline" &&
|
|
2487
|
+
!supervised &&
|
|
2488
|
+
!process.deadlineCancellationRequestedAt;
|
|
2489
|
+
const probeOutcome = await probe(
|
|
1587
2490
|
settings,
|
|
1588
2491
|
lease!,
|
|
1589
2492
|
process,
|
|
1590
|
-
claim.ownerState === "background_stopping"
|
|
2493
|
+
(claim.ownerState === "background_stopping" &&
|
|
2494
|
+
!process.deadlineCancellationRequestedAt) ||
|
|
2495
|
+
legacyDeadlineStop
|
|
2496
|
+
? "cancel"
|
|
2497
|
+
: "observe",
|
|
1591
2498
|
async (result, chunkId, stream, streamFidelity) => {
|
|
1592
2499
|
if (
|
|
1593
2500
|
typeof result !== "string" ||
|
|
@@ -1610,7 +2517,25 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1610
2517
|
.catch(() => undefined);
|
|
1611
2518
|
},
|
|
1612
2519
|
commandPersistence,
|
|
2520
|
+
drainBudget,
|
|
2521
|
+
).then(
|
|
2522
|
+
(value) => ({ ok: true as const, value }),
|
|
2523
|
+
(error: unknown) => ({ ok: false as const, error }),
|
|
1613
2524
|
);
|
|
2525
|
+
// Legacy PTY input reserves Ctrl-C before the cancellation fence.
|
|
2526
|
+
// Non-PTY has no command signal in the Modal SDK: probe it without
|
|
2527
|
+
// injecting Ctrl-C as data, then fence and save after the grace.
|
|
2528
|
+
if (
|
|
2529
|
+
legacyDeadlineStop &&
|
|
2530
|
+
!(await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
2531
|
+
)
|
|
2532
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
2533
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
2534
|
+
observation = probeOutcome.value;
|
|
2535
|
+
wholeBoxGone =
|
|
2536
|
+
observation.status === "proved" &&
|
|
2537
|
+
observation.proof.outcome === "lost" &&
|
|
2538
|
+
observation.proof.reason === "provider_instance_not_found";
|
|
1614
2539
|
if (supervised) {
|
|
1615
2540
|
supervisionMetric(
|
|
1616
2541
|
(await commandPersistence.loadSupervisionReceipt())
|
|
@@ -1665,6 +2590,7 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1665
2590
|
expected,
|
|
1666
2591
|
claim.claimId,
|
|
1667
2592
|
observation.reason,
|
|
2593
|
+
observation.outputBacklog === true,
|
|
1668
2594
|
);
|
|
1669
2595
|
continue;
|
|
1670
2596
|
}
|
|
@@ -1687,6 +2613,79 @@ async function reconcileTerminalRetainedProcesses(
|
|
|
1687
2613
|
}
|
|
1688
2614
|
}
|
|
1689
2615
|
|
|
2616
|
+
if (wholeBoxGone) {
|
|
2617
|
+
// Settle every blocker of the exact lost box in one transaction: all its
|
|
2618
|
+
// commands, open requests, PTYs and process holders, then cold the lease
|
|
2619
|
+
// and wake waiters. Settling one command per probe took a 20-per-sweep
|
|
2620
|
+
// queue 11 minutes for 35 commands in staging session 5040c525 and kept
|
|
2621
|
+
// the session waiting on a box that was already gone.
|
|
2622
|
+
// Commands of this box that a closed turn never adopted get their
|
|
2623
|
+
// session background record first, exactly as their own claim would
|
|
2624
|
+
// have, so the agent hears about them when the box settles below.
|
|
2625
|
+
const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;
|
|
2626
|
+
for (const other of claims) {
|
|
2627
|
+
const sibling = other.process;
|
|
2628
|
+
if (
|
|
2629
|
+
other === claim ||
|
|
2630
|
+
`${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey ||
|
|
2631
|
+
sibling.ownerActorKind !== "turn" ||
|
|
2632
|
+
sibling.providerBackend !== "modal" ||
|
|
2633
|
+
other.ownerState.startsWith("background_") ||
|
|
2634
|
+
retainedProcessReconciliationProof(sibling)
|
|
2635
|
+
)
|
|
2636
|
+
continue;
|
|
2637
|
+
try {
|
|
2638
|
+
await recoverManagedSessionBackgroundCommand(db, {
|
|
2639
|
+
accountId: sibling.accountId,
|
|
2640
|
+
workspaceId: sibling.workspaceId,
|
|
2641
|
+
sessionId: sibling.sessionId,
|
|
2642
|
+
processId: sibling.id,
|
|
2643
|
+
expected: retainedProcessSettlementIdentity(sibling),
|
|
2644
|
+
reconciliationClaimId: other.claimId,
|
|
2645
|
+
});
|
|
2646
|
+
} catch (error) {
|
|
2647
|
+
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
2648
|
+
processId: sibling.id,
|
|
2649
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2650
|
+
});
|
|
2651
|
+
}
|
|
2652
|
+
}
|
|
2653
|
+
try {
|
|
2654
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
2655
|
+
accountId: process.accountId,
|
|
2656
|
+
workspaceId: process.workspaceId,
|
|
2657
|
+
sandboxGroupId: process.sandboxGroupId,
|
|
2658
|
+
expectedEpoch: process.leaseEpoch,
|
|
2659
|
+
expectedInstanceId: process.providerInstanceId,
|
|
2660
|
+
expectedBackend: process.providerBackend,
|
|
2661
|
+
});
|
|
2662
|
+
if (marked.status === "marked") {
|
|
2663
|
+
lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);
|
|
2664
|
+
recordRetainedProcessReconciliation(observability, "provider_lost_whole_box");
|
|
2665
|
+
if (marked.backgroundCommandEvents?.length) {
|
|
2666
|
+
await publishDurableSessionEvents(
|
|
2667
|
+
bus,
|
|
2668
|
+
process.workspaceId,
|
|
2669
|
+
marked.backgroundCommandEvents,
|
|
2670
|
+
(error) => {
|
|
2671
|
+
observability.warn("sandbox reaper: lost-box command event fanout failed", {
|
|
2672
|
+
sandboxGroupId: process.sandboxGroupId,
|
|
2673
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2674
|
+
});
|
|
2675
|
+
},
|
|
2676
|
+
);
|
|
2677
|
+
}
|
|
2678
|
+
continue;
|
|
2679
|
+
}
|
|
2680
|
+
} catch (error) {
|
|
2681
|
+
observability.warn("sandbox reaper: whole-box loss settlement failed", {
|
|
2682
|
+
processId: process.id,
|
|
2683
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2684
|
+
});
|
|
2685
|
+
}
|
|
2686
|
+
// Stale (a draining drain or another observer owns the box): fall back
|
|
2687
|
+
// to this command's own exact settlement below.
|
|
2688
|
+
}
|
|
1690
2689
|
try {
|
|
1691
2690
|
const settlement = await settleRetainedProcess(db, {
|
|
1692
2691
|
accountId: process.accountId,
|
|
@@ -1745,9 +2744,17 @@ async function deferRetainedProcessClaim(
|
|
|
1745
2744
|
expected: ReturnType<typeof retainedProcessSettlementIdentity>,
|
|
1746
2745
|
claimId: string,
|
|
1747
2746
|
outcome: RetainedProcessDeferralOutcome,
|
|
2747
|
+
outputBacklog = false,
|
|
1748
2748
|
): Promise<void> {
|
|
1749
|
-
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
|
|
2749
|
+
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
|
|
1750
2750
|
try {
|
|
2751
|
+
// A healthy long-lived command can already be on a five-minute backoff
|
|
2752
|
+
// when rotation becomes due. Wake it at the lead boundary, then at the
|
|
2753
|
+
// reaper cadence, so observation does not consume the capture window.
|
|
2754
|
+
const lease =
|
|
2755
|
+
process.providerBackend === "modal" && process.routeTargetId === null
|
|
2756
|
+
? await readLease(db, process.workspaceId, process.sandboxGroupId)
|
|
2757
|
+
: null;
|
|
1751
2758
|
await deferRetainedProcessReconciliation(db, {
|
|
1752
2759
|
accountId: process.accountId,
|
|
1753
2760
|
workspaceId: process.workspaceId,
|
|
@@ -1756,7 +2763,7 @@ async function deferRetainedProcessClaim(
|
|
|
1756
2763
|
expected,
|
|
1757
2764
|
claimId,
|
|
1758
2765
|
outcome: deferral.durableOutcome,
|
|
1759
|
-
retryAfterMs: deferral.retryAfterMs,
|
|
2766
|
+
retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs),
|
|
1760
2767
|
});
|
|
1761
2768
|
recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
|
|
1762
2769
|
} catch (error) {
|
|
@@ -1776,6 +2783,7 @@ export function retainedProcessReconciliationDeferral(
|
|
|
1776
2783
|
settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">,
|
|
1777
2784
|
process: Pick<SandboxRetainedProcess, "reconcileAttempts">,
|
|
1778
2785
|
outcome: RetainedProcessDeferralOutcome,
|
|
2786
|
+
outputBacklog = false,
|
|
1779
2787
|
): {
|
|
1780
2788
|
durableOutcome: string;
|
|
1781
2789
|
metricOutcome: RetainedProcessReconciliationOutcome;
|
|
@@ -1800,11 +2808,14 @@ export function retainedProcessReconciliationDeferral(
|
|
|
1800
2808
|
quarantined: true,
|
|
1801
2809
|
};
|
|
1802
2810
|
}
|
|
1803
|
-
const
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
2811
|
+
const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000);
|
|
2812
|
+
// A backlog still being drained is progress, not an unhealthy provider:
|
|
2813
|
+
// come back at the reaper cadence, without backoff.
|
|
2814
|
+
const exponent =
|
|
2815
|
+
outcome === "provider_running" && outputBacklog
|
|
2816
|
+
? 0
|
|
2817
|
+
: Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
2818
|
+
const retryAfterMs = Math.min(5 * 60_000, baseRetryMs * 2 ** exponent);
|
|
1808
2819
|
return {
|
|
1809
2820
|
durableOutcome: outcome,
|
|
1810
2821
|
metricOutcome: outcome,
|
|
@@ -1831,6 +2842,77 @@ type RetainedProcessProbeSession = ProviderCommandSession & {
|
|
|
1831
2842
|
|
|
1832
2843
|
const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
1833
2844
|
|
|
2845
|
+
/** A command's exit is reported only after both output streams reach EOF, so a
|
|
2846
|
+
* finished command with a large backlog looks running until it is read. While a
|
|
2847
|
+
* backlog remains, keep reading within the per-claim and per-sweep budgets, so
|
|
2848
|
+
* it settles in this claim instead of one page per backoff step. Every page,
|
|
2849
|
+
* including `first`, is captured here. */
|
|
2850
|
+
export async function drainRetainedCommandBacklog(
|
|
2851
|
+
session: RetainedProcessProbeSession,
|
|
2852
|
+
providerSessionId: number,
|
|
2853
|
+
first: unknown,
|
|
2854
|
+
capturePage: (value: unknown) => Promise<void>,
|
|
2855
|
+
budget?: RetainedProcessDrainBudget,
|
|
2856
|
+
): Promise<RetainedProcessProbeResult> {
|
|
2857
|
+
// Measure before capture: capturing a byte-offset page consumes its receipt.
|
|
2858
|
+
let backlog = retainedBacklogRemains(session, first);
|
|
2859
|
+
await capturePage(first);
|
|
2860
|
+
let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
|
|
2861
|
+
if (typeof session.writeStdin !== "function") return observation;
|
|
2862
|
+
const until = Math.min(
|
|
2863
|
+
Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
|
|
2864
|
+
budget?.until ?? Number.POSITIVE_INFINITY,
|
|
2865
|
+
);
|
|
2866
|
+
for (
|
|
2867
|
+
let reads = 0;
|
|
2868
|
+
backlog &&
|
|
2869
|
+
observation.status === "deferred" &&
|
|
2870
|
+
observation.reason === "provider_running" &&
|
|
2871
|
+
reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS &&
|
|
2872
|
+
Date.now() < until;
|
|
2873
|
+
reads++
|
|
2874
|
+
) {
|
|
2875
|
+
let next: unknown;
|
|
2876
|
+
try {
|
|
2877
|
+
next = await withRetainedProcessProbeTimeout(
|
|
2878
|
+
session.writeStdin({
|
|
2879
|
+
sessionId: providerSessionId,
|
|
2880
|
+
chars: "",
|
|
2881
|
+
yieldTimeMs: 1_000,
|
|
2882
|
+
maxOutputTokens: 2_000,
|
|
2883
|
+
}),
|
|
2884
|
+
);
|
|
2885
|
+
} catch {
|
|
2886
|
+
// The observation already made stays authoritative; the next sweep
|
|
2887
|
+
// classifies any provider failure through the ordinary path.
|
|
2888
|
+
break;
|
|
2889
|
+
}
|
|
2890
|
+
backlog = retainedBacklogRemains(session, next);
|
|
2891
|
+
await capturePage(next);
|
|
2892
|
+
observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
|
|
2893
|
+
}
|
|
2894
|
+
return backlog && observation.status === "deferred" && observation.reason === "provider_running"
|
|
2895
|
+
? { ...observation, outputBacklog: true }
|
|
2896
|
+
: observation;
|
|
2897
|
+
}
|
|
2898
|
+
|
|
2899
|
+
/** Whether a byte-offset page leaves a backlog worth reading now: the provider
|
|
2900
|
+
* reports the process exited but its output is not yet read to EOF, or a live
|
|
2901
|
+
* process produced a large page. Must run before the page is captured, which
|
|
2902
|
+
* consumes its receipt. Supervised commands are left to their proof path. */
|
|
2903
|
+
function retainedBacklogRemains(session: RetainedProcessProbeSession, result: unknown): boolean {
|
|
2904
|
+
const page = session.getProviderCommandOutput?.(result);
|
|
2905
|
+
if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
|
|
2906
|
+
if (page.command.supervision || page.exitCode !== null) return false;
|
|
2907
|
+
if (page.providerExited) return true;
|
|
2908
|
+
const { command, expected } = page;
|
|
2909
|
+
return (["stdout", "stderr"] as const).some(
|
|
2910
|
+
(stream) =>
|
|
2911
|
+
command.streams[stream].byteOffset - expected.streams[stream].byteOffset >=
|
|
2912
|
+
RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,
|
|
2913
|
+
);
|
|
2914
|
+
}
|
|
2915
|
+
|
|
1834
2916
|
async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
|
|
1835
2917
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1836
2918
|
try {
|
|
@@ -1879,6 +2961,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1879
2961
|
streamFidelity?: "separate" | "merged",
|
|
1880
2962
|
) => Promise<void>,
|
|
1881
2963
|
providerPersistence?: ProviderCommandPersistence,
|
|
2964
|
+
drainBudget?: RetainedProcessDrainBudget,
|
|
1882
2965
|
): Promise<RetainedProcessProbeResult> {
|
|
1883
2966
|
if (
|
|
1884
2967
|
lease.id !== process.leaseId ||
|
|
@@ -1899,7 +2982,13 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1899
2982
|
if (pending) {
|
|
1900
2983
|
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
1901
2984
|
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
1902
|
-
|
|
2985
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
2986
|
+
pending.result,
|
|
2987
|
+
process.providerSessionId,
|
|
2988
|
+
);
|
|
2989
|
+
// Capture of a previous page must not consume the only deadline stop
|
|
2990
|
+
// attempt. Terminal proof needs no further provider call.
|
|
2991
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
1903
2992
|
}
|
|
1904
2993
|
const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
|
|
1905
2994
|
if (
|
|
@@ -1996,6 +3085,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
1996
3085
|
if (typeof session.writeStdin !== "function") {
|
|
1997
3086
|
return { status: "deferred", reason: "backend_unsupported" };
|
|
1998
3087
|
}
|
|
3088
|
+
let legacyPty = false;
|
|
1999
3089
|
if (process.providerBackend === "modal") {
|
|
2000
3090
|
let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;
|
|
2001
3091
|
try {
|
|
@@ -2027,6 +3117,7 @@ export async function probeRetainedProcessAtProvider(
|
|
|
2027
3117
|
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
2028
3118
|
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
2029
3119
|
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
3120
|
+
legacyPty = command.pty === true;
|
|
2030
3121
|
}
|
|
2031
3122
|
const capturePage = async (value: unknown): Promise<void> => {
|
|
2032
3123
|
if (!captureOutput) {
|
|
@@ -2062,7 +3153,14 @@ export async function probeRetainedProcessAtProvider(
|
|
|
2062
3153
|
result = await withRetainedProcessProbeTimeout(
|
|
2063
3154
|
session.writeStdin({
|
|
2064
3155
|
sessionId: process.providerSessionId,
|
|
2065
|
-
|
|
3156
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
3157
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
3158
|
+
chars:
|
|
3159
|
+
mode === "cancel" &&
|
|
3160
|
+
!supervisedCancelled &&
|
|
3161
|
+
(process.providerBackend !== "modal" || legacyPty)
|
|
3162
|
+
? "\u0003"
|
|
3163
|
+
: "",
|
|
2066
3164
|
yieldTimeMs: 1_000,
|
|
2067
3165
|
maxOutputTokens: 2_000,
|
|
2068
3166
|
}),
|
|
@@ -2086,12 +3184,22 @@ export async function probeRetainedProcessAtProvider(
|
|
|
2086
3184
|
}
|
|
2087
3185
|
return { status: "deferred", reason: "provider_error" };
|
|
2088
3186
|
}
|
|
2089
|
-
|
|
2090
|
-
|
|
3187
|
+
// Cancellation keeps draining with empty reads too: a stopped command's exit
|
|
3188
|
+
// is also reported only after its unread output reaches EOF.
|
|
3189
|
+
const observation = await drainRetainedCommandBacklog(
|
|
3190
|
+
session,
|
|
3191
|
+
process.providerSessionId,
|
|
3192
|
+
result,
|
|
3193
|
+
capturePage,
|
|
3194
|
+
drainBudget,
|
|
3195
|
+
);
|
|
2091
3196
|
if (
|
|
2092
3197
|
observation.status === "deferred" &&
|
|
2093
3198
|
observation.reason === "provider_running" &&
|
|
2094
3199
|
lease.rotationRequestedAt !== null &&
|
|
3200
|
+
mode !== "cancel" &&
|
|
3201
|
+
!process.cancellationRequestedAt &&
|
|
3202
|
+
legacyPty &&
|
|
2095
3203
|
!(await providerPersistence
|
|
2096
3204
|
?.load()
|
|
2097
3205
|
.then((command) => command?.kind === "modal-router-v1" && command.supervision))
|
|
@@ -2197,19 +3305,25 @@ async function accrueWarmTick(
|
|
|
2197
3305
|
}
|
|
2198
3306
|
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
2199
3307
|
try {
|
|
2200
|
-
const rate =
|
|
3308
|
+
const rate =
|
|
3309
|
+
settings.sandboxWarmBillingMode === "usage_only"
|
|
3310
|
+
? 0
|
|
3311
|
+
: sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
2201
3312
|
const result = await accrueWarmSeconds(db, {
|
|
2202
3313
|
accountId: lease.accountId,
|
|
2203
3314
|
workspaceId: lease.workspaceId,
|
|
2204
3315
|
sandboxGroupId: lease.sandboxGroupId,
|
|
2205
3316
|
expectedEpoch: lease.leaseEpoch,
|
|
2206
3317
|
warmRateMicrosPerSecond: rate,
|
|
3318
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
2207
3319
|
subjectId: lease.sandboxGroupId,
|
|
2208
3320
|
});
|
|
2209
3321
|
if (result.accrued) {
|
|
2210
3322
|
accrued += 1;
|
|
2211
3323
|
}
|
|
2212
|
-
|
|
3324
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
3325
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3326
|
+
}
|
|
2213
3327
|
} catch (error) {
|
|
2214
3328
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
2215
3329
|
workspaceId: lease.workspaceId,
|
|
@@ -2221,6 +3335,54 @@ async function accrueWarmTick(
|
|
|
2221
3335
|
return { accrued, workspaceIds };
|
|
2222
3336
|
}
|
|
2223
3337
|
|
|
3338
|
+
/**
|
|
3339
|
+
* The reaper maintenance pass's read-only projections, for a reaper tick that
|
|
3340
|
+
* started no workflow because there was no work. Same gauges, same process:
|
|
3341
|
+
* the control worker both runs the cadence and polls the lifecycle queue.
|
|
3342
|
+
*/
|
|
3343
|
+
export async function refreshIdleSandboxReaperProjections(input: {
|
|
3344
|
+
db: ActivityServices["db"];
|
|
3345
|
+
settings: ActivityServices["settings"];
|
|
3346
|
+
observability: ActivityServices["observability"];
|
|
3347
|
+
inspectOpenSandboxInventory?: InspectOpenSandboxKubernetesInventoryFn;
|
|
3348
|
+
}): Promise<void> {
|
|
3349
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
3350
|
+
input.db,
|
|
3351
|
+
input.settings,
|
|
3352
|
+
input.observability,
|
|
3353
|
+
input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory,
|
|
3354
|
+
);
|
|
3355
|
+
}
|
|
3356
|
+
|
|
3357
|
+
/**
|
|
3358
|
+
* Whether the Modal orphan sweep could act: the deployment uses Modal and its
|
|
3359
|
+
* app has an unfinished sandbox. Called only once the database checks found no
|
|
3360
|
+
* warm lease, so an empty listing is the sweep's complete, empty inventory and
|
|
3361
|
+
* is projected exactly as the sweep would project it.
|
|
3362
|
+
*/
|
|
3363
|
+
export async function modalOrphanSweepMayHaveWork(input: {
|
|
3364
|
+
settings: ActivityServices["settings"];
|
|
3365
|
+
observability: ActivityServices["observability"];
|
|
3366
|
+
hasRunningSandboxes?: (settings: ActivityServices["settings"]) => Promise<boolean>;
|
|
3367
|
+
}): Promise<boolean> {
|
|
3368
|
+
const { settings, observability } = input;
|
|
3369
|
+
if (!modalOrphanSweepConfigured(settings)) return false;
|
|
3370
|
+
if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;
|
|
3371
|
+
recordModalSandboxInventoryGauges(observability, {
|
|
3372
|
+
running: 0,
|
|
3373
|
+
unleased: 0,
|
|
3374
|
+
liveLeaseInstancesMissing: 0,
|
|
3375
|
+
});
|
|
3376
|
+
recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
|
|
3377
|
+
return false;
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
function modalOrphanSweepConfigured(settings: ActivityServices["settings"]): boolean {
|
|
3381
|
+
return (
|
|
3382
|
+
settings.sandboxBackend === "modal" || !!settings.modalTokenId || !!settings.modalTokenSecret
|
|
3383
|
+
);
|
|
3384
|
+
}
|
|
3385
|
+
|
|
2224
3386
|
async function refreshQueueLeaseAndCreditGauges(
|
|
2225
3387
|
db: ActivityServices["db"],
|
|
2226
3388
|
settings: ActivityServices["settings"],
|
|
@@ -2251,6 +3413,17 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
2251
3413
|
);
|
|
2252
3414
|
},
|
|
2253
3415
|
),
|
|
3416
|
+
refreshSandboxInventoryGauge(
|
|
3417
|
+
observability,
|
|
3418
|
+
"recovery_observations",
|
|
3419
|
+
"recovery-observations",
|
|
3420
|
+
async () => {
|
|
3421
|
+
recordSandboxRecoveryObservationGauges(
|
|
3422
|
+
observability,
|
|
3423
|
+
await readRecentSandboxRecoveryObservations(db),
|
|
3424
|
+
);
|
|
3425
|
+
},
|
|
3426
|
+
),
|
|
2254
3427
|
refreshSandboxInventoryGauge(
|
|
2255
3428
|
observability,
|
|
2256
3429
|
"rotation_backlog",
|
|
@@ -2259,6 +3432,17 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
2259
3432
|
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
2260
3433
|
},
|
|
2261
3434
|
),
|
|
3435
|
+
refreshSandboxInventoryGauge(
|
|
3436
|
+
observability,
|
|
3437
|
+
"checkpoint_staleness",
|
|
3438
|
+
"checkpoint-staleness",
|
|
3439
|
+
async () => {
|
|
3440
|
+
recordSandboxCheckpointStalenessGauges(
|
|
3441
|
+
observability,
|
|
3442
|
+
await readSandboxCheckpointStaleness(db),
|
|
3443
|
+
);
|
|
3444
|
+
},
|
|
3445
|
+
),
|
|
2262
3446
|
refreshSandboxInventoryGauge(
|
|
2263
3447
|
observability,
|
|
2264
3448
|
"retained_processes",
|
|
@@ -2270,6 +3454,17 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
2270
3454
|
);
|
|
2271
3455
|
},
|
|
2272
3456
|
),
|
|
3457
|
+
refreshSandboxInventoryGauge(
|
|
3458
|
+
observability,
|
|
3459
|
+
"interaction_idle",
|
|
3460
|
+
"interaction-only-lease",
|
|
3461
|
+
async () => {
|
|
3462
|
+
recordInteractionOnlyLeaseGauges(
|
|
3463
|
+
observability,
|
|
3464
|
+
await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db)),
|
|
3465
|
+
);
|
|
3466
|
+
},
|
|
3467
|
+
),
|
|
2273
3468
|
refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
|
|
2274
3469
|
recordExpiredDrainingSandboxLeaseGauges(
|
|
2275
3470
|
observability,
|
|
@@ -2303,6 +3498,51 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
2303
3498
|
});
|
|
2304
3499
|
}
|
|
2305
3500
|
})(),
|
|
3501
|
+
(async () => {
|
|
3502
|
+
try {
|
|
3503
|
+
recordActiveUserGauges(observability, await countActiveUsers(db));
|
|
3504
|
+
} catch (error) {
|
|
3505
|
+
observability.warn("sandbox reaper: active-user gauge refresh failed", {
|
|
3506
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3507
|
+
});
|
|
3508
|
+
}
|
|
3509
|
+
})(),
|
|
3510
|
+
(async () => {
|
|
3511
|
+
try {
|
|
3512
|
+
recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
|
|
3513
|
+
} catch (error) {
|
|
3514
|
+
observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
|
|
3515
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3516
|
+
});
|
|
3517
|
+
}
|
|
3518
|
+
})(),
|
|
3519
|
+
(async () => {
|
|
3520
|
+
recordVerifiedSignupTrialDeploymentFlagGauge(
|
|
3521
|
+
observability,
|
|
3522
|
+
settings.verifiedSignupTrialCreditsEnabled,
|
|
3523
|
+
);
|
|
3524
|
+
try {
|
|
3525
|
+
const trialSwitch = await readVerifiedSignupTrialSwitch(db);
|
|
3526
|
+
recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
|
|
3527
|
+
} catch (error) {
|
|
3528
|
+
observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
|
|
3529
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3530
|
+
});
|
|
3531
|
+
}
|
|
3532
|
+
})(),
|
|
3533
|
+
(async () => {
|
|
3534
|
+
try {
|
|
3535
|
+
const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
|
|
3536
|
+
recordManagedAuthNewSignupsSwitchGauge(
|
|
3537
|
+
observability,
|
|
3538
|
+
signupsSwitch?.signupsEnabled !== false,
|
|
3539
|
+
);
|
|
3540
|
+
} catch (error) {
|
|
3541
|
+
observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
|
|
3542
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3543
|
+
});
|
|
3544
|
+
}
|
|
3545
|
+
})(),
|
|
2306
3546
|
]);
|
|
2307
3547
|
}
|
|
2308
3548
|
|
|
@@ -2325,8 +3565,8 @@ async function refreshSandboxInventoryGauge(
|
|
|
2325
3565
|
|
|
2326
3566
|
/**
|
|
2327
3567
|
* The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
|
|
2328
|
-
* live warm box, under the usage lock: if it is
|
|
2329
|
-
*
|
|
3568
|
+
* live warm box, under the usage lock: if it is over its optional warm cap,
|
|
3569
|
+
* accelerate only holderless boxes to draining
|
|
2330
3570
|
* (guarded turn_holders=0 — a paying turn is never killed). Returns the count of
|
|
2331
3571
|
* viewer-only boxes force-drained. Per-workspace best-effort.
|
|
2332
3572
|
*/
|
|
@@ -2336,8 +3576,7 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
2336
3576
|
workspaceIds: Set<string>,
|
|
2337
3577
|
observability: ActivityServices["observability"],
|
|
2338
3578
|
): Promise<number> {
|
|
2339
|
-
const enforceBalance =
|
|
2340
|
-
settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
3579
|
+
const enforceBalance = false;
|
|
2341
3580
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
2342
3581
|
let forceDrained = 0;
|
|
2343
3582
|
await forEachWithConcurrency(
|
|
@@ -2345,13 +3584,8 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
2345
3584
|
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
2346
3585
|
async (workspaceId) => {
|
|
2347
3586
|
try {
|
|
2348
|
-
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
2349
|
-
const balance = enforceBalance
|
|
2350
|
-
? await getBillingBalance(db, accountId)
|
|
2351
|
-
: ({ balanceMicros: 1 } as { balanceMicros: number });
|
|
2352
3587
|
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
2353
3588
|
workspaceId,
|
|
2354
|
-
balanceMicros: balance.balanceMicros,
|
|
2355
3589
|
enforceBalance,
|
|
2356
3590
|
maxWarmSecondsPerWorkspace: cap,
|
|
2357
3591
|
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
@@ -2375,9 +3609,72 @@ async function forceDrainOverLimitWorkspaces(
|
|
|
2375
3609
|
return forceDrained;
|
|
2376
3610
|
}
|
|
2377
3611
|
|
|
3612
|
+
export async function reconcileModalProviderCreates(
|
|
3613
|
+
db: ActivityServices["db"],
|
|
3614
|
+
settings: ActivityServices["settings"],
|
|
3615
|
+
observability: ActivityServices["observability"],
|
|
3616
|
+
discover: typeof findModalProviderCreateReceipt = findModalProviderCreateReceipt,
|
|
3617
|
+
): Promise<void> {
|
|
3618
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
|
|
3619
|
+
return;
|
|
3620
|
+
const candidates = await listPendingModalProviderCreates(db);
|
|
3621
|
+
await forEachWithConcurrency(candidates, 2, async (candidate) => {
|
|
3622
|
+
try {
|
|
3623
|
+
const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
|
|
3624
|
+
const identity = { ...candidate, accountId };
|
|
3625
|
+
const attempt = await claimModalProviderCreateRecovery(db, identity);
|
|
3626
|
+
if (!attempt) return;
|
|
3627
|
+
const instanceId = await discover(settings, attempt);
|
|
3628
|
+
if (!instanceId) return; // Absence never settles an unknown dispatch.
|
|
3629
|
+
if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
|
|
3630
|
+
observability.info("sandbox reaper: recovered Modal create receipt", {
|
|
3631
|
+
...candidate,
|
|
3632
|
+
operationId: attempt.operationId,
|
|
3633
|
+
instanceId,
|
|
3634
|
+
});
|
|
3635
|
+
}
|
|
3636
|
+
} catch (error) {
|
|
3637
|
+
observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
|
|
3638
|
+
...candidate,
|
|
3639
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3640
|
+
});
|
|
3641
|
+
}
|
|
3642
|
+
});
|
|
3643
|
+
}
|
|
3644
|
+
|
|
2378
3645
|
export function modalOrphanTerminationStillEligible(
|
|
2379
3646
|
latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
|
|
2380
3647
|
candidate: ModalOrphanSweepTermination,
|
|
3648
|
+
): boolean {
|
|
3649
|
+
// The provider may have accepted a create whose reply has not arrived.
|
|
3650
|
+
// Before attribution, neither its ID nor its separately-applied tags are
|
|
3651
|
+
// available here. Postpone orphan deletion while any such owner exists;
|
|
3652
|
+
// age/absence cannot distinguish its provider from an abandoned one.
|
|
3653
|
+
if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
|
|
3654
|
+
return false;
|
|
3655
|
+
}
|
|
3656
|
+
return modalOrphanCandidateStillUnowned(latest, candidate);
|
|
3657
|
+
}
|
|
3658
|
+
|
|
3659
|
+
/** Live leases still WARM on an instance the complete listing did not show.
|
|
3660
|
+
* A draining lease legitimately loses its box mid-drain (capture, terminate,
|
|
3661
|
+
* then cold), and warming has not published a box yet, so only a warm lease
|
|
3662
|
+
* whose exact box is gone is a zombie. */
|
|
3663
|
+
export function countWarmModalLeasesMissingInstance(
|
|
3664
|
+
latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
|
|
3665
|
+
missingInstanceIds: string[],
|
|
3666
|
+
): number {
|
|
3667
|
+
return missingInstanceIds.filter((instanceId) =>
|
|
3668
|
+
latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId),
|
|
3669
|
+
).length;
|
|
3670
|
+
}
|
|
3671
|
+
|
|
3672
|
+
/** Whether no live lease owns the candidate box by exact instance or by its
|
|
3673
|
+
* attribution tags. Unlike termination eligibility this ignores the pending
|
|
3674
|
+
* create postponement: a postponed orphan is still running without a lease. */
|
|
3675
|
+
export function modalOrphanCandidateStillUnowned(
|
|
3676
|
+
latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
|
|
3677
|
+
candidate: ModalOrphanSweepTermination,
|
|
2381
3678
|
): boolean {
|
|
2382
3679
|
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
2383
3680
|
return false;
|
|
@@ -2409,7 +3706,7 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
2409
3706
|
// Keep the provider-side sweep tightly scoped to deployments that have a Modal
|
|
2410
3707
|
// app path configured. Local/docker-only workers should not attempt Modal API
|
|
2411
3708
|
// calls just because modalAppName has a default.
|
|
2412
|
-
if (
|
|
3709
|
+
if (!modalOrphanSweepConfigured(settings)) {
|
|
2413
3710
|
return 0;
|
|
2414
3711
|
}
|
|
2415
3712
|
let liveLeases: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>;
|
|
@@ -2428,6 +3725,31 @@ async function sweepModalOrphansForConfiguredBackend(
|
|
|
2428
3725
|
return modalOrphanTerminationStillEligible(latest, candidate);
|
|
2429
3726
|
},
|
|
2430
3727
|
});
|
|
3728
|
+
// Only a complete listing is an inventory; a pass cut short by the
|
|
3729
|
+
// termination budget leaves the previous projection to age out as stale.
|
|
3730
|
+
// Re-read durable ownership after the listing so a lease that started or
|
|
3731
|
+
// finished while the app was being listed is not reported as a leak.
|
|
3732
|
+
if (result.inventory.complete) {
|
|
3733
|
+
try {
|
|
3734
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
3735
|
+
recordModalSandboxInventoryGauges(observability, {
|
|
3736
|
+
running: result.inventory.running,
|
|
3737
|
+
unleased: result.inventory.unterminated.filter((candidate) =>
|
|
3738
|
+
modalOrphanCandidateStillUnowned(latest, candidate),
|
|
3739
|
+
).length,
|
|
3740
|
+
liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
|
|
3741
|
+
latest,
|
|
3742
|
+
result.inventory.missingLiveLeaseInstanceIds,
|
|
3743
|
+
),
|
|
3744
|
+
});
|
|
3745
|
+
recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
|
|
3746
|
+
} catch (error) {
|
|
3747
|
+
recordSandboxInventoryProjectionFailure(observability, "modal_provider");
|
|
3748
|
+
observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
|
|
3749
|
+
error: error instanceof Error ? error.message : String(error),
|
|
3750
|
+
});
|
|
3751
|
+
}
|
|
3752
|
+
}
|
|
2431
3753
|
for (const terminated of result.terminated) {
|
|
2432
3754
|
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
2433
3755
|
sandboxId: terminated.sandboxId,
|
|
@@ -2566,6 +3888,7 @@ async function terminateDrainableBox(
|
|
|
2566
3888
|
probeDrainableProvider: DrainableProviderProbeFn,
|
|
2567
3889
|
attempt: SandboxDrainCaptureAttempt,
|
|
2568
3890
|
objectStorage: ObjectStorage | null,
|
|
3891
|
+
bus: ActivityServices["bus"] | null = null,
|
|
2569
3892
|
): Promise<boolean> {
|
|
2570
3893
|
// Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
|
|
2571
3894
|
// returns no account_id; the workspace->account map is the bootstrap read).
|
|
@@ -2602,6 +3925,20 @@ async function terminateDrainableBox(
|
|
|
2602
3925
|
}
|
|
2603
3926
|
|
|
2604
3927
|
const captureTimeoutMs = attempt.captureTimeoutMs;
|
|
3928
|
+
if (
|
|
3929
|
+
managedProvider &&
|
|
3930
|
+
!lease.archiveCapture?.publishedAt &&
|
|
3931
|
+
sandboxDrainCaptureSettlingInProcess(lease)
|
|
3932
|
+
) {
|
|
3933
|
+
// An earlier attempt's capture is still reading, publishing late, or
|
|
3934
|
+
// removing its spool in this worker. Leave its claim untouched and do not
|
|
3935
|
+
// start a second physical capture of the same workspace; a later retry
|
|
3936
|
+
// tears down from its published archive or, if it failed, captures anew.
|
|
3937
|
+
// An already published archive still resumes teardown immediately.
|
|
3938
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3939
|
+
`sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`,
|
|
3940
|
+
);
|
|
3941
|
+
}
|
|
2605
3942
|
let captureClaim:
|
|
2606
3943
|
| NonNullable<
|
|
2607
3944
|
Extract<Awaited<ReturnType<typeof claimWorkspaceArchiveCapture>>, { status: "claimed" }>
|
|
@@ -2766,6 +4103,8 @@ async function terminateDrainableBox(
|
|
|
2766
4103
|
liveness: "draining",
|
|
2767
4104
|
captureTimeoutMs,
|
|
2768
4105
|
minIntervalMs: 0,
|
|
4106
|
+
pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),
|
|
4107
|
+
deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
|
|
2769
4108
|
providerReplaySafe: capturePolicy?.takeover === "same_request",
|
|
2770
4109
|
takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive",
|
|
2771
4110
|
});
|
|
@@ -2939,6 +4278,20 @@ async function terminateDrainableBox(
|
|
|
2939
4278
|
) {
|
|
2940
4279
|
return false;
|
|
2941
4280
|
}
|
|
4281
|
+
// Stamp the charge horizon after verified capture and immediately before
|
|
4282
|
+
// provider stop. Capture can fail and release its claim, allowing re-arm.
|
|
4283
|
+
// A retry after stop success cannot charge post-termination wall time.
|
|
4284
|
+
let stopCutoffMarked = false;
|
|
4285
|
+
const beforeProviderStop = async (): Promise<void> => {
|
|
4286
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
4287
|
+
accountId,
|
|
4288
|
+
workspaceId: row.workspaceId,
|
|
4289
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4290
|
+
expectedEpoch: row.leaseEpoch,
|
|
4291
|
+
});
|
|
4292
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
4293
|
+
stopCutoffMarked = true;
|
|
4294
|
+
};
|
|
2942
4295
|
const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
|
|
2943
4296
|
? { terminated: true, providerMissingBeforeCapture: true }
|
|
2944
4297
|
: await terminateBox(
|
|
@@ -2962,12 +4315,66 @@ async function terminateDrainableBox(
|
|
|
2962
4315
|
});
|
|
2963
4316
|
},
|
|
2964
4317
|
row.workspaceId,
|
|
4318
|
+
beforeProviderStop,
|
|
4319
|
+
backend === "docker"
|
|
4320
|
+
? async () => {
|
|
4321
|
+
if (!captureClaim || !lease.instanceId)
|
|
4322
|
+
throw new Error("Docker drain has no current claimed capture");
|
|
4323
|
+
return await verifyDockerDrainCaptureFence(db, {
|
|
4324
|
+
accountId,
|
|
4325
|
+
workspaceId: row.workspaceId,
|
|
4326
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4327
|
+
leaseId: lease.id,
|
|
4328
|
+
leaseEpoch: row.leaseEpoch,
|
|
4329
|
+
instanceId: lease.instanceId,
|
|
4330
|
+
workspaceGeneration: captureClaim.workspaceGeneration,
|
|
4331
|
+
captureId: captureClaim.id,
|
|
4332
|
+
providerRequestId: captureClaim.providerRequestId,
|
|
4333
|
+
});
|
|
4334
|
+
}
|
|
4335
|
+
: undefined,
|
|
2965
4336
|
);
|
|
2966
4337
|
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
2967
4338
|
if (!terminated) {
|
|
2968
4339
|
return false;
|
|
2969
4340
|
}
|
|
2970
4341
|
|
|
4342
|
+
// Self-hosted/none and an unassigned lease have no provider stop callback;
|
|
4343
|
+
// finish their usage at logical detach without touching customer compute.
|
|
4344
|
+
// A managed instance that was stopped without its callback must fail closed.
|
|
4345
|
+
const providerMissingDuringStop =
|
|
4346
|
+
providerMissingBeforeCapture ||
|
|
4347
|
+
(typeof termination !== "boolean" && termination.providerMissingBeforeCapture);
|
|
4348
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
4349
|
+
if (managedProvider && lease.instanceId) {
|
|
4350
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
4351
|
+
}
|
|
4352
|
+
await beforeProviderStop();
|
|
4353
|
+
}
|
|
4354
|
+
|
|
4355
|
+
// Stop was confirmed but the lease is not cold yet. This final epoch-fenced
|
|
4356
|
+
// tick settles the interval since the last heartbeat/reaper sweep, including
|
|
4357
|
+
// a box drained before its first periodic tick. If settlement fails, keep
|
|
4358
|
+
// the draining receipt for retry instead of silently losing the charge.
|
|
4359
|
+
const finalRate =
|
|
4360
|
+
settings.sandboxWarmBillingMode === "usage_only"
|
|
4361
|
+
? 0
|
|
4362
|
+
: sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
4363
|
+
if (!providerMissingDuringStop) {
|
|
4364
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
4365
|
+
accountId,
|
|
4366
|
+
workspaceId: row.workspaceId,
|
|
4367
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4368
|
+
expectedEpoch: row.leaseEpoch,
|
|
4369
|
+
warmRateMicrosPerSecond: finalRate,
|
|
4370
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
4371
|
+
finalDrain: true,
|
|
4372
|
+
});
|
|
4373
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
4374
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
4375
|
+
}
|
|
4376
|
+
}
|
|
4377
|
+
|
|
2971
4378
|
const providerMissing =
|
|
2972
4379
|
providerMissingBeforeCapture ||
|
|
2973
4380
|
(typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
@@ -2975,15 +4382,44 @@ async function terminateDrainableBox(
|
|
|
2975
4382
|
// with draining->cold. Until this succeeds, arrivals remain fenced by that
|
|
2976
4383
|
// exact claim; a timestamp or a failed provider call can never reopen a box
|
|
2977
4384
|
// while termination may still be in flight.
|
|
2978
|
-
const { wentCold } = await confirmDrainCold(
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
4385
|
+
const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
|
|
4386
|
+
db,
|
|
4387
|
+
{
|
|
4388
|
+
accountId,
|
|
4389
|
+
workspaceId: row.workspaceId,
|
|
4390
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4391
|
+
expectedEpoch: row.leaseEpoch,
|
|
4392
|
+
...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
|
|
4393
|
+
providerMissingBeforeCapture: providerMissing,
|
|
4394
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
|
|
4395
|
+
// Mirrors terminateProviderBox: a selfhosted machine is never stopped.
|
|
4396
|
+
providerStopped:
|
|
4397
|
+
managedProvider && lease.backend !== "selfhosted" && lease.resumeBackendId !== "selfhosted",
|
|
4398
|
+
},
|
|
4399
|
+
);
|
|
4400
|
+
// The command terminal events and agent inputs are already durable in the
|
|
4401
|
+
// cold commit; this is only best-effort live fanout.
|
|
4402
|
+
await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
|
|
4403
|
+
observability.warn("sandbox reaper: contained command event fanout failed", {
|
|
4404
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
4405
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4406
|
+
});
|
|
2985
4407
|
});
|
|
2986
4408
|
if (wentCold) {
|
|
4409
|
+
// Only the exact successful cold commit counts provider loss. A missing
|
|
4410
|
+
// probe, a stale capture, a failed commit, or a retried child is not another
|
|
4411
|
+
// observed loss. Keep this outside the best-effort session event writer.
|
|
4412
|
+
// An unpublished warming replacement (its creator died before warm
|
|
4413
|
+
// publication) held no workspace; its absence is not a capture loss.
|
|
4414
|
+
if (providerMissing && !unpublishedProviderLost) {
|
|
4415
|
+
recordSandboxProviderMissingBeforeCapture(observability, backend);
|
|
4416
|
+
}
|
|
4417
|
+
if (lease.unobservableCommandDrainIds?.length) {
|
|
4418
|
+
recordSandboxCommandContainment(
|
|
4419
|
+
observability,
|
|
4420
|
+
providerMissing ? "provider_missing" : "contained",
|
|
4421
|
+
);
|
|
4422
|
+
}
|
|
2987
4423
|
// Durable termination record (sandbox-file-persistence observability): who
|
|
2988
4424
|
// ended this box and whether its /workspace was captured first, appended to
|
|
2989
4425
|
// every session sharing the group's box. Best-effort: attribution must
|
|
@@ -3084,6 +4520,73 @@ function providerDrainLogIdentity(
|
|
|
3084
4520
|
* draining until its adapter can prove termination. A truly instance-less or
|
|
3085
4521
|
* `none`-backed row is a no-op.
|
|
3086
4522
|
*/
|
|
4523
|
+
/** Internal server pre-read fence; not a grant to arbitrary callers. The
|
|
4524
|
+
* owning drain already derived account/workspace/group authority and claimed
|
|
4525
|
+
* capture through the canonical DB lifecycle. No claim is fabricated here. */
|
|
4526
|
+
export async function verifyDockerDrainCaptureFence(
|
|
4527
|
+
db: ActivityServices["db"],
|
|
4528
|
+
input: {
|
|
4529
|
+
accountId: string;
|
|
4530
|
+
workspaceId: string;
|
|
4531
|
+
sandboxGroupId: string;
|
|
4532
|
+
leaseId: string;
|
|
4533
|
+
leaseEpoch: number;
|
|
4534
|
+
instanceId: string;
|
|
4535
|
+
workspaceGeneration: number;
|
|
4536
|
+
captureId: string;
|
|
4537
|
+
providerRequestId: string;
|
|
4538
|
+
},
|
|
4539
|
+
readers = { readLease, readWorkspaceArchiveCapturePreflight },
|
|
4540
|
+
): Promise<{ archivePublished: boolean }> {
|
|
4541
|
+
const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);
|
|
4542
|
+
if (
|
|
4543
|
+
!current ||
|
|
4544
|
+
current.id !== input.leaseId ||
|
|
4545
|
+
current.liveness !== "draining" ||
|
|
4546
|
+
current.leaseEpoch !== input.leaseEpoch ||
|
|
4547
|
+
current.instanceId !== input.instanceId ||
|
|
4548
|
+
current.backend !== "docker" ||
|
|
4549
|
+
current.workspaceGeneration !== input.workspaceGeneration ||
|
|
4550
|
+
current.archiveCapture?.id !== input.captureId ||
|
|
4551
|
+
current.archiveCapture.providerRequestId !== input.providerRequestId ||
|
|
4552
|
+
current.archiveCapture.workspaceGeneration !== input.workspaceGeneration
|
|
4553
|
+
)
|
|
4554
|
+
throw new Error("Docker drain capture lease/epoch/claim changed");
|
|
4555
|
+
const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {
|
|
4556
|
+
accountId: input.accountId,
|
|
4557
|
+
workspaceId: input.workspaceId,
|
|
4558
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
4559
|
+
expectedEpoch: input.leaseEpoch,
|
|
4560
|
+
expectedInstanceId: input.instanceId,
|
|
4561
|
+
liveness: "draining",
|
|
4562
|
+
});
|
|
4563
|
+
if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)
|
|
4564
|
+
throw new Error("Docker drain capture has an active writer or changed generation");
|
|
4565
|
+
return {
|
|
4566
|
+
archivePublished:
|
|
4567
|
+
current.archiveCapture.publishedAt instanceof Date &&
|
|
4568
|
+
Number.isFinite(current.archiveCapture.publishedAt.getTime()) &&
|
|
4569
|
+
current.archiveComplete === true,
|
|
4570
|
+
};
|
|
4571
|
+
}
|
|
4572
|
+
|
|
4573
|
+
function observeDockerWorkspaceRelease(
|
|
4574
|
+
observability: ActivityServices["observability"],
|
|
4575
|
+
session: unknown,
|
|
4576
|
+
logIdentity: ReturnType<typeof providerDrainLogIdentity>,
|
|
4577
|
+
): void {
|
|
4578
|
+
const release = (session as { hostWorkspaceRelease?: DockerHostWorkspaceRelease | null })
|
|
4579
|
+
.hostWorkspaceRelease;
|
|
4580
|
+
if (!release) return;
|
|
4581
|
+
recordSandboxDockerWorkspaceRelease(observability, release.status);
|
|
4582
|
+
if (release.status === "retained") {
|
|
4583
|
+
observability.warn(
|
|
4584
|
+
"sandbox reaper: Docker drain kept its host workspace directory after teardown",
|
|
4585
|
+
{ ...logIdentity, reason: release.reason },
|
|
4586
|
+
);
|
|
4587
|
+
}
|
|
4588
|
+
}
|
|
4589
|
+
|
|
3087
4590
|
export async function terminateProviderBox(
|
|
3088
4591
|
settings: ActivityServices["settings"],
|
|
3089
4592
|
lease: NonNullable<Awaited<ReturnType<typeof readLease>>>,
|
|
@@ -3097,6 +4600,8 @@ export async function terminateProviderBox(
|
|
|
3097
4600
|
diskBackedArchives = false,
|
|
3098
4601
|
releaseFailedCapture?: () => Promise<void>,
|
|
3099
4602
|
workspaceId?: string,
|
|
4603
|
+
beforeProviderStop?: () => Promise<void>,
|
|
4604
|
+
beforeWorkspaceCapture?: () => Promise<{ archivePublished: boolean }>,
|
|
3100
4605
|
): Promise<ProviderTerminationOutcome> {
|
|
3101
4606
|
const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
|
|
3102
4607
|
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
@@ -3161,6 +4666,11 @@ export async function terminateProviderBox(
|
|
|
3161
4666
|
// fail closed instead of silently leaving a live provider behind while the
|
|
3162
4667
|
// caller commits the lease cold.
|
|
3163
4668
|
if (!lease.instanceId) {
|
|
4669
|
+
if (lease.providerCreateAttempt?.instanceId === null) {
|
|
4670
|
+
throw new Error(
|
|
4671
|
+
"provider_create_outcome_unknown: cannot infer termination from missing instance ID",
|
|
4672
|
+
);
|
|
4673
|
+
}
|
|
3164
4674
|
if (persistedInstanceId) {
|
|
3165
4675
|
throw new Error(
|
|
3166
4676
|
`sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,
|
|
@@ -3196,7 +4706,13 @@ export async function terminateProviderBox(
|
|
|
3196
4706
|
return { terminated: false, providerMissingBeforeCapture: false };
|
|
3197
4707
|
}
|
|
3198
4708
|
try {
|
|
3199
|
-
await
|
|
4709
|
+
await beforeProviderStop?.();
|
|
4710
|
+
await terminateModalById(
|
|
4711
|
+
settings,
|
|
4712
|
+
lease.instanceId,
|
|
4713
|
+
undefined,
|
|
4714
|
+
lease.providerCreateAttempt?.providerBindingKey,
|
|
4715
|
+
);
|
|
3200
4716
|
} catch (error) {
|
|
3201
4717
|
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
3202
4718
|
throw error;
|
|
@@ -3234,7 +4750,7 @@ export async function terminateProviderBox(
|
|
|
3234
4750
|
let session: PersistableSession | undefined;
|
|
3235
4751
|
let sessionState: Parameters<typeof terminateManagedSandboxSession>[1] | undefined;
|
|
3236
4752
|
try {
|
|
3237
|
-
if (!client.
|
|
4753
|
+
if (!client.deserializeSessionState || (backend !== "docker" && !client.resume)) {
|
|
3238
4754
|
// A cloud backend that cannot prove provider state is not safely
|
|
3239
4755
|
// terminable: treating this as success would cold the lease while the
|
|
3240
4756
|
// provider may still be live. Leave it draining for a later retry.
|
|
@@ -3261,16 +4777,25 @@ export async function terminateProviderBox(
|
|
|
3261
4777
|
if (resumedState === undefined) {
|
|
3262
4778
|
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
3263
4779
|
}
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
|
|
4780
|
+
if (backend === "docker") {
|
|
4781
|
+
if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {
|
|
4782
|
+
throw new Error(
|
|
4783
|
+
"Docker drain requires a creation-free attachment and current capture fence",
|
|
4784
|
+
);
|
|
4785
|
+
}
|
|
4786
|
+
await beforeWorkspaceCapture();
|
|
4787
|
+
session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);
|
|
4788
|
+
sessionState = resumedState;
|
|
4789
|
+
} else {
|
|
4790
|
+
const resumed = await resumeExactSandboxSession(
|
|
4791
|
+
client,
|
|
4792
|
+
backend,
|
|
4793
|
+
resumedState,
|
|
4794
|
+
lease.instanceId,
|
|
4795
|
+
);
|
|
4796
|
+
session = resumed.session as PersistableSession;
|
|
4797
|
+
sessionState = resumed.sessionState;
|
|
4798
|
+
}
|
|
3274
4799
|
} catch (error) {
|
|
3275
4800
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3276
4801
|
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
@@ -3315,8 +4840,15 @@ export async function terminateProviderBox(
|
|
|
3315
4840
|
resolveModalCheckpointProviderBindingForSession(settings, session),
|
|
3316
4841
|
)
|
|
3317
4842
|
: null;
|
|
4843
|
+
// Physical capture bookkeeping for this worker: released once the capture
|
|
4844
|
+
// and its spool are settled, or by the late path that inherits them.
|
|
4845
|
+
let releaseSettlingCapture: () => void = () => undefined;
|
|
3318
4846
|
try {
|
|
3319
4847
|
if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
|
|
4848
|
+
releaseSettlingCapture = trackSettlingDrainCapture(lease);
|
|
4849
|
+
// The budget bounds only this attempt's wait, never the capture: a slow
|
|
4850
|
+
// but finite capture keeps reading and publishes late, and the retry
|
|
4851
|
+
// that finds it still settling here defers instead of starting another.
|
|
3320
4852
|
const capture = captureWorkspaceArchiveForStorage(
|
|
3321
4853
|
session,
|
|
3322
4854
|
Date.now(),
|
|
@@ -3368,9 +4900,13 @@ export async function terminateProviderBox(
|
|
|
3368
4900
|
void releaseAfterCaptureFailure();
|
|
3369
4901
|
warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
|
|
3370
4902
|
},
|
|
4903
|
+
onLateSettled: () => releaseSettlingCapture(),
|
|
3371
4904
|
});
|
|
3372
4905
|
}
|
|
3373
4906
|
} catch (error) {
|
|
4907
|
+
// Every rejection except the caller's own timeout means the capture has
|
|
4908
|
+
// settled; a timed-out capture is released by its late path instead.
|
|
4909
|
+
if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();
|
|
3374
4910
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3375
4911
|
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
3376
4912
|
...logIdentity,
|
|
@@ -3412,7 +4948,11 @@ export async function terminateProviderBox(
|
|
|
3412
4948
|
checkpointBinding,
|
|
3413
4949
|
));
|
|
3414
4950
|
} finally {
|
|
3415
|
-
|
|
4951
|
+
try {
|
|
4952
|
+
await disposeWorkspaceArchive(verifiedArchive);
|
|
4953
|
+
} finally {
|
|
4954
|
+
releaseSettlingCapture();
|
|
4955
|
+
}
|
|
3416
4956
|
}
|
|
3417
4957
|
if (!wrote) {
|
|
3418
4958
|
observability.info(
|
|
@@ -3439,7 +4979,9 @@ export async function terminateProviderBox(
|
|
|
3439
4979
|
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
3440
4980
|
}
|
|
3441
4981
|
prepareProviderForTeardownAfterCapture(backend, session);
|
|
4982
|
+
await beforeProviderStop?.();
|
|
3442
4983
|
await terminateManagedSandboxSession(client, sessionState, session);
|
|
4984
|
+
if (backend === "docker") observeDockerWorkspaceRelease(observability, session, logIdentity);
|
|
3443
4985
|
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3444
4986
|
} catch (error) {
|
|
3445
4987
|
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|