@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -0,0 +1,3746 @@
|
|
|
1
|
+
import {
|
|
2
|
+
captureConnectedCommandOutput,
|
|
3
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
4
|
+
persistWorkspaceArchiveCandidate,
|
|
5
|
+
publishDurableSessionEvents,
|
|
6
|
+
putVersion1TarArchiveOrInline,
|
|
7
|
+
warnDrainSnapshotFailure
|
|
8
|
+
} from "./chunk-NQDVJ6RI.js";
|
|
9
|
+
import {
|
|
10
|
+
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
11
|
+
SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS,
|
|
12
|
+
SANDBOX_REAPER_CHILD_DISPATCH_LIMIT,
|
|
13
|
+
assertSandboxDrainInputTiming,
|
|
14
|
+
sandboxDrainTimeoutClass,
|
|
15
|
+
sandboxDrainTiming,
|
|
16
|
+
sandboxIdleCheckpointWorkflowId
|
|
17
|
+
} from "./chunk-24GP3BX5.js";
|
|
18
|
+
import {
|
|
19
|
+
inspectOpenSandboxKubernetesInventory,
|
|
20
|
+
recordActiveUserGauges,
|
|
21
|
+
recordCreditBalanceGauges,
|
|
22
|
+
recordCreditGrantGauges,
|
|
23
|
+
recordCreditMicros,
|
|
24
|
+
recordExpiredDrainingSandboxLeaseGauges,
|
|
25
|
+
recordInteractionOnlyLeaseGauges,
|
|
26
|
+
recordManagedAuthNewSignupsSwitchGauge,
|
|
27
|
+
recordModalSandboxInventoryGauges,
|
|
28
|
+
recordOpenSandboxKubernetesInventoryGauges,
|
|
29
|
+
recordRetainedProcessInventoryGauges,
|
|
30
|
+
recordRetainedProcessReconciliation,
|
|
31
|
+
recordSandboxCheckpointArtifactGauges,
|
|
32
|
+
recordSandboxCheckpointArtifactOutcome,
|
|
33
|
+
recordSandboxCheckpointStalenessGauges,
|
|
34
|
+
recordSandboxCommandContainment,
|
|
35
|
+
recordSandboxDeadlineRotationsRequested,
|
|
36
|
+
recordSandboxDockerWorkspaceRelease,
|
|
37
|
+
recordSandboxIdleInteractionRelease,
|
|
38
|
+
recordSandboxInventoryProjectionFailure,
|
|
39
|
+
recordSandboxInventoryProjectionSuccess,
|
|
40
|
+
recordSandboxLeaseGauges,
|
|
41
|
+
recordSandboxOrphansTerminated,
|
|
42
|
+
recordSandboxProviderMissingBeforeCapture,
|
|
43
|
+
recordSandboxRecoveryObservationGauges,
|
|
44
|
+
recordSandboxRotationBacklogGauges,
|
|
45
|
+
recordTurnsQueuedGauge,
|
|
46
|
+
recordVerifiedSignupTrialDeploymentFlagGauge,
|
|
47
|
+
recordVerifiedSignupTrialSwitchGauge,
|
|
48
|
+
runtimeMetricsHooksForObservability
|
|
49
|
+
} from "./chunk-5NNS4327.js";
|
|
50
|
+
import {
|
|
51
|
+
reconcilePendingParentSystemUpdates
|
|
52
|
+
} from "./chunk-W2QSM5HR.js";
|
|
53
|
+
|
|
54
|
+
// src/retained-process-diagnostics.ts
|
|
55
|
+
import {
|
|
56
|
+
databaseFailureCode,
|
|
57
|
+
nestedPostgresSqlState,
|
|
58
|
+
safeDatabaseErrorFacts,
|
|
59
|
+
SandboxWorkspaceMutationFencedError
|
|
60
|
+
} from "@opengeni/db";
|
|
61
|
+
function warnRetainedProcessProofFailure(observability, error, process) {
|
|
62
|
+
let correlationId;
|
|
63
|
+
try {
|
|
64
|
+
const sqlState = nestedPostgresSqlState(error);
|
|
65
|
+
const constraint = safeDatabaseErrorFacts(error).constraint;
|
|
66
|
+
correlationId = observability.recordFailureDiagnostic({
|
|
67
|
+
code: error instanceof SandboxWorkspaceMutationFencedError ? "retained_process_fenced" : sqlState ? databaseFailureCode(sqlState) : "retained_process_proof_failed",
|
|
68
|
+
stage: "sandbox_retained_processes.proof",
|
|
69
|
+
error,
|
|
70
|
+
processId: process.id,
|
|
71
|
+
sessionId: process.sessionId,
|
|
72
|
+
...process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {},
|
|
73
|
+
...process.ownerTurnId ? { turnId: process.ownerTurnId } : {},
|
|
74
|
+
attempts: process.reconcileAttempts,
|
|
75
|
+
sqlState,
|
|
76
|
+
...constraint ? { constraint } : {}
|
|
77
|
+
});
|
|
78
|
+
} catch {
|
|
79
|
+
}
|
|
80
|
+
try {
|
|
81
|
+
observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
|
|
82
|
+
...correlationId ? { correlationId } : {}
|
|
83
|
+
});
|
|
84
|
+
} catch {
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// src/retained-process-retry.ts
|
|
89
|
+
function retainedProcessDeadlineRetryMs(process, lease, settings, retryAfterMs, now = Date.now()) {
|
|
90
|
+
if (process.providerBackend !== "modal" || process.routeTargetId !== null || !lease || lease.id !== process.leaseId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.providerDeadlineAt)
|
|
91
|
+
return retryAfterMs;
|
|
92
|
+
const untilRotation = lease.providerDeadlineAt.getTime() - settings.sandboxRotationLeadMs - now;
|
|
93
|
+
if (!Number.isFinite(untilRotation)) return retryAfterMs;
|
|
94
|
+
return Math.min(
|
|
95
|
+
retryAfterMs,
|
|
96
|
+
Math.max(1e3, settings.sandboxLeaseReaperPeriodMs, untilRotation)
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// src/activities/sandbox-lease.ts
|
|
101
|
+
import { createHash, randomUUID } from "crypto";
|
|
102
|
+
import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
|
|
103
|
+
import { Context as Context2 } from "@temporalio/activity";
|
|
104
|
+
import { OpLostReason, OpState } from "@opengeni/agent-proto";
|
|
105
|
+
import {
|
|
106
|
+
accrueWarmSeconds,
|
|
107
|
+
retainedProviderCommandPersistence,
|
|
108
|
+
adoptLegacyModalCheckpointArtifact,
|
|
109
|
+
confirmDrainCold,
|
|
110
|
+
markWarmLeaseInstanceLost,
|
|
111
|
+
appendSessionEventToSandboxGroup,
|
|
112
|
+
bindRetainedProcessProviderIdentity,
|
|
113
|
+
claimWorkspaceArchiveCapture,
|
|
114
|
+
leaseMayHaveUntrackedWriters,
|
|
115
|
+
releaseWorkspaceArchiveCapture,
|
|
116
|
+
listIdleCheckpointCandidates,
|
|
117
|
+
recordIdleCheckpointAttempt,
|
|
118
|
+
claimSandboxCheckpointArtifactsForGc,
|
|
119
|
+
claimTerminalRetainedProcesses,
|
|
120
|
+
getEnrollment,
|
|
121
|
+
countActiveRetainedProcessesByOwnerState,
|
|
122
|
+
countSandboxCheckpointArtifactsByState,
|
|
123
|
+
countExpiredDrainingSandboxLeases,
|
|
124
|
+
countQueuedTurns,
|
|
125
|
+
countSandboxLeasesByLiveness,
|
|
126
|
+
deferRetainedProcessReconciliation,
|
|
127
|
+
forceDrainOverLimitViewerOnlyBoxes,
|
|
128
|
+
listCreditBalancesByAccount,
|
|
129
|
+
countActiveUsers,
|
|
130
|
+
readCreditGrantTotals,
|
|
131
|
+
readManagedAuthNewSignupsSwitch,
|
|
132
|
+
readVerifiedSignupTrialSwitch,
|
|
133
|
+
listLegacyModalCheckpointSlots,
|
|
134
|
+
listLiveModalSandboxLeaseAttributions,
|
|
135
|
+
listPendingModalProviderCreates,
|
|
136
|
+
claimModalProviderCreateRecovery,
|
|
137
|
+
recordRecoveredModalProviderCreate,
|
|
138
|
+
markWarmBillingStopCutoff,
|
|
139
|
+
listMeterableWarmLeases,
|
|
140
|
+
countInteractionOnlyWarmLeasesByIdle,
|
|
141
|
+
listSandboxViewerForceDrainWorkspaceIds,
|
|
142
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
143
|
+
persistDrainSnapshot,
|
|
144
|
+
pruneDeletedSandboxCheckpointArtifacts,
|
|
145
|
+
registerSandboxCheckpointArtifact,
|
|
146
|
+
recordRetainedProcessReconciliationProof,
|
|
147
|
+
replaceWorkspaceArchiveCaptureAfterProof,
|
|
148
|
+
listEndedEpochWorkspaceBlockers,
|
|
149
|
+
listEndedEpochSandboxSetupUncertainties,
|
|
150
|
+
reconcileCompletedSandboxSetup,
|
|
151
|
+
readLease as readLease2,
|
|
152
|
+
settleEndedEpochWorkspaceBlockers,
|
|
153
|
+
readWorkspaceArchiveCapturePreflight,
|
|
154
|
+
reapExpiredSessionListSnapshots,
|
|
155
|
+
reapStaleLeaseHoldersGlobal,
|
|
156
|
+
requestDueSandboxRotationsGlobal,
|
|
157
|
+
readSandboxRotationBacklog,
|
|
158
|
+
readSandboxCheckpointStaleness,
|
|
159
|
+
readRecentSandboxRecoveryObservations,
|
|
160
|
+
workspaceArchiveCaptureDeadlineElapsed,
|
|
161
|
+
retainedProcessReconciliationProof,
|
|
162
|
+
retainedProcessSettlementIdentity,
|
|
163
|
+
recoverManagedSessionBackgroundCommand,
|
|
164
|
+
settleClaimedConnectedMachineBackgroundCommand,
|
|
165
|
+
settleSandboxCheckpointArtifactGc,
|
|
166
|
+
rlsContextForWorkspace,
|
|
167
|
+
settleRetainedProcess
|
|
168
|
+
} from "@opengeni/db";
|
|
169
|
+
import {
|
|
170
|
+
claimConnectedCommandOutputReleases,
|
|
171
|
+
claimConnectedMachineSessionBackgroundCommands,
|
|
172
|
+
deferConnectedMachineBackgroundCommandReconciliation,
|
|
173
|
+
recordConnectedMachineBackgroundCommandProof,
|
|
174
|
+
recordConnectedCommandOutputConsumption,
|
|
175
|
+
settleConnectedCommandOutputReleaseClaim
|
|
176
|
+
} from "@opengeni/db/session-background-commands";
|
|
177
|
+
import {
|
|
178
|
+
SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS,
|
|
179
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
180
|
+
sandboxDeadlineMandatoryCaptureLeadMs,
|
|
181
|
+
sandboxWarmRateMicrosPerSecond
|
|
182
|
+
} from "@opengeni/config";
|
|
183
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
184
|
+
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
185
|
+
import {
|
|
186
|
+
captureWorkspaceArchiveForStorage,
|
|
187
|
+
disposeWorkspaceArchive,
|
|
188
|
+
cancelSelfhostedOp,
|
|
189
|
+
assertConsistentSandboxProviderIdentity,
|
|
190
|
+
createSandboxClientForBackend as createSandboxClientForBackend2,
|
|
191
|
+
deleteModalCheckpointSnapshot,
|
|
192
|
+
deserializeSandboxSessionStateEnvelope as deserializeSandboxSessionStateEnvelope2,
|
|
193
|
+
establishSandboxSessionFromEnvelope,
|
|
194
|
+
inspectModalSandboxLifecycle,
|
|
195
|
+
isExecSessionLostBanner,
|
|
196
|
+
isProviderSandboxNotFoundError,
|
|
197
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
198
|
+
NatsControlRpc,
|
|
199
|
+
NatsOpStreamTransport,
|
|
200
|
+
OpStreamExecClient,
|
|
201
|
+
defaultSelfhostedRetryClock,
|
|
202
|
+
stripExecBanner,
|
|
203
|
+
parseExecBannerExitCode,
|
|
204
|
+
prepareProviderForTeardownAfterCapture,
|
|
205
|
+
providerWorkspaceCapturePolicy,
|
|
206
|
+
providerWorkspaceCaptureIsPointInTime,
|
|
207
|
+
resolveModalCheckpointProviderBindingForLiveSandbox,
|
|
208
|
+
resolveModalCheckpointProviderBinding,
|
|
209
|
+
findModalProviderCreateReceipt,
|
|
210
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
211
|
+
querySelfhostedOp,
|
|
212
|
+
resumeExactSandboxSession as resumeExactSandboxSession2,
|
|
213
|
+
sandboxBackendForSdkBackendId,
|
|
214
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
215
|
+
sandboxCommandExitCode,
|
|
216
|
+
sandboxCommandStillRunning,
|
|
217
|
+
SelfhostedControlError,
|
|
218
|
+
subjectFor,
|
|
219
|
+
modalAppHasRunningSandboxes,
|
|
220
|
+
sweepModalOrphanSandboxes,
|
|
221
|
+
terminateManagedSandboxSession,
|
|
222
|
+
terminateModalSandboxById,
|
|
223
|
+
verifySandboxExecReadiness
|
|
224
|
+
} from "@opengeni/runtime/sandbox";
|
|
225
|
+
|
|
226
|
+
// src/activities/browser-deadline-checkpoint.ts
|
|
227
|
+
import { Context } from "@temporalio/activity";
|
|
228
|
+
import {
|
|
229
|
+
environmentsEncryptionKeyBytes,
|
|
230
|
+
resolveFirstPartyDelegationSecret
|
|
231
|
+
} from "@opengeni/config";
|
|
232
|
+
import { BROWSER_STATE_ARTIFACT_CONTENT_TYPE } from "@opengeni/contracts";
|
|
233
|
+
import {
|
|
234
|
+
browserDeadlineCheckpoint,
|
|
235
|
+
clearSuspendedBrowserSessionController,
|
|
236
|
+
commitBrowserSessionSuspension,
|
|
237
|
+
dispatchBrowserSessionOperation,
|
|
238
|
+
failBrowserSessionSuspension,
|
|
239
|
+
listBrowserDeadlineCheckpoints,
|
|
240
|
+
readLease,
|
|
241
|
+
releaseLeaseHolder,
|
|
242
|
+
withRlsContext
|
|
243
|
+
} from "@opengeni/db";
|
|
244
|
+
import { deriveBrowserControllerAdminToken } from "@opengeni/interaction/browser-controller-authority";
|
|
245
|
+
import {
|
|
246
|
+
browserStateArtifactAad,
|
|
247
|
+
browserStateManifestDigest,
|
|
248
|
+
browserStateObjectKey,
|
|
249
|
+
deriveBrowserStateDataKey,
|
|
250
|
+
wrapBrowserStateDataKey
|
|
251
|
+
} from "@opengeni/interaction/browser-state-authority";
|
|
252
|
+
import {
|
|
253
|
+
BrowserControlClient,
|
|
254
|
+
BrowserControlRequestError,
|
|
255
|
+
createSandboxClientForBackend,
|
|
256
|
+
deserializeSandboxSessionStateEnvelope,
|
|
257
|
+
resumeExactSandboxSession
|
|
258
|
+
} from "@opengeni/runtime";
|
|
259
|
+
var UPLOAD_CLEANUP_GRACE_MS = 24 * 60 * 6e4;
|
|
260
|
+
function heartbeat() {
|
|
261
|
+
try {
|
|
262
|
+
Context.current().heartbeat();
|
|
263
|
+
} catch {
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
async function withActivityHeartbeat(run) {
|
|
267
|
+
heartbeat();
|
|
268
|
+
const timer = setInterval(heartbeat, 5e3);
|
|
269
|
+
try {
|
|
270
|
+
return await run();
|
|
271
|
+
} finally {
|
|
272
|
+
clearInterval(timer);
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async function resumeCheckpointController({
|
|
276
|
+
target,
|
|
277
|
+
lease,
|
|
278
|
+
settings,
|
|
279
|
+
controllerRoot
|
|
280
|
+
}) {
|
|
281
|
+
const provider = createSandboxClientForBackend("modal", settings);
|
|
282
|
+
const envelope = lease.resumeState;
|
|
283
|
+
const state = await deserializeSandboxSessionStateEnvelope(
|
|
284
|
+
provider,
|
|
285
|
+
envelope.sessionState ?? lease.resumeState,
|
|
286
|
+
target.instanceId
|
|
287
|
+
);
|
|
288
|
+
const resumed = await resumeExactSandboxSession(provider, "modal", state, target.instanceId);
|
|
289
|
+
return new BrowserControlClient(resumed.session, {
|
|
290
|
+
adminToken: deriveBrowserControllerAdminToken({
|
|
291
|
+
rootSecret: controllerRoot,
|
|
292
|
+
accountId: target.accountId,
|
|
293
|
+
workspaceId: target.workspaceId,
|
|
294
|
+
placement: { kind: "sandbox_group", sandboxGroupId: target.sandboxGroupId },
|
|
295
|
+
placementInstanceId: target.instanceId
|
|
296
|
+
})
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
function browserCheckpointPrerequisites(settings, objectStorage) {
|
|
300
|
+
if (!settings.sandboxOwnershipEnabled || !objectStorage || !resolveFirstPartyDelegationSecret(settings))
|
|
301
|
+
return false;
|
|
302
|
+
const root = environmentsEncryptionKeyBytes(settings);
|
|
303
|
+
if (!root) return false;
|
|
304
|
+
root.fill(0);
|
|
305
|
+
return true;
|
|
306
|
+
}
|
|
307
|
+
function createBrowserDeadlineCheckpointActivities(services, connectController = resumeCheckpointController) {
|
|
308
|
+
return {
|
|
309
|
+
async listDueBrowserCheckpoints() {
|
|
310
|
+
return await withActivityHeartbeat(async () => {
|
|
311
|
+
const { db, settings, objectStorage } = await services();
|
|
312
|
+
if (!browserCheckpointPrerequisites(settings, objectStorage)) return [];
|
|
313
|
+
return await listBrowserDeadlineCheckpoints(db);
|
|
314
|
+
});
|
|
315
|
+
},
|
|
316
|
+
async checkpointBrowserBeforeDeadline(target) {
|
|
317
|
+
return await withActivityHeartbeat(async () => {
|
|
318
|
+
const { db, settings, objectStorage } = await services();
|
|
319
|
+
const rootKey = environmentsEncryptionKeyBytes(settings);
|
|
320
|
+
const controllerRoot = resolveFirstPartyDelegationSecret(settings);
|
|
321
|
+
if (!rootKey || !objectStorage || !controllerRoot) {
|
|
322
|
+
rootKey?.fill(0);
|
|
323
|
+
throw new Error("browser deadline checkpoint storage or encryption is unavailable");
|
|
324
|
+
}
|
|
325
|
+
let dataKey = null;
|
|
326
|
+
let aad = null;
|
|
327
|
+
let timer = null;
|
|
328
|
+
try {
|
|
329
|
+
const claim = await browserDeadlineCheckpoint(db, target, {
|
|
330
|
+
prepare: target.reason !== "idle",
|
|
331
|
+
touch: true
|
|
332
|
+
});
|
|
333
|
+
if (!claim) return { status: "skipped" };
|
|
334
|
+
let pulseRunning = false;
|
|
335
|
+
let authorityLost = false;
|
|
336
|
+
const pulse = async () => {
|
|
337
|
+
if (pulseRunning) return;
|
|
338
|
+
pulseRunning = true;
|
|
339
|
+
try {
|
|
340
|
+
if (!await browserDeadlineCheckpoint(db, target, { touch: true }))
|
|
341
|
+
authorityLost = true;
|
|
342
|
+
} finally {
|
|
343
|
+
pulseRunning = false;
|
|
344
|
+
}
|
|
345
|
+
};
|
|
346
|
+
timer = setInterval(() => {
|
|
347
|
+
void pulse().catch(() => void 0);
|
|
348
|
+
}, 5e3);
|
|
349
|
+
const lease = await readLease(db, target.workspaceId, target.sandboxGroupId);
|
|
350
|
+
if (!lease || lease.instanceId !== target.instanceId || lease.leaseEpoch !== target.leaseEpoch || !lease.resumeState) {
|
|
351
|
+
throw new Error("browser deadline checkpoint placement changed");
|
|
352
|
+
}
|
|
353
|
+
const placement = {
|
|
354
|
+
kind: "sandbox_group",
|
|
355
|
+
sandboxGroupId: target.sandboxGroupId
|
|
356
|
+
};
|
|
357
|
+
const client = await connectController({ target, lease, settings, controllerRoot });
|
|
358
|
+
if (authorityLost || !await browserDeadlineCheckpoint(db, target, { touch: true })) {
|
|
359
|
+
throw new Error("browser deadline checkpoint authority changed");
|
|
360
|
+
}
|
|
361
|
+
if (claim.state !== "completed") {
|
|
362
|
+
const scope = { ...target, operationId: claim.operationId };
|
|
363
|
+
const objectKey = browserStateObjectKey(target.workspaceId, claim.operationId);
|
|
364
|
+
dataKey = deriveBrowserStateDataKey(rootKey, { ...scope, objectKey });
|
|
365
|
+
aad = browserStateArtifactAad({ ...scope, objectKey });
|
|
366
|
+
await dispatchBrowserSessionOperation(db, {
|
|
367
|
+
...scope,
|
|
368
|
+
deadlineTarget: target,
|
|
369
|
+
stateUpload: {
|
|
370
|
+
objectKey,
|
|
371
|
+
cleanupAfter: new Date(Date.now() + UPLOAD_CLEANUP_GRACE_MS)
|
|
372
|
+
}
|
|
373
|
+
});
|
|
374
|
+
const signed = await objectStorage.createPutUrl({
|
|
375
|
+
key: objectKey,
|
|
376
|
+
contentType: BROWSER_STATE_ARTIFACT_CONTENT_TYPE,
|
|
377
|
+
audience: "sandbox"
|
|
378
|
+
});
|
|
379
|
+
const receipt = await client.captureState({
|
|
380
|
+
...scope,
|
|
381
|
+
objectKey,
|
|
382
|
+
afterCapture: "stop",
|
|
383
|
+
dataKey,
|
|
384
|
+
aad,
|
|
385
|
+
upload: {
|
|
386
|
+
url: signed.url,
|
|
387
|
+
requiredHeaders: signed.requiredHeaders,
|
|
388
|
+
expiresAt: signed.expiresAt.toISOString()
|
|
389
|
+
}
|
|
390
|
+
}).catch(async (error) => {
|
|
391
|
+
if (error instanceof BrowserControlRequestError && (error.error.code === "outcome_unknown" || !error.error.retryable)) {
|
|
392
|
+
await failBrowserSessionSuspension(db, { ...scope, error: error.error });
|
|
393
|
+
}
|
|
394
|
+
throw error;
|
|
395
|
+
});
|
|
396
|
+
if (authorityLost) throw new Error("browser deadline checkpoint authority changed");
|
|
397
|
+
const portable = receipt.manifest.profileCrypto !== "platform_bound";
|
|
398
|
+
await commitBrowserSessionSuspension(db, {
|
|
399
|
+
...scope,
|
|
400
|
+
deadlineTarget: target,
|
|
401
|
+
artifact: {
|
|
402
|
+
kind: "chromium_profile",
|
|
403
|
+
format: receipt.format,
|
|
404
|
+
objectKey,
|
|
405
|
+
artifactDigest: receipt.artifactDigest,
|
|
406
|
+
contentDigest: receipt.contentDigest,
|
|
407
|
+
manifestDigest: browserStateManifestDigest(receipt.manifest),
|
|
408
|
+
sizeBytes: receipt.sizeBytes,
|
|
409
|
+
encryptedDataKey: wrapBrowserStateDataKey(rootKey, dataKey, {
|
|
410
|
+
...scope,
|
|
411
|
+
objectKey,
|
|
412
|
+
artifactDigest: receipt.artifactDigest,
|
|
413
|
+
contentDigest: receipt.contentDigest
|
|
414
|
+
}),
|
|
415
|
+
materialization: {
|
|
416
|
+
portability: portable ? "portable" : "placement_bound",
|
|
417
|
+
reason: portable ? null : "Profile encryption depends on the source operating-system credential store.",
|
|
418
|
+
platform: receipt.manifest.platform,
|
|
419
|
+
architecture: receipt.manifest.architecture,
|
|
420
|
+
engine: receipt.manifest.engine,
|
|
421
|
+
engineVersion: receipt.manifest.engineVersion,
|
|
422
|
+
driverId: receipt.manifest.driverId,
|
|
423
|
+
driverSchemaVersion: receipt.manifest.driverSchemaVersion,
|
|
424
|
+
profileCrypto: receipt.manifest.profileCrypto,
|
|
425
|
+
providerId: null,
|
|
426
|
+
placement: portable ? null : placement
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
});
|
|
430
|
+
}
|
|
431
|
+
await client.endSession(target, { removeState: true }).catch((error) => {
|
|
432
|
+
if (!(error instanceof BrowserControlRequestError && error.status === 404)) throw error;
|
|
433
|
+
});
|
|
434
|
+
await withRlsContext(
|
|
435
|
+
db,
|
|
436
|
+
target,
|
|
437
|
+
async (tx) => {
|
|
438
|
+
if ((await browserDeadlineCheckpoint(tx, target))?.state !== "completed") {
|
|
439
|
+
throw new Error("browser deadline checkpoint cleanup authority changed");
|
|
440
|
+
}
|
|
441
|
+
await clearSuspendedBrowserSessionController(tx, {
|
|
442
|
+
...target,
|
|
443
|
+
expectedControllerGeneration: target.controllerGeneration
|
|
444
|
+
});
|
|
445
|
+
await releaseLeaseHolder(tx, {
|
|
446
|
+
...target,
|
|
447
|
+
kind: "interaction",
|
|
448
|
+
holderId: `browser-session:${target.browserSessionId}`,
|
|
449
|
+
// The idle window already elapsed before this save began, so
|
|
450
|
+
// a box the saved browser was last to hold drains now.
|
|
451
|
+
idleGraceMs: target.reason === "idle" ? 0 : settings.sandboxIdleGraceMs,
|
|
452
|
+
preserveHoldersChangedAt: target.reason === "idle"
|
|
453
|
+
});
|
|
454
|
+
},
|
|
455
|
+
void 0,
|
|
456
|
+
"none"
|
|
457
|
+
);
|
|
458
|
+
return { status: "suspended" };
|
|
459
|
+
} finally {
|
|
460
|
+
if (timer) clearInterval(timer);
|
|
461
|
+
dataKey?.fill(0);
|
|
462
|
+
aad?.fill(0);
|
|
463
|
+
rootKey.fill(0);
|
|
464
|
+
}
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
};
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// src/activities/sandbox-lease.ts
|
|
471
|
+
import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni/observability";
|
|
472
|
+
var RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
|
|
473
|
+
var RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
474
|
+
var RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5e3;
|
|
475
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
|
|
476
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 1e4;
|
|
477
|
+
var RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 6e4;
|
|
478
|
+
var RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
|
|
479
|
+
var RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
|
|
480
|
+
var RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 6e4;
|
|
481
|
+
var CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
|
|
482
|
+
var CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS = 5 * 6e4;
|
|
483
|
+
var SandboxProviderCaptureTimeoutError = class extends Error {
|
|
484
|
+
constructor(sandboxGroupId, backend, timeoutMs, leaseEpoch, instanceId) {
|
|
485
|
+
super(
|
|
486
|
+
`sandbox ${sandboxGroupId} epoch ${leaseEpoch} ${backend} instance ${instanceId} workspace capture exceeded ${timeoutMs}ms; the exact durable claim remains fenced while its provider response settles`
|
|
487
|
+
);
|
|
488
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
489
|
+
this.backend = backend;
|
|
490
|
+
this.timeoutMs = timeoutMs;
|
|
491
|
+
this.leaseEpoch = leaseEpoch;
|
|
492
|
+
this.instanceId = instanceId;
|
|
493
|
+
}
|
|
494
|
+
name = "SandboxProviderCaptureTimeoutError";
|
|
495
|
+
};
|
|
496
|
+
async function awaitProviderCaptureWithLatePublication(input) {
|
|
497
|
+
let timeout;
|
|
498
|
+
const timeoutError = input.timeoutError;
|
|
499
|
+
try {
|
|
500
|
+
return await Promise.race([
|
|
501
|
+
input.capture,
|
|
502
|
+
new Promise((_, reject) => {
|
|
503
|
+
timeout = setTimeout(() => reject(timeoutError), input.timeoutMs);
|
|
504
|
+
})
|
|
505
|
+
]);
|
|
506
|
+
} catch (error) {
|
|
507
|
+
if (error !== timeoutError) throw error;
|
|
508
|
+
const settled = () => {
|
|
509
|
+
try {
|
|
510
|
+
input.onLateSettled?.();
|
|
511
|
+
} catch {
|
|
512
|
+
}
|
|
513
|
+
};
|
|
514
|
+
void input.capture.then(input.publishLate, input.observeLateFailure).catch(input.observeLatePublicationFailure).then(settled, settled);
|
|
515
|
+
throw error;
|
|
516
|
+
} finally {
|
|
517
|
+
if (timeout) clearTimeout(timeout);
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
function startSandboxReaperHeartbeat(details) {
|
|
521
|
+
let context;
|
|
522
|
+
try {
|
|
523
|
+
context = Context2.current();
|
|
524
|
+
} catch {
|
|
525
|
+
return () => void 0;
|
|
526
|
+
}
|
|
527
|
+
const heartbeat2 = () => {
|
|
528
|
+
try {
|
|
529
|
+
context.heartbeat({ ...details, heartbeatAt: (/* @__PURE__ */ new Date()).toISOString() });
|
|
530
|
+
} catch {
|
|
531
|
+
}
|
|
532
|
+
};
|
|
533
|
+
heartbeat2();
|
|
534
|
+
const timer = setInterval(heartbeat2, SANDBOX_REAPER_ACTIVITY_HEARTBEAT_INTERVAL_MS);
|
|
535
|
+
if ("unref" in timer && typeof timer.unref === "function") timer.unref();
|
|
536
|
+
return () => clearInterval(timer);
|
|
537
|
+
}
|
|
538
|
+
var SANDBOX_IDLE_CHECKPOINT_BATCH = Math.max(
|
|
539
|
+
1,
|
|
540
|
+
Math.floor(CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES / 2)
|
|
541
|
+
);
|
|
542
|
+
var SandboxDrainRecoveryDeferredError = class extends Error {
|
|
543
|
+
name = "SandboxDrainRecoveryDeferredError";
|
|
544
|
+
};
|
|
545
|
+
var settlingDrainCaptures = /* @__PURE__ */ new Map();
|
|
546
|
+
function drainCaptureKey(lease) {
|
|
547
|
+
return JSON.stringify([lease.sandboxGroupId, lease.leaseEpoch, lease.instanceId]);
|
|
548
|
+
}
|
|
549
|
+
function sandboxDrainCaptureSettlingInProcess(lease, nowMs = Date.now()) {
|
|
550
|
+
if (!lease.instanceId) return false;
|
|
551
|
+
const entry = settlingDrainCaptures.get(
|
|
552
|
+
drainCaptureKey({ ...lease, instanceId: lease.instanceId })
|
|
553
|
+
);
|
|
554
|
+
return entry !== void 0 && nowMs - entry.startedAtMs < SANDBOX_ARCHIVE_CAPTURE_MAX_TIMEOUT_MS;
|
|
555
|
+
}
|
|
556
|
+
function trackSettlingDrainCapture(lease) {
|
|
557
|
+
if (!lease.instanceId) return () => void 0;
|
|
558
|
+
const key = drainCaptureKey({ ...lease, instanceId: lease.instanceId });
|
|
559
|
+
const entry = { startedAtMs: Date.now() };
|
|
560
|
+
settlingDrainCaptures.set(key, entry);
|
|
561
|
+
return () => {
|
|
562
|
+
if (settlingDrainCaptures.get(key) === entry) settlingDrainCaptures.delete(key);
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
function sandboxDrainCaptureId(operationId, attempt) {
|
|
566
|
+
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(operationId)) {
|
|
567
|
+
throw new Error("Sandbox drain operation id is invalid");
|
|
568
|
+
}
|
|
569
|
+
if (!Number.isSafeInteger(attempt) || attempt <= 0) {
|
|
570
|
+
throw new Error("Sandbox drain activity attempt is invalid");
|
|
571
|
+
}
|
|
572
|
+
if (attempt === 1) return operationId.toLowerCase();
|
|
573
|
+
const bytes = createHash("sha256").update(`opengeni:sandbox-drain:${operationId.toLowerCase()}:${attempt}`).digest().subarray(0, 16);
|
|
574
|
+
bytes[6] = bytes[6] & 15 | 80;
|
|
575
|
+
bytes[8] = bytes[8] & 63 | 128;
|
|
576
|
+
const hex = bytes.toString("hex");
|
|
577
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
578
|
+
}
|
|
579
|
+
function sandboxDrainActivityAttempt() {
|
|
580
|
+
try {
|
|
581
|
+
const attempt = Context2.current().info.attempt;
|
|
582
|
+
return Number.isSafeInteger(attempt) && attempt > 0 ? attempt : 1;
|
|
583
|
+
} catch {
|
|
584
|
+
return 1;
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
function createSandboxLeaseActivities(services, options = {}) {
|
|
588
|
+
const terminateBox = (options.terminateBox ? async (...args) => {
|
|
589
|
+
await args[10]?.();
|
|
590
|
+
return options.terminateBox(...args);
|
|
591
|
+
} : null) ?? (async (settings, lease, observability, persistArchive, providerCaptureRequestId, captureDisposition, capturePolicy, diskBackedArchives, releaseFailedCapture, workspaceId, beforeProviderStop, beforeWorkspaceCapture) => await terminateProviderBox(
|
|
592
|
+
settings,
|
|
593
|
+
lease,
|
|
594
|
+
observability,
|
|
595
|
+
persistArchive,
|
|
596
|
+
options.createClientForBackend ?? createSandboxClientForBackend2,
|
|
597
|
+
terminateModalSandboxById,
|
|
598
|
+
providerCaptureRequestId,
|
|
599
|
+
captureDisposition,
|
|
600
|
+
capturePolicy,
|
|
601
|
+
diskBackedArchives,
|
|
602
|
+
releaseFailedCapture,
|
|
603
|
+
workspaceId,
|
|
604
|
+
beforeProviderStop,
|
|
605
|
+
beforeWorkspaceCapture
|
|
606
|
+
));
|
|
607
|
+
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
608
|
+
const inspectOpenSandboxInventory = options.inspectOpenSandboxKubernetesInventory ?? inspectOpenSandboxKubernetesInventory;
|
|
609
|
+
const probeRetainedProcess = options.probeRetainedProcess ?? probeRetainedProcessAtProvider;
|
|
610
|
+
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
611
|
+
const resumeIdleCheckpointSession = options.resumeIdleCheckpointSession ?? resumeIdleCheckpointSessionAtProvider;
|
|
612
|
+
const embeddedIdleCheckpoints = /* @__PURE__ */ new Set();
|
|
613
|
+
const endedEpochProbeBackoff = /* @__PURE__ */ new Map();
|
|
614
|
+
async function prepareSandboxLeaseSweep() {
|
|
615
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "prepare" });
|
|
616
|
+
try {
|
|
617
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
618
|
+
const timing = sandboxDrainTiming(settings);
|
|
619
|
+
const commandContainment = {
|
|
620
|
+
// Browsers and desktops follow the general idle grace: unused for it,
|
|
621
|
+
// they stop holding the box (saved browsers resume on demand).
|
|
622
|
+
// Never under a minute, so a visible live view's 30 s heartbeat
|
|
623
|
+
// always lands inside the window.
|
|
624
|
+
idleInteractionReleaseMs: Math.max(settings.sandboxIdleGraceMs, 6e4),
|
|
625
|
+
idleBrowserCheckpoints: browserCheckpointPrerequisites(settings, objectStorage),
|
|
626
|
+
onIdleInteractionRelease: (outcome) => recordSandboxIdleInteractionRelease(observability, outcome),
|
|
627
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
|
|
628
|
+
deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
|
|
629
|
+
onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
|
|
630
|
+
onCommandContainmentError: (error) => {
|
|
631
|
+
observability.warn("sandbox reaper: retained command containment inspection failed", {
|
|
632
|
+
error: error instanceof Error ? error.message : String(error)
|
|
633
|
+
});
|
|
634
|
+
}
|
|
635
|
+
};
|
|
636
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
637
|
+
const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
|
|
638
|
+
...commandContainment,
|
|
639
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
640
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
641
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
642
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
643
|
+
});
|
|
644
|
+
const drainable2 = drainableInventory2.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
645
|
+
return {
|
|
646
|
+
drainable: drainable2,
|
|
647
|
+
timeoutClass: timing.timeoutClass,
|
|
648
|
+
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
649
|
+
captureTimeoutMs: timing.captureTimeoutMs,
|
|
650
|
+
rotationsRequested: 0
|
|
651
|
+
};
|
|
652
|
+
}
|
|
653
|
+
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
654
|
+
db,
|
|
655
|
+
settings.sandboxRotationLeadMs,
|
|
656
|
+
settings.sandboxRotationBatchSize
|
|
657
|
+
).catch((error) => {
|
|
658
|
+
observability.warn("sandbox reaper: provider-deadline rotation request failed", {
|
|
659
|
+
error: error instanceof Error ? error.message : String(error)
|
|
660
|
+
});
|
|
661
|
+
return 0;
|
|
662
|
+
});
|
|
663
|
+
recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
|
|
664
|
+
const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
|
|
665
|
+
...commandContainment,
|
|
666
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
667
|
+
// Dead-worker turn holders: a live holder is touched every 10s from the
|
|
668
|
+
// moment it is registered (resumeBoxForTurn's holder-liveness loop covers
|
|
669
|
+
// the whole warmup — waitForWarm/establish/display-stack — and the turn
|
|
670
|
+
// heartbeat covers the run), so NO live path is ever silent for more than
|
|
671
|
+
// one tick. The ordinary warm lease TTL is therefore already a generous
|
|
672
|
+
// dead-worker horizon. Canonical holders whose turn is durably closed are
|
|
673
|
+
// removed immediately by the DB function; no path-duration guess belongs
|
|
674
|
+
// in this lifecycle contract.
|
|
675
|
+
turnHolderTtlMs: settings.sandboxLeaseTtlMs,
|
|
676
|
+
interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
|
|
677
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
678
|
+
});
|
|
679
|
+
const drainable = drainableInventory.slice(0, SANDBOX_REAPER_CHILD_DISPATCH_LIMIT);
|
|
680
|
+
if (drainableInventory.length > drainable.length) {
|
|
681
|
+
observability.info("sandbox reaper: drain inventory continues next sweep", {
|
|
682
|
+
admitted: drainable.length,
|
|
683
|
+
deferred: drainableInventory.length - drainable.length
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
return {
|
|
687
|
+
drainable,
|
|
688
|
+
timeoutClass: timing.timeoutClass,
|
|
689
|
+
snapshotTimeoutMs: timing.snapshotTimeoutMs,
|
|
690
|
+
captureTimeoutMs: timing.captureTimeoutMs,
|
|
691
|
+
rotationsRequested
|
|
692
|
+
};
|
|
693
|
+
} finally {
|
|
694
|
+
stopHeartbeat();
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
async function listIdleSandboxCheckpoints() {
|
|
698
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({ phase: "idle_checkpoint_inventory" });
|
|
699
|
+
try {
|
|
700
|
+
const { db, settings } = await services();
|
|
701
|
+
const timeoutClass = sandboxDrainTimeoutClass(sandboxArchiveCaptureTimeoutMs(settings));
|
|
702
|
+
if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
|
|
703
|
+
return { targets: [], timeoutClass };
|
|
704
|
+
}
|
|
705
|
+
const rows = await listIdleCheckpointCandidates(db, {
|
|
706
|
+
limit: SANDBOX_IDLE_CHECKPOINT_BATCH,
|
|
707
|
+
intervalMs: settings.sandboxSnapshotIntervalMs
|
|
708
|
+
});
|
|
709
|
+
return {
|
|
710
|
+
targets: rows.map(({ accountId: _accountId, ...target }) => target),
|
|
711
|
+
timeoutClass
|
|
712
|
+
};
|
|
713
|
+
} finally {
|
|
714
|
+
stopHeartbeat();
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
async function checkpointIdleSandboxLease(input) {
|
|
718
|
+
const target = input.target;
|
|
719
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
720
|
+
phase: "idle_checkpoint",
|
|
721
|
+
workspaceId: target.workspaceId,
|
|
722
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
723
|
+
leaseEpoch: target.leaseEpoch,
|
|
724
|
+
instanceId: target.instanceId
|
|
725
|
+
});
|
|
726
|
+
try {
|
|
727
|
+
const { db, settings, observability, objectStorage } = await services();
|
|
728
|
+
const fields = {
|
|
729
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
|
|
730
|
+
leaseEpoch: target.leaseEpoch
|
|
731
|
+
};
|
|
732
|
+
const skipped = (reason) => {
|
|
733
|
+
observability.info("sandbox reaper: idle checkpoint skipped", { ...fields, reason });
|
|
734
|
+
return { status: "skipped", reason };
|
|
735
|
+
};
|
|
736
|
+
if (!settings.sandboxOwnershipEnabled || settings.sandboxSnapshotIntervalMs <= 0) {
|
|
737
|
+
return skipped("disabled");
|
|
738
|
+
}
|
|
739
|
+
const lease = await readLease2(db, target.workspaceId, target.sandboxGroupId);
|
|
740
|
+
if (!lease || lease.liveness !== "warm" || lease.leaseEpoch !== target.leaseEpoch || lease.instanceId !== target.instanceId) {
|
|
741
|
+
return skipped("lease_fenced");
|
|
742
|
+
}
|
|
743
|
+
if (lease.turnHolders > 0) return skipped("turn_held");
|
|
744
|
+
if (lease.refcount === 0) return skipped("drain_owned");
|
|
745
|
+
const { accountId } = await rlsContextForWorkspace(db, target.workspaceId);
|
|
746
|
+
const stamped = await recordIdleCheckpointAttempt(db, {
|
|
747
|
+
accountId,
|
|
748
|
+
workspaceId: target.workspaceId,
|
|
749
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
750
|
+
expectedEpoch: target.leaseEpoch,
|
|
751
|
+
expectedInstanceId: target.instanceId
|
|
752
|
+
});
|
|
753
|
+
if (!stamped) return skipped("lease_fenced");
|
|
754
|
+
if (!providerWorkspaceCaptureIsPointInTime(lease.backend, lease.resumeState)) {
|
|
755
|
+
return skipped("capture_policy");
|
|
756
|
+
}
|
|
757
|
+
if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) return skipped("clean");
|
|
758
|
+
if (lease.archiveCapture) {
|
|
759
|
+
const tookOver = await releaseWorkspaceArchiveCapture(db, {
|
|
760
|
+
accountId,
|
|
761
|
+
workspaceId: target.workspaceId,
|
|
762
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
763
|
+
captureId: lease.archiveCapture.id,
|
|
764
|
+
expectedEpoch: target.leaseEpoch,
|
|
765
|
+
expectedInstanceId: target.instanceId,
|
|
766
|
+
onlyExpiredWarmTakeover: true
|
|
767
|
+
});
|
|
768
|
+
if (!tookOver) return skipped("capture_in_progress");
|
|
769
|
+
observability.warn("sandbox reaper: released an expired warm capture claim", {
|
|
770
|
+
...fields,
|
|
771
|
+
captureId: lease.archiveCapture.id
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
let session;
|
|
775
|
+
try {
|
|
776
|
+
session = await resumeIdleCheckpointSession(settings, lease);
|
|
777
|
+
} catch (error) {
|
|
778
|
+
observability.warn("sandbox reaper: idle checkpoint resume failed", {
|
|
779
|
+
...fields,
|
|
780
|
+
error: error instanceof Error ? error.message : String(error)
|
|
781
|
+
});
|
|
782
|
+
return skipped("provider_error");
|
|
783
|
+
}
|
|
784
|
+
if (session === null) return skipped("provider_missing");
|
|
785
|
+
const capture = maybePersistWarmWorkspaceSnapshot(
|
|
786
|
+
{ db, settings, objectStorage, observability },
|
|
787
|
+
{
|
|
788
|
+
accountId,
|
|
789
|
+
workspaceId: target.workspaceId,
|
|
790
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
791
|
+
idleCheckpoint: true
|
|
792
|
+
},
|
|
793
|
+
session,
|
|
794
|
+
target.leaseEpoch
|
|
795
|
+
);
|
|
796
|
+
const persisted = await capture;
|
|
797
|
+
await capture.settled;
|
|
798
|
+
if (!persisted) return skipped("not_captured");
|
|
799
|
+
observability.info("sandbox reaper: idle checkpoint published", fields);
|
|
800
|
+
return { status: "checkpointed" };
|
|
801
|
+
} finally {
|
|
802
|
+
stopHeartbeat();
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
async function drainSandboxLease(input) {
|
|
806
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
807
|
+
phase: "drain",
|
|
808
|
+
workspaceId: input.target.workspaceId,
|
|
809
|
+
sandboxGroupId: input.target.sandboxGroupId,
|
|
810
|
+
leaseEpoch: input.target.leaseEpoch,
|
|
811
|
+
instanceId: input.target.instanceId
|
|
812
|
+
});
|
|
813
|
+
try {
|
|
814
|
+
const { db, settings, observability, objectStorage, bus } = await services();
|
|
815
|
+
assertSandboxDrainInputTiming(input);
|
|
816
|
+
const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
|
|
817
|
+
const row = input.target;
|
|
818
|
+
const activityAttempt = sandboxDrainActivityAttempt();
|
|
819
|
+
const captureAttempt = {
|
|
820
|
+
operationId: input.operationId,
|
|
821
|
+
captureId: sandboxDrainCaptureId(input.operationId, activityAttempt),
|
|
822
|
+
attempt: activityAttempt,
|
|
823
|
+
captureTimeoutMs: input.captureTimeoutMs
|
|
824
|
+
};
|
|
825
|
+
try {
|
|
826
|
+
const drainedCold = await terminateDrainableBox(
|
|
827
|
+
db,
|
|
828
|
+
drainSettings,
|
|
829
|
+
row,
|
|
830
|
+
observability,
|
|
831
|
+
terminateBox,
|
|
832
|
+
probeDrainableProvider,
|
|
833
|
+
captureAttempt,
|
|
834
|
+
objectStorage,
|
|
835
|
+
bus
|
|
836
|
+
);
|
|
837
|
+
return { status: drainedCold ? "terminated" : "skipped" };
|
|
838
|
+
} catch (error) {
|
|
839
|
+
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId).catch(() => null);
|
|
840
|
+
const details = {
|
|
841
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(row.workspaceId, row.sandboxGroupId),
|
|
842
|
+
workspaceId: row.workspaceId,
|
|
843
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
844
|
+
leaseEpoch: row.leaseEpoch,
|
|
845
|
+
backend: lease?.backend ?? "unknown",
|
|
846
|
+
instanceId: lease?.instanceId ?? row.instanceId,
|
|
847
|
+
captureId: lease?.archiveCapture?.id ?? null,
|
|
848
|
+
captureOperationId: lease?.archiveCapture?.operationId ?? input.operationId,
|
|
849
|
+
captureAttempt: lease?.archiveCapture?.attempt ?? activityAttempt,
|
|
850
|
+
timeoutClass: input.timeoutClass,
|
|
851
|
+
error: error instanceof Error ? error.message : String(error)
|
|
852
|
+
};
|
|
853
|
+
if (error instanceof SandboxDrainRecoveryDeferredError) {
|
|
854
|
+
observability.info("sandbox reaper: per-sandbox recovery deferred safely", details);
|
|
855
|
+
} else {
|
|
856
|
+
observability.warn("sandbox reaper: per-sandbox drain failed", details);
|
|
857
|
+
}
|
|
858
|
+
throw error;
|
|
859
|
+
}
|
|
860
|
+
} finally {
|
|
861
|
+
stopHeartbeat();
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
async function maintainSandboxLeaseSweep(input) {
|
|
865
|
+
const stopHeartbeat = startSandboxReaperHeartbeat({
|
|
866
|
+
phase: "maintain",
|
|
867
|
+
examined: input.examined,
|
|
868
|
+
started: input.started
|
|
869
|
+
});
|
|
870
|
+
try {
|
|
871
|
+
const service = await services();
|
|
872
|
+
const { db, settings, observability } = service;
|
|
873
|
+
const parentUpdates = await reconcilePendingParentSystemUpdates(service, 100).catch(
|
|
874
|
+
(error) => {
|
|
875
|
+
observability.warn("system-update outbox reconciliation failed", {
|
|
876
|
+
error: error instanceof Error ? error.message : String(error)
|
|
877
|
+
});
|
|
878
|
+
return { claimed: 0, delivered: 0, failed: 1 };
|
|
879
|
+
}
|
|
880
|
+
);
|
|
881
|
+
const expiredSessionListSnapshots = await reapExpiredSessionListSnapshots(db, 500).catch(
|
|
882
|
+
(error) => {
|
|
883
|
+
observability.warn("session-list snapshot reconciliation failed", {
|
|
884
|
+
error: error instanceof Error ? error.message : String(error)
|
|
885
|
+
});
|
|
886
|
+
return 0;
|
|
887
|
+
}
|
|
888
|
+
);
|
|
889
|
+
if (expiredSessionListSnapshots > 0) {
|
|
890
|
+
observability.info("expired session-list snapshots reaped", {
|
|
891
|
+
deleted: expiredSessionListSnapshots
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
if (parentUpdates.claimed > 0) {
|
|
895
|
+
observability.info("system-update outbox reconciled", parentUpdates);
|
|
896
|
+
}
|
|
897
|
+
let metered = 0;
|
|
898
|
+
let forceDrained = 0;
|
|
899
|
+
const rotationsRequested = input.rotationsRequested;
|
|
900
|
+
let modalOrphansTerminated = 0;
|
|
901
|
+
await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
|
|
902
|
+
await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
|
|
903
|
+
try {
|
|
904
|
+
await reconcileModalProviderCreates(
|
|
905
|
+
db,
|
|
906
|
+
settings,
|
|
907
|
+
observability,
|
|
908
|
+
options.findModalProviderCreateReceipt
|
|
909
|
+
);
|
|
910
|
+
} catch (error) {
|
|
911
|
+
observability.warn("sandbox reaper: Modal create recovery inventory failed", {
|
|
912
|
+
error: error instanceof Error ? error.message : String(error)
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
916
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
917
|
+
db,
|
|
918
|
+
settings,
|
|
919
|
+
observability,
|
|
920
|
+
inspectOpenSandboxInventory
|
|
921
|
+
);
|
|
922
|
+
return {
|
|
923
|
+
metered,
|
|
924
|
+
forceDrained,
|
|
925
|
+
rotationsRequested,
|
|
926
|
+
modalOrphansTerminated
|
|
927
|
+
};
|
|
928
|
+
}
|
|
929
|
+
const meterResult = await accrueWarmTick(db, settings, observability);
|
|
930
|
+
metered = meterResult.accrued;
|
|
931
|
+
const forceDrainWorkspaceIds = /* @__PURE__ */ new Set();
|
|
932
|
+
if (settings.sandboxWarmBillingMode === "credits" || settings.sandboxMaxWarmSecondsPerWorkspace > 0) {
|
|
933
|
+
for (const workspaceId of meterResult.workspaceIds) {
|
|
934
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
935
|
+
}
|
|
936
|
+
}
|
|
937
|
+
try {
|
|
938
|
+
for (const workspaceId of await listSandboxViewerForceDrainWorkspaceIds(db)) {
|
|
939
|
+
forceDrainWorkspaceIds.add(workspaceId);
|
|
940
|
+
}
|
|
941
|
+
} catch (error) {
|
|
942
|
+
observability.warn("sandbox reaper: viewer force-drain workspace read failed", {
|
|
943
|
+
error: error instanceof Error ? error.message : String(error)
|
|
944
|
+
});
|
|
945
|
+
}
|
|
946
|
+
forceDrained = await forceDrainOverLimitWorkspaces(
|
|
947
|
+
db,
|
|
948
|
+
settings,
|
|
949
|
+
forceDrainWorkspaceIds,
|
|
950
|
+
observability
|
|
951
|
+
);
|
|
952
|
+
await reconcileTerminalRetainedProcesses(
|
|
953
|
+
db,
|
|
954
|
+
settings,
|
|
955
|
+
observability,
|
|
956
|
+
probeRetainedProcess,
|
|
957
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
958
|
+
service.bus
|
|
959
|
+
);
|
|
960
|
+
await settleEndedEpochBlockers(
|
|
961
|
+
db,
|
|
962
|
+
settings,
|
|
963
|
+
observability,
|
|
964
|
+
options.inspectHistoricalModalSandbox ?? inspectModalSandboxLifecycle,
|
|
965
|
+
endedEpochProbeBackoff
|
|
966
|
+
);
|
|
967
|
+
try {
|
|
968
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
969
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
970
|
+
} catch (error) {
|
|
971
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
972
|
+
error: error instanceof Error ? error.message : String(error)
|
|
973
|
+
});
|
|
974
|
+
}
|
|
975
|
+
const checkpointGc = await gcSandboxCheckpointArtifacts(db, settings, observability).catch(
|
|
976
|
+
(error) => {
|
|
977
|
+
observability.warn("sandbox reaper: checkpoint artifact GC failed", {
|
|
978
|
+
error: error instanceof Error ? error.message : String(error)
|
|
979
|
+
});
|
|
980
|
+
return { claimed: 0, deleted: 0, failed: 0 };
|
|
981
|
+
}
|
|
982
|
+
);
|
|
983
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
984
|
+
db,
|
|
985
|
+
settings,
|
|
986
|
+
observability,
|
|
987
|
+
inspectOpenSandboxInventory
|
|
988
|
+
);
|
|
989
|
+
if (input.examined > 0 || metered > 0 || forceDrained > 0 || modalOrphansTerminated > 0 || rotationsRequested > 0 || checkpointGc.claimed > 0) {
|
|
990
|
+
observability.info("sandbox reaper dispatched", {
|
|
991
|
+
drainable: input.examined,
|
|
992
|
+
childrenStarted: input.started,
|
|
993
|
+
childrenAlreadyRunning: input.alreadyRunning,
|
|
994
|
+
childrenStartFailed: input.startFailed,
|
|
995
|
+
metered,
|
|
996
|
+
forceDrained,
|
|
997
|
+
modalOrphansTerminated,
|
|
998
|
+
rotationsRequested,
|
|
999
|
+
checkpointArtifactsClaimed: checkpointGc.claimed,
|
|
1000
|
+
checkpointArtifactsDeleted: checkpointGc.deleted,
|
|
1001
|
+
checkpointArtifactsFailed: checkpointGc.failed
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
return {
|
|
1005
|
+
metered,
|
|
1006
|
+
forceDrained,
|
|
1007
|
+
rotationsRequested,
|
|
1008
|
+
modalOrphansTerminated
|
|
1009
|
+
};
|
|
1010
|
+
} finally {
|
|
1011
|
+
stopHeartbeat();
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
async function reapSandboxLeases() {
|
|
1015
|
+
const service = await services();
|
|
1016
|
+
if (service.startSandboxReaperWorkflow) {
|
|
1017
|
+
const dispatch = await service.startSandboxReaperWorkflow();
|
|
1018
|
+
service.observability.info("sandbox reaper routed to versioned workflow", {
|
|
1019
|
+
dispatch
|
|
1020
|
+
});
|
|
1021
|
+
return {
|
|
1022
|
+
examined: 0,
|
|
1023
|
+
terminated: 0,
|
|
1024
|
+
skipped: 0,
|
|
1025
|
+
metered: 0,
|
|
1026
|
+
forceDrained: 0,
|
|
1027
|
+
modalOrphansTerminated: 0
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
const plan = await prepareSandboxLeaseSweep();
|
|
1031
|
+
const admitted = plan.drainable.slice(0, CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES);
|
|
1032
|
+
const outcomes = await Promise.allSettled(
|
|
1033
|
+
admitted.map(
|
|
1034
|
+
(target) => drainSandboxLease({
|
|
1035
|
+
target,
|
|
1036
|
+
timeoutClass: plan.timeoutClass,
|
|
1037
|
+
snapshotTimeoutMs: plan.snapshotTimeoutMs,
|
|
1038
|
+
captureTimeoutMs: plan.captureTimeoutMs,
|
|
1039
|
+
operationId: randomUUID()
|
|
1040
|
+
})
|
|
1041
|
+
)
|
|
1042
|
+
);
|
|
1043
|
+
try {
|
|
1044
|
+
const { targets, timeoutClass } = await listIdleSandboxCheckpoints();
|
|
1045
|
+
for (const target of targets) {
|
|
1046
|
+
const key = sandboxIdleCheckpointWorkflowId(target);
|
|
1047
|
+
if (embeddedIdleCheckpoints.has(key)) continue;
|
|
1048
|
+
embeddedIdleCheckpoints.add(key);
|
|
1049
|
+
void checkpointIdleSandboxLease({ target, timeoutClass }).catch((error) => {
|
|
1050
|
+
service.observability.warn("sandbox reaper: idle checkpoint failed", {
|
|
1051
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey(target.workspaceId, target.sandboxGroupId),
|
|
1052
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1053
|
+
});
|
|
1054
|
+
}).finally(() => embeddedIdleCheckpoints.delete(key));
|
|
1055
|
+
}
|
|
1056
|
+
} catch (error) {
|
|
1057
|
+
service.observability.warn("sandbox reaper: idle checkpoint inventory failed", {
|
|
1058
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1059
|
+
});
|
|
1060
|
+
}
|
|
1061
|
+
const terminated = outcomes.filter(
|
|
1062
|
+
(outcome) => outcome.status === "fulfilled" && outcome.value.status === "terminated"
|
|
1063
|
+
).length;
|
|
1064
|
+
const skipped = outcomes.length - terminated;
|
|
1065
|
+
const maintenance = await maintainSandboxLeaseSweep({
|
|
1066
|
+
examined: admitted.length,
|
|
1067
|
+
started: admitted.length,
|
|
1068
|
+
alreadyRunning: 0,
|
|
1069
|
+
startFailed: 0,
|
|
1070
|
+
rotationsRequested: plan.rotationsRequested
|
|
1071
|
+
});
|
|
1072
|
+
return {
|
|
1073
|
+
examined: admitted.length,
|
|
1074
|
+
terminated,
|
|
1075
|
+
skipped,
|
|
1076
|
+
metered: maintenance.metered,
|
|
1077
|
+
forceDrained: maintenance.forceDrained,
|
|
1078
|
+
modalOrphansTerminated: maintenance.modalOrphansTerminated
|
|
1079
|
+
};
|
|
1080
|
+
}
|
|
1081
|
+
return {
|
|
1082
|
+
prepareSandboxLeaseSweep,
|
|
1083
|
+
drainSandboxLease,
|
|
1084
|
+
listIdleSandboxCheckpoints,
|
|
1085
|
+
checkpointIdleSandboxLease,
|
|
1086
|
+
maintainSandboxLeaseSweep,
|
|
1087
|
+
reapSandboxLeases
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
async function resumeIdleCheckpointSessionAtProvider(settings, lease) {
|
|
1091
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
1092
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
1093
|
+
if (backend !== "modal" || !lease.instanceId || !lease.resumeState) {
|
|
1094
|
+
throw new Error(`sandbox backend ${backend} has no resumable idle checkpoint box`);
|
|
1095
|
+
}
|
|
1096
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== lease.instanceId) {
|
|
1097
|
+
throw new Error("idle checkpoint lease envelope does not name its provider instance");
|
|
1098
|
+
}
|
|
1099
|
+
const client = createSandboxClientForBackend2(backend, settings);
|
|
1100
|
+
if (!client?.resume) {
|
|
1101
|
+
throw new Error(`sandbox backend ${backend} cannot resume an idle checkpoint box`);
|
|
1102
|
+
}
|
|
1103
|
+
try {
|
|
1104
|
+
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
1105
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope2(
|
|
1106
|
+
client,
|
|
1107
|
+
envelopeSessionState,
|
|
1108
|
+
lease.instanceId
|
|
1109
|
+
);
|
|
1110
|
+
if (resumedState === void 0) {
|
|
1111
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
1112
|
+
}
|
|
1113
|
+
const resumed = await withRetainedProcessProbeTimeout(
|
|
1114
|
+
resumeExactSandboxSession2(client, backend, resumedState, lease.instanceId)
|
|
1115
|
+
);
|
|
1116
|
+
return resumed.session;
|
|
1117
|
+
} catch (error) {
|
|
1118
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
1119
|
+
throw new Error("idle checkpoint provider resume timed out", { cause: error });
|
|
1120
|
+
}
|
|
1121
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) return null;
|
|
1122
|
+
throw error;
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
var CHECKPOINT_GC_LIMIT = 50;
|
|
1126
|
+
var CHECKPOINT_GC_CLAIM_TTL_MS = 10 * 6e4;
|
|
1127
|
+
var SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS = CHECKPOINT_GC_CLAIM_TTL_MS;
|
|
1128
|
+
var CHECKPOINT_TOMBSTONE_RETENTION_MS = 30 * 24 * 60 * 6e4;
|
|
1129
|
+
var CHECKPOINT_TOMBSTONE_PRUNE_LIMIT = 500;
|
|
1130
|
+
var SANDBOX_MAINTENANCE_ITEM_CONCURRENCY = 8;
|
|
1131
|
+
function recordConnectedCommandReconciliation(observability, outcome) {
|
|
1132
|
+
observability.incrementCounter({
|
|
1133
|
+
name: "opengeni_connected_background_command_reconciliation_total",
|
|
1134
|
+
help: "Exact-locator Connected Machine background-command reconciliation outcomes.",
|
|
1135
|
+
labels: { outcome }
|
|
1136
|
+
});
|
|
1137
|
+
}
|
|
1138
|
+
var CONNECTED_RUNNER_STOPPED_REASONS = /* @__PURE__ */ new Set([
|
|
1139
|
+
"GOING_OFFLINE_REASON_USER_STOP",
|
|
1140
|
+
"GOING_OFFLINE_REASON_UPDATE",
|
|
1141
|
+
"GOING_OFFLINE_REASON_HOST_SHUTDOWN"
|
|
1142
|
+
]);
|
|
1143
|
+
async function connectedCommandRunnerRetirement(db, claim, error, lookups) {
|
|
1144
|
+
if (!(error instanceof SelfhostedControlError) || !error.agentOffline || !error.neverSent) {
|
|
1145
|
+
return null;
|
|
1146
|
+
}
|
|
1147
|
+
const key = JSON.stringify([
|
|
1148
|
+
claim.controlWorkspaceId,
|
|
1149
|
+
claim.enrollmentId,
|
|
1150
|
+
claim.connectionInstanceId
|
|
1151
|
+
]);
|
|
1152
|
+
let lookup = lookups.get(key);
|
|
1153
|
+
if (!lookup) {
|
|
1154
|
+
lookup = (async () => {
|
|
1155
|
+
const enrollment = await getEnrollment(db, claim.controlWorkspaceId, claim.enrollmentId);
|
|
1156
|
+
if (!enrollment) return null;
|
|
1157
|
+
if (enrollment.status === "revoked") return "enrollment_revoked";
|
|
1158
|
+
if (enrollment.connectionInstanceId) {
|
|
1159
|
+
return enrollment.connectionInstanceId === claim.connectionInstanceId ? null : "connection_replaced";
|
|
1160
|
+
}
|
|
1161
|
+
return CONNECTED_RUNNER_STOPPED_REASONS.has(enrollment.wentOfflineReason ?? "") ? "connection_stopped" : null;
|
|
1162
|
+
})().catch(() => null);
|
|
1163
|
+
lookups.set(key, lookup);
|
|
1164
|
+
}
|
|
1165
|
+
return await lookup;
|
|
1166
|
+
}
|
|
1167
|
+
async function forEachWithConcurrency(values, concurrency, visit) {
|
|
1168
|
+
if (values.length === 0) return;
|
|
1169
|
+
const workerCount = Math.min(values.length, Math.max(1, Math.floor(concurrency)));
|
|
1170
|
+
let nextIndex = 0;
|
|
1171
|
+
await Promise.all(
|
|
1172
|
+
Array.from({ length: workerCount }, async () => {
|
|
1173
|
+
while (nextIndex < values.length) {
|
|
1174
|
+
const index = nextIndex;
|
|
1175
|
+
nextIndex += 1;
|
|
1176
|
+
await visit(values[index], index);
|
|
1177
|
+
}
|
|
1178
|
+
})
|
|
1179
|
+
);
|
|
1180
|
+
}
|
|
1181
|
+
async function gcSandboxCheckpointArtifacts(db, settings, observability) {
|
|
1182
|
+
const adopted = await adoptLegacyModalCheckpointReceipts(db, settings, observability).catch(
|
|
1183
|
+
(error) => {
|
|
1184
|
+
observability.warn("sandbox reaper: legacy Modal checkpoint discovery failed", {
|
|
1185
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1186
|
+
});
|
|
1187
|
+
return 0;
|
|
1188
|
+
}
|
|
1189
|
+
);
|
|
1190
|
+
if (adopted > 0) {
|
|
1191
|
+
recordSandboxCheckpointArtifactOutcome(observability, "legacy_adopted", adopted);
|
|
1192
|
+
observability.info("sandbox reaper: adopted legacy Modal checkpoint receipts", { adopted });
|
|
1193
|
+
}
|
|
1194
|
+
const claimId = crypto.randomUUID();
|
|
1195
|
+
const claims = await claimSandboxCheckpointArtifactsForGc(db, {
|
|
1196
|
+
claimId,
|
|
1197
|
+
limit: CHECKPOINT_GC_LIMIT,
|
|
1198
|
+
claimTtlMs: CHECKPOINT_GC_CLAIM_TTL_MS
|
|
1199
|
+
});
|
|
1200
|
+
recordSandboxCheckpointArtifactOutcome(observability, "claimed", claims.length);
|
|
1201
|
+
let deleted = 0;
|
|
1202
|
+
let failed = 0;
|
|
1203
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1204
|
+
try {
|
|
1205
|
+
if (claim.providerBackend !== "modal") {
|
|
1206
|
+
throw new Error(`Unsupported checkpoint provider ${claim.providerBackend}`);
|
|
1207
|
+
}
|
|
1208
|
+
await deleteModalCheckpointSnapshot(settings, claim.providerBindingKey, claim.objectId);
|
|
1209
|
+
const settled = await settleSandboxCheckpointArtifactGc(db, {
|
|
1210
|
+
artifactId: claim.id,
|
|
1211
|
+
claimId,
|
|
1212
|
+
deleted: true,
|
|
1213
|
+
retryAfterMs: 1e3
|
|
1214
|
+
});
|
|
1215
|
+
if (settled) {
|
|
1216
|
+
deleted += 1;
|
|
1217
|
+
recordSandboxCheckpointArtifactOutcome(observability, "deleted");
|
|
1218
|
+
}
|
|
1219
|
+
} catch (error) {
|
|
1220
|
+
failed += 1;
|
|
1221
|
+
recordSandboxCheckpointArtifactOutcome(observability, "delete_failed");
|
|
1222
|
+
const retryAfterMs = Math.min(
|
|
1223
|
+
6 * 60 * 6e4,
|
|
1224
|
+
5e3 * 2 ** Math.min(claim.deleteAttempts, 17)
|
|
1225
|
+
);
|
|
1226
|
+
await settleSandboxCheckpointArtifactGc(db, {
|
|
1227
|
+
artifactId: claim.id,
|
|
1228
|
+
claimId,
|
|
1229
|
+
deleted: false,
|
|
1230
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1231
|
+
retryAfterMs
|
|
1232
|
+
}).catch(() => false);
|
|
1233
|
+
observability.warn("sandbox reaper: checkpoint artifact delete failed", {
|
|
1234
|
+
artifactId: claim.id,
|
|
1235
|
+
providerBackend: claim.providerBackend,
|
|
1236
|
+
objectKind: claim.objectKind,
|
|
1237
|
+
attempt: claim.deleteAttempts,
|
|
1238
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
});
|
|
1242
|
+
const pruned = await pruneDeletedSandboxCheckpointArtifacts(
|
|
1243
|
+
db,
|
|
1244
|
+
CHECKPOINT_TOMBSTONE_RETENTION_MS,
|
|
1245
|
+
CHECKPOINT_TOMBSTONE_PRUNE_LIMIT
|
|
1246
|
+
);
|
|
1247
|
+
recordSandboxCheckpointArtifactOutcome(observability, "tombstone_pruned", pruned);
|
|
1248
|
+
return { claimed: claims.length, deleted, failed };
|
|
1249
|
+
}
|
|
1250
|
+
async function adoptLegacyModalCheckpointReceipts(db, settings, observability) {
|
|
1251
|
+
const slots = await listLegacyModalCheckpointSlots(db, 25);
|
|
1252
|
+
if (slots.length === 0) return 0;
|
|
1253
|
+
const bindings = /* @__PURE__ */ new Map();
|
|
1254
|
+
let adopted = 0;
|
|
1255
|
+
await forEachWithConcurrency(slots, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (slot) => {
|
|
1256
|
+
try {
|
|
1257
|
+
let binding = bindings.get(slot.instanceId);
|
|
1258
|
+
if (!binding) {
|
|
1259
|
+
binding = resolveModalCheckpointProviderBindingForLiveSandbox(settings, slot.instanceId);
|
|
1260
|
+
bindings.set(slot.instanceId, binding);
|
|
1261
|
+
}
|
|
1262
|
+
const resolvedBinding = await binding;
|
|
1263
|
+
if (await adoptLegacyModalCheckpointArtifact(db, {
|
|
1264
|
+
accountId: slot.accountId,
|
|
1265
|
+
workspaceId: slot.workspaceId,
|
|
1266
|
+
sandboxGroupId: slot.sandboxGroupId,
|
|
1267
|
+
leaseId: slot.leaseId,
|
|
1268
|
+
leaseEpoch: slot.leaseEpoch,
|
|
1269
|
+
workspaceGeneration: slot.workspaceGeneration,
|
|
1270
|
+
slot: slot.slot,
|
|
1271
|
+
archiveBase64: slot.archiveBase64,
|
|
1272
|
+
descriptor: slot.descriptor,
|
|
1273
|
+
providerBindingKey: resolvedBinding.key,
|
|
1274
|
+
providerBinding: resolvedBinding.binding
|
|
1275
|
+
})) {
|
|
1276
|
+
adopted += 1;
|
|
1277
|
+
}
|
|
1278
|
+
} catch (error) {
|
|
1279
|
+
observability.warn("sandbox reaper: legacy Modal checkpoint adoption failed", {
|
|
1280
|
+
workspaceId: slot.workspaceId,
|
|
1281
|
+
sandboxGroupId: slot.sandboxGroupId,
|
|
1282
|
+
slot: slot.slot,
|
|
1283
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
});
|
|
1287
|
+
return adopted;
|
|
1288
|
+
}
|
|
1289
|
+
async function reconcileConnectedMachineBackgroundCommands(db, settings, observability, bus, controlRpcOverride) {
|
|
1290
|
+
const dueBefore = /* @__PURE__ */ new Date();
|
|
1291
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
1292
|
+
const offlineConnections = /* @__PURE__ */ new Set();
|
|
1293
|
+
const firstConnectionProbes = /* @__PURE__ */ new Map();
|
|
1294
|
+
do {
|
|
1295
|
+
const claimId = crypto.randomUUID();
|
|
1296
|
+
let claims;
|
|
1297
|
+
try {
|
|
1298
|
+
claims = await claimConnectedMachineSessionBackgroundCommands(db, {
|
|
1299
|
+
claimId,
|
|
1300
|
+
dueBefore,
|
|
1301
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1302
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
1303
|
+
});
|
|
1304
|
+
} catch (error) {
|
|
1305
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1306
|
+
observability.warn("sandbox reaper: connected-command claim failed", {
|
|
1307
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1308
|
+
});
|
|
1309
|
+
return;
|
|
1310
|
+
}
|
|
1311
|
+
const controlRpc = controlRpcOverride ?? new NatsControlRpc(
|
|
1312
|
+
async () => bus.getRequestConnection()
|
|
1313
|
+
);
|
|
1314
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1315
|
+
let proof = claim.proof;
|
|
1316
|
+
let outputReplay = null;
|
|
1317
|
+
const connectionKey = JSON.stringify([
|
|
1318
|
+
claim.controlWorkspaceId,
|
|
1319
|
+
claim.enrollmentId,
|
|
1320
|
+
claim.connectionInstanceId
|
|
1321
|
+
]);
|
|
1322
|
+
if (!proof) {
|
|
1323
|
+
const firstProbe = firstConnectionProbes.get(connectionKey);
|
|
1324
|
+
if (firstProbe) await firstProbe;
|
|
1325
|
+
if (offlineConnections.has(connectionKey)) {
|
|
1326
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "provider_offline");
|
|
1327
|
+
return;
|
|
1328
|
+
}
|
|
1329
|
+
let finishFirstProbe;
|
|
1330
|
+
if (!firstProbe) {
|
|
1331
|
+
firstConnectionProbes.set(
|
|
1332
|
+
connectionKey,
|
|
1333
|
+
new Promise((resolve) => {
|
|
1334
|
+
finishFirstProbe = resolve;
|
|
1335
|
+
})
|
|
1336
|
+
);
|
|
1337
|
+
}
|
|
1338
|
+
try {
|
|
1339
|
+
proof = await probeConnectedMachineBackgroundCommand(
|
|
1340
|
+
claim,
|
|
1341
|
+
controlRpc,
|
|
1342
|
+
Math.min(
|
|
1343
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1344
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
1345
|
+
)
|
|
1346
|
+
);
|
|
1347
|
+
if (proof?.outcome !== "lost") {
|
|
1348
|
+
if (!bus.getOpStreamConnection)
|
|
1349
|
+
throw new Error("Connected command output transport unavailable");
|
|
1350
|
+
const client = new OpStreamExecClient({
|
|
1351
|
+
workspaceId: claim.controlWorkspaceId,
|
|
1352
|
+
agentId: claim.enrollmentId,
|
|
1353
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1354
|
+
epoch: 0,
|
|
1355
|
+
controlRpc,
|
|
1356
|
+
rpcSubject: subjectFor(
|
|
1357
|
+
claim.controlWorkspaceId,
|
|
1358
|
+
claim.enrollmentId,
|
|
1359
|
+
claim.connectionInstanceId
|
|
1360
|
+
),
|
|
1361
|
+
transport: new NatsOpStreamTransport(
|
|
1362
|
+
async () => bus.getOpStreamConnection?.() ?? null
|
|
1363
|
+
),
|
|
1364
|
+
controlTimeoutMs: Math.min(
|
|
1365
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1366
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
1367
|
+
),
|
|
1368
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
1369
|
+
journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {
|
|
1370
|
+
} }
|
|
1371
|
+
});
|
|
1372
|
+
outputReplay = await replayConnectedCommandOutput(
|
|
1373
|
+
client,
|
|
1374
|
+
claim.opId,
|
|
1375
|
+
Boolean(proof),
|
|
1376
|
+
async (frames) => {
|
|
1377
|
+
await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
|
|
1378
|
+
}
|
|
1379
|
+
);
|
|
1380
|
+
}
|
|
1381
|
+
if (!proof) {
|
|
1382
|
+
await deferConnectedCommandClaim(
|
|
1383
|
+
db,
|
|
1384
|
+
settings,
|
|
1385
|
+
observability,
|
|
1386
|
+
claim,
|
|
1387
|
+
"provider_running"
|
|
1388
|
+
);
|
|
1389
|
+
return;
|
|
1390
|
+
}
|
|
1391
|
+
} catch (error) {
|
|
1392
|
+
const offline = error instanceof SelfhostedControlError && (error.agentOffline || error.reason === "agent_reconnecting" || error.draining);
|
|
1393
|
+
if (offline) offlineConnections.add(connectionKey);
|
|
1394
|
+
if (!offline) {
|
|
1395
|
+
observability.warn("sandbox reaper: connected-command provider probe failed", {
|
|
1396
|
+
commandId: claim.commandId,
|
|
1397
|
+
enrollmentId: claim.enrollmentId,
|
|
1398
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1399
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1400
|
+
});
|
|
1401
|
+
}
|
|
1402
|
+
await deferConnectedCommandClaim(
|
|
1403
|
+
db,
|
|
1404
|
+
settings,
|
|
1405
|
+
observability,
|
|
1406
|
+
claim,
|
|
1407
|
+
offline ? "provider_offline" : "provider_error"
|
|
1408
|
+
);
|
|
1409
|
+
return;
|
|
1410
|
+
} finally {
|
|
1411
|
+
finishFirstProbe?.();
|
|
1412
|
+
}
|
|
1413
|
+
try {
|
|
1414
|
+
await recordConnectedMachineBackgroundCommandProof(db, { claim, proof });
|
|
1415
|
+
recordConnectedCommandReconciliation(observability, `proof_${proof.outcome}`);
|
|
1416
|
+
} catch (error) {
|
|
1417
|
+
recordConnectedCommandReconciliation(observability, "proof_checkpoint_failed");
|
|
1418
|
+
observability.warn("sandbox reaper: connected-command proof checkpoint failed", {
|
|
1419
|
+
commandId: claim.commandId,
|
|
1420
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1421
|
+
});
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
try {
|
|
1426
|
+
const settlement = await settleClaimedConnectedMachineBackgroundCommand(db, { claim });
|
|
1427
|
+
if (!settlement.settled) {
|
|
1428
|
+
throw new Error("Connected command settlement lost its exact claim or proof");
|
|
1429
|
+
}
|
|
1430
|
+
const receipt = outputReplay?.outputReceipt ?? (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : void 0);
|
|
1431
|
+
const terminal = outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
|
|
1432
|
+
if (proof.outcome === "exited" && receipt && terminal) {
|
|
1433
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
1434
|
+
...claim,
|
|
1435
|
+
receipt,
|
|
1436
|
+
exitCode: terminal.outcome.response.exitCode
|
|
1437
|
+
});
|
|
1438
|
+
}
|
|
1439
|
+
if (settlement.events.length > 0) {
|
|
1440
|
+
try {
|
|
1441
|
+
await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
|
|
1442
|
+
} catch (error) {
|
|
1443
|
+
observability.warn("sandbox reaper: connected-command event fanout failed", {
|
|
1444
|
+
commandId: claim.commandId,
|
|
1445
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1446
|
+
});
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
recordConnectedCommandReconciliation(observability, `settled_${proof.outcome}`);
|
|
1450
|
+
} catch (error) {
|
|
1451
|
+
recordConnectedCommandReconciliation(observability, "settlement_failed");
|
|
1452
|
+
observability.warn("sandbox reaper: connected-command settlement failed", {
|
|
1453
|
+
commandId: claim.commandId,
|
|
1454
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1455
|
+
});
|
|
1456
|
+
await deferConnectedCommandClaim(db, settings, observability, claim, "settlement_failed");
|
|
1457
|
+
}
|
|
1458
|
+
});
|
|
1459
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
1460
|
+
} while (Date.now() < deadline);
|
|
1461
|
+
}
|
|
1462
|
+
async function replayConnectedCommandOutput(client, opId, terminalKnown, capture) {
|
|
1463
|
+
let capturedThrough = 0n;
|
|
1464
|
+
while (true) {
|
|
1465
|
+
const before = capturedThrough;
|
|
1466
|
+
const replay = await client.readExisting(opId, terminalKnown ? 5e3 : 250, capture);
|
|
1467
|
+
if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
|
|
1468
|
+
capturedThrough = BigInt(replay.replaySequence);
|
|
1469
|
+
if (capturedThrough === before) {
|
|
1470
|
+
throw new Error("Connected command terminal output replay has not reached its exit frontier");
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
async function reconcileConnectedCommandOutputReleases(db, settings, observability, bus, controlRpcOverride) {
|
|
1475
|
+
const dueBefore = /* @__PURE__ */ new Date();
|
|
1476
|
+
const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
|
|
1477
|
+
const controlRpc = controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
|
|
1478
|
+
const runnerLookups = /* @__PURE__ */ new Map();
|
|
1479
|
+
do {
|
|
1480
|
+
let claims;
|
|
1481
|
+
try {
|
|
1482
|
+
claims = await claimConnectedCommandOutputReleases(db, {
|
|
1483
|
+
claimId: crypto.randomUUID(),
|
|
1484
|
+
dueBefore,
|
|
1485
|
+
limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
1486
|
+
claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS
|
|
1487
|
+
});
|
|
1488
|
+
} catch (error) {
|
|
1489
|
+
recordConnectedCommandReconciliation(observability, "claim_failed");
|
|
1490
|
+
observability.warn("sandbox reaper: connected-command output claim failed", {
|
|
1491
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1492
|
+
});
|
|
1493
|
+
return;
|
|
1494
|
+
}
|
|
1495
|
+
await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
|
|
1496
|
+
let receipt = claim.receipt;
|
|
1497
|
+
try {
|
|
1498
|
+
if (!bus.getOpStreamConnection)
|
|
1499
|
+
throw new Error("Connected command output transport unavailable");
|
|
1500
|
+
const rpcSubject = subjectFor(
|
|
1501
|
+
claim.controlWorkspaceId,
|
|
1502
|
+
claim.enrollmentId,
|
|
1503
|
+
claim.connectionInstanceId
|
|
1504
|
+
);
|
|
1505
|
+
const controlTimeoutMs = Math.min(
|
|
1506
|
+
settings.sandboxSelfhostedControlTimeoutMs,
|
|
1507
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
1508
|
+
);
|
|
1509
|
+
const client = new OpStreamExecClient({
|
|
1510
|
+
workspaceId: claim.controlWorkspaceId,
|
|
1511
|
+
agentId: claim.enrollmentId,
|
|
1512
|
+
connectionInstanceId: claim.connectionInstanceId,
|
|
1513
|
+
epoch: 0,
|
|
1514
|
+
controlRpc,
|
|
1515
|
+
rpcSubject,
|
|
1516
|
+
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
1517
|
+
controlTimeoutMs,
|
|
1518
|
+
retryClock: defaultSelfhostedRetryClock,
|
|
1519
|
+
journal: {
|
|
1520
|
+
attachGeneration: () => String(Date.now()),
|
|
1521
|
+
persistSettled: () => {
|
|
1522
|
+
throw new Error("Background output release requires its database receipt");
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
});
|
|
1526
|
+
if (!receipt) {
|
|
1527
|
+
const status = await querySelfhostedOp({
|
|
1528
|
+
controlRpc,
|
|
1529
|
+
rpcSubject,
|
|
1530
|
+
opId: claim.opId,
|
|
1531
|
+
controlTimeoutMs
|
|
1532
|
+
});
|
|
1533
|
+
if (status.state === OpState.OP_STATE_LOST) {
|
|
1534
|
+
if (!await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1535
|
+
claim,
|
|
1536
|
+
outcome: "unavailable",
|
|
1537
|
+
retryAfterMs: 0
|
|
1538
|
+
}))
|
|
1539
|
+
throw new Error("Connected command output loss no longer owns its exact claim");
|
|
1540
|
+
recordConnectedCommandReconciliation(observability, "output_unavailable");
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
if (status.state !== OpState.OP_STATE_COMPLETE)
|
|
1544
|
+
throw new Error("Terminal command output returned a live operation");
|
|
1545
|
+
const replay = await replayConnectedCommandOutput(
|
|
1546
|
+
client,
|
|
1547
|
+
claim.opId,
|
|
1548
|
+
true,
|
|
1549
|
+
async (frames) => captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames)
|
|
1550
|
+
);
|
|
1551
|
+
const terminal = replay.status === "completed" ? replay : replay.terminal;
|
|
1552
|
+
receipt = replay.outputReceipt ?? (replay.status === "running" ? replay.terminal?.outputReceipt : void 0) ?? null;
|
|
1553
|
+
if (!terminal || !receipt)
|
|
1554
|
+
throw new Error("Connected command output replay has no complete custody receipt");
|
|
1555
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
1556
|
+
...claim,
|
|
1557
|
+
receipt,
|
|
1558
|
+
exitCode: terminal.outcome.response.exitCode
|
|
1559
|
+
});
|
|
1560
|
+
}
|
|
1561
|
+
const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
|
|
1562
|
+
if (!await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1563
|
+
claim,
|
|
1564
|
+
outcome,
|
|
1565
|
+
retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0
|
|
1566
|
+
}))
|
|
1567
|
+
throw new Error("Connected command output release no longer owns its exact claim");
|
|
1568
|
+
recordConnectedCommandReconciliation(
|
|
1569
|
+
observability,
|
|
1570
|
+
outcome === "published" ? "output_ack_published" : "output_released"
|
|
1571
|
+
);
|
|
1572
|
+
} catch (error) {
|
|
1573
|
+
const retired = await connectedCommandRunnerRetirement(db, claim, error, runnerLookups);
|
|
1574
|
+
if (retired && await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1575
|
+
claim,
|
|
1576
|
+
outcome: receipt ? "not_retained" : "unavailable",
|
|
1577
|
+
retryAfterMs: 0
|
|
1578
|
+
}).catch(() => false)) {
|
|
1579
|
+
recordConnectedCommandReconciliation(
|
|
1580
|
+
observability,
|
|
1581
|
+
receipt ? "output_released" : "output_unavailable"
|
|
1582
|
+
);
|
|
1583
|
+
return;
|
|
1584
|
+
}
|
|
1585
|
+
await settleConnectedCommandOutputReleaseClaim(db, {
|
|
1586
|
+
claim,
|
|
1587
|
+
outcome: "retry",
|
|
1588
|
+
retryAfterMs: Math.min(
|
|
1589
|
+
5 * 6e4,
|
|
1590
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** Math.min(4, claim.reconcileAttempts - 1)
|
|
1591
|
+
)
|
|
1592
|
+
}).catch(() => false);
|
|
1593
|
+
observability.warn("sandbox reaper: connected-command output release deferred", {
|
|
1594
|
+
commandId: claim.commandId,
|
|
1595
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1596
|
+
});
|
|
1597
|
+
}
|
|
1598
|
+
});
|
|
1599
|
+
if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
|
|
1600
|
+
} while (Date.now() < deadline);
|
|
1601
|
+
}
|
|
1602
|
+
async function probeConnectedMachineBackgroundCommand(claim, controlRpc, controlTimeoutMs, observedAt = /* @__PURE__ */ new Date()) {
|
|
1603
|
+
const input = {
|
|
1604
|
+
controlRpc,
|
|
1605
|
+
rpcSubject: subjectFor(
|
|
1606
|
+
claim.controlWorkspaceId,
|
|
1607
|
+
claim.enrollmentId,
|
|
1608
|
+
claim.connectionInstanceId
|
|
1609
|
+
),
|
|
1610
|
+
opId: claim.opId,
|
|
1611
|
+
controlTimeoutMs
|
|
1612
|
+
};
|
|
1613
|
+
const status = claim.state === "stopping" ? await cancelSelfhostedOp(input) : await querySelfhostedOp(input);
|
|
1614
|
+
return connectedCommandProofFromStatus(status, observedAt);
|
|
1615
|
+
}
|
|
1616
|
+
function connectedCommandProofFromStatus(status, observedAt) {
|
|
1617
|
+
switch (status.state) {
|
|
1618
|
+
case OpState.OP_STATE_ACCEPTED:
|
|
1619
|
+
case OpState.OP_STATE_RUNNING:
|
|
1620
|
+
return null;
|
|
1621
|
+
case OpState.OP_STATE_COMPLETE: {
|
|
1622
|
+
const exit = status.exit;
|
|
1623
|
+
if (!exit) {
|
|
1624
|
+
throw new Error("Connected command completed without an exit record");
|
|
1625
|
+
}
|
|
1626
|
+
const failureCode = exit.failureCode.replace(/[^A-Za-z0-9_-]/g, "_").slice(0, 128);
|
|
1627
|
+
const reason = failureCode ? `op_failure_${failureCode}` : exit.cancelled ? "op_cancelled" : exit.timedOut ? "op_timed_out" : "op_exit";
|
|
1628
|
+
return {
|
|
1629
|
+
outcome: "exited",
|
|
1630
|
+
exitCode: exit.exitCode,
|
|
1631
|
+
reason,
|
|
1632
|
+
...failureCode ? {
|
|
1633
|
+
failure: { code: failureCode, detail: exit.failureDetail, retryable: false }
|
|
1634
|
+
} : {},
|
|
1635
|
+
observedAt
|
|
1636
|
+
};
|
|
1637
|
+
}
|
|
1638
|
+
case OpState.OP_STATE_LOST:
|
|
1639
|
+
return {
|
|
1640
|
+
outcome: "lost",
|
|
1641
|
+
exitCode: null,
|
|
1642
|
+
reason: status.lostReason === OpLostReason.OP_LOST_REASON_EVICTED ? "op_lost_evicted" : status.lostReason === OpLostReason.OP_LOST_REASON_AGENT_RESTARTED ? "op_lost_agent_restarted" : "op_lost_unspecified",
|
|
1643
|
+
observedAt
|
|
1644
|
+
};
|
|
1645
|
+
default:
|
|
1646
|
+
throw new Error(`Connected command returned invalid op state ${status.state}`);
|
|
1647
|
+
}
|
|
1648
|
+
}
|
|
1649
|
+
async function deferConnectedCommandClaim(db, settings, observability, claim, outcome) {
|
|
1650
|
+
const exponent = Math.min(4, Math.max(0, claim.reconcileAttempts - 1));
|
|
1651
|
+
const retryAfterMs = Math.min(
|
|
1652
|
+
5 * 6e4,
|
|
1653
|
+
Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4) * 2 ** exponent
|
|
1654
|
+
);
|
|
1655
|
+
try {
|
|
1656
|
+
if (!await deferConnectedMachineBackgroundCommandReconciliation(db, {
|
|
1657
|
+
claim,
|
|
1658
|
+
outcome,
|
|
1659
|
+
retryAfterMs
|
|
1660
|
+
})) {
|
|
1661
|
+
throw new Error("Connected command defer lost its exact claim");
|
|
1662
|
+
}
|
|
1663
|
+
recordConnectedCommandReconciliation(observability, outcome);
|
|
1664
|
+
} catch (error) {
|
|
1665
|
+
recordConnectedCommandReconciliation(observability, "defer_failed");
|
|
1666
|
+
observability.warn("sandbox reaper: connected-command defer failed", {
|
|
1667
|
+
commandId: claim.commandId,
|
|
1668
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
var ENDED_EPOCH_BLOCKER_LIST_LIMIT = 200;
|
|
1673
|
+
var ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS = 6e4;
|
|
1674
|
+
var ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS = 5 * 6e4;
|
|
1675
|
+
var ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS = 6 * 60 * 6e4;
|
|
1676
|
+
async function settleEndedEpochBlockers(db, settings, observability, inspectHistoricalModalSandbox, backoff) {
|
|
1677
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
1678
|
+
return;
|
|
1679
|
+
}
|
|
1680
|
+
const startedAtMs = Date.now();
|
|
1681
|
+
const observed = /* @__PURE__ */ new Map();
|
|
1682
|
+
const kept = {};
|
|
1683
|
+
async function observe(instanceId, details) {
|
|
1684
|
+
const known = observed.get(instanceId);
|
|
1685
|
+
if (known !== void 0) return known;
|
|
1686
|
+
const nowMs = Date.now();
|
|
1687
|
+
if ((backoff.get(instanceId)?.nextProbeAtMs ?? 0) > nowMs) return null;
|
|
1688
|
+
if (nowMs - startedAtMs >= ENDED_EPOCH_BLOCKER_SWEEP_BUDGET_MS) return null;
|
|
1689
|
+
let status;
|
|
1690
|
+
try {
|
|
1691
|
+
status = (await withRetainedProcessProbeTimeout(
|
|
1692
|
+
inspectHistoricalModalSandbox(settings, instanceId, null)
|
|
1693
|
+
)).status;
|
|
1694
|
+
} catch (error) {
|
|
1695
|
+
status = error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "error";
|
|
1696
|
+
observability.warn("sandbox reaper: ended-epoch box probe failed", {
|
|
1697
|
+
...details,
|
|
1698
|
+
error: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : error instanceof Error ? error.message : String(error)
|
|
1699
|
+
});
|
|
1700
|
+
}
|
|
1701
|
+
observed.set(instanceId, status);
|
|
1702
|
+
if (status === "terminated") {
|
|
1703
|
+
backoff.delete(instanceId);
|
|
1704
|
+
} else {
|
|
1705
|
+
deferEndedEpochProbe(backoff, instanceId);
|
|
1706
|
+
kept[status] = (kept[status] ?? 0) + 1;
|
|
1707
|
+
}
|
|
1708
|
+
return status;
|
|
1709
|
+
}
|
|
1710
|
+
let tuples;
|
|
1711
|
+
try {
|
|
1712
|
+
tuples = await listEndedEpochWorkspaceBlockers(db, { limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT });
|
|
1713
|
+
} catch (error) {
|
|
1714
|
+
observability.warn("sandbox reaper: ended-epoch blocker inventory failed", {
|
|
1715
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1716
|
+
});
|
|
1717
|
+
tuples = [];
|
|
1718
|
+
}
|
|
1719
|
+
for (const tuple of tuples) {
|
|
1720
|
+
const instanceId = tuple.lostInstanceId;
|
|
1721
|
+
const details = {
|
|
1722
|
+
workspaceId: tuple.workspaceId,
|
|
1723
|
+
sandboxGroupId: tuple.sandboxGroupId,
|
|
1724
|
+
lostEpoch: tuple.lostEpoch,
|
|
1725
|
+
instanceId
|
|
1726
|
+
};
|
|
1727
|
+
try {
|
|
1728
|
+
const status = await observe(instanceId, details);
|
|
1729
|
+
if (status !== "terminated") continue;
|
|
1730
|
+
const settled = await settleEndedEpochWorkspaceBlockers(db, tuple);
|
|
1731
|
+
if (settled) {
|
|
1732
|
+
observability.info("sandbox reaper: settled blockers of an ended lease epoch", {
|
|
1733
|
+
...details,
|
|
1734
|
+
...settled
|
|
1735
|
+
});
|
|
1736
|
+
}
|
|
1737
|
+
} catch (error) {
|
|
1738
|
+
observability.warn("sandbox reaper: ended-epoch blocker settlement failed", {
|
|
1739
|
+
...details,
|
|
1740
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
}
|
|
1744
|
+
let setups;
|
|
1745
|
+
try {
|
|
1746
|
+
setups = await listEndedEpochSandboxSetupUncertainties(db, {
|
|
1747
|
+
limit: ENDED_EPOCH_BLOCKER_LIST_LIMIT
|
|
1748
|
+
});
|
|
1749
|
+
} catch (error) {
|
|
1750
|
+
observability.warn("sandbox reaper: ended-epoch setup inventory failed", {
|
|
1751
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1752
|
+
});
|
|
1753
|
+
setups = [];
|
|
1754
|
+
}
|
|
1755
|
+
for (const setup of setups) {
|
|
1756
|
+
const instanceId = setup.box.providerInstanceId;
|
|
1757
|
+
const turnKey = `setup:${setup.turnId}:${setup.attemptId}`;
|
|
1758
|
+
if ((backoff.get(turnKey)?.nextProbeAtMs ?? 0) > Date.now()) continue;
|
|
1759
|
+
const details = {
|
|
1760
|
+
workspaceId: setup.workspaceId,
|
|
1761
|
+
sessionId: setup.sessionId,
|
|
1762
|
+
turnId: setup.turnId,
|
|
1763
|
+
sandboxGroupId: setup.sandboxGroupId,
|
|
1764
|
+
lostEpoch: setup.box.leaseEpoch,
|
|
1765
|
+
instanceId
|
|
1766
|
+
};
|
|
1767
|
+
try {
|
|
1768
|
+
const status = await observe(instanceId, details);
|
|
1769
|
+
if (status !== "terminated") continue;
|
|
1770
|
+
const { reconciled } = await reconcileCompletedSandboxSetup(db, {
|
|
1771
|
+
accountId: setup.accountId,
|
|
1772
|
+
workspaceId: setup.workspaceId,
|
|
1773
|
+
sessionId: setup.sessionId,
|
|
1774
|
+
turnId: setup.turnId,
|
|
1775
|
+
attemptId: setup.attemptId,
|
|
1776
|
+
terminatedBox: setup.box
|
|
1777
|
+
});
|
|
1778
|
+
if (reconciled) {
|
|
1779
|
+
backoff.delete(turnKey);
|
|
1780
|
+
observability.info(
|
|
1781
|
+
"sandbox reaper: resumed a turn whose unknown setup ran on a terminated box",
|
|
1782
|
+
details
|
|
1783
|
+
);
|
|
1784
|
+
} else {
|
|
1785
|
+
deferEndedEpochProbe(backoff, turnKey);
|
|
1786
|
+
}
|
|
1787
|
+
} catch (error) {
|
|
1788
|
+
deferEndedEpochProbe(backoff, turnKey);
|
|
1789
|
+
observability.warn("sandbox reaper: ended-epoch setup reconciliation failed", {
|
|
1790
|
+
...details,
|
|
1791
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1792
|
+
});
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
if (Object.keys(kept).length) {
|
|
1796
|
+
observability.info("sandbox reaper: ended-epoch blockers kept without terminal proof", kept);
|
|
1797
|
+
}
|
|
1798
|
+
}
|
|
1799
|
+
function deferEndedEpochProbe(backoff, key) {
|
|
1800
|
+
const failures = (backoff.get(key)?.failures ?? 0) + 1;
|
|
1801
|
+
backoff.set(key, {
|
|
1802
|
+
failures,
|
|
1803
|
+
nextProbeAtMs: Date.now() + Math.min(
|
|
1804
|
+
ENDED_EPOCH_BLOCKER_BACKOFF_MAX_MS,
|
|
1805
|
+
ENDED_EPOCH_BLOCKER_BACKOFF_BASE_MS * 2 ** Math.min(failures - 1, 16)
|
|
1806
|
+
)
|
|
1807
|
+
});
|
|
1808
|
+
}
|
|
1809
|
+
async function reconcileTerminalRetainedProcesses(db, settings, observability, probe, inspectHistoricalModalSandbox, bus) {
|
|
1810
|
+
const claimId = crypto.randomUUID();
|
|
1811
|
+
let claims;
|
|
1812
|
+
try {
|
|
1813
|
+
claims = await claimTerminalRetainedProcesses(db, {
|
|
1814
|
+
claimId,
|
|
1815
|
+
limit: RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
1816
|
+
claimTtlMs: RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS
|
|
1817
|
+
});
|
|
1818
|
+
} catch (error) {
|
|
1819
|
+
recordRetainedProcessReconciliation(observability, "claim_failed");
|
|
1820
|
+
observability.warn("sandbox reaper: retained-process claim failed", {
|
|
1821
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1822
|
+
});
|
|
1823
|
+
return;
|
|
1824
|
+
}
|
|
1825
|
+
const drainBudget = {
|
|
1826
|
+
until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS
|
|
1827
|
+
};
|
|
1828
|
+
const lostBoxes = /* @__PURE__ */ new Set();
|
|
1829
|
+
for (const claim of claims) {
|
|
1830
|
+
let process = claim.process;
|
|
1831
|
+
if (lostBoxes.has(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`))
|
|
1832
|
+
continue;
|
|
1833
|
+
const expected = retainedProcessSettlementIdentity(process);
|
|
1834
|
+
let proof = retainedProcessReconciliationProof(process);
|
|
1835
|
+
let wholeBoxGone = false;
|
|
1836
|
+
const processScope = {
|
|
1837
|
+
accountId: process.accountId,
|
|
1838
|
+
workspaceId: process.workspaceId,
|
|
1839
|
+
sessionId: process.sessionId,
|
|
1840
|
+
processId: process.id
|
|
1841
|
+
};
|
|
1842
|
+
if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
|
|
1843
|
+
try {
|
|
1844
|
+
const command = await recoverManagedSessionBackgroundCommand(db, {
|
|
1845
|
+
...processScope,
|
|
1846
|
+
expected,
|
|
1847
|
+
reconciliationClaimId: claim.claimId
|
|
1848
|
+
});
|
|
1849
|
+
if (command) {
|
|
1850
|
+
claim.ownerState = `background_${command.state}`;
|
|
1851
|
+
recordRetainedProcessReconciliation(observability, "background_recovered");
|
|
1852
|
+
}
|
|
1853
|
+
} catch (error) {
|
|
1854
|
+
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
1855
|
+
processId: process.id,
|
|
1856
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1857
|
+
});
|
|
1858
|
+
}
|
|
1859
|
+
}
|
|
1860
|
+
const storedCommandPersistence = retainedProviderCommandPersistence(
|
|
1861
|
+
db,
|
|
1862
|
+
processScope,
|
|
1863
|
+
bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : void 0
|
|
1864
|
+
);
|
|
1865
|
+
const commandPersistence = {
|
|
1866
|
+
...storedCommandPersistence,
|
|
1867
|
+
requestCancellation: async (reason) => {
|
|
1868
|
+
if (reason === "provider_deadline") {
|
|
1869
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
1870
|
+
if (!await storedCommandPersistence.cancellationRequested())
|
|
1871
|
+
throw new Error(
|
|
1872
|
+
"Supervised deadline cancellation no longer owns its original rotating lease"
|
|
1873
|
+
);
|
|
1874
|
+
} else await storedCommandPersistence.requestCancellation(reason);
|
|
1875
|
+
}
|
|
1876
|
+
};
|
|
1877
|
+
if (proof && process.providerBackend === "modal") {
|
|
1878
|
+
try {
|
|
1879
|
+
const command = await commandPersistence.load();
|
|
1880
|
+
if (command?.kind === "modal-router-v1" && command.supervision && (!await commandPersistence.loadSupervisionReceipt() || ![command.streams.stdout, command.streams.stderr].every(
|
|
1881
|
+
(stream) => stream.eof && stream.exitCode === 0
|
|
1882
|
+
)))
|
|
1883
|
+
proof = null;
|
|
1884
|
+
} catch {
|
|
1885
|
+
proof = null;
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
if (!proof) {
|
|
1889
|
+
let observation = null;
|
|
1890
|
+
if (process.providerBackend === "modal" && process.routeTargetId === null && !process.providerBindingKey) {
|
|
1891
|
+
try {
|
|
1892
|
+
const lifecycle = await withRetainedProcessProbeTimeout(
|
|
1893
|
+
inspectHistoricalModalSandbox(settings, process.providerInstanceId, null)
|
|
1894
|
+
);
|
|
1895
|
+
if (lifecycle.status === "not_found") {
|
|
1896
|
+
observation = {
|
|
1897
|
+
status: "deferred",
|
|
1898
|
+
reason: "provider_binding_missing"
|
|
1899
|
+
};
|
|
1900
|
+
} else {
|
|
1901
|
+
process = await bindRetainedProcessProviderIdentity(db, {
|
|
1902
|
+
accountId: process.accountId,
|
|
1903
|
+
workspaceId: process.workspaceId,
|
|
1904
|
+
sessionId: process.sessionId,
|
|
1905
|
+
processId: process.id,
|
|
1906
|
+
expected,
|
|
1907
|
+
claimId: claim.claimId,
|
|
1908
|
+
providerBindingKey: lifecycle.providerBindingKey,
|
|
1909
|
+
providerBinding: lifecycle.providerBinding
|
|
1910
|
+
});
|
|
1911
|
+
recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
|
|
1912
|
+
if (lifecycle.status === "terminated") {
|
|
1913
|
+
observation = {
|
|
1914
|
+
status: "proved",
|
|
1915
|
+
proof: {
|
|
1916
|
+
outcome: "lost",
|
|
1917
|
+
exitCode: null,
|
|
1918
|
+
reason: "provider_instance_terminated"
|
|
1919
|
+
}
|
|
1920
|
+
};
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
} catch (error) {
|
|
1924
|
+
observation = {
|
|
1925
|
+
status: "deferred",
|
|
1926
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
|
|
1927
|
+
};
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
if (observation?.status === "deferred" && process.providerBackend === "modal" && !process.providerBindingKey && !process.deadlineCancellationRequestedAt) {
|
|
1931
|
+
try {
|
|
1932
|
+
await requestRetainedProcessDeadlineCancellation(db, processScope);
|
|
1933
|
+
} catch (error) {
|
|
1934
|
+
observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
|
|
1935
|
+
processId: process.id,
|
|
1936
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1937
|
+
});
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
if (observation === null) {
|
|
1941
|
+
const lease = await readLease2(db, process.workspaceId, process.sandboxGroupId).catch(
|
|
1942
|
+
() => null
|
|
1943
|
+
);
|
|
1944
|
+
const currentLeaseIdentityMatches = Boolean(lease) && lease.id === process.leaseId && lease.leaseEpoch === process.leaseEpoch && lease.backend === process.providerBackend && lease.instanceId === process.providerInstanceId && process.routeTargetId === null;
|
|
1945
|
+
if (!currentLeaseIdentityMatches) {
|
|
1946
|
+
if (process.providerBackend !== "modal" || process.routeTargetId !== null) {
|
|
1947
|
+
observation = { status: "deferred", reason: "identity_mismatch" };
|
|
1948
|
+
} else {
|
|
1949
|
+
try {
|
|
1950
|
+
const lifecycle = await withRetainedProcessProbeTimeout(
|
|
1951
|
+
inspectHistoricalModalSandbox(
|
|
1952
|
+
settings,
|
|
1953
|
+
process.providerInstanceId,
|
|
1954
|
+
process.providerBindingKey
|
|
1955
|
+
)
|
|
1956
|
+
);
|
|
1957
|
+
observation = lifecycle.status === "not_found" ? {
|
|
1958
|
+
status: "proved",
|
|
1959
|
+
proof: {
|
|
1960
|
+
outcome: "lost",
|
|
1961
|
+
exitCode: null,
|
|
1962
|
+
reason: "provider_instance_not_found"
|
|
1963
|
+
}
|
|
1964
|
+
} : lifecycle.status === "terminated" ? {
|
|
1965
|
+
status: "proved",
|
|
1966
|
+
proof: {
|
|
1967
|
+
outcome: "lost",
|
|
1968
|
+
exitCode: null,
|
|
1969
|
+
reason: "provider_instance_terminated"
|
|
1970
|
+
}
|
|
1971
|
+
} : { status: "deferred", reason: "identity_mismatch" };
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
observation = {
|
|
1974
|
+
status: "deferred",
|
|
1975
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
|
|
1976
|
+
};
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
} else {
|
|
1980
|
+
try {
|
|
1981
|
+
const command = await commandPersistence.load();
|
|
1982
|
+
const supervised = command?.kind === "modal-router-v1" && Boolean(command.supervision);
|
|
1983
|
+
const supervisionMetric = (outcome) => observability.incrementCounter({
|
|
1984
|
+
name: "opengeni_command_supervision_total",
|
|
1985
|
+
help: "Native retained command supervision reconciliation outcomes.",
|
|
1986
|
+
labels: { outcome }
|
|
1987
|
+
});
|
|
1988
|
+
if (supervised && (lease.rotationReason === "provider_deadline" || claim.ownerState === "background_stopping")) {
|
|
1989
|
+
await commandPersistence.requestCancellation(
|
|
1990
|
+
lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
|
|
1991
|
+
);
|
|
1992
|
+
supervisionMetric("cancellation_intent");
|
|
1993
|
+
}
|
|
1994
|
+
const legacyDeadlineStop = lease.rotationReason === "provider_deadline" && !supervised && !process.deadlineCancellationRequestedAt;
|
|
1995
|
+
const probeOutcome = await probe(
|
|
1996
|
+
settings,
|
|
1997
|
+
lease,
|
|
1998
|
+
process,
|
|
1999
|
+
claim.ownerState === "background_stopping" && !process.deadlineCancellationRequestedAt || legacyDeadlineStop ? "cancel" : "observe",
|
|
2000
|
+
async (result, chunkId, stream, streamFidelity) => {
|
|
2001
|
+
if (typeof result !== "string" || !stream && isExecSessionLostBanner(result, process.providerSessionId))
|
|
2002
|
+
return;
|
|
2003
|
+
const events = await appendSessionCommandOutput(db, {
|
|
2004
|
+
accountId: process.accountId,
|
|
2005
|
+
workspaceId: process.workspaceId,
|
|
2006
|
+
sessionId: process.sessionId,
|
|
2007
|
+
commandId: process.id,
|
|
2008
|
+
chunkId,
|
|
2009
|
+
stream: stream ?? "stdout",
|
|
2010
|
+
streamFidelity: streamFidelity ?? (stream ? "separate" : "merged"),
|
|
2011
|
+
chunk: stream ? result : stripExecBanner(result)
|
|
2012
|
+
});
|
|
2013
|
+
if (events.length && bus)
|
|
2014
|
+
await bus.publish(process.workspaceId, process.sessionId, events).catch(() => void 0);
|
|
2015
|
+
},
|
|
2016
|
+
commandPersistence,
|
|
2017
|
+
drainBudget
|
|
2018
|
+
).then(
|
|
2019
|
+
(value) => ({ ok: true, value }),
|
|
2020
|
+
(error) => ({ ok: false, error })
|
|
2021
|
+
);
|
|
2022
|
+
if (legacyDeadlineStop && !await requestRetainedProcessDeadlineCancellation(db, processScope))
|
|
2023
|
+
throw new Error("Deadline cancellation no longer owns its rotating lease");
|
|
2024
|
+
if (!probeOutcome.ok) throw probeOutcome.error;
|
|
2025
|
+
observation = probeOutcome.value;
|
|
2026
|
+
wholeBoxGone = observation.status === "proved" && observation.proof.outcome === "lost" && observation.proof.reason === "provider_instance_not_found";
|
|
2027
|
+
if (supervised) {
|
|
2028
|
+
supervisionMetric(
|
|
2029
|
+
await commandPersistence.loadSupervisionReceipt() ? "proof_retained" : "proof_missing"
|
|
2030
|
+
);
|
|
2031
|
+
if (observation.status === "deferred")
|
|
2032
|
+
supervisionMetric(
|
|
2033
|
+
observation.reason === "provider_running" ? "checkpoint_blocked" : "provider_failure"
|
|
2034
|
+
);
|
|
2035
|
+
}
|
|
2036
|
+
} catch (error) {
|
|
2037
|
+
observability.warn("sandbox reaper: retained-process provider probe failed", {
|
|
2038
|
+
processId: process.id,
|
|
2039
|
+
providerBackend: process.providerBackend,
|
|
2040
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2041
|
+
});
|
|
2042
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
if (observation.status === "binding") {
|
|
2047
|
+
try {
|
|
2048
|
+
process = await bindRetainedProcessProviderIdentity(db, {
|
|
2049
|
+
accountId: process.accountId,
|
|
2050
|
+
workspaceId: process.workspaceId,
|
|
2051
|
+
sessionId: process.sessionId,
|
|
2052
|
+
processId: process.id,
|
|
2053
|
+
expected,
|
|
2054
|
+
claimId: claim.claimId,
|
|
2055
|
+
providerBindingKey: observation.providerBindingKey,
|
|
2056
|
+
providerBinding: observation.providerBinding
|
|
2057
|
+
});
|
|
2058
|
+
recordRetainedProcessReconciliation(observability, "provider_binding_adopted");
|
|
2059
|
+
observation = { status: "deferred", reason: "provider_running" };
|
|
2060
|
+
} catch (error) {
|
|
2061
|
+
observability.warn("sandbox reaper: retained-process binding adoption failed", {
|
|
2062
|
+
processId: process.id,
|
|
2063
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2064
|
+
});
|
|
2065
|
+
observation = { status: "deferred", reason: "provider_error" };
|
|
2066
|
+
}
|
|
2067
|
+
}
|
|
2068
|
+
if (observation.status === "deferred") {
|
|
2069
|
+
await deferRetainedProcessClaim(
|
|
2070
|
+
db,
|
|
2071
|
+
settings,
|
|
2072
|
+
observability,
|
|
2073
|
+
process,
|
|
2074
|
+
expected,
|
|
2075
|
+
claim.claimId,
|
|
2076
|
+
observation.reason,
|
|
2077
|
+
observation.outputBacklog === true
|
|
2078
|
+
);
|
|
2079
|
+
continue;
|
|
2080
|
+
}
|
|
2081
|
+
try {
|
|
2082
|
+
process = await recordRetainedProcessReconciliationProof(db, {
|
|
2083
|
+
accountId: process.accountId,
|
|
2084
|
+
workspaceId: process.workspaceId,
|
|
2085
|
+
sessionId: process.sessionId,
|
|
2086
|
+
processId: process.id,
|
|
2087
|
+
expected,
|
|
2088
|
+
claimId: claim.claimId,
|
|
2089
|
+
proof: observation.proof
|
|
2090
|
+
});
|
|
2091
|
+
proof = observation.proof;
|
|
2092
|
+
recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
|
|
2093
|
+
} catch (error) {
|
|
2094
|
+
recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
|
|
2095
|
+
warnRetainedProcessProofFailure(observability, error, process);
|
|
2096
|
+
continue;
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
if (wholeBoxGone) {
|
|
2100
|
+
const boxKey = `${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`;
|
|
2101
|
+
for (const other of claims) {
|
|
2102
|
+
const sibling = other.process;
|
|
2103
|
+
if (other === claim || `${sibling.leaseId}:${sibling.leaseEpoch}:${sibling.providerInstanceId}` !== boxKey || sibling.ownerActorKind !== "turn" || sibling.providerBackend !== "modal" || other.ownerState.startsWith("background_") || retainedProcessReconciliationProof(sibling))
|
|
2104
|
+
continue;
|
|
2105
|
+
try {
|
|
2106
|
+
await recoverManagedSessionBackgroundCommand(db, {
|
|
2107
|
+
accountId: sibling.accountId,
|
|
2108
|
+
workspaceId: sibling.workspaceId,
|
|
2109
|
+
sessionId: sibling.sessionId,
|
|
2110
|
+
processId: sibling.id,
|
|
2111
|
+
expected: retainedProcessSettlementIdentity(sibling),
|
|
2112
|
+
reconciliationClaimId: other.claimId
|
|
2113
|
+
});
|
|
2114
|
+
} catch (error) {
|
|
2115
|
+
observability.warn("sandbox reaper: retained-command background recovery failed", {
|
|
2116
|
+
processId: sibling.id,
|
|
2117
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2118
|
+
});
|
|
2119
|
+
}
|
|
2120
|
+
}
|
|
2121
|
+
try {
|
|
2122
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
2123
|
+
accountId: process.accountId,
|
|
2124
|
+
workspaceId: process.workspaceId,
|
|
2125
|
+
sandboxGroupId: process.sandboxGroupId,
|
|
2126
|
+
expectedEpoch: process.leaseEpoch,
|
|
2127
|
+
expectedInstanceId: process.providerInstanceId,
|
|
2128
|
+
expectedBackend: process.providerBackend
|
|
2129
|
+
});
|
|
2130
|
+
if (marked.status === "marked") {
|
|
2131
|
+
lostBoxes.add(`${process.leaseId}:${process.leaseEpoch}:${process.providerInstanceId}`);
|
|
2132
|
+
recordRetainedProcessReconciliation(observability, "provider_lost_whole_box");
|
|
2133
|
+
if (marked.backgroundCommandEvents?.length) {
|
|
2134
|
+
await publishDurableSessionEvents(
|
|
2135
|
+
bus,
|
|
2136
|
+
process.workspaceId,
|
|
2137
|
+
marked.backgroundCommandEvents,
|
|
2138
|
+
(error) => {
|
|
2139
|
+
observability.warn("sandbox reaper: lost-box command event fanout failed", {
|
|
2140
|
+
sandboxGroupId: process.sandboxGroupId,
|
|
2141
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2142
|
+
});
|
|
2143
|
+
}
|
|
2144
|
+
);
|
|
2145
|
+
}
|
|
2146
|
+
continue;
|
|
2147
|
+
}
|
|
2148
|
+
} catch (error) {
|
|
2149
|
+
observability.warn("sandbox reaper: whole-box loss settlement failed", {
|
|
2150
|
+
processId: process.id,
|
|
2151
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2152
|
+
});
|
|
2153
|
+
}
|
|
2154
|
+
}
|
|
2155
|
+
try {
|
|
2156
|
+
const settlement = await settleRetainedProcess(db, {
|
|
2157
|
+
accountId: process.accountId,
|
|
2158
|
+
workspaceId: process.workspaceId,
|
|
2159
|
+
sessionId: process.sessionId,
|
|
2160
|
+
processId: process.id,
|
|
2161
|
+
expected,
|
|
2162
|
+
reconciliationClaimId: claim.claimId,
|
|
2163
|
+
outcome: proof.outcome,
|
|
2164
|
+
exitCode: proof.exitCode,
|
|
2165
|
+
reason: proof.reason,
|
|
2166
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
2167
|
+
});
|
|
2168
|
+
if (settlement.process.state === "active") {
|
|
2169
|
+
throw new Error("Retained-process reconciliation returned an active durable process");
|
|
2170
|
+
}
|
|
2171
|
+
if (settlement.backgroundCommandEvents.length > 0) {
|
|
2172
|
+
try {
|
|
2173
|
+
await bus.publish(
|
|
2174
|
+
process.workspaceId,
|
|
2175
|
+
process.sessionId,
|
|
2176
|
+
settlement.backgroundCommandEvents
|
|
2177
|
+
);
|
|
2178
|
+
} catch (error) {
|
|
2179
|
+
observability.warn("sandbox reaper: retained-command event fanout failed", {
|
|
2180
|
+
processId: process.id,
|
|
2181
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2182
|
+
});
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
recordRetainedProcessReconciliation(observability, `settled_${proof.outcome}`);
|
|
2186
|
+
} catch (error) {
|
|
2187
|
+
recordRetainedProcessReconciliation(observability, "settlement_failed");
|
|
2188
|
+
observability.warn("sandbox reaper: retained-process settlement failed", {
|
|
2189
|
+
processId: process.id,
|
|
2190
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2191
|
+
});
|
|
2192
|
+
await deferRetainedProcessClaim(
|
|
2193
|
+
db,
|
|
2194
|
+
settings,
|
|
2195
|
+
observability,
|
|
2196
|
+
process,
|
|
2197
|
+
expected,
|
|
2198
|
+
claim.claimId,
|
|
2199
|
+
"settlement_failed"
|
|
2200
|
+
);
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
async function deferRetainedProcessClaim(db, settings, observability, process, expected, claimId, outcome, outputBacklog = false) {
|
|
2205
|
+
const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
|
|
2206
|
+
try {
|
|
2207
|
+
const lease = process.providerBackend === "modal" && process.routeTargetId === null ? await readLease2(db, process.workspaceId, process.sandboxGroupId) : null;
|
|
2208
|
+
await deferRetainedProcessReconciliation(db, {
|
|
2209
|
+
accountId: process.accountId,
|
|
2210
|
+
workspaceId: process.workspaceId,
|
|
2211
|
+
sessionId: process.sessionId,
|
|
2212
|
+
processId: process.id,
|
|
2213
|
+
expected,
|
|
2214
|
+
claimId,
|
|
2215
|
+
outcome: deferral.durableOutcome,
|
|
2216
|
+
retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs)
|
|
2217
|
+
});
|
|
2218
|
+
recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
|
|
2219
|
+
} catch (error) {
|
|
2220
|
+
recordRetainedProcessReconciliation(observability, "defer_failed");
|
|
2221
|
+
observability.warn("sandbox reaper: retained-process defer failed", {
|
|
2222
|
+
processId: process.id,
|
|
2223
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2224
|
+
});
|
|
2225
|
+
}
|
|
2226
|
+
}
|
|
2227
|
+
function retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog = false) {
|
|
2228
|
+
const bindingQuarantine = process.reconcileAttempts >= RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS && (outcome === "provider_binding_missing" || outcome === "provider_binding_mismatch" || outcome === "process_observation_unavailable");
|
|
2229
|
+
if (bindingQuarantine) {
|
|
2230
|
+
return {
|
|
2231
|
+
durableOutcome: `quarantined_${outcome}`,
|
|
2232
|
+
metricOutcome: outcome === "provider_binding_missing" ? "quarantined_binding_missing" : outcome === "provider_binding_mismatch" ? "quarantined_binding_mismatch" : "quarantined_process_observation_unavailable",
|
|
2233
|
+
retryAfterMs: RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
|
|
2234
|
+
quarantined: true
|
|
2235
|
+
};
|
|
2236
|
+
}
|
|
2237
|
+
const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 3e4);
|
|
2238
|
+
const exponent = outcome === "provider_running" && outputBacklog ? 0 : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
|
|
2239
|
+
const retryAfterMs = Math.min(5 * 6e4, baseRetryMs * 2 ** exponent);
|
|
2240
|
+
return {
|
|
2241
|
+
durableOutcome: outcome,
|
|
2242
|
+
metricOutcome: outcome,
|
|
2243
|
+
retryAfterMs,
|
|
2244
|
+
quarantined: false
|
|
2245
|
+
};
|
|
2246
|
+
}
|
|
2247
|
+
var RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
|
|
2248
|
+
async function drainRetainedCommandBacklog(session, providerSessionId, first, capturePage, budget) {
|
|
2249
|
+
let backlog = retainedBacklogRemains(session, first);
|
|
2250
|
+
await capturePage(first);
|
|
2251
|
+
let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
|
|
2252
|
+
if (typeof session.writeStdin !== "function") return observation;
|
|
2253
|
+
const until = Math.min(
|
|
2254
|
+
Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
|
|
2255
|
+
budget?.until ?? Number.POSITIVE_INFINITY
|
|
2256
|
+
);
|
|
2257
|
+
for (let reads = 0; backlog && observation.status === "deferred" && observation.reason === "provider_running" && reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS && Date.now() < until; reads++) {
|
|
2258
|
+
let next;
|
|
2259
|
+
try {
|
|
2260
|
+
next = await withRetainedProcessProbeTimeout(
|
|
2261
|
+
session.writeStdin({
|
|
2262
|
+
sessionId: providerSessionId,
|
|
2263
|
+
chars: "",
|
|
2264
|
+
yieldTimeMs: 1e3,
|
|
2265
|
+
maxOutputTokens: 2e3
|
|
2266
|
+
})
|
|
2267
|
+
);
|
|
2268
|
+
} catch {
|
|
2269
|
+
break;
|
|
2270
|
+
}
|
|
2271
|
+
backlog = retainedBacklogRemains(session, next);
|
|
2272
|
+
await capturePage(next);
|
|
2273
|
+
observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
|
|
2274
|
+
}
|
|
2275
|
+
return backlog && observation.status === "deferred" && observation.reason === "provider_running" ? { ...observation, outputBacklog: true } : observation;
|
|
2276
|
+
}
|
|
2277
|
+
function retainedBacklogRemains(session, result) {
|
|
2278
|
+
const page = session.getProviderCommandOutput?.(result);
|
|
2279
|
+
if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
|
|
2280
|
+
if (page.command.supervision || page.exitCode !== null) return false;
|
|
2281
|
+
if (page.providerExited) return true;
|
|
2282
|
+
const { command, expected } = page;
|
|
2283
|
+
return ["stdout", "stderr"].some(
|
|
2284
|
+
(stream) => command.streams[stream].byteOffset - expected.streams[stream].byteOffset >= RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES
|
|
2285
|
+
);
|
|
2286
|
+
}
|
|
2287
|
+
async function withRetainedProcessProbeTimeout(promise) {
|
|
2288
|
+
let timer;
|
|
2289
|
+
try {
|
|
2290
|
+
return await Promise.race([
|
|
2291
|
+
promise,
|
|
2292
|
+
new Promise((_resolve, reject) => {
|
|
2293
|
+
timer = setTimeout(
|
|
2294
|
+
() => reject(RETAINED_PROCESS_PROBE_TIMEOUT),
|
|
2295
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
2296
|
+
);
|
|
2297
|
+
timer.unref?.();
|
|
2298
|
+
})
|
|
2299
|
+
]);
|
|
2300
|
+
} finally {
|
|
2301
|
+
if (timer) clearTimeout(timer);
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
var pendingRetainedProbeOutput = /* @__PURE__ */ new Map();
|
|
2305
|
+
async function captureRetainedProbeOutput(processId, result, capture) {
|
|
2306
|
+
const pending = pendingRetainedProbeOutput.get(processId) ?? {
|
|
2307
|
+
result,
|
|
2308
|
+
chunkId: crypto.randomUUID()
|
|
2309
|
+
};
|
|
2310
|
+
pendingRetainedProbeOutput.set(processId, pending);
|
|
2311
|
+
await capture(pending.result, pending.chunkId);
|
|
2312
|
+
pendingRetainedProbeOutput.delete(processId);
|
|
2313
|
+
}
|
|
2314
|
+
async function probeRetainedProcessAtProvider(settings, lease, process, mode = "observe", captureOutput, providerPersistence, drainBudget) {
|
|
2315
|
+
if (lease.id !== process.leaseId || lease.sandboxGroupId !== process.sandboxGroupId || lease.leaseEpoch !== process.leaseEpoch || lease.backend !== process.providerBackend || lease.instanceId !== process.providerInstanceId || !lease.resumeBackendId || sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend || process.routeTargetId !== null) {
|
|
2316
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
2317
|
+
}
|
|
2318
|
+
if (!lease.resumeState) {
|
|
2319
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
2320
|
+
}
|
|
2321
|
+
const pending = pendingRetainedProbeOutput.get(process.id);
|
|
2322
|
+
if (pending) {
|
|
2323
|
+
if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
|
|
2324
|
+
await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
|
|
2325
|
+
const pendingObservation = classifyRetainedProcessPollResult(
|
|
2326
|
+
pending.result,
|
|
2327
|
+
process.providerSessionId
|
|
2328
|
+
);
|
|
2329
|
+
if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
|
|
2330
|
+
}
|
|
2331
|
+
const envelopeBackend = lease.resumeState.backendId;
|
|
2332
|
+
if (envelopeBackend !== void 0 && (typeof envelopeBackend !== "string" || sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)) {
|
|
2333
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
2334
|
+
}
|
|
2335
|
+
if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
|
|
2336
|
+
return { status: "deferred", reason: "identity_mismatch" };
|
|
2337
|
+
}
|
|
2338
|
+
let configuredModalBinding = null;
|
|
2339
|
+
if (process.providerBackend === "modal" && process.providerBindingKey) {
|
|
2340
|
+
try {
|
|
2341
|
+
configuredModalBinding = await withRetainedProcessProbeTimeout(
|
|
2342
|
+
resolveModalCheckpointProviderBinding(settings)
|
|
2343
|
+
);
|
|
2344
|
+
} catch (error) {
|
|
2345
|
+
return {
|
|
2346
|
+
status: "deferred",
|
|
2347
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
if (configuredModalBinding.key !== process.providerBindingKey) {
|
|
2351
|
+
return { status: "deferred", reason: "provider_binding_mismatch" };
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
let client;
|
|
2355
|
+
try {
|
|
2356
|
+
client = createSandboxClientForBackend2(process.providerBackend, settings);
|
|
2357
|
+
} catch {
|
|
2358
|
+
return { status: "deferred", reason: "provider_error" };
|
|
2359
|
+
}
|
|
2360
|
+
if (!client || sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend || !client.resume) {
|
|
2361
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
2362
|
+
}
|
|
2363
|
+
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
2364
|
+
let resumedState;
|
|
2365
|
+
try {
|
|
2366
|
+
resumedState = await deserializeSandboxSessionStateEnvelope2(
|
|
2367
|
+
client,
|
|
2368
|
+
envelopeSessionState,
|
|
2369
|
+
process.providerInstanceId
|
|
2370
|
+
);
|
|
2371
|
+
} catch {
|
|
2372
|
+
return { status: "deferred", reason: "provider_error" };
|
|
2373
|
+
}
|
|
2374
|
+
if (resumedState === void 0) {
|
|
2375
|
+
return { status: "deferred", reason: "resume_state_missing" };
|
|
2376
|
+
}
|
|
2377
|
+
let session;
|
|
2378
|
+
try {
|
|
2379
|
+
const resumed = await withRetainedProcessProbeTimeout(
|
|
2380
|
+
resumeExactSandboxSession2(
|
|
2381
|
+
client,
|
|
2382
|
+
process.providerBackend,
|
|
2383
|
+
resumedState,
|
|
2384
|
+
process.providerInstanceId
|
|
2385
|
+
)
|
|
2386
|
+
);
|
|
2387
|
+
session = resumed.session;
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
2390
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
2391
|
+
}
|
|
2392
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
2393
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
2394
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
2395
|
+
}
|
|
2396
|
+
return {
|
|
2397
|
+
status: "proved",
|
|
2398
|
+
proof: {
|
|
2399
|
+
outcome: "lost",
|
|
2400
|
+
exitCode: null,
|
|
2401
|
+
reason: "provider_instance_not_found"
|
|
2402
|
+
}
|
|
2403
|
+
};
|
|
2404
|
+
}
|
|
2405
|
+
return { status: "deferred", reason: "provider_error" };
|
|
2406
|
+
}
|
|
2407
|
+
if (typeof session.writeStdin !== "function") {
|
|
2408
|
+
return { status: "deferred", reason: "backend_unsupported" };
|
|
2409
|
+
}
|
|
2410
|
+
let legacyPty = false;
|
|
2411
|
+
if (process.providerBackend === "modal") {
|
|
2412
|
+
let liveBinding;
|
|
2413
|
+
try {
|
|
2414
|
+
liveBinding = await withRetainedProcessProbeTimeout(
|
|
2415
|
+
resolveModalCheckpointProviderBindingForSession(settings, session)
|
|
2416
|
+
);
|
|
2417
|
+
} catch (error) {
|
|
2418
|
+
return {
|
|
2419
|
+
status: "deferred",
|
|
2420
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
if (!process.providerBindingKey) {
|
|
2424
|
+
return {
|
|
2425
|
+
status: "binding",
|
|
2426
|
+
providerBindingKey: liveBinding.key,
|
|
2427
|
+
providerBinding: liveBinding.binding
|
|
2428
|
+
};
|
|
2429
|
+
}
|
|
2430
|
+
if (liveBinding.key !== process.providerBindingKey || configuredModalBinding?.key !== process.providerBindingKey) {
|
|
2431
|
+
return { status: "deferred", reason: "provider_binding_mismatch" };
|
|
2432
|
+
}
|
|
2433
|
+
const command = await providerPersistence?.load();
|
|
2434
|
+
if (!command || !providerPersistence || !session.bindProviderCommand)
|
|
2435
|
+
return { status: "deferred", reason: "process_observation_unavailable" };
|
|
2436
|
+
session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
|
|
2437
|
+
legacyPty = command.pty === true;
|
|
2438
|
+
}
|
|
2439
|
+
const capturePage = async (value) => {
|
|
2440
|
+
if (!captureOutput) {
|
|
2441
|
+
if (session.getProviderCommandOutput?.(value))
|
|
2442
|
+
throw new Error("Provider command output requires durable capture before settlement");
|
|
2443
|
+
return;
|
|
2444
|
+
}
|
|
2445
|
+
const page = session.getProviderCommandOutput?.(value);
|
|
2446
|
+
if (page?.command.kind === "modal-router-v1" && typeof value === "string") {
|
|
2447
|
+
if (!session.captureCommandOutput || !await session.captureCommandOutput(value))
|
|
2448
|
+
throw new Error("Byte-offset output requires atomic capture before settlement");
|
|
2449
|
+
return;
|
|
2450
|
+
}
|
|
2451
|
+
if (page) {
|
|
2452
|
+
for (const chunk of page.chunks)
|
|
2453
|
+
if (chunk.text)
|
|
2454
|
+
await captureOutput(chunk.text, chunk.chunkId, chunk.stream, page.streamFidelity);
|
|
2455
|
+
} else {
|
|
2456
|
+
await captureRetainedProbeOutput(process.id, value, captureOutput);
|
|
2457
|
+
}
|
|
2458
|
+
if (typeof value === "string") await session.acknowledgeCommandOutput?.(value);
|
|
2459
|
+
};
|
|
2460
|
+
let result;
|
|
2461
|
+
try {
|
|
2462
|
+
const supervisedCancelled = mode === "cancel" || lease.rotationReason === "provider_deadline" ? await session.cancelSupervisedCommand?.(
|
|
2463
|
+
process.providerSessionId,
|
|
2464
|
+
lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop"
|
|
2465
|
+
) ?? false : false;
|
|
2466
|
+
result = await withRetainedProcessProbeTimeout(
|
|
2467
|
+
session.writeStdin({
|
|
2468
|
+
sessionId: process.providerSessionId,
|
|
2469
|
+
// Ctrl-C is a signal only through a PTY. For a non-PTY command it is
|
|
2470
|
+
// ordinary stdin data and can corrupt the command's input.
|
|
2471
|
+
chars: mode === "cancel" && !supervisedCancelled && (process.providerBackend !== "modal" || legacyPty) ? "" : "",
|
|
2472
|
+
yieldTimeMs: 1e3,
|
|
2473
|
+
maxOutputTokens: 2e3
|
|
2474
|
+
})
|
|
2475
|
+
);
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
if (error === RETAINED_PROCESS_PROBE_TIMEOUT) {
|
|
2478
|
+
return { status: "deferred", reason: "provider_timeout" };
|
|
2479
|
+
}
|
|
2480
|
+
if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
|
|
2481
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
2482
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
2483
|
+
}
|
|
2484
|
+
return {
|
|
2485
|
+
status: "proved",
|
|
2486
|
+
proof: {
|
|
2487
|
+
outcome: "lost",
|
|
2488
|
+
exitCode: null,
|
|
2489
|
+
reason: "provider_instance_not_found"
|
|
2490
|
+
}
|
|
2491
|
+
};
|
|
2492
|
+
}
|
|
2493
|
+
return { status: "deferred", reason: "provider_error" };
|
|
2494
|
+
}
|
|
2495
|
+
const observation = await drainRetainedCommandBacklog(
|
|
2496
|
+
session,
|
|
2497
|
+
process.providerSessionId,
|
|
2498
|
+
result,
|
|
2499
|
+
capturePage,
|
|
2500
|
+
drainBudget
|
|
2501
|
+
);
|
|
2502
|
+
if (observation.status === "deferred" && observation.reason === "provider_running" && lease.rotationRequestedAt !== null && mode !== "cancel" && !process.cancellationRequestedAt && legacyPty && !await providerPersistence?.load().then((command) => command?.kind === "modal-router-v1" && command.supervision)) {
|
|
2503
|
+
try {
|
|
2504
|
+
const interrupted = await withRetainedProcessProbeTimeout(
|
|
2505
|
+
session.writeStdin({
|
|
2506
|
+
sessionId: process.providerSessionId,
|
|
2507
|
+
chars: "",
|
|
2508
|
+
yieldTimeMs: 1e3,
|
|
2509
|
+
maxOutputTokens: 2e3
|
|
2510
|
+
})
|
|
2511
|
+
);
|
|
2512
|
+
await capturePage(interrupted);
|
|
2513
|
+
return classifyRetainedProcessPollResult(interrupted, process.providerSessionId, session);
|
|
2514
|
+
} catch (error) {
|
|
2515
|
+
if (isProviderSandboxGoneDuringRoutedOperation(client.backendId, error)) {
|
|
2516
|
+
if (process.providerBackend === "modal" && !process.providerBindingKey) {
|
|
2517
|
+
return { status: "deferred", reason: "provider_binding_missing" };
|
|
2518
|
+
}
|
|
2519
|
+
return {
|
|
2520
|
+
status: "proved",
|
|
2521
|
+
proof: {
|
|
2522
|
+
outcome: "lost",
|
|
2523
|
+
exitCode: null,
|
|
2524
|
+
reason: "provider_instance_not_found"
|
|
2525
|
+
}
|
|
2526
|
+
};
|
|
2527
|
+
}
|
|
2528
|
+
return {
|
|
2529
|
+
status: "deferred",
|
|
2530
|
+
reason: error === RETAINED_PROCESS_PROBE_TIMEOUT ? "provider_timeout" : "provider_error"
|
|
2531
|
+
};
|
|
2532
|
+
}
|
|
2533
|
+
}
|
|
2534
|
+
return observation;
|
|
2535
|
+
}
|
|
2536
|
+
function classifyRetainedProcessPollResult(result, providerSessionId, source) {
|
|
2537
|
+
if (typeof result !== "string") {
|
|
2538
|
+
return { status: "deferred", reason: "provider_unknown" };
|
|
2539
|
+
}
|
|
2540
|
+
if (isExecSessionLostBanner(result, providerSessionId, source)) {
|
|
2541
|
+
return {
|
|
2542
|
+
status: "proved",
|
|
2543
|
+
proof: {
|
|
2544
|
+
outcome: "lost",
|
|
2545
|
+
exitCode: null,
|
|
2546
|
+
reason: "provider_session_lost_banner"
|
|
2547
|
+
}
|
|
2548
|
+
};
|
|
2549
|
+
}
|
|
2550
|
+
const exitCode = parseExecBannerExitCode(result);
|
|
2551
|
+
if (exitCode !== null) {
|
|
2552
|
+
return {
|
|
2553
|
+
status: "proved",
|
|
2554
|
+
proof: { outcome: "exited", exitCode, reason: "provider_exit_banner" }
|
|
2555
|
+
};
|
|
2556
|
+
}
|
|
2557
|
+
return {
|
|
2558
|
+
status: "deferred",
|
|
2559
|
+
reason: result.includes("Process running with session ID") ? "provider_running" : "provider_unknown"
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
async function accrueWarmTick(db, settings, observability) {
|
|
2563
|
+
const workspaceIds = /* @__PURE__ */ new Set();
|
|
2564
|
+
let accrued = 0;
|
|
2565
|
+
let leases = [];
|
|
2566
|
+
try {
|
|
2567
|
+
leases = await listMeterableWarmLeases(db);
|
|
2568
|
+
} catch (error) {
|
|
2569
|
+
observability.warn("sandbox reaper: warm-lease read failed", {
|
|
2570
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2571
|
+
});
|
|
2572
|
+
return { accrued, workspaceIds };
|
|
2573
|
+
}
|
|
2574
|
+
for (const lease of leases) {
|
|
2575
|
+
workspaceIds.add(lease.workspaceId);
|
|
2576
|
+
}
|
|
2577
|
+
await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
|
|
2578
|
+
try {
|
|
2579
|
+
const rate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
|
|
2580
|
+
const result = await accrueWarmSeconds(db, {
|
|
2581
|
+
accountId: lease.accountId,
|
|
2582
|
+
workspaceId: lease.workspaceId,
|
|
2583
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
2584
|
+
expectedEpoch: lease.leaseEpoch,
|
|
2585
|
+
warmRateMicrosPerSecond: rate,
|
|
2586
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
2587
|
+
subjectId: lease.sandboxGroupId
|
|
2588
|
+
});
|
|
2589
|
+
if (result.accrued) {
|
|
2590
|
+
accrued += 1;
|
|
2591
|
+
}
|
|
2592
|
+
if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
2593
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
2594
|
+
}
|
|
2595
|
+
} catch (error) {
|
|
2596
|
+
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
2597
|
+
workspaceId: lease.workspaceId,
|
|
2598
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
2599
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2600
|
+
});
|
|
2601
|
+
}
|
|
2602
|
+
});
|
|
2603
|
+
return { accrued, workspaceIds };
|
|
2604
|
+
}
|
|
2605
|
+
async function refreshIdleSandboxReaperProjections(input) {
|
|
2606
|
+
await refreshQueueLeaseAndCreditGauges(
|
|
2607
|
+
input.db,
|
|
2608
|
+
input.settings,
|
|
2609
|
+
input.observability,
|
|
2610
|
+
input.inspectOpenSandboxInventory ?? inspectOpenSandboxKubernetesInventory
|
|
2611
|
+
);
|
|
2612
|
+
}
|
|
2613
|
+
async function modalOrphanSweepMayHaveWork(input) {
|
|
2614
|
+
const { settings, observability } = input;
|
|
2615
|
+
if (!modalOrphanSweepConfigured(settings)) return false;
|
|
2616
|
+
if (await (input.hasRunningSandboxes ?? modalAppHasRunningSandboxes)(settings)) return true;
|
|
2617
|
+
recordModalSandboxInventoryGauges(observability, {
|
|
2618
|
+
running: 0,
|
|
2619
|
+
unleased: 0,
|
|
2620
|
+
liveLeaseInstancesMissing: 0
|
|
2621
|
+
});
|
|
2622
|
+
recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
|
|
2623
|
+
return false;
|
|
2624
|
+
}
|
|
2625
|
+
function modalOrphanSweepConfigured(settings) {
|
|
2626
|
+
return settings.sandboxBackend === "modal" || !!settings.modalTokenId || !!settings.modalTokenSecret;
|
|
2627
|
+
}
|
|
2628
|
+
async function refreshQueueLeaseAndCreditGauges(db, settings, observability, inspectOpenSandboxInventory) {
|
|
2629
|
+
await Promise.all([
|
|
2630
|
+
(async () => {
|
|
2631
|
+
try {
|
|
2632
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
2633
|
+
} catch (error) {
|
|
2634
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
2635
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2636
|
+
});
|
|
2637
|
+
}
|
|
2638
|
+
})(),
|
|
2639
|
+
refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
2640
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
2641
|
+
}),
|
|
2642
|
+
refreshSandboxInventoryGauge(
|
|
2643
|
+
observability,
|
|
2644
|
+
"checkpoint_artifacts",
|
|
2645
|
+
"checkpoint-artifact",
|
|
2646
|
+
async () => {
|
|
2647
|
+
recordSandboxCheckpointArtifactGauges(
|
|
2648
|
+
observability,
|
|
2649
|
+
await countSandboxCheckpointArtifactsByState(db)
|
|
2650
|
+
);
|
|
2651
|
+
}
|
|
2652
|
+
),
|
|
2653
|
+
refreshSandboxInventoryGauge(
|
|
2654
|
+
observability,
|
|
2655
|
+
"recovery_observations",
|
|
2656
|
+
"recovery-observations",
|
|
2657
|
+
async () => {
|
|
2658
|
+
recordSandboxRecoveryObservationGauges(
|
|
2659
|
+
observability,
|
|
2660
|
+
await readRecentSandboxRecoveryObservations(db)
|
|
2661
|
+
);
|
|
2662
|
+
}
|
|
2663
|
+
),
|
|
2664
|
+
refreshSandboxInventoryGauge(
|
|
2665
|
+
observability,
|
|
2666
|
+
"rotation_backlog",
|
|
2667
|
+
"rotation-backlog",
|
|
2668
|
+
async () => {
|
|
2669
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
2670
|
+
}
|
|
2671
|
+
),
|
|
2672
|
+
refreshSandboxInventoryGauge(
|
|
2673
|
+
observability,
|
|
2674
|
+
"checkpoint_staleness",
|
|
2675
|
+
"checkpoint-staleness",
|
|
2676
|
+
async () => {
|
|
2677
|
+
recordSandboxCheckpointStalenessGauges(
|
|
2678
|
+
observability,
|
|
2679
|
+
await readSandboxCheckpointStaleness(db)
|
|
2680
|
+
);
|
|
2681
|
+
}
|
|
2682
|
+
),
|
|
2683
|
+
refreshSandboxInventoryGauge(
|
|
2684
|
+
observability,
|
|
2685
|
+
"retained_processes",
|
|
2686
|
+
"retained-process",
|
|
2687
|
+
async () => {
|
|
2688
|
+
recordRetainedProcessInventoryGauges(
|
|
2689
|
+
observability,
|
|
2690
|
+
await countActiveRetainedProcessesByOwnerState(db)
|
|
2691
|
+
);
|
|
2692
|
+
}
|
|
2693
|
+
),
|
|
2694
|
+
refreshSandboxInventoryGauge(
|
|
2695
|
+
observability,
|
|
2696
|
+
"interaction_idle",
|
|
2697
|
+
"interaction-only-lease",
|
|
2698
|
+
async () => {
|
|
2699
|
+
recordInteractionOnlyLeaseGauges(
|
|
2700
|
+
observability,
|
|
2701
|
+
await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db))
|
|
2702
|
+
);
|
|
2703
|
+
}
|
|
2704
|
+
),
|
|
2705
|
+
refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
|
|
2706
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
2707
|
+
observability,
|
|
2708
|
+
await countExpiredDrainingSandboxLeases(db)
|
|
2709
|
+
);
|
|
2710
|
+
}),
|
|
2711
|
+
...settings.sandboxBackend === "opensandbox" && settings.openSandboxKubernetesInventoryNamespace ? [
|
|
2712
|
+
refreshSandboxInventoryGauge(
|
|
2713
|
+
observability,
|
|
2714
|
+
"opensandbox_kubernetes",
|
|
2715
|
+
"OpenSandbox Kubernetes",
|
|
2716
|
+
async () => {
|
|
2717
|
+
recordOpenSandboxKubernetesInventoryGauges(
|
|
2718
|
+
observability,
|
|
2719
|
+
await inspectOpenSandboxInventory({
|
|
2720
|
+
namespace: settings.openSandboxKubernetesInventoryNamespace
|
|
2721
|
+
})
|
|
2722
|
+
);
|
|
2723
|
+
}
|
|
2724
|
+
)
|
|
2725
|
+
] : [],
|
|
2726
|
+
(async () => {
|
|
2727
|
+
try {
|
|
2728
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
2729
|
+
} catch (error) {
|
|
2730
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
2731
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2732
|
+
});
|
|
2733
|
+
}
|
|
2734
|
+
})(),
|
|
2735
|
+
(async () => {
|
|
2736
|
+
try {
|
|
2737
|
+
recordActiveUserGauges(observability, await countActiveUsers(db));
|
|
2738
|
+
} catch (error) {
|
|
2739
|
+
observability.warn("sandbox reaper: active-user gauge refresh failed", {
|
|
2740
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2741
|
+
});
|
|
2742
|
+
}
|
|
2743
|
+
})(),
|
|
2744
|
+
(async () => {
|
|
2745
|
+
try {
|
|
2746
|
+
recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
|
|
2747
|
+
} catch (error) {
|
|
2748
|
+
observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
|
|
2749
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2750
|
+
});
|
|
2751
|
+
}
|
|
2752
|
+
})(),
|
|
2753
|
+
(async () => {
|
|
2754
|
+
recordVerifiedSignupTrialDeploymentFlagGauge(
|
|
2755
|
+
observability,
|
|
2756
|
+
settings.verifiedSignupTrialCreditsEnabled
|
|
2757
|
+
);
|
|
2758
|
+
try {
|
|
2759
|
+
const trialSwitch = await readVerifiedSignupTrialSwitch(db);
|
|
2760
|
+
recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
|
|
2761
|
+
} catch (error) {
|
|
2762
|
+
observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
|
|
2763
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2764
|
+
});
|
|
2765
|
+
}
|
|
2766
|
+
})(),
|
|
2767
|
+
(async () => {
|
|
2768
|
+
try {
|
|
2769
|
+
const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
|
|
2770
|
+
recordManagedAuthNewSignupsSwitchGauge(
|
|
2771
|
+
observability,
|
|
2772
|
+
signupsSwitch?.signupsEnabled !== false
|
|
2773
|
+
);
|
|
2774
|
+
} catch (error) {
|
|
2775
|
+
observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
|
|
2776
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2777
|
+
});
|
|
2778
|
+
}
|
|
2779
|
+
})()
|
|
2780
|
+
]);
|
|
2781
|
+
}
|
|
2782
|
+
async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
|
|
2783
|
+
try {
|
|
2784
|
+
await refresh();
|
|
2785
|
+
recordSandboxInventoryProjectionSuccess(observability, domain);
|
|
2786
|
+
} catch (error) {
|
|
2787
|
+
recordSandboxInventoryProjectionFailure(observability, domain);
|
|
2788
|
+
observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
|
|
2789
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2790
|
+
});
|
|
2791
|
+
}
|
|
2792
|
+
}
|
|
2793
|
+
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
2794
|
+
const enforceBalance = false;
|
|
2795
|
+
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
2796
|
+
let forceDrained = 0;
|
|
2797
|
+
await forEachWithConcurrency(
|
|
2798
|
+
[...workspaceIds],
|
|
2799
|
+
SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
|
|
2800
|
+
async (workspaceId) => {
|
|
2801
|
+
try {
|
|
2802
|
+
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
2803
|
+
workspaceId,
|
|
2804
|
+
enforceBalance,
|
|
2805
|
+
maxWarmSecondsPerWorkspace: cap,
|
|
2806
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
2807
|
+
});
|
|
2808
|
+
if (result.overLimit && result.drained.length > 0) {
|
|
2809
|
+
forceDrained += result.drained.length;
|
|
2810
|
+
observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
|
|
2811
|
+
workspaceId,
|
|
2812
|
+
reason: result.reason,
|
|
2813
|
+
drained: result.drained.length
|
|
2814
|
+
});
|
|
2815
|
+
}
|
|
2816
|
+
} catch (error) {
|
|
2817
|
+
observability.warn("sandbox reaper: force-drain check failed for workspace", {
|
|
2818
|
+
workspaceId,
|
|
2819
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2820
|
+
});
|
|
2821
|
+
}
|
|
2822
|
+
}
|
|
2823
|
+
);
|
|
2824
|
+
return forceDrained;
|
|
2825
|
+
}
|
|
2826
|
+
async function reconcileModalProviderCreates(db, settings, observability, discover = findModalProviderCreateReceipt) {
|
|
2827
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
|
|
2828
|
+
return;
|
|
2829
|
+
const candidates = await listPendingModalProviderCreates(db);
|
|
2830
|
+
await forEachWithConcurrency(candidates, 2, async (candidate) => {
|
|
2831
|
+
try {
|
|
2832
|
+
const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
|
|
2833
|
+
const identity = { ...candidate, accountId };
|
|
2834
|
+
const attempt = await claimModalProviderCreateRecovery(db, identity);
|
|
2835
|
+
if (!attempt) return;
|
|
2836
|
+
const instanceId = await discover(settings, attempt);
|
|
2837
|
+
if (!instanceId) return;
|
|
2838
|
+
if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
|
|
2839
|
+
observability.info("sandbox reaper: recovered Modal create receipt", {
|
|
2840
|
+
...candidate,
|
|
2841
|
+
operationId: attempt.operationId,
|
|
2842
|
+
instanceId
|
|
2843
|
+
});
|
|
2844
|
+
}
|
|
2845
|
+
} catch (error) {
|
|
2846
|
+
observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
|
|
2847
|
+
...candidate,
|
|
2848
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2849
|
+
});
|
|
2850
|
+
}
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
function modalOrphanTerminationStillEligible(latest, candidate) {
|
|
2854
|
+
if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
|
|
2855
|
+
return false;
|
|
2856
|
+
}
|
|
2857
|
+
return modalOrphanCandidateStillUnowned(latest, candidate);
|
|
2858
|
+
}
|
|
2859
|
+
function countWarmModalLeasesMissingInstance(latest, missingInstanceIds) {
|
|
2860
|
+
return missingInstanceIds.filter(
|
|
2861
|
+
(instanceId) => latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId)
|
|
2862
|
+
).length;
|
|
2863
|
+
}
|
|
2864
|
+
function modalOrphanCandidateStillUnowned(latest, candidate) {
|
|
2865
|
+
if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
|
|
2866
|
+
return false;
|
|
2867
|
+
}
|
|
2868
|
+
const leaseId = candidate.tags.opengeni_lease_id;
|
|
2869
|
+
const workspaceId = candidate.tags.opengeni_workspace_id;
|
|
2870
|
+
const sandboxGroupId = candidate.tags.opengeni_sandbox_group_id;
|
|
2871
|
+
if (!leaseId || !workspaceId || !sandboxGroupId) {
|
|
2872
|
+
return true;
|
|
2873
|
+
}
|
|
2874
|
+
const activeAttribution = latest.find(
|
|
2875
|
+
(lease) => lease.leaseId === leaseId && lease.workspaceId === workspaceId && lease.sandboxGroupId === sandboxGroupId
|
|
2876
|
+
);
|
|
2877
|
+
return Boolean(
|
|
2878
|
+
!activeAttribution || activeAttribution.instanceId !== null && activeAttribution.instanceId !== candidate.sandboxId
|
|
2879
|
+
);
|
|
2880
|
+
}
|
|
2881
|
+
async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
|
|
2882
|
+
if (!modalOrphanSweepConfigured(settings)) {
|
|
2883
|
+
return 0;
|
|
2884
|
+
}
|
|
2885
|
+
let liveLeases;
|
|
2886
|
+
try {
|
|
2887
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
2888
|
+
} catch (error) {
|
|
2889
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
2890
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2891
|
+
});
|
|
2892
|
+
return 0;
|
|
2893
|
+
}
|
|
2894
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases, {
|
|
2895
|
+
revalidateTermination: async (candidate) => {
|
|
2896
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
2897
|
+
return modalOrphanTerminationStillEligible(latest, candidate);
|
|
2898
|
+
}
|
|
2899
|
+
});
|
|
2900
|
+
if (result.inventory.complete) {
|
|
2901
|
+
try {
|
|
2902
|
+
const latest = await listLiveModalSandboxLeaseAttributions(db);
|
|
2903
|
+
recordModalSandboxInventoryGauges(observability, {
|
|
2904
|
+
running: result.inventory.running,
|
|
2905
|
+
unleased: result.inventory.unterminated.filter(
|
|
2906
|
+
(candidate) => modalOrphanCandidateStillUnowned(latest, candidate)
|
|
2907
|
+
).length,
|
|
2908
|
+
liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
|
|
2909
|
+
latest,
|
|
2910
|
+
result.inventory.missingLiveLeaseInstanceIds
|
|
2911
|
+
)
|
|
2912
|
+
});
|
|
2913
|
+
recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
|
|
2914
|
+
} catch (error) {
|
|
2915
|
+
recordSandboxInventoryProjectionFailure(observability, "modal_provider");
|
|
2916
|
+
observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
|
|
2917
|
+
error: error instanceof Error ? error.message : String(error)
|
|
2918
|
+
});
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
for (const terminated of result.terminated) {
|
|
2922
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
2923
|
+
sandboxId: terminated.sandboxId,
|
|
2924
|
+
reason: terminated.reason,
|
|
2925
|
+
tags: JSON.stringify(terminated.tags)
|
|
2926
|
+
});
|
|
2927
|
+
}
|
|
2928
|
+
if (result.examined > 0) {
|
|
2929
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
2930
|
+
examined: result.examined,
|
|
2931
|
+
terminated: result.terminated.length,
|
|
2932
|
+
skipped: result.skipped,
|
|
2933
|
+
appName: settings.modalAppName
|
|
2934
|
+
});
|
|
2935
|
+
}
|
|
2936
|
+
return result.terminated.length;
|
|
2937
|
+
}
|
|
2938
|
+
async function probeDrainableProviderReadiness(settings, lease) {
|
|
2939
|
+
if (!lease.instanceId || !lease.resumeState) {
|
|
2940
|
+
throw new Error("Expired workspace capture has no resumable provider identity");
|
|
2941
|
+
}
|
|
2942
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
2943
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
2944
|
+
let established;
|
|
2945
|
+
try {
|
|
2946
|
+
established = await establishSandboxSessionFromEnvelope(settings, lease.resumeState, {
|
|
2947
|
+
sessionId: `sandbox-capture-recovery:${lease.sandboxGroupId}`,
|
|
2948
|
+
recovery: "resume-only",
|
|
2949
|
+
backendOverride: backend
|
|
2950
|
+
});
|
|
2951
|
+
} catch (error) {
|
|
2952
|
+
if (isProviderSandboxNotFoundError(backend, error)) return "missing";
|
|
2953
|
+
throw error;
|
|
2954
|
+
}
|
|
2955
|
+
if (established.instanceId !== lease.instanceId) {
|
|
2956
|
+
throw new Error("Expired workspace capture resumed a different provider instance");
|
|
2957
|
+
}
|
|
2958
|
+
try {
|
|
2959
|
+
if (established.backendId === "modal") {
|
|
2960
|
+
await verifySandboxExecReadiness(established, RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS);
|
|
2961
|
+
return "ready";
|
|
2962
|
+
}
|
|
2963
|
+
const session = established.session;
|
|
2964
|
+
const run = session.exec ?? session.execCommand;
|
|
2965
|
+
if (!run) throw new Error("Expired workspace capture provider has no readiness command");
|
|
2966
|
+
let timer;
|
|
2967
|
+
const result = await Promise.race([
|
|
2968
|
+
run.call(session, {
|
|
2969
|
+
cmd: "true",
|
|
2970
|
+
yieldTimeMs: 1e3,
|
|
2971
|
+
maxOutputTokens: 1e3
|
|
2972
|
+
}),
|
|
2973
|
+
new Promise((_, reject) => {
|
|
2974
|
+
timer = setTimeout(
|
|
2975
|
+
() => reject(new Error("Expired workspace capture readiness probe timed out")),
|
|
2976
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS
|
|
2977
|
+
);
|
|
2978
|
+
})
|
|
2979
|
+
]).finally(() => {
|
|
2980
|
+
if (timer) clearTimeout(timer);
|
|
2981
|
+
});
|
|
2982
|
+
if (sandboxCommandStillRunning(result) || sandboxCommandExitCode(result) !== 0) {
|
|
2983
|
+
throw new Error("Expired workspace capture provider is not command-ready");
|
|
2984
|
+
}
|
|
2985
|
+
return "ready";
|
|
2986
|
+
} catch (error) {
|
|
2987
|
+
if (isProviderSandboxNotFoundError(backend, error)) return "missing";
|
|
2988
|
+
throw error;
|
|
2989
|
+
}
|
|
2990
|
+
}
|
|
2991
|
+
function sandboxDrainReason(lease) {
|
|
2992
|
+
if (lease.rotationRequestedAt === null) return "idle";
|
|
2993
|
+
return lease.rotationReason ?? "rotation_requested";
|
|
2994
|
+
}
|
|
2995
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
|
|
2996
|
+
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
2997
|
+
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
2998
|
+
if (!lease) {
|
|
2999
|
+
return false;
|
|
3000
|
+
}
|
|
3001
|
+
if (lease.liveness !== "draining" || lease.refcount !== 0 && !lease.unobservableCommandDrainIds?.length || lease.leaseEpoch !== row.leaseEpoch) {
|
|
3002
|
+
return false;
|
|
3003
|
+
}
|
|
3004
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
3005
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
3006
|
+
const managedProvider = backend !== "none" && backend !== "selfhosted";
|
|
3007
|
+
const capturePolicy = providerWorkspaceCapturePolicy(backend, lease.resumeState);
|
|
3008
|
+
if (managedProvider && !capturePolicy) {
|
|
3009
|
+
throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
|
|
3010
|
+
}
|
|
3011
|
+
if (capturePolicy?.liveInstance === "replaced") {
|
|
3012
|
+
throw new Error(
|
|
3013
|
+
`sandbox backend ${backend} workspace capture would replace the fenced live instance`
|
|
3014
|
+
);
|
|
3015
|
+
}
|
|
3016
|
+
const captureTimeoutMs = attempt.captureTimeoutMs;
|
|
3017
|
+
if (managedProvider && !lease.archiveCapture?.publishedAt && sandboxDrainCaptureSettlingInProcess(lease)) {
|
|
3018
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3019
|
+
`sandbox ${row.sandboxGroupId} epoch ${row.leaseEpoch} still has a workspace capture settling in this worker`
|
|
3020
|
+
);
|
|
3021
|
+
}
|
|
3022
|
+
let captureClaim = null;
|
|
3023
|
+
let providerMissingBeforeCapture = false;
|
|
3024
|
+
let captureDisposition = "capture_required";
|
|
3025
|
+
if (lease.instanceId && managedProvider) {
|
|
3026
|
+
if (lease.archiveCapture) {
|
|
3027
|
+
const priorCapture = lease.archiveCapture;
|
|
3028
|
+
const instanceId = lease.instanceId;
|
|
3029
|
+
if (priorCapture.publishedAt) {
|
|
3030
|
+
captureClaim = {
|
|
3031
|
+
...priorCapture,
|
|
3032
|
+
leaseId: lease.id,
|
|
3033
|
+
leaseEpoch: lease.leaseEpoch,
|
|
3034
|
+
instanceId,
|
|
3035
|
+
archiveGeneration: lease.archiveGeneration,
|
|
3036
|
+
archiveComplete: lease.archiveComplete
|
|
3037
|
+
};
|
|
3038
|
+
captureDisposition = "archive_published";
|
|
3039
|
+
} else {
|
|
3040
|
+
if (priorCapture.operationId === attempt.operationId && priorCapture.attempt >= attempt.attempt) {
|
|
3041
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3042
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is owned by attempt ${priorCapture.attempt}`
|
|
3043
|
+
);
|
|
3044
|
+
}
|
|
3045
|
+
const deadlineExpired = await workspaceArchiveCaptureDeadlineElapsed(db, {
|
|
3046
|
+
accountId,
|
|
3047
|
+
workspaceId: row.workspaceId,
|
|
3048
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3049
|
+
captureId: priorCapture.id,
|
|
3050
|
+
expectedEpoch: row.leaseEpoch,
|
|
3051
|
+
expectedInstanceId: instanceId
|
|
3052
|
+
});
|
|
3053
|
+
const unpublishedWarmingProvider = backend === "modal" && !sandboxProviderInstanceIdFromEnvelope(lease.resumeState, backend) && lease.recovery.provider.status === "creating" && lease.recovery.provider.instanceId === instanceId && lease.recovery.workspace.status === "not_ready";
|
|
3054
|
+
const replace = async () => await replaceWorkspaceArchiveCaptureAfterProof(db, {
|
|
3055
|
+
accountId,
|
|
3056
|
+
workspaceId: row.workspaceId,
|
|
3057
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3058
|
+
priorCaptureId: priorCapture.id,
|
|
3059
|
+
captureId: attempt.captureId,
|
|
3060
|
+
operationId: attempt.operationId,
|
|
3061
|
+
attempt: attempt.attempt,
|
|
3062
|
+
expectedEpoch: row.leaseEpoch,
|
|
3063
|
+
expectedInstanceId: instanceId,
|
|
3064
|
+
captureTimeoutMs
|
|
3065
|
+
});
|
|
3066
|
+
const currentTakeoverSafe = capturePolicy !== null && capturePolicy.takeover !== "exclusive";
|
|
3067
|
+
if (!lease.resumeState) {
|
|
3068
|
+
if (unpublishedWarmingProvider && priorCapture.takeoverSafe && currentTakeoverSafe) {
|
|
3069
|
+
const replacement = await replace();
|
|
3070
|
+
if (!replacement) {
|
|
3071
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3072
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during safe warming takeover`
|
|
3073
|
+
);
|
|
3074
|
+
}
|
|
3075
|
+
captureClaim = replacement;
|
|
3076
|
+
} else {
|
|
3077
|
+
if (!deadlineExpired) {
|
|
3078
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3079
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`
|
|
3080
|
+
);
|
|
3081
|
+
}
|
|
3082
|
+
if (!unpublishedWarmingProvider) {
|
|
3083
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3084
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} has no resumable provider state`
|
|
3085
|
+
);
|
|
3086
|
+
}
|
|
3087
|
+
const replacement = await replace();
|
|
3088
|
+
if (!replacement) {
|
|
3089
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3090
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during warming takeover`
|
|
3091
|
+
);
|
|
3092
|
+
}
|
|
3093
|
+
captureClaim = replacement;
|
|
3094
|
+
}
|
|
3095
|
+
} else if (priorCapture.takeoverSafe && currentTakeoverSafe) {
|
|
3096
|
+
const replacement = await replace();
|
|
3097
|
+
if (!replacement) {
|
|
3098
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3099
|
+
`sandbox ${row.sandboxGroupId} takeover-safe capture ${priorCapture.id} changed during takeover`
|
|
3100
|
+
);
|
|
3101
|
+
}
|
|
3102
|
+
captureClaim = replacement;
|
|
3103
|
+
} else {
|
|
3104
|
+
if (!deadlineExpired) {
|
|
3105
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3106
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} is still within its recovery window`
|
|
3107
|
+
);
|
|
3108
|
+
}
|
|
3109
|
+
const providerState = await probeDrainableProvider(settings, lease);
|
|
3110
|
+
if (providerState === "missing") {
|
|
3111
|
+
const replacement = await replace();
|
|
3112
|
+
if (!replacement) {
|
|
3113
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3114
|
+
`sandbox ${row.sandboxGroupId} capture ${priorCapture.id} changed during missing-provider takeover`
|
|
3115
|
+
);
|
|
3116
|
+
}
|
|
3117
|
+
providerMissingBeforeCapture = true;
|
|
3118
|
+
captureClaim = replacement;
|
|
3119
|
+
} else {
|
|
3120
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3121
|
+
`sandbox ${row.sandboxGroupId} exclusive capture ${priorCapture.id} remains ambiguous on a live provider`
|
|
3122
|
+
);
|
|
3123
|
+
}
|
|
3124
|
+
}
|
|
3125
|
+
}
|
|
3126
|
+
} else {
|
|
3127
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
3128
|
+
accountId,
|
|
3129
|
+
workspaceId: row.workspaceId,
|
|
3130
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3131
|
+
captureId: attempt.captureId,
|
|
3132
|
+
operationId: attempt.operationId,
|
|
3133
|
+
attempt: attempt.attempt,
|
|
3134
|
+
expectedEpoch: row.leaseEpoch,
|
|
3135
|
+
expectedInstanceId: lease.instanceId,
|
|
3136
|
+
liveness: "draining",
|
|
3137
|
+
captureTimeoutMs,
|
|
3138
|
+
minIntervalMs: 0,
|
|
3139
|
+
pointInTimeCapture: providerWorkspaceCaptureIsPointInTime(backend, lease.resumeState),
|
|
3140
|
+
deadlineMandatoryCaptureLeadMs: sandboxDeadlineMandatoryCaptureLeadMs(settings),
|
|
3141
|
+
providerReplaySafe: capturePolicy?.takeover === "same_request",
|
|
3142
|
+
takeoverSafe: capturePolicy !== null && capturePolicy.takeover !== "exclusive"
|
|
3143
|
+
});
|
|
3144
|
+
if (claimed.status === "claimed") {
|
|
3145
|
+
captureClaim = claimed.claim;
|
|
3146
|
+
} else if (claimed.status === "mutation_in_progress") {
|
|
3147
|
+
const providerState = await probeDrainableProvider(settings, lease);
|
|
3148
|
+
if (providerState !== "missing") return false;
|
|
3149
|
+
providerMissingBeforeCapture = true;
|
|
3150
|
+
} else if (claimed.status === "capture_in_progress") {
|
|
3151
|
+
throw new SandboxDrainRecoveryDeferredError(
|
|
3152
|
+
`sandbox ${row.sandboxGroupId} acquired a concurrent capture owner`
|
|
3153
|
+
);
|
|
3154
|
+
} else {
|
|
3155
|
+
return false;
|
|
3156
|
+
}
|
|
3157
|
+
}
|
|
3158
|
+
}
|
|
3159
|
+
let persisted = captureDisposition === "archive_published";
|
|
3160
|
+
let archiveRevision = captureDisposition === "archive_published" ? lease.recovery.archive.current?.revision ?? null : null;
|
|
3161
|
+
const persistArchive = async (archiveBase64, archiveMetadata, _providerSession, providerBinding) => {
|
|
3162
|
+
if (!lease.instanceId || !captureClaim) {
|
|
3163
|
+
return { wrote: false, archiveRevision: null };
|
|
3164
|
+
}
|
|
3165
|
+
const archiveMetrics = runtimeMetricsHooksForObservability(observability);
|
|
3166
|
+
let checkpointArtifactId = null;
|
|
3167
|
+
if (typeof archiveBase64 === "string" && archiveBase64 && archiveMetadata?.version === 2 && (archiveMetadata.provider === "modal_snapshot_filesystem" || archiveMetadata.provider === "modal_snapshot_directory")) {
|
|
3168
|
+
if (!providerBinding) {
|
|
3169
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
3170
|
+
}
|
|
3171
|
+
const candidate = await registerSandboxCheckpointArtifact(db, {
|
|
3172
|
+
accountId,
|
|
3173
|
+
workspaceId: row.workspaceId,
|
|
3174
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3175
|
+
sourceLeaseId: lease.id,
|
|
3176
|
+
sourceLeaseEpoch: row.leaseEpoch,
|
|
3177
|
+
sourceInstanceId: lease.instanceId,
|
|
3178
|
+
sourceWorkspaceGeneration: captureClaim.workspaceGeneration,
|
|
3179
|
+
providerBindingKey: providerBinding.key,
|
|
3180
|
+
providerBinding: providerBinding.binding,
|
|
3181
|
+
workspaceArchive: archiveBase64,
|
|
3182
|
+
workspaceArchiveMeta: archiveMetadata
|
|
3183
|
+
});
|
|
3184
|
+
checkpointArtifactId = candidate.id;
|
|
3185
|
+
}
|
|
3186
|
+
const baseInput = {
|
|
3187
|
+
accountId,
|
|
3188
|
+
workspaceId: row.workspaceId,
|
|
3189
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3190
|
+
expectedLeaseId: lease.id,
|
|
3191
|
+
expectedEpoch: row.leaseEpoch,
|
|
3192
|
+
expectedInstanceId: lease.instanceId,
|
|
3193
|
+
expectedWorkspaceGeneration: captureClaim.workspaceGeneration,
|
|
3194
|
+
captureId: captureClaim.id,
|
|
3195
|
+
providerRequestId: captureClaim.providerRequestId
|
|
3196
|
+
};
|
|
3197
|
+
let result;
|
|
3198
|
+
if (archiveBase64 === null) {
|
|
3199
|
+
result = await persistDrainSnapshot(db, {
|
|
3200
|
+
...baseInput,
|
|
3201
|
+
workspaceArchive: null
|
|
3202
|
+
});
|
|
3203
|
+
} else {
|
|
3204
|
+
if (!archiveMetadata) {
|
|
3205
|
+
throw new Error("Sandbox snapshot publication requires a verified archive descriptor");
|
|
3206
|
+
}
|
|
3207
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
3208
|
+
backend: lease.backend,
|
|
3209
|
+
objectStorage,
|
|
3210
|
+
accountId,
|
|
3211
|
+
workspaceId: row.workspaceId,
|
|
3212
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3213
|
+
archive: typeof archiveBase64 === "object" ? archiveBase64 : {
|
|
3214
|
+
bytes: Buffer.from(archiveBase64, "base64"),
|
|
3215
|
+
descriptor: archiveMetadata,
|
|
3216
|
+
base64: archiveBase64
|
|
3217
|
+
},
|
|
3218
|
+
metrics: archiveMetrics
|
|
3219
|
+
});
|
|
3220
|
+
result = await persistWorkspaceArchiveCandidate({
|
|
3221
|
+
objectStorage,
|
|
3222
|
+
...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
|
|
3223
|
+
persist: () => persistDrainSnapshot(db, {
|
|
3224
|
+
...baseInput,
|
|
3225
|
+
workspaceArchiveMeta: archiveMetadata,
|
|
3226
|
+
...published,
|
|
3227
|
+
...checkpointArtifactId ? { checkpointArtifactId } : {}
|
|
3228
|
+
})
|
|
3229
|
+
});
|
|
3230
|
+
}
|
|
3231
|
+
if (!result.wrote && checkpointArtifactId) {
|
|
3232
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
3233
|
+
accountId,
|
|
3234
|
+
workspaceId: row.workspaceId,
|
|
3235
|
+
artifactId: checkpointArtifactId,
|
|
3236
|
+
reason: "drain_snapshot_publication_fenced"
|
|
3237
|
+
}).catch(() => void 0);
|
|
3238
|
+
}
|
|
3239
|
+
if (result.wrote && archiveBase64 !== null) {
|
|
3240
|
+
persisted = true;
|
|
3241
|
+
archiveRevision = result.archiveRevision;
|
|
3242
|
+
}
|
|
3243
|
+
return result;
|
|
3244
|
+
};
|
|
3245
|
+
if (!providerMissingBeforeCapture && backend === "modal" && lease.instanceId && !await readWorkspaceArchiveCapturePreflight(db, {
|
|
3246
|
+
accountId,
|
|
3247
|
+
workspaceId: row.workspaceId,
|
|
3248
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3249
|
+
expectedEpoch: row.leaseEpoch,
|
|
3250
|
+
expectedInstanceId: lease.instanceId,
|
|
3251
|
+
liveness: "draining"
|
|
3252
|
+
})) {
|
|
3253
|
+
return false;
|
|
3254
|
+
}
|
|
3255
|
+
let stopCutoffMarked = false;
|
|
3256
|
+
const beforeProviderStop = async () => {
|
|
3257
|
+
const cutoff = await markWarmBillingStopCutoff(db, {
|
|
3258
|
+
accountId,
|
|
3259
|
+
workspaceId: row.workspaceId,
|
|
3260
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3261
|
+
expectedEpoch: row.leaseEpoch
|
|
3262
|
+
});
|
|
3263
|
+
if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
|
|
3264
|
+
stopCutoffMarked = true;
|
|
3265
|
+
};
|
|
3266
|
+
const termination = providerMissingBeforeCapture ? { terminated: true, providerMissingBeforeCapture: true } : await terminateBox(
|
|
3267
|
+
settings,
|
|
3268
|
+
lease,
|
|
3269
|
+
observability,
|
|
3270
|
+
persistArchive,
|
|
3271
|
+
captureClaim?.providerRequestId ?? attempt.operationId,
|
|
3272
|
+
captureDisposition,
|
|
3273
|
+
capturePolicy,
|
|
3274
|
+
Boolean(objectStorage),
|
|
3275
|
+
async () => {
|
|
3276
|
+
if (!captureClaim || !lease.instanceId) return;
|
|
3277
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
3278
|
+
accountId,
|
|
3279
|
+
workspaceId: row.workspaceId,
|
|
3280
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3281
|
+
captureId: captureClaim.id,
|
|
3282
|
+
expectedEpoch: row.leaseEpoch,
|
|
3283
|
+
expectedInstanceId: lease.instanceId
|
|
3284
|
+
});
|
|
3285
|
+
},
|
|
3286
|
+
row.workspaceId,
|
|
3287
|
+
beforeProviderStop,
|
|
3288
|
+
backend === "docker" ? async () => {
|
|
3289
|
+
if (!captureClaim || !lease.instanceId)
|
|
3290
|
+
throw new Error("Docker drain has no current claimed capture");
|
|
3291
|
+
return await verifyDockerDrainCaptureFence(db, {
|
|
3292
|
+
accountId,
|
|
3293
|
+
workspaceId: row.workspaceId,
|
|
3294
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3295
|
+
leaseId: lease.id,
|
|
3296
|
+
leaseEpoch: row.leaseEpoch,
|
|
3297
|
+
instanceId: lease.instanceId,
|
|
3298
|
+
workspaceGeneration: captureClaim.workspaceGeneration,
|
|
3299
|
+
captureId: captureClaim.id,
|
|
3300
|
+
providerRequestId: captureClaim.providerRequestId
|
|
3301
|
+
});
|
|
3302
|
+
} : void 0
|
|
3303
|
+
);
|
|
3304
|
+
const terminated = typeof termination === "boolean" ? termination : termination.terminated;
|
|
3305
|
+
if (!terminated) {
|
|
3306
|
+
return false;
|
|
3307
|
+
}
|
|
3308
|
+
const providerMissingDuringStop = providerMissingBeforeCapture || typeof termination !== "boolean" && termination.providerMissingBeforeCapture;
|
|
3309
|
+
if (!stopCutoffMarked && !providerMissingDuringStop) {
|
|
3310
|
+
if (managedProvider && lease.instanceId) {
|
|
3311
|
+
throw new Error("managed sandbox stopped without a warm usage cutoff");
|
|
3312
|
+
}
|
|
3313
|
+
await beforeProviderStop();
|
|
3314
|
+
}
|
|
3315
|
+
const finalRate = settings.sandboxWarmBillingMode === "usage_only" ? 0 : sandboxWarmRateMicrosPerSecond(settings, backend);
|
|
3316
|
+
if (!providerMissingDuringStop) {
|
|
3317
|
+
const finalTick = await accrueWarmSeconds(db, {
|
|
3318
|
+
accountId,
|
|
3319
|
+
workspaceId: row.workspaceId,
|
|
3320
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3321
|
+
expectedEpoch: row.leaseEpoch,
|
|
3322
|
+
warmRateMicrosPerSecond: finalRate,
|
|
3323
|
+
billingMode: settings.sandboxWarmBillingMode,
|
|
3324
|
+
finalDrain: true
|
|
3325
|
+
});
|
|
3326
|
+
if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
|
|
3327
|
+
recordCreditMicros(observability, "usage", finalTick.costMicros);
|
|
3328
|
+
}
|
|
3329
|
+
}
|
|
3330
|
+
const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
|
|
3331
|
+
const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
|
|
3332
|
+
db,
|
|
3333
|
+
{
|
|
3334
|
+
accountId,
|
|
3335
|
+
workspaceId: row.workspaceId,
|
|
3336
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3337
|
+
expectedEpoch: row.leaseEpoch,
|
|
3338
|
+
...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
|
|
3339
|
+
providerMissingBeforeCapture: providerMissing,
|
|
3340
|
+
idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
|
|
3341
|
+
// Mirrors terminateProviderBox: a selfhosted machine is never stopped.
|
|
3342
|
+
providerStopped: managedProvider && lease.backend !== "selfhosted" && lease.resumeBackendId !== "selfhosted"
|
|
3343
|
+
}
|
|
3344
|
+
);
|
|
3345
|
+
await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
|
|
3346
|
+
observability.warn("sandbox reaper: contained command event fanout failed", {
|
|
3347
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3348
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3349
|
+
});
|
|
3350
|
+
});
|
|
3351
|
+
if (wentCold) {
|
|
3352
|
+
if (providerMissing && !unpublishedProviderLost) {
|
|
3353
|
+
recordSandboxProviderMissingBeforeCapture(observability, backend);
|
|
3354
|
+
}
|
|
3355
|
+
if (lease.unobservableCommandDrainIds?.length) {
|
|
3356
|
+
recordSandboxCommandContainment(
|
|
3357
|
+
observability,
|
|
3358
|
+
providerMissing ? "provider_missing" : "contained"
|
|
3359
|
+
);
|
|
3360
|
+
}
|
|
3361
|
+
try {
|
|
3362
|
+
await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
|
|
3363
|
+
type: "sandbox.box.terminated",
|
|
3364
|
+
payload: {
|
|
3365
|
+
actor: "reaper",
|
|
3366
|
+
persisted,
|
|
3367
|
+
archiveRevision,
|
|
3368
|
+
instanceId: lease.instanceId,
|
|
3369
|
+
providerMissingBeforeCapture: providerMissing,
|
|
3370
|
+
// Why the box drained: the lease's bounded rotation reason when one
|
|
3371
|
+
// was requested (provider deadline, operator/image conflict, teardown
|
|
3372
|
+
// claim), otherwise the ordinary idle drain. Additive diagnostic
|
|
3373
|
+
// fact; no identifiers beyond the ones already carried above.
|
|
3374
|
+
drainReason: sandboxDrainReason(lease)
|
|
3375
|
+
}
|
|
3376
|
+
});
|
|
3377
|
+
} catch (eventError) {
|
|
3378
|
+
observability.warn(
|
|
3379
|
+
"sandbox reaper: box-terminated event write failed (drain outcome unaffected)",
|
|
3380
|
+
{
|
|
3381
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3382
|
+
leaseEpoch: row.leaseEpoch,
|
|
3383
|
+
backend: lease.backend,
|
|
3384
|
+
instanceId: lease.instanceId,
|
|
3385
|
+
error: eventError instanceof Error ? eventError.message : String(eventError)
|
|
3386
|
+
}
|
|
3387
|
+
);
|
|
3388
|
+
}
|
|
3389
|
+
}
|
|
3390
|
+
return wentCold;
|
|
3391
|
+
}
|
|
3392
|
+
function providerDrainLogIdentity(lease, backend) {
|
|
3393
|
+
return {
|
|
3394
|
+
leaseId: lease.id,
|
|
3395
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3396
|
+
leaseEpoch: lease.leaseEpoch,
|
|
3397
|
+
backend,
|
|
3398
|
+
instanceId: lease.instanceId
|
|
3399
|
+
};
|
|
3400
|
+
}
|
|
3401
|
+
async function verifyDockerDrainCaptureFence(db, input, readers = { readLease: readLease2, readWorkspaceArchiveCapturePreflight }) {
|
|
3402
|
+
const current = await readers.readLease(db, input.workspaceId, input.sandboxGroupId);
|
|
3403
|
+
if (!current || current.id !== input.leaseId || current.liveness !== "draining" || current.leaseEpoch !== input.leaseEpoch || current.instanceId !== input.instanceId || current.backend !== "docker" || current.workspaceGeneration !== input.workspaceGeneration || current.archiveCapture?.id !== input.captureId || current.archiveCapture.providerRequestId !== input.providerRequestId || current.archiveCapture.workspaceGeneration !== input.workspaceGeneration)
|
|
3404
|
+
throw new Error("Docker drain capture lease/epoch/claim changed");
|
|
3405
|
+
const preflight = await readers.readWorkspaceArchiveCapturePreflight(db, {
|
|
3406
|
+
accountId: input.accountId,
|
|
3407
|
+
workspaceId: input.workspaceId,
|
|
3408
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
3409
|
+
expectedEpoch: input.leaseEpoch,
|
|
3410
|
+
expectedInstanceId: input.instanceId,
|
|
3411
|
+
liveness: "draining"
|
|
3412
|
+
});
|
|
3413
|
+
if (!preflight || preflight.workspaceGeneration !== input.workspaceGeneration)
|
|
3414
|
+
throw new Error("Docker drain capture has an active writer or changed generation");
|
|
3415
|
+
return {
|
|
3416
|
+
archivePublished: current.archiveCapture.publishedAt instanceof Date && Number.isFinite(current.archiveCapture.publishedAt.getTime()) && current.archiveComplete === true
|
|
3417
|
+
};
|
|
3418
|
+
}
|
|
3419
|
+
function observeDockerWorkspaceRelease(observability, session, logIdentity) {
|
|
3420
|
+
const release = session.hostWorkspaceRelease;
|
|
3421
|
+
if (!release) return;
|
|
3422
|
+
recordSandboxDockerWorkspaceRelease(observability, release.status);
|
|
3423
|
+
if (release.status === "retained") {
|
|
3424
|
+
observability.warn(
|
|
3425
|
+
"sandbox reaper: Docker drain kept its host workspace directory after teardown",
|
|
3426
|
+
{ ...logIdentity, reason: release.reason }
|
|
3427
|
+
);
|
|
3428
|
+
}
|
|
3429
|
+
}
|
|
3430
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend2, terminateModalById = terminateModalSandboxById, providerCaptureRequestId, captureDisposition = "capture_required", claimedCapturePolicy, diskBackedArchives = false, releaseFailedCapture, workspaceId, beforeProviderStop, beforeWorkspaceCapture) {
|
|
3431
|
+
const durableBackendId = lease.resumeBackendId ?? lease.backend;
|
|
3432
|
+
const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
|
|
3433
|
+
const logIdentity = providerDrainLogIdentity(lease, backend);
|
|
3434
|
+
const warnCaptureFailure = (message, error) => warnDrainSnapshotFailure(observability, message, error, {
|
|
3435
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3436
|
+
leaseEpoch: lease.leaseEpoch,
|
|
3437
|
+
...workspaceId ? { workspaceId } : {}
|
|
3438
|
+
});
|
|
3439
|
+
const releaseAfterCaptureFailure = async () => {
|
|
3440
|
+
try {
|
|
3441
|
+
await releaseFailedCapture?.();
|
|
3442
|
+
} catch (error) {
|
|
3443
|
+
warnCaptureFailure("sandbox reaper: failed capture claim release failed", error);
|
|
3444
|
+
}
|
|
3445
|
+
};
|
|
3446
|
+
if (!backend || backend === "none") {
|
|
3447
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3448
|
+
}
|
|
3449
|
+
if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
|
|
3450
|
+
observability.info(
|
|
3451
|
+
"sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped \u2014 it is the user's machine)",
|
|
3452
|
+
{
|
|
3453
|
+
...logIdentity
|
|
3454
|
+
}
|
|
3455
|
+
);
|
|
3456
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3457
|
+
}
|
|
3458
|
+
const durableCapturePolicy = claimedCapturePolicy ?? providerWorkspaceCapturePolicy(backend, lease.resumeState);
|
|
3459
|
+
if (!durableCapturePolicy) {
|
|
3460
|
+
throw new Error(`sandbox backend ${backend} declares no workspace capture policy`);
|
|
3461
|
+
}
|
|
3462
|
+
if (durableCapturePolicy.liveInstance !== "preserved") {
|
|
3463
|
+
throw new Error(
|
|
3464
|
+
`sandbox backend ${backend} workspace capture would replace the fenced live instance`
|
|
3465
|
+
);
|
|
3466
|
+
}
|
|
3467
|
+
const persistedInstanceId = assertConsistentSandboxProviderIdentity(backend, lease.resumeState);
|
|
3468
|
+
if (!lease.instanceId) {
|
|
3469
|
+
if (lease.providerCreateAttempt?.instanceId === null) {
|
|
3470
|
+
throw new Error(
|
|
3471
|
+
"provider_create_outcome_unknown: cannot infer termination from missing instance ID"
|
|
3472
|
+
);
|
|
3473
|
+
}
|
|
3474
|
+
if (persistedInstanceId) {
|
|
3475
|
+
throw new Error(
|
|
3476
|
+
`sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`
|
|
3477
|
+
);
|
|
3478
|
+
}
|
|
3479
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3480
|
+
}
|
|
3481
|
+
if (persistedInstanceId && persistedInstanceId !== lease.instanceId) {
|
|
3482
|
+
throw new Error(
|
|
3483
|
+
`sandbox backend ${backend} lease instance ${lease.instanceId} does not match persisted provider identity ${persistedInstanceId}; refusing teardown`
|
|
3484
|
+
);
|
|
3485
|
+
}
|
|
3486
|
+
const isUnpublishedWarmingProvider = backend === "modal" && !persistedInstanceId && lease.recovery.provider.status === "creating" && lease.recovery.provider.instanceId === lease.instanceId && lease.recovery.workspace.status === "not_ready";
|
|
3487
|
+
if (isUnpublishedWarmingProvider) {
|
|
3488
|
+
const { wrote } = await persistArchive(null);
|
|
3489
|
+
if (!wrote) {
|
|
3490
|
+
observability.info(
|
|
3491
|
+
"sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING",
|
|
3492
|
+
{
|
|
3493
|
+
...logIdentity
|
|
3494
|
+
}
|
|
3495
|
+
);
|
|
3496
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
3497
|
+
}
|
|
3498
|
+
try {
|
|
3499
|
+
await beforeProviderStop?.();
|
|
3500
|
+
await terminateModalById(
|
|
3501
|
+
settings,
|
|
3502
|
+
lease.instanceId,
|
|
3503
|
+
void 0,
|
|
3504
|
+
lease.providerCreateAttempt?.providerBindingKey
|
|
3505
|
+
);
|
|
3506
|
+
} catch (error) {
|
|
3507
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
3508
|
+
throw error;
|
|
3509
|
+
}
|
|
3510
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
3511
|
+
...logIdentity
|
|
3512
|
+
});
|
|
3513
|
+
}
|
|
3514
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3515
|
+
}
|
|
3516
|
+
if (!persistedInstanceId || !lease.resumeState) {
|
|
3517
|
+
throw new Error(
|
|
3518
|
+
`sandbox backend ${backend} instance ${lease.instanceId} has no resumable provider envelope; refusing unverified teardown`
|
|
3519
|
+
);
|
|
3520
|
+
}
|
|
3521
|
+
const client = createClientForBackend(backend, settings);
|
|
3522
|
+
if (!client) {
|
|
3523
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3524
|
+
}
|
|
3525
|
+
let session;
|
|
3526
|
+
let sessionState;
|
|
3527
|
+
try {
|
|
3528
|
+
if (!client.deserializeSessionState || backend !== "docker" && !client.resume) {
|
|
3529
|
+
throw new Error(`sandbox backend ${backend} cannot resume a drainable provider box`);
|
|
3530
|
+
}
|
|
3531
|
+
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
3532
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope2(
|
|
3533
|
+
client,
|
|
3534
|
+
envelopeSessionState,
|
|
3535
|
+
lease.instanceId
|
|
3536
|
+
);
|
|
3537
|
+
if (resumedState === void 0) {
|
|
3538
|
+
throw new Error(`sandbox backend ${backend} returned no resumable provider state`);
|
|
3539
|
+
}
|
|
3540
|
+
if (backend === "docker") {
|
|
3541
|
+
if (!client.attachWorkspaceForDrain || !beforeWorkspaceCapture) {
|
|
3542
|
+
throw new Error(
|
|
3543
|
+
"Docker drain requires a creation-free attachment and current capture fence"
|
|
3544
|
+
);
|
|
3545
|
+
}
|
|
3546
|
+
await beforeWorkspaceCapture();
|
|
3547
|
+
session = await client.attachWorkspaceForDrain(resumedState, beforeWorkspaceCapture);
|
|
3548
|
+
sessionState = resumedState;
|
|
3549
|
+
} else {
|
|
3550
|
+
const resumed = await resumeExactSandboxSession2(
|
|
3551
|
+
client,
|
|
3552
|
+
backend,
|
|
3553
|
+
resumedState,
|
|
3554
|
+
lease.instanceId
|
|
3555
|
+
);
|
|
3556
|
+
session = resumed.session;
|
|
3557
|
+
sessionState = resumed.sessionState;
|
|
3558
|
+
}
|
|
3559
|
+
} catch (error) {
|
|
3560
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3561
|
+
observability.info("sandbox reaper: drainable box already gone before workspace capture", {
|
|
3562
|
+
...logIdentity
|
|
3563
|
+
});
|
|
3564
|
+
return {
|
|
3565
|
+
terminated: true,
|
|
3566
|
+
providerMissingBeforeCapture: captureDisposition !== "archive_published"
|
|
3567
|
+
};
|
|
3568
|
+
}
|
|
3569
|
+
throw error;
|
|
3570
|
+
}
|
|
3571
|
+
const liveCapturePolicy = providerWorkspaceCapturePolicy(backend, session);
|
|
3572
|
+
if (!liveCapturePolicy || liveCapturePolicy.takeover !== durableCapturePolicy.takeover || liveCapturePolicy.strategy !== durableCapturePolicy.strategy || liveCapturePolicy.liveInstance !== durableCapturePolicy.liveInstance) {
|
|
3573
|
+
throw new Error(
|
|
3574
|
+
`sandbox backend ${backend} live workspace capture policy differs from its durable lease contract`
|
|
3575
|
+
);
|
|
3576
|
+
}
|
|
3577
|
+
let verifiedArchive;
|
|
3578
|
+
const sessionWorkspacePersistence = session?.state?.workspacePersistence;
|
|
3579
|
+
const checkpointBinding = captureDisposition === "capture_required" && backend === "modal" && (sessionWorkspacePersistence === "snapshot_filesystem" || sessionWorkspacePersistence === "snapshot_directory") ? await withRetainedProcessProbeTimeout(
|
|
3580
|
+
resolveModalCheckpointProviderBindingForSession(settings, session)
|
|
3581
|
+
) : null;
|
|
3582
|
+
let releaseSettlingCapture = () => void 0;
|
|
3583
|
+
try {
|
|
3584
|
+
if (captureDisposition !== "archive_published" && session?.persistWorkspace) {
|
|
3585
|
+
releaseSettlingCapture = trackSettlingDrainCapture(lease);
|
|
3586
|
+
const capture = captureWorkspaceArchiveForStorage(
|
|
3587
|
+
session,
|
|
3588
|
+
Date.now(),
|
|
3589
|
+
{
|
|
3590
|
+
requestId: providerCaptureRequestId ?? randomUUID(),
|
|
3591
|
+
strategy: liveCapturePolicy.strategy
|
|
3592
|
+
},
|
|
3593
|
+
diskBackedArchives
|
|
3594
|
+
);
|
|
3595
|
+
verifiedArchive = await awaitProviderCaptureWithLatePublication({
|
|
3596
|
+
capture,
|
|
3597
|
+
timeoutMs: settings.sandboxSnapshotTimeoutMs,
|
|
3598
|
+
timeoutError: new SandboxProviderCaptureTimeoutError(
|
|
3599
|
+
lease.sandboxGroupId,
|
|
3600
|
+
backend,
|
|
3601
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
3602
|
+
lease.leaseEpoch,
|
|
3603
|
+
lease.instanceId
|
|
3604
|
+
),
|
|
3605
|
+
publishLate: async (archive) => {
|
|
3606
|
+
try {
|
|
3607
|
+
const result = await persistArchive(
|
|
3608
|
+
archive.kind === "host_spool" ? archive : archive.base64,
|
|
3609
|
+
archive.descriptor,
|
|
3610
|
+
session,
|
|
3611
|
+
checkpointBinding
|
|
3612
|
+
);
|
|
3613
|
+
observability.info(
|
|
3614
|
+
result.wrote ? "sandbox reaper: late workspace capture published under durable claim" : "sandbox reaper: late workspace capture fenced by successor",
|
|
3615
|
+
{
|
|
3616
|
+
...logIdentity
|
|
3617
|
+
}
|
|
3618
|
+
);
|
|
3619
|
+
} catch (error) {
|
|
3620
|
+
warnCaptureFailure("sandbox reaper: late workspace capture publication failed", error);
|
|
3621
|
+
} finally {
|
|
3622
|
+
await disposeWorkspaceArchive(archive);
|
|
3623
|
+
}
|
|
3624
|
+
},
|
|
3625
|
+
observeLatePublicationFailure: (error) => {
|
|
3626
|
+
warnCaptureFailure("sandbox reaper: late capture publication cleanup failed", error);
|
|
3627
|
+
},
|
|
3628
|
+
observeLateFailure: (error) => {
|
|
3629
|
+
void releaseAfterCaptureFailure();
|
|
3630
|
+
warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
|
|
3631
|
+
},
|
|
3632
|
+
onLateSettled: () => releaseSettlingCapture()
|
|
3633
|
+
});
|
|
3634
|
+
}
|
|
3635
|
+
} catch (error) {
|
|
3636
|
+
if (!(error instanceof SandboxProviderCaptureTimeoutError)) releaseSettlingCapture();
|
|
3637
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3638
|
+
observability.info("sandbox reaper: box gone during workspace capture", {
|
|
3639
|
+
...logIdentity
|
|
3640
|
+
});
|
|
3641
|
+
return { terminated: true, providerMissingBeforeCapture: true };
|
|
3642
|
+
}
|
|
3643
|
+
warnCaptureFailure(
|
|
3644
|
+
"sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)",
|
|
3645
|
+
error
|
|
3646
|
+
);
|
|
3647
|
+
if (!(error instanceof SandboxProviderCaptureTimeoutError)) {
|
|
3648
|
+
await releaseAfterCaptureFailure();
|
|
3649
|
+
}
|
|
3650
|
+
throw error;
|
|
3651
|
+
}
|
|
3652
|
+
if (captureDisposition === "archive_published") {
|
|
3653
|
+
observability.info("sandbox reaper: resuming teardown from durable archive publication", {
|
|
3654
|
+
...logIdentity
|
|
3655
|
+
});
|
|
3656
|
+
} else if (verifiedArchive) {
|
|
3657
|
+
let wrote;
|
|
3658
|
+
try {
|
|
3659
|
+
({ wrote } = await persistArchive(
|
|
3660
|
+
verifiedArchive.kind === "host_spool" ? verifiedArchive : verifiedArchive.base64,
|
|
3661
|
+
verifiedArchive.descriptor,
|
|
3662
|
+
session,
|
|
3663
|
+
checkpointBinding
|
|
3664
|
+
));
|
|
3665
|
+
} finally {
|
|
3666
|
+
try {
|
|
3667
|
+
await disposeWorkspaceArchive(verifiedArchive);
|
|
3668
|
+
} finally {
|
|
3669
|
+
releaseSettlingCapture();
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
if (!wrote) {
|
|
3673
|
+
observability.info(
|
|
3674
|
+
"sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)",
|
|
3675
|
+
{
|
|
3676
|
+
...logIdentity
|
|
3677
|
+
}
|
|
3678
|
+
);
|
|
3679
|
+
return { terminated: false, providerMissingBeforeCapture: false };
|
|
3680
|
+
}
|
|
3681
|
+
} else {
|
|
3682
|
+
throw new Error(
|
|
3683
|
+
`sandbox backend ${backend} produced no verified workspace archive during drain`
|
|
3684
|
+
);
|
|
3685
|
+
}
|
|
3686
|
+
try {
|
|
3687
|
+
if (!session || !sessionState) {
|
|
3688
|
+
throw new Error(`sandbox backend ${backend} returned no terminable session state`);
|
|
3689
|
+
}
|
|
3690
|
+
prepareProviderForTeardownAfterCapture(backend, session);
|
|
3691
|
+
await beforeProviderStop?.();
|
|
3692
|
+
await terminateManagedSandboxSession(client, sessionState, session);
|
|
3693
|
+
if (backend === "docker") observeDockerWorkspaceRelease(observability, session, logIdentity);
|
|
3694
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3695
|
+
} catch (error) {
|
|
3696
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3697
|
+
observability.info("sandbox reaper: provider already gone during terminate", {
|
|
3698
|
+
...logIdentity
|
|
3699
|
+
});
|
|
3700
|
+
return { terminated: true, providerMissingBeforeCapture: false };
|
|
3701
|
+
}
|
|
3702
|
+
throw error;
|
|
3703
|
+
}
|
|
3704
|
+
}
|
|
3705
|
+
|
|
3706
|
+
export {
|
|
3707
|
+
createBrowserDeadlineCheckpointActivities,
|
|
3708
|
+
RETAINED_PROCESS_RECONCILIATION_LIMIT,
|
|
3709
|
+
RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS,
|
|
3710
|
+
RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
|
|
3711
|
+
RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS,
|
|
3712
|
+
RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
|
|
3713
|
+
RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,
|
|
3714
|
+
RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,
|
|
3715
|
+
RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS,
|
|
3716
|
+
RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS,
|
|
3717
|
+
CONNECTED_COMMAND_RECONCILIATION_LIMIT,
|
|
3718
|
+
CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
|
|
3719
|
+
SandboxProviderCaptureTimeoutError,
|
|
3720
|
+
awaitProviderCaptureWithLatePublication,
|
|
3721
|
+
sandboxDrainCaptureId,
|
|
3722
|
+
createSandboxLeaseActivities,
|
|
3723
|
+
resumeIdleCheckpointSessionAtProvider,
|
|
3724
|
+
SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS,
|
|
3725
|
+
reconcileConnectedMachineBackgroundCommands,
|
|
3726
|
+
replayConnectedCommandOutput,
|
|
3727
|
+
reconcileConnectedCommandOutputReleases,
|
|
3728
|
+
probeConnectedMachineBackgroundCommand,
|
|
3729
|
+
connectedCommandProofFromStatus,
|
|
3730
|
+
retainedProcessReconciliationDeferral,
|
|
3731
|
+
drainRetainedCommandBacklog,
|
|
3732
|
+
captureRetainedProbeOutput,
|
|
3733
|
+
probeRetainedProcessAtProvider,
|
|
3734
|
+
classifyRetainedProcessPollResult,
|
|
3735
|
+
refreshIdleSandboxReaperProjections,
|
|
3736
|
+
modalOrphanSweepMayHaveWork,
|
|
3737
|
+
reconcileModalProviderCreates,
|
|
3738
|
+
modalOrphanTerminationStillEligible,
|
|
3739
|
+
countWarmModalLeasesMissingInstance,
|
|
3740
|
+
modalOrphanCandidateStillUnowned,
|
|
3741
|
+
sandboxDrainReason,
|
|
3742
|
+
verifyDockerDrainCaptureFence,
|
|
3743
|
+
terminateProviderBox,
|
|
3744
|
+
sandboxLeaseTelemetryKey2 as sandboxLeaseTelemetryKey
|
|
3745
|
+
};
|
|
3746
|
+
//# sourceMappingURL=chunk-MVESOYFC.js.map
|