@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -0,0 +1,2823 @@
|
|
|
1
|
+
// src/sandbox-snapshot-diagnostics.ts
|
|
2
|
+
import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
|
|
3
|
+
function safeSnapshotError(error, origin = "sandbox-resume") {
|
|
4
|
+
const fields = {
|
|
5
|
+
errorClass: "SnapshotOperationError",
|
|
6
|
+
errorCode: "snapshot_operation_failed",
|
|
7
|
+
origin
|
|
8
|
+
};
|
|
9
|
+
try {
|
|
10
|
+
if (error && typeof error === "object") {
|
|
11
|
+
const candidate = error;
|
|
12
|
+
if (typeof candidate.name === "string" && /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)) {
|
|
13
|
+
fields.causeName = candidate.name;
|
|
14
|
+
}
|
|
15
|
+
if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
|
|
16
|
+
fields.integrityCode = candidate.code;
|
|
17
|
+
}
|
|
18
|
+
const status = Number(candidate.status ?? candidate.statusCode);
|
|
19
|
+
if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
|
|
20
|
+
}
|
|
21
|
+
} catch {
|
|
22
|
+
}
|
|
23
|
+
const read = (value, key) => {
|
|
24
|
+
try {
|
|
25
|
+
return value && typeof value === "object" ? Reflect.get(value, key) : void 0;
|
|
26
|
+
} catch {
|
|
27
|
+
return void 0;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
const details = read(error, "details");
|
|
31
|
+
const providerName = read(details, "errorName");
|
|
32
|
+
if (typeof providerName === "string" && [
|
|
33
|
+
"ClientError",
|
|
34
|
+
"TimeoutError",
|
|
35
|
+
"ConnectionError",
|
|
36
|
+
"AuthError",
|
|
37
|
+
"NotFoundError",
|
|
38
|
+
"InvalidError",
|
|
39
|
+
"RemoteError",
|
|
40
|
+
"AbortError"
|
|
41
|
+
].includes(providerName))
|
|
42
|
+
fields.providerErrorName = providerName;
|
|
43
|
+
const grpcCode = read(details, "errorCode");
|
|
44
|
+
if (providerName === "ClientError" && typeof grpcCode === "number" && Number.isInteger(grpcCode) && grpcCode >= 0 && grpcCode <= 16)
|
|
45
|
+
fields.providerGrpcCode = grpcCode;
|
|
46
|
+
for (const key of ["status", "httpStatus", "responseStatus"]) {
|
|
47
|
+
const status = read(details, key);
|
|
48
|
+
if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) {
|
|
49
|
+
fields.providerHttpStatus = status;
|
|
50
|
+
break;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
const retryable = read(error, "retryable");
|
|
54
|
+
if (typeof retryable === "boolean") fields.providerRetryable = retryable;
|
|
55
|
+
return fields;
|
|
56
|
+
}
|
|
57
|
+
function warnDrainSnapshotFailure(observability, message, error, lease) {
|
|
58
|
+
try {
|
|
59
|
+
observability.warn(message, {
|
|
60
|
+
...safeSnapshotError(error, "worker-lifecycle"),
|
|
61
|
+
...lease.workspaceId ? { sandboxLeaseKey: sandboxLeaseTelemetryKey(lease.workspaceId, lease.sandboxGroupId) } : {},
|
|
62
|
+
leaseEpoch: lease.leaseEpoch
|
|
63
|
+
});
|
|
64
|
+
} catch {
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// src/session-event-fanout.ts
|
|
69
|
+
async function publishDurableSessionEvents(bus, workspaceId, events, onError = () => void 0) {
|
|
70
|
+
if (!bus || !events?.length) return;
|
|
71
|
+
for (const sessionId of new Set(events.map((event) => event.sessionId))) {
|
|
72
|
+
await bus.publish(
|
|
73
|
+
workspaceId,
|
|
74
|
+
sessionId,
|
|
75
|
+
events.filter((event) => event.sessionId === sessionId)
|
|
76
|
+
).catch(onError);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// src/sandbox-archive-storage.ts
|
|
81
|
+
import { uploadWorkspaceArchiveSpool } from "@opengeni/storage";
|
|
82
|
+
import { randomUUID } from "crypto";
|
|
83
|
+
import {
|
|
84
|
+
parseWorkspaceArchiveObjectRef,
|
|
85
|
+
workspaceArchiveObjectKey
|
|
86
|
+
} from "@opengeni/contracts";
|
|
87
|
+
var WorkspaceArchiveObjectStorageRequiredError = class extends Error {
|
|
88
|
+
code = "workspace_archive_object_storage_required";
|
|
89
|
+
constructor(backend) {
|
|
90
|
+
super(`${backend} workspace archives require object storage`);
|
|
91
|
+
this.name = "WorkspaceArchiveObjectStorageRequiredError";
|
|
92
|
+
}
|
|
93
|
+
};
|
|
94
|
+
async function putTarWorkspaceArchiveObject(input) {
|
|
95
|
+
const descriptor = structuredClone(input.archive.descriptor);
|
|
96
|
+
if (descriptor.version !== 1) {
|
|
97
|
+
throw new Error("Object-storage workspace archives are portable tar only");
|
|
98
|
+
}
|
|
99
|
+
const key = workspaceArchiveObjectKey({
|
|
100
|
+
accountId: input.accountId,
|
|
101
|
+
workspaceId: input.workspaceId,
|
|
102
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
103
|
+
revision: descriptor.revision,
|
|
104
|
+
uploadId: randomUUID()
|
|
105
|
+
});
|
|
106
|
+
const archive = input.archive;
|
|
107
|
+
if ("spool" in archive) {
|
|
108
|
+
if (archive.spool.byteSize !== descriptor.archiveBytes || archive.spool.sha256 !== descriptor.archiveSha256) {
|
|
109
|
+
throw new Error("Workspace archive spool does not match its verified descriptor");
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
const source = "spool" in archive ? archive.spool : {
|
|
113
|
+
path: "",
|
|
114
|
+
byteSize: archive.bytes.byteLength,
|
|
115
|
+
sha256: descriptor.archiveSha256,
|
|
116
|
+
async *open() {
|
|
117
|
+
for (let offset = 0; offset < archive.bytes.length; offset += 1024 * 1024)
|
|
118
|
+
yield archive.bytes.subarray(offset, offset + 1024 * 1024);
|
|
119
|
+
},
|
|
120
|
+
async dispose() {
|
|
121
|
+
}
|
|
122
|
+
};
|
|
123
|
+
try {
|
|
124
|
+
await uploadWorkspaceArchiveSpool(input.objectStorage, key, {
|
|
125
|
+
...source,
|
|
126
|
+
byteSize: descriptor.archiveBytes,
|
|
127
|
+
sha256: descriptor.archiveSha256,
|
|
128
|
+
open: source.open.bind(source)
|
|
129
|
+
});
|
|
130
|
+
} catch (error) {
|
|
131
|
+
await input.objectStorage.deleteObject(key).catch(() => void 0);
|
|
132
|
+
throw error;
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
schema: "sandbox_archive_object_v1",
|
|
136
|
+
key,
|
|
137
|
+
sha256: descriptor.archiveSha256,
|
|
138
|
+
bytes: descriptor.archiveBytes,
|
|
139
|
+
backend: input.objectStorage.backend
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
async function persistWorkspaceArchiveCandidate(input) {
|
|
143
|
+
const objectStorage = input.objectStorage;
|
|
144
|
+
const ref = input.ref ? { ...input.ref } : void 0;
|
|
145
|
+
const result = await input.persist();
|
|
146
|
+
if (objectStorage && ref && result.candidateDisposition === "unused") {
|
|
147
|
+
await deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref);
|
|
148
|
+
}
|
|
149
|
+
return result;
|
|
150
|
+
}
|
|
151
|
+
async function putVersion1TarArchiveOrInline(input) {
|
|
152
|
+
if (input.archive.descriptor.version !== 1) {
|
|
153
|
+
if ("spool" in input.archive) throw new Error("Native snapshots cannot use a portable spool");
|
|
154
|
+
return { workspaceArchive: input.archive.base64 };
|
|
155
|
+
}
|
|
156
|
+
if (input.backend === "opensandbox" && !input.objectStorage) {
|
|
157
|
+
throw new WorkspaceArchiveObjectStorageRequiredError("opensandbox");
|
|
158
|
+
}
|
|
159
|
+
if (!input.objectStorage) {
|
|
160
|
+
if ("spool" in input.archive)
|
|
161
|
+
throw new WorkspaceArchiveObjectStorageRequiredError(input.backend);
|
|
162
|
+
return { workspaceArchive: input.archive.base64 };
|
|
163
|
+
}
|
|
164
|
+
try {
|
|
165
|
+
const workspaceArchiveRef = await putTarWorkspaceArchiveObject({
|
|
166
|
+
objectStorage: input.objectStorage,
|
|
167
|
+
accountId: input.accountId,
|
|
168
|
+
workspaceId: input.workspaceId,
|
|
169
|
+
sandboxGroupId: input.sandboxGroupId,
|
|
170
|
+
archive: input.archive
|
|
171
|
+
});
|
|
172
|
+
try {
|
|
173
|
+
input.metrics?.onWorkspaceArchiveObject?.({
|
|
174
|
+
outcome: "put",
|
|
175
|
+
backend: input.objectStorage.backend
|
|
176
|
+
});
|
|
177
|
+
} catch {
|
|
178
|
+
}
|
|
179
|
+
console.info("workspace archive object put", {
|
|
180
|
+
key: workspaceArchiveRef.key,
|
|
181
|
+
bytes: workspaceArchiveRef.bytes,
|
|
182
|
+
sha256: workspaceArchiveRef.sha256
|
|
183
|
+
});
|
|
184
|
+
return { workspaceArchiveRef };
|
|
185
|
+
} catch (error) {
|
|
186
|
+
try {
|
|
187
|
+
input.metrics?.onWorkspaceArchiveObject?.({
|
|
188
|
+
outcome: "put_failed",
|
|
189
|
+
backend: input.objectStorage.backend
|
|
190
|
+
});
|
|
191
|
+
} catch {
|
|
192
|
+
}
|
|
193
|
+
throw error;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metrics) {
|
|
197
|
+
if (!objectStorage || !ref) return;
|
|
198
|
+
try {
|
|
199
|
+
await objectStorage.deleteObject(ref.key);
|
|
200
|
+
try {
|
|
201
|
+
metrics?.onWorkspaceArchiveObject?.({
|
|
202
|
+
outcome: "deleted_unpublished",
|
|
203
|
+
backend: objectStorage.backend
|
|
204
|
+
});
|
|
205
|
+
} catch {
|
|
206
|
+
}
|
|
207
|
+
} catch {
|
|
208
|
+
console.error("unpublished workspace archive object delete failed", { key: ref.key });
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// src/sandbox-resume.ts
|
|
213
|
+
import {
|
|
214
|
+
effectiveSandboxLifecycle,
|
|
215
|
+
sandboxArchiveCaptureTimeoutMs,
|
|
216
|
+
sandboxLifecycleTransitionWaitMs,
|
|
217
|
+
sandboxWarmRateMicrosPerSecond
|
|
218
|
+
} from "@opengeni/config";
|
|
219
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
220
|
+
|
|
221
|
+
// src/warm-capture-holder.ts
|
|
222
|
+
function retainsHolderForWarmCapture(lease, expectedEpoch, now = Date.now()) {
|
|
223
|
+
return Boolean(
|
|
224
|
+
lease?.liveness === "warm" && lease.leaseEpoch === expectedEpoch && lease.archiveCapture && lease.archiveCapture.deadlineAt.getTime() > now
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/sandbox-resume.ts
|
|
229
|
+
import {
|
|
230
|
+
acquireLease,
|
|
231
|
+
adoptLegacyModalCheckpointArtifact,
|
|
232
|
+
beginSandboxRematerialization,
|
|
233
|
+
sessionHoldsFreshWorkspaceRecovery,
|
|
234
|
+
claimWorkspaceArchiveCapture,
|
|
235
|
+
commitWarmingToWarm,
|
|
236
|
+
failSandboxRematerialization,
|
|
237
|
+
failWarmingToCold,
|
|
238
|
+
beginModalProviderCreate,
|
|
239
|
+
getSandboxSessionEnvelope,
|
|
240
|
+
heartbeatLeaseHolderStatus,
|
|
241
|
+
markSandboxRestoreVerifying,
|
|
242
|
+
markWarmLeaseInstanceLost,
|
|
243
|
+
markSandboxCheckpointArtifactDeletePending,
|
|
244
|
+
persistWarmSnapshot,
|
|
245
|
+
readLease,
|
|
246
|
+
registerSandboxCheckpointArtifact,
|
|
247
|
+
recordWarmingSandboxCreated,
|
|
248
|
+
reinstateTurnLeaseHolder,
|
|
249
|
+
leaseMayHaveUntrackedWriters,
|
|
250
|
+
releaseWorkspaceArchiveCapture,
|
|
251
|
+
releaseLeaseHolder,
|
|
252
|
+
touchLeaseHolder,
|
|
253
|
+
SandboxLeaseRecoveryBlockedError,
|
|
254
|
+
SandboxLeaseSupersededError,
|
|
255
|
+
SandboxLeaseTransitionError
|
|
256
|
+
} from "@opengeni/db";
|
|
257
|
+
import {
|
|
258
|
+
captureWorkspaceArchiveForStorage,
|
|
259
|
+
disposeWorkspaceArchive,
|
|
260
|
+
describeLegacyNativeSnapshotArchive,
|
|
261
|
+
inlineWorkspaceArchiveForRestore,
|
|
262
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
263
|
+
SandboxExecReadinessError,
|
|
264
|
+
WorkspaceArchiveIntegrityError,
|
|
265
|
+
establishSandboxSessionFromEnvelope,
|
|
266
|
+
isProviderSandboxNotFoundError,
|
|
267
|
+
parseWorkspaceArchiveDescriptor,
|
|
268
|
+
providerWorkspaceCapturePolicy,
|
|
269
|
+
SandboxProviderContinuityUnavailableError,
|
|
270
|
+
requirePersistableReplacementSandboxEnvelope,
|
|
271
|
+
renewSandboxProviderExpiration,
|
|
272
|
+
modalSessionMatchesCheckpointProviderBinding,
|
|
273
|
+
resolveModalCheckpointProviderBindingForSession,
|
|
274
|
+
serializeReplacementSandboxEnvelope,
|
|
275
|
+
tagModalSandbox,
|
|
276
|
+
terminateUnpublishedSandboxSession,
|
|
277
|
+
verifySandboxExecReadiness,
|
|
278
|
+
withoutSandboxProviderIdentity,
|
|
279
|
+
workspaceArchiveDownloadTemporaryDirectory
|
|
280
|
+
} from "@opengeni/runtime";
|
|
281
|
+
import {
|
|
282
|
+
downloadWorkspaceArchiveSpool,
|
|
283
|
+
WorkspaceArchiveStorageError
|
|
284
|
+
} from "@opengeni/storage";
|
|
285
|
+
import { sandboxLeaseTelemetryKey as sandboxLeaseTelemetryKey2 } from "@opengeni/observability";
|
|
286
|
+
import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
|
|
287
|
+
function sandboxLeaseHolderIdForAttempt(attemptId) {
|
|
288
|
+
const normalized = attemptId.trim();
|
|
289
|
+
if (!normalized) {
|
|
290
|
+
throw new Error("Sandbox lease holder requires a turn attempt id");
|
|
291
|
+
}
|
|
292
|
+
return `turn-attempt:${normalized}`;
|
|
293
|
+
}
|
|
294
|
+
function isRetryableDegradedRestore(restore) {
|
|
295
|
+
return restore.status === "degraded" && restore.retryable === true;
|
|
296
|
+
}
|
|
297
|
+
var SandboxProvisionStageError = class extends Error {
|
|
298
|
+
constructor(stage, source) {
|
|
299
|
+
super(source instanceof Error ? source.message : String(source), { cause: source });
|
|
300
|
+
this.stage = stage;
|
|
301
|
+
this.source = source;
|
|
302
|
+
}
|
|
303
|
+
name = "SandboxProvisionStageError";
|
|
304
|
+
};
|
|
305
|
+
var sandboxProvisionFailureStages = /* @__PURE__ */ new WeakMap();
|
|
306
|
+
function sandboxProvisionFailureStage(error) {
|
|
307
|
+
if (!error || typeof error !== "object") return null;
|
|
308
|
+
return sandboxProvisionFailureStages.get(error) ?? (error instanceof SandboxProvisionStageError ? error.stage : null);
|
|
309
|
+
}
|
|
310
|
+
function sandboxProvisionStageError(stage, error) {
|
|
311
|
+
if (error && typeof error === "object") {
|
|
312
|
+
sandboxProvisionFailureStages.set(error, stage);
|
|
313
|
+
return error;
|
|
314
|
+
}
|
|
315
|
+
return new SandboxProvisionStageError(stage, error);
|
|
316
|
+
}
|
|
317
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
318
|
+
constructor(backend, timeoutMs, stage, sandboxGroupId, instanceId) {
|
|
319
|
+
const timeoutSeconds = Math.ceil(timeoutMs / 1e3);
|
|
320
|
+
const identity = [
|
|
321
|
+
sandboxGroupId ? `group ${sandboxGroupId}` : null,
|
|
322
|
+
instanceId ? `sandbox ${instanceId}` : null
|
|
323
|
+
].filter((part) => part !== null).join(", ");
|
|
324
|
+
const target = identity ? ` (${identity})` : "";
|
|
325
|
+
super(
|
|
326
|
+
stage === "exec_readiness" ? `Sandbox backend "${backend}"${target} did not become command-ready within ${timeoutSeconds}s after creation or restore.` : `Sandbox backend "${backend}"${target} did not finish warming within ${timeoutSeconds}s while waiting for the elected sandbox creator.`
|
|
327
|
+
);
|
|
328
|
+
this.backend = backend;
|
|
329
|
+
this.timeoutMs = timeoutMs;
|
|
330
|
+
this.stage = stage;
|
|
331
|
+
this.sandboxGroupId = sandboxGroupId;
|
|
332
|
+
this.instanceId = instanceId;
|
|
333
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
334
|
+
}
|
|
335
|
+
};
|
|
336
|
+
var SandboxExecReadinessTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
337
|
+
code = "sandbox_exec_readiness_timeout";
|
|
338
|
+
constructor(backend, timeoutMs, identity = {}) {
|
|
339
|
+
super(
|
|
340
|
+
backend,
|
|
341
|
+
timeoutMs,
|
|
342
|
+
"exec_readiness",
|
|
343
|
+
identity.sandboxGroupId ?? null,
|
|
344
|
+
identity.instanceId ?? null
|
|
345
|
+
);
|
|
346
|
+
this.name = "SandboxExecReadinessTimeoutError";
|
|
347
|
+
}
|
|
348
|
+
};
|
|
349
|
+
var SandboxSiblingWarmingTimeoutError = class extends SandboxWarmingTimeoutError {
|
|
350
|
+
code = "sandbox_sibling_warming_timeout";
|
|
351
|
+
constructor(backend, timeoutMs, identity) {
|
|
352
|
+
super(
|
|
353
|
+
backend,
|
|
354
|
+
timeoutMs,
|
|
355
|
+
"sibling_warming",
|
|
356
|
+
identity.sandboxGroupId,
|
|
357
|
+
identity.instanceId ?? null
|
|
358
|
+
);
|
|
359
|
+
this.name = "SandboxSiblingWarmingTimeoutError";
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
var disposedUnreadyFreshSandboxErrors = /* @__PURE__ */ new WeakSet();
|
|
363
|
+
function isReplaceableFreshSandboxReadinessTimeout(error) {
|
|
364
|
+
return error instanceof SandboxExecReadinessTimeoutError && disposedUnreadyFreshSandboxErrors.has(error);
|
|
365
|
+
}
|
|
366
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2e3;
|
|
367
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8e3;
|
|
368
|
+
function freshSandboxReadinessReplacementDelayMs(random = Math.random) {
|
|
369
|
+
const sample = random();
|
|
370
|
+
const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
|
|
371
|
+
return FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS + Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS);
|
|
372
|
+
}
|
|
373
|
+
var FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
|
|
374
|
+
function createFreshSandboxReadinessReplacementBudget() {
|
|
375
|
+
return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
|
|
376
|
+
}
|
|
377
|
+
function recordFreshSandboxReadinessReplacement(metrics, backend, outcome) {
|
|
378
|
+
try {
|
|
379
|
+
metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
|
|
380
|
+
} catch {
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
async function sleepUnlessCancelled(ms, signal) {
|
|
384
|
+
if (signal?.aborted) return false;
|
|
385
|
+
if (ms <= 0) return true;
|
|
386
|
+
return await new Promise((resolve) => {
|
|
387
|
+
const onAbort = () => {
|
|
388
|
+
clearTimeout(timer);
|
|
389
|
+
resolve(false);
|
|
390
|
+
};
|
|
391
|
+
const timer = setTimeout(() => {
|
|
392
|
+
signal?.removeEventListener("abort", onAbort);
|
|
393
|
+
resolve(true);
|
|
394
|
+
}, ms);
|
|
395
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
|
|
399
|
+
constructor(sandboxGroupId, leaseEpoch, lostInstanceId) {
|
|
400
|
+
super(sandboxGroupId, leaseEpoch);
|
|
401
|
+
this.lostInstanceId = lostInstanceId;
|
|
402
|
+
this.name = "SandboxLeaseInstanceLostError";
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
var WARMING_POLL_INTERVAL_MS = 250;
|
|
406
|
+
async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}, signal) {
|
|
407
|
+
try {
|
|
408
|
+
await verifySandboxExecReadiness(established, timeoutMs, signal);
|
|
409
|
+
} catch (error) {
|
|
410
|
+
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
411
|
+
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
412
|
+
sandboxGroupId: identity.sandboxGroupId ?? null,
|
|
413
|
+
instanceId: established.instanceId
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
throw error;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
var SnapshotTimeoutError = class extends Error {
|
|
420
|
+
constructor(timeoutMs) {
|
|
421
|
+
super(`workspace snapshot timed out after ${timeoutMs}ms`);
|
|
422
|
+
this.name = "SnapshotTimeoutError";
|
|
423
|
+
}
|
|
424
|
+
};
|
|
425
|
+
async function waitForWarmSnapshot(snapshot, timeoutMs, signal) {
|
|
426
|
+
if (signal?.aborted) return false;
|
|
427
|
+
let timeout;
|
|
428
|
+
let cancelListener;
|
|
429
|
+
try {
|
|
430
|
+
await Promise.race([
|
|
431
|
+
snapshot,
|
|
432
|
+
...signal ? [
|
|
433
|
+
new Promise((_resolve, reject) => {
|
|
434
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot wait cancelled"));
|
|
435
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
436
|
+
if (signal.aborted) cancelListener();
|
|
437
|
+
})
|
|
438
|
+
] : [],
|
|
439
|
+
new Promise((_, reject) => {
|
|
440
|
+
timeout = setTimeout(() => reject(new SnapshotTimeoutError(timeoutMs)), timeoutMs);
|
|
441
|
+
if (timeout && "unref" in timeout && typeof timeout.unref === "function") {
|
|
442
|
+
timeout.unref();
|
|
443
|
+
}
|
|
444
|
+
})
|
|
445
|
+
]);
|
|
446
|
+
return true;
|
|
447
|
+
} catch (error) {
|
|
448
|
+
if (signal?.aborted) return false;
|
|
449
|
+
if (error instanceof SnapshotTimeoutError) {
|
|
450
|
+
console.error(
|
|
451
|
+
"mid-session workspace snapshot wait timed out (turn unaffected)",
|
|
452
|
+
safeSnapshotError(error)
|
|
453
|
+
);
|
|
454
|
+
return false;
|
|
455
|
+
}
|
|
456
|
+
return true;
|
|
457
|
+
} finally {
|
|
458
|
+
if (timeout) {
|
|
459
|
+
clearTimeout(timeout);
|
|
460
|
+
}
|
|
461
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
async function resolveModalCheckpointBindingBeforeCapture(settings, session, signal) {
|
|
465
|
+
if (signal?.aborted) {
|
|
466
|
+
throw signal.reason ?? new Error("workspace snapshot identity lookup cancelled");
|
|
467
|
+
}
|
|
468
|
+
const pending = resolveModalCheckpointProviderBindingForSession(settings, session);
|
|
469
|
+
let timeout;
|
|
470
|
+
let cancelListener;
|
|
471
|
+
try {
|
|
472
|
+
return await Promise.race([
|
|
473
|
+
pending,
|
|
474
|
+
...signal ? [
|
|
475
|
+
new Promise((_resolve, reject) => {
|
|
476
|
+
cancelListener = () => reject(signal.reason ?? new Error("workspace snapshot identity lookup cancelled"));
|
|
477
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
478
|
+
if (signal.aborted) cancelListener();
|
|
479
|
+
})
|
|
480
|
+
] : [],
|
|
481
|
+
new Promise((_resolve, reject) => {
|
|
482
|
+
timeout = setTimeout(
|
|
483
|
+
() => reject(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
484
|
+
settings.sandboxSnapshotTimeoutMs
|
|
485
|
+
);
|
|
486
|
+
timeout.unref?.();
|
|
487
|
+
})
|
|
488
|
+
]);
|
|
489
|
+
} finally {
|
|
490
|
+
if (timeout) clearTimeout(timeout);
|
|
491
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
492
|
+
}
|
|
493
|
+
}
|
|
494
|
+
async function terminateEstablishedSandbox(established) {
|
|
495
|
+
if (!established) return true;
|
|
496
|
+
try {
|
|
497
|
+
await terminateUnpublishedSandboxSession(established);
|
|
498
|
+
return true;
|
|
499
|
+
} catch (error) {
|
|
500
|
+
if (isProviderSandboxNotFoundError(established.backendId, error)) {
|
|
501
|
+
return true;
|
|
502
|
+
}
|
|
503
|
+
return false;
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
507
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
510
|
+
try {
|
|
511
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend, stage: error.stage });
|
|
512
|
+
} catch {
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
function workspaceArchiveFieldsFromEnvelope(envelope) {
|
|
516
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
517
|
+
const archive = sessionState?.workspaceArchive;
|
|
518
|
+
const archiveRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchiveRef);
|
|
519
|
+
const hasInline = typeof archive === "string" && archive.length > 0;
|
|
520
|
+
if (!hasInline && !archiveRef) {
|
|
521
|
+
return null;
|
|
522
|
+
}
|
|
523
|
+
const previous = sessionState?.workspaceArchivePrev;
|
|
524
|
+
const previousRef = parseWorkspaceArchiveObjectRef2(sessionState?.workspaceArchivePrevRef);
|
|
525
|
+
const metadata = sessionState?.workspaceArchiveMeta;
|
|
526
|
+
const previousMetadata = sessionState?.workspaceArchivePrevMeta;
|
|
527
|
+
const capturedAt = sessionState?.workspaceArchiveAt;
|
|
528
|
+
return {
|
|
529
|
+
...hasInline && !archiveRef ? { workspaceArchive: archive } : {},
|
|
530
|
+
...archiveRef ? { workspaceArchiveRef: archiveRef } : {},
|
|
531
|
+
...metadata !== void 0 ? { workspaceArchiveMeta: metadata } : {},
|
|
532
|
+
...typeof previous === "string" && previous.length > 0 && !previousRef ? { workspaceArchivePrev: previous } : {},
|
|
533
|
+
...previousRef ? { workspaceArchivePrevRef: previousRef } : {},
|
|
534
|
+
...previousMetadata !== void 0 ? { workspaceArchivePrevMeta: previousMetadata } : {},
|
|
535
|
+
...typeof capturedAt === "string" && capturedAt.length > 0 ? { workspaceArchiveAt: capturedAt } : {}
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
|
|
539
|
+
if (!envelope || typeof envelope !== "object") return envelope;
|
|
540
|
+
const record = envelope;
|
|
541
|
+
const sessionState = record.sessionState && typeof record.sessionState === "object" && !Array.isArray(record.sessionState) ? record.sessionState : null;
|
|
542
|
+
if (!sessionState || !parseWorkspaceArchiveObjectRef2(sessionState.workspaceArchiveRef)) {
|
|
543
|
+
return envelope;
|
|
544
|
+
}
|
|
545
|
+
if (!objectStorage) {
|
|
546
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
547
|
+
"archive_storage_unavailable",
|
|
548
|
+
"workspace archive object storage is not configured",
|
|
549
|
+
{ retryable: true }
|
|
550
|
+
);
|
|
551
|
+
}
|
|
552
|
+
const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
553
|
+
if (process.platform === "linux" && descriptor?.version === 1 && descriptor.workspace.projection === "sdk_local_archive_v1") {
|
|
554
|
+
return envelope;
|
|
555
|
+
}
|
|
556
|
+
return {
|
|
557
|
+
...record,
|
|
558
|
+
sessionState: await inlineWorkspaceArchiveForRestore(
|
|
559
|
+
sessionState,
|
|
560
|
+
(key) => objectStorage.getObjectBytes(key)
|
|
561
|
+
)
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
function legacyNativeArchiveFromEnvelope(envelope) {
|
|
565
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
566
|
+
if (!sessionState) return null;
|
|
567
|
+
const existing = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
568
|
+
if (existing?.version === 2) return null;
|
|
569
|
+
return describeLegacyNativeSnapshotArchive(
|
|
570
|
+
sessionState.workspaceArchive,
|
|
571
|
+
existing?.version === 1 ? Date.parse(existing.capturedAt) : Date.now()
|
|
572
|
+
);
|
|
573
|
+
}
|
|
574
|
+
function maybePersistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force = false) {
|
|
575
|
+
let continuationRegistered = false;
|
|
576
|
+
let resolvePhysicalSettlement = () => void 0;
|
|
577
|
+
const physicalSettlement = new Promise((resolve) => {
|
|
578
|
+
resolvePhysicalSettlement = resolve;
|
|
579
|
+
});
|
|
580
|
+
const result = persistWarmWorkspaceSnapshot(
|
|
581
|
+
services,
|
|
582
|
+
ids,
|
|
583
|
+
session,
|
|
584
|
+
leaseEpoch,
|
|
585
|
+
signal,
|
|
586
|
+
force,
|
|
587
|
+
(continuation) => {
|
|
588
|
+
continuationRegistered = true;
|
|
589
|
+
void continuation.then(resolvePhysicalSettlement, resolvePhysicalSettlement);
|
|
590
|
+
}
|
|
591
|
+
);
|
|
592
|
+
void result.then(
|
|
593
|
+
() => {
|
|
594
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
595
|
+
},
|
|
596
|
+
() => {
|
|
597
|
+
if (!continuationRegistered) resolvePhysicalSettlement();
|
|
598
|
+
}
|
|
599
|
+
);
|
|
600
|
+
return Object.assign(result, { settled: physicalSettlement });
|
|
601
|
+
}
|
|
602
|
+
async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch, signal, force, registerOwnedContinuation) {
|
|
603
|
+
const { db, settings } = services;
|
|
604
|
+
const warnSnapshotFailure = (message, error) => {
|
|
605
|
+
const fields = {
|
|
606
|
+
...safeSnapshotError(error),
|
|
607
|
+
sandboxLeaseKey: sandboxLeaseTelemetryKey2(ids.workspaceId, ids.sandboxGroupId),
|
|
608
|
+
leaseEpoch
|
|
609
|
+
};
|
|
610
|
+
try {
|
|
611
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
612
|
+
else console.warn(message, fields);
|
|
613
|
+
} catch {
|
|
614
|
+
}
|
|
615
|
+
};
|
|
616
|
+
const reportSkipped = (backend, reason) => {
|
|
617
|
+
try {
|
|
618
|
+
services.sandboxMetrics?.onWorkspaceCaptureSkipped?.({
|
|
619
|
+
backend: typeof backend === "string" ? backend : "unknown",
|
|
620
|
+
reason
|
|
621
|
+
});
|
|
622
|
+
} catch {
|
|
623
|
+
}
|
|
624
|
+
};
|
|
625
|
+
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
626
|
+
if (intervalMs <= 0 && !force) {
|
|
627
|
+
return false;
|
|
628
|
+
}
|
|
629
|
+
const persistable = session;
|
|
630
|
+
if (typeof persistable.persistWorkspace !== "function" && typeof persistable.persistWorkspaceTar !== "function") {
|
|
631
|
+
console.error("mid-session workspace snapshot skipped (no persist primitive)", {
|
|
632
|
+
backendId: typeof persistable.backendId === "string" ? persistable.backendId : null
|
|
633
|
+
});
|
|
634
|
+
reportSkipped(persistable.backendId, "no_persist_primitive");
|
|
635
|
+
return false;
|
|
636
|
+
}
|
|
637
|
+
const workspacePersistence = persistable.state?.workspacePersistence ?? persistable.state?.providerState?.workspacePersistence;
|
|
638
|
+
if (signal?.aborted) {
|
|
639
|
+
return false;
|
|
640
|
+
}
|
|
641
|
+
try {
|
|
642
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
643
|
+
const canWarmCapture = lease?.liveness === "warm" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
|
|
644
|
+
const canForceDrainingCapture = force === true && lease?.liveness === "draining" && lease.instanceId !== null && lease.leaseEpoch === leaseEpoch;
|
|
645
|
+
const turnOwner = ids.idleCheckpoint === true ? null : ids;
|
|
646
|
+
if (!lease || !canWarmCapture && (!canForceDrainingCapture || !turnOwner)) {
|
|
647
|
+
console.error("mid-session workspace snapshot skipped (lease not warm)", {
|
|
648
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
649
|
+
expectedEpoch: leaseEpoch,
|
|
650
|
+
liveness: lease?.liveness ?? null,
|
|
651
|
+
leaseEpoch: lease?.leaseEpoch ?? null,
|
|
652
|
+
hasInstance: lease?.instanceId !== null && lease?.instanceId !== void 0
|
|
653
|
+
});
|
|
654
|
+
reportSkipped(lease?.backend ?? persistable.backendId, "lease_not_warm");
|
|
655
|
+
return false;
|
|
656
|
+
}
|
|
657
|
+
if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) {
|
|
658
|
+
return false;
|
|
659
|
+
}
|
|
660
|
+
if (lease.instanceId === null) {
|
|
661
|
+
return false;
|
|
662
|
+
}
|
|
663
|
+
const instanceId = lease.instanceId;
|
|
664
|
+
const captureLiveness = canWarmCapture ? "warm" : "draining";
|
|
665
|
+
const capturePolicy = providerWorkspaceCapturePolicy(lease.backend, session);
|
|
666
|
+
if (!capturePolicy || capturePolicy.liveInstance !== "preserved") {
|
|
667
|
+
console.error("mid-session workspace snapshot skipped (capture policy)", {
|
|
668
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
669
|
+
backend: lease.backend,
|
|
670
|
+
liveInstance: capturePolicy?.liveInstance ?? null
|
|
671
|
+
});
|
|
672
|
+
reportSkipped(lease.backend, "capture_policy");
|
|
673
|
+
return false;
|
|
674
|
+
}
|
|
675
|
+
const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
|
|
676
|
+
if (!turnOwner && !nativeModalPersistence) {
|
|
677
|
+
reportSkipped(lease.backend, "capture_policy");
|
|
678
|
+
return false;
|
|
679
|
+
}
|
|
680
|
+
const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
|
|
681
|
+
if (signal?.aborted) return false;
|
|
682
|
+
const captureId = randomUUID2();
|
|
683
|
+
const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
|
|
684
|
+
const claimed = await claimWorkspaceArchiveCapture(db, {
|
|
685
|
+
accountId: ids.accountId,
|
|
686
|
+
workspaceId: ids.workspaceId,
|
|
687
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
688
|
+
captureId,
|
|
689
|
+
expectedEpoch: leaseEpoch,
|
|
690
|
+
expectedInstanceId: instanceId,
|
|
691
|
+
liveness: captureLiveness,
|
|
692
|
+
captureTimeoutMs,
|
|
693
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
694
|
+
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
695
|
+
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
696
|
+
// Modal pauses the box for a native snapshot, so the image is one
|
|
697
|
+
// instant even while background commands run.
|
|
698
|
+
pointInTimeCapture: nativeModalPersistence,
|
|
699
|
+
...captureLiveness !== "warm" ? {} : turnOwner ? {
|
|
700
|
+
warmAttempt: {
|
|
701
|
+
sessionId: turnOwner.sessionId,
|
|
702
|
+
turnId: turnOwner.turnId,
|
|
703
|
+
attemptId: turnOwner.attemptId,
|
|
704
|
+
holderId: sandboxLeaseHolderIdForAttempt(turnOwner.attemptId)
|
|
705
|
+
}
|
|
706
|
+
} : { idleCheckpoint: true }
|
|
707
|
+
});
|
|
708
|
+
if (claimed.status !== "claimed") {
|
|
709
|
+
if (claimed.status === "throttled" || claimed.status === "clean") return false;
|
|
710
|
+
console.error("mid-session workspace snapshot skipped (capture claim)", {
|
|
711
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
712
|
+
status: claimed.status
|
|
713
|
+
});
|
|
714
|
+
reportSkipped(lease.backend, claimed.status);
|
|
715
|
+
return false;
|
|
716
|
+
}
|
|
717
|
+
let timeout;
|
|
718
|
+
const capturedAtMs = Date.now();
|
|
719
|
+
const registerCandidate = async (archive) => {
|
|
720
|
+
if (archive.kind === "host_spool" || archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
|
|
721
|
+
return null;
|
|
722
|
+
}
|
|
723
|
+
if (!checkpointBinding) {
|
|
724
|
+
throw new Error("Modal native snapshot has no exact session provider identity");
|
|
725
|
+
}
|
|
726
|
+
return await registerSandboxCheckpointArtifact(db, {
|
|
727
|
+
accountId: ids.accountId,
|
|
728
|
+
workspaceId: ids.workspaceId,
|
|
729
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
730
|
+
sourceLeaseId: claimed.claim.leaseId,
|
|
731
|
+
sourceLeaseEpoch: leaseEpoch,
|
|
732
|
+
sourceInstanceId: instanceId,
|
|
733
|
+
sourceWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
734
|
+
providerBindingKey: checkpointBinding.key,
|
|
735
|
+
providerBinding: checkpointBinding.binding,
|
|
736
|
+
workspaceArchive: archive.base64,
|
|
737
|
+
workspaceArchiveMeta: archive.descriptor
|
|
738
|
+
});
|
|
739
|
+
};
|
|
740
|
+
const abandonCandidate = async (artifactId, reason) => {
|
|
741
|
+
await markSandboxCheckpointArtifactDeletePending(db, {
|
|
742
|
+
accountId: ids.accountId,
|
|
743
|
+
workspaceId: ids.workspaceId,
|
|
744
|
+
artifactId,
|
|
745
|
+
reason
|
|
746
|
+
}).catch(() => void 0);
|
|
747
|
+
};
|
|
748
|
+
const captureAbort = new AbortController();
|
|
749
|
+
const captureAbortTimer = setTimeout(
|
|
750
|
+
() => captureAbort.abort(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
751
|
+
settings.sandboxSnapshotTimeoutMs
|
|
752
|
+
);
|
|
753
|
+
captureAbortTimer.unref?.();
|
|
754
|
+
const captureAndPublish = (async () => {
|
|
755
|
+
const captureStarted = performance.now();
|
|
756
|
+
let captureOutcome = "failed";
|
|
757
|
+
let archive;
|
|
758
|
+
let candidate = null;
|
|
759
|
+
let publicationAttempted = false;
|
|
760
|
+
try {
|
|
761
|
+
archive = await captureWorkspaceArchiveForStorage(
|
|
762
|
+
session,
|
|
763
|
+
capturedAtMs,
|
|
764
|
+
{
|
|
765
|
+
requestId: claimed.claim.providerRequestId,
|
|
766
|
+
strategy: capturePolicy.strategy,
|
|
767
|
+
signal: captureAbort.signal
|
|
768
|
+
},
|
|
769
|
+
Boolean(services.objectStorage)
|
|
770
|
+
);
|
|
771
|
+
clearTimeout(captureAbortTimer);
|
|
772
|
+
candidate = await registerCandidate(archive);
|
|
773
|
+
const archiveDescriptor = archive.descriptor;
|
|
774
|
+
const published = await putVersion1TarArchiveOrInline({
|
|
775
|
+
backend: lease.backend,
|
|
776
|
+
objectStorage: services.objectStorage,
|
|
777
|
+
accountId: ids.accountId,
|
|
778
|
+
workspaceId: ids.workspaceId,
|
|
779
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
780
|
+
archive,
|
|
781
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
782
|
+
});
|
|
783
|
+
publicationAttempted = true;
|
|
784
|
+
const { wrote } = await persistWorkspaceArchiveCandidate({
|
|
785
|
+
...services.objectStorage ? { objectStorage: services.objectStorage } : {},
|
|
786
|
+
...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
|
|
787
|
+
persist: () => persistWarmSnapshot(db, {
|
|
788
|
+
accountId: ids.accountId,
|
|
789
|
+
workspaceId: ids.workspaceId,
|
|
790
|
+
...turnOwner ? {
|
|
791
|
+
sessionId: turnOwner.sessionId,
|
|
792
|
+
turnId: turnOwner.turnId,
|
|
793
|
+
attemptId: turnOwner.attemptId
|
|
794
|
+
} : { idleCheckpoint: true },
|
|
795
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
796
|
+
expectedEpoch: leaseEpoch,
|
|
797
|
+
expectedInstanceId: instanceId,
|
|
798
|
+
expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
|
|
799
|
+
captureId,
|
|
800
|
+
workspaceArchiveMeta: archiveDescriptor,
|
|
801
|
+
...published,
|
|
802
|
+
checkpointArtifactId: candidate?.id ?? null,
|
|
803
|
+
minIntervalMs: force ? 0 : intervalMs,
|
|
804
|
+
capturedAtMs
|
|
805
|
+
})
|
|
806
|
+
});
|
|
807
|
+
if (!wrote && candidate) {
|
|
808
|
+
await abandonCandidate(candidate.id, "snapshot_publication_fenced");
|
|
809
|
+
}
|
|
810
|
+
if (wrote) captureOutcome = "completed";
|
|
811
|
+
return wrote;
|
|
812
|
+
} catch (error) {
|
|
813
|
+
if (candidate && !publicationAttempted)
|
|
814
|
+
await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
815
|
+
throw error;
|
|
816
|
+
} finally {
|
|
817
|
+
clearTimeout(captureAbortTimer);
|
|
818
|
+
await disposeWorkspaceArchive(archive).catch(() => {
|
|
819
|
+
console.warn("workspace archive spool cleanup failed");
|
|
820
|
+
});
|
|
821
|
+
await releaseWorkspaceArchiveCapture(db, {
|
|
822
|
+
accountId: ids.accountId,
|
|
823
|
+
workspaceId: ids.workspaceId,
|
|
824
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
825
|
+
captureId,
|
|
826
|
+
expectedEpoch: leaseEpoch,
|
|
827
|
+
expectedInstanceId: instanceId
|
|
828
|
+
}).catch((error) => {
|
|
829
|
+
captureOutcome = "failed";
|
|
830
|
+
warnSnapshotFailure("mid-session workspace capture gate release failed", error);
|
|
831
|
+
});
|
|
832
|
+
try {
|
|
833
|
+
services.sandboxMetrics?.onWorkspaceCapture?.({
|
|
834
|
+
backend: lease.backend,
|
|
835
|
+
outcome: captureOutcome,
|
|
836
|
+
durationSeconds: Math.max(0, performance.now() - captureStarted) / 1e3
|
|
837
|
+
});
|
|
838
|
+
} catch {
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
})();
|
|
842
|
+
const settled = captureAndPublish.then(
|
|
843
|
+
(persisted) => ({ kind: "settled", persisted }),
|
|
844
|
+
(error) => {
|
|
845
|
+
warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
|
|
846
|
+
return { kind: "settled", persisted: false };
|
|
847
|
+
}
|
|
848
|
+
);
|
|
849
|
+
registerOwnedContinuation(settled.then(() => void 0));
|
|
850
|
+
let cancelListener;
|
|
851
|
+
const outcome = await Promise.race([
|
|
852
|
+
settled,
|
|
853
|
+
...signal ? [
|
|
854
|
+
new Promise((resolve) => {
|
|
855
|
+
cancelListener = () => resolve({ kind: "cancelled" });
|
|
856
|
+
signal.addEventListener("abort", cancelListener, { once: true });
|
|
857
|
+
if (signal.aborted) cancelListener();
|
|
858
|
+
})
|
|
859
|
+
] : [],
|
|
860
|
+
new Promise((resolve) => {
|
|
861
|
+
timeout = setTimeout(
|
|
862
|
+
() => resolve({ kind: "timed_out" }),
|
|
863
|
+
settings.sandboxSnapshotTimeoutMs
|
|
864
|
+
);
|
|
865
|
+
})
|
|
866
|
+
]).finally(() => {
|
|
867
|
+
if (timeout) clearTimeout(timeout);
|
|
868
|
+
if (cancelListener) signal?.removeEventListener("abort", cancelListener);
|
|
869
|
+
});
|
|
870
|
+
return outcome.kind === "settled" ? outcome.persisted : false;
|
|
871
|
+
} catch (error) {
|
|
872
|
+
warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
|
|
873
|
+
return false;
|
|
874
|
+
}
|
|
875
|
+
}
|
|
876
|
+
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
877
|
+
try {
|
|
878
|
+
return await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
879
|
+
} catch (error) {
|
|
880
|
+
if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
|
|
881
|
+
const readiness = error;
|
|
882
|
+
const warn = (message, fields) => {
|
|
883
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
884
|
+
else console.warn(message, fields);
|
|
885
|
+
};
|
|
886
|
+
const identity = {
|
|
887
|
+
workspaceId: ids.workspaceId,
|
|
888
|
+
sessionId: ids.sessionId,
|
|
889
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
890
|
+
backend: readiness.backend,
|
|
891
|
+
instanceId: readiness.instanceId,
|
|
892
|
+
readinessTimeoutMs: readiness.timeoutMs
|
|
893
|
+
};
|
|
894
|
+
const budget = services.freshSandboxReadinessReplacementBudget ?? createFreshSandboxReadinessReplacementBudget();
|
|
895
|
+
if (budget.remaining <= 0) {
|
|
896
|
+
recordFreshSandboxReadinessReplacement(
|
|
897
|
+
services.sandboxMetrics,
|
|
898
|
+
readiness.backend,
|
|
899
|
+
"budget_spent"
|
|
900
|
+
);
|
|
901
|
+
warn(
|
|
902
|
+
"sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
|
|
903
|
+
identity
|
|
904
|
+
);
|
|
905
|
+
throw error;
|
|
906
|
+
}
|
|
907
|
+
budget.remaining -= 1;
|
|
908
|
+
const delayMs = await (services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs)();
|
|
909
|
+
warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
|
|
910
|
+
...identity,
|
|
911
|
+
replacementDelayMs: delayMs
|
|
912
|
+
});
|
|
913
|
+
if (!await sleepUnlessCancelled(delayMs, services.cancellationSignal)) {
|
|
914
|
+
recordFreshSandboxReadinessReplacement(
|
|
915
|
+
services.sandboxMetrics,
|
|
916
|
+
readiness.backend,
|
|
917
|
+
"cancelled"
|
|
918
|
+
);
|
|
919
|
+
throw error;
|
|
920
|
+
}
|
|
921
|
+
try {
|
|
922
|
+
const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
923
|
+
recordFreshSandboxReadinessReplacement(
|
|
924
|
+
services.sandboxMetrics,
|
|
925
|
+
readiness.backend,
|
|
926
|
+
"replaced"
|
|
927
|
+
);
|
|
928
|
+
return replaced;
|
|
929
|
+
} catch (replacementError) {
|
|
930
|
+
recordFreshSandboxReadinessReplacement(
|
|
931
|
+
services.sandboxMetrics,
|
|
932
|
+
readiness.backend,
|
|
933
|
+
services.cancellationSignal?.aborted ? "cancelled" : replacementError instanceof SandboxExecReadinessTimeoutError ? "failed_again" : "replacement_failed"
|
|
934
|
+
);
|
|
935
|
+
throw replacementError;
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
|
|
940
|
+
const { db, settings } = services;
|
|
941
|
+
const os = ids.os ?? "linux";
|
|
942
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
943
|
+
const cancellationSignal = services.cancellationSignal;
|
|
944
|
+
let releaseStarted = false;
|
|
945
|
+
let holderRelease = null;
|
|
946
|
+
let quiescedRelease = null;
|
|
947
|
+
let holderLivenessTimer;
|
|
948
|
+
let cancellationListener;
|
|
949
|
+
const release = (options) => {
|
|
950
|
+
const writersQuiesced = options?.workspaceWritersQuiesced === true;
|
|
951
|
+
if (!releaseStarted) {
|
|
952
|
+
releaseStarted = true;
|
|
953
|
+
if (cancellationListener) {
|
|
954
|
+
cancellationSignal?.removeEventListener("abort", cancellationListener);
|
|
955
|
+
cancellationListener = void 0;
|
|
956
|
+
}
|
|
957
|
+
if (holderLivenessTimer) {
|
|
958
|
+
clearInterval(holderLivenessTimer);
|
|
959
|
+
holderLivenessTimer = void 0;
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
const persistRelease = async (workspaceWritersQuiesced) => {
|
|
963
|
+
await releaseLeaseHolder(db, {
|
|
964
|
+
accountId: ids.accountId,
|
|
965
|
+
workspaceId: ids.workspaceId,
|
|
966
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
967
|
+
kind,
|
|
968
|
+
holderId,
|
|
969
|
+
idleGraceMs: settings.sandboxIdleGraceMs,
|
|
970
|
+
...workspaceWritersQuiesced ? { workspaceWritersQuiesced: true } : {}
|
|
971
|
+
});
|
|
972
|
+
};
|
|
973
|
+
if (writersQuiesced) {
|
|
974
|
+
if (!quiescedRelease) {
|
|
975
|
+
const predecessor = holderRelease;
|
|
976
|
+
const attempt = (predecessor ? predecessor.catch(() => void 0) : Promise.resolve()).then(
|
|
977
|
+
async () => await persistRelease(true)
|
|
978
|
+
);
|
|
979
|
+
const tracked = attempt.catch((error) => {
|
|
980
|
+
if (quiescedRelease === tracked) quiescedRelease = null;
|
|
981
|
+
throw error;
|
|
982
|
+
});
|
|
983
|
+
quiescedRelease = tracked;
|
|
984
|
+
holderRelease = tracked;
|
|
985
|
+
}
|
|
986
|
+
return quiescedRelease;
|
|
987
|
+
}
|
|
988
|
+
if (!holderRelease) {
|
|
989
|
+
const attempt = persistRelease(false);
|
|
990
|
+
const tracked = attempt.catch((error) => {
|
|
991
|
+
if (holderRelease === tracked) holderRelease = null;
|
|
992
|
+
throw error;
|
|
993
|
+
});
|
|
994
|
+
holderRelease = tracked;
|
|
995
|
+
}
|
|
996
|
+
return holderRelease;
|
|
997
|
+
};
|
|
998
|
+
const cancellationError = () => cancellationSignal?.reason instanceof Error ? cancellationSignal.reason : new Error("Sandbox resume was cancelled with its owning turn attempt");
|
|
999
|
+
const throwIfReleasedOrCancelled = () => {
|
|
1000
|
+
if (releaseStarted || cancellationSignal?.aborted) {
|
|
1001
|
+
throw cancellationError();
|
|
1002
|
+
}
|
|
1003
|
+
};
|
|
1004
|
+
const acquired = await acquireLease(db, {
|
|
1005
|
+
accountId: ids.accountId,
|
|
1006
|
+
workspaceId: ids.workspaceId,
|
|
1007
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1008
|
+
kind,
|
|
1009
|
+
holderId,
|
|
1010
|
+
subjectId: ids.sessionId,
|
|
1011
|
+
backend: ids.backend,
|
|
1012
|
+
warmBilling: {
|
|
1013
|
+
mode: settings.sandboxWarmBillingMode,
|
|
1014
|
+
rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend)
|
|
1015
|
+
},
|
|
1016
|
+
os,
|
|
1017
|
+
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
1018
|
+
// conflicts on a live box already running a different required image. A
|
|
1019
|
+
// SandboxImageConflictError propagates while another holder is active; a
|
|
1020
|
+
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
1021
|
+
// after the cold successor can safely stamp the new image.
|
|
1022
|
+
...ids.image ? { image: ids.image } : {},
|
|
1023
|
+
...ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {},
|
|
1024
|
+
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
1025
|
+
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
1026
|
+
// propagates while another holder is active; a solo change uses the same
|
|
1027
|
+
// durable capture-and-drain rotation as an image change.
|
|
1028
|
+
...ids.rigVersionId ? { rigVersionId: ids.rigVersionId } : {},
|
|
1029
|
+
leaseTtlMs,
|
|
1030
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs,
|
|
1031
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
|
|
1032
|
+
...cancellationSignal ? { waitSignal: cancellationSignal } : {}
|
|
1033
|
+
});
|
|
1034
|
+
if (cancellationSignal) {
|
|
1035
|
+
cancellationListener = () => {
|
|
1036
|
+
void release().catch(() => void 0);
|
|
1037
|
+
};
|
|
1038
|
+
cancellationSignal.addEventListener("abort", cancellationListener, { once: true });
|
|
1039
|
+
}
|
|
1040
|
+
if (cancellationSignal?.aborted) {
|
|
1041
|
+
await release();
|
|
1042
|
+
throw cancellationError();
|
|
1043
|
+
}
|
|
1044
|
+
let holderLeaseHeartbeat = acquired.role === "spawner" ? {
|
|
1045
|
+
expectedEpoch: acquired.lease.leaseEpoch,
|
|
1046
|
+
leaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
1047
|
+
} : (acquired.role === "attached" || acquired.role === "rearmed") && acquired.lease.liveness === "warm" ? { expectedEpoch: acquired.lease.leaseEpoch, leaseTtlMs } : null;
|
|
1048
|
+
let providerRenewalTarget = acquired.lease.liveness === "warm" && acquired.lease.instanceId ? {
|
|
1049
|
+
backend: ids.backend,
|
|
1050
|
+
instanceId: acquired.lease.instanceId
|
|
1051
|
+
} : null;
|
|
1052
|
+
let providerRenewedAtMs = Date.now();
|
|
1053
|
+
let providerRenewalInFlight = null;
|
|
1054
|
+
const maybeRenewProviderExpiration = async (force = false) => {
|
|
1055
|
+
const target = providerRenewalTarget;
|
|
1056
|
+
if (!target || providerRenewalInFlight) return;
|
|
1057
|
+
const ttlSeconds = effectiveSandboxLifecycle(settings, target.backend).renewableTtlSeconds;
|
|
1058
|
+
if (ttlSeconds === null) return;
|
|
1059
|
+
const intervalMs = Math.max(1e4, Math.floor(ttlSeconds * 1e3 / 3));
|
|
1060
|
+
if (!force && Date.now() - providerRenewedAtMs < intervalMs) return;
|
|
1061
|
+
providerRenewalInFlight = renewSandboxProviderExpiration({
|
|
1062
|
+
backend: target.backend,
|
|
1063
|
+
settings,
|
|
1064
|
+
instanceId: target.instanceId,
|
|
1065
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
1066
|
+
}).then(() => {
|
|
1067
|
+
providerRenewedAtMs = Date.now();
|
|
1068
|
+
}).finally(() => {
|
|
1069
|
+
providerRenewalInFlight = null;
|
|
1070
|
+
});
|
|
1071
|
+
await providerRenewalInFlight;
|
|
1072
|
+
};
|
|
1073
|
+
const releaseDeadHolder = async (reason) => {
|
|
1074
|
+
if (reason === "holder_gone" && kind === "turn" && holderLeaseHeartbeat) {
|
|
1075
|
+
const expectedEpoch = holderLeaseHeartbeat.expectedEpoch;
|
|
1076
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1077
|
+
if (retainsHolderForWarmCapture(lease, expectedEpoch)) return;
|
|
1078
|
+
}
|
|
1079
|
+
const message = "sandbox resume holder released: holder no longer live";
|
|
1080
|
+
const fields = {
|
|
1081
|
+
workspaceId: ids.workspaceId,
|
|
1082
|
+
sessionId: ids.sessionId,
|
|
1083
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1084
|
+
holderKind: kind,
|
|
1085
|
+
reason
|
|
1086
|
+
};
|
|
1087
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
1088
|
+
else console.warn(message, fields);
|
|
1089
|
+
void release().catch(() => void 0);
|
|
1090
|
+
};
|
|
1091
|
+
holderLivenessTimer = setInterval(() => {
|
|
1092
|
+
const heartbeat = holderLeaseHeartbeat;
|
|
1093
|
+
if (heartbeat) {
|
|
1094
|
+
void heartbeatLeaseHolderStatus(db, {
|
|
1095
|
+
accountId: ids.accountId,
|
|
1096
|
+
workspaceId: ids.workspaceId,
|
|
1097
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1098
|
+
kind,
|
|
1099
|
+
holderId,
|
|
1100
|
+
expectedEpoch: heartbeat.expectedEpoch,
|
|
1101
|
+
leaseTtlMs: heartbeat.leaseTtlMs,
|
|
1102
|
+
billingMode: services.settings.sandboxWarmBillingMode
|
|
1103
|
+
}).then(async (status) => {
|
|
1104
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
1105
|
+
if (!status.holderAlive) {
|
|
1106
|
+
await releaseDeadHolder(status.fence);
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
if (status.fence === "epoch") {
|
|
1110
|
+
await releaseDeadHolder(status.fence);
|
|
1111
|
+
return;
|
|
1112
|
+
}
|
|
1113
|
+
if (status.leaseExtended || status.fence === "rotation_requested") {
|
|
1114
|
+
void maybeRenewProviderExpiration().catch(() => void 0);
|
|
1115
|
+
}
|
|
1116
|
+
}).catch(() => void 0);
|
|
1117
|
+
return;
|
|
1118
|
+
}
|
|
1119
|
+
void touchLeaseHolder(db, {
|
|
1120
|
+
accountId: ids.accountId,
|
|
1121
|
+
workspaceId: ids.workspaceId,
|
|
1122
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1123
|
+
kind,
|
|
1124
|
+
holderId
|
|
1125
|
+
}).then(async (touched) => {
|
|
1126
|
+
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
1127
|
+
if (!touched) {
|
|
1128
|
+
await releaseDeadHolder("holder_gone");
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
void maybeRenewProviderExpiration().catch(() => void 0);
|
|
1132
|
+
}).catch(() => void 0);
|
|
1133
|
+
}, services.holderLivenessIntervalMs ?? 1e4);
|
|
1134
|
+
if ("unref" in holderLivenessTimer && typeof holderLivenessTimer.unref === "function") {
|
|
1135
|
+
holderLivenessTimer.unref();
|
|
1136
|
+
}
|
|
1137
|
+
if (acquired.role === "blocked") {
|
|
1138
|
+
await release();
|
|
1139
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1140
|
+
ids.sandboxGroupId,
|
|
1141
|
+
acquired.lease.leaseEpoch,
|
|
1142
|
+
acquired.code,
|
|
1143
|
+
acquired.lease.recovery
|
|
1144
|
+
);
|
|
1145
|
+
}
|
|
1146
|
+
if (acquired.role === "fenced") {
|
|
1147
|
+
await release();
|
|
1148
|
+
if (acquired.reason !== "superseded") {
|
|
1149
|
+
throw new SandboxLeaseTransitionError(
|
|
1150
|
+
ids.sandboxGroupId,
|
|
1151
|
+
acquired.lease.leaseEpoch,
|
|
1152
|
+
acquired.reason,
|
|
1153
|
+
acquired.lease.backend,
|
|
1154
|
+
acquired.lease.instanceId,
|
|
1155
|
+
acquired.lease.liveness
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch, {
|
|
1159
|
+
backend: acquired.lease.backend,
|
|
1160
|
+
instanceId: acquired.lease.instanceId,
|
|
1161
|
+
liveness: acquired.lease.liveness
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
if (acquired.role === "spawner") {
|
|
1165
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1166
|
+
let createdEstablished = null;
|
|
1167
|
+
let published = false;
|
|
1168
|
+
let providerCreateOperationId;
|
|
1169
|
+
let providerCreateBindingKey;
|
|
1170
|
+
let rematerialization = null;
|
|
1171
|
+
try {
|
|
1172
|
+
const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
|
|
1173
|
+
const envelope = freshWorkspaceRecoveryId ? null : await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
1174
|
+
const legacyFallbackRefused = !freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null && await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId);
|
|
1175
|
+
const fallbackArchiveEnvelope = !freshWorkspaceRecoveryId && !legacyFallbackRefused && acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(envelope) !== null ? withoutSandboxProviderIdentity(envelope) : null;
|
|
1176
|
+
let spawnEnvelope = freshWorkspaceRecoveryId ? null : legacyFallbackRefused ? acquired.lease.resumeState ?? null : fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope;
|
|
1177
|
+
const archiveSource = acquired.lease.recovery.archive.status === "none" ? fallbackArchiveEnvelope : acquired.lease.resumeState;
|
|
1178
|
+
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
1179
|
+
if (!freshWorkspaceRecoveryId && (acquired.lease.recovery.archive.status === "available" && (acquired.lease.archiveComplete || acquired.lease.historicalRecoveryAuthorized === true) || acquired.lease.recovery.archive.status === "none" && workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)) {
|
|
1180
|
+
const rematerializationId = crypto.randomUUID();
|
|
1181
|
+
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
1182
|
+
const begun = await beginSandboxRematerialization(db, {
|
|
1183
|
+
accountId: ids.accountId,
|
|
1184
|
+
workspaceId: ids.workspaceId,
|
|
1185
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1186
|
+
expectedEpoch,
|
|
1187
|
+
rematerializationId,
|
|
1188
|
+
archiveSource,
|
|
1189
|
+
legacyNativeArchive
|
|
1190
|
+
});
|
|
1191
|
+
if (begun.status !== "started") {
|
|
1192
|
+
if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
|
|
1193
|
+
throw new SandboxLeaseSupersededError(
|
|
1194
|
+
ids.sandboxGroupId,
|
|
1195
|
+
begun.lease?.leaseEpoch ?? expectedEpoch
|
|
1196
|
+
);
|
|
1197
|
+
}
|
|
1198
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1199
|
+
ids.sandboxGroupId,
|
|
1200
|
+
begun.lease?.leaseEpoch ?? expectedEpoch,
|
|
1201
|
+
begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
|
|
1202
|
+
begun.lease?.recovery ?? acquired.lease.recovery
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1205
|
+
spawnEnvelope = begun.lease.resumeState ?? spawnEnvelope;
|
|
1206
|
+
const selectedRevision = begun.lease.recovery.restore.selectedRevision;
|
|
1207
|
+
if (!selectedRevision) {
|
|
1208
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1209
|
+
"archive_metadata_invalid",
|
|
1210
|
+
"sandbox rematerialization selected no durable archive revision"
|
|
1211
|
+
);
|
|
1212
|
+
}
|
|
1213
|
+
rematerialization = {
|
|
1214
|
+
id: rematerializationId,
|
|
1215
|
+
selectedRevision,
|
|
1216
|
+
workspaceGeneration: begun.lease.workspaceGeneration,
|
|
1217
|
+
providerBindingKey: begun.checkpointArtifact?.providerBindingKey ?? null,
|
|
1218
|
+
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
1219
|
+
legacyProviderBinding: null
|
|
1220
|
+
};
|
|
1221
|
+
} else if (!freshWorkspaceRecoveryId && acquired.lease.recovery.archive.status !== "none" && !continuityRecovery) {
|
|
1222
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1223
|
+
ids.sandboxGroupId,
|
|
1224
|
+
expectedEpoch,
|
|
1225
|
+
"restore_degraded",
|
|
1226
|
+
acquired.lease.recovery
|
|
1227
|
+
);
|
|
1228
|
+
}
|
|
1229
|
+
const providerCreateStartedAt = /* @__PURE__ */ new Date();
|
|
1230
|
+
const persistArchiveSource = spawnEnvelope && typeof spawnEnvelope === "object" ? spawnEnvelope : null;
|
|
1231
|
+
const hydrateEnvelopeRaw = await materializeSpawnEnvelopeArchive(
|
|
1232
|
+
persistArchiveSource,
|
|
1233
|
+
services.objectStorage
|
|
1234
|
+
);
|
|
1235
|
+
const hydrateEnvelope = hydrateEnvelopeRaw && typeof hydrateEnvelopeRaw === "object" ? hydrateEnvelopeRaw : null;
|
|
1236
|
+
const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
|
|
1237
|
+
sessionId: ids.sessionId,
|
|
1238
|
+
recovery: "create-or-restore",
|
|
1239
|
+
...services.objectStorage ? {
|
|
1240
|
+
loadHostWorkspaceArchive: async (ref) => {
|
|
1241
|
+
try {
|
|
1242
|
+
return await downloadWorkspaceArchiveSpool(
|
|
1243
|
+
services.objectStorage,
|
|
1244
|
+
ref.key,
|
|
1245
|
+
{ bytes: ref.bytes, sha256: ref.sha256 },
|
|
1246
|
+
{ temporaryDirectory: workspaceArchiveDownloadTemporaryDirectory }
|
|
1247
|
+
);
|
|
1248
|
+
} catch (error) {
|
|
1249
|
+
if (error instanceof WorkspaceArchiveStorageError) {
|
|
1250
|
+
throw new WorkspaceArchiveIntegrityError(error.code, error.message, {
|
|
1251
|
+
retryable: error.retryable,
|
|
1252
|
+
cause: error
|
|
1253
|
+
});
|
|
1254
|
+
}
|
|
1255
|
+
throw error;
|
|
1256
|
+
}
|
|
1257
|
+
}
|
|
1258
|
+
} : {},
|
|
1259
|
+
backendOverride: ids.backend,
|
|
1260
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1261
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
1262
|
+
...services.logicalFallbackSettings ? { logicalFallbackSettings: services.logicalFallbackSettings } : {},
|
|
1263
|
+
onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
|
|
1264
|
+
if (ids.backend !== "modal") return;
|
|
1265
|
+
if (!intent || !providerContext)
|
|
1266
|
+
throw new Error("Modal create requires the provider dispatch boundary");
|
|
1267
|
+
throwIfReleasedOrCancelled();
|
|
1268
|
+
const binding = await resolveModalCheckpointProviderBindingForSession(
|
|
1269
|
+
createSettings,
|
|
1270
|
+
providerContext
|
|
1271
|
+
);
|
|
1272
|
+
const operationId = intent.operationId;
|
|
1273
|
+
await beginModalProviderCreate(db, {
|
|
1274
|
+
accountId: ids.accountId,
|
|
1275
|
+
workspaceId: ids.workspaceId,
|
|
1276
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1277
|
+
expectedEpoch,
|
|
1278
|
+
operationId,
|
|
1279
|
+
providerBindingKey: binding.key,
|
|
1280
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
1281
|
+
selectedRevision: rematerialization?.selectedRevision ?? null,
|
|
1282
|
+
imageId: intent.imageId,
|
|
1283
|
+
imageRef: createSettings.modalImageRef ?? null,
|
|
1284
|
+
appId: intent.appId,
|
|
1285
|
+
providerName: intent.name,
|
|
1286
|
+
requestSha256: intent.requestSha256
|
|
1287
|
+
});
|
|
1288
|
+
providerCreateOperationId = operationId;
|
|
1289
|
+
providerCreateBindingKey = binding.key;
|
|
1290
|
+
},
|
|
1291
|
+
onSandboxCreated: async (created) => {
|
|
1292
|
+
createdEstablished = created;
|
|
1293
|
+
providerRenewalTarget = {
|
|
1294
|
+
backend: ids.backend,
|
|
1295
|
+
instanceId: created.instanceId
|
|
1296
|
+
};
|
|
1297
|
+
providerRenewedAtMs = Date.now();
|
|
1298
|
+
if (providerCreateBindingKey && !await modalSessionMatchesCheckpointProviderBinding(
|
|
1299
|
+
settings,
|
|
1300
|
+
created.session,
|
|
1301
|
+
providerCreateBindingKey
|
|
1302
|
+
)) {
|
|
1303
|
+
throw new Error("Modal creation receipt crossed the fenced provider namespace");
|
|
1304
|
+
}
|
|
1305
|
+
if (rematerialization && (rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)) {
|
|
1306
|
+
if (created.backendId !== "modal") {
|
|
1307
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1308
|
+
"native_snapshot_reference_invalid",
|
|
1309
|
+
"Modal checkpoint restore resolved a non-Modal sandbox backend"
|
|
1310
|
+
);
|
|
1311
|
+
}
|
|
1312
|
+
if (rematerialization.providerBindingKey) {
|
|
1313
|
+
if (!await modalSessionMatchesCheckpointProviderBinding(
|
|
1314
|
+
settings,
|
|
1315
|
+
created.session,
|
|
1316
|
+
rematerialization.providerBindingKey
|
|
1317
|
+
)) {
|
|
1318
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1319
|
+
"native_snapshot_reference_invalid",
|
|
1320
|
+
"Modal checkpoint restore refused because the authenticated provider workspace changed"
|
|
1321
|
+
);
|
|
1322
|
+
}
|
|
1323
|
+
} else {
|
|
1324
|
+
const resolved = await resolveModalCheckpointProviderBindingForSession(
|
|
1325
|
+
settings,
|
|
1326
|
+
created.session
|
|
1327
|
+
);
|
|
1328
|
+
if (rematerialization.legacyProviderBinding && rematerialization.legacyProviderBinding.key !== resolved.key) {
|
|
1329
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1330
|
+
"native_snapshot_reference_invalid",
|
|
1331
|
+
"Legacy Modal checkpoint restore crossed authenticated provider workspaces"
|
|
1332
|
+
);
|
|
1333
|
+
}
|
|
1334
|
+
rematerialization.legacyProviderBinding = resolved;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
const resumeEnvelope2 = requirePersistableReplacementSandboxEnvelope(
|
|
1338
|
+
await serializeReplacementSandboxEnvelope(created, persistArchiveSource),
|
|
1339
|
+
created.backendId
|
|
1340
|
+
);
|
|
1341
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
1342
|
+
accountId: ids.accountId,
|
|
1343
|
+
workspaceId: ids.workspaceId,
|
|
1344
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1345
|
+
expectedEpoch,
|
|
1346
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
1347
|
+
...providerCreateOperationId ? { providerCreateOperationId } : {},
|
|
1348
|
+
...created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {},
|
|
1349
|
+
instanceId: created.instanceId,
|
|
1350
|
+
resumeBackendId: created.backendId,
|
|
1351
|
+
resumeState: resumeEnvelope2,
|
|
1352
|
+
...created.backendId === "modal" ? {
|
|
1353
|
+
providerCreatedAt: providerCreateStartedAt,
|
|
1354
|
+
providerDeadlineAt: new Date(
|
|
1355
|
+
providerCreateStartedAt.getTime() + settings.modalTimeoutSeconds * 1e3
|
|
1356
|
+
)
|
|
1357
|
+
} : {},
|
|
1358
|
+
leaseTtlMs,
|
|
1359
|
+
// Keep the warming budget after create(): manifest setup and
|
|
1360
|
+
// commitWarmingToWarm still run, and can exceed the 90s turn TTL.
|
|
1361
|
+
warmingLeaseTtlMs: settings.sandboxLeaseWarmingTtlMs
|
|
1362
|
+
});
|
|
1363
|
+
if (!recorded.recorded) {
|
|
1364
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1365
|
+
}
|
|
1366
|
+
if (created.backendId === "modal") {
|
|
1367
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
1368
|
+
leaseId: acquired.lease.id,
|
|
1369
|
+
workspaceId: ids.workspaceId,
|
|
1370
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1371
|
+
}).catch(() => void 0);
|
|
1372
|
+
}
|
|
1373
|
+
throwIfReleasedOrCancelled();
|
|
1374
|
+
},
|
|
1375
|
+
onWorkspaceRestoreVerifying: async (descriptor) => {
|
|
1376
|
+
if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
|
|
1377
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1378
|
+
"archive_metadata_invalid",
|
|
1379
|
+
`hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
if (rematerialization.legacyCheckpoint) {
|
|
1383
|
+
const binding = rematerialization.legacyProviderBinding;
|
|
1384
|
+
if (!binding) {
|
|
1385
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1386
|
+
"native_snapshot_reference_invalid",
|
|
1387
|
+
"Legacy Modal checkpoint restore produced no authenticated provider identity"
|
|
1388
|
+
);
|
|
1389
|
+
}
|
|
1390
|
+
const adopted = await adoptLegacyModalCheckpointArtifact(db, {
|
|
1391
|
+
accountId: ids.accountId,
|
|
1392
|
+
workspaceId: ids.workspaceId,
|
|
1393
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1394
|
+
leaseId: acquired.lease.id,
|
|
1395
|
+
leaseEpoch: expectedEpoch,
|
|
1396
|
+
workspaceGeneration: rematerialization.workspaceGeneration,
|
|
1397
|
+
slot: "current",
|
|
1398
|
+
archiveBase64: rematerialization.legacyCheckpoint.archiveBase64,
|
|
1399
|
+
descriptor: rematerialization.legacyCheckpoint.descriptor,
|
|
1400
|
+
providerBindingKey: binding.key,
|
|
1401
|
+
providerBinding: binding.binding,
|
|
1402
|
+
rematerializationId: rematerialization.id
|
|
1403
|
+
});
|
|
1404
|
+
if (!adopted) {
|
|
1405
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1406
|
+
}
|
|
1407
|
+
rematerialization.providerBindingKey = binding.key;
|
|
1408
|
+
}
|
|
1409
|
+
const verifying = await markSandboxRestoreVerifying(db, {
|
|
1410
|
+
accountId: ids.accountId,
|
|
1411
|
+
workspaceId: ids.workspaceId,
|
|
1412
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1413
|
+
expectedEpoch,
|
|
1414
|
+
rematerializationId: rematerialization.id
|
|
1415
|
+
});
|
|
1416
|
+
if (!verifying.wrote) {
|
|
1417
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1418
|
+
}
|
|
1419
|
+
}
|
|
1420
|
+
});
|
|
1421
|
+
createdEstablished = established;
|
|
1422
|
+
throwIfReleasedOrCancelled();
|
|
1423
|
+
if (services.verifySpawnedSandboxReadiness) {
|
|
1424
|
+
await services.verifySpawnedSandboxReadiness(established, {
|
|
1425
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1426
|
+
});
|
|
1427
|
+
} else {
|
|
1428
|
+
await waitForSandboxExecReadiness(
|
|
1429
|
+
established,
|
|
1430
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
1431
|
+
{
|
|
1432
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1433
|
+
},
|
|
1434
|
+
cancellationSignal
|
|
1435
|
+
);
|
|
1436
|
+
}
|
|
1437
|
+
await maybeRenewProviderExpiration(true);
|
|
1438
|
+
throwIfReleasedOrCancelled();
|
|
1439
|
+
const resumeEnvelope = requirePersistableReplacementSandboxEnvelope(
|
|
1440
|
+
await serializeReplacementSandboxEnvelope(established, persistArchiveSource),
|
|
1441
|
+
established.backendId
|
|
1442
|
+
);
|
|
1443
|
+
throwIfReleasedOrCancelled();
|
|
1444
|
+
if (rematerialization && !established.providerContinuity && established.restoredArchive?.revision !== rematerialization.selectedRevision) {
|
|
1445
|
+
throw new WorkspaceArchiveIntegrityError(
|
|
1446
|
+
"workspace_fingerprint_mismatch",
|
|
1447
|
+
"sandbox restore completed without the exact selected durable archive revision"
|
|
1448
|
+
);
|
|
1449
|
+
}
|
|
1450
|
+
const committed = await commitWarmingToWarm(db, {
|
|
1451
|
+
accountId: ids.accountId,
|
|
1452
|
+
workspaceId: ids.workspaceId,
|
|
1453
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1454
|
+
expectedEpoch,
|
|
1455
|
+
instanceId: established.instanceId,
|
|
1456
|
+
// Viewer attach resolves and records the tunnel URL lazily. A headless
|
|
1457
|
+
// turn must not touch the optional desktop data plane.
|
|
1458
|
+
dataPlaneUrl: null,
|
|
1459
|
+
resumeBackendId: established.backendId,
|
|
1460
|
+
resumeState: resumeEnvelope,
|
|
1461
|
+
...freshWorkspaceRecoveryId ? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } } : established.providerContinuity ? { continuityRecovery: established.providerContinuity } : rematerialization ? {
|
|
1462
|
+
rematerialization: {
|
|
1463
|
+
id: rematerialization.id,
|
|
1464
|
+
verifiedRevision: rematerialization.selectedRevision
|
|
1465
|
+
}
|
|
1466
|
+
} : {},
|
|
1467
|
+
leaseTtlMs
|
|
1468
|
+
});
|
|
1469
|
+
if (!committed.committed || !committed.lease) {
|
|
1470
|
+
const terminated = await terminateEstablishedSandbox(established);
|
|
1471
|
+
if (terminated && rematerialization && !established.providerContinuity) {
|
|
1472
|
+
await failSandboxRematerialization(db, {
|
|
1473
|
+
accountId: ids.accountId,
|
|
1474
|
+
workspaceId: ids.workspaceId,
|
|
1475
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1476
|
+
expectedEpoch,
|
|
1477
|
+
rematerializationId: rematerialization.id,
|
|
1478
|
+
failureCode: committed.reason ?? "warm_commit_rejected",
|
|
1479
|
+
retryable: false
|
|
1480
|
+
});
|
|
1481
|
+
} else if (terminated) {
|
|
1482
|
+
await failWarmingToCold(db, {
|
|
1483
|
+
accountId: ids.accountId,
|
|
1484
|
+
workspaceId: ids.workspaceId,
|
|
1485
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1486
|
+
expectedEpoch
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
await release();
|
|
1490
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1491
|
+
}
|
|
1492
|
+
published = true;
|
|
1493
|
+
holderLeaseHeartbeat = {
|
|
1494
|
+
expectedEpoch: committed.lease.leaseEpoch,
|
|
1495
|
+
leaseTtlMs
|
|
1496
|
+
};
|
|
1497
|
+
await services.onSpawnedSandboxPublished?.();
|
|
1498
|
+
throwIfReleasedOrCancelled();
|
|
1499
|
+
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1500
|
+
} catch (error) {
|
|
1501
|
+
if (published) {
|
|
1502
|
+
await release();
|
|
1503
|
+
throw error;
|
|
1504
|
+
}
|
|
1505
|
+
if (error instanceof SandboxLeaseSupersededError) {
|
|
1506
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
1507
|
+
await release();
|
|
1508
|
+
throw error;
|
|
1509
|
+
}
|
|
1510
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
1511
|
+
if (terminated) {
|
|
1512
|
+
const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
|
|
1513
|
+
if (rematerialization && !createdEstablished?.providerContinuity) {
|
|
1514
|
+
await failSandboxRematerialization(db, {
|
|
1515
|
+
accountId: ids.accountId,
|
|
1516
|
+
workspaceId: ids.workspaceId,
|
|
1517
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1518
|
+
expectedEpoch,
|
|
1519
|
+
rematerializationId: rematerialization.id,
|
|
1520
|
+
failureCode: continuityUnavailable ? "provider_continuity_unavailable" : error instanceof WorkspaceArchiveIntegrityError ? error.code : "sandbox_rematerialization_failed",
|
|
1521
|
+
retryable: continuityUnavailable ? error.retryable : error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
|
|
1522
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
1523
|
+
});
|
|
1524
|
+
} else {
|
|
1525
|
+
await failWarmingToCold(db, {
|
|
1526
|
+
accountId: ids.accountId,
|
|
1527
|
+
workspaceId: ids.workspaceId,
|
|
1528
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1529
|
+
expectedEpoch,
|
|
1530
|
+
...continuityUnavailable ? { discardContinuity: true } : {}
|
|
1531
|
+
});
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
await release();
|
|
1535
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1536
|
+
if (terminated && createdEstablished && error instanceof SandboxExecReadinessTimeoutError && error.instanceId === createdEstablished.instanceId && (createdEstablished.origin === "created" || createdEstablished.origin === "restored") && !createdEstablished.providerContinuity) {
|
|
1537
|
+
disposedUnreadyFreshSandboxErrors.add(error);
|
|
1538
|
+
}
|
|
1539
|
+
throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
try {
|
|
1543
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1544
|
+
if (acquired.lease.liveness === "warming") {
|
|
1545
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1546
|
+
holderLeaseHeartbeat = { expectedEpoch: leaseEpoch, leaseTtlMs };
|
|
1547
|
+
}
|
|
1548
|
+
throwIfReleasedOrCancelled();
|
|
1549
|
+
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1550
|
+
if (!live || live.liveness !== "warm" || live.leaseEpoch !== leaseEpoch || live.instanceId === null) {
|
|
1551
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, live?.leaseEpoch ?? leaseEpoch);
|
|
1552
|
+
}
|
|
1553
|
+
let established;
|
|
1554
|
+
try {
|
|
1555
|
+
const establish = services.establishAttachedSandbox ?? establishSandboxSessionFromEnvelope;
|
|
1556
|
+
established = await establish(settings, live.resumeState, {
|
|
1557
|
+
sessionId: ids.sessionId,
|
|
1558
|
+
recovery: "resume-only",
|
|
1559
|
+
backendOverride: ids.backend,
|
|
1560
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1561
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
1562
|
+
});
|
|
1563
|
+
throwIfReleasedOrCancelled();
|
|
1564
|
+
if (services.verifyAttachedSandboxReadiness) {
|
|
1565
|
+
await services.verifyAttachedSandboxReadiness(established);
|
|
1566
|
+
} else {
|
|
1567
|
+
await waitForSandboxExecReadiness(
|
|
1568
|
+
established,
|
|
1569
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
1570
|
+
{
|
|
1571
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1572
|
+
},
|
|
1573
|
+
cancellationSignal
|
|
1574
|
+
);
|
|
1575
|
+
}
|
|
1576
|
+
providerRenewalTarget = {
|
|
1577
|
+
backend: ids.backend,
|
|
1578
|
+
instanceId: established.instanceId
|
|
1579
|
+
};
|
|
1580
|
+
await maybeRenewProviderExpiration(true);
|
|
1581
|
+
throwIfReleasedOrCancelled();
|
|
1582
|
+
} catch (error) {
|
|
1583
|
+
if (!isProviderSandboxNotFoundError(ids.backend, error)) {
|
|
1584
|
+
throw sandboxProvisionStageError("resume", error);
|
|
1585
|
+
}
|
|
1586
|
+
const marked = await markWarmLeaseInstanceLost(db, {
|
|
1587
|
+
accountId: ids.accountId,
|
|
1588
|
+
workspaceId: ids.workspaceId,
|
|
1589
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1590
|
+
expectedEpoch: leaseEpoch,
|
|
1591
|
+
expectedInstanceId: live.instanceId,
|
|
1592
|
+
expectedBackend: ids.backend
|
|
1593
|
+
});
|
|
1594
|
+
if (marked.status === "marked") {
|
|
1595
|
+
await publishDurableSessionEvents(
|
|
1596
|
+
services.bus,
|
|
1597
|
+
ids.workspaceId,
|
|
1598
|
+
marked.backgroundCommandEvents
|
|
1599
|
+
);
|
|
1600
|
+
await services.onSandboxLost?.({
|
|
1601
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1602
|
+
instanceId: live.instanceId,
|
|
1603
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
1604
|
+
});
|
|
1605
|
+
throw new SandboxLeaseInstanceLostError(
|
|
1606
|
+
ids.sandboxGroupId,
|
|
1607
|
+
marked.lease.leaseEpoch,
|
|
1608
|
+
live.instanceId
|
|
1609
|
+
);
|
|
1610
|
+
}
|
|
1611
|
+
throw new SandboxLeaseSupersededError(
|
|
1612
|
+
ids.sandboxGroupId,
|
|
1613
|
+
marked.lease?.leaseEpoch ?? leaseEpoch
|
|
1614
|
+
);
|
|
1615
|
+
}
|
|
1616
|
+
throwIfReleasedOrCancelled();
|
|
1617
|
+
return { established, leaseEpoch, release };
|
|
1618
|
+
} catch (error) {
|
|
1619
|
+
await release();
|
|
1620
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
1621
|
+
throw error;
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
async function waitForWarm(services, ids) {
|
|
1625
|
+
const { db, settings } = services;
|
|
1626
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1627
|
+
let instanceId = null;
|
|
1628
|
+
while (Date.now() < deadline) {
|
|
1629
|
+
if (!await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal)) {
|
|
1630
|
+
throw services.cancellationSignal?.reason instanceof Error ? services.cancellationSignal.reason : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
|
|
1631
|
+
}
|
|
1632
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1633
|
+
if (!lease) {
|
|
1634
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1635
|
+
}
|
|
1636
|
+
instanceId = lease.instanceId;
|
|
1637
|
+
if (lease.liveness === "warm") {
|
|
1638
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
1639
|
+
}
|
|
1640
|
+
if (lease.liveness === "draining") {
|
|
1641
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1642
|
+
}
|
|
1643
|
+
if (lease.liveness === "cold") {
|
|
1644
|
+
if (isRetryableDegradedRestore(lease.recovery.restore)) {
|
|
1645
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1646
|
+
}
|
|
1647
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.restore.status === "unrecoverable") {
|
|
1648
|
+
throw new SandboxLeaseRecoveryBlockedError(
|
|
1649
|
+
ids.sandboxGroupId,
|
|
1650
|
+
lease.leaseEpoch,
|
|
1651
|
+
lease.recovery.restore.status === "degraded" ? "restore_degraded" : "restore_unrecoverable",
|
|
1652
|
+
lease.recovery
|
|
1653
|
+
);
|
|
1654
|
+
}
|
|
1655
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
throw new SandboxSiblingWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs, {
|
|
1659
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1660
|
+
instanceId
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
// src/sandbox-routing.ts
|
|
1665
|
+
import { isDeepStrictEqual } from "util";
|
|
1666
|
+
import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
|
|
1667
|
+
import {
|
|
1668
|
+
createProviderCommandRetainer,
|
|
1669
|
+
supervisedCommandProtocolReady
|
|
1670
|
+
} from "@opengeni/db/retained-provider-commands";
|
|
1671
|
+
import {
|
|
1672
|
+
adoptConnectedMachineSessionBackgroundCommand,
|
|
1673
|
+
adoptManagedSessionBackgroundCommand,
|
|
1674
|
+
advanceWorkspaceGenerationForRetainedProcess,
|
|
1675
|
+
retainedProviderCommandPersistence,
|
|
1676
|
+
advanceWorkspaceGeneration,
|
|
1677
|
+
getRetainedProcess,
|
|
1678
|
+
retainWorkspaceMutationProcess,
|
|
1679
|
+
retainedProcessSettlementIdentity,
|
|
1680
|
+
settleConnectedMachineSessionBackgroundCommand,
|
|
1681
|
+
settleRetainedProcess,
|
|
1682
|
+
verifyRetainedProcessMutationSettlement,
|
|
1683
|
+
verifyWorkspaceMutationSettlement,
|
|
1684
|
+
getSandbox,
|
|
1685
|
+
getLiveEnrollmentConnection,
|
|
1686
|
+
markWarmLeaseInstanceLost as markWarmLeaseInstanceLost2,
|
|
1687
|
+
readLease as readLease2,
|
|
1688
|
+
readActiveSandbox,
|
|
1689
|
+
resolvePersonalMachineConnectionForAttempt,
|
|
1690
|
+
SandboxRetainedProcessPromotionFencedError,
|
|
1691
|
+
SandboxRetainedProcessTerminalError,
|
|
1692
|
+
SandboxWorkspaceMutationOutputRejectedError
|
|
1693
|
+
} from "@opengeni/db";
|
|
1694
|
+
import {
|
|
1695
|
+
observeSessionBackgroundCommandCompletion,
|
|
1696
|
+
recordConnectedCommandOutputConsumption
|
|
1697
|
+
} from "@opengeni/db/session-background-commands";
|
|
1698
|
+
import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
|
|
1699
|
+
import {
|
|
1700
|
+
buildSelfhostedBackendSession,
|
|
1701
|
+
ActiveBackendUnresolvableError,
|
|
1702
|
+
establishSandboxSessionFromEnvelope as establishSandboxSessionFromEnvelope2,
|
|
1703
|
+
isProviderSandboxGoneDuringRoutedOperation,
|
|
1704
|
+
isProviderSandboxNotFoundError as isProviderSandboxNotFoundError2,
|
|
1705
|
+
makeActiveBackendResolver,
|
|
1706
|
+
NatsControlRpc,
|
|
1707
|
+
NatsOpStreamTransport,
|
|
1708
|
+
RoutingBackendRecoveryRequiredError,
|
|
1709
|
+
RoutingSandboxSession,
|
|
1710
|
+
RoutingMutationOutputRejectedError,
|
|
1711
|
+
resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
|
|
1712
|
+
resolveConnectedMachineWorkspaceRoot,
|
|
1713
|
+
sandboxProviderInstanceIdFromEnvelope,
|
|
1714
|
+
sandboxBackendForSdkBackendId,
|
|
1715
|
+
selectBackend,
|
|
1716
|
+
verifySandboxExecReadiness as verifySandboxExecReadiness2
|
|
1717
|
+
} from "@opengeni/runtime";
|
|
1718
|
+
var retainWorkspaceProviderCommand = createProviderCommandRetainer(
|
|
1719
|
+
retainWorkspaceMutationProcess,
|
|
1720
|
+
(error) => error instanceof SandboxRetainedProcessPromotionFencedError ? error.process : null
|
|
1721
|
+
);
|
|
1722
|
+
function admittedCommandHandle(value) {
|
|
1723
|
+
if (!value || typeof value !== "object") return void 0;
|
|
1724
|
+
return value.admission?.workspaceGeneration;
|
|
1725
|
+
}
|
|
1726
|
+
function relayConfigFromSettings(settings) {
|
|
1727
|
+
const raw = settings.selfhostedRelayUrl?.trim();
|
|
1728
|
+
if (!raw) {
|
|
1729
|
+
return { host: "relay.opengeni.local", port: 443, tls: true };
|
|
1730
|
+
}
|
|
1731
|
+
try {
|
|
1732
|
+
const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
|
|
1733
|
+
const tls = url.protocol === "wss:" || url.protocol === "https:";
|
|
1734
|
+
const port = url.port ? Number(url.port) : tls ? 443 : 80;
|
|
1735
|
+
return { host: url.hostname, port, tls };
|
|
1736
|
+
} catch {
|
|
1737
|
+
return { host: raw, port: 443, tls: true };
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
function selfhostedTimeoutsFromSettings(settings) {
|
|
1741
|
+
return {
|
|
1742
|
+
timeoutMs: settings.sandboxSelfhostedControlTimeoutMs,
|
|
1743
|
+
execTimeoutMs: settings.sandboxSelfhostedExecTimeoutMs
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
function selfhostedResolverTimeouts(settings) {
|
|
1747
|
+
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
1748
|
+
return {
|
|
1749
|
+
selfhostedTimeoutMs: timeoutMs,
|
|
1750
|
+
selfhostedExecTimeoutMs: execTimeoutMs
|
|
1751
|
+
};
|
|
1752
|
+
}
|
|
1753
|
+
function homeRouteRecoveryDisposition(lease) {
|
|
1754
|
+
if (!lease) return "unrecoverable";
|
|
1755
|
+
if (lease.liveness === "warming") return "pending";
|
|
1756
|
+
if (lease.liveness === "draining") return "superseded";
|
|
1757
|
+
if (lease.recovery.restore.status === "pending" || lease.recovery.restore.status === "restoring" || lease.recovery.restore.status === "verifying") {
|
|
1758
|
+
return "pending";
|
|
1759
|
+
}
|
|
1760
|
+
if (lease.recovery.restore.status === "degraded" || lease.recovery.workspace.status === "degraded" || lease.recovery.archive.status === "unverified" || lease.recovery.archive.status === "invalid") {
|
|
1761
|
+
return "degraded";
|
|
1762
|
+
}
|
|
1763
|
+
return "unrecoverable";
|
|
1764
|
+
}
|
|
1765
|
+
function homeRouteRecoveryError(lease, fallbackEpoch) {
|
|
1766
|
+
return new RoutingBackendRecoveryRequiredError(
|
|
1767
|
+
"resolve_home_backend",
|
|
1768
|
+
lease?.leaseEpoch ?? fallbackEpoch,
|
|
1769
|
+
homeRouteRecoveryDisposition(lease)
|
|
1770
|
+
);
|
|
1771
|
+
}
|
|
1772
|
+
function providerIdentityFromResumeState(resumeState) {
|
|
1773
|
+
return sandboxProviderInstanceIdFromEnvelope(resumeState);
|
|
1774
|
+
}
|
|
1775
|
+
async function resolveCurrentHomeBackend(services, ids, established) {
|
|
1776
|
+
const lease = await readLease2(services.db, ids.workspaceId, ids.sandboxGroupId);
|
|
1777
|
+
const fallbackEpoch = lease?.leaseEpoch ?? 0;
|
|
1778
|
+
if (!lease || lease.liveness !== "warm" || lease.recovery.provider.status !== "exists" || lease.instanceId === null || lease.recovery.provider.instanceId !== lease.instanceId || lease.recovery.workspace.status !== "ready" || lease.recovery.restore.status !== "not_required" && lease.recovery.restore.status !== "ready") {
|
|
1779
|
+
throw homeRouteRecoveryError(lease, fallbackEpoch);
|
|
1780
|
+
}
|
|
1781
|
+
if (lease.instanceId === established.instanceId) {
|
|
1782
|
+
await services.onHomeSandboxRebound?.({
|
|
1783
|
+
established,
|
|
1784
|
+
leaseEpoch: lease.leaseEpoch
|
|
1785
|
+
});
|
|
1786
|
+
return {
|
|
1787
|
+
session: established.session,
|
|
1788
|
+
sandboxId: null,
|
|
1789
|
+
kind: established.backendId,
|
|
1790
|
+
leaseEpoch: lease.leaseEpoch,
|
|
1791
|
+
providerInstanceId: lease.instanceId
|
|
1792
|
+
};
|
|
1793
|
+
}
|
|
1794
|
+
const resumeState = lease.resumeState;
|
|
1795
|
+
const durableResumeBackend = lease.resumeBackendId ?? lease.backend ?? ids.backend;
|
|
1796
|
+
const resumeBackend = sandboxBackendForSdkBackendId(durableResumeBackend) ?? durableResumeBackend;
|
|
1797
|
+
if (!resumeState || resumeBackend !== ids.backend || providerIdentityFromResumeState(resumeState) !== lease.instanceId) {
|
|
1798
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
1799
|
+
}
|
|
1800
|
+
let rebound;
|
|
1801
|
+
try {
|
|
1802
|
+
rebound = await establishSandboxSessionFromEnvelope2(services.settings, resumeState, {
|
|
1803
|
+
sessionId: ids.sessionId,
|
|
1804
|
+
recovery: "resume-only",
|
|
1805
|
+
backendOverride: resumeBackend
|
|
1806
|
+
});
|
|
1807
|
+
await verifySandboxExecReadiness2(rebound);
|
|
1808
|
+
} catch (error) {
|
|
1809
|
+
if (!isProviderSandboxNotFoundError2(resumeBackend, error)) {
|
|
1810
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
1811
|
+
"resolve_home_backend",
|
|
1812
|
+
lease.leaseEpoch,
|
|
1813
|
+
"pending"
|
|
1814
|
+
);
|
|
1815
|
+
}
|
|
1816
|
+
const marked = await markWarmLeaseInstanceLost2(services.db, {
|
|
1817
|
+
accountId: ids.accountId,
|
|
1818
|
+
workspaceId: ids.workspaceId,
|
|
1819
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1820
|
+
expectedEpoch: lease.leaseEpoch,
|
|
1821
|
+
expectedInstanceId: lease.instanceId,
|
|
1822
|
+
expectedBackend: lease.backend,
|
|
1823
|
+
diagnostic: "provider_not_found_during_home_route_rebind"
|
|
1824
|
+
});
|
|
1825
|
+
if (marked.status === "marked") {
|
|
1826
|
+
await publishDurableSessionEvents(
|
|
1827
|
+
services.bus,
|
|
1828
|
+
ids.workspaceId,
|
|
1829
|
+
marked.backgroundCommandEvents
|
|
1830
|
+
);
|
|
1831
|
+
await services.onHomeSandboxLost?.({
|
|
1832
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1833
|
+
instanceId: lease.instanceId,
|
|
1834
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
1835
|
+
});
|
|
1836
|
+
}
|
|
1837
|
+
throw new RoutingBackendRecoveryRequiredError(
|
|
1838
|
+
"resolve_home_backend",
|
|
1839
|
+
marked.lease?.leaseEpoch ?? lease.leaseEpoch,
|
|
1840
|
+
marked.status === "stale" ? "superseded" : homeRouteRecoveryDisposition(marked.lease)
|
|
1841
|
+
);
|
|
1842
|
+
}
|
|
1843
|
+
const reboundBackend = sandboxBackendForSdkBackendId(rebound.backendId) ?? rebound.backendId;
|
|
1844
|
+
if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
|
|
1845
|
+
throw homeRouteRecoveryError(lease, lease.leaseEpoch);
|
|
1846
|
+
}
|
|
1847
|
+
await services.onHomeSandboxRebound?.({
|
|
1848
|
+
established: rebound,
|
|
1849
|
+
leaseEpoch: lease.leaseEpoch
|
|
1850
|
+
});
|
|
1851
|
+
Object.assign(established, rebound);
|
|
1852
|
+
return {
|
|
1853
|
+
session: rebound.session,
|
|
1854
|
+
sandboxId: null,
|
|
1855
|
+
kind: rebound.backendId,
|
|
1856
|
+
leaseEpoch: lease.leaseEpoch,
|
|
1857
|
+
providerInstanceId: lease.instanceId
|
|
1858
|
+
};
|
|
1859
|
+
}
|
|
1860
|
+
function beforePersistableHomeMutation(services, ids, home) {
|
|
1861
|
+
const fence = ids.workspaceMutationFence;
|
|
1862
|
+
if (!home || !fence) return void 0;
|
|
1863
|
+
return async ({ op, backend, onCaptureWait }) => {
|
|
1864
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
1865
|
+
return null;
|
|
1866
|
+
}
|
|
1867
|
+
const providerBinding = home.backend === "modal" ? await resolveModalCheckpointProviderBindingForSession2(services.settings, backend.session) : null;
|
|
1868
|
+
const admission = await advanceWorkspaceGeneration(services.db, {
|
|
1869
|
+
accountId: fence.accountId,
|
|
1870
|
+
workspaceId: ids.workspaceId,
|
|
1871
|
+
sessionId: ids.sessionId,
|
|
1872
|
+
turnId: fence.turnId,
|
|
1873
|
+
executionGeneration: fence.executionGeneration,
|
|
1874
|
+
attemptId: fence.attemptId,
|
|
1875
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
1876
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
1877
|
+
expectedEpoch: backend.leaseEpoch,
|
|
1878
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
1879
|
+
operation: op,
|
|
1880
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
1881
|
+
...services.waitSignal ? { waitSignal: services.waitSignal } : {},
|
|
1882
|
+
...onCaptureWait ? { onCaptureWait } : {}
|
|
1883
|
+
});
|
|
1884
|
+
return { admission, providerBinding };
|
|
1885
|
+
};
|
|
1886
|
+
}
|
|
1887
|
+
function afterPersistableHomeMutation(services, ids, home) {
|
|
1888
|
+
const fence = ids.workspaceMutationFence;
|
|
1889
|
+
if (!home || !fence) return void 0;
|
|
1890
|
+
return async ({ op, backend, admission, outcome, retainedProcess }) => {
|
|
1891
|
+
if (backend.sandboxId !== null || backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0 || !admission || typeof admission !== "object") {
|
|
1892
|
+
return;
|
|
1893
|
+
}
|
|
1894
|
+
const boundAdmission = admission;
|
|
1895
|
+
const exactAdmission = boundAdmission.admission;
|
|
1896
|
+
if (!exactAdmission || typeof exactAdmission.id !== "string" || typeof exactAdmission.workspaceGeneration !== "number" || !("providerBinding" in boundAdmission)) {
|
|
1897
|
+
throw new Error("Persistable home mutation settlement lacked its bound admission");
|
|
1898
|
+
}
|
|
1899
|
+
if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
|
|
1900
|
+
try {
|
|
1901
|
+
await retainWorkspaceProviderCommand(services.db, {
|
|
1902
|
+
accountId: fence.accountId,
|
|
1903
|
+
workspaceId: ids.workspaceId,
|
|
1904
|
+
sessionId: ids.sessionId,
|
|
1905
|
+
processId: retainedProcess.id,
|
|
1906
|
+
providerSessionId: retainedProcess.providerSessionId,
|
|
1907
|
+
...retainedProcess.providerCommand ? { providerCommand: retainedProcess.providerCommand } : {},
|
|
1908
|
+
admissionId: exactAdmission.id,
|
|
1909
|
+
admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
|
|
1910
|
+
operation: op,
|
|
1911
|
+
providerBinding: boundAdmission.providerBinding ?? null,
|
|
1912
|
+
owner: {
|
|
1913
|
+
kind: "turn",
|
|
1914
|
+
turnId: fence.turnId,
|
|
1915
|
+
executionGeneration: fence.executionGeneration,
|
|
1916
|
+
attemptId: fence.attemptId,
|
|
1917
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
1918
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
1919
|
+
expectedEpoch: backend.leaseEpoch,
|
|
1920
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
1921
|
+
routeKind: exactAdmission.routeKind,
|
|
1922
|
+
routeTargetId: exactAdmission.routeTargetId,
|
|
1923
|
+
routeEpoch: exactAdmission.routeEpoch
|
|
1924
|
+
}
|
|
1925
|
+
});
|
|
1926
|
+
} catch (error) {
|
|
1927
|
+
if (!(error instanceof SandboxRetainedProcessPromotionFencedError)) throw error;
|
|
1928
|
+
const durable = error.process;
|
|
1929
|
+
if (durable.id !== retainedProcess.id || durable.providerSessionId !== retainedProcess.providerSessionId || durable.sessionId !== ids.sessionId || durable.sandboxGroupId !== home.sandboxGroupId || durable.parentAdmissionId !== exactAdmission.id || durable.leaseEpoch !== backend.leaseEpoch || durable.providerInstanceId !== backend.providerInstanceId || durable.routeKind !== exactAdmission.routeKind || durable.routeTargetId !== exactAdmission.routeTargetId || durable.routeEpoch !== exactAdmission.routeEpoch || durable.state !== "active") {
|
|
1930
|
+
throw new Error("Durable retained-process promotion returned a mismatched identity", {
|
|
1931
|
+
cause: error
|
|
1932
|
+
});
|
|
1933
|
+
}
|
|
1934
|
+
return {
|
|
1935
|
+
status: "retained_process_durable_output_rejected",
|
|
1936
|
+
retainedProcess: {
|
|
1937
|
+
id: durable.id,
|
|
1938
|
+
providerSessionId: durable.providerSessionId
|
|
1939
|
+
}
|
|
1940
|
+
};
|
|
1941
|
+
}
|
|
1942
|
+
return;
|
|
1943
|
+
}
|
|
1944
|
+
if (outcome === "outcome_unknown")
|
|
1945
|
+
throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
|
|
1946
|
+
try {
|
|
1947
|
+
await verifyWorkspaceMutationSettlement(services.db, {
|
|
1948
|
+
accountId: fence.accountId,
|
|
1949
|
+
workspaceId: ids.workspaceId,
|
|
1950
|
+
sessionId: ids.sessionId,
|
|
1951
|
+
turnId: fence.turnId,
|
|
1952
|
+
executionGeneration: fence.executionGeneration,
|
|
1953
|
+
attemptId: fence.attemptId,
|
|
1954
|
+
holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
|
|
1955
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
1956
|
+
expectedEpoch: backend.leaseEpoch,
|
|
1957
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
1958
|
+
admission: exactAdmission,
|
|
1959
|
+
operation: op,
|
|
1960
|
+
outcome
|
|
1961
|
+
});
|
|
1962
|
+
} catch (error) {
|
|
1963
|
+
if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
|
|
1964
|
+
accountId: fence.accountId,
|
|
1965
|
+
workspaceId: ids.workspaceId,
|
|
1966
|
+
admission: exactAdmission,
|
|
1967
|
+
operation: op,
|
|
1968
|
+
outcome
|
|
1969
|
+
})) {
|
|
1970
|
+
throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
|
|
1971
|
+
}
|
|
1972
|
+
throw error;
|
|
1973
|
+
}
|
|
1974
|
+
};
|
|
1975
|
+
}
|
|
1976
|
+
function beforeRetainedProcessMutation(services, ids) {
|
|
1977
|
+
const fence = ids.workspaceMutationFence;
|
|
1978
|
+
if (!fence) return void 0;
|
|
1979
|
+
return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
|
|
1980
|
+
accountId: fence.accountId,
|
|
1981
|
+
workspaceId: ids.workspaceId,
|
|
1982
|
+
sessionId: ids.sessionId,
|
|
1983
|
+
processId: process2.id,
|
|
1984
|
+
operation: op,
|
|
1985
|
+
captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
|
|
1986
|
+
...services.waitSignal ? { waitSignal: services.waitSignal } : {}
|
|
1987
|
+
});
|
|
1988
|
+
}
|
|
1989
|
+
function afterRetainedProcessMutation(services, ids) {
|
|
1990
|
+
const fence = ids.workspaceMutationFence;
|
|
1991
|
+
if (!fence) return void 0;
|
|
1992
|
+
return async ({ op, process: process2, admission, outcome }) => {
|
|
1993
|
+
if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
|
|
1994
|
+
throw new Error("Retained-process mutation settlement lacked its exact admission");
|
|
1995
|
+
}
|
|
1996
|
+
try {
|
|
1997
|
+
await verifyRetainedProcessMutationSettlement(services.db, {
|
|
1998
|
+
accountId: fence.accountId,
|
|
1999
|
+
workspaceId: ids.workspaceId,
|
|
2000
|
+
sessionId: ids.sessionId,
|
|
2001
|
+
processId: process2.id,
|
|
2002
|
+
admission,
|
|
2003
|
+
operation: op,
|
|
2004
|
+
outcome
|
|
2005
|
+
});
|
|
2006
|
+
} catch (error) {
|
|
2007
|
+
if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
|
|
2008
|
+
accountId: fence.accountId,
|
|
2009
|
+
workspaceId: ids.workspaceId,
|
|
2010
|
+
admission,
|
|
2011
|
+
operation: op,
|
|
2012
|
+
outcome
|
|
2013
|
+
})) {
|
|
2014
|
+
if (error.retainedProcessTerminal) {
|
|
2015
|
+
throw new SandboxRetainedProcessTerminalError(
|
|
2016
|
+
error.retainedProcessTerminal.state,
|
|
2017
|
+
error.retainedProcessTerminal.exitCode
|
|
2018
|
+
);
|
|
2019
|
+
}
|
|
2020
|
+
throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
|
|
2021
|
+
}
|
|
2022
|
+
throw error;
|
|
2023
|
+
}
|
|
2024
|
+
};
|
|
2025
|
+
}
|
|
2026
|
+
function captureConnectedCommandOutput(db, ids, bus) {
|
|
2027
|
+
return async (commandId, frames) => {
|
|
2028
|
+
for (const frame of frames) {
|
|
2029
|
+
const events = await appendSessionCommandOutput(db, {
|
|
2030
|
+
...ids,
|
|
2031
|
+
commandId,
|
|
2032
|
+
chunkId: `op-frame:${frame.sequence}`,
|
|
2033
|
+
stream: frame.stream,
|
|
2034
|
+
chunk: frame.chunk
|
|
2035
|
+
});
|
|
2036
|
+
if (events.length && bus)
|
|
2037
|
+
await bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => void 0);
|
|
2038
|
+
}
|
|
2039
|
+
};
|
|
2040
|
+
}
|
|
2041
|
+
function captureRetainedProcessOutputForTurn(services, ids) {
|
|
2042
|
+
const accountId = ids.workspaceMutationFence?.accountId;
|
|
2043
|
+
if (!accountId) return void 0;
|
|
2044
|
+
return async ({
|
|
2045
|
+
process: process2,
|
|
2046
|
+
chunkId,
|
|
2047
|
+
chunk,
|
|
2048
|
+
stream,
|
|
2049
|
+
streamFidelity
|
|
2050
|
+
}) => {
|
|
2051
|
+
const events = await appendSessionCommandOutput(services.db, {
|
|
2052
|
+
accountId,
|
|
2053
|
+
workspaceId: ids.workspaceId,
|
|
2054
|
+
sessionId: ids.sessionId,
|
|
2055
|
+
commandId: process2.id,
|
|
2056
|
+
chunkId,
|
|
2057
|
+
chunk,
|
|
2058
|
+
stream,
|
|
2059
|
+
streamFidelity
|
|
2060
|
+
});
|
|
2061
|
+
if (events.length && services.bus)
|
|
2062
|
+
await services.bus.publish(ids.workspaceId, ids.sessionId, events).catch(() => void 0);
|
|
2063
|
+
};
|
|
2064
|
+
}
|
|
2065
|
+
function observeRetainedProcessTerminalForTurn(services, ids) {
|
|
2066
|
+
const fence = ids.workspaceMutationFence;
|
|
2067
|
+
if (!fence) return void 0;
|
|
2068
|
+
return async ({ process: process2 }) => {
|
|
2069
|
+
await observeSessionBackgroundCommandCompletion(services.db, {
|
|
2070
|
+
accountId: fence.accountId,
|
|
2071
|
+
workspaceId: ids.workspaceId,
|
|
2072
|
+
sessionId: ids.sessionId,
|
|
2073
|
+
commandId: process2.id
|
|
2074
|
+
});
|
|
2075
|
+
};
|
|
2076
|
+
}
|
|
2077
|
+
function settleRetainedProcessForTurn(services, ids) {
|
|
2078
|
+
const fence = ids.workspaceMutationFence;
|
|
2079
|
+
if (!fence) return void 0;
|
|
2080
|
+
return async ({ backend, process: process2, proof }) => {
|
|
2081
|
+
const durable = await getRetainedProcess(services.db, {
|
|
2082
|
+
workspaceId: ids.workspaceId,
|
|
2083
|
+
sessionId: ids.sessionId,
|
|
2084
|
+
processId: process2.id
|
|
2085
|
+
});
|
|
2086
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
2087
|
+
throw new Error("Retained-process settlement lost its exact durable backend identity");
|
|
2088
|
+
}
|
|
2089
|
+
const settlement = await settleRetainedProcess(services.db, {
|
|
2090
|
+
accountId: fence.accountId,
|
|
2091
|
+
workspaceId: ids.workspaceId,
|
|
2092
|
+
sessionId: ids.sessionId,
|
|
2093
|
+
processId: process2.id,
|
|
2094
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
2095
|
+
outcome: proof.outcome,
|
|
2096
|
+
exitCode: proof.exitCode,
|
|
2097
|
+
reason: proof.reason,
|
|
2098
|
+
idleGraceMs: services.settings.sandboxIdleGraceMs
|
|
2099
|
+
});
|
|
2100
|
+
if (settlement.process.state === "active") {
|
|
2101
|
+
throw new Error("Retained-process settlement returned an active durable process");
|
|
2102
|
+
}
|
|
2103
|
+
if (settlement.backgroundCommandEvents.length > 0 && services.bus) {
|
|
2104
|
+
await services.bus.publish(ids.workspaceId, ids.sessionId, settlement.backgroundCommandEvents).catch(() => void 0);
|
|
2105
|
+
}
|
|
2106
|
+
};
|
|
2107
|
+
}
|
|
2108
|
+
function isRetainedProcessSettledForTurn(services, ids) {
|
|
2109
|
+
const fence = ids.workspaceMutationFence;
|
|
2110
|
+
if (!fence) return void 0;
|
|
2111
|
+
return async ({
|
|
2112
|
+
backend,
|
|
2113
|
+
process: process2
|
|
2114
|
+
}) => {
|
|
2115
|
+
const durable = await getRetainedProcess(services.db, {
|
|
2116
|
+
workspaceId: ids.workspaceId,
|
|
2117
|
+
sessionId: ids.sessionId,
|
|
2118
|
+
processId: process2.id
|
|
2119
|
+
});
|
|
2120
|
+
if (!durable || durable.state === "active" || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch)
|
|
2121
|
+
return false;
|
|
2122
|
+
if (process2.providerCommand) {
|
|
2123
|
+
const command = await retainedProviderCommandPersistence(services.db, {
|
|
2124
|
+
accountId: fence.accountId,
|
|
2125
|
+
workspaceId: ids.workspaceId,
|
|
2126
|
+
sessionId: ids.sessionId,
|
|
2127
|
+
processId: process2.id
|
|
2128
|
+
}).load();
|
|
2129
|
+
if (!command || command.kind !== process2.providerCommand.kind || command.execId !== process2.providerCommand.execId || command.taskId !== process2.providerCommand.taskId || command.sandboxId !== process2.providerCommand.sandboxId || Boolean(command.pty) !== Boolean(process2.providerCommand.pty) || command.kind === "modal-router-v1" && process2.providerCommand.kind === "modal-router-v1" && !isDeepStrictEqual(command.supervision, process2.providerCommand.supervision))
|
|
2130
|
+
return false;
|
|
2131
|
+
}
|
|
2132
|
+
return true;
|
|
2133
|
+
};
|
|
2134
|
+
}
|
|
2135
|
+
function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
|
|
2136
|
+
const fence = ids.workspaceMutationFence;
|
|
2137
|
+
if (!fence) return void 0;
|
|
2138
|
+
return async ({ backend, process: process2, command }) => {
|
|
2139
|
+
const durable = await getRetainedProcess(services.db, {
|
|
2140
|
+
workspaceId: ids.workspaceId,
|
|
2141
|
+
sessionId: ids.sessionId,
|
|
2142
|
+
processId: process2.id
|
|
2143
|
+
});
|
|
2144
|
+
if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
|
|
2145
|
+
throw new Error("Background command adoption lost its exact retained-process identity");
|
|
2146
|
+
}
|
|
2147
|
+
await adoptManagedSessionBackgroundCommand(services.db, {
|
|
2148
|
+
accountId: fence.accountId,
|
|
2149
|
+
workspaceId: ids.workspaceId,
|
|
2150
|
+
sessionId: ids.sessionId,
|
|
2151
|
+
turnId: fence.turnId,
|
|
2152
|
+
executionGeneration: fence.executionGeneration,
|
|
2153
|
+
attemptId: fence.attemptId,
|
|
2154
|
+
processId: process2.id,
|
|
2155
|
+
expected: retainedProcessSettlementIdentity(durable),
|
|
2156
|
+
command: command ?? "Background command"
|
|
2157
|
+
});
|
|
2158
|
+
};
|
|
2159
|
+
}
|
|
2160
|
+
function controlRpcFactory(bus) {
|
|
2161
|
+
return () => new NatsControlRpc(async () => {
|
|
2162
|
+
if (!bus) {
|
|
2163
|
+
return null;
|
|
2164
|
+
}
|
|
2165
|
+
return bus.getRequestConnection();
|
|
2166
|
+
});
|
|
2167
|
+
}
|
|
2168
|
+
function wrapTurnBoxWithRouting(services, ids, established) {
|
|
2169
|
+
const { db, settings, bus, onOp } = services;
|
|
2170
|
+
const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
|
|
2171
|
+
const backgroundCommandAttempt = ids.workspaceMutationFence;
|
|
2172
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, ids.homeLease);
|
|
2173
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, ids.homeLease);
|
|
2174
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
2175
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
2176
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
2177
|
+
const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
|
|
2178
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
2179
|
+
services,
|
|
2180
|
+
ids
|
|
2181
|
+
);
|
|
2182
|
+
const resolver = makeActiveBackendResolver({
|
|
2183
|
+
workspaceId: ids.workspaceId,
|
|
2184
|
+
defaultBackend: established.session,
|
|
2185
|
+
defaultKind: established.backendId,
|
|
2186
|
+
getSandbox: async (sandboxId) => {
|
|
2187
|
+
const sandbox = await getSandbox(
|
|
2188
|
+
db,
|
|
2189
|
+
ids.resourceAccountId && ids.resourceSubjectId ? {
|
|
2190
|
+
accountId: ids.resourceAccountId,
|
|
2191
|
+
workspaceId: ids.workspaceId,
|
|
2192
|
+
subjectId: ids.resourceSubjectId
|
|
2193
|
+
} : ids.workspaceId,
|
|
2194
|
+
sandboxId
|
|
2195
|
+
);
|
|
2196
|
+
return sandbox ? {
|
|
2197
|
+
id: sandbox.id,
|
|
2198
|
+
workspaceId: sandbox.workspaceId,
|
|
2199
|
+
kind: sandbox.kind,
|
|
2200
|
+
name: sandbox.name,
|
|
2201
|
+
...sandbox.scope ? { scope: sandbox.scope } : {},
|
|
2202
|
+
enrollmentId: sandbox.enrollmentId
|
|
2203
|
+
} : null;
|
|
2204
|
+
},
|
|
2205
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2206
|
+
resolveSelfhostedConnection: async (sandbox) => {
|
|
2207
|
+
if (!sandbox.enrollmentId) return null;
|
|
2208
|
+
const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
|
|
2209
|
+
return connectionBindingFor(services, enrollment);
|
|
2210
|
+
},
|
|
2211
|
+
relay: relayConfigFromSettings(settings),
|
|
2212
|
+
// A selfhosted swap target runs real commands too, so give it the same split
|
|
2213
|
+
// deadlines the machine-primary establish path uses (short control, long exec).
|
|
2214
|
+
...selfhostedResolverTimeouts(settings),
|
|
2215
|
+
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
2216
|
+
...backgroundCommandAccountId && backgroundCommandAttempt ? {
|
|
2217
|
+
selfhostedAdoptBackgroundCommand: async (command) => {
|
|
2218
|
+
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
2219
|
+
accountId: backgroundCommandAccountId,
|
|
2220
|
+
workspaceId: ids.workspaceId,
|
|
2221
|
+
sessionId: ids.sessionId,
|
|
2222
|
+
turnId: backgroundCommandAttempt.turnId,
|
|
2223
|
+
executionGeneration: backgroundCommandAttempt.executionGeneration,
|
|
2224
|
+
attemptId: backgroundCommandAttempt.attemptId,
|
|
2225
|
+
commandId: crypto.randomUUID(),
|
|
2226
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2227
|
+
enrollmentId: command.enrollmentId,
|
|
2228
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2229
|
+
opId: command.opId,
|
|
2230
|
+
command: command.command
|
|
2231
|
+
});
|
|
2232
|
+
return { commandId: adopted.id };
|
|
2233
|
+
},
|
|
2234
|
+
selfhostedSettleBackgroundCommand: async (command) => {
|
|
2235
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
2236
|
+
accountId: backgroundCommandAccountId,
|
|
2237
|
+
workspaceId: ids.workspaceId,
|
|
2238
|
+
sessionId: ids.sessionId,
|
|
2239
|
+
commandId: command.commandId,
|
|
2240
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2241
|
+
enrollmentId: command.enrollmentId,
|
|
2242
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2243
|
+
opId: command.opId,
|
|
2244
|
+
outcome: command.outcome,
|
|
2245
|
+
exitCode: command.exitCode,
|
|
2246
|
+
reason: command.reason,
|
|
2247
|
+
...command.failure ? { failure: command.failure } : {}
|
|
2248
|
+
});
|
|
2249
|
+
if (settlement && bus) {
|
|
2250
|
+
await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
|
|
2251
|
+
}
|
|
2252
|
+
},
|
|
2253
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
2254
|
+
db,
|
|
2255
|
+
{
|
|
2256
|
+
accountId: backgroundCommandAccountId,
|
|
2257
|
+
workspaceId: ids.workspaceId,
|
|
2258
|
+
sessionId: ids.sessionId
|
|
2259
|
+
},
|
|
2260
|
+
bus
|
|
2261
|
+
)
|
|
2262
|
+
} : {},
|
|
2263
|
+
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
2264
|
+
// SDK's per-turn manifest-env delta is empty (no "cannot change manifest
|
|
2265
|
+
// environment variables" throw when the turn pins to a vm). Mirrors the group
|
|
2266
|
+
// box, which is created WITH this same environment (resumeBoxForTurn).
|
|
2267
|
+
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
2268
|
+
...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {},
|
|
2269
|
+
// Stage D machine-primary: pin THIS established SelfhostedSession for the machine
|
|
2270
|
+
// pointer so the resolver returns the SAME instance (no two-instance manifest
|
|
2271
|
+
// divergence). `established.session` is the SelfhostedSession the establish branch
|
|
2272
|
+
// bound to (sandboxId, epoch).
|
|
2273
|
+
...ids.pinnedSelfhosted ? {
|
|
2274
|
+
pinnedSelfhosted: {
|
|
2275
|
+
sandboxId: ids.pinnedSelfhosted.sandboxId,
|
|
2276
|
+
epoch: ids.pinnedSelfhosted.epoch,
|
|
2277
|
+
session: established.session
|
|
2278
|
+
}
|
|
2279
|
+
} : {},
|
|
2280
|
+
// A modal swap target in the turn path would need its own lease resume-by-id;
|
|
2281
|
+
// that is a future cross-group-box concern. Until then a modal swap target is
|
|
2282
|
+
// unresolvable (the swap tool validates liveness, so this only triggers if a
|
|
2283
|
+
// session points at a sibling modal box the turn cannot resume here) and the
|
|
2284
|
+
// op surfaces unresolvable — never a silent wrong-box landing.
|
|
2285
|
+
//
|
|
2286
|
+
// For a machine-primary turn of a Modal-HOME session (pinned to a machine, no
|
|
2287
|
+
// group box established this turn), a mid-turn clear-to-null must NOT fall back to
|
|
2288
|
+
// the pinned machine — passing defaultIsHome:false makes the null branch throw typed
|
|
2289
|
+
// `home_unavailable_this_turn` instead. Forward the explicit boolean (including false).
|
|
2290
|
+
...ids.defaultIsHome !== void 0 ? { defaultIsHome: ids.defaultIsHome } : {},
|
|
2291
|
+
...ids.homeLease ? {
|
|
2292
|
+
resolveDefaultBackend: () => resolveCurrentHomeBackend(
|
|
2293
|
+
services,
|
|
2294
|
+
{
|
|
2295
|
+
workspaceId: ids.workspaceId,
|
|
2296
|
+
sessionId: ids.sessionId,
|
|
2297
|
+
accountId: ids.homeLease.accountId,
|
|
2298
|
+
sandboxGroupId: ids.homeLease.sandboxGroupId,
|
|
2299
|
+
backend: ids.homeLease.backend
|
|
2300
|
+
},
|
|
2301
|
+
established
|
|
2302
|
+
)
|
|
2303
|
+
} : {}
|
|
2304
|
+
});
|
|
2305
|
+
const proxy = new RoutingSandboxSession({
|
|
2306
|
+
// Seed the DEFAULT backend (the established group box) at construction so
|
|
2307
|
+
// `session.state` is the real backend's state object BEFORE the first op. The
|
|
2308
|
+
// SDK reads `session.state.manifest` at turn START (and writes it back); an
|
|
2309
|
+
// empty `{}` there crashes serializeManifestEnvironment /
|
|
2310
|
+
// validateProvidedSessionManifestUpdate. This is byte-identical to what the
|
|
2311
|
+
// resolver returns for the default pointer (`activeSandboxId === null`).
|
|
2312
|
+
defaultResolved: {
|
|
2313
|
+
session: established.session,
|
|
2314
|
+
sandboxId: null,
|
|
2315
|
+
kind: established.backendId,
|
|
2316
|
+
...ids.homeLease ? {
|
|
2317
|
+
leaseEpoch: ids.homeLease.leaseEpoch,
|
|
2318
|
+
providerInstanceId: ids.homeLease.instanceId
|
|
2319
|
+
} : {}
|
|
2320
|
+
},
|
|
2321
|
+
readPointer: async () => {
|
|
2322
|
+
if (!routingEnabled(settings)) {
|
|
2323
|
+
return { activeSandboxId: null, activeEpoch: 0 };
|
|
2324
|
+
}
|
|
2325
|
+
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
2326
|
+
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
2327
|
+
},
|
|
2328
|
+
resolveActiveBackend: resolver,
|
|
2329
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
2330
|
+
...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
|
|
2331
|
+
providerCommandHandle: admittedCommandHandle,
|
|
2332
|
+
providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
|
|
2333
|
+
...ids.workspaceMutationFence ? {
|
|
2334
|
+
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
2335
|
+
db,
|
|
2336
|
+
{
|
|
2337
|
+
accountId: ids.workspaceMutationFence.accountId,
|
|
2338
|
+
workspaceId: ids.workspaceId,
|
|
2339
|
+
sessionId: ids.sessionId,
|
|
2340
|
+
processId: process2.id
|
|
2341
|
+
},
|
|
2342
|
+
services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
|
|
2343
|
+
)
|
|
2344
|
+
} : {},
|
|
2345
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
2346
|
+
...afterMutation ? { afterMutation } : {},
|
|
2347
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
2348
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
2349
|
+
...settleProcess ? { settleProcess } : {},
|
|
2350
|
+
...isProcessSettled ? { isProcessSettled } : {},
|
|
2351
|
+
...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
|
|
2352
|
+
...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
|
|
2353
|
+
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
2354
|
+
...ids.homeLease ? {
|
|
2355
|
+
onDefaultBackendError: async ({
|
|
2356
|
+
error,
|
|
2357
|
+
backend
|
|
2358
|
+
}) => {
|
|
2359
|
+
const home = ids.homeLease;
|
|
2360
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
2361
|
+
const expectedEpoch = backend.leaseEpoch ?? home.leaseEpoch;
|
|
2362
|
+
const expectedInstanceId = backend.providerInstanceId ?? home.instanceId;
|
|
2363
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
2364
|
+
accountId: home.accountId,
|
|
2365
|
+
workspaceId: ids.workspaceId,
|
|
2366
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2367
|
+
expectedEpoch,
|
|
2368
|
+
expectedInstanceId,
|
|
2369
|
+
expectedBackend: home.backend,
|
|
2370
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
2371
|
+
});
|
|
2372
|
+
if (marked.status === "marked") {
|
|
2373
|
+
await publishDurableSessionEvents(
|
|
2374
|
+
services.bus,
|
|
2375
|
+
ids.workspaceId,
|
|
2376
|
+
marked.backgroundCommandEvents
|
|
2377
|
+
);
|
|
2378
|
+
await services.onHomeSandboxLost?.({
|
|
2379
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2380
|
+
instanceId: expectedInstanceId,
|
|
2381
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2382
|
+
});
|
|
2383
|
+
}
|
|
2384
|
+
const lease = marked.lease;
|
|
2385
|
+
const restore = lease?.recovery.restore.status;
|
|
2386
|
+
return {
|
|
2387
|
+
leaseEpoch: lease?.leaseEpoch ?? expectedEpoch,
|
|
2388
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
2389
|
+
};
|
|
2390
|
+
}
|
|
2391
|
+
} : {}
|
|
2392
|
+
});
|
|
2393
|
+
return { ...established, session: proxy };
|
|
2394
|
+
}
|
|
2395
|
+
function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
2396
|
+
const { db, settings, bus, onOp } = services;
|
|
2397
|
+
const backgroundCommandAccountId = ids.workspaceMutationFence?.accountId;
|
|
2398
|
+
const backgroundCommandAttempt = ids.workspaceMutationFence;
|
|
2399
|
+
const beforeMutation = beforePersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
2400
|
+
const afterMutation = afterPersistableHomeMutation(services, ids, args.homeLeaseIdentity);
|
|
2401
|
+
const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
|
|
2402
|
+
const afterProcessMutation = afterRetainedProcessMutation(services, ids);
|
|
2403
|
+
const settleProcess = settleRetainedProcessForTurn(services, ids);
|
|
2404
|
+
const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
|
|
2405
|
+
const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
|
|
2406
|
+
services,
|
|
2407
|
+
ids
|
|
2408
|
+
);
|
|
2409
|
+
const defaultBackend = sandboxBackendForSdkBackendId(args.backendId);
|
|
2410
|
+
const defaultSupportsPty = defaultBackend ? selectBackend(defaultBackend).capabilities.Terminal.pty : false;
|
|
2411
|
+
const syntheticSession = {
|
|
2412
|
+
state: { manifest: args.agentDefaultManifest },
|
|
2413
|
+
// The SDK binds shell tools synchronously before lazy provisioning. Preserve
|
|
2414
|
+
// the authoritative descriptor's PTY shape so a Modal home exposes
|
|
2415
|
+
// write_stdin on its first turn operation; the async cancellation transport
|
|
2416
|
+
// below still resolves the exact live route before executing a command.
|
|
2417
|
+
supportsPty: () => defaultSupportsPty
|
|
2418
|
+
};
|
|
2419
|
+
const routedResolver = makeActiveBackendResolver({
|
|
2420
|
+
workspaceId: ids.workspaceId,
|
|
2421
|
+
defaultBackend: syntheticSession,
|
|
2422
|
+
defaultKind: "unprovisioned",
|
|
2423
|
+
getSandbox: async (sandboxId) => {
|
|
2424
|
+
const sandbox = await getSandbox(
|
|
2425
|
+
db,
|
|
2426
|
+
ids.resourceAccountId && ids.resourceSubjectId ? {
|
|
2427
|
+
accountId: ids.resourceAccountId,
|
|
2428
|
+
workspaceId: ids.workspaceId,
|
|
2429
|
+
subjectId: ids.resourceSubjectId
|
|
2430
|
+
} : ids.workspaceId,
|
|
2431
|
+
sandboxId
|
|
2432
|
+
);
|
|
2433
|
+
return sandbox ? {
|
|
2434
|
+
id: sandbox.id,
|
|
2435
|
+
workspaceId: sandbox.workspaceId,
|
|
2436
|
+
kind: sandbox.kind,
|
|
2437
|
+
name: sandbox.name,
|
|
2438
|
+
...sandbox.scope ? { scope: sandbox.scope } : {},
|
|
2439
|
+
enrollmentId: sandbox.enrollmentId
|
|
2440
|
+
} : null;
|
|
2441
|
+
},
|
|
2442
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2443
|
+
resolveSelfhostedConnection: async (sandbox) => {
|
|
2444
|
+
if (!sandbox.enrollmentId) return null;
|
|
2445
|
+
const enrollment = await resolveRoutedMachineConnection(db, ids, sandbox);
|
|
2446
|
+
return connectionBindingFor(services, enrollment);
|
|
2447
|
+
},
|
|
2448
|
+
relay: relayConfigFromSettings(settings),
|
|
2449
|
+
...selfhostedResolverTimeouts(settings),
|
|
2450
|
+
...onOp !== void 0 ? { selfhostedOnOp: onOp } : {},
|
|
2451
|
+
...backgroundCommandAccountId && backgroundCommandAttempt ? {
|
|
2452
|
+
selfhostedAdoptBackgroundCommand: async (command) => {
|
|
2453
|
+
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
2454
|
+
accountId: backgroundCommandAccountId,
|
|
2455
|
+
workspaceId: ids.workspaceId,
|
|
2456
|
+
sessionId: ids.sessionId,
|
|
2457
|
+
turnId: backgroundCommandAttempt.turnId,
|
|
2458
|
+
executionGeneration: backgroundCommandAttempt.executionGeneration,
|
|
2459
|
+
attemptId: backgroundCommandAttempt.attemptId,
|
|
2460
|
+
commandId: crypto.randomUUID(),
|
|
2461
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2462
|
+
enrollmentId: command.enrollmentId,
|
|
2463
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2464
|
+
opId: command.opId,
|
|
2465
|
+
command: command.command
|
|
2466
|
+
});
|
|
2467
|
+
return { commandId: adopted.id };
|
|
2468
|
+
},
|
|
2469
|
+
selfhostedSettleBackgroundCommand: async (command) => {
|
|
2470
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
2471
|
+
accountId: backgroundCommandAccountId,
|
|
2472
|
+
workspaceId: ids.workspaceId,
|
|
2473
|
+
sessionId: ids.sessionId,
|
|
2474
|
+
commandId: command.commandId,
|
|
2475
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2476
|
+
enrollmentId: command.enrollmentId,
|
|
2477
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2478
|
+
opId: command.opId,
|
|
2479
|
+
outcome: command.outcome,
|
|
2480
|
+
exitCode: command.exitCode,
|
|
2481
|
+
reason: command.reason,
|
|
2482
|
+
...command.failure ? { failure: command.failure } : {}
|
|
2483
|
+
});
|
|
2484
|
+
if (settlement && bus) {
|
|
2485
|
+
await bus.publish(ids.workspaceId, ids.sessionId, settlement.events).catch(() => void 0);
|
|
2486
|
+
}
|
|
2487
|
+
},
|
|
2488
|
+
selfhostedCaptureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
2489
|
+
db,
|
|
2490
|
+
{
|
|
2491
|
+
accountId: backgroundCommandAccountId,
|
|
2492
|
+
workspaceId: ids.workspaceId,
|
|
2493
|
+
sessionId: ids.sessionId
|
|
2494
|
+
},
|
|
2495
|
+
bus
|
|
2496
|
+
)
|
|
2497
|
+
} : {},
|
|
2498
|
+
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
2499
|
+
...ids.transientExecEnvironment !== void 0 ? { transientExecEnvironment: ids.transientExecEnvironment } : {}
|
|
2500
|
+
});
|
|
2501
|
+
const proxy = new RoutingSandboxSession({
|
|
2502
|
+
// Before the first op the SDK reads `state.manifest`; the synthetic backend
|
|
2503
|
+
// points at agent.defaultManifest BY REFERENCE so the provided-session delta is
|
|
2504
|
+
// empty. The first default-pointer op resolves the real box through the
|
|
2505
|
+
// provisioner and `state` switches to that real backend by reference.
|
|
2506
|
+
defaultResolved: {
|
|
2507
|
+
session: syntheticSession,
|
|
2508
|
+
sandboxId: null,
|
|
2509
|
+
kind: "unprovisioned"
|
|
2510
|
+
},
|
|
2511
|
+
readPointer: async () => {
|
|
2512
|
+
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
2513
|
+
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
2514
|
+
},
|
|
2515
|
+
resolveActiveBackend: async (pointer) => {
|
|
2516
|
+
if (pointer.activeSandboxId === null || !routingEnabled(settings)) {
|
|
2517
|
+
const provisioned = await args.provisioner.get();
|
|
2518
|
+
if (args.homeLeaseIdentity && provisioned.leaseEpoch !== void 0) {
|
|
2519
|
+
return resolveCurrentHomeBackend(
|
|
2520
|
+
services,
|
|
2521
|
+
{
|
|
2522
|
+
workspaceId: ids.workspaceId,
|
|
2523
|
+
sessionId: ids.sessionId,
|
|
2524
|
+
accountId: args.homeLeaseIdentity.accountId,
|
|
2525
|
+
sandboxGroupId: args.homeLeaseIdentity.sandboxGroupId,
|
|
2526
|
+
backend: args.homeLeaseIdentity.backend
|
|
2527
|
+
},
|
|
2528
|
+
provisioned.established
|
|
2529
|
+
);
|
|
2530
|
+
}
|
|
2531
|
+
return {
|
|
2532
|
+
session: provisioned.established.session,
|
|
2533
|
+
sandboxId: null,
|
|
2534
|
+
kind: provisioned.established.backendId
|
|
2535
|
+
};
|
|
2536
|
+
}
|
|
2537
|
+
return routedResolver(pointer);
|
|
2538
|
+
},
|
|
2539
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
2540
|
+
...services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {},
|
|
2541
|
+
...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
|
|
2542
|
+
providerCommandHandle: admittedCommandHandle,
|
|
2543
|
+
providerSupervisionReady: async () => settings.modalCommandSupervisionEnabled && await supervisedCommandProtocolReady(db),
|
|
2544
|
+
...ids.workspaceMutationFence ? {
|
|
2545
|
+
providerCommandPersistence: (process2) => retainedProviderCommandPersistence(
|
|
2546
|
+
db,
|
|
2547
|
+
{
|
|
2548
|
+
accountId: ids.workspaceMutationFence.accountId,
|
|
2549
|
+
workspaceId: ids.workspaceId,
|
|
2550
|
+
sessionId: ids.sessionId,
|
|
2551
|
+
processId: process2.id
|
|
2552
|
+
},
|
|
2553
|
+
services.bus ? (events) => services.bus.publish(ids.workspaceId, ids.sessionId, events) : void 0
|
|
2554
|
+
)
|
|
2555
|
+
} : {},
|
|
2556
|
+
...beforeMutation ? { beforeMutation } : {},
|
|
2557
|
+
...afterMutation ? { afterMutation } : {},
|
|
2558
|
+
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
2559
|
+
...afterProcessMutation ? { afterProcessMutation } : {},
|
|
2560
|
+
...settleProcess ? { settleProcess } : {},
|
|
2561
|
+
...isProcessSettled ? { isProcessSettled } : {},
|
|
2562
|
+
...ids.workspaceMutationFence ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids) } : {},
|
|
2563
|
+
...ids.workspaceMutationFence ? { captureProcessOutput: captureRetainedProcessOutputForTurn(services, ids) } : {},
|
|
2564
|
+
...adoptProcessAsBackgroundCommand ? { adoptProcessAsBackgroundCommand } : {},
|
|
2565
|
+
...args.homeLeaseIdentity ? {
|
|
2566
|
+
onDefaultBackendError: async ({
|
|
2567
|
+
error,
|
|
2568
|
+
backend
|
|
2569
|
+
}) => {
|
|
2570
|
+
const home = args.homeLeaseIdentity;
|
|
2571
|
+
if (!isProviderSandboxGoneDuringRoutedOperation(home.backend, error)) return null;
|
|
2572
|
+
if (backend.leaseEpoch === void 0 || backend.providerInstanceId === void 0) {
|
|
2573
|
+
return null;
|
|
2574
|
+
}
|
|
2575
|
+
const marked = await markWarmLeaseInstanceLost2(db, {
|
|
2576
|
+
accountId: home.accountId,
|
|
2577
|
+
workspaceId: ids.workspaceId,
|
|
2578
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2579
|
+
expectedEpoch: backend.leaseEpoch,
|
|
2580
|
+
expectedInstanceId: backend.providerInstanceId,
|
|
2581
|
+
expectedBackend: home.backend,
|
|
2582
|
+
diagnostic: "provider_not_found_during_routed_operation"
|
|
2583
|
+
});
|
|
2584
|
+
if (marked.status === "marked") {
|
|
2585
|
+
await publishDurableSessionEvents(
|
|
2586
|
+
services.bus,
|
|
2587
|
+
ids.workspaceId,
|
|
2588
|
+
marked.backgroundCommandEvents
|
|
2589
|
+
);
|
|
2590
|
+
await services.onHomeSandboxLost?.({
|
|
2591
|
+
sandboxGroupId: home.sandboxGroupId,
|
|
2592
|
+
instanceId: backend.providerInstanceId,
|
|
2593
|
+
leaseEpoch: marked.lease.leaseEpoch
|
|
2594
|
+
});
|
|
2595
|
+
}
|
|
2596
|
+
const lease = marked.lease;
|
|
2597
|
+
const restore = lease?.recovery.restore.status;
|
|
2598
|
+
return {
|
|
2599
|
+
leaseEpoch: lease?.leaseEpoch ?? backend.leaseEpoch,
|
|
2600
|
+
recovery: marked.status === "stale" ? "superseded" : restore === "pending" ? "pending" : restore === "degraded" ? "degraded" : "unrecoverable"
|
|
2601
|
+
};
|
|
2602
|
+
}
|
|
2603
|
+
} : {}
|
|
2604
|
+
});
|
|
2605
|
+
return {
|
|
2606
|
+
client: args.client,
|
|
2607
|
+
session: proxy,
|
|
2608
|
+
sessionState: void 0,
|
|
2609
|
+
instanceId: "unprovisioned",
|
|
2610
|
+
backendId: args.backendId
|
|
2611
|
+
};
|
|
2612
|
+
}
|
|
2613
|
+
function opStreamDepsFor(services, machineAdvertisesOpStream) {
|
|
2614
|
+
const { settings, bus, opJournal } = services;
|
|
2615
|
+
if (!machineAdvertisesOpStream || settings.agentOpStreamEnabled !== true || !bus) {
|
|
2616
|
+
return void 0;
|
|
2617
|
+
}
|
|
2618
|
+
return {
|
|
2619
|
+
transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
|
|
2620
|
+
...opJournal !== void 0 ? { journal: opJournal } : {}
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
function connectionBindingFor(services, enrollment) {
|
|
2624
|
+
if (!enrollment?.connectionInstanceId) return null;
|
|
2625
|
+
const opStream = opStreamDepsFor(services, enrollment.opStream === true);
|
|
2626
|
+
return {
|
|
2627
|
+
workspaceId: enrollment.workspaceId,
|
|
2628
|
+
connectionInstanceId: enrollment.connectionInstanceId,
|
|
2629
|
+
workspaceRoot: enrollment.workspaceRoot,
|
|
2630
|
+
...opStream !== void 0 ? { opStream } : {},
|
|
2631
|
+
operationResourcePolicy: enrollment.operationPolicy,
|
|
2632
|
+
operationResourcePolicySupported: enrollment.agentCapabilities.operationResourcePolicy === true,
|
|
2633
|
+
operationCpuQuotaSupported: enrollment.agentCapabilities.operationCpuQuota === true,
|
|
2634
|
+
transactionalFsWriteSupported: enrollment.agentCapabilities.transactionalFsWrite === true
|
|
2635
|
+
};
|
|
2636
|
+
}
|
|
2637
|
+
async function resolveRoutedMachineConnection(db, ids, sandbox) {
|
|
2638
|
+
if (!sandbox.enrollmentId) return null;
|
|
2639
|
+
const current = await getLiveEnrollmentConnection(db, sandbox.workspaceId, sandbox.enrollmentId);
|
|
2640
|
+
if (!current) return null;
|
|
2641
|
+
if (current.scope === "user") {
|
|
2642
|
+
const authority = ids.personalMachineAttempt;
|
|
2643
|
+
if (!authority) return null;
|
|
2644
|
+
return await resolvePersonalMachineConnectionForAttempt(db, {
|
|
2645
|
+
accountId: authority.accountId,
|
|
2646
|
+
workspaceId: ids.workspaceId,
|
|
2647
|
+
subjectId: authority.subjectId,
|
|
2648
|
+
sessionId: ids.sessionId,
|
|
2649
|
+
turnId: authority.turnId,
|
|
2650
|
+
attemptId: authority.attemptId,
|
|
2651
|
+
executionGeneration: authority.executionGeneration,
|
|
2652
|
+
enrollmentId: sandbox.enrollmentId,
|
|
2653
|
+
requireActiveSandbox: true
|
|
2654
|
+
});
|
|
2655
|
+
}
|
|
2656
|
+
if (!ids.resourceAccountId || !ids.resourceSubjectId) return current;
|
|
2657
|
+
return await getLiveEnrollmentConnection(
|
|
2658
|
+
db,
|
|
2659
|
+
{
|
|
2660
|
+
accountId: ids.resourceAccountId,
|
|
2661
|
+
workspaceId: ids.workspaceId,
|
|
2662
|
+
subjectId: ids.resourceSubjectId
|
|
2663
|
+
},
|
|
2664
|
+
sandbox.enrollmentId
|
|
2665
|
+
);
|
|
2666
|
+
}
|
|
2667
|
+
async function establishSelfhostedTurnSession(services, args) {
|
|
2668
|
+
const { db, settings, bus, onOp } = services;
|
|
2669
|
+
const { timeoutMs, execTimeoutMs } = selfhostedTimeoutsFromSettings(settings);
|
|
2670
|
+
const opStream = opStreamDepsFor(services, args.opStream);
|
|
2671
|
+
const { client, session } = await buildSelfhostedBackendSession({
|
|
2672
|
+
workspaceId: args.workspaceId,
|
|
2673
|
+
...args.controlWorkspaceId !== void 0 ? { controlWorkspaceId: args.controlWorkspaceId } : {},
|
|
2674
|
+
agentId: args.agentId,
|
|
2675
|
+
connectionInstanceId: args.connectionInstanceId,
|
|
2676
|
+
workspaceRoot: args.workspaceRoot,
|
|
2677
|
+
relay: relayConfigFromSettings(settings),
|
|
2678
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
2679
|
+
epoch: args.epoch,
|
|
2680
|
+
environment: args.environment,
|
|
2681
|
+
...args.transientExecEnvironment !== void 0 ? { transientExecEnvironment: args.transientExecEnvironment } : {},
|
|
2682
|
+
// Give this turn's exec ops the long deadline (control ops stay short) so a real
|
|
2683
|
+
// command is not killed at the control wall.
|
|
2684
|
+
timeoutMs,
|
|
2685
|
+
execTimeoutMs,
|
|
2686
|
+
operationResourcePolicy: args.operationResourcePolicy,
|
|
2687
|
+
operationResourcePolicySupported: args.operationResourcePolicySupported,
|
|
2688
|
+
operationCpuQuotaSupported: args.operationCpuQuotaSupported,
|
|
2689
|
+
transactionalFsWriteSupported: args.transactionalFsWriteSupported === true,
|
|
2690
|
+
resolveOperationAdmission: async () => {
|
|
2691
|
+
const current = await getLiveEnrollmentConnection(
|
|
2692
|
+
db,
|
|
2693
|
+
args.controlWorkspaceId ?? args.workspaceId,
|
|
2694
|
+
args.agentId
|
|
2695
|
+
);
|
|
2696
|
+
const enrollment = current?.scope !== "user" ? current : args.personalMachineAttempt ? await resolvePersonalMachineConnectionForAttempt(db, {
|
|
2697
|
+
accountId: args.personalMachineAttempt.accountId,
|
|
2698
|
+
workspaceId: args.workspaceId,
|
|
2699
|
+
subjectId: args.personalMachineAttempt.subjectId,
|
|
2700
|
+
sessionId: args.personalMachineAttempt.sessionId,
|
|
2701
|
+
turnId: args.personalMachineAttempt.turnId,
|
|
2702
|
+
attemptId: args.personalMachineAttempt.attemptId,
|
|
2703
|
+
executionGeneration: args.personalMachineAttempt.executionGeneration,
|
|
2704
|
+
enrollmentId: args.agentId,
|
|
2705
|
+
requireActiveSandbox: true
|
|
2706
|
+
}) : null;
|
|
2707
|
+
const binding = connectionBindingFor(services, enrollment);
|
|
2708
|
+
if (!binding) return null;
|
|
2709
|
+
if (!binding.workspaceRoot) {
|
|
2710
|
+
throw new ActiveBackendUnresolvableError(
|
|
2711
|
+
"workspace_root_unavailable",
|
|
2712
|
+
"Connected Machine no longer has a reported workspace root"
|
|
2713
|
+
);
|
|
2714
|
+
}
|
|
2715
|
+
return {
|
|
2716
|
+
...binding,
|
|
2717
|
+
workspaceRoot: resolveConnectedMachineWorkspaceRoot(binding.workspaceRoot, args.workingDir)
|
|
2718
|
+
};
|
|
2719
|
+
},
|
|
2720
|
+
...args.backgroundCommandAttempt ? {
|
|
2721
|
+
adoptBackgroundCommand: async (command) => {
|
|
2722
|
+
const adopted = await adoptConnectedMachineSessionBackgroundCommand(db, {
|
|
2723
|
+
accountId: args.accountId,
|
|
2724
|
+
workspaceId: args.workspaceId,
|
|
2725
|
+
sessionId: args.sessionId,
|
|
2726
|
+
turnId: args.backgroundCommandAttempt.turnId,
|
|
2727
|
+
executionGeneration: args.backgroundCommandAttempt.executionGeneration,
|
|
2728
|
+
attemptId: args.backgroundCommandAttempt.attemptId,
|
|
2729
|
+
commandId: crypto.randomUUID(),
|
|
2730
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2731
|
+
enrollmentId: command.enrollmentId,
|
|
2732
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2733
|
+
opId: command.opId,
|
|
2734
|
+
command: command.command
|
|
2735
|
+
});
|
|
2736
|
+
return { commandId: adopted.id };
|
|
2737
|
+
}
|
|
2738
|
+
} : {},
|
|
2739
|
+
captureBackgroundCommandOutput: captureConnectedCommandOutput(
|
|
2740
|
+
db,
|
|
2741
|
+
{
|
|
2742
|
+
accountId: args.accountId,
|
|
2743
|
+
workspaceId: args.workspaceId,
|
|
2744
|
+
sessionId: args.sessionId
|
|
2745
|
+
},
|
|
2746
|
+
bus
|
|
2747
|
+
),
|
|
2748
|
+
settleBackgroundCommand: async (command) => {
|
|
2749
|
+
const settlement = await settleConnectedMachineSessionBackgroundCommand(db, {
|
|
2750
|
+
accountId: args.accountId,
|
|
2751
|
+
workspaceId: args.workspaceId,
|
|
2752
|
+
sessionId: args.sessionId,
|
|
2753
|
+
commandId: command.commandId,
|
|
2754
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2755
|
+
enrollmentId: command.enrollmentId,
|
|
2756
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2757
|
+
opId: command.opId,
|
|
2758
|
+
outcome: command.outcome,
|
|
2759
|
+
exitCode: command.exitCode,
|
|
2760
|
+
reason: command.reason,
|
|
2761
|
+
...command.failure ? { failure: command.failure } : {}
|
|
2762
|
+
});
|
|
2763
|
+
if (command.outputReceipt && command.outcome === "exited" && command.exitCode !== null) {
|
|
2764
|
+
await recordConnectedCommandOutputConsumption(db, {
|
|
2765
|
+
accountId: args.accountId,
|
|
2766
|
+
workspaceId: args.workspaceId,
|
|
2767
|
+
sessionId: args.sessionId,
|
|
2768
|
+
commandId: command.commandId,
|
|
2769
|
+
controlWorkspaceId: command.controlWorkspaceId,
|
|
2770
|
+
enrollmentId: command.enrollmentId,
|
|
2771
|
+
connectionInstanceId: command.connectionInstanceId,
|
|
2772
|
+
opId: command.opId,
|
|
2773
|
+
receipt: command.outputReceipt,
|
|
2774
|
+
exitCode: command.exitCode
|
|
2775
|
+
});
|
|
2776
|
+
}
|
|
2777
|
+
if (settlement && bus) {
|
|
2778
|
+
await bus.publish(args.workspaceId, args.sessionId, settlement.events).catch(() => void 0);
|
|
2779
|
+
}
|
|
2780
|
+
},
|
|
2781
|
+
// Meter every control op (out-of-band telemetry) — no-op when unwired.
|
|
2782
|
+
...onOp !== void 0 ? { onOp } : {},
|
|
2783
|
+
// The streaming exec transport — present iff the machine advertised the
|
|
2784
|
+
// capability AND the server flag is on. Connected Machine exec fails closed
|
|
2785
|
+
// when it is absent; there is no request/reply downgrade.
|
|
2786
|
+
...opStream !== void 0 ? { opStream } : {}
|
|
2787
|
+
});
|
|
2788
|
+
return {
|
|
2789
|
+
client,
|
|
2790
|
+
session,
|
|
2791
|
+
sessionState: { agentId: args.agentId },
|
|
2792
|
+
instanceId: args.agentId,
|
|
2793
|
+
backendId: "selfhosted"
|
|
2794
|
+
};
|
|
2795
|
+
}
|
|
2796
|
+
function routingEnabled(settings) {
|
|
2797
|
+
return settings.sandboxSelfhostedEnabled === true;
|
|
2798
|
+
}
|
|
2799
|
+
function lazyProvisionEnabled(settings) {
|
|
2800
|
+
return settings.sandboxLazyProvisionEnabled === true && settings.sandboxOwnershipEnabled === true;
|
|
2801
|
+
}
|
|
2802
|
+
|
|
2803
|
+
export {
|
|
2804
|
+
warnDrainSnapshotFailure,
|
|
2805
|
+
publishDurableSessionEvents,
|
|
2806
|
+
persistWorkspaceArchiveCandidate,
|
|
2807
|
+
putVersion1TarArchiveOrInline,
|
|
2808
|
+
sandboxLeaseHolderIdForAttempt,
|
|
2809
|
+
SandboxProvisionStageError,
|
|
2810
|
+
sandboxProvisionFailureStage,
|
|
2811
|
+
SandboxWarmingTimeoutError,
|
|
2812
|
+
createFreshSandboxReadinessReplacementBudget,
|
|
2813
|
+
waitForWarmSnapshot,
|
|
2814
|
+
maybePersistWarmWorkspaceSnapshot,
|
|
2815
|
+
resumeBoxForTurn,
|
|
2816
|
+
captureConnectedCommandOutput,
|
|
2817
|
+
wrapTurnBoxWithRouting,
|
|
2818
|
+
wrapLazyTurnBoxWithRouting,
|
|
2819
|
+
establishSelfhostedTurnSession,
|
|
2820
|
+
routingEnabled,
|
|
2821
|
+
lazyProvisionEnabled
|
|
2822
|
+
};
|
|
2823
|
+
//# sourceMappingURL=chunk-NQDVJ6RI.js.map
|