@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
package/src/sandbox-resume.ts
CHANGED
|
@@ -20,10 +20,13 @@
|
|
|
20
20
|
//
|
|
21
21
|
// Liveness between turns is the lease refcount; there is no keepalive loop.
|
|
22
22
|
|
|
23
|
+
import type { EventBus } from "@opengeni/events";
|
|
24
|
+
import { publishDurableSessionEvents } from "./session-event-fanout";
|
|
23
25
|
import {
|
|
24
26
|
effectiveSandboxLifecycle,
|
|
25
27
|
sandboxArchiveCaptureTimeoutMs,
|
|
26
28
|
sandboxLifecycleTransitionWaitMs,
|
|
29
|
+
sandboxWarmRateMicrosPerSecond,
|
|
27
30
|
type Settings,
|
|
28
31
|
} from "@opengeni/config";
|
|
29
32
|
import { randomUUID } from "node:crypto";
|
|
@@ -32,10 +35,12 @@ import {
|
|
|
32
35
|
acquireLease,
|
|
33
36
|
adoptLegacyModalCheckpointArtifact,
|
|
34
37
|
beginSandboxRematerialization,
|
|
38
|
+
sessionHoldsFreshWorkspaceRecovery,
|
|
35
39
|
claimWorkspaceArchiveCapture,
|
|
36
40
|
commitWarmingToWarm,
|
|
37
41
|
failSandboxRematerialization,
|
|
38
42
|
failWarmingToCold,
|
|
43
|
+
beginModalProviderCreate,
|
|
39
44
|
getSandboxSessionEnvelope,
|
|
40
45
|
heartbeatLeaseHolderStatus,
|
|
41
46
|
markSandboxRestoreVerifying,
|
|
@@ -46,6 +51,7 @@ import {
|
|
|
46
51
|
registerSandboxCheckpointArtifact,
|
|
47
52
|
recordWarmingSandboxCreated,
|
|
48
53
|
reinstateTurnLeaseHolder,
|
|
54
|
+
leaseMayHaveUntrackedWriters,
|
|
49
55
|
releaseWorkspaceArchiveCapture,
|
|
50
56
|
releaseLeaseHolder,
|
|
51
57
|
touchLeaseHolder,
|
|
@@ -78,8 +84,11 @@ import {
|
|
|
78
84
|
terminateUnpublishedSandboxSession,
|
|
79
85
|
verifySandboxExecReadiness,
|
|
80
86
|
withoutSandboxProviderIdentity,
|
|
87
|
+
workspaceArchiveDownloadTemporaryDirectory,
|
|
81
88
|
type EstablishedSandboxSession,
|
|
82
89
|
type RuntimeMetricsHooks,
|
|
90
|
+
type WorkspaceCaptureSkipReason,
|
|
91
|
+
type SandboxReadinessReplacementOutcome,
|
|
83
92
|
type WorkspaceArchiveDescriptor,
|
|
84
93
|
} from "@opengeni/runtime";
|
|
85
94
|
import {
|
|
@@ -149,6 +158,29 @@ export type SandboxResumeServices = {
|
|
|
149
158
|
/** Test seam for the bounded command-readiness proof performed before an
|
|
150
159
|
* attached provider box is handed to the agent. */
|
|
151
160
|
verifyAttachedSandboxReadiness?: (established: EstablishedSandboxSession) => Promise<void>;
|
|
161
|
+
/** Test seam for the bounded command-readiness proof the elected spawner runs
|
|
162
|
+
* on its freshly created box before publishing the lease warm. Production
|
|
163
|
+
* uses {@link waitForSandboxExecReadiness} with the shared 60 s budget. */
|
|
164
|
+
verifySpawnedSandboxReadiness?: (
|
|
165
|
+
established: EstablishedSandboxSession,
|
|
166
|
+
identity: { sandboxGroupId: string },
|
|
167
|
+
) => Promise<void>;
|
|
168
|
+
/** Test seam for the jittered pause before the single fresh-box readiness
|
|
169
|
+
* replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
|
|
170
|
+
* It may be async so a test can observe the rolled-back lease in between. */
|
|
171
|
+
freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
|
|
172
|
+
/** Test seam: runs immediately after the elected spawner published its box
|
|
173
|
+
* warm, before the final cancellation check. */
|
|
174
|
+
onSpawnedSandboxPublished?: () => void | Promise<void>;
|
|
175
|
+
/**
|
|
176
|
+
* The turn attempt's fresh-box readiness replacement budget. The lazy
|
|
177
|
+
* provisioner may call resumeBoxForTurn again after a typed lease
|
|
178
|
+
* supersession, so the turn creates the budget once and shares it with every
|
|
179
|
+
* call. Absent, the call gets its own single replacement.
|
|
180
|
+
*/
|
|
181
|
+
freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
|
|
182
|
+
/** Live fanout for background commands a provider loss settled. */
|
|
183
|
+
bus?: EventBus | null;
|
|
152
184
|
/** Called only by the observer that wins the exact warm->cold loss CAS. */
|
|
153
185
|
onSandboxLost?: (input: {
|
|
154
186
|
sandboxGroupId: string;
|
|
@@ -188,6 +220,9 @@ export type ResumeBoxIds = {
|
|
|
188
220
|
* never conflicts).
|
|
189
221
|
*/
|
|
190
222
|
image?: string;
|
|
223
|
+
/** Deployment/workspace image pins apply only to new creates; an existing
|
|
224
|
+
* group keeps its image. Omission preserves explicit image matching (B3). */
|
|
225
|
+
imagePolicy?: "require_match" | "new_creates_only";
|
|
191
226
|
/**
|
|
192
227
|
* RIG IS SHARED STATE (M3): the frozen rig version this run rides. Threaded to
|
|
193
228
|
* acquireLease, which stamps it on the cold-create and conflicts on a live box
|
|
@@ -317,6 +352,80 @@ export class SandboxSiblingWarmingTimeoutError extends SandboxWarmingTimeoutErro
|
|
|
317
352
|
}
|
|
318
353
|
}
|
|
319
354
|
|
|
355
|
+
/**
|
|
356
|
+
* Out-of-band proof attached to one exact {@link SandboxExecReadinessTimeoutError}
|
|
357
|
+
* thrown by the elected spawner: the box that missed its readiness budget was
|
|
358
|
+
* freshly created by this caller (not an attached, resumed, or provider-continuity
|
|
359
|
+
* box), it was never published warm, the provider terminated it, and the holder
|
|
360
|
+
* was released. Only such an error may be replaced by re-entering admission.
|
|
361
|
+
*/
|
|
362
|
+
const disposedUnreadyFreshSandboxErrors = new WeakSet<object>();
|
|
363
|
+
|
|
364
|
+
/** True only for a spawner readiness timeout whose unready fresh box is proven gone. */
|
|
365
|
+
export function isReplaceableFreshSandboxReadinessTimeout(error: unknown): boolean {
|
|
366
|
+
return (
|
|
367
|
+
error instanceof SandboxExecReadinessTimeoutError &&
|
|
368
|
+
disposedUnreadyFreshSandboxErrors.has(error)
|
|
369
|
+
);
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/**
|
|
373
|
+
* A burst of simultaneous turn starts can leave many freshly created provider
|
|
374
|
+
* boxes command-unready at once. The single replacement re-enters admission
|
|
375
|
+
* after a jittered pause so the replacement creates do not re-synchronize into
|
|
376
|
+
* the same burst. Bounded to [2 s, 10 s).
|
|
377
|
+
*/
|
|
378
|
+
export const FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS = 2_000;
|
|
379
|
+
export const FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS = 8_000;
|
|
380
|
+
|
|
381
|
+
export function freshSandboxReadinessReplacementDelayMs(
|
|
382
|
+
random: () => number = Math.random,
|
|
383
|
+
): number {
|
|
384
|
+
const sample = random();
|
|
385
|
+
const unit = Number.isFinite(sample) ? Math.min(Math.max(sample, 0), 1 - Number.EPSILON) : 0;
|
|
386
|
+
return (
|
|
387
|
+
FRESH_SANDBOX_READINESS_REPLACEMENT_BASE_DELAY_MS +
|
|
388
|
+
Math.floor(unit * FRESH_SANDBOX_READINESS_REPLACEMENT_JITTER_MS)
|
|
389
|
+
);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/** Replacements allowed per turn attempt, across every resumeBoxForTurn call. */
|
|
393
|
+
export const FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT = 1;
|
|
394
|
+
|
|
395
|
+
export type FreshSandboxReadinessReplacementBudget = { remaining: number };
|
|
396
|
+
|
|
397
|
+
export function createFreshSandboxReadinessReplacementBudget(): FreshSandboxReadinessReplacementBudget {
|
|
398
|
+
return { remaining: FRESH_SANDBOX_READINESS_REPLACEMENTS_PER_TURN_ATTEMPT };
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
function recordFreshSandboxReadinessReplacement(
|
|
402
|
+
metrics: RuntimeMetricsHooks | undefined,
|
|
403
|
+
backend: string,
|
|
404
|
+
outcome: SandboxReadinessReplacementOutcome,
|
|
405
|
+
): void {
|
|
406
|
+
try {
|
|
407
|
+
metrics?.onSandboxReadinessReplacement?.({ backend, outcome });
|
|
408
|
+
} catch {
|
|
409
|
+
// Metrics emission must never affect sandbox recovery or error propagation.
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
async function sleepUnlessCancelled(ms: number, signal: AbortSignal | undefined): Promise<boolean> {
|
|
414
|
+
if (signal?.aborted) return false;
|
|
415
|
+
if (ms <= 0) return true;
|
|
416
|
+
return await new Promise<boolean>((resolve) => {
|
|
417
|
+
const onAbort = () => {
|
|
418
|
+
clearTimeout(timer);
|
|
419
|
+
resolve(false);
|
|
420
|
+
};
|
|
421
|
+
const timer = setTimeout(() => {
|
|
422
|
+
signal?.removeEventListener("abort", onAbort);
|
|
423
|
+
resolve(true);
|
|
424
|
+
}, ms);
|
|
425
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
|
|
320
429
|
/** The exact attached caller that won warm->cold after proving the provider
|
|
321
430
|
* instance gone. It is a lease supersession (the same logical turn recovers),
|
|
322
431
|
* plus the lost id needed for one durable observability event. */
|
|
@@ -335,10 +444,6 @@ export class SandboxLeaseInstanceLostError extends SandboxLeaseSupersededError {
|
|
|
335
444
|
// user-facing and separate from the lease TTL heartbeat/reaper horizon.
|
|
336
445
|
const WARMING_POLL_INTERVAL_MS = 250;
|
|
337
446
|
|
|
338
|
-
async function sleep(ms: number): Promise<void> {
|
|
339
|
-
await new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
340
|
-
}
|
|
341
|
-
|
|
342
447
|
/**
|
|
343
448
|
* A remote provider may return a sandbox handle before its command router
|
|
344
449
|
* accepts the first exec. The upstream session's yieldTimeMs starts only after
|
|
@@ -349,9 +454,10 @@ export async function waitForSandboxExecReadiness(
|
|
|
349
454
|
established: EstablishedSandboxSession,
|
|
350
455
|
timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
351
456
|
identity: { sandboxGroupId?: string | null } = {},
|
|
457
|
+
signal?: AbortSignal,
|
|
352
458
|
): Promise<void> {
|
|
353
459
|
try {
|
|
354
|
-
await verifySandboxExecReadiness(established, timeoutMs);
|
|
460
|
+
await verifySandboxExecReadiness(established, timeoutMs, signal);
|
|
355
461
|
} catch (error) {
|
|
356
462
|
if (error instanceof SandboxExecReadinessError && error.code === "exec_probe_timeout") {
|
|
357
463
|
throw new SandboxExecReadinessTimeoutError(established.backendId, timeoutMs, {
|
|
@@ -535,8 +641,9 @@ async function materializeSpawnEnvelopeArchive(
|
|
|
535
641
|
}
|
|
536
642
|
if (!objectStorage) {
|
|
537
643
|
throw new WorkspaceArchiveIntegrityError(
|
|
538
|
-
"
|
|
644
|
+
"archive_storage_unavailable",
|
|
539
645
|
"workspace archive object storage is not configured",
|
|
646
|
+
{ retryable: true },
|
|
540
647
|
);
|
|
541
648
|
}
|
|
542
649
|
const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
|
|
@@ -680,16 +787,23 @@ export type WarmWorkspaceSnapshotPromise = Promise<boolean> & {
|
|
|
680
787
|
readonly settled: Promise<void>;
|
|
681
788
|
};
|
|
682
789
|
|
|
790
|
+
/** Who owns a warm checkpoint: the exact turn attempt holding the box, or
|
|
791
|
+
* (`idleCheckpoint`) the reaper checkpointing a box that no turn holds and
|
|
792
|
+
* viewers, interactions or running background commands keep warm. The idle
|
|
793
|
+
* owner is fenced by its exact capture claim alone and is limited to
|
|
794
|
+
* point-in-time Modal captures. */
|
|
795
|
+
export type WarmWorkspaceSnapshotOwner = {
|
|
796
|
+
accountId: string;
|
|
797
|
+
workspaceId: string;
|
|
798
|
+
sandboxGroupId: string;
|
|
799
|
+
} & (
|
|
800
|
+
| { sessionId: string; turnId: string; attemptId: string; idleCheckpoint?: never }
|
|
801
|
+
| { idleCheckpoint: true; sessionId?: never; turnId?: never; attemptId?: never }
|
|
802
|
+
);
|
|
803
|
+
|
|
683
804
|
export function maybePersistWarmWorkspaceSnapshot(
|
|
684
805
|
services: SandboxResumeServices,
|
|
685
|
-
ids:
|
|
686
|
-
accountId: string;
|
|
687
|
-
workspaceId: string;
|
|
688
|
-
sessionId: string;
|
|
689
|
-
turnId: string;
|
|
690
|
-
attemptId: string;
|
|
691
|
-
sandboxGroupId: string;
|
|
692
|
-
},
|
|
806
|
+
ids: WarmWorkspaceSnapshotOwner,
|
|
693
807
|
session: unknown,
|
|
694
808
|
leaseEpoch: number,
|
|
695
809
|
signal?: AbortSignal,
|
|
@@ -725,14 +839,7 @@ export function maybePersistWarmWorkspaceSnapshot(
|
|
|
725
839
|
|
|
726
840
|
async function persistWarmWorkspaceSnapshot(
|
|
727
841
|
services: SandboxResumeServices,
|
|
728
|
-
ids:
|
|
729
|
-
accountId: string;
|
|
730
|
-
workspaceId: string;
|
|
731
|
-
sessionId: string;
|
|
732
|
-
turnId: string;
|
|
733
|
-
attemptId: string;
|
|
734
|
-
sandboxGroupId: string;
|
|
735
|
-
},
|
|
842
|
+
ids: WarmWorkspaceSnapshotOwner,
|
|
736
843
|
session: unknown,
|
|
737
844
|
leaseEpoch: number,
|
|
738
845
|
signal: AbortSignal | undefined,
|
|
@@ -753,6 +860,16 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
753
860
|
// Diagnostics cannot change provider settlement or capture-gate cleanup.
|
|
754
861
|
}
|
|
755
862
|
};
|
|
863
|
+
const reportSkipped = (backend: unknown, reason: WorkspaceCaptureSkipReason): void => {
|
|
864
|
+
try {
|
|
865
|
+
services.sandboxMetrics?.onWorkspaceCaptureSkipped?.({
|
|
866
|
+
backend: typeof backend === "string" ? backend : "unknown",
|
|
867
|
+
reason,
|
|
868
|
+
});
|
|
869
|
+
} catch {
|
|
870
|
+
// Telemetry never changes whether a checkpoint is attempted.
|
|
871
|
+
}
|
|
872
|
+
};
|
|
756
873
|
const intervalMs = settings.sandboxSnapshotIntervalMs;
|
|
757
874
|
if (intervalMs <= 0 && !force) {
|
|
758
875
|
return false;
|
|
@@ -773,12 +890,13 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
773
890
|
console.error("mid-session workspace snapshot skipped (no persist primitive)", {
|
|
774
891
|
backendId: typeof persistable.backendId === "string" ? persistable.backendId : null,
|
|
775
892
|
});
|
|
893
|
+
reportSkipped(persistable.backendId, "no_persist_primitive");
|
|
776
894
|
return false;
|
|
777
895
|
}
|
|
778
896
|
// Filesystem and directory snapshots create retained Images without
|
|
779
897
|
// terminating the source Sandbox. (Modal's termination warning applies to
|
|
780
898
|
// memory snapshots.) They are therefore the preferred warm-checkpoint path:
|
|
781
|
-
// the durable capture gate pauses
|
|
899
|
+
// the durable capture gate pauses Opengeni commands while the provider reads
|
|
782
900
|
// the filesystem, then the same live instance continues serving the turn.
|
|
783
901
|
const workspacePersistence =
|
|
784
902
|
persistable.state?.workspacePersistence ??
|
|
@@ -798,7 +916,8 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
798
916
|
lease?.liveness === "draining" &&
|
|
799
917
|
lease.instanceId !== null &&
|
|
800
918
|
lease.leaseEpoch === leaseEpoch;
|
|
801
|
-
|
|
919
|
+
const turnOwner = ids.idleCheckpoint === true ? null : ids;
|
|
920
|
+
if (!lease || (!canWarmCapture && (!canForceDrainingCapture || !turnOwner))) {
|
|
802
921
|
console.error("mid-session workspace snapshot skipped (lease not warm)", {
|
|
803
922
|
sandboxGroupId: ids.sandboxGroupId,
|
|
804
923
|
expectedEpoch: leaseEpoch,
|
|
@@ -806,12 +925,15 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
806
925
|
leaseEpoch: lease?.leaseEpoch ?? null,
|
|
807
926
|
hasInstance: lease?.instanceId !== null && lease?.instanceId !== undefined,
|
|
808
927
|
});
|
|
928
|
+
reportSkipped(lease?.backend ?? persistable.backendId, "lease_not_warm");
|
|
809
929
|
return false;
|
|
810
930
|
}
|
|
811
931
|
// A checkpoint of this exact mutation generation already protects every
|
|
812
932
|
// settled operation admitted so far. Capturing it again cannot improve the
|
|
813
|
-
// recovery point, even when the wall-clock interval has elapsed.
|
|
814
|
-
|
|
933
|
+
// recovery point, even when the wall-clock interval has elapsed. An
|
|
934
|
+
// attached viewer or interaction can write without advancing the
|
|
935
|
+
// generation; the claim then decides whether the box is really clean.
|
|
936
|
+
if (lease.archiveComplete && !leaseMayHaveUntrackedWriters(lease)) {
|
|
815
937
|
return false;
|
|
816
938
|
}
|
|
817
939
|
if (lease.instanceId === null) {
|
|
@@ -826,11 +948,18 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
826
948
|
backend: lease.backend,
|
|
827
949
|
liveInstance: capturePolicy?.liveInstance ?? null,
|
|
828
950
|
});
|
|
951
|
+
reportSkipped(lease.backend, "capture_policy");
|
|
829
952
|
return false;
|
|
830
953
|
}
|
|
831
954
|
const nativeModalPersistence =
|
|
832
955
|
workspacePersistence === "snapshot_filesystem" ||
|
|
833
956
|
workspacePersistence === "snapshot_directory";
|
|
957
|
+
if (!turnOwner && !nativeModalPersistence) {
|
|
958
|
+
// Only a paused-box image may run around the commands keeping an idle
|
|
959
|
+
// box warm; a file-by-file read of the running box could tear.
|
|
960
|
+
reportSkipped(lease.backend, "capture_policy");
|
|
961
|
+
return false;
|
|
962
|
+
}
|
|
834
963
|
const checkpointBinding = nativeModalPersistence
|
|
835
964
|
? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal)
|
|
836
965
|
: null;
|
|
@@ -850,25 +979,31 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
850
979
|
minIntervalMs: force ? 0 : intervalMs,
|
|
851
980
|
providerReplaySafe: capturePolicy.takeover === "same_request",
|
|
852
981
|
takeoverSafe: capturePolicy.takeover !== "exclusive",
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
982
|
+
// Modal pauses the box for a native snapshot, so the image is one
|
|
983
|
+
// instant even while background commands run.
|
|
984
|
+
pointInTimeCapture: nativeModalPersistence,
|
|
985
|
+
...(captureLiveness !== "warm"
|
|
986
|
+
? {}
|
|
987
|
+
: turnOwner
|
|
988
|
+
? {
|
|
989
|
+
warmAttempt: {
|
|
990
|
+
sessionId: turnOwner.sessionId,
|
|
991
|
+
turnId: turnOwner.turnId,
|
|
992
|
+
attemptId: turnOwner.attemptId,
|
|
993
|
+
holderId: sandboxLeaseHolderIdForAttempt(turnOwner.attemptId),
|
|
994
|
+
},
|
|
995
|
+
}
|
|
996
|
+
: { idleCheckpoint: true }),
|
|
863
997
|
});
|
|
864
998
|
if (claimed.status !== "claimed") {
|
|
865
999
|
// A scheduled interval not being due is normal, including after a failed
|
|
866
1000
|
// attempt. Do not turn every ten-second heartbeat into an error log.
|
|
867
|
-
if (claimed.status === "throttled") return false;
|
|
1001
|
+
if (claimed.status === "throttled" || claimed.status === "clean") return false;
|
|
868
1002
|
console.error("mid-session workspace snapshot skipped (capture claim)", {
|
|
869
1003
|
sandboxGroupId: ids.sandboxGroupId,
|
|
870
1004
|
status: claimed.status,
|
|
871
1005
|
});
|
|
1006
|
+
reportSkipped(lease.backend, claimed.status);
|
|
872
1007
|
return false;
|
|
873
1008
|
}
|
|
874
1009
|
|
|
@@ -915,10 +1050,20 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
915
1050
|
reason,
|
|
916
1051
|
}).catch(() => undefined);
|
|
917
1052
|
};
|
|
918
|
-
//
|
|
1053
|
+
// A provider-native capture is not cancellable. Keep one owned continuation
|
|
919
1054
|
// alive through provider settlement even if this caller's bounded wait or
|
|
920
1055
|
// turn signal resolves first. Its finally block is the only normal release
|
|
921
1056
|
// of the exact admission gate; a late callback cannot release a successor.
|
|
1057
|
+
// A host-backed spool capture reads the workspace locally, so it is
|
|
1058
|
+
// stopped at the snapshot timeout instead: its rejection is the physical
|
|
1059
|
+
// end of the reads, and holding the write fence for an unbounded local
|
|
1060
|
+
// read would turn a slow host into failed workspace writes.
|
|
1061
|
+
const captureAbort = new AbortController();
|
|
1062
|
+
const captureAbortTimer = setTimeout(
|
|
1063
|
+
() => captureAbort.abort(new SnapshotTimeoutError(settings.sandboxSnapshotTimeoutMs)),
|
|
1064
|
+
settings.sandboxSnapshotTimeoutMs,
|
|
1065
|
+
);
|
|
1066
|
+
captureAbortTimer.unref?.();
|
|
922
1067
|
const captureAndPublish = (async (): Promise<boolean> => {
|
|
923
1068
|
const captureStarted = performance.now();
|
|
924
1069
|
let captureOutcome: "completed" | "failed" = "failed";
|
|
@@ -932,9 +1077,11 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
932
1077
|
{
|
|
933
1078
|
requestId: claimed.claim.providerRequestId,
|
|
934
1079
|
strategy: capturePolicy.strategy,
|
|
1080
|
+
signal: captureAbort.signal,
|
|
935
1081
|
},
|
|
936
1082
|
Boolean(services.objectStorage),
|
|
937
1083
|
);
|
|
1084
|
+
clearTimeout(captureAbortTimer);
|
|
938
1085
|
candidate = await registerCandidate(archive);
|
|
939
1086
|
const archiveDescriptor = archive.descriptor;
|
|
940
1087
|
const published = await putVersion1TarArchiveOrInline({
|
|
@@ -954,9 +1101,13 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
954
1101
|
persistWarmSnapshot(db, {
|
|
955
1102
|
accountId: ids.accountId,
|
|
956
1103
|
workspaceId: ids.workspaceId,
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
1104
|
+
...(turnOwner
|
|
1105
|
+
? {
|
|
1106
|
+
sessionId: turnOwner.sessionId,
|
|
1107
|
+
turnId: turnOwner.turnId,
|
|
1108
|
+
attemptId: turnOwner.attemptId,
|
|
1109
|
+
}
|
|
1110
|
+
: { idleCheckpoint: true as const }),
|
|
960
1111
|
sandboxGroupId: ids.sandboxGroupId,
|
|
961
1112
|
expectedEpoch: leaseEpoch,
|
|
962
1113
|
expectedInstanceId: instanceId,
|
|
@@ -979,6 +1130,7 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
979
1130
|
await abandonCandidate(candidate.id, "snapshot_capture_failed");
|
|
980
1131
|
throw error;
|
|
981
1132
|
} finally {
|
|
1133
|
+
clearTimeout(captureAbortTimer);
|
|
982
1134
|
await disposeWorkspaceArchive(archive).catch(() => {
|
|
983
1135
|
console.warn("workspace archive spool cleanup failed");
|
|
984
1136
|
});
|
|
@@ -1056,12 +1208,103 @@ async function persistWarmWorkspaceSnapshot(
|
|
|
1056
1208
|
* holderId is the globally unique durable turn-attempt id. It must not be a
|
|
1057
1209
|
* Temporal activity id, because activity ids are only workflow-local and
|
|
1058
1210
|
* collide when sibling sessions share one sandbox group.
|
|
1211
|
+
*
|
|
1212
|
+
* A freshly created box that misses its bounded command-readiness budget is
|
|
1213
|
+
* replaced at most once per turn attempt (the budget is shared by every call
|
|
1214
|
+
* the turn's provisioner makes). The first establish has already terminated that exact
|
|
1215
|
+
* unpublished box, rolled its exact warming epoch back to cold (advancing the
|
|
1216
|
+
* epoch), and released its holder, so the replacement is an ordinary new
|
|
1217
|
+
* admission: it re-runs the cold->warming CAS, persists its own provider
|
|
1218
|
+
* instance before readiness, and may instead attach to a sibling that won the
|
|
1219
|
+
* CAS first. Nothing model- or tool-visible ran on the discarded box, so no
|
|
1220
|
+
* side effect is replayed. A second readiness timeout fails the turn exactly
|
|
1221
|
+
* as before; attached/resumed boxes and unconfirmed terminations never
|
|
1222
|
+
* replace.
|
|
1059
1223
|
*/
|
|
1060
1224
|
export async function resumeBoxForTurn(
|
|
1061
1225
|
services: SandboxResumeServices,
|
|
1062
1226
|
ids: ResumeBoxIds,
|
|
1063
1227
|
kind: "turn",
|
|
1064
1228
|
holderId: TurnSandboxLeaseHolderId,
|
|
1229
|
+
): Promise<ResumedTurnSandbox> {
|
|
1230
|
+
try {
|
|
1231
|
+
return await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
1232
|
+
} catch (error) {
|
|
1233
|
+
if (!isReplaceableFreshSandboxReadinessTimeout(error)) throw error;
|
|
1234
|
+
const readiness = error as SandboxExecReadinessTimeoutError;
|
|
1235
|
+
const warn = (message: string, fields: Parameters<Observability["warn"]>[1]) => {
|
|
1236
|
+
if (services.observability) services.observability.warn(message, fields);
|
|
1237
|
+
else console.warn(message, fields);
|
|
1238
|
+
};
|
|
1239
|
+
const identity = {
|
|
1240
|
+
workspaceId: ids.workspaceId,
|
|
1241
|
+
sessionId: ids.sessionId,
|
|
1242
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1243
|
+
backend: readiness.backend,
|
|
1244
|
+
instanceId: readiness.instanceId,
|
|
1245
|
+
readinessTimeoutMs: readiness.timeoutMs,
|
|
1246
|
+
};
|
|
1247
|
+
const budget =
|
|
1248
|
+
services.freshSandboxReadinessReplacementBudget ??
|
|
1249
|
+
createFreshSandboxReadinessReplacementBudget();
|
|
1250
|
+
if (budget.remaining <= 0) {
|
|
1251
|
+
recordFreshSandboxReadinessReplacement(
|
|
1252
|
+
services.sandboxMetrics,
|
|
1253
|
+
readiness.backend,
|
|
1254
|
+
"budget_spent",
|
|
1255
|
+
);
|
|
1256
|
+
warn(
|
|
1257
|
+
"sandbox command-readiness timed out on a fresh box; this turn attempt already used its replacement",
|
|
1258
|
+
identity,
|
|
1259
|
+
);
|
|
1260
|
+
throw error;
|
|
1261
|
+
}
|
|
1262
|
+
budget.remaining -= 1;
|
|
1263
|
+
const delayMs = await (
|
|
1264
|
+
services.freshSandboxReadinessReplacementDelayMs ?? freshSandboxReadinessReplacementDelayMs
|
|
1265
|
+
)();
|
|
1266
|
+
warn("sandbox command-readiness timed out on a fresh box; replacing it once after jitter", {
|
|
1267
|
+
...identity,
|
|
1268
|
+
replacementDelayMs: delayMs,
|
|
1269
|
+
});
|
|
1270
|
+
if (!(await sleepUnlessCancelled(delayMs, services.cancellationSignal))) {
|
|
1271
|
+
// Cancellation owns the turn boundary; surface the original typed
|
|
1272
|
+
// readiness failure rather than starting a replacement.
|
|
1273
|
+
recordFreshSandboxReadinessReplacement(
|
|
1274
|
+
services.sandboxMetrics,
|
|
1275
|
+
readiness.backend,
|
|
1276
|
+
"cancelled",
|
|
1277
|
+
);
|
|
1278
|
+
throw error;
|
|
1279
|
+
}
|
|
1280
|
+
try {
|
|
1281
|
+
const replaced = await resumeBoxForTurnOnce(services, ids, kind, holderId);
|
|
1282
|
+
recordFreshSandboxReadinessReplacement(
|
|
1283
|
+
services.sandboxMetrics,
|
|
1284
|
+
readiness.backend,
|
|
1285
|
+
"replaced",
|
|
1286
|
+
);
|
|
1287
|
+
return replaced;
|
|
1288
|
+
} catch (replacementError) {
|
|
1289
|
+
recordFreshSandboxReadinessReplacement(
|
|
1290
|
+
services.sandboxMetrics,
|
|
1291
|
+
readiness.backend,
|
|
1292
|
+
services.cancellationSignal?.aborted
|
|
1293
|
+
? "cancelled"
|
|
1294
|
+
: replacementError instanceof SandboxExecReadinessTimeoutError
|
|
1295
|
+
? "failed_again"
|
|
1296
|
+
: "replacement_failed",
|
|
1297
|
+
);
|
|
1298
|
+
throw replacementError;
|
|
1299
|
+
}
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
|
|
1303
|
+
async function resumeBoxForTurnOnce(
|
|
1304
|
+
services: SandboxResumeServices,
|
|
1305
|
+
ids: ResumeBoxIds,
|
|
1306
|
+
kind: "turn",
|
|
1307
|
+
holderId: TurnSandboxLeaseHolderId,
|
|
1065
1308
|
): Promise<ResumedTurnSandbox> {
|
|
1066
1309
|
const { db, settings } = services;
|
|
1067
1310
|
const os = ids.os ?? "linux";
|
|
@@ -1153,13 +1396,18 @@ export async function resumeBoxForTurn(
|
|
|
1153
1396
|
holderId,
|
|
1154
1397
|
subjectId: ids.sessionId,
|
|
1155
1398
|
backend: ids.backend,
|
|
1399
|
+
warmBilling: {
|
|
1400
|
+
mode: settings.sandboxWarmBillingMode,
|
|
1401
|
+
rateMicrosPerSecond: sandboxWarmRateMicrosPerSecond(settings, ids.backend),
|
|
1402
|
+
},
|
|
1156
1403
|
os,
|
|
1157
1404
|
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
1158
|
-
// conflicts on a live box already running a different image. A
|
|
1405
|
+
// conflicts on a live box already running a different required image. A
|
|
1159
1406
|
// SandboxImageConflictError propagates while another holder is active; a
|
|
1160
1407
|
// solo change requests a capture-and-drain rotation and this attempt retries
|
|
1161
1408
|
// after the cold successor can safely stamp the new image.
|
|
1162
1409
|
...(ids.image ? { image: ids.image } : {}),
|
|
1410
|
+
...(ids.imagePolicy ? { imagePolicy: ids.imagePolicy } : {}),
|
|
1163
1411
|
// RIG IS SHARED STATE (M3): thread the frozen rig version so the lease stamps it
|
|
1164
1412
|
// + conflicts on a live box under a different rig. A SandboxRigConflictError
|
|
1165
1413
|
// propagates while another holder is active; a solo change uses the same
|
|
@@ -1283,6 +1531,7 @@ export async function resumeBoxForTurn(
|
|
|
1283
1531
|
holderId,
|
|
1284
1532
|
expectedEpoch: heartbeat.expectedEpoch,
|
|
1285
1533
|
leaseTtlMs: heartbeat.leaseTtlMs,
|
|
1534
|
+
billingMode: services.settings.sandboxWarmBillingMode,
|
|
1286
1535
|
})
|
|
1287
1536
|
.then(async (status) => {
|
|
1288
1537
|
if (heartbeat !== holderLeaseHeartbeat) return;
|
|
@@ -1374,6 +1623,14 @@ export async function resumeBoxForTurn(
|
|
|
1374
1623
|
if (acquired.role === "spawner") {
|
|
1375
1624
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1376
1625
|
let createdEstablished: EstablishedSandboxSession | null = null;
|
|
1626
|
+
// Set once commitWarmingToWarm publishes this box. From then on the box is
|
|
1627
|
+
// the group's shared, attachable workspace: a later cancellation (Pause,
|
|
1628
|
+
// Steer, worker shutdown) only drops this attempt's holder. Terminating it
|
|
1629
|
+
// here would hand the next attempt a warm lease naming a dead box, whose
|
|
1630
|
+
// exact-id resume then records a lost workspace that never lost anything.
|
|
1631
|
+
let published = false;
|
|
1632
|
+
let providerCreateOperationId: string | undefined;
|
|
1633
|
+
let providerCreateBindingKey: string | undefined;
|
|
1377
1634
|
let rematerialization: {
|
|
1378
1635
|
id: string;
|
|
1379
1636
|
selectedRevision: string;
|
|
@@ -1385,30 +1642,53 @@ export async function resumeBoxForTurn(
|
|
|
1385
1642
|
> | null;
|
|
1386
1643
|
} | null = null;
|
|
1387
1644
|
try {
|
|
1388
|
-
|
|
1645
|
+
// An audited decision to continue a definitively lost workspace on a new
|
|
1646
|
+
// EMPTY box: hydrate nothing (not even a per-session legacy archive) and
|
|
1647
|
+
// resume no prior provider identity. Every group member was already told
|
|
1648
|
+
// the workspace is empty; the DB accepts only this exact publication.
|
|
1649
|
+
const freshWorkspaceRecoveryId = acquired.lease.freshWorkspaceRecoveryId ?? null;
|
|
1650
|
+
const envelope = freshWorkspaceRecoveryId
|
|
1651
|
+
? null
|
|
1652
|
+
: await getSandboxSessionEnvelope(db, ids.workspaceId, ids.sessionId);
|
|
1389
1653
|
// The lease is authoritative. A legacy per-session fallback archive may
|
|
1390
1654
|
// only be used after beginSandboxRematerialization imports its archive
|
|
1391
1655
|
// fields under the warming-row lock and records one selected revision.
|
|
1392
1656
|
// Select the fallback only when the lease has no archive truth at all. A
|
|
1393
1657
|
// lease-carried unverified/invalid archive must fail closed rather than
|
|
1394
1658
|
// silently substituting another revision.
|
|
1659
|
+
// Backstop: a session already told its workspace is empty never gets a
|
|
1660
|
+
// legacy per-session archive back, even if the lease marker was lost.
|
|
1661
|
+
const legacyFallbackRefused =
|
|
1662
|
+
!freshWorkspaceRecoveryId &&
|
|
1663
|
+
acquired.lease.recovery.archive.status === "none" &&
|
|
1664
|
+
workspaceArchiveFieldsFromEnvelope(envelope) !== null &&
|
|
1665
|
+
(await sessionHoldsFreshWorkspaceRecovery(db, ids.workspaceId, ids.sessionId));
|
|
1395
1666
|
const fallbackArchiveEnvelope =
|
|
1667
|
+
!freshWorkspaceRecoveryId &&
|
|
1668
|
+
!legacyFallbackRefused &&
|
|
1396
1669
|
acquired.lease.recovery.archive.status === "none" &&
|
|
1397
1670
|
workspaceArchiveFieldsFromEnvelope(envelope) !== null
|
|
1398
1671
|
? withoutSandboxProviderIdentity(envelope)
|
|
1399
1672
|
: null;
|
|
1400
|
-
let spawnEnvelope =
|
|
1673
|
+
let spawnEnvelope = freshWorkspaceRecoveryId
|
|
1674
|
+
? null
|
|
1675
|
+
: legacyFallbackRefused
|
|
1676
|
+
? (acquired.lease.resumeState ?? null)
|
|
1677
|
+
: (fallbackArchiveEnvelope ?? acquired.lease.resumeState ?? envelope);
|
|
1401
1678
|
const archiveSource =
|
|
1402
1679
|
acquired.lease.recovery.archive.status === "none"
|
|
1403
1680
|
? fallbackArchiveEnvelope
|
|
1404
1681
|
: acquired.lease.resumeState;
|
|
1405
1682
|
const continuityRecovery = acquired.lease.recovery.continuity;
|
|
1683
|
+
// A fresh-workspace spawn is deliberately archive-free; commitWarmingToWarm
|
|
1684
|
+
// verifies the exact decision before publication.
|
|
1406
1685
|
if (
|
|
1407
|
-
|
|
1686
|
+
!freshWorkspaceRecoveryId &&
|
|
1687
|
+
((acquired.lease.recovery.archive.status === "available" &&
|
|
1408
1688
|
(acquired.lease.archiveComplete ||
|
|
1409
1689
|
acquired.lease.historicalRecoveryAuthorized === true)) ||
|
|
1410
|
-
|
|
1411
|
-
|
|
1690
|
+
(acquired.lease.recovery.archive.status === "none" &&
|
|
1691
|
+
workspaceArchiveFieldsFromEnvelope(archiveSource) !== null))
|
|
1412
1692
|
) {
|
|
1413
1693
|
const rematerializationId = crypto.randomUUID();
|
|
1414
1694
|
const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
|
|
@@ -1451,7 +1731,11 @@ export async function resumeBoxForTurn(
|
|
|
1451
1731
|
legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
|
|
1452
1732
|
legacyProviderBinding: null,
|
|
1453
1733
|
};
|
|
1454
|
-
} else if (
|
|
1734
|
+
} else if (
|
|
1735
|
+
!freshWorkspaceRecoveryId &&
|
|
1736
|
+
acquired.lease.recovery.archive.status !== "none" &&
|
|
1737
|
+
!continuityRecovery
|
|
1738
|
+
) {
|
|
1455
1739
|
throw new SandboxLeaseRecoveryBlockedError(
|
|
1456
1740
|
ids.sandboxGroupId,
|
|
1457
1741
|
expectedEpoch,
|
|
@@ -1488,10 +1772,12 @@ export async function resumeBoxForTurn(
|
|
|
1488
1772
|
? {
|
|
1489
1773
|
loadHostWorkspaceArchive: async (ref) => {
|
|
1490
1774
|
try {
|
|
1491
|
-
return await downloadWorkspaceArchiveSpool(
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1775
|
+
return await downloadWorkspaceArchiveSpool(
|
|
1776
|
+
services.objectStorage!,
|
|
1777
|
+
ref.key,
|
|
1778
|
+
{ bytes: ref.bytes, sha256: ref.sha256 },
|
|
1779
|
+
{ temporaryDirectory: workspaceArchiveDownloadTemporaryDirectory },
|
|
1780
|
+
);
|
|
1495
1781
|
} catch (error) {
|
|
1496
1782
|
if (error instanceof WorkspaceArchiveStorageError) {
|
|
1497
1783
|
throw new WorkspaceArchiveIntegrityError(error.code, error.message, {
|
|
@@ -1510,6 +1796,34 @@ export async function resumeBoxForTurn(
|
|
|
1510
1796
|
...(services.logicalFallbackSettings
|
|
1511
1797
|
? { logicalFallbackSettings: services.logicalFallbackSettings }
|
|
1512
1798
|
: {}),
|
|
1799
|
+
onBeforeSandboxCreate: async (createSettings, intent, providerContext) => {
|
|
1800
|
+
if (ids.backend !== "modal") return;
|
|
1801
|
+
if (!intent || !providerContext)
|
|
1802
|
+
throw new Error("Modal create requires the provider dispatch boundary");
|
|
1803
|
+
throwIfReleasedOrCancelled();
|
|
1804
|
+
const binding = await resolveModalCheckpointProviderBindingForSession(
|
|
1805
|
+
createSettings,
|
|
1806
|
+
providerContext,
|
|
1807
|
+
);
|
|
1808
|
+
const operationId = intent.operationId;
|
|
1809
|
+
await beginModalProviderCreate(db, {
|
|
1810
|
+
accountId: ids.accountId,
|
|
1811
|
+
workspaceId: ids.workspaceId,
|
|
1812
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1813
|
+
expectedEpoch,
|
|
1814
|
+
operationId,
|
|
1815
|
+
providerBindingKey: binding.key,
|
|
1816
|
+
rematerializationId: rematerialization?.id ?? null,
|
|
1817
|
+
selectedRevision: rematerialization?.selectedRevision ?? null,
|
|
1818
|
+
imageId: intent.imageId,
|
|
1819
|
+
imageRef: createSettings.modalImageRef ?? null,
|
|
1820
|
+
appId: intent.appId,
|
|
1821
|
+
providerName: intent.name,
|
|
1822
|
+
requestSha256: intent.requestSha256,
|
|
1823
|
+
});
|
|
1824
|
+
providerCreateOperationId = operationId;
|
|
1825
|
+
providerCreateBindingKey = binding.key;
|
|
1826
|
+
},
|
|
1513
1827
|
onSandboxCreated: async (created) => {
|
|
1514
1828
|
createdEstablished = created;
|
|
1515
1829
|
providerRenewalTarget = {
|
|
@@ -1517,7 +1831,16 @@ export async function resumeBoxForTurn(
|
|
|
1517
1831
|
instanceId: created.instanceId,
|
|
1518
1832
|
};
|
|
1519
1833
|
providerRenewedAtMs = Date.now();
|
|
1520
|
-
|
|
1834
|
+
if (
|
|
1835
|
+
providerCreateBindingKey &&
|
|
1836
|
+
!(await modalSessionMatchesCheckpointProviderBinding(
|
|
1837
|
+
settings,
|
|
1838
|
+
created.session,
|
|
1839
|
+
providerCreateBindingKey,
|
|
1840
|
+
))
|
|
1841
|
+
) {
|
|
1842
|
+
throw new Error("Modal creation receipt crossed the fenced provider namespace");
|
|
1843
|
+
}
|
|
1521
1844
|
if (
|
|
1522
1845
|
rematerialization &&
|
|
1523
1846
|
(rematerialization.providerBindingKey || rematerialization.legacyCheckpoint)
|
|
@@ -1568,6 +1891,7 @@ export async function resumeBoxForTurn(
|
|
|
1568
1891
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1569
1892
|
expectedEpoch,
|
|
1570
1893
|
rematerializationId: rematerialization?.id ?? null,
|
|
1894
|
+
...(providerCreateOperationId ? { providerCreateOperationId } : {}),
|
|
1571
1895
|
...(created.providerContinuity
|
|
1572
1896
|
? { continuityRecovery: created.providerContinuity }
|
|
1573
1897
|
: {}),
|
|
@@ -1650,11 +1974,22 @@ export async function resumeBoxForTurn(
|
|
|
1650
1974
|
// A sandbox handle is not sufficient evidence that an asynchronous
|
|
1651
1975
|
// provider's command router is live. Do not publish a warm lease until
|
|
1652
1976
|
// one bounded no-op exec works.
|
|
1653
|
-
// On timeout the catch below terminates the box and rolls warming -> cold
|
|
1654
|
-
//
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1977
|
+
// On timeout the catch below terminates the box and rolls warming -> cold;
|
|
1978
|
+
// resumeBoxForTurn may then replace a proven-terminated fresh box once.
|
|
1979
|
+
if (services.verifySpawnedSandboxReadiness) {
|
|
1980
|
+
await services.verifySpawnedSandboxReadiness(established, {
|
|
1981
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1982
|
+
});
|
|
1983
|
+
} else {
|
|
1984
|
+
await waitForSandboxExecReadiness(
|
|
1985
|
+
established,
|
|
1986
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
1987
|
+
{
|
|
1988
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1989
|
+
},
|
|
1990
|
+
cancellationSignal,
|
|
1991
|
+
);
|
|
1992
|
+
}
|
|
1658
1993
|
await maybeRenewProviderExpiration(true);
|
|
1659
1994
|
throwIfReleasedOrCancelled();
|
|
1660
1995
|
// Fold the LIVE box into a re-resumable envelope and persist it as the
|
|
@@ -1700,16 +2035,18 @@ export async function resumeBoxForTurn(
|
|
|
1700
2035
|
dataPlaneUrl: null,
|
|
1701
2036
|
resumeBackendId: established.backendId,
|
|
1702
2037
|
resumeState: resumeEnvelope,
|
|
1703
|
-
...(
|
|
1704
|
-
? {
|
|
1705
|
-
:
|
|
1706
|
-
? {
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
2038
|
+
...(freshWorkspaceRecoveryId
|
|
2039
|
+
? { freshWorkspace: { operationId: freshWorkspaceRecoveryId } }
|
|
2040
|
+
: established.providerContinuity
|
|
2041
|
+
? { continuityRecovery: established.providerContinuity }
|
|
2042
|
+
: rematerialization
|
|
2043
|
+
? {
|
|
2044
|
+
rematerialization: {
|
|
2045
|
+
id: rematerialization.id,
|
|
2046
|
+
verifiedRevision: rematerialization.selectedRevision,
|
|
2047
|
+
},
|
|
2048
|
+
}
|
|
2049
|
+
: {}),
|
|
1713
2050
|
leaseTtlMs,
|
|
1714
2051
|
});
|
|
1715
2052
|
if (!committed.committed || !committed.lease) {
|
|
@@ -1739,13 +2076,21 @@ export async function resumeBoxForTurn(
|
|
|
1739
2076
|
await release();
|
|
1740
2077
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1741
2078
|
}
|
|
2079
|
+
published = true;
|
|
1742
2080
|
holderLeaseHeartbeat = {
|
|
1743
2081
|
expectedEpoch: committed.lease.leaseEpoch,
|
|
1744
2082
|
leaseTtlMs,
|
|
1745
2083
|
};
|
|
2084
|
+
await services.onSpawnedSandboxPublished?.();
|
|
1746
2085
|
throwIfReleasedOrCancelled();
|
|
1747
2086
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1748
2087
|
} catch (error) {
|
|
2088
|
+
if (published) {
|
|
2089
|
+
// The published warm box stays for the replacement attempt to resume
|
|
2090
|
+
// by exact provider id; ordinary idle drain owns its capture/teardown.
|
|
2091
|
+
await release();
|
|
2092
|
+
throw error;
|
|
2093
|
+
}
|
|
1749
2094
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
1750
2095
|
await terminateEstablishedSandbox(createdEstablished);
|
|
1751
2096
|
await release();
|
|
@@ -1790,6 +2135,16 @@ export async function resumeBoxForTurn(
|
|
|
1790
2135
|
}
|
|
1791
2136
|
await release();
|
|
1792
2137
|
recordSandboxWarmingTimeout(services.sandboxMetrics, error);
|
|
2138
|
+
if (
|
|
2139
|
+
terminated &&
|
|
2140
|
+
createdEstablished &&
|
|
2141
|
+
error instanceof SandboxExecReadinessTimeoutError &&
|
|
2142
|
+
error.instanceId === createdEstablished.instanceId &&
|
|
2143
|
+
(createdEstablished.origin === "created" || createdEstablished.origin === "restored") &&
|
|
2144
|
+
!createdEstablished.providerContinuity
|
|
2145
|
+
) {
|
|
2146
|
+
disposedUnreadyFreshSandboxErrors.add(error);
|
|
2147
|
+
}
|
|
1793
2148
|
throw sandboxProvisionStageError(rematerialization ? "archive_recovery" : "create", error);
|
|
1794
2149
|
}
|
|
1795
2150
|
}
|
|
@@ -1832,16 +2187,21 @@ export async function resumeBoxForTurn(
|
|
|
1832
2187
|
});
|
|
1833
2188
|
throwIfReleasedOrCancelled();
|
|
1834
2189
|
// A durable `warm` row is an ownership assertion, not provider liveness.
|
|
1835
|
-
// A provider may have ended the exact box while
|
|
2190
|
+
// A provider may have ended the exact box while Opengeni was idle. Prove
|
|
1836
2191
|
// the command router before handing the session to the
|
|
1837
2192
|
// agent so terminal evidence enters the atomic warm->cold recovery path
|
|
1838
2193
|
// below instead of surfacing inside a model-visible tool call.
|
|
1839
2194
|
if (services.verifyAttachedSandboxReadiness) {
|
|
1840
2195
|
await services.verifyAttachedSandboxReadiness(established);
|
|
1841
2196
|
} else {
|
|
1842
|
-
await waitForSandboxExecReadiness(
|
|
1843
|
-
|
|
1844
|
-
|
|
2197
|
+
await waitForSandboxExecReadiness(
|
|
2198
|
+
established,
|
|
2199
|
+
MODAL_EXEC_READINESS_TIMEOUT_MS,
|
|
2200
|
+
{
|
|
2201
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
2202
|
+
},
|
|
2203
|
+
cancellationSignal,
|
|
2204
|
+
);
|
|
1845
2205
|
}
|
|
1846
2206
|
providerRenewalTarget = {
|
|
1847
2207
|
backend: ids.backend,
|
|
@@ -1862,6 +2222,11 @@ export async function resumeBoxForTurn(
|
|
|
1862
2222
|
expectedBackend: ids.backend,
|
|
1863
2223
|
});
|
|
1864
2224
|
if (marked.status === "marked") {
|
|
2225
|
+
await publishDurableSessionEvents(
|
|
2226
|
+
services.bus,
|
|
2227
|
+
ids.workspaceId,
|
|
2228
|
+
marked.backgroundCommandEvents,
|
|
2229
|
+
);
|
|
1865
2230
|
await services.onSandboxLost?.({
|
|
1866
2231
|
sandboxGroupId: ids.sandboxGroupId,
|
|
1867
2232
|
instanceId: live.instanceId,
|
|
@@ -1901,7 +2266,13 @@ async function waitForWarm(
|
|
|
1901
2266
|
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1902
2267
|
let instanceId: string | null = null;
|
|
1903
2268
|
while (Date.now() < deadline) {
|
|
1904
|
-
|
|
2269
|
+
// A cancelled waiter owns no box and must not keep polling for up to the
|
|
2270
|
+
// warming budget: its activity finalizer joins this exact promise.
|
|
2271
|
+
if (!(await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal))) {
|
|
2272
|
+
throw services.cancellationSignal?.reason instanceof Error
|
|
2273
|
+
? services.cancellationSignal.reason
|
|
2274
|
+
: new Error("Sandbox warming wait was cancelled with its owning turn attempt");
|
|
2275
|
+
}
|
|
1905
2276
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1906
2277
|
if (!lease) {
|
|
1907
2278
|
// Lease vanished (cold-reaped). Re-dispatch from scratch.
|