@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
import type { Worker, WorkerStatus } from "@temporalio/worker";
|
|
3
|
+
import { turnTaskQueueMetricLabels, type TurnTaskQueueIdentity } from "./observability-metrics";
|
|
4
|
+
import {
|
|
5
|
+
createTelemetryClock,
|
|
6
|
+
telemetryInterval,
|
|
7
|
+
telemetryScheduler,
|
|
8
|
+
type TelemetryScheduler,
|
|
9
|
+
} from "./telemetry-scheduler";
|
|
10
|
+
import type { WorkerRunTarget } from "./worker-service-lifecycle";
|
|
11
|
+
|
|
12
|
+
export const TURN_WORKER_ACTIVITY_METRICS = {
|
|
13
|
+
inflight: "opengeni_turn_worker_activities_inflight",
|
|
14
|
+
timestamp: "opengeni_turn_worker_activities_last_observed_timestamp_seconds",
|
|
15
|
+
valid: "opengeni_turn_worker_activities_last_read_success",
|
|
16
|
+
} as const;
|
|
17
|
+
|
|
18
|
+
export type TurnWorkerActivityTelemetry = {
|
|
19
|
+
/** Settles only when Worker.run AND all SDK activity execution have settled. */
|
|
20
|
+
settled: Promise<void>;
|
|
21
|
+
isSettled(): boolean;
|
|
22
|
+
closeIfNeverRun(): void;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** Preserve scraping after an SDK force-failure without blocking resource close
|
|
26
|
+
* or extending the host shutdown/containment budgets. */
|
|
27
|
+
export function closeWorkerHttpAfterActivityExecution(
|
|
28
|
+
telemetry: TurnWorkerActivityTelemetry | undefined,
|
|
29
|
+
close: () => void | Promise<void>,
|
|
30
|
+
): void {
|
|
31
|
+
telemetry?.closeIfNeverRun();
|
|
32
|
+
if (!telemetry || telemetry.isSettled()) {
|
|
33
|
+
try {
|
|
34
|
+
void Promise.resolve(close()).catch(() => undefined);
|
|
35
|
+
} catch {
|
|
36
|
+
/* Preserve other resource cleanup. */
|
|
37
|
+
}
|
|
38
|
+
} else {
|
|
39
|
+
void telemetry.settled.then(close).catch(() => undefined);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
type SdkActivityWorker = Pick<Worker, "run" | "shutdown"> & {
|
|
44
|
+
getStatus(): Pick<
|
|
45
|
+
WorkerStatus,
|
|
46
|
+
"numInFlightNonLocalActivities" | "numInFlightLocalActivities" | "numInFlightActivities"
|
|
47
|
+
>;
|
|
48
|
+
numInFlightActivities$: {
|
|
49
|
+
subscribe(observer: { next(value: number): void; error(error: unknown): void }): {
|
|
50
|
+
unsubscribe(): void;
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/** Only public SDK1.22 hooks. Never count agent claims/slots, poll reservations,
|
|
56
|
+
* heartbeat activity counts, or cancellation intent as executing occupancy.
|
|
57
|
+
* Requires a fresh metric registry/schema and one SDK turn-worker owner per
|
|
58
|
+
* registry + namespace + queue + physical UID. This bridge does not aggregate
|
|
59
|
+
* multiple workers or migrate legacy pre-registered unlabelled gauges. */
|
|
60
|
+
export function withTurnWorkerActivityTelemetry(
|
|
61
|
+
sdkWorker: SdkActivityWorker,
|
|
62
|
+
input: {
|
|
63
|
+
observability: Observability;
|
|
64
|
+
identity: TurnTaskQueueIdentity;
|
|
65
|
+
podUid: string;
|
|
66
|
+
intervalMs?: number;
|
|
67
|
+
now?: () => number;
|
|
68
|
+
scheduler?: TelemetryScheduler;
|
|
69
|
+
},
|
|
70
|
+
): { worker: WorkerRunTarget; activityTelemetry: TurnWorkerActivityTelemetry } {
|
|
71
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
72
|
+
const intervalMs = telemetryInterval(input.intervalMs ?? 15_000, "activity telemetry intervalMs");
|
|
73
|
+
const clock = createTelemetryClock(input.now ?? Date.now);
|
|
74
|
+
const labels = { ...turnTaskQueueMetricLabels(input.identity), worker_pod_uid: input.podUid };
|
|
75
|
+
let runStarted = false;
|
|
76
|
+
let runSettled = false;
|
|
77
|
+
let runPromise: Promise<void> | undefined;
|
|
78
|
+
let disposed = false;
|
|
79
|
+
let queued = false;
|
|
80
|
+
let subscription: { unsubscribe(): void } | undefined;
|
|
81
|
+
let stopTimer: (() => void) | undefined;
|
|
82
|
+
let resolveSettled!: () => void;
|
|
83
|
+
const settled = new Promise<void>((resolve) => {
|
|
84
|
+
resolveSettled = resolve;
|
|
85
|
+
});
|
|
86
|
+
const set = (name: string, help: string, value: number) =>
|
|
87
|
+
input.observability.setGauge({ name, help, labels, value });
|
|
88
|
+
const invalidate = () => {
|
|
89
|
+
if (disposed) return;
|
|
90
|
+
try {
|
|
91
|
+
set(
|
|
92
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
93
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
94
|
+
0,
|
|
95
|
+
);
|
|
96
|
+
} catch {
|
|
97
|
+
/* Metrics cannot fail an activity or the worker. Age also fences failed exporters. */
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
const dispose = () => {
|
|
101
|
+
if (disposed) return;
|
|
102
|
+
disposed = true;
|
|
103
|
+
stopTimer?.();
|
|
104
|
+
try {
|
|
105
|
+
subscription?.unsubscribe();
|
|
106
|
+
} catch {
|
|
107
|
+
/* Host lifecycle remains unchanged. */
|
|
108
|
+
}
|
|
109
|
+
resolveSettled();
|
|
110
|
+
};
|
|
111
|
+
const sample = () => {
|
|
112
|
+
if (disposed) return;
|
|
113
|
+
let idle = false;
|
|
114
|
+
try {
|
|
115
|
+
const status = sdkWorker.getStatus();
|
|
116
|
+
const count = status.numInFlightNonLocalActivities;
|
|
117
|
+
const local = status.numInFlightLocalActivities;
|
|
118
|
+
const total = status.numInFlightActivities;
|
|
119
|
+
if (
|
|
120
|
+
![count, local, total].every((value) => Number.isSafeInteger(value) && value >= 0) ||
|
|
121
|
+
count + local !== total
|
|
122
|
+
) {
|
|
123
|
+
throw new Error("invalid SDK executing activity state");
|
|
124
|
+
}
|
|
125
|
+
idle = total === 0;
|
|
126
|
+
const observedAt = clock.read();
|
|
127
|
+
if (observedAt === null) throw new Error("invalid activity observation clock");
|
|
128
|
+
// Publish validity last; a partial or failed metric update cannot license
|
|
129
|
+
// combining a new timestamp with old occupancy. Never invent idle zero.
|
|
130
|
+
set(
|
|
131
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
132
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
133
|
+
0,
|
|
134
|
+
);
|
|
135
|
+
set(
|
|
136
|
+
TURN_WORKER_ACTIVITY_METRICS.inflight,
|
|
137
|
+
"SDK executing non-local activities on this turn worker, including preclaim, video and cancellation-unresolved execution.",
|
|
138
|
+
count,
|
|
139
|
+
);
|
|
140
|
+
set(
|
|
141
|
+
TURN_WORKER_ACTIVITY_METRICS.timestamp,
|
|
142
|
+
"Unix timestamp of the latest successful SDK activity occupancy observation.",
|
|
143
|
+
observedAt / 1_000,
|
|
144
|
+
);
|
|
145
|
+
set(
|
|
146
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
147
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
148
|
+
1,
|
|
149
|
+
);
|
|
150
|
+
} catch {
|
|
151
|
+
invalidate();
|
|
152
|
+
}
|
|
153
|
+
// SDK force-shutdown rejection alone is not activity quiescence. Counters remain
|
|
154
|
+
// live until the underlying JS promises settle, including after run rejects.
|
|
155
|
+
if (runSettled && idle) dispose();
|
|
156
|
+
};
|
|
157
|
+
const scheduleSample = () => {
|
|
158
|
+
if (disposed || queued) return;
|
|
159
|
+
queued = true;
|
|
160
|
+
// SDK increments its aggregate observable BEFORE its local/nonlocal
|
|
161
|
+
// breakdown. Read after that synchronous transition, not inside it.
|
|
162
|
+
scheduler.defer(() => {
|
|
163
|
+
queued = false;
|
|
164
|
+
sample();
|
|
165
|
+
});
|
|
166
|
+
};
|
|
167
|
+
try {
|
|
168
|
+
subscription = sdkWorker.numInFlightActivities$.subscribe({
|
|
169
|
+
next: scheduleSample,
|
|
170
|
+
error: invalidate,
|
|
171
|
+
});
|
|
172
|
+
} catch {
|
|
173
|
+
invalidate();
|
|
174
|
+
}
|
|
175
|
+
stopTimer = scheduler.interval(sample, intervalMs);
|
|
176
|
+
sample(); // INITIALIZED workers expose true idle zero before the first poll.
|
|
177
|
+
const activityTelemetry: TurnWorkerActivityTelemetry = {
|
|
178
|
+
settled,
|
|
179
|
+
isSettled: () => disposed,
|
|
180
|
+
closeIfNeverRun: () => {
|
|
181
|
+
if (!runStarted) {
|
|
182
|
+
runSettled = true;
|
|
183
|
+
sample();
|
|
184
|
+
}
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
return {
|
|
188
|
+
activityTelemetry,
|
|
189
|
+
worker: {
|
|
190
|
+
shutdown: () => sdkWorker.shutdown(), // Drain never unregisters observation.
|
|
191
|
+
run: () => {
|
|
192
|
+
if (!runPromise && runSettled)
|
|
193
|
+
return Promise.reject(new Error("cannot run a closed turn worker"));
|
|
194
|
+
// A second SDK run rejects immediately while the first may still poll.
|
|
195
|
+
// Keep one execution owner so that rejection cannot detach telemetry.
|
|
196
|
+
runPromise ??= (async () => {
|
|
197
|
+
runStarted = true;
|
|
198
|
+
try {
|
|
199
|
+
await sdkWorker.run();
|
|
200
|
+
} finally {
|
|
201
|
+
runSettled = true;
|
|
202
|
+
sample();
|
|
203
|
+
}
|
|
204
|
+
})();
|
|
205
|
+
return runPromise;
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
}
|
|
@@ -126,7 +126,7 @@ export function createWorkerServiceLifecycle(input: {
|
|
|
126
126
|
}
|
|
127
127
|
const previousState = state;
|
|
128
128
|
safeLifecycleLog(() =>
|
|
129
|
-
input.observability.info("
|
|
129
|
+
input.observability.info("Opengeni worker draining (graceful shutdown)", {
|
|
130
130
|
role: input.role,
|
|
131
131
|
errorClass: "WorkerLifecycleOperation",
|
|
132
132
|
errorCode: "worker_draining",
|
|
@@ -172,3 +172,56 @@ function safeLifecycleLog(log: () => void): void {
|
|
|
172
172
|
// Telemetry cannot change worker lifecycle state or shutdown delivery.
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
|
+
|
|
176
|
+
/** A cleanup stall first drains peers through WORKER_SHUTDOWN. The standalone
|
|
177
|
+
* process owns the final exit, because Temporal cannot kill JavaScript writers.
|
|
178
|
+
* Embedded hosts may supply their own termination policy at this same edge. */
|
|
179
|
+
export function createWorkerCleanupContainment(input: {
|
|
180
|
+
drain: () => boolean;
|
|
181
|
+
terminate?: () => void;
|
|
182
|
+
observability: Observability;
|
|
183
|
+
timeoutMs?: number;
|
|
184
|
+
}) {
|
|
185
|
+
let requested = false;
|
|
186
|
+
let finished = false;
|
|
187
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
188
|
+
return {
|
|
189
|
+
request() {
|
|
190
|
+
if (requested || finished) return;
|
|
191
|
+
requested = true;
|
|
192
|
+
if (input.terminate) {
|
|
193
|
+
timer = setTimeout(() => {
|
|
194
|
+
safeLifecycleLog(() =>
|
|
195
|
+
input.observability.error("worker cleanup drain exhausted; terminating host", {
|
|
196
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
197
|
+
errorCode: "worker_cleanup_drain_exhausted",
|
|
198
|
+
origin: "worker-lifecycle",
|
|
199
|
+
}),
|
|
200
|
+
);
|
|
201
|
+
input.terminate?.();
|
|
202
|
+
}, input.timeoutMs ?? 100_000);
|
|
203
|
+
timer.unref?.();
|
|
204
|
+
}
|
|
205
|
+
let accepted = false;
|
|
206
|
+
try {
|
|
207
|
+
accepted = input.drain();
|
|
208
|
+
} catch {
|
|
209
|
+
/* The host backstop remains armed. */
|
|
210
|
+
}
|
|
211
|
+
if (!accepted)
|
|
212
|
+
safeLifecycleLog(() =>
|
|
213
|
+
input.observability.error("worker cleanup drain request failed", {
|
|
214
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
215
|
+
errorCode: "worker_shutdown_request_failed",
|
|
216
|
+
origin: "worker-lifecycle",
|
|
217
|
+
}),
|
|
218
|
+
);
|
|
219
|
+
},
|
|
220
|
+
finished() {
|
|
221
|
+
// Call only after Worker.run resolves. ForceShutdownError does not prove
|
|
222
|
+
// its activity promises stopped, so rejection must retain the backstop.
|
|
223
|
+
finished = true;
|
|
224
|
+
clearTimeout(timer);
|
|
225
|
+
},
|
|
226
|
+
};
|
|
227
|
+
}
|
|
@@ -1,4 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ActivityCancellationType,
|
|
3
|
+
ActivityFailure,
|
|
4
|
+
ApplicationFailure,
|
|
5
|
+
proxyActivities,
|
|
6
|
+
} from "@temporalio/workflow";
|
|
2
7
|
import type * as activities from "../activities";
|
|
3
8
|
import {
|
|
4
9
|
KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
@@ -10,12 +15,14 @@ type WorkflowControlActivities = Pick<
|
|
|
10
15
|
| "enqueueGoalRetryWake"
|
|
11
16
|
| "expireSessionHumanInput"
|
|
12
17
|
| "expireSessionInteractionIntervention"
|
|
18
|
+
| "expireScheduledRunHumanWait"
|
|
13
19
|
| "failSessionAttempt"
|
|
14
20
|
| "getCodexCapacityWait"
|
|
15
21
|
| "markSessionIdle"
|
|
16
22
|
| "peekSessionWork"
|
|
17
23
|
| "persistSessionAttemptQuiescence"
|
|
18
24
|
| "reconcileSessionAttemptQuiescence"
|
|
25
|
+
| "reconcileSettledSessionAttempt"
|
|
19
26
|
| "reconcileCodexCapacityWait"
|
|
20
27
|
| "recoverDispatch"
|
|
21
28
|
| "recoverEscapedMcpTimeout"
|
|
@@ -160,6 +167,14 @@ export const knowledgeSourceSyncActivity = proxyActivities<
|
|
|
160
167
|
});
|
|
161
168
|
|
|
162
169
|
export function workflowFailureMessage(error: unknown): string {
|
|
170
|
+
if (
|
|
171
|
+
error instanceof ActivityFailure &&
|
|
172
|
+
error.activityType === "runAgentTurn" &&
|
|
173
|
+
error.cause instanceof ApplicationFailure &&
|
|
174
|
+
error.cause.type === "TurnExecutionPolicyDefinitionMismatchError"
|
|
175
|
+
) {
|
|
176
|
+
return error.cause.message;
|
|
177
|
+
}
|
|
163
178
|
if (error instanceof Error) {
|
|
164
179
|
return error.message;
|
|
165
180
|
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ParentClosePolicy,
|
|
3
|
+
WorkflowIdReusePolicy,
|
|
4
|
+
log,
|
|
5
|
+
proxyActivities,
|
|
6
|
+
startChild,
|
|
7
|
+
workflowInfo,
|
|
8
|
+
} from "@temporalio/workflow";
|
|
9
|
+
import type { BrowserDeadlineCheckpointTarget } from "@opengeni/db";
|
|
10
|
+
import type * as activities from "../activities";
|
|
11
|
+
|
|
12
|
+
const inventory = proxyActivities<Pick<typeof activities, "listDueBrowserCheckpoints">>({
|
|
13
|
+
startToCloseTimeout: "5 minutes",
|
|
14
|
+
heartbeatTimeout: "20 seconds",
|
|
15
|
+
});
|
|
16
|
+
const checkpoint = proxyActivities<Pick<typeof activities, "checkpointBrowserBeforeDeadline">>({
|
|
17
|
+
startToCloseTimeout: "25 minutes",
|
|
18
|
+
heartbeatTimeout: "20 seconds",
|
|
19
|
+
retry: { initialInterval: "1 second", maximumInterval: "30 seconds" },
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
export async function browserDeadlineCheckpointWorkflow(target: BrowserDeadlineCheckpointTarget) {
|
|
23
|
+
return await checkpoint.checkpointBrowserBeforeDeadline(target);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Runs from the existing lease-sweep dispatch. Inventory never performs provider
|
|
27
|
+
* I/O; each exact browser has an independent durable capture/cleanup child.
|
|
28
|
+
* Provider-deadline saves and idle saves (prepared by the lease reaper) share it. */
|
|
29
|
+
export async function browserDeadlineCheckpointSweepWorkflow(): Promise<void> {
|
|
30
|
+
const targets = await inventory.listDueBrowserCheckpoints();
|
|
31
|
+
await Promise.all(
|
|
32
|
+
targets.map(async (target) => {
|
|
33
|
+
try {
|
|
34
|
+
await startChild(browserDeadlineCheckpointWorkflow, {
|
|
35
|
+
// One id shape for both reasons: the id never depends on fields an
|
|
36
|
+
// older worker strips, so replay is deterministic across a rolling
|
|
37
|
+
// deploy. An idle and a deadline save of one controller generation
|
|
38
|
+
// are mutually exclusive in the database anyway.
|
|
39
|
+
workflowId: `browser-deadline:${target.browserSessionId}:${target.leaseEpoch}:${target.controllerGeneration}`,
|
|
40
|
+
taskQueue: workflowInfo().taskQueue,
|
|
41
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
42
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
43
|
+
args: [target],
|
|
44
|
+
});
|
|
45
|
+
} catch (error) {
|
|
46
|
+
if (error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError") return;
|
|
47
|
+
// Another sweep retries start; it cannot retire the browser's authority.
|
|
48
|
+
log.warn("browser deadline checkpoint child start deferred", {
|
|
49
|
+
browserSessionId: target.browserSessionId,
|
|
50
|
+
error: error instanceof Error ? error.name : "unknown",
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
}),
|
|
54
|
+
);
|
|
55
|
+
}
|
|
@@ -8,6 +8,7 @@ const reaperActivity = proxyActivities<
|
|
|
8
8
|
| "maintainRetainedScreenshots"
|
|
9
9
|
| "maintainBrowserStateArtifacts"
|
|
10
10
|
| "recoverVideoGenerationWorkflows"
|
|
11
|
+
| "reconcileRecentModelCallFacts"
|
|
11
12
|
>
|
|
12
13
|
>({
|
|
13
14
|
startToCloseTimeout: "5 minutes",
|
|
@@ -20,4 +21,5 @@ export async function fileUploadReaperWorkflow(): Promise<void> {
|
|
|
20
21
|
await reaperActivity.maintainRetainedScreenshots();
|
|
21
22
|
await reaperActivity.maintainBrowserStateArtifacts();
|
|
22
23
|
await reaperActivity.recoverVideoGenerationWorkflows();
|
|
24
|
+
await reaperActivity.reconcileRecentModelCallFacts();
|
|
23
25
|
}
|
|
@@ -20,9 +20,11 @@ import {
|
|
|
20
20
|
SANDBOX_REAPER_MAINTENANCE_WORKFLOW_ID,
|
|
21
21
|
SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
22
22
|
sandboxDrainWorkflowId,
|
|
23
|
+
sandboxIdleCheckpointWorkflowId,
|
|
23
24
|
sandboxLifecycleTaskQueue,
|
|
24
25
|
type SandboxDrainActivityInput,
|
|
25
26
|
type SandboxDrainWorkflowInput,
|
|
27
|
+
type SandboxIdleCheckpointWorkflowInput,
|
|
26
28
|
type SandboxLeaseSweepMaintenanceInput,
|
|
27
29
|
} from "../sandbox-reaper-contract";
|
|
28
30
|
|
|
@@ -57,6 +59,40 @@ function maintenanceActivity() {
|
|
|
57
59
|
});
|
|
58
60
|
}
|
|
59
61
|
|
|
62
|
+
function idleCheckpointInventoryActivity() {
|
|
63
|
+
return proxyActivities<Pick<typeof activities, "listIdleSandboxCheckpoints">>({
|
|
64
|
+
taskQueue: lifecycleTaskQueue(),
|
|
65
|
+
startToCloseTimeout: SANDBOX_REAPER_SCAN_ACTIVITY_TIMEOUT_MS,
|
|
66
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
67
|
+
// Bounded so the fixed-id sweep closes and the next tick starts a fresh
|
|
68
|
+
// one, including during a rolling deploy whose old pollers lack it.
|
|
69
|
+
retry: {
|
|
70
|
+
initialInterval: "1 second",
|
|
71
|
+
backoffCoefficient: 2,
|
|
72
|
+
maximumInterval: "30 seconds",
|
|
73
|
+
maximumAttempts: 3,
|
|
74
|
+
},
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function idleCheckpointActivity(startToCloseTimeout: number) {
|
|
79
|
+
return proxyActivities<Pick<typeof activities, "checkpointIdleSandboxLease">>({
|
|
80
|
+
taskQueue: lifecycleTaskQueue(),
|
|
81
|
+
startToCloseTimeout,
|
|
82
|
+
heartbeatTimeout: SANDBOX_REAPER_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
|
|
83
|
+
// A protective checkpoint, not a lifecycle transition: the exact capture
|
|
84
|
+
// claim fences a late or repeated attempt, and the next sweep lists the
|
|
85
|
+
// box again once its snapshot interval is due. A few retries cover a
|
|
86
|
+
// crashed worker or a rolling deploy whose old pollers lack the activity.
|
|
87
|
+
retry: {
|
|
88
|
+
initialInterval: "1 second",
|
|
89
|
+
backoffCoefficient: 2,
|
|
90
|
+
maximumInterval: "30 seconds",
|
|
91
|
+
maximumAttempts: 3,
|
|
92
|
+
},
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
|
|
60
96
|
function drainActivity(startToCloseTimeout: number) {
|
|
61
97
|
return proxyActivities<Pick<typeof activities, "drainSandboxLease">>({
|
|
62
98
|
taskQueue: lifecycleTaskQueue(),
|
|
@@ -97,6 +133,58 @@ export async function sandboxDrainWorkflow(input: SandboxDrainWorkflowInput): Pr
|
|
|
97
133
|
}
|
|
98
134
|
}
|
|
99
135
|
|
|
136
|
+
/** Checkpoint one warm box between turns (see checkpointIdleSandboxLease).
|
|
137
|
+
* Failure is not fatal: the next sweep lists the box again when it is due. */
|
|
138
|
+
export async function sandboxIdleCheckpointWorkflow(
|
|
139
|
+
input: SandboxIdleCheckpointWorkflowInput,
|
|
140
|
+
): Promise<void> {
|
|
141
|
+
try {
|
|
142
|
+
await idleCheckpointActivity(
|
|
143
|
+
input.timeoutClass === "fast"
|
|
144
|
+
? SANDBOX_DRAIN_FAST_ACTIVITY_TIMEOUT_MS
|
|
145
|
+
: SANDBOX_DRAIN_EXTENDED_ACTIVITY_TIMEOUT_MS,
|
|
146
|
+
).checkpointIdleSandboxLease(input);
|
|
147
|
+
} catch (error) {
|
|
148
|
+
log.warn("sandbox idle checkpoint deferred to a later sweep", {
|
|
149
|
+
workspaceId: input.target.workspaceId,
|
|
150
|
+
sandboxGroupId: input.target.sandboxGroupId,
|
|
151
|
+
leaseEpoch: input.target.leaseEpoch,
|
|
152
|
+
error: error instanceof Error ? error.message : String(error),
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Idle checkpoint sweep, started from the same tick as the drain inventory
|
|
158
|
+
* but as its own workflow type, so the V2 drain workflow's command history is
|
|
159
|
+
* unchanged across a rolling deploy. Inventory is DB only; each exact box gets
|
|
160
|
+
* one durable child per lease epoch, and a still-running child for that epoch
|
|
161
|
+
* is the coalesced capture. */
|
|
162
|
+
export async function sandboxIdleCheckpointSweepWorkflow(): Promise<void> {
|
|
163
|
+
const { targets, timeoutClass } =
|
|
164
|
+
await idleCheckpointInventoryActivity().listIdleSandboxCheckpoints();
|
|
165
|
+
await Promise.all(
|
|
166
|
+
targets.map(async (target) => {
|
|
167
|
+
try {
|
|
168
|
+
await startChild(sandboxIdleCheckpointWorkflow, {
|
|
169
|
+
workflowId: sandboxIdleCheckpointWorkflowId(target),
|
|
170
|
+
taskQueue: lifecycleTaskQueue(),
|
|
171
|
+
workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
|
|
172
|
+
parentClosePolicy: ParentClosePolicy.ABANDON,
|
|
173
|
+
args: [{ target, timeoutClass }],
|
|
174
|
+
});
|
|
175
|
+
} catch (error) {
|
|
176
|
+
if (alreadyRunningChild(error)) return;
|
|
177
|
+
log.warn("sandbox idle checkpoint child start deferred to a later sweep", {
|
|
178
|
+
workspaceId: target.workspaceId,
|
|
179
|
+
sandboxGroupId: target.sandboxGroupId,
|
|
180
|
+
leaseEpoch: target.leaseEpoch,
|
|
181
|
+
error: error instanceof Error ? error.message : String(error),
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
}),
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
100
188
|
/** Ancillary billing/reconciliation/GC is durable but never allowed to hold the
|
|
101
189
|
* fixed-ID inventory workflow open behind a provider backlog. One singleton
|
|
102
190
|
* maintenance child may queue on the lifecycle workers while every schedule
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
WorkflowIdReusePolicy,
|
|
8
8
|
} from "@temporalio/workflow";
|
|
9
9
|
import type { TurnInitiator } from "@opengeni/contracts";
|
|
10
|
+
import type { DispatchScheduledTaskRunInput } from "../activities/types";
|
|
10
11
|
import { scheduledTaskActivity } from "./activities";
|
|
11
12
|
import {
|
|
12
13
|
knowledgeSourceSyncWake,
|
|
@@ -26,14 +27,38 @@ export type ScheduledTaskFireWorkflowInput = ScheduledTaskFireWorkflowBase &
|
|
|
26
27
|
triggerType: "scheduled";
|
|
27
28
|
agentRunUsageIdempotencyKey?: never;
|
|
28
29
|
initiator?: never;
|
|
30
|
+
credentialRestriction?: never;
|
|
29
31
|
}
|
|
30
32
|
| {
|
|
31
33
|
triggerType: "manual" | "initial" | "provider_event" | "retry" | "repair";
|
|
32
34
|
agentRunUsageIdempotencyKey: string;
|
|
33
35
|
initiator: TurnInitiator;
|
|
36
|
+
credentialRestriction?: "developer_setup";
|
|
34
37
|
}
|
|
35
38
|
);
|
|
36
39
|
|
|
40
|
+
export function scheduledTaskDispatchInput(
|
|
41
|
+
input: ScheduledTaskFireWorkflowInput,
|
|
42
|
+
producerKey: string,
|
|
43
|
+
): DispatchScheduledTaskRunInput {
|
|
44
|
+
const base = {
|
|
45
|
+
workspaceId: input.workspaceId,
|
|
46
|
+
taskId: input.taskId,
|
|
47
|
+
producerKey,
|
|
48
|
+
};
|
|
49
|
+
return input.triggerType !== "scheduled"
|
|
50
|
+
? {
|
|
51
|
+
...base,
|
|
52
|
+
triggerType: input.triggerType,
|
|
53
|
+
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
|
|
54
|
+
initiator: input.initiator,
|
|
55
|
+
...(input.credentialRestriction === "developer_setup"
|
|
56
|
+
? { credentialRestriction: input.credentialRestriction }
|
|
57
|
+
: {}),
|
|
58
|
+
}
|
|
59
|
+
: { ...base, triggerType: "scheduled" };
|
|
60
|
+
}
|
|
61
|
+
|
|
37
62
|
export async function scheduledTaskFireWorkflow(
|
|
38
63
|
input: ScheduledTaskFireWorkflowInput,
|
|
39
64
|
): Promise<void> {
|
|
@@ -47,20 +72,8 @@ export async function scheduledTaskFireWorkflow(
|
|
|
47
72
|
) {
|
|
48
73
|
return;
|
|
49
74
|
}
|
|
50
|
-
const base = {
|
|
51
|
-
workspaceId: input.workspaceId,
|
|
52
|
-
taskId: input.taskId,
|
|
53
|
-
producerKey: workflowInfo().workflowId,
|
|
54
|
-
};
|
|
55
75
|
const result = await scheduledTaskActivity.dispatchScheduledTaskRun(
|
|
56
|
-
input.
|
|
57
|
-
? {
|
|
58
|
-
...base,
|
|
59
|
-
triggerType: input.triggerType,
|
|
60
|
-
agentRunUsageIdempotencyKey: input.agentRunUsageIdempotencyKey,
|
|
61
|
-
initiator: input.initiator,
|
|
62
|
-
}
|
|
63
|
-
: { ...base, triggerType: "scheduled" },
|
|
76
|
+
scheduledTaskDispatchInput(input, workflowInfo().workflowId),
|
|
64
77
|
);
|
|
65
78
|
if (result.action !== "knowledge_source_sync") return;
|
|
66
79
|
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { proxyActivities } from "@temporalio/workflow";
|
|
2
|
+
import type * as activities from "../activities";
|
|
3
|
+
|
|
4
|
+
const storageActivity = proxyActivities<Pick<typeof activities, "maintainSessionStorage">>({
|
|
5
|
+
startToCloseTimeout: "15 minutes",
|
|
6
|
+
retry: { maximumAttempts: 1 },
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
const archiveActivity = proxyActivities<Pick<typeof activities, "archiveIdleSessions">>({
|
|
10
|
+
// The activity keeps its own soft budget below this bound.
|
|
11
|
+
startToCloseTimeout: "60 minutes",
|
|
12
|
+
retry: { maximumAttempts: 1 },
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
/** One bounded session storage maintenance pass; the Temporal Schedule owns the cadence. */
|
|
16
|
+
export async function sessionStorageMaintenanceWorkflow(): Promise<void> {
|
|
17
|
+
await storageActivity.maintainSessionStorage();
|
|
18
|
+
await archiveActivity.archiveIdleSessions();
|
|
19
|
+
}
|