@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
package/src/index.ts
CHANGED
|
@@ -10,10 +10,10 @@ import {
|
|
|
10
10
|
import {
|
|
11
11
|
assertRuntimeDatabasePosture,
|
|
12
12
|
isRetryableRuntimeDatabaseStartupError,
|
|
13
|
-
countSessionRecoveryBacklog,
|
|
14
13
|
createDb,
|
|
15
14
|
getContextCompactionPendingSummary,
|
|
16
15
|
markSessionWorkflowWakeDelivered,
|
|
16
|
+
summarizeSessionRecoveryBacklog,
|
|
17
17
|
type Database,
|
|
18
18
|
type RuntimeDatabasePostureOptions,
|
|
19
19
|
} from "@opengeni/db";
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
} from "@temporalio/client";
|
|
38
38
|
import { NativeConnection, Worker, type WorkflowBundleOption } from "@temporalio/worker";
|
|
39
39
|
import { existsSync } from "node:fs";
|
|
40
|
+
import { prepareObjectStorageProvider } from "@opengeni/storage";
|
|
40
41
|
import { fileURLToPath } from "node:url";
|
|
41
42
|
import type { createControlActivities } from "./activities-control";
|
|
42
43
|
import type { createTurnActivities } from "./activities-turn";
|
|
@@ -50,6 +51,13 @@ import type {
|
|
|
50
51
|
WakeSessionWorkflowSignal,
|
|
51
52
|
} from "./activities/types";
|
|
52
53
|
import { turnTaskQueue } from "./workflows/activities";
|
|
54
|
+
import { createTurnTaskQueueStatsClient } from "./turn-task-queue-reader";
|
|
55
|
+
import { installTemporalRuntimeLogging } from "./temporal-runtime-logger";
|
|
56
|
+
import {
|
|
57
|
+
closeWorkerHttpAfterActivityExecution,
|
|
58
|
+
withTurnWorkerActivityTelemetry,
|
|
59
|
+
type TurnWorkerActivityTelemetry,
|
|
60
|
+
} from "./turn-worker-activity-telemetry";
|
|
53
61
|
import {
|
|
54
62
|
dbReadyCheck,
|
|
55
63
|
natsReadyCheck,
|
|
@@ -60,20 +68,19 @@ import {
|
|
|
60
68
|
import {
|
|
61
69
|
initializeContextCompactionMetrics,
|
|
62
70
|
initializeWorkerOutcomeMetrics,
|
|
63
|
-
|
|
64
|
-
observabilityEventLogger,
|
|
71
|
+
observabilityEventBusOptions,
|
|
65
72
|
startContextCompactionPendingMonitor,
|
|
66
73
|
startSessionRecoveryMonitor,
|
|
67
74
|
startTurnCapacityMonitor,
|
|
68
75
|
type TurnTaskQueueStats,
|
|
76
|
+
type TurnTaskQueueReadOptions,
|
|
77
|
+
type TurnTaskQueueIdentity,
|
|
69
78
|
} from "./observability-metrics";
|
|
70
79
|
import {
|
|
71
80
|
resolveCatalogSettings,
|
|
72
|
-
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
73
|
-
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
74
|
-
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
75
81
|
TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS,
|
|
76
82
|
} from "@opengeni/core";
|
|
83
|
+
import { startInternalMaintenanceCadences } from "./internal-maintenance";
|
|
77
84
|
import {
|
|
78
85
|
CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
79
86
|
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
@@ -85,6 +92,7 @@ import {
|
|
|
85
92
|
combineWorkerRunTargets,
|
|
86
93
|
constructWithOwnedConnection,
|
|
87
94
|
createWorkerServiceLifecycle,
|
|
95
|
+
createWorkerCleanupContainment,
|
|
88
96
|
type WorkerServiceLifecycle,
|
|
89
97
|
type WorkerRunTarget,
|
|
90
98
|
} from "./worker-service-lifecycle";
|
|
@@ -96,9 +104,14 @@ import {
|
|
|
96
104
|
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
97
105
|
import { installWorkerUnhandledRejectionBoundary } from "./unhandled-rejection-boundary";
|
|
98
106
|
import {
|
|
107
|
+
SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
99
108
|
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
100
109
|
sandboxLifecycleTaskQueue,
|
|
101
110
|
} from "./sandbox-reaper-contract";
|
|
111
|
+
import {
|
|
112
|
+
BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
113
|
+
browserDeadlineCheckpointTaskQueue,
|
|
114
|
+
} from "./browser-deadline-checkpoint-contract";
|
|
102
115
|
|
|
103
116
|
export {
|
|
104
117
|
createHostExportPump,
|
|
@@ -108,23 +121,27 @@ export {
|
|
|
108
121
|
type HostEventExport,
|
|
109
122
|
type HostEventExportBatch,
|
|
110
123
|
type HostEventSink,
|
|
124
|
+
type HostLifecycleFactExport,
|
|
125
|
+
type HostLifecycleFactExportBatch,
|
|
126
|
+
type HostLifecycleFactSink,
|
|
111
127
|
type HostUsageExport,
|
|
112
128
|
type HostUsageExportBatch,
|
|
113
129
|
type HostUsageSink,
|
|
114
130
|
} from "./host-export";
|
|
131
|
+
export {
|
|
132
|
+
INTERNAL_MAINTENANCE_JOBS,
|
|
133
|
+
KNOWLEDGE_INDEXING_PERIOD_MS,
|
|
134
|
+
KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
135
|
+
SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
136
|
+
SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
137
|
+
startInternalMaintenanceCadences,
|
|
138
|
+
type InternalMaintenanceJob,
|
|
139
|
+
} from "./internal-maintenance";
|
|
115
140
|
|
|
116
|
-
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
117
|
-
// create() is idempotent across every worker in the pool: the first worker to
|
|
118
|
-
// boot creates it, all others collide on this id (ScheduleAlreadyRunning) and
|
|
119
|
-
// no-op — so the Schedule is registered EXACTLY ONCE per deployment regardless
|
|
120
|
-
// of replica count.
|
|
121
|
-
const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
122
|
-
export const KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
|
|
123
|
-
export const KNOWLEDGE_INDEXING_PERIOD_MS = 15_000;
|
|
124
141
|
export const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
125
142
|
export const FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1_000;
|
|
126
|
-
export const
|
|
127
|
-
export const
|
|
143
|
+
export const SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID = "opengeni-session-storage-maintenance";
|
|
144
|
+
export const SESSION_STORAGE_MAINTENANCE_PERIOD_MS = 5 * 60 * 1_000;
|
|
128
145
|
export type OpenGeniWorkerRole = "control" | "turn";
|
|
129
146
|
|
|
130
147
|
export type WorkerOptions = {
|
|
@@ -178,6 +195,9 @@ export async function createDefaultWorkerActivities(
|
|
|
178
195
|
role: OpenGeniWorkerRole,
|
|
179
196
|
dependencies: ActivityDependencies = {},
|
|
180
197
|
) {
|
|
198
|
+
if (dependencies.objectStorage == null) {
|
|
199
|
+
prepareObjectStorageProvider(dependencies.settings ?? getSettings());
|
|
200
|
+
}
|
|
181
201
|
return role === "control"
|
|
182
202
|
? (await import("./activities-control")).createControlActivities(dependencies)
|
|
183
203
|
: (await import("./activities-turn")).createTurnActivities(dependencies);
|
|
@@ -203,7 +223,7 @@ export function resolveOpenGeniWorkflowDefinition(
|
|
|
203
223
|
const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
|
|
204
224
|
if (!existsSync(codePath)) {
|
|
205
225
|
throw new Error(
|
|
206
|
-
`
|
|
226
|
+
`Opengeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`,
|
|
207
227
|
);
|
|
208
228
|
}
|
|
209
229
|
return { workflowBundle: { codePath } };
|
|
@@ -219,6 +239,8 @@ export function turnCancellationHeartbeatThrottleOptions() {
|
|
|
219
239
|
export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
220
240
|
worker: WorkerRunTarget;
|
|
221
241
|
connection: NativeConnection;
|
|
242
|
+
activityTelemetry?: TurnWorkerActivityTelemetry;
|
|
243
|
+
turnQueueIdentity?: TurnTaskQueueIdentity;
|
|
222
244
|
}> {
|
|
223
245
|
const settings = options.settings ?? getSettings();
|
|
224
246
|
if (options.role === "control") {
|
|
@@ -264,10 +286,20 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
264
286
|
},
|
|
265
287
|
),
|
|
266
288
|
async (connection) => {
|
|
289
|
+
let turnWorker: Worker | undefined;
|
|
290
|
+
let cleanupDrainRequested = false;
|
|
267
291
|
const activityDependencies = {
|
|
268
292
|
...options.activityDependencies,
|
|
269
293
|
settings,
|
|
270
294
|
observability,
|
|
295
|
+
requestWorkerDrain:
|
|
296
|
+
options.activityDependencies?.requestWorkerDrain ??
|
|
297
|
+
(() => {
|
|
298
|
+
if (cleanupDrainRequested) return;
|
|
299
|
+
if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
|
|
300
|
+
turnWorker.shutdown();
|
|
301
|
+
cleanupDrainRequested = true;
|
|
302
|
+
}),
|
|
271
303
|
};
|
|
272
304
|
const activities =
|
|
273
305
|
options.activities ??
|
|
@@ -316,32 +348,58 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
|
|
|
316
348
|
: turnConcurrency!.options),
|
|
317
349
|
});
|
|
318
350
|
if (options.role !== "control") {
|
|
351
|
+
turnWorker = worker;
|
|
319
352
|
turnConcurrency?.admission?.finalizeStartupBaseline();
|
|
320
|
-
|
|
353
|
+
const turnQueueIdentity = {
|
|
354
|
+
temporalNamespace: worker.options.namespace,
|
|
355
|
+
taskQueue: worker.options.taskQueue,
|
|
356
|
+
};
|
|
357
|
+
return {
|
|
358
|
+
...withTurnWorkerActivityTelemetry(worker, {
|
|
359
|
+
observability,
|
|
360
|
+
identity: turnQueueIdentity,
|
|
361
|
+
// No invented identity when running outside Kubernetes. Scaler
|
|
362
|
+
// consumers must reject empty/missing UID coverage.
|
|
363
|
+
podUid: process.env.OPENGENI_POD_UID ?? "",
|
|
364
|
+
}),
|
|
365
|
+
connection,
|
|
366
|
+
turnQueueIdentity,
|
|
367
|
+
};
|
|
321
368
|
}
|
|
322
369
|
|
|
370
|
+
const ownedWorkers = [worker];
|
|
323
371
|
try {
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
372
|
+
// The new activity type has its own queue. Retain the V1 poller so old
|
|
373
|
+
// drain histories finish; old binaries never receive checkpoint work.
|
|
374
|
+
for (const taskQueue of [
|
|
375
|
+
sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
376
|
+
browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
|
|
377
|
+
]) {
|
|
378
|
+
const lifecycleWorker = await Worker.create({
|
|
379
|
+
...sharedWorkerOptions,
|
|
380
|
+
...workflowDefinition,
|
|
381
|
+
taskQueue,
|
|
382
|
+
reuseV8Context: true,
|
|
383
|
+
workflowThreadPoolSize: 1,
|
|
384
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
385
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
386
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
387
|
+
});
|
|
388
|
+
ownedWorkers.push(lifecycleWorker);
|
|
389
|
+
}
|
|
334
390
|
return {
|
|
335
|
-
worker: combineWorkerRunTargets(
|
|
391
|
+
worker: combineWorkerRunTargets(ownedWorkers),
|
|
336
392
|
connection,
|
|
337
393
|
};
|
|
338
394
|
} catch (error) {
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
395
|
+
for (const ownedWorker of ownedWorkers) {
|
|
396
|
+
try {
|
|
397
|
+
ownedWorker.shutdown();
|
|
398
|
+
} catch {
|
|
399
|
+
// Preserve the lifecycle worker construction failure. The base poller
|
|
400
|
+
// still received its shutdown request; a secondary synchronous
|
|
401
|
+
// shutdown error must not replace the startup root cause.
|
|
402
|
+
}
|
|
345
403
|
}
|
|
346
404
|
throw error;
|
|
347
405
|
}
|
|
@@ -363,13 +421,18 @@ export async function createWorkerWorkflowSignaler(
|
|
|
363
421
|
signalSessionAttemptQuiesced: SignalSessionAttemptQuiesced;
|
|
364
422
|
inspectSessionAttemptActivity: InspectSessionAttemptActivity;
|
|
365
423
|
signalCodexCapacityWorkflow: SignalCodexCapacityWorkflow;
|
|
366
|
-
getTurnTaskQueueStats: (
|
|
424
|
+
getTurnTaskQueueStats: (
|
|
425
|
+
options?: TurnTaskQueueReadOptions,
|
|
426
|
+
identity?: TurnTaskQueueIdentity,
|
|
427
|
+
) => Promise<TurnTaskQueueStats>;
|
|
367
428
|
startSandboxReaperWorkflow: StartSandboxReaperWorkflow;
|
|
368
429
|
startVideoGenerationWorkflow: StartVideoGenerationWorkflow;
|
|
369
430
|
check: () => Promise<void>;
|
|
370
431
|
close: () => Promise<void>;
|
|
371
432
|
}> {
|
|
372
|
-
const
|
|
433
|
+
const connectionOptions = temporalConnectionOptions(settings);
|
|
434
|
+
const connection = await Connection.connect(connectionOptions);
|
|
435
|
+
const statsClient = createTurnTaskQueueStatsClient(connectionOptions);
|
|
373
436
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
374
437
|
return {
|
|
375
438
|
wakeSessionWorkflow: async ({
|
|
@@ -462,33 +525,48 @@ export async function createWorkerWorkflowSignaler(
|
|
|
462
525
|
// another producer may have advanced it with a Pause/Steer that requires
|
|
463
526
|
// sessionControl. The global dispatcher owns that acknowledgement.
|
|
464
527
|
},
|
|
465
|
-
getTurnTaskQueueStats:
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
taskQueueType: 2,
|
|
473
|
-
reportStats: true,
|
|
474
|
-
});
|
|
475
|
-
return normalizeTurnTaskQueueStats(response.stats);
|
|
476
|
-
},
|
|
528
|
+
getTurnTaskQueueStats: (
|
|
529
|
+
readOptions = { signal: new AbortController().signal, deadline: Date.now() + 5_000 },
|
|
530
|
+
identity = {
|
|
531
|
+
temporalNamespace: settings.temporalNamespace,
|
|
532
|
+
taskQueue: turnTaskQueue(settings.temporalTaskQueue),
|
|
533
|
+
},
|
|
534
|
+
) => statsClient.read(readOptions, identity),
|
|
477
535
|
startSandboxReaperWorkflow: async () => {
|
|
478
|
-
|
|
479
|
-
|
|
536
|
+
// Same backend tick, no additional schedule. A bounded independent
|
|
537
|
+
// inventory cannot prevent the existing per-box drain from starting.
|
|
538
|
+
const starts = await Promise.allSettled([
|
|
539
|
+
temporal.workflow.start("sandboxReaperWorkflowV2", {
|
|
480
540
|
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
481
541
|
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
482
542
|
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
483
543
|
args: [],
|
|
484
|
-
})
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
544
|
+
}),
|
|
545
|
+
temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
|
|
546
|
+
taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
|
|
547
|
+
workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
548
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
549
|
+
args: [],
|
|
550
|
+
}),
|
|
551
|
+
// Its own workflow type, so the drain workflow's history is unchanged.
|
|
552
|
+
temporal.workflow.start("sandboxIdleCheckpointSweepWorkflow", {
|
|
553
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
554
|
+
workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
555
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
556
|
+
args: [],
|
|
557
|
+
}),
|
|
558
|
+
]);
|
|
559
|
+
for (const start of starts) {
|
|
560
|
+
if (
|
|
561
|
+
start.status === "rejected" &&
|
|
562
|
+
!(start.reason instanceof WorkflowExecutionAlreadyStartedError)
|
|
563
|
+
) {
|
|
564
|
+
// The other start already settled. Retry the router without losing
|
|
565
|
+
// ordinary draining or silently omitting the checkpoint sweep.
|
|
566
|
+
throw start.reason;
|
|
489
567
|
}
|
|
490
|
-
throw error;
|
|
491
568
|
}
|
|
569
|
+
return starts[0].status === "fulfilled" ? "started" : "already_running";
|
|
492
570
|
},
|
|
493
571
|
startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
|
|
494
572
|
try {
|
|
@@ -521,143 +599,11 @@ export async function createWorkerWorkflowSignaler(
|
|
|
521
599
|
await connection.workflowService.getSystemInfo({});
|
|
522
600
|
},
|
|
523
601
|
close: async () => {
|
|
524
|
-
await connection.close();
|
|
602
|
+
await Promise.all([statsClient.close(), connection.close()]);
|
|
525
603
|
},
|
|
526
604
|
};
|
|
527
605
|
}
|
|
528
606
|
|
|
529
|
-
/**
|
|
530
|
-
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
531
|
-
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
532
|
-
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
533
|
-
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
534
|
-
*
|
|
535
|
-
* The Schedule permanently fires sandboxReaperWorkflow on the legacy/base queue
|
|
536
|
-
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
537
|
-
* `reaperPeriod < viewerHolderTTL` and `reaperPeriod + idleGrace < providerLifetime`
|
|
538
|
-
* validates in packages/config — wiring the schedule period to it). SKIP overlap means a slow
|
|
539
|
-
* sweep never overlaps itself. Idempotent: a duplicate scheduleId across the
|
|
540
|
-
* worker pool collides on ScheduleAlreadyRunning and reconciles the same
|
|
541
|
-
* definition. New workers route onto the versioned lifecycle queue; old workers
|
|
542
|
-
* can still run the legacy action during rollout or a full binary rollback.
|
|
543
|
-
*
|
|
544
|
-
* Returns a `close()` for the dedicated client connection (separate from the
|
|
545
|
-
* worker's NativeConnection — the Schedule client is a @temporalio/client).
|
|
546
|
-
*/
|
|
547
|
-
export async function registerSandboxReaperSchedule(
|
|
548
|
-
settings: Settings,
|
|
549
|
-
observability: Observability,
|
|
550
|
-
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
551
|
-
assertSandboxReaperActivityTimeout(settings);
|
|
552
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
553
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
554
|
-
const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
|
|
555
|
-
const action = {
|
|
556
|
-
type: "startWorkflow" as const,
|
|
557
|
-
workflowType: "sandboxReaperWorkflow",
|
|
558
|
-
taskQueue: settings.temporalTaskQueue,
|
|
559
|
-
args: [] as [],
|
|
560
|
-
};
|
|
561
|
-
const policies = {
|
|
562
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
563
|
-
catchupWindow: "1m",
|
|
564
|
-
pauseOnFailure: false,
|
|
565
|
-
} as const;
|
|
566
|
-
try {
|
|
567
|
-
await temporal.schedule.create({
|
|
568
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
569
|
-
// @every-style interval: fire once per reaper period. The boot invariant
|
|
570
|
-
// guarantees reaperPeriod < viewerHolderTTL and
|
|
571
|
-
// reaperPeriod + idleGrace < providerLifetime.
|
|
572
|
-
spec,
|
|
573
|
-
action,
|
|
574
|
-
// The base workflow is a short router on new binaries; legacy binaries
|
|
575
|
-
// still execute the pre-refactor composite under this same overlap fence.
|
|
576
|
-
policies,
|
|
577
|
-
});
|
|
578
|
-
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
579
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
580
|
-
reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs,
|
|
581
|
-
});
|
|
582
|
-
return {
|
|
583
|
-
registered: true,
|
|
584
|
-
close: async () => {
|
|
585
|
-
await connection.close();
|
|
586
|
-
},
|
|
587
|
-
};
|
|
588
|
-
} catch (error) {
|
|
589
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
590
|
-
// Never move this action off the base queue: doing so would leave a full
|
|
591
|
-
// rollback with no worker polling the only reaper Schedule. The workflow
|
|
592
|
-
// implementation itself owns the forward-compatible queue handoff.
|
|
593
|
-
await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
|
|
594
|
-
spec,
|
|
595
|
-
action,
|
|
596
|
-
policies,
|
|
597
|
-
state: {
|
|
598
|
-
paused: previous.state.paused,
|
|
599
|
-
...(previous.state.note ? { note: previous.state.note } : {}),
|
|
600
|
-
...(previous.state.remainingActions !== undefined
|
|
601
|
-
? { remainingActions: previous.state.remainingActions }
|
|
602
|
-
: {}),
|
|
603
|
-
},
|
|
604
|
-
}));
|
|
605
|
-
observability.info("Reconciled the global sandbox-lease reaper Schedule", {
|
|
606
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
607
|
-
temporalTaskQueue: action.taskQueue,
|
|
608
|
-
});
|
|
609
|
-
return {
|
|
610
|
-
registered: false,
|
|
611
|
-
close: async () => {
|
|
612
|
-
await connection.close();
|
|
613
|
-
},
|
|
614
|
-
};
|
|
615
|
-
}
|
|
616
|
-
await connection.close().catch(() => undefined);
|
|
617
|
-
throw error;
|
|
618
|
-
}
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
/** Register the deployment-wide worker for rebuildable Knowledge search projections. */
|
|
622
|
-
export async function registerKnowledgeIndexingSchedule(
|
|
623
|
-
settings: Settings,
|
|
624
|
-
observability: Observability,
|
|
625
|
-
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
626
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
627
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
628
|
-
try {
|
|
629
|
-
await temporal.schedule.create({
|
|
630
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
631
|
-
spec: { intervals: [{ every: KNOWLEDGE_INDEXING_PERIOD_MS }] },
|
|
632
|
-
action: {
|
|
633
|
-
type: "startWorkflow",
|
|
634
|
-
workflowType: "knowledgeIndexingWorkflow",
|
|
635
|
-
taskQueue: settings.temporalTaskQueue,
|
|
636
|
-
args: [],
|
|
637
|
-
},
|
|
638
|
-
policies: {
|
|
639
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
640
|
-
catchupWindow: "1m",
|
|
641
|
-
pauseOnFailure: false,
|
|
642
|
-
},
|
|
643
|
-
});
|
|
644
|
-
observability.info("Registered the global Knowledge indexer Schedule", {
|
|
645
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
646
|
-
indexingPeriodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
|
|
647
|
-
});
|
|
648
|
-
return { registered: true, close: async () => connection.close() };
|
|
649
|
-
} catch (error) {
|
|
650
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
651
|
-
observability.info("Global Knowledge indexer Schedule already registered", {
|
|
652
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
653
|
-
});
|
|
654
|
-
return { registered: false, close: async () => connection.close() };
|
|
655
|
-
}
|
|
656
|
-
await connection.close().catch(() => undefined);
|
|
657
|
-
throw error;
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
607
|
/**
|
|
662
608
|
* Register the one provider-neutral expired direct-upload cleanup Schedule.
|
|
663
609
|
* Unlike sandbox GC this is always registered: file uploads can be enabled in
|
|
@@ -703,54 +649,12 @@ export async function registerFileUploadReaperSchedule(
|
|
|
703
649
|
}
|
|
704
650
|
}
|
|
705
651
|
|
|
706
|
-
/** Register the deployment-wide maintained-auth dispatcher. Its one-minute
|
|
707
|
-
* cadence matches the minimum public policy interval; DB claims own overlap,
|
|
708
|
-
* crash recovery, and exact session idempotency. */
|
|
709
|
-
export async function registerSiteAuthMaintenanceSchedule(
|
|
710
|
-
settings: Settings,
|
|
711
|
-
observability: Observability,
|
|
712
|
-
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
713
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
714
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
715
|
-
try {
|
|
716
|
-
await temporal.schedule.create({
|
|
717
|
-
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
718
|
-
spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
|
|
719
|
-
action: {
|
|
720
|
-
type: "startWorkflow",
|
|
721
|
-
workflowType: "siteAuthMaintenanceWorkflow",
|
|
722
|
-
taskQueue: settings.temporalTaskQueue,
|
|
723
|
-
args: [],
|
|
724
|
-
},
|
|
725
|
-
policies: {
|
|
726
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
727
|
-
catchupWindow: "1m",
|
|
728
|
-
pauseOnFailure: false,
|
|
729
|
-
},
|
|
730
|
-
});
|
|
731
|
-
observability.info("Registered the global site-auth maintenance Schedule", {
|
|
732
|
-
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
733
|
-
maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
734
|
-
});
|
|
735
|
-
return { registered: true, close: async () => connection.close() };
|
|
736
|
-
} catch (error) {
|
|
737
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
738
|
-
observability.info("Global site-auth maintenance Schedule already registered", {
|
|
739
|
-
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
740
|
-
});
|
|
741
|
-
return { registered: false, close: async () => connection.close() };
|
|
742
|
-
}
|
|
743
|
-
await connection.close().catch(() => undefined);
|
|
744
|
-
throw error;
|
|
745
|
-
}
|
|
746
|
-
}
|
|
747
|
-
|
|
748
652
|
/**
|
|
749
|
-
* Register the
|
|
750
|
-
*
|
|
751
|
-
*
|
|
653
|
+
* Register the deployment-wide session storage maintenance Schedule. Always
|
|
654
|
+
* registered: legacy content compaction is lossless. Overlap is skipped so one
|
|
655
|
+
* pass never races another.
|
|
752
656
|
*/
|
|
753
|
-
export async function
|
|
657
|
+
export async function registerSessionStorageMaintenanceSchedule(
|
|
754
658
|
settings: Settings,
|
|
755
659
|
observability: Observability,
|
|
756
660
|
): Promise<{ registered: boolean; close: () => Promise<void> }> {
|
|
@@ -758,11 +662,11 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
758
662
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
759
663
|
try {
|
|
760
664
|
await temporal.schedule.create({
|
|
761
|
-
scheduleId:
|
|
762
|
-
spec: { intervals: [{ every:
|
|
665
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
666
|
+
spec: { intervals: [{ every: SESSION_STORAGE_MAINTENANCE_PERIOD_MS }] },
|
|
763
667
|
action: {
|
|
764
668
|
type: "startWorkflow",
|
|
765
|
-
workflowType:
|
|
669
|
+
workflowType: "sessionStorageMaintenanceWorkflow",
|
|
766
670
|
taskQueue: settings.temporalTaskQueue,
|
|
767
671
|
args: [],
|
|
768
672
|
},
|
|
@@ -772,15 +676,15 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
772
676
|
pauseOnFailure: false,
|
|
773
677
|
},
|
|
774
678
|
});
|
|
775
|
-
observability.info("Registered the session
|
|
776
|
-
scheduleId:
|
|
777
|
-
periodMs:
|
|
679
|
+
observability.info("Registered the session storage maintenance Schedule", {
|
|
680
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
681
|
+
periodMs: SESSION_STORAGE_MAINTENANCE_PERIOD_MS,
|
|
778
682
|
});
|
|
779
683
|
return { registered: true, close: async () => connection.close() };
|
|
780
684
|
} catch (error) {
|
|
781
685
|
if (error instanceof ScheduleAlreadyRunning) {
|
|
782
|
-
observability.info("Session
|
|
783
|
-
scheduleId:
|
|
686
|
+
observability.info("Session storage maintenance Schedule already registered", {
|
|
687
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
784
688
|
});
|
|
785
689
|
return { registered: false, close: async () => connection.close() };
|
|
786
690
|
}
|
|
@@ -791,13 +695,16 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
|
|
|
791
695
|
|
|
792
696
|
export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
|
|
793
697
|
activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
|
|
698
|
+
/** Host-owned final containment after a stalled cleanup has drained peers.
|
|
699
|
+
* The standalone process exits; embedded hosts own their termination policy. */
|
|
700
|
+
terminateWorker?: () => void;
|
|
794
701
|
/**
|
|
795
702
|
* Exact catalog posture required by the standalone runtime. Embedded hosts
|
|
796
703
|
* may omit this when they own an equivalent database isolation contract.
|
|
797
704
|
*/
|
|
798
705
|
databasePosture?: RuntimeDatabasePostureOptions;
|
|
799
706
|
/**
|
|
800
|
-
* `role-default` registers
|
|
707
|
+
* `role-default` registers Opengeni's internal maintenance schedules on a
|
|
801
708
|
* control worker only. These are engine maintenance schedules, not a host's
|
|
802
709
|
* product-level scheduled-agent jobs. Use `none` when another control worker
|
|
803
710
|
* in the same deployment owns them.
|
|
@@ -847,6 +754,7 @@ export async function createOpenGeniWorkerService(
|
|
|
847
754
|
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
|
|
848
755
|
logStartupDependencyRetry(observability, event);
|
|
849
756
|
let lifecycle: WorkerServiceLifecycle | undefined;
|
|
757
|
+
let cleanupContainment: ReturnType<typeof createWorkerCleanupContainment> | undefined;
|
|
850
758
|
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
851
759
|
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
852
760
|
let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
|
|
@@ -864,7 +772,7 @@ export async function createOpenGeniWorkerService(
|
|
|
864
772
|
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
865
773
|
{ ...retryOptions, onRetry },
|
|
866
774
|
);
|
|
867
|
-
observability.info("
|
|
775
|
+
observability.info("Opengeni model catalog resolved", {
|
|
868
776
|
role: options.role,
|
|
869
777
|
catalogSource: resolvedCatalog.source,
|
|
870
778
|
catalogVersion: resolvedCatalog.version,
|
|
@@ -909,7 +817,7 @@ export async function createOpenGeniWorkerService(
|
|
|
909
817
|
!startSandboxReaperWorkflow ||
|
|
910
818
|
!startVideoGenerationWorkflow
|
|
911
819
|
) {
|
|
912
|
-
throw new Error("
|
|
820
|
+
throw new Error("Opengeni worker lifecycle could not resolve its workflow signalers");
|
|
913
821
|
}
|
|
914
822
|
workerBundle = await createOpenGeniWorker({
|
|
915
823
|
role: options.role,
|
|
@@ -926,6 +834,10 @@ export async function createOpenGeniWorkerService(
|
|
|
926
834
|
signalCodexCapacityWorkflow,
|
|
927
835
|
startSandboxReaperWorkflow,
|
|
928
836
|
startVideoGenerationWorkflow,
|
|
837
|
+
requestWorkerDrain: () => {
|
|
838
|
+
if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
|
|
839
|
+
cleanupContainment.request();
|
|
840
|
+
},
|
|
929
841
|
},
|
|
930
842
|
});
|
|
931
843
|
|
|
@@ -933,14 +845,19 @@ export async function createOpenGeniWorkerService(
|
|
|
933
845
|
if (!signaler) {
|
|
934
846
|
throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
|
|
935
847
|
}
|
|
848
|
+
const statsClient = signaler;
|
|
849
|
+
const identity = workerBundle.turnQueueIdentity;
|
|
850
|
+
if (!identity)
|
|
851
|
+
throw new Error("Turn worker did not expose its actual Temporal queue identity");
|
|
936
852
|
turnCapacityMonitor = startTurnCapacityMonitor({
|
|
937
853
|
observability,
|
|
938
|
-
read:
|
|
854
|
+
read: (readOptions) => statsClient.getTurnTaskQueueStats(readOptions, identity),
|
|
855
|
+
identity,
|
|
939
856
|
});
|
|
940
857
|
} else {
|
|
941
858
|
sessionRecoveryMonitor = startSessionRecoveryMonitor({
|
|
942
859
|
observability,
|
|
943
|
-
read: async () => await
|
|
860
|
+
read: async () => await summarizeSessionRecoveryBacklog(options.activityDependencies.db),
|
|
944
861
|
});
|
|
945
862
|
contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
|
|
946
863
|
observability,
|
|
@@ -949,20 +866,6 @@ export async function createOpenGeniWorkerService(
|
|
|
949
866
|
}
|
|
950
867
|
|
|
951
868
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
952
|
-
schedules.push(
|
|
953
|
-
await retryStartupDependency(
|
|
954
|
-
"Temporal schedule (sandbox reaper)",
|
|
955
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
956
|
-
{ ...retryOptions, onRetry },
|
|
957
|
-
),
|
|
958
|
-
);
|
|
959
|
-
schedules.push(
|
|
960
|
-
await retryStartupDependency(
|
|
961
|
-
"Temporal schedule (Knowledge indexing)",
|
|
962
|
-
() => registerKnowledgeIndexingSchedule(settings, observability),
|
|
963
|
-
{ ...retryOptions, onRetry },
|
|
964
|
-
),
|
|
965
|
-
);
|
|
966
869
|
schedules.push(
|
|
967
870
|
await retryStartupDependency(
|
|
968
871
|
"Temporal schedule (file upload reaper)",
|
|
@@ -972,33 +875,38 @@ export async function createOpenGeniWorkerService(
|
|
|
972
875
|
);
|
|
973
876
|
schedules.push(
|
|
974
877
|
await retryStartupDependency(
|
|
975
|
-
"Temporal schedule (
|
|
976
|
-
() =>
|
|
878
|
+
"Temporal schedule (session storage maintenance)",
|
|
879
|
+
() => registerSessionStorageMaintenanceSchedule(settings, observability),
|
|
977
880
|
{ ...retryOptions, onRetry },
|
|
978
881
|
),
|
|
979
882
|
);
|
|
980
883
|
schedules.push(
|
|
981
884
|
await retryStartupDependency(
|
|
982
|
-
"
|
|
983
|
-
() =>
|
|
885
|
+
"work-checked maintenance cadences",
|
|
886
|
+
() =>
|
|
887
|
+
startInternalMaintenanceCadences({
|
|
888
|
+
settings,
|
|
889
|
+
observability,
|
|
890
|
+
db: options.activityDependencies.db,
|
|
891
|
+
...(options.activityDependencies.documentServices
|
|
892
|
+
? { documentServices: options.activityDependencies.documentServices }
|
|
893
|
+
: {}),
|
|
894
|
+
}),
|
|
984
895
|
{ ...retryOptions, onRetry },
|
|
985
896
|
),
|
|
986
897
|
);
|
|
987
898
|
}
|
|
988
899
|
|
|
989
900
|
if (options.http !== false) {
|
|
990
|
-
const
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
);
|
|
901
|
+
const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
|
|
902
|
+
const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
|
|
903
|
+
await resolveCatalogSettings(readinessDb, settings);
|
|
904
|
+
});
|
|
994
905
|
httpServer = startWorkerHttpServer({
|
|
995
906
|
settings,
|
|
996
907
|
observability,
|
|
997
908
|
checks: {
|
|
998
|
-
db:
|
|
999
|
-
await databaseReady();
|
|
1000
|
-
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
1001
|
-
},
|
|
909
|
+
db: databaseReady,
|
|
1002
910
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
1003
911
|
temporal: temporalReadyCheck(workerBundle.connection),
|
|
1004
912
|
},
|
|
@@ -1007,6 +915,7 @@ export async function createOpenGeniWorkerService(
|
|
|
1007
915
|
});
|
|
1008
916
|
}
|
|
1009
917
|
} catch (error) {
|
|
918
|
+
workerBundle?.activityTelemetry?.closeIfNeverRun();
|
|
1010
919
|
httpServer?.stop(true);
|
|
1011
920
|
await Promise.allSettled([
|
|
1012
921
|
turnCapacityMonitor?.close(),
|
|
@@ -1022,16 +931,30 @@ export async function createOpenGeniWorkerService(
|
|
|
1022
931
|
const activeWorkerBundle = workerBundle;
|
|
1023
932
|
const activeSignaler = signaler;
|
|
1024
933
|
if (!activeWorkerBundle) {
|
|
1025
|
-
throw new Error("
|
|
934
|
+
throw new Error("Opengeni worker service initialization did not complete");
|
|
1026
935
|
}
|
|
1027
936
|
|
|
937
|
+
cleanupContainment = createWorkerCleanupContainment({
|
|
938
|
+
drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
|
|
939
|
+
...(options.terminateWorker ? { terminate: options.terminateWorker } : {}),
|
|
940
|
+
observability,
|
|
941
|
+
});
|
|
942
|
+
const activeCleanupContainment = cleanupContainment;
|
|
1028
943
|
lifecycle = createWorkerServiceLifecycle({
|
|
1029
944
|
role: options.role,
|
|
1030
|
-
worker:
|
|
945
|
+
worker: {
|
|
946
|
+
shutdown: () => activeWorkerBundle.worker.shutdown(),
|
|
947
|
+
run: async () => {
|
|
948
|
+
await activeWorkerBundle.worker.run();
|
|
949
|
+
activeCleanupContainment.finished();
|
|
950
|
+
},
|
|
951
|
+
},
|
|
1031
952
|
observability,
|
|
1032
953
|
closeOwnedResources: async () => {
|
|
1033
954
|
memoryPressureGuard?.close();
|
|
1034
|
-
|
|
955
|
+
closeWorkerHttpAfterActivityExecution(activeWorkerBundle.activityTelemetry, () =>
|
|
956
|
+
httpServer?.stop(true),
|
|
957
|
+
);
|
|
1035
958
|
await Promise.allSettled([
|
|
1036
959
|
turnCapacityMonitor?.close(),
|
|
1037
960
|
sessionRecoveryMonitor?.close(),
|
|
@@ -1042,7 +965,7 @@ export async function createOpenGeniWorkerService(
|
|
|
1042
965
|
]);
|
|
1043
966
|
},
|
|
1044
967
|
onReady: () => {
|
|
1045
|
-
observability.info("
|
|
968
|
+
observability.info("Opengeni worker listening", {
|
|
1046
969
|
role: options.role,
|
|
1047
970
|
temporalTaskQueue:
|
|
1048
971
|
options.role === "control"
|
|
@@ -1103,6 +1026,22 @@ export type RunOpenGeniWorkerOptions = OpenGeniWorkerServiceOptions & {
|
|
|
1103
1026
|
/** Start, drain, and close one embedded worker process. */
|
|
1104
1027
|
export async function runOpenGeniWorker(options: RunOpenGeniWorkerOptions): Promise<void> {
|
|
1105
1028
|
const service = await createOpenGeniWorkerService(options);
|
|
1029
|
+
// Reclaim multi-gigabyte workspace archive spools left by a previous worker
|
|
1030
|
+
// process that stopped mid-capture/upload. Only provably dead owners' spools
|
|
1031
|
+
// are removed, so peers sharing this TMPDIR are unaffected.
|
|
1032
|
+
void import("@opengeni/runtime/sandbox")
|
|
1033
|
+
.then(({ sweepOrphanedHostArchiveTemporaryDirectoriesOnce }) =>
|
|
1034
|
+
sweepOrphanedHostArchiveTemporaryDirectoriesOnce(),
|
|
1035
|
+
)
|
|
1036
|
+
.then((removed) => {
|
|
1037
|
+
if (removed.length > 0) {
|
|
1038
|
+
options.activityDependencies.observability?.info(
|
|
1039
|
+
"worker removed orphaned workspace archive spools",
|
|
1040
|
+
{ count: removed.length },
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
})
|
|
1044
|
+
.catch(() => undefined);
|
|
1106
1045
|
const signals =
|
|
1107
1046
|
options.shutdownSignals === false ? [] : (options.shutdownSignals ?? ["SIGTERM", "SIGINT"]);
|
|
1108
1047
|
const handlers = signals.map((signal) => {
|
|
@@ -1127,6 +1066,8 @@ export async function startWorker() {
|
|
|
1127
1066
|
}
|
|
1128
1067
|
const settings = getSettings();
|
|
1129
1068
|
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
1069
|
+
// Before any NativeConnection: SDK and core logs become structured lines.
|
|
1070
|
+
installTemporalRuntimeLogging(observability);
|
|
1130
1071
|
// The Agents SDK also listens for unhandled rejections and exits the process.
|
|
1131
1072
|
// Keep detached SDK/transport promises observational here: durable turn
|
|
1132
1073
|
// activity boundaries own failures, while process exit causes cross-session
|
|
@@ -1154,8 +1095,6 @@ export async function startWorker() {
|
|
|
1154
1095
|
rlsStrategy: settings.rlsStrategy,
|
|
1155
1096
|
expectedRole: settings.runtimeDatabaseRole,
|
|
1156
1097
|
targetSchema: settings.dbSchema.trim() || "public",
|
|
1157
|
-
organizationTenancyCanonicalActivationEnabled:
|
|
1158
|
-
settings.organizationTenancyCanonicalActivationEnabled,
|
|
1159
1098
|
} as const;
|
|
1160
1099
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
1161
1100
|
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
@@ -1173,12 +1112,13 @@ export async function startWorker() {
|
|
|
1173
1112
|
controlPlaneAuth
|
|
1174
1113
|
? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password }
|
|
1175
1114
|
: undefined,
|
|
1176
|
-
|
|
1115
|
+
observabilityEventBusOptions(observability),
|
|
1177
1116
|
),
|
|
1178
1117
|
{ ...retryOptions, onRetry },
|
|
1179
1118
|
);
|
|
1180
1119
|
await runOpenGeniWorker({
|
|
1181
1120
|
role,
|
|
1121
|
+
terminateWorker: () => process.exit(1),
|
|
1182
1122
|
settings,
|
|
1183
1123
|
databasePosture,
|
|
1184
1124
|
http: { readinessDb: readinessDbClient.db },
|