@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
package/dist/index.js
CHANGED
|
@@ -1,25 +1,30 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createHostExportPump
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-4ER3YXOT.js";
|
|
4
4
|
import {
|
|
5
5
|
CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
6
6
|
CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES,
|
|
7
7
|
CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
8
|
+
SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
8
9
|
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
9
10
|
assertSandboxReaperActivityTimeout,
|
|
10
11
|
createTurnWorkerConcurrencyPlan,
|
|
11
12
|
sandboxLifecycleTaskQueue,
|
|
12
13
|
turnWorkerConcurrencyLogFields
|
|
13
|
-
} from "./chunk-
|
|
14
|
+
} from "./chunk-24GP3BX5.js";
|
|
14
15
|
import {
|
|
16
|
+
createTelemetryClock,
|
|
15
17
|
initializeContextCompactionMetrics,
|
|
16
18
|
initializeWorkerOutcomeMetrics,
|
|
17
19
|
normalizeTurnTaskQueueStats,
|
|
18
|
-
|
|
20
|
+
observabilityEventBusOptions,
|
|
19
21
|
startContextCompactionPendingMonitor,
|
|
20
22
|
startSessionRecoveryMonitor,
|
|
21
|
-
startTurnCapacityMonitor
|
|
22
|
-
|
|
23
|
+
startTurnCapacityMonitor,
|
|
24
|
+
telemetryInterval,
|
|
25
|
+
telemetryScheduler,
|
|
26
|
+
turnTaskQueueMetricLabels
|
|
27
|
+
} from "./chunk-5NNS4327.js";
|
|
23
28
|
import "./chunk-PZ5AY32C.js";
|
|
24
29
|
|
|
25
30
|
// src/index.ts
|
|
@@ -29,15 +34,15 @@ import {
|
|
|
29
34
|
resolveNatsControlPlaneAuth,
|
|
30
35
|
retryStartupDependency,
|
|
31
36
|
startupRetryOptions,
|
|
32
|
-
temporalConnectionOptions
|
|
37
|
+
temporalConnectionOptions as temporalConnectionOptions2
|
|
33
38
|
} from "@opengeni/config";
|
|
34
39
|
import {
|
|
35
40
|
assertRuntimeDatabasePosture as assertRuntimeDatabasePosture2,
|
|
36
41
|
isRetryableRuntimeDatabaseStartupError,
|
|
37
|
-
countSessionRecoveryBacklog,
|
|
38
42
|
createDb,
|
|
39
43
|
getContextCompactionPendingSummary,
|
|
40
|
-
markSessionWorkflowWakeDelivered
|
|
44
|
+
markSessionWorkflowWakeDelivered,
|
|
45
|
+
summarizeSessionRecoveryBacklog
|
|
41
46
|
} from "@opengeni/db";
|
|
42
47
|
import {
|
|
43
48
|
createNatsEventBus,
|
|
@@ -48,19 +53,25 @@ import {
|
|
|
48
53
|
logStartupDependencyRetry
|
|
49
54
|
} from "@opengeni/observability";
|
|
50
55
|
import {
|
|
51
|
-
Connection,
|
|
56
|
+
Connection as Connection3,
|
|
52
57
|
isGrpcServiceError,
|
|
53
58
|
ScheduleAlreadyRunning,
|
|
54
59
|
ScheduleOverlapPolicy,
|
|
55
|
-
Client as
|
|
56
|
-
WorkflowExecutionAlreadyStartedError
|
|
60
|
+
Client as TemporalClient2,
|
|
61
|
+
WorkflowExecutionAlreadyStartedError as WorkflowExecutionAlreadyStartedError2
|
|
57
62
|
} from "@temporalio/client";
|
|
58
63
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
59
64
|
import { existsSync } from "fs";
|
|
65
|
+
import { prepareObjectStorageProvider } from "@opengeni/storage";
|
|
60
66
|
import { fileURLToPath } from "url";
|
|
61
67
|
|
|
62
68
|
// src/workflows/activities.ts
|
|
63
|
-
import {
|
|
69
|
+
import {
|
|
70
|
+
ActivityCancellationType,
|
|
71
|
+
ActivityFailure,
|
|
72
|
+
ApplicationFailure,
|
|
73
|
+
proxyActivities
|
|
74
|
+
} from "@temporalio/workflow";
|
|
64
75
|
|
|
65
76
|
// src/knowledge-source-sync-activity-policy.ts
|
|
66
77
|
var KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS = 12e4;
|
|
@@ -130,6 +141,251 @@ var knowledgeSourceSyncActivity = proxyActivities({
|
|
|
130
141
|
}
|
|
131
142
|
});
|
|
132
143
|
|
|
144
|
+
// src/turn-task-queue-reader.ts
|
|
145
|
+
import { Connection } from "@temporalio/client";
|
|
146
|
+
function createTurnTaskQueueStatsClient(connectionOptions, input = {}) {
|
|
147
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
148
|
+
const closeTimeoutMs = telemetryInterval(
|
|
149
|
+
input.closeTimeoutMs ?? 1e3,
|
|
150
|
+
"queue client closeTimeoutMs"
|
|
151
|
+
);
|
|
152
|
+
let connection;
|
|
153
|
+
let closed = false;
|
|
154
|
+
let closing;
|
|
155
|
+
const active = /* @__PURE__ */ new Set();
|
|
156
|
+
return {
|
|
157
|
+
read: async ({ signal, deadline }, identity) => {
|
|
158
|
+
signal.throwIfAborted();
|
|
159
|
+
if (closed) throw new Error("turn task-queue diagnostic client is closed");
|
|
160
|
+
const controller = new AbortController();
|
|
161
|
+
const abort = () => controller.abort(signal.reason);
|
|
162
|
+
signal.addEventListener("abort", abort, { once: true });
|
|
163
|
+
active.add(controller);
|
|
164
|
+
try {
|
|
165
|
+
connection ??= Connection.lazy({ ...connectionOptions, interceptors: [] });
|
|
166
|
+
const stats = await createTurnTaskQueueStatsReader(
|
|
167
|
+
connection,
|
|
168
|
+
identity
|
|
169
|
+
)({
|
|
170
|
+
signal: controller.signal,
|
|
171
|
+
deadline
|
|
172
|
+
});
|
|
173
|
+
controller.signal.throwIfAborted();
|
|
174
|
+
return stats;
|
|
175
|
+
} finally {
|
|
176
|
+
active.delete(controller);
|
|
177
|
+
signal.removeEventListener("abort", abort);
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
close: () => {
|
|
181
|
+
if (closing) return closing;
|
|
182
|
+
closed = true;
|
|
183
|
+
for (const controller of active) controller.abort();
|
|
184
|
+
const owned = connection;
|
|
185
|
+
if (!owned) return closing = Promise.resolve();
|
|
186
|
+
closing = new Promise((resolve) => {
|
|
187
|
+
const cancelTimeout = scheduler.timeout(resolve, closeTimeoutMs);
|
|
188
|
+
const settled = () => {
|
|
189
|
+
cancelTimeout();
|
|
190
|
+
resolve();
|
|
191
|
+
};
|
|
192
|
+
try {
|
|
193
|
+
void owned.close().then(settled, settled);
|
|
194
|
+
} catch {
|
|
195
|
+
settled();
|
|
196
|
+
}
|
|
197
|
+
});
|
|
198
|
+
return closing;
|
|
199
|
+
}
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
function createTurnTaskQueueStatsReader(connection, identity) {
|
|
203
|
+
return async ({ signal, deadline }) => {
|
|
204
|
+
signal.throwIfAborted();
|
|
205
|
+
return connection.withAbortSignal(
|
|
206
|
+
signal,
|
|
207
|
+
() => connection.withDeadline(deadline, async () => {
|
|
208
|
+
signal.throwIfAborted();
|
|
209
|
+
const response = await connection.workflowService.describeTaskQueue({
|
|
210
|
+
namespace: identity.temporalNamespace,
|
|
211
|
+
taskQueue: { name: identity.taskQueue },
|
|
212
|
+
// TASK_QUEUE_TYPE_ACTIVITY, supported DEFAULT mode. This is the shared
|
|
213
|
+
// turn-worker activity queue (also video/retries), not a turn count.
|
|
214
|
+
taskQueueType: 2,
|
|
215
|
+
reportStats: true
|
|
216
|
+
});
|
|
217
|
+
return normalizeTurnTaskQueueStats(response.stats);
|
|
218
|
+
})
|
|
219
|
+
);
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// src/temporal-runtime-logger.ts
|
|
224
|
+
import { createTemporalSdkLogger } from "@opengeni/observability";
|
|
225
|
+
import { makeTelemetryFilterString, Runtime } from "@temporalio/worker";
|
|
226
|
+
function temporalRuntimeLoggingOptions(observability) {
|
|
227
|
+
return {
|
|
228
|
+
logger: createTemporalSdkLogger(observability),
|
|
229
|
+
telemetryOptions: {
|
|
230
|
+
logging: {
|
|
231
|
+
// The SDK's default core filter, now forwarded instead of printed.
|
|
232
|
+
filter: makeTelemetryFilterString({ core: "WARN", other: "ERROR" }),
|
|
233
|
+
forward: {}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
};
|
|
237
|
+
}
|
|
238
|
+
function installTemporalRuntimeLogging(observability, runtime = Runtime) {
|
|
239
|
+
if (runtime._instance !== void 0) return false;
|
|
240
|
+
runtime.install(temporalRuntimeLoggingOptions(observability));
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// src/turn-worker-activity-telemetry.ts
|
|
245
|
+
var TURN_WORKER_ACTIVITY_METRICS = {
|
|
246
|
+
inflight: "opengeni_turn_worker_activities_inflight",
|
|
247
|
+
timestamp: "opengeni_turn_worker_activities_last_observed_timestamp_seconds",
|
|
248
|
+
valid: "opengeni_turn_worker_activities_last_read_success"
|
|
249
|
+
};
|
|
250
|
+
function closeWorkerHttpAfterActivityExecution(telemetry, close) {
|
|
251
|
+
telemetry?.closeIfNeverRun();
|
|
252
|
+
if (!telemetry || telemetry.isSettled()) {
|
|
253
|
+
try {
|
|
254
|
+
void Promise.resolve(close()).catch(() => void 0);
|
|
255
|
+
} catch {
|
|
256
|
+
}
|
|
257
|
+
} else {
|
|
258
|
+
void telemetry.settled.then(close).catch(() => void 0);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
function withTurnWorkerActivityTelemetry(sdkWorker, input) {
|
|
262
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
263
|
+
const intervalMs = telemetryInterval(input.intervalMs ?? 15e3, "activity telemetry intervalMs");
|
|
264
|
+
const clock = createTelemetryClock(input.now ?? Date.now);
|
|
265
|
+
const labels = { ...turnTaskQueueMetricLabels(input.identity), worker_pod_uid: input.podUid };
|
|
266
|
+
let runStarted = false;
|
|
267
|
+
let runSettled = false;
|
|
268
|
+
let runPromise;
|
|
269
|
+
let disposed = false;
|
|
270
|
+
let queued = false;
|
|
271
|
+
let subscription;
|
|
272
|
+
let stopTimer;
|
|
273
|
+
let resolveSettled;
|
|
274
|
+
const settled = new Promise((resolve) => {
|
|
275
|
+
resolveSettled = resolve;
|
|
276
|
+
});
|
|
277
|
+
const set = (name, help, value) => input.observability.setGauge({ name, help, labels, value });
|
|
278
|
+
const invalidate = () => {
|
|
279
|
+
if (disposed) return;
|
|
280
|
+
try {
|
|
281
|
+
set(
|
|
282
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
283
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
284
|
+
0
|
|
285
|
+
);
|
|
286
|
+
} catch {
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
const dispose = () => {
|
|
290
|
+
if (disposed) return;
|
|
291
|
+
disposed = true;
|
|
292
|
+
stopTimer?.();
|
|
293
|
+
try {
|
|
294
|
+
subscription?.unsubscribe();
|
|
295
|
+
} catch {
|
|
296
|
+
}
|
|
297
|
+
resolveSettled();
|
|
298
|
+
};
|
|
299
|
+
const sample = () => {
|
|
300
|
+
if (disposed) return;
|
|
301
|
+
let idle = false;
|
|
302
|
+
try {
|
|
303
|
+
const status = sdkWorker.getStatus();
|
|
304
|
+
const count = status.numInFlightNonLocalActivities;
|
|
305
|
+
const local = status.numInFlightLocalActivities;
|
|
306
|
+
const total = status.numInFlightActivities;
|
|
307
|
+
if (![count, local, total].every((value) => Number.isSafeInteger(value) && value >= 0) || count + local !== total) {
|
|
308
|
+
throw new Error("invalid SDK executing activity state");
|
|
309
|
+
}
|
|
310
|
+
idle = total === 0;
|
|
311
|
+
const observedAt = clock.read();
|
|
312
|
+
if (observedAt === null) throw new Error("invalid activity observation clock");
|
|
313
|
+
set(
|
|
314
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
315
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
316
|
+
0
|
|
317
|
+
);
|
|
318
|
+
set(
|
|
319
|
+
TURN_WORKER_ACTIVITY_METRICS.inflight,
|
|
320
|
+
"SDK executing non-local activities on this turn worker, including preclaim, video and cancellation-unresolved execution.",
|
|
321
|
+
count
|
|
322
|
+
);
|
|
323
|
+
set(
|
|
324
|
+
TURN_WORKER_ACTIVITY_METRICS.timestamp,
|
|
325
|
+
"Unix timestamp of the latest successful SDK activity occupancy observation.",
|
|
326
|
+
observedAt / 1e3
|
|
327
|
+
);
|
|
328
|
+
set(
|
|
329
|
+
TURN_WORKER_ACTIVITY_METRICS.valid,
|
|
330
|
+
"Whether the latest SDK activity occupancy observation succeeded.",
|
|
331
|
+
1
|
|
332
|
+
);
|
|
333
|
+
} catch {
|
|
334
|
+
invalidate();
|
|
335
|
+
}
|
|
336
|
+
if (runSettled && idle) dispose();
|
|
337
|
+
};
|
|
338
|
+
const scheduleSample = () => {
|
|
339
|
+
if (disposed || queued) return;
|
|
340
|
+
queued = true;
|
|
341
|
+
scheduler.defer(() => {
|
|
342
|
+
queued = false;
|
|
343
|
+
sample();
|
|
344
|
+
});
|
|
345
|
+
};
|
|
346
|
+
try {
|
|
347
|
+
subscription = sdkWorker.numInFlightActivities$.subscribe({
|
|
348
|
+
next: scheduleSample,
|
|
349
|
+
error: invalidate
|
|
350
|
+
});
|
|
351
|
+
} catch {
|
|
352
|
+
invalidate();
|
|
353
|
+
}
|
|
354
|
+
stopTimer = scheduler.interval(sample, intervalMs);
|
|
355
|
+
sample();
|
|
356
|
+
const activityTelemetry = {
|
|
357
|
+
settled,
|
|
358
|
+
isSettled: () => disposed,
|
|
359
|
+
closeIfNeverRun: () => {
|
|
360
|
+
if (!runStarted) {
|
|
361
|
+
runSettled = true;
|
|
362
|
+
sample();
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
};
|
|
366
|
+
return {
|
|
367
|
+
activityTelemetry,
|
|
368
|
+
worker: {
|
|
369
|
+
shutdown: () => sdkWorker.shutdown(),
|
|
370
|
+
// Drain never unregisters observation.
|
|
371
|
+
run: () => {
|
|
372
|
+
if (!runPromise && runSettled)
|
|
373
|
+
return Promise.reject(new Error("cannot run a closed turn worker"));
|
|
374
|
+
runPromise ??= (async () => {
|
|
375
|
+
runStarted = true;
|
|
376
|
+
try {
|
|
377
|
+
await sdkWorker.run();
|
|
378
|
+
} finally {
|
|
379
|
+
runSettled = true;
|
|
380
|
+
sample();
|
|
381
|
+
}
|
|
382
|
+
})();
|
|
383
|
+
return runPromise;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
|
|
133
389
|
// src/http.ts
|
|
134
390
|
import {
|
|
135
391
|
assertRuntimeDatabasePosture,
|
|
@@ -210,16 +466,17 @@ async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
|
210
466
|
checks: result
|
|
211
467
|
};
|
|
212
468
|
}
|
|
213
|
-
function dbReadyCheck(db, posture) {
|
|
469
|
+
function dbReadyCheck(db, posture, afterDatabaseReady) {
|
|
214
470
|
let inFlight;
|
|
215
471
|
return () => {
|
|
216
472
|
if (inFlight) return inFlight;
|
|
217
473
|
inFlight = (async () => {
|
|
218
474
|
if (posture) {
|
|
219
475
|
await assertRuntimeDatabasePosture(db, posture);
|
|
220
|
-
|
|
476
|
+
} else {
|
|
477
|
+
await db.execute(dbSql`select 1`);
|
|
221
478
|
}
|
|
222
|
-
await
|
|
479
|
+
await afterDatabaseReady?.();
|
|
223
480
|
})().finally(() => {
|
|
224
481
|
inFlight = void 0;
|
|
225
482
|
});
|
|
@@ -264,11 +521,362 @@ async function withTimeout(promise, timeoutMs) {
|
|
|
264
521
|
// src/index.ts
|
|
265
522
|
import {
|
|
266
523
|
resolveCatalogSettings,
|
|
524
|
+
TURN_ACTIVITY_CANCELLATION_HEARTBEAT_INTERVAL_MS
|
|
525
|
+
} from "@opengeni/core";
|
|
526
|
+
|
|
527
|
+
// src/internal-maintenance.ts
|
|
528
|
+
import { temporalConnectionOptions } from "@opengeni/config";
|
|
529
|
+
import {
|
|
530
|
+
SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
|
|
267
531
|
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
268
532
|
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
269
|
-
|
|
270
|
-
|
|
533
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
|
|
534
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE
|
|
271
535
|
} from "@opengeni/core";
|
|
536
|
+
import {
|
|
537
|
+
automaticSessionTitleFanoutDue,
|
|
538
|
+
browserDeadlineCheckpointDue,
|
|
539
|
+
knowledgeIndexDue,
|
|
540
|
+
sandboxIdleCheckpointDue,
|
|
541
|
+
sandboxLeaseReaperDue,
|
|
542
|
+
sandboxMaintenanceDue,
|
|
543
|
+
sessionWorkflowWakesDue,
|
|
544
|
+
siteAuthMaintenanceDue,
|
|
545
|
+
workspacePauseTimerDue
|
|
546
|
+
} from "@opengeni/db";
|
|
547
|
+
import {
|
|
548
|
+
Connection as Connection2,
|
|
549
|
+
ScheduleNotFoundError,
|
|
550
|
+
Client as TemporalClient,
|
|
551
|
+
WorkflowExecutionAlreadyStartedError
|
|
552
|
+
} from "@temporalio/client";
|
|
553
|
+
|
|
554
|
+
// src/maintenance-cadence.ts
|
|
555
|
+
var TICK_OUTCOMES = [
|
|
556
|
+
"idle",
|
|
557
|
+
"started",
|
|
558
|
+
"already_running",
|
|
559
|
+
"probe_failed",
|
|
560
|
+
"start_failed",
|
|
561
|
+
"busy"
|
|
562
|
+
];
|
|
563
|
+
function startMaintenanceCadences(input) {
|
|
564
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
565
|
+
const now = input.now ?? Date.now;
|
|
566
|
+
const jobs = /* @__PURE__ */ new Map();
|
|
567
|
+
for (const cadence of input.cadences) {
|
|
568
|
+
if (jobs.has(cadence.job)) throw new Error(`duplicate maintenance cadence ${cadence.job}`);
|
|
569
|
+
telemetryInterval(cadence.periodMs, `${cadence.job} maintenance period`);
|
|
570
|
+
jobs.set(cadence.job, cadence);
|
|
571
|
+
for (const outcome of TICK_OUTCOMES) {
|
|
572
|
+
input.observability.incrementCounter({
|
|
573
|
+
name: "opengeni_maintenance_cadence_ticks_total",
|
|
574
|
+
help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
|
|
575
|
+
labels: { job: cadence.job, outcome },
|
|
576
|
+
amount: 0
|
|
577
|
+
});
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
const inFlight = /* @__PURE__ */ new Map();
|
|
581
|
+
const cancels = [];
|
|
582
|
+
let stopped = false;
|
|
583
|
+
const record = (job, outcome) => {
|
|
584
|
+
input.observability.incrementCounter({
|
|
585
|
+
name: "opengeni_maintenance_cadence_ticks_total",
|
|
586
|
+
help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
|
|
587
|
+
labels: { job, outcome }
|
|
588
|
+
});
|
|
589
|
+
if (outcome !== "busy" && outcome !== "start_failed") {
|
|
590
|
+
input.observability.setGauge({
|
|
591
|
+
name: "opengeni_maintenance_cadence_last_success_timestamp_seconds",
|
|
592
|
+
help: "Unix time of the latest maintenance cadence tick that checked for work and, when needed, started its workflow.",
|
|
593
|
+
labels: { job },
|
|
594
|
+
value: now() / 1e3
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
return outcome;
|
|
598
|
+
};
|
|
599
|
+
const run = async (cadence) => {
|
|
600
|
+
let probeFailed = false;
|
|
601
|
+
try {
|
|
602
|
+
if (!await cadence.hasWork()) return record(cadence.job, "idle");
|
|
603
|
+
} catch (error) {
|
|
604
|
+
probeFailed = true;
|
|
605
|
+
input.observability.warn("maintenance cadence work check failed; starting the job anyway", {
|
|
606
|
+
job: cadence.job,
|
|
607
|
+
error: error instanceof Error ? error.message : String(error)
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
try {
|
|
611
|
+
const started = await cadence.start();
|
|
612
|
+
return record(cadence.job, probeFailed ? "probe_failed" : started);
|
|
613
|
+
} catch (error) {
|
|
614
|
+
input.observability.warn("maintenance cadence could not start its workflow", {
|
|
615
|
+
job: cadence.job,
|
|
616
|
+
error: error instanceof Error ? error.message : String(error)
|
|
617
|
+
});
|
|
618
|
+
return record(cadence.job, "start_failed");
|
|
619
|
+
}
|
|
620
|
+
};
|
|
621
|
+
const tick = async (job) => {
|
|
622
|
+
const cadence = jobs.get(job);
|
|
623
|
+
if (!cadence) throw new Error(`unknown maintenance cadence ${job}`);
|
|
624
|
+
if (stopped) return "busy";
|
|
625
|
+
if (inFlight.has(job)) return record(job, "busy");
|
|
626
|
+
const running = run(cadence).finally(() => inFlight.delete(job));
|
|
627
|
+
inFlight.set(job, running);
|
|
628
|
+
return await running;
|
|
629
|
+
};
|
|
630
|
+
if (input.autoStart !== false) {
|
|
631
|
+
for (const cadence of jobs.values()) {
|
|
632
|
+
const fire = () => {
|
|
633
|
+
void tick(cadence.job);
|
|
634
|
+
};
|
|
635
|
+
scheduler.defer(fire);
|
|
636
|
+
cancels.push(scheduler.interval(fire, cadence.periodMs));
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
return {
|
|
640
|
+
tick,
|
|
641
|
+
close: async () => {
|
|
642
|
+
stopped = true;
|
|
643
|
+
for (const cancel of cancels.splice(0)) cancel();
|
|
644
|
+
await Promise.allSettled([...inFlight.values()]);
|
|
645
|
+
}
|
|
646
|
+
};
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
// src/browser-deadline-checkpoint-contract.ts
|
|
650
|
+
var BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID = "opengeni-browser-deadline-checkpoints-v1";
|
|
651
|
+
var QUEUE_SUFFIX = "-browser-checkpoint-v1";
|
|
652
|
+
function browserDeadlineCheckpointTaskQueue(base) {
|
|
653
|
+
return base.endsWith(QUEUE_SUFFIX) ? base : `${base}${QUEUE_SUFFIX}`;
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// src/internal-maintenance.ts
|
|
657
|
+
var KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
|
|
658
|
+
var KNOWLEDGE_INDEXING_WORKFLOW_ID = "opengeni-knowledge-indexing";
|
|
659
|
+
var KNOWLEDGE_INDEXING_PERIOD_MS = 15e3;
|
|
660
|
+
var SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
|
|
661
|
+
var SITE_AUTH_MAINTENANCE_WORKFLOW_ID = "opengeni-site-auth-maintenance";
|
|
662
|
+
var SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1e3;
|
|
663
|
+
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
664
|
+
var INTERNAL_MAINTENANCE_JOBS = [
|
|
665
|
+
"session-workflow-wake-dispatcher",
|
|
666
|
+
"knowledge-indexing",
|
|
667
|
+
"site-auth-maintenance",
|
|
668
|
+
"sandbox-lease-reaper",
|
|
669
|
+
"sandbox-idle-checkpoint",
|
|
670
|
+
"browser-deadline-checkpoint"
|
|
671
|
+
];
|
|
672
|
+
var LEGACY_SCHEDULE_IDS = {
|
|
673
|
+
"session-workflow-wake-dispatcher": SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
674
|
+
"knowledge-indexing": KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
675
|
+
"site-auth-maintenance": SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
676
|
+
// The old reaper Schedule fired a router workflow that started the drain
|
|
677
|
+
// inventory, both checkpoint sweeps and the maintenance child on every tick.
|
|
678
|
+
"sandbox-lease-reaper": SANDBOX_REAPER_SCHEDULE_ID
|
|
679
|
+
};
|
|
680
|
+
async function startMaintenanceWorkflowOnce(workflow, input) {
|
|
681
|
+
try {
|
|
682
|
+
await workflow.start(input.workflowType, {
|
|
683
|
+
taskQueue: input.taskQueue,
|
|
684
|
+
workflowId: input.workflowId,
|
|
685
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
686
|
+
args: []
|
|
687
|
+
});
|
|
688
|
+
return "started";
|
|
689
|
+
} catch (error) {
|
|
690
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) return "already_running";
|
|
691
|
+
throw error;
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
async function deleteLegacyMaintenanceSchedules(schedule, jobs, observability) {
|
|
695
|
+
for (const job of jobs) {
|
|
696
|
+
const scheduleId = LEGACY_SCHEDULE_IDS[job];
|
|
697
|
+
if (!scheduleId) continue;
|
|
698
|
+
try {
|
|
699
|
+
await schedule.getHandle(scheduleId).delete();
|
|
700
|
+
observability.info(
|
|
701
|
+
"Deleted a per-tick maintenance Schedule replaced by a work-checked cadence",
|
|
702
|
+
{
|
|
703
|
+
job,
|
|
704
|
+
scheduleId
|
|
705
|
+
}
|
|
706
|
+
);
|
|
707
|
+
} catch (error) {
|
|
708
|
+
if (!(error instanceof ScheduleNotFoundError)) throw error;
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
async function internalMaintenanceCadences(input) {
|
|
713
|
+
const { settings, observability, db, workflow } = input;
|
|
714
|
+
const jobs = new Set(input.jobs ?? INTERNAL_MAINTENANCE_JOBS);
|
|
715
|
+
const taskQueue = settings.temporalTaskQueue;
|
|
716
|
+
const cadences = [];
|
|
717
|
+
if (jobs.has("session-workflow-wake-dispatcher")) {
|
|
718
|
+
const { repairMissedSessionWorkflowWakes } = await import("./parent-wake-PCLH7A3Y.js");
|
|
719
|
+
cadences.push({
|
|
720
|
+
job: "session-workflow-wake-dispatcher",
|
|
721
|
+
periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
722
|
+
hasWork: async () => {
|
|
723
|
+
await repairMissedSessionWorkflowWakes({ db, observability });
|
|
724
|
+
return await sessionWorkflowWakesDue(db) || await workspacePauseTimerDue(db) || await automaticSessionTitleFanoutDue(db);
|
|
725
|
+
},
|
|
726
|
+
start: async () => {
|
|
727
|
+
await workflow.signalWithStart(SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE, {
|
|
728
|
+
taskQueue,
|
|
729
|
+
workflowId: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
|
|
730
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
731
|
+
args: [],
|
|
732
|
+
signal: SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL
|
|
733
|
+
});
|
|
734
|
+
return "started";
|
|
735
|
+
}
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
if (jobs.has("knowledge-indexing")) {
|
|
739
|
+
let embedding;
|
|
740
|
+
let unavailableReported = false;
|
|
741
|
+
const resolveEmbedding = async () => {
|
|
742
|
+
const services = input.documentServices ?? (await import("@opengeni/documents")).createDocumentServices(settings);
|
|
743
|
+
return {
|
|
744
|
+
model: services.embedder.model,
|
|
745
|
+
dimensions: services.embedder.dimensions,
|
|
746
|
+
available: services.embedder.available !== false
|
|
747
|
+
};
|
|
748
|
+
};
|
|
749
|
+
cadences.push({
|
|
750
|
+
job: "knowledge-indexing",
|
|
751
|
+
periodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
|
|
752
|
+
hasWork: async () => {
|
|
753
|
+
embedding ??= resolveEmbedding().catch((error) => {
|
|
754
|
+
embedding = void 0;
|
|
755
|
+
throw error;
|
|
756
|
+
});
|
|
757
|
+
const current = await embedding;
|
|
758
|
+
if (!current.available) {
|
|
759
|
+
if (!unavailableReported) {
|
|
760
|
+
unavailableReported = true;
|
|
761
|
+
observability.warn(
|
|
762
|
+
"Knowledge indexing is idle: no document embedding credential is configured",
|
|
763
|
+
{ job: "knowledge-indexing", model: current.model }
|
|
764
|
+
);
|
|
765
|
+
}
|
|
766
|
+
return false;
|
|
767
|
+
}
|
|
768
|
+
return await knowledgeIndexDue(db, current);
|
|
769
|
+
},
|
|
770
|
+
start: async () => await startMaintenanceWorkflowOnce(workflow, {
|
|
771
|
+
workflowType: "knowledgeIndexingWorkflow",
|
|
772
|
+
workflowId: KNOWLEDGE_INDEXING_WORKFLOW_ID,
|
|
773
|
+
taskQueue
|
|
774
|
+
})
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
if (jobs.has("site-auth-maintenance")) {
|
|
778
|
+
const { SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS } = await import("./site-auth-maintenance-WK3MR2PV.js");
|
|
779
|
+
cadences.push({
|
|
780
|
+
job: "site-auth-maintenance",
|
|
781
|
+
periodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
782
|
+
hasWork: async () => await siteAuthMaintenanceDue(db, {
|
|
783
|
+
claimTimeoutMs: SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS
|
|
784
|
+
}),
|
|
785
|
+
start: async () => await startMaintenanceWorkflowOnce(workflow, {
|
|
786
|
+
workflowType: "siteAuthMaintenanceWorkflow",
|
|
787
|
+
workflowId: SITE_AUTH_MAINTENANCE_WORKFLOW_ID,
|
|
788
|
+
taskQueue
|
|
789
|
+
})
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
const sandboxJobs = ["sandbox-lease-reaper", "sandbox-idle-checkpoint", "browser-deadline-checkpoint"].filter((job) => jobs.has(job));
|
|
793
|
+
if (sandboxJobs.length > 0) {
|
|
794
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
795
|
+
const periodMs = settings.sandboxLeaseReaperPeriodMs;
|
|
796
|
+
const lifecycleQueue = sandboxLifecycleTaskQueue(taskQueue);
|
|
797
|
+
const sandbox = await import("./sandbox-lease-BJYU72N3.js");
|
|
798
|
+
if (jobs.has("sandbox-lease-reaper")) {
|
|
799
|
+
const { SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS } = sandbox;
|
|
800
|
+
cadences.push({
|
|
801
|
+
job: "sandbox-lease-reaper",
|
|
802
|
+
periodMs,
|
|
803
|
+
// The drain inventory and its maintenance pass act only on warm or held
|
|
804
|
+
// leases, due reconciliation rows, or (with Modal) boxes still running in
|
|
805
|
+
// the provider app. With none of those, the tick only refreshes the
|
|
806
|
+
// pass's read-only gauges in this process.
|
|
807
|
+
hasWork: async () => {
|
|
808
|
+
if (await sandboxLeaseReaperDue(db) || await sandboxMaintenanceDue(db, {
|
|
809
|
+
checkpointArtifactClaimTtlMs: SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS
|
|
810
|
+
}) || await sandbox.modalOrphanSweepMayHaveWork({ settings, observability })) {
|
|
811
|
+
return true;
|
|
812
|
+
}
|
|
813
|
+
await sandbox.refreshIdleSandboxReaperProjections({ db, settings, observability });
|
|
814
|
+
return false;
|
|
815
|
+
},
|
|
816
|
+
start: async () => await startMaintenanceWorkflowOnce(workflow, {
|
|
817
|
+
workflowType: "sandboxReaperWorkflowV2",
|
|
818
|
+
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
819
|
+
taskQueue: lifecycleQueue
|
|
820
|
+
})
|
|
821
|
+
});
|
|
822
|
+
}
|
|
823
|
+
if (jobs.has("sandbox-idle-checkpoint")) {
|
|
824
|
+
cadences.push({
|
|
825
|
+
job: "sandbox-idle-checkpoint",
|
|
826
|
+
periodMs,
|
|
827
|
+
// The sweep's inventory is empty when ownership or snapshots are off.
|
|
828
|
+
hasWork: async () => settings.sandboxOwnershipEnabled && settings.sandboxSnapshotIntervalMs > 0 && await sandboxIdleCheckpointDue(db, { intervalMs: settings.sandboxSnapshotIntervalMs }),
|
|
829
|
+
start: async () => await startMaintenanceWorkflowOnce(workflow, {
|
|
830
|
+
workflowType: "sandboxIdleCheckpointSweepWorkflow",
|
|
831
|
+
workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
832
|
+
taskQueue: lifecycleQueue
|
|
833
|
+
})
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
if (jobs.has("browser-deadline-checkpoint")) {
|
|
837
|
+
cadences.push({
|
|
838
|
+
job: "browser-deadline-checkpoint",
|
|
839
|
+
periodMs,
|
|
840
|
+
hasWork: async () => await browserDeadlineCheckpointDue(db),
|
|
841
|
+
start: async () => await startMaintenanceWorkflowOnce(workflow, {
|
|
842
|
+
workflowType: "browserDeadlineCheckpointSweepWorkflow",
|
|
843
|
+
workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
844
|
+
taskQueue: browserDeadlineCheckpointTaskQueue(taskQueue)
|
|
845
|
+
})
|
|
846
|
+
});
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
return cadences;
|
|
850
|
+
}
|
|
851
|
+
async function startInternalMaintenanceCadences(input) {
|
|
852
|
+
const connection = await Connection2.connect(temporalConnectionOptions(input.settings));
|
|
853
|
+
try {
|
|
854
|
+
const temporal = new TemporalClient({
|
|
855
|
+
connection,
|
|
856
|
+
namespace: input.settings.temporalNamespace
|
|
857
|
+
});
|
|
858
|
+
const jobs = input.jobs ?? INTERNAL_MAINTENANCE_JOBS;
|
|
859
|
+
await deleteLegacyMaintenanceSchedules(temporal.schedule, jobs, input.observability);
|
|
860
|
+
const cadences = startMaintenanceCadences({
|
|
861
|
+
cadences: await internalMaintenanceCadences({ ...input, jobs, workflow: temporal.workflow }),
|
|
862
|
+
observability: input.observability,
|
|
863
|
+
...input.scheduler ? { scheduler: input.scheduler } : {}
|
|
864
|
+
});
|
|
865
|
+
input.observability.info("Started work-checked maintenance cadences", {
|
|
866
|
+
jobs: jobs.join(",")
|
|
867
|
+
});
|
|
868
|
+
return {
|
|
869
|
+
cadences,
|
|
870
|
+
close: async () => {
|
|
871
|
+
await cadences.close();
|
|
872
|
+
await connection.close();
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
} catch (error) {
|
|
876
|
+
await connection.close().catch(() => void 0);
|
|
877
|
+
throw error;
|
|
878
|
+
}
|
|
879
|
+
}
|
|
272
880
|
|
|
273
881
|
// src/worker-service-lifecycle.ts
|
|
274
882
|
async function constructWithOwnedConnection(connect, construct, close) {
|
|
@@ -366,7 +974,7 @@ function createWorkerServiceLifecycle(input) {
|
|
|
366
974
|
}
|
|
367
975
|
const previousState = state;
|
|
368
976
|
safeLifecycleLog(
|
|
369
|
-
() => input.observability.info("
|
|
977
|
+
() => input.observability.info("Opengeni worker draining (graceful shutdown)", {
|
|
370
978
|
role: input.role,
|
|
371
979
|
errorClass: "WorkerLifecycleOperation",
|
|
372
980
|
errorCode: "worker_draining",
|
|
@@ -409,6 +1017,47 @@ function safeLifecycleLog(log) {
|
|
|
409
1017
|
} catch {
|
|
410
1018
|
}
|
|
411
1019
|
}
|
|
1020
|
+
function createWorkerCleanupContainment(input) {
|
|
1021
|
+
let requested = false;
|
|
1022
|
+
let finished = false;
|
|
1023
|
+
let timer;
|
|
1024
|
+
return {
|
|
1025
|
+
request() {
|
|
1026
|
+
if (requested || finished) return;
|
|
1027
|
+
requested = true;
|
|
1028
|
+
if (input.terminate) {
|
|
1029
|
+
timer = setTimeout(() => {
|
|
1030
|
+
safeLifecycleLog(
|
|
1031
|
+
() => input.observability.error("worker cleanup drain exhausted; terminating host", {
|
|
1032
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
1033
|
+
errorCode: "worker_cleanup_drain_exhausted",
|
|
1034
|
+
origin: "worker-lifecycle"
|
|
1035
|
+
})
|
|
1036
|
+
);
|
|
1037
|
+
input.terminate?.();
|
|
1038
|
+
}, input.timeoutMs ?? 1e5);
|
|
1039
|
+
timer.unref?.();
|
|
1040
|
+
}
|
|
1041
|
+
let accepted = false;
|
|
1042
|
+
try {
|
|
1043
|
+
accepted = input.drain();
|
|
1044
|
+
} catch {
|
|
1045
|
+
}
|
|
1046
|
+
if (!accepted)
|
|
1047
|
+
safeLifecycleLog(
|
|
1048
|
+
() => input.observability.error("worker cleanup drain request failed", {
|
|
1049
|
+
errorClass: "WorkerLifecycleOperationError",
|
|
1050
|
+
errorCode: "worker_shutdown_request_failed",
|
|
1051
|
+
origin: "worker-lifecycle"
|
|
1052
|
+
})
|
|
1053
|
+
);
|
|
1054
|
+
},
|
|
1055
|
+
finished() {
|
|
1056
|
+
finished = true;
|
|
1057
|
+
clearTimeout(timer);
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
412
1061
|
|
|
413
1062
|
// src/memory-pressure-guard.ts
|
|
414
1063
|
import { readFileSync } from "fs";
|
|
@@ -764,13 +1413,10 @@ function installWorkerUnhandledRejectionBoundary(runtimeProcess = process, repor
|
|
|
764
1413
|
}
|
|
765
1414
|
|
|
766
1415
|
// src/index.ts
|
|
767
|
-
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
768
|
-
var KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
|
|
769
|
-
var KNOWLEDGE_INDEXING_PERIOD_MS = 15e3;
|
|
770
1416
|
var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
771
1417
|
var FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1e3;
|
|
772
|
-
var
|
|
773
|
-
var
|
|
1418
|
+
var SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID = "opengeni-session-storage-maintenance";
|
|
1419
|
+
var SESSION_STORAGE_MAINTENANCE_PERIOD_MS = 5 * 60 * 1e3;
|
|
774
1420
|
function temporalTimeMs(value) {
|
|
775
1421
|
if (value?.seconds === void 0 || value.seconds === null) return null;
|
|
776
1422
|
return Number(String(value.seconds)) * 1e3 + Math.floor((value.nanos ?? 0) / 1e6);
|
|
@@ -785,7 +1431,10 @@ function temporalWorkflowExecutionNotFound(error) {
|
|
|
785
1431
|
return isGrpcServiceError(error) && error.code === 5;
|
|
786
1432
|
}
|
|
787
1433
|
async function createDefaultWorkerActivities(role, dependencies = {}) {
|
|
788
|
-
|
|
1434
|
+
if (dependencies.objectStorage == null) {
|
|
1435
|
+
prepareObjectStorageProvider(dependencies.settings ?? getSettings());
|
|
1436
|
+
}
|
|
1437
|
+
return role === "control" ? (await import("./activities-control-5IIVRXLA.js")).createControlActivities(dependencies) : (await import("./activities-turn-FLYJOM54.js")).createTurnActivities(dependencies);
|
|
789
1438
|
}
|
|
790
1439
|
function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
791
1440
|
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
@@ -795,7 +1444,7 @@ function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
|
795
1444
|
const codePath = fileURLToPath(new URL("./workflow-bundle.js", moduleUrl));
|
|
796
1445
|
if (!existsSync(codePath)) {
|
|
797
1446
|
throw new Error(
|
|
798
|
-
`
|
|
1447
|
+
`Opengeni workflow bundle is missing at ${codePath}; rebuild or reinstall @opengeni/worker-bundle`
|
|
799
1448
|
);
|
|
800
1449
|
}
|
|
801
1450
|
return { workflowBundle: { codePath } };
|
|
@@ -833,17 +1482,25 @@ async function createOpenGeniWorker(options) {
|
|
|
833
1482
|
return constructWithOwnedConnection(
|
|
834
1483
|
() => retryStartupDependency(
|
|
835
1484
|
"Temporal",
|
|
836
|
-
() => NativeConnection.connect(
|
|
1485
|
+
() => NativeConnection.connect(temporalConnectionOptions2(settings)),
|
|
837
1486
|
{
|
|
838
1487
|
...startupRetryOptions(settings),
|
|
839
1488
|
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
840
1489
|
}
|
|
841
1490
|
),
|
|
842
1491
|
async (connection) => {
|
|
1492
|
+
let turnWorker;
|
|
1493
|
+
let cleanupDrainRequested = false;
|
|
843
1494
|
const activityDependencies = {
|
|
844
1495
|
...options.activityDependencies,
|
|
845
1496
|
settings,
|
|
846
|
-
observability
|
|
1497
|
+
observability,
|
|
1498
|
+
requestWorkerDrain: options.activityDependencies?.requestWorkerDrain ?? (() => {
|
|
1499
|
+
if (cleanupDrainRequested) return;
|
|
1500
|
+
if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
|
|
1501
|
+
turnWorker.shutdown();
|
|
1502
|
+
cleanupDrainRequested = true;
|
|
1503
|
+
})
|
|
847
1504
|
};
|
|
848
1505
|
const activities = options.activities ?? await createDefaultWorkerActivities(options.role, activityDependencies);
|
|
849
1506
|
const turnConcurrency = options.role === "turn" ? createTurnWorkerConcurrencyPlan(settings, { observability }) : null;
|
|
@@ -877,28 +1534,52 @@ async function createOpenGeniWorker(options) {
|
|
|
877
1534
|
} : turnConcurrency.options
|
|
878
1535
|
});
|
|
879
1536
|
if (options.role !== "control") {
|
|
1537
|
+
turnWorker = worker;
|
|
880
1538
|
turnConcurrency?.admission?.finalizeStartupBaseline();
|
|
881
|
-
|
|
1539
|
+
const turnQueueIdentity = {
|
|
1540
|
+
temporalNamespace: worker.options.namespace,
|
|
1541
|
+
taskQueue: worker.options.taskQueue
|
|
1542
|
+
};
|
|
1543
|
+
return {
|
|
1544
|
+
...withTurnWorkerActivityTelemetry(worker, {
|
|
1545
|
+
observability,
|
|
1546
|
+
identity: turnQueueIdentity,
|
|
1547
|
+
// No invented identity when running outside Kubernetes. Scaler
|
|
1548
|
+
// consumers must reject empty/missing UID coverage.
|
|
1549
|
+
podUid: process.env.OPENGENI_POD_UID ?? ""
|
|
1550
|
+
}),
|
|
1551
|
+
connection,
|
|
1552
|
+
turnQueueIdentity
|
|
1553
|
+
};
|
|
882
1554
|
}
|
|
1555
|
+
const ownedWorkers = [worker];
|
|
883
1556
|
try {
|
|
884
|
-
const
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1557
|
+
for (const taskQueue of [
|
|
1558
|
+
sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
1559
|
+
browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue)
|
|
1560
|
+
]) {
|
|
1561
|
+
const lifecycleWorker = await Worker.create({
|
|
1562
|
+
...sharedWorkerOptions,
|
|
1563
|
+
...workflowDefinition,
|
|
1564
|
+
taskQueue,
|
|
1565
|
+
reuseV8Context: true,
|
|
1566
|
+
workflowThreadPoolSize: 1,
|
|
1567
|
+
maxCachedWorkflows: CONTROL_WORKER_MAX_CACHED_WORKFLOWS,
|
|
1568
|
+
maxConcurrentWorkflowTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_WORKFLOW_TASKS,
|
|
1569
|
+
maxConcurrentActivityTaskExecutions: CONTROL_WORKER_MAX_CONCURRENT_ACTIVITIES
|
|
1570
|
+
});
|
|
1571
|
+
ownedWorkers.push(lifecycleWorker);
|
|
1572
|
+
}
|
|
894
1573
|
return {
|
|
895
|
-
worker: combineWorkerRunTargets(
|
|
1574
|
+
worker: combineWorkerRunTargets(ownedWorkers),
|
|
896
1575
|
connection
|
|
897
1576
|
};
|
|
898
1577
|
} catch (error) {
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
1578
|
+
for (const ownedWorker of ownedWorkers) {
|
|
1579
|
+
try {
|
|
1580
|
+
ownedWorker.shutdown();
|
|
1581
|
+
} catch {
|
|
1582
|
+
}
|
|
902
1583
|
}
|
|
903
1584
|
throw error;
|
|
904
1585
|
}
|
|
@@ -907,8 +1588,10 @@ async function createOpenGeniWorker(options) {
|
|
|
907
1588
|
);
|
|
908
1589
|
}
|
|
909
1590
|
async function createWorkerWorkflowSignaler(settings, db) {
|
|
910
|
-
const
|
|
911
|
-
const
|
|
1591
|
+
const connectionOptions = temporalConnectionOptions2(settings);
|
|
1592
|
+
const connection = await Connection3.connect(connectionOptions);
|
|
1593
|
+
const statsClient = createTurnTaskQueueStatsClient(connectionOptions);
|
|
1594
|
+
const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
|
|
912
1595
|
return {
|
|
913
1596
|
wakeSessionWorkflow: async ({
|
|
914
1597
|
accountId,
|
|
@@ -994,33 +1677,38 @@ async function createWorkerWorkflowSignaler(settings, db) {
|
|
|
994
1677
|
signalArgs: [wakeRevision]
|
|
995
1678
|
});
|
|
996
1679
|
},
|
|
997
|
-
getTurnTaskQueueStats:
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
// temporal.api.enums.v1.TASK_QUEUE_TYPE_ACTIVITY. Keep this request in
|
|
1002
|
-
// the supported DEFAULT mode; `stats.approximateBacklogCount` is the
|
|
1003
|
-
// server-documented scaling signal.
|
|
1004
|
-
taskQueueType: 2,
|
|
1005
|
-
reportStats: true
|
|
1006
|
-
});
|
|
1007
|
-
return normalizeTurnTaskQueueStats(response.stats);
|
|
1008
|
-
},
|
|
1680
|
+
getTurnTaskQueueStats: (readOptions = { signal: new AbortController().signal, deadline: Date.now() + 5e3 }, identity = {
|
|
1681
|
+
temporalNamespace: settings.temporalNamespace,
|
|
1682
|
+
taskQueue: turnTaskQueue(settings.temporalTaskQueue)
|
|
1683
|
+
}) => statsClient.read(readOptions, identity),
|
|
1009
1684
|
startSandboxReaperWorkflow: async () => {
|
|
1010
|
-
|
|
1011
|
-
|
|
1685
|
+
const starts = await Promise.allSettled([
|
|
1686
|
+
temporal.workflow.start("sandboxReaperWorkflowV2", {
|
|
1012
1687
|
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
1013
1688
|
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
1014
1689
|
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
1015
1690
|
args: []
|
|
1016
|
-
})
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1691
|
+
}),
|
|
1692
|
+
temporal.workflow.start("browserDeadlineCheckpointSweepWorkflow", {
|
|
1693
|
+
taskQueue: browserDeadlineCheckpointTaskQueue(settings.temporalTaskQueue),
|
|
1694
|
+
workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
1695
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
1696
|
+
args: []
|
|
1697
|
+
}),
|
|
1698
|
+
// Its own workflow type, so the drain workflow's history is unchanged.
|
|
1699
|
+
temporal.workflow.start("sandboxIdleCheckpointSweepWorkflow", {
|
|
1700
|
+
taskQueue: sandboxLifecycleTaskQueue(settings.temporalTaskQueue),
|
|
1701
|
+
workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
1702
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
1703
|
+
args: []
|
|
1704
|
+
})
|
|
1705
|
+
]);
|
|
1706
|
+
for (const start of starts) {
|
|
1707
|
+
if (start.status === "rejected" && !(start.reason instanceof WorkflowExecutionAlreadyStartedError2)) {
|
|
1708
|
+
throw start.reason;
|
|
1021
1709
|
}
|
|
1022
|
-
throw error;
|
|
1023
1710
|
}
|
|
1711
|
+
return starts[0].status === "fulfilled" ? "started" : "already_running";
|
|
1024
1712
|
},
|
|
1025
1713
|
startVideoGenerationWorkflow: async ({ accountId, workspaceId, operationId }) => {
|
|
1026
1714
|
try {
|
|
@@ -1043,7 +1731,7 @@ async function createWorkerWorkflowSignaler(settings, db) {
|
|
|
1043
1731
|
});
|
|
1044
1732
|
return "started";
|
|
1045
1733
|
} catch (error) {
|
|
1046
|
-
if (error instanceof
|
|
1734
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError2) {
|
|
1047
1735
|
return "already_running";
|
|
1048
1736
|
}
|
|
1049
1737
|
throw error;
|
|
@@ -1053,113 +1741,13 @@ async function createWorkerWorkflowSignaler(settings, db) {
|
|
|
1053
1741
|
await connection.workflowService.getSystemInfo({});
|
|
1054
1742
|
},
|
|
1055
1743
|
close: async () => {
|
|
1056
|
-
await connection.close();
|
|
1744
|
+
await Promise.all([statsClient.close(), connection.close()]);
|
|
1057
1745
|
}
|
|
1058
1746
|
};
|
|
1059
1747
|
}
|
|
1060
|
-
async function registerSandboxReaperSchedule(settings, observability) {
|
|
1061
|
-
assertSandboxReaperActivityTimeout(settings);
|
|
1062
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
1063
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
1064
|
-
const spec = { intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }] };
|
|
1065
|
-
const action = {
|
|
1066
|
-
type: "startWorkflow",
|
|
1067
|
-
workflowType: "sandboxReaperWorkflow",
|
|
1068
|
-
taskQueue: settings.temporalTaskQueue,
|
|
1069
|
-
args: []
|
|
1070
|
-
};
|
|
1071
|
-
const policies = {
|
|
1072
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
1073
|
-
catchupWindow: "1m",
|
|
1074
|
-
pauseOnFailure: false
|
|
1075
|
-
};
|
|
1076
|
-
try {
|
|
1077
|
-
await temporal.schedule.create({
|
|
1078
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
1079
|
-
// @every-style interval: fire once per reaper period. The boot invariant
|
|
1080
|
-
// guarantees reaperPeriod < viewerHolderTTL and
|
|
1081
|
-
// reaperPeriod + idleGrace < providerLifetime.
|
|
1082
|
-
spec,
|
|
1083
|
-
action,
|
|
1084
|
-
// The base workflow is a short router on new binaries; legacy binaries
|
|
1085
|
-
// still execute the pre-refactor composite under this same overlap fence.
|
|
1086
|
-
policies
|
|
1087
|
-
});
|
|
1088
|
-
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
1089
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
1090
|
-
reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs
|
|
1091
|
-
});
|
|
1092
|
-
return {
|
|
1093
|
-
registered: true,
|
|
1094
|
-
close: async () => {
|
|
1095
|
-
await connection.close();
|
|
1096
|
-
}
|
|
1097
|
-
};
|
|
1098
|
-
} catch (error) {
|
|
1099
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
1100
|
-
await temporal.schedule.getHandle(SANDBOX_REAPER_SCHEDULE_ID).update((previous) => ({
|
|
1101
|
-
spec,
|
|
1102
|
-
action,
|
|
1103
|
-
policies,
|
|
1104
|
-
state: {
|
|
1105
|
-
paused: previous.state.paused,
|
|
1106
|
-
...previous.state.note ? { note: previous.state.note } : {},
|
|
1107
|
-
...previous.state.remainingActions !== void 0 ? { remainingActions: previous.state.remainingActions } : {}
|
|
1108
|
-
}
|
|
1109
|
-
}));
|
|
1110
|
-
observability.info("Reconciled the global sandbox-lease reaper Schedule", {
|
|
1111
|
-
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
1112
|
-
temporalTaskQueue: action.taskQueue
|
|
1113
|
-
});
|
|
1114
|
-
return {
|
|
1115
|
-
registered: false,
|
|
1116
|
-
close: async () => {
|
|
1117
|
-
await connection.close();
|
|
1118
|
-
}
|
|
1119
|
-
};
|
|
1120
|
-
}
|
|
1121
|
-
await connection.close().catch(() => void 0);
|
|
1122
|
-
throw error;
|
|
1123
|
-
}
|
|
1124
|
-
}
|
|
1125
|
-
async function registerKnowledgeIndexingSchedule(settings, observability) {
|
|
1126
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
1127
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
1128
|
-
try {
|
|
1129
|
-
await temporal.schedule.create({
|
|
1130
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
1131
|
-
spec: { intervals: [{ every: KNOWLEDGE_INDEXING_PERIOD_MS }] },
|
|
1132
|
-
action: {
|
|
1133
|
-
type: "startWorkflow",
|
|
1134
|
-
workflowType: "knowledgeIndexingWorkflow",
|
|
1135
|
-
taskQueue: settings.temporalTaskQueue,
|
|
1136
|
-
args: []
|
|
1137
|
-
},
|
|
1138
|
-
policies: {
|
|
1139
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
1140
|
-
catchupWindow: "1m",
|
|
1141
|
-
pauseOnFailure: false
|
|
1142
|
-
}
|
|
1143
|
-
});
|
|
1144
|
-
observability.info("Registered the global Knowledge indexer Schedule", {
|
|
1145
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
1146
|
-
indexingPeriodMs: KNOWLEDGE_INDEXING_PERIOD_MS
|
|
1147
|
-
});
|
|
1148
|
-
return { registered: true, close: async () => connection.close() };
|
|
1149
|
-
} catch (error) {
|
|
1150
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
1151
|
-
observability.info("Global Knowledge indexer Schedule already registered", {
|
|
1152
|
-
scheduleId: KNOWLEDGE_INDEXING_SCHEDULE_ID
|
|
1153
|
-
});
|
|
1154
|
-
return { registered: false, close: async () => connection.close() };
|
|
1155
|
-
}
|
|
1156
|
-
await connection.close().catch(() => void 0);
|
|
1157
|
-
throw error;
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
1748
|
async function registerFileUploadReaperSchedule(settings, observability) {
|
|
1161
|
-
const connection = await
|
|
1162
|
-
const temporal = new
|
|
1749
|
+
const connection = await Connection3.connect(temporalConnectionOptions2(settings));
|
|
1750
|
+
const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
|
|
1163
1751
|
try {
|
|
1164
1752
|
await temporal.schedule.create({
|
|
1165
1753
|
scheduleId: FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
@@ -1192,16 +1780,16 @@ async function registerFileUploadReaperSchedule(settings, observability) {
|
|
|
1192
1780
|
throw error;
|
|
1193
1781
|
}
|
|
1194
1782
|
}
|
|
1195
|
-
async function
|
|
1196
|
-
const connection = await
|
|
1197
|
-
const temporal = new
|
|
1783
|
+
async function registerSessionStorageMaintenanceSchedule(settings, observability) {
|
|
1784
|
+
const connection = await Connection3.connect(temporalConnectionOptions2(settings));
|
|
1785
|
+
const temporal = new TemporalClient2({ connection, namespace: settings.temporalNamespace });
|
|
1198
1786
|
try {
|
|
1199
1787
|
await temporal.schedule.create({
|
|
1200
|
-
scheduleId:
|
|
1201
|
-
spec: { intervals: [{ every:
|
|
1788
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
1789
|
+
spec: { intervals: [{ every: SESSION_STORAGE_MAINTENANCE_PERIOD_MS }] },
|
|
1202
1790
|
action: {
|
|
1203
1791
|
type: "startWorkflow",
|
|
1204
|
-
workflowType: "
|
|
1792
|
+
workflowType: "sessionStorageMaintenanceWorkflow",
|
|
1205
1793
|
taskQueue: settings.temporalTaskQueue,
|
|
1206
1794
|
args: []
|
|
1207
1795
|
},
|
|
@@ -1211,50 +1799,15 @@ async function registerSiteAuthMaintenanceSchedule(settings, observability) {
|
|
|
1211
1799
|
pauseOnFailure: false
|
|
1212
1800
|
}
|
|
1213
1801
|
});
|
|
1214
|
-
observability.info("Registered the
|
|
1215
|
-
scheduleId:
|
|
1216
|
-
|
|
1802
|
+
observability.info("Registered the session storage maintenance Schedule", {
|
|
1803
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
1804
|
+
periodMs: SESSION_STORAGE_MAINTENANCE_PERIOD_MS
|
|
1217
1805
|
});
|
|
1218
1806
|
return { registered: true, close: async () => connection.close() };
|
|
1219
1807
|
} catch (error) {
|
|
1220
1808
|
if (error instanceof ScheduleAlreadyRunning) {
|
|
1221
|
-
observability.info("
|
|
1222
|
-
scheduleId:
|
|
1223
|
-
});
|
|
1224
|
-
return { registered: false, close: async () => connection.close() };
|
|
1225
|
-
}
|
|
1226
|
-
await connection.close().catch(() => void 0);
|
|
1227
|
-
throw error;
|
|
1228
|
-
}
|
|
1229
|
-
}
|
|
1230
|
-
async function registerSessionWorkflowWakeDispatcherSchedule(settings, observability) {
|
|
1231
|
-
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
1232
|
-
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
1233
|
-
try {
|
|
1234
|
-
await temporal.schedule.create({
|
|
1235
|
-
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
1236
|
-
spec: { intervals: [{ every: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS }] },
|
|
1237
|
-
action: {
|
|
1238
|
-
type: "startWorkflow",
|
|
1239
|
-
workflowType: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
1240
|
-
taskQueue: settings.temporalTaskQueue,
|
|
1241
|
-
args: []
|
|
1242
|
-
},
|
|
1243
|
-
policies: {
|
|
1244
|
-
overlap: ScheduleOverlapPolicy.SKIP,
|
|
1245
|
-
catchupWindow: "1m",
|
|
1246
|
-
pauseOnFailure: false
|
|
1247
|
-
}
|
|
1248
|
-
});
|
|
1249
|
-
observability.info("Registered the session-workflow wake dispatcher Schedule", {
|
|
1250
|
-
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
1251
|
-
periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS
|
|
1252
|
-
});
|
|
1253
|
-
return { registered: true, close: async () => connection.close() };
|
|
1254
|
-
} catch (error) {
|
|
1255
|
-
if (error instanceof ScheduleAlreadyRunning) {
|
|
1256
|
-
observability.info("Session-workflow wake dispatcher Schedule already registered", {
|
|
1257
|
-
scheduleId: SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID
|
|
1809
|
+
observability.info("Session storage maintenance Schedule already registered", {
|
|
1810
|
+
scheduleId: SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID
|
|
1258
1811
|
});
|
|
1259
1812
|
return { registered: false, close: async () => connection.close() };
|
|
1260
1813
|
}
|
|
@@ -1272,6 +1825,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1272
1825
|
const retryOptions = startupRetryOptions(settings);
|
|
1273
1826
|
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
1274
1827
|
let lifecycle;
|
|
1828
|
+
let cleanupContainment;
|
|
1275
1829
|
let signaler;
|
|
1276
1830
|
let workerBundle;
|
|
1277
1831
|
let turnCapacityMonitor;
|
|
@@ -1286,7 +1840,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1286
1840
|
() => resolveCatalogSettings(options.activityDependencies.db, settings),
|
|
1287
1841
|
{ ...retryOptions, onRetry }
|
|
1288
1842
|
);
|
|
1289
|
-
observability.info("
|
|
1843
|
+
observability.info("Opengeni model catalog resolved", {
|
|
1290
1844
|
role: options.role,
|
|
1291
1845
|
catalogSource: resolvedCatalog.source,
|
|
1292
1846
|
catalogVersion: resolvedCatalog.version
|
|
@@ -1306,7 +1860,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1306
1860
|
const startSandboxReaperWorkflow = options.activityDependencies.startSandboxReaperWorkflow ?? signaler?.startSandboxReaperWorkflow;
|
|
1307
1861
|
const startVideoGenerationWorkflow = options.activityDependencies.startVideoGenerationWorkflow ?? signaler?.startVideoGenerationWorkflow;
|
|
1308
1862
|
if (!wakeSessionWorkflow || !signalSessionAttemptQuiesced || !inspectSessionAttemptActivity || !signalCodexCapacityWorkflow || !startSandboxReaperWorkflow || !startVideoGenerationWorkflow) {
|
|
1309
|
-
throw new Error("
|
|
1863
|
+
throw new Error("Opengeni worker lifecycle could not resolve its workflow signalers");
|
|
1310
1864
|
}
|
|
1311
1865
|
workerBundle = await createOpenGeniWorker({
|
|
1312
1866
|
role: options.role,
|
|
@@ -1322,21 +1876,30 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1322
1876
|
inspectSessionAttemptActivity,
|
|
1323
1877
|
signalCodexCapacityWorkflow,
|
|
1324
1878
|
startSandboxReaperWorkflow,
|
|
1325
|
-
startVideoGenerationWorkflow
|
|
1879
|
+
startVideoGenerationWorkflow,
|
|
1880
|
+
requestWorkerDrain: () => {
|
|
1881
|
+
if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
|
|
1882
|
+
cleanupContainment.request();
|
|
1883
|
+
}
|
|
1326
1884
|
}
|
|
1327
1885
|
});
|
|
1328
1886
|
if (options.role === "turn") {
|
|
1329
1887
|
if (!signaler) {
|
|
1330
1888
|
throw new Error("Turn worker capacity monitor could not resolve its Temporal stats client");
|
|
1331
1889
|
}
|
|
1890
|
+
const statsClient = signaler;
|
|
1891
|
+
const identity = workerBundle.turnQueueIdentity;
|
|
1892
|
+
if (!identity)
|
|
1893
|
+
throw new Error("Turn worker did not expose its actual Temporal queue identity");
|
|
1332
1894
|
turnCapacityMonitor = startTurnCapacityMonitor({
|
|
1333
1895
|
observability,
|
|
1334
|
-
read:
|
|
1896
|
+
read: (readOptions) => statsClient.getTurnTaskQueueStats(readOptions, identity),
|
|
1897
|
+
identity
|
|
1335
1898
|
});
|
|
1336
1899
|
} else {
|
|
1337
1900
|
sessionRecoveryMonitor = startSessionRecoveryMonitor({
|
|
1338
1901
|
observability,
|
|
1339
|
-
read: async () => await
|
|
1902
|
+
read: async () => await summarizeSessionRecoveryBacklog(options.activityDependencies.db)
|
|
1340
1903
|
});
|
|
1341
1904
|
contextCompactionPendingMonitor = startContextCompactionPendingMonitor({
|
|
1342
1905
|
observability,
|
|
@@ -1344,20 +1907,6 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1344
1907
|
});
|
|
1345
1908
|
}
|
|
1346
1909
|
if (workerOwnsInternalSchedules(options.role, options.internalSchedules)) {
|
|
1347
|
-
schedules.push(
|
|
1348
|
-
await retryStartupDependency(
|
|
1349
|
-
"Temporal schedule (sandbox reaper)",
|
|
1350
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
1351
|
-
{ ...retryOptions, onRetry }
|
|
1352
|
-
)
|
|
1353
|
-
);
|
|
1354
|
-
schedules.push(
|
|
1355
|
-
await retryStartupDependency(
|
|
1356
|
-
"Temporal schedule (Knowledge indexing)",
|
|
1357
|
-
() => registerKnowledgeIndexingSchedule(settings, observability),
|
|
1358
|
-
{ ...retryOptions, onRetry }
|
|
1359
|
-
)
|
|
1360
|
-
);
|
|
1361
1910
|
schedules.push(
|
|
1362
1911
|
await retryStartupDependency(
|
|
1363
1912
|
"Temporal schedule (file upload reaper)",
|
|
@@ -1367,32 +1916,34 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1367
1916
|
);
|
|
1368
1917
|
schedules.push(
|
|
1369
1918
|
await retryStartupDependency(
|
|
1370
|
-
"Temporal schedule (
|
|
1371
|
-
() =>
|
|
1919
|
+
"Temporal schedule (session storage maintenance)",
|
|
1920
|
+
() => registerSessionStorageMaintenanceSchedule(settings, observability),
|
|
1372
1921
|
{ ...retryOptions, onRetry }
|
|
1373
1922
|
)
|
|
1374
1923
|
);
|
|
1375
1924
|
schedules.push(
|
|
1376
1925
|
await retryStartupDependency(
|
|
1377
|
-
"
|
|
1378
|
-
() =>
|
|
1926
|
+
"work-checked maintenance cadences",
|
|
1927
|
+
() => startInternalMaintenanceCadences({
|
|
1928
|
+
settings,
|
|
1929
|
+
observability,
|
|
1930
|
+
db: options.activityDependencies.db,
|
|
1931
|
+
...options.activityDependencies.documentServices ? { documentServices: options.activityDependencies.documentServices } : {}
|
|
1932
|
+
}),
|
|
1379
1933
|
{ ...retryOptions, onRetry }
|
|
1380
1934
|
)
|
|
1381
1935
|
);
|
|
1382
1936
|
}
|
|
1383
1937
|
if (options.http !== false) {
|
|
1384
|
-
const
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
);
|
|
1938
|
+
const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
|
|
1939
|
+
const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
|
|
1940
|
+
await resolveCatalogSettings(readinessDb, settings);
|
|
1941
|
+
});
|
|
1388
1942
|
httpServer = startWorkerHttpServer({
|
|
1389
1943
|
settings,
|
|
1390
1944
|
observability,
|
|
1391
1945
|
checks: {
|
|
1392
|
-
db:
|
|
1393
|
-
await databaseReady();
|
|
1394
|
-
await resolveCatalogSettings(options.activityDependencies.db, settings);
|
|
1395
|
-
},
|
|
1946
|
+
db: databaseReady,
|
|
1396
1947
|
nats: natsReadyCheck(options.activityDependencies.bus),
|
|
1397
1948
|
temporal: temporalReadyCheck(workerBundle.connection)
|
|
1398
1949
|
},
|
|
@@ -1401,6 +1952,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1401
1952
|
});
|
|
1402
1953
|
}
|
|
1403
1954
|
} catch (error) {
|
|
1955
|
+
workerBundle?.activityTelemetry?.closeIfNeverRun();
|
|
1404
1956
|
httpServer?.stop(true);
|
|
1405
1957
|
await Promise.allSettled([
|
|
1406
1958
|
turnCapacityMonitor?.close(),
|
|
@@ -1415,15 +1967,30 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1415
1967
|
const activeWorkerBundle = workerBundle;
|
|
1416
1968
|
const activeSignaler = signaler;
|
|
1417
1969
|
if (!activeWorkerBundle) {
|
|
1418
|
-
throw new Error("
|
|
1970
|
+
throw new Error("Opengeni worker service initialization did not complete");
|
|
1419
1971
|
}
|
|
1972
|
+
cleanupContainment = createWorkerCleanupContainment({
|
|
1973
|
+
drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
|
|
1974
|
+
...options.terminateWorker ? { terminate: options.terminateWorker } : {},
|
|
1975
|
+
observability
|
|
1976
|
+
});
|
|
1977
|
+
const activeCleanupContainment = cleanupContainment;
|
|
1420
1978
|
lifecycle = createWorkerServiceLifecycle({
|
|
1421
1979
|
role: options.role,
|
|
1422
|
-
worker:
|
|
1980
|
+
worker: {
|
|
1981
|
+
shutdown: () => activeWorkerBundle.worker.shutdown(),
|
|
1982
|
+
run: async () => {
|
|
1983
|
+
await activeWorkerBundle.worker.run();
|
|
1984
|
+
activeCleanupContainment.finished();
|
|
1985
|
+
}
|
|
1986
|
+
},
|
|
1423
1987
|
observability,
|
|
1424
1988
|
closeOwnedResources: async () => {
|
|
1425
1989
|
memoryPressureGuard?.close();
|
|
1426
|
-
|
|
1990
|
+
closeWorkerHttpAfterActivityExecution(
|
|
1991
|
+
activeWorkerBundle.activityTelemetry,
|
|
1992
|
+
() => httpServer?.stop(true)
|
|
1993
|
+
);
|
|
1427
1994
|
await Promise.allSettled([
|
|
1428
1995
|
turnCapacityMonitor?.close(),
|
|
1429
1996
|
sessionRecoveryMonitor?.close(),
|
|
@@ -1434,7 +2001,7 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1434
2001
|
]);
|
|
1435
2002
|
},
|
|
1436
2003
|
onReady: () => {
|
|
1437
|
-
observability.info("
|
|
2004
|
+
observability.info("Opengeni worker listening", {
|
|
1438
2005
|
role: options.role,
|
|
1439
2006
|
temporalTaskQueue: options.role === "control" ? settings.temporalTaskQueue : turnTaskQueue(settings.temporalTaskQueue),
|
|
1440
2007
|
...options.role === "turn" ? turnWorkerConcurrencyLogFields(settings) : {
|
|
@@ -1480,6 +2047,16 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1480
2047
|
}
|
|
1481
2048
|
async function runOpenGeniWorker(options) {
|
|
1482
2049
|
const service = await createOpenGeniWorkerService(options);
|
|
2050
|
+
void import("@opengeni/runtime/sandbox").then(
|
|
2051
|
+
({ sweepOrphanedHostArchiveTemporaryDirectoriesOnce }) => sweepOrphanedHostArchiveTemporaryDirectoriesOnce()
|
|
2052
|
+
).then((removed) => {
|
|
2053
|
+
if (removed.length > 0) {
|
|
2054
|
+
options.activityDependencies.observability?.info(
|
|
2055
|
+
"worker removed orphaned workspace archive spools",
|
|
2056
|
+
{ count: removed.length }
|
|
2057
|
+
);
|
|
2058
|
+
}
|
|
2059
|
+
}).catch(() => void 0);
|
|
1483
2060
|
const signals = options.shutdownSignals === false ? [] : options.shutdownSignals ?? ["SIGTERM", "SIGINT"];
|
|
1484
2061
|
const handlers = signals.map((signal) => {
|
|
1485
2062
|
const handler = () => service.drain(signal);
|
|
@@ -1502,6 +2079,7 @@ async function startWorker() {
|
|
|
1502
2079
|
}
|
|
1503
2080
|
const settings = getSettings();
|
|
1504
2081
|
const observability = createObservability(settings, { component: `worker-${role}` });
|
|
2082
|
+
installTemporalRuntimeLogging(observability);
|
|
1505
2083
|
const disposeUnhandledRejectionBoundary = installWorkerUnhandledRejectionBoundary();
|
|
1506
2084
|
const retryOptions = startupRetryOptions(settings);
|
|
1507
2085
|
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
@@ -1518,8 +2096,7 @@ async function startWorker() {
|
|
|
1518
2096
|
const databasePosture = {
|
|
1519
2097
|
rlsStrategy: settings.rlsStrategy,
|
|
1520
2098
|
expectedRole: settings.runtimeDatabaseRole,
|
|
1521
|
-
targetSchema: settings.dbSchema.trim() || "public"
|
|
1522
|
-
organizationTenancyCanonicalActivationEnabled: settings.organizationTenancyCanonicalActivationEnabled
|
|
2099
|
+
targetSchema: settings.dbSchema.trim() || "public"
|
|
1523
2100
|
};
|
|
1524
2101
|
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
1525
2102
|
let bus;
|
|
@@ -1534,12 +2111,13 @@ async function startWorker() {
|
|
|
1534
2111
|
() => createNatsEventBus(
|
|
1535
2112
|
settings.natsUrl,
|
|
1536
2113
|
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
1537
|
-
|
|
2114
|
+
observabilityEventBusOptions(observability)
|
|
1538
2115
|
),
|
|
1539
2116
|
{ ...retryOptions, onRetry }
|
|
1540
2117
|
);
|
|
1541
2118
|
await runOpenGeniWorker({
|
|
1542
2119
|
role,
|
|
2120
|
+
terminateWorker: () => process.exit(1),
|
|
1543
2121
|
settings,
|
|
1544
2122
|
databasePosture,
|
|
1545
2123
|
http: { readinessDb: readinessDbClient.db },
|
|
@@ -1570,8 +2148,11 @@ if (import.meta.main) {
|
|
|
1570
2148
|
export {
|
|
1571
2149
|
FILE_UPLOAD_REAPER_PERIOD_MS,
|
|
1572
2150
|
FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
2151
|
+
INTERNAL_MAINTENANCE_JOBS,
|
|
1573
2152
|
KNOWLEDGE_INDEXING_PERIOD_MS,
|
|
1574
2153
|
KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
2154
|
+
SESSION_STORAGE_MAINTENANCE_PERIOD_MS,
|
|
2155
|
+
SESSION_STORAGE_MAINTENANCE_SCHEDULE_ID,
|
|
1575
2156
|
SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
1576
2157
|
SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
1577
2158
|
createDefaultWorkerActivities,
|
|
@@ -1580,12 +2161,10 @@ export {
|
|
|
1580
2161
|
createOpenGeniWorkerService,
|
|
1581
2162
|
createWorkerWorkflowSignaler,
|
|
1582
2163
|
registerFileUploadReaperSchedule,
|
|
1583
|
-
|
|
1584
|
-
registerSandboxReaperSchedule,
|
|
1585
|
-
registerSessionWorkflowWakeDispatcherSchedule,
|
|
1586
|
-
registerSiteAuthMaintenanceSchedule,
|
|
2164
|
+
registerSessionStorageMaintenanceSchedule,
|
|
1587
2165
|
resolveOpenGeniWorkflowDefinition,
|
|
1588
2166
|
runOpenGeniWorker,
|
|
2167
|
+
startInternalMaintenanceCadences,
|
|
1589
2168
|
startWorker,
|
|
1590
2169
|
temporalActivityLeaseSettled,
|
|
1591
2170
|
temporalWorkflowExecutionNotFound,
|