@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38686408778001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +172 -28
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +3 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +30 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-NTXZHYET.js} +1552 -2876
- package/dist/activities-control-NTXZHYET.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-RKRI5GY4.js} +9238 -4392
- package/dist/activities-turn-RKRI5GY4.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +104 -35
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-7BIHFUGB.js} +2293 -3956
- package/dist/chunk-7BIHFUGB.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1134 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +26 -4
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +599 -110
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +162 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +116 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +758 -141
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +79 -38
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
import { temporalConnectionOptions, type Settings } from "@opengeni/config";
|
|
2
|
+
import {
|
|
3
|
+
SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
|
|
4
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
5
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
6
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
|
|
7
|
+
SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE,
|
|
8
|
+
} from "@opengeni/core";
|
|
9
|
+
import {
|
|
10
|
+
automaticSessionTitleFanoutDue,
|
|
11
|
+
browserDeadlineCheckpointDue,
|
|
12
|
+
knowledgeIndexDue,
|
|
13
|
+
sandboxIdleCheckpointDue,
|
|
14
|
+
sandboxLeaseReaperDue,
|
|
15
|
+
sandboxMaintenanceDue,
|
|
16
|
+
sessionWorkflowWakesDue,
|
|
17
|
+
siteAuthMaintenanceDue,
|
|
18
|
+
workspacePauseTimerDue,
|
|
19
|
+
type Database,
|
|
20
|
+
} from "@opengeni/db";
|
|
21
|
+
import type { DocumentServices } from "@opengeni/documents";
|
|
22
|
+
import type { Observability } from "@opengeni/observability";
|
|
23
|
+
import {
|
|
24
|
+
Connection,
|
|
25
|
+
ScheduleNotFoundError,
|
|
26
|
+
Client as TemporalClient,
|
|
27
|
+
WorkflowExecutionAlreadyStartedError,
|
|
28
|
+
} from "@temporalio/client";
|
|
29
|
+
import {
|
|
30
|
+
startMaintenanceCadences,
|
|
31
|
+
type MaintenanceCadence,
|
|
32
|
+
type MaintenanceCadences,
|
|
33
|
+
type MaintenanceStartResult,
|
|
34
|
+
} from "./maintenance-cadence";
|
|
35
|
+
import {
|
|
36
|
+
BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
37
|
+
browserDeadlineCheckpointTaskQueue,
|
|
38
|
+
} from "./browser-deadline-checkpoint-contract";
|
|
39
|
+
import {
|
|
40
|
+
SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
41
|
+
SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
42
|
+
sandboxLifecycleTaskQueue,
|
|
43
|
+
} from "./sandbox-reaper-contract";
|
|
44
|
+
import { assertSandboxReaperActivityTimeout } from "./sandbox-reaper-timeout";
|
|
45
|
+
import type { TelemetryScheduler } from "./telemetry-scheduler";
|
|
46
|
+
|
|
47
|
+
export const KNOWLEDGE_INDEXING_SCHEDULE_ID = "opengeni-knowledge-indexing";
|
|
48
|
+
export const KNOWLEDGE_INDEXING_WORKFLOW_ID = "opengeni-knowledge-indexing";
|
|
49
|
+
export const KNOWLEDGE_INDEXING_PERIOD_MS = 15_000;
|
|
50
|
+
export const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
|
|
51
|
+
export const SITE_AUTH_MAINTENANCE_WORKFLOW_ID = "opengeni-site-auth-maintenance";
|
|
52
|
+
/** Matches the minimum public site-auth health policy interval. */
|
|
53
|
+
export const SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1_000;
|
|
54
|
+
/** The Schedule earlier releases used to drive the sandbox lease reaper. */
|
|
55
|
+
export const SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
56
|
+
|
|
57
|
+
export const INTERNAL_MAINTENANCE_JOBS = [
|
|
58
|
+
"session-workflow-wake-dispatcher",
|
|
59
|
+
"knowledge-indexing",
|
|
60
|
+
"site-auth-maintenance",
|
|
61
|
+
"sandbox-lease-reaper",
|
|
62
|
+
"sandbox-idle-checkpoint",
|
|
63
|
+
"browser-deadline-checkpoint",
|
|
64
|
+
] as const;
|
|
65
|
+
export type InternalMaintenanceJob = (typeof INTERNAL_MAINTENANCE_JOBS)[number];
|
|
66
|
+
|
|
67
|
+
/** Schedules that earlier releases created for these jobs. A paused Schedule
|
|
68
|
+
* still runs its scheduler workflow on every tick, so they are deleted. */
|
|
69
|
+
const LEGACY_SCHEDULE_IDS: Partial<Record<InternalMaintenanceJob, string>> = {
|
|
70
|
+
"session-workflow-wake-dispatcher": SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID,
|
|
71
|
+
"knowledge-indexing": KNOWLEDGE_INDEXING_SCHEDULE_ID,
|
|
72
|
+
"site-auth-maintenance": SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
73
|
+
// The old reaper Schedule fired a router workflow that started the drain
|
|
74
|
+
// inventory, both checkpoint sweeps and the maintenance child on every tick.
|
|
75
|
+
"sandbox-lease-reaper": SANDBOX_REAPER_SCHEDULE_ID,
|
|
76
|
+
};
|
|
77
|
+
|
|
78
|
+
type WorkflowStarter = Pick<TemporalClient["workflow"], "start" | "signalWithStart">;
|
|
79
|
+
type ScheduleDeleter = { getHandle(id: string): { delete(): Promise<void> } };
|
|
80
|
+
|
|
81
|
+
/** Start a fixed-id maintenance workflow unless one is already running; the
|
|
82
|
+
* running sweep's own claims cover whatever this tick saw. */
|
|
83
|
+
export async function startMaintenanceWorkflowOnce(
|
|
84
|
+
workflow: WorkflowStarter,
|
|
85
|
+
input: { workflowType: string; workflowId: string; taskQueue: string },
|
|
86
|
+
): Promise<MaintenanceStartResult> {
|
|
87
|
+
try {
|
|
88
|
+
await workflow.start(input.workflowType, {
|
|
89
|
+
taskQueue: input.taskQueue,
|
|
90
|
+
workflowId: input.workflowId,
|
|
91
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
92
|
+
args: [],
|
|
93
|
+
});
|
|
94
|
+
return "started";
|
|
95
|
+
} catch (error) {
|
|
96
|
+
if (error instanceof WorkflowExecutionAlreadyStartedError) return "already_running";
|
|
97
|
+
throw error;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Remove the per-tick Schedules this release replaces. Absent is success. */
|
|
102
|
+
export async function deleteLegacyMaintenanceSchedules(
|
|
103
|
+
schedule: ScheduleDeleter,
|
|
104
|
+
jobs: readonly InternalMaintenanceJob[],
|
|
105
|
+
observability: Observability,
|
|
106
|
+
): Promise<void> {
|
|
107
|
+
for (const job of jobs) {
|
|
108
|
+
const scheduleId = LEGACY_SCHEDULE_IDS[job];
|
|
109
|
+
if (!scheduleId) continue;
|
|
110
|
+
try {
|
|
111
|
+
await schedule.getHandle(scheduleId).delete();
|
|
112
|
+
observability.info(
|
|
113
|
+
"Deleted a per-tick maintenance Schedule replaced by a work-checked cadence",
|
|
114
|
+
{
|
|
115
|
+
job,
|
|
116
|
+
scheduleId,
|
|
117
|
+
},
|
|
118
|
+
);
|
|
119
|
+
} catch (error) {
|
|
120
|
+
if (!(error instanceof ScheduleNotFoundError)) throw error;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* Build the work-checked cadences for the deployment-wide dispatchers. Each
|
|
127
|
+
* tick runs one cheap read-only check and touches Temporal only when the check
|
|
128
|
+
* may find work, so an idle deployment writes no workflow history at all.
|
|
129
|
+
*/
|
|
130
|
+
export async function internalMaintenanceCadences(input: {
|
|
131
|
+
settings: Settings;
|
|
132
|
+
observability: Observability;
|
|
133
|
+
db: Database;
|
|
134
|
+
workflow: WorkflowStarter;
|
|
135
|
+
documentServices?: DocumentServices;
|
|
136
|
+
jobs?: readonly InternalMaintenanceJob[];
|
|
137
|
+
}): Promise<MaintenanceCadence[]> {
|
|
138
|
+
const { settings, observability, db, workflow } = input;
|
|
139
|
+
const jobs = new Set(input.jobs ?? INTERNAL_MAINTENANCE_JOBS);
|
|
140
|
+
const taskQueue = settings.temporalTaskQueue;
|
|
141
|
+
const cadences: MaintenanceCadence[] = [];
|
|
142
|
+
if (jobs.has("session-workflow-wake-dispatcher")) {
|
|
143
|
+
const { repairMissedSessionWorkflowWakes } = await import("./activities/parent-wake");
|
|
144
|
+
cadences.push({
|
|
145
|
+
job: "session-workflow-wake-dispatcher",
|
|
146
|
+
periodMs: SESSION_WORKFLOW_WAKE_DISPATCHER_PERIOD_MS,
|
|
147
|
+
hasWork: async () => {
|
|
148
|
+
// Registration is the dispatcher's own idempotent repair: it writes
|
|
149
|
+
// only when a producer committed work but never recorded its wake.
|
|
150
|
+
await repairMissedSessionWorkflowWakes({ db, observability });
|
|
151
|
+
return (
|
|
152
|
+
(await sessionWorkflowWakesDue(db)) ||
|
|
153
|
+
(await workspacePauseTimerDue(db)) ||
|
|
154
|
+
(await automaticSessionTitleFanoutDue(db))
|
|
155
|
+
);
|
|
156
|
+
},
|
|
157
|
+
start: async () => {
|
|
158
|
+
await workflow.signalWithStart(SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_TYPE, {
|
|
159
|
+
taskQueue,
|
|
160
|
+
workflowId: SESSION_WORKFLOW_WAKE_DISPATCHER_WORKFLOW_ID,
|
|
161
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
162
|
+
args: [],
|
|
163
|
+
signal: SESSION_WORKFLOW_WAKE_DISPATCH_REQUESTED_SIGNAL,
|
|
164
|
+
});
|
|
165
|
+
return "started";
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
if (jobs.has("knowledge-indexing")) {
|
|
170
|
+
let embedding: Promise<{ model: string; dimensions: number; available: boolean }> | undefined;
|
|
171
|
+
let unavailableReported = false;
|
|
172
|
+
const resolveEmbedding = async () => {
|
|
173
|
+
const services =
|
|
174
|
+
input.documentServices ??
|
|
175
|
+
(await import("@opengeni/documents")).createDocumentServices(settings);
|
|
176
|
+
return {
|
|
177
|
+
model: services.embedder.model,
|
|
178
|
+
dimensions: services.embedder.dimensions,
|
|
179
|
+
available: services.embedder.available !== false,
|
|
180
|
+
};
|
|
181
|
+
};
|
|
182
|
+
cadences.push({
|
|
183
|
+
job: "knowledge-indexing",
|
|
184
|
+
periodMs: KNOWLEDGE_INDEXING_PERIOD_MS,
|
|
185
|
+
hasWork: async () => {
|
|
186
|
+
embedding ??= resolveEmbedding().catch((error) => {
|
|
187
|
+
embedding = undefined;
|
|
188
|
+
throw error;
|
|
189
|
+
});
|
|
190
|
+
const current = await embedding;
|
|
191
|
+
// Without an embedding credential every batch fails before reaching the
|
|
192
|
+
// provider, so queued jobs are not work this process can do. They stay
|
|
193
|
+
// queued and index once the deployment configures a credential.
|
|
194
|
+
if (!current.available) {
|
|
195
|
+
if (!unavailableReported) {
|
|
196
|
+
unavailableReported = true;
|
|
197
|
+
observability.warn(
|
|
198
|
+
"Knowledge indexing is idle: no document embedding credential is configured",
|
|
199
|
+
{ job: "knowledge-indexing", model: current.model },
|
|
200
|
+
);
|
|
201
|
+
}
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
return await knowledgeIndexDue(db, current);
|
|
205
|
+
},
|
|
206
|
+
start: async () =>
|
|
207
|
+
await startMaintenanceWorkflowOnce(workflow, {
|
|
208
|
+
workflowType: "knowledgeIndexingWorkflow",
|
|
209
|
+
workflowId: KNOWLEDGE_INDEXING_WORKFLOW_ID,
|
|
210
|
+
taskQueue,
|
|
211
|
+
}),
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
if (jobs.has("site-auth-maintenance")) {
|
|
215
|
+
const { SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS } =
|
|
216
|
+
await import("./activities/site-auth-maintenance");
|
|
217
|
+
cadences.push({
|
|
218
|
+
job: "site-auth-maintenance",
|
|
219
|
+
periodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
220
|
+
hasWork: async () =>
|
|
221
|
+
await siteAuthMaintenanceDue(db, {
|
|
222
|
+
claimTimeoutMs: SITE_AUTH_MAINTENANCE_CLAIM_TIMEOUT_MS,
|
|
223
|
+
}),
|
|
224
|
+
start: async () =>
|
|
225
|
+
await startMaintenanceWorkflowOnce(workflow, {
|
|
226
|
+
workflowType: "siteAuthMaintenanceWorkflow",
|
|
227
|
+
workflowId: SITE_AUTH_MAINTENANCE_WORKFLOW_ID,
|
|
228
|
+
taskQueue,
|
|
229
|
+
}),
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
const sandboxJobs = (
|
|
233
|
+
["sandbox-lease-reaper", "sandbox-idle-checkpoint", "browser-deadline-checkpoint"] as const
|
|
234
|
+
).filter((job) => jobs.has(job));
|
|
235
|
+
if (sandboxJobs.length > 0) {
|
|
236
|
+
// The same period the old reaper Schedule used, so the boot invariants
|
|
237
|
+
// (reaper period below the viewer holder TTL, and with the idle grace
|
|
238
|
+
// below the provider lifetime) hold for every sandbox cadence.
|
|
239
|
+
assertSandboxReaperActivityTimeout(settings);
|
|
240
|
+
const periodMs = settings.sandboxLeaseReaperPeriodMs;
|
|
241
|
+
const lifecycleQueue = sandboxLifecycleTaskQueue(taskQueue);
|
|
242
|
+
const sandbox = await import("./activities/sandbox-lease");
|
|
243
|
+
if (jobs.has("sandbox-lease-reaper")) {
|
|
244
|
+
const { SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS } = sandbox;
|
|
245
|
+
cadences.push({
|
|
246
|
+
job: "sandbox-lease-reaper",
|
|
247
|
+
periodMs,
|
|
248
|
+
// The drain inventory and its maintenance pass act only on warm or held
|
|
249
|
+
// leases, due reconciliation rows, or (with Modal) boxes still running in
|
|
250
|
+
// the provider app. With none of those, the tick only refreshes the
|
|
251
|
+
// pass's read-only gauges in this process.
|
|
252
|
+
hasWork: async () => {
|
|
253
|
+
if (
|
|
254
|
+
(await sandboxLeaseReaperDue(db)) ||
|
|
255
|
+
(await sandboxMaintenanceDue(db, {
|
|
256
|
+
checkpointArtifactClaimTtlMs: SANDBOX_CHECKPOINT_GC_CLAIM_TTL_MS,
|
|
257
|
+
})) ||
|
|
258
|
+
(await sandbox.modalOrphanSweepMayHaveWork({ settings, observability }))
|
|
259
|
+
) {
|
|
260
|
+
return true;
|
|
261
|
+
}
|
|
262
|
+
await sandbox.refreshIdleSandboxReaperProjections({ db, settings, observability });
|
|
263
|
+
return false;
|
|
264
|
+
},
|
|
265
|
+
start: async () =>
|
|
266
|
+
await startMaintenanceWorkflowOnce(workflow, {
|
|
267
|
+
workflowType: "sandboxReaperWorkflowV2",
|
|
268
|
+
workflowId: SANDBOX_REAPER_V2_WORKFLOW_ID,
|
|
269
|
+
taskQueue: lifecycleQueue,
|
|
270
|
+
}),
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
if (jobs.has("sandbox-idle-checkpoint")) {
|
|
274
|
+
cadences.push({
|
|
275
|
+
job: "sandbox-idle-checkpoint",
|
|
276
|
+
periodMs,
|
|
277
|
+
// The sweep's inventory is empty when ownership or snapshots are off.
|
|
278
|
+
hasWork: async () =>
|
|
279
|
+
settings.sandboxOwnershipEnabled &&
|
|
280
|
+
settings.sandboxSnapshotIntervalMs > 0 &&
|
|
281
|
+
(await sandboxIdleCheckpointDue(db, { intervalMs: settings.sandboxSnapshotIntervalMs })),
|
|
282
|
+
start: async () =>
|
|
283
|
+
await startMaintenanceWorkflowOnce(workflow, {
|
|
284
|
+
workflowType: "sandboxIdleCheckpointSweepWorkflow",
|
|
285
|
+
workflowId: SANDBOX_IDLE_CHECKPOINT_SWEEP_WORKFLOW_ID,
|
|
286
|
+
taskQueue: lifecycleQueue,
|
|
287
|
+
}),
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
if (jobs.has("browser-deadline-checkpoint")) {
|
|
291
|
+
cadences.push({
|
|
292
|
+
job: "browser-deadline-checkpoint",
|
|
293
|
+
periodMs,
|
|
294
|
+
hasWork: async () => await browserDeadlineCheckpointDue(db),
|
|
295
|
+
start: async () =>
|
|
296
|
+
await startMaintenanceWorkflowOnce(workflow, {
|
|
297
|
+
workflowType: "browserDeadlineCheckpointSweepWorkflow",
|
|
298
|
+
workflowId: BROWSER_DEADLINE_CHECKPOINT_SWEEP_ID,
|
|
299
|
+
taskQueue: browserDeadlineCheckpointTaskQueue(taskQueue),
|
|
300
|
+
}),
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
return cadences;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* Run the deployment-wide maintenance dispatchers from this control worker.
|
|
309
|
+
* Several control replicas may each run the cadence: an idle tick is only a
|
|
310
|
+
* read, and a busy one converges on the same fixed workflow id.
|
|
311
|
+
*/
|
|
312
|
+
export async function startInternalMaintenanceCadences(input: {
|
|
313
|
+
settings: Settings;
|
|
314
|
+
observability: Observability;
|
|
315
|
+
db: Database;
|
|
316
|
+
documentServices?: DocumentServices;
|
|
317
|
+
jobs?: readonly InternalMaintenanceJob[];
|
|
318
|
+
scheduler?: TelemetryScheduler;
|
|
319
|
+
}): Promise<{ cadences: MaintenanceCadences; close: () => Promise<void> }> {
|
|
320
|
+
const connection = await Connection.connect(temporalConnectionOptions(input.settings));
|
|
321
|
+
try {
|
|
322
|
+
const temporal = new TemporalClient({
|
|
323
|
+
connection,
|
|
324
|
+
namespace: input.settings.temporalNamespace,
|
|
325
|
+
});
|
|
326
|
+
const jobs = input.jobs ?? INTERNAL_MAINTENANCE_JOBS;
|
|
327
|
+
await deleteLegacyMaintenanceSchedules(temporal.schedule, jobs, input.observability);
|
|
328
|
+
const cadences = startMaintenanceCadences({
|
|
329
|
+
cadences: await internalMaintenanceCadences({ ...input, jobs, workflow: temporal.workflow }),
|
|
330
|
+
observability: input.observability,
|
|
331
|
+
...(input.scheduler ? { scheduler: input.scheduler } : {}),
|
|
332
|
+
});
|
|
333
|
+
input.observability.info("Started work-checked maintenance cadences", {
|
|
334
|
+
jobs: jobs.join(","),
|
|
335
|
+
});
|
|
336
|
+
return {
|
|
337
|
+
cadences,
|
|
338
|
+
close: async () => {
|
|
339
|
+
await cadences.close();
|
|
340
|
+
await connection.close();
|
|
341
|
+
},
|
|
342
|
+
};
|
|
343
|
+
} catch (error) {
|
|
344
|
+
await connection.close().catch(() => undefined);
|
|
345
|
+
throw error;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import type { Observability } from "@opengeni/observability";
|
|
2
|
+
import {
|
|
3
|
+
telemetryInterval,
|
|
4
|
+
telemetryScheduler,
|
|
5
|
+
type TelemetryScheduler,
|
|
6
|
+
} from "./telemetry-scheduler";
|
|
7
|
+
|
|
8
|
+
export type MaintenanceStartResult = "started" | "already_running";
|
|
9
|
+
|
|
10
|
+
export type MaintenanceTickOutcome =
|
|
11
|
+
/** The work check proved there is nothing to do; no workflow was touched. */
|
|
12
|
+
| "idle"
|
|
13
|
+
| "started"
|
|
14
|
+
| "already_running"
|
|
15
|
+
/** The check failed; the workflow was started anyway (fail open). */
|
|
16
|
+
| "probe_failed"
|
|
17
|
+
| "start_failed"
|
|
18
|
+
/** This process's previous tick for the job has not finished. */
|
|
19
|
+
| "busy";
|
|
20
|
+
|
|
21
|
+
const TICK_OUTCOMES: readonly MaintenanceTickOutcome[] = [
|
|
22
|
+
"idle",
|
|
23
|
+
"started",
|
|
24
|
+
"already_running",
|
|
25
|
+
"probe_failed",
|
|
26
|
+
"start_failed",
|
|
27
|
+
"busy",
|
|
28
|
+
];
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* One deployment-wide maintenance job driven by a worker-process cadence
|
|
32
|
+
* instead of a Temporal Schedule. A Schedule starts a workflow execution on
|
|
33
|
+
* every tick, so an idle deployment pays a history and visibility write per
|
|
34
|
+
* tick for no work. A cadence instead asks a cheap read-only question first and
|
|
35
|
+
* starts the job's workflow only when the answer may be yes.
|
|
36
|
+
*/
|
|
37
|
+
export type MaintenanceCadence = {
|
|
38
|
+
/** Stable, low-cardinality metric label. */
|
|
39
|
+
job: string;
|
|
40
|
+
periodMs: number;
|
|
41
|
+
/**
|
|
42
|
+
* A superset of the job's claim predicate: `false` must prove that a run
|
|
43
|
+
* would find nothing. It may perform the job's own idempotent repair
|
|
44
|
+
* registration, but never claims or delivers work.
|
|
45
|
+
*/
|
|
46
|
+
hasWork: () => Promise<boolean>;
|
|
47
|
+
/** Start or nudge the job's single dispatcher workflow. */
|
|
48
|
+
start: () => Promise<MaintenanceStartResult>;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
export type MaintenanceCadences = {
|
|
52
|
+
/** Run one tick for a job now; used by tests and by the cadence itself. */
|
|
53
|
+
tick: (job: string) => Promise<MaintenanceTickOutcome>;
|
|
54
|
+
close: () => Promise<void>;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export function startMaintenanceCadences(input: {
|
|
58
|
+
cadences: readonly MaintenanceCadence[];
|
|
59
|
+
observability: Observability;
|
|
60
|
+
scheduler?: TelemetryScheduler;
|
|
61
|
+
now?: () => number;
|
|
62
|
+
/** Tests drive ticks explicitly. */
|
|
63
|
+
autoStart?: boolean;
|
|
64
|
+
}): MaintenanceCadences {
|
|
65
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
66
|
+
const now = input.now ?? Date.now;
|
|
67
|
+
const jobs = new Map<string, MaintenanceCadence>();
|
|
68
|
+
for (const cadence of input.cadences) {
|
|
69
|
+
if (jobs.has(cadence.job)) throw new Error(`duplicate maintenance cadence ${cadence.job}`);
|
|
70
|
+
telemetryInterval(cadence.periodMs, `${cadence.job} maintenance period`);
|
|
71
|
+
jobs.set(cadence.job, cadence);
|
|
72
|
+
for (const outcome of TICK_OUTCOMES) {
|
|
73
|
+
input.observability.incrementCounter({
|
|
74
|
+
name: "opengeni_maintenance_cadence_ticks_total",
|
|
75
|
+
help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
|
|
76
|
+
labels: { job: cadence.job, outcome },
|
|
77
|
+
amount: 0,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
const inFlight = new Map<string, Promise<MaintenanceTickOutcome>>();
|
|
82
|
+
const cancels: Array<() => void> = [];
|
|
83
|
+
let stopped = false;
|
|
84
|
+
|
|
85
|
+
const record = (job: string, outcome: MaintenanceTickOutcome) => {
|
|
86
|
+
input.observability.incrementCounter({
|
|
87
|
+
name: "opengeni_maintenance_cadence_ticks_total",
|
|
88
|
+
help: "Maintenance cadence ticks by job and outcome; idle ticks start no workflow.",
|
|
89
|
+
labels: { job, outcome },
|
|
90
|
+
});
|
|
91
|
+
if (outcome !== "busy" && outcome !== "start_failed") {
|
|
92
|
+
input.observability.setGauge({
|
|
93
|
+
name: "opengeni_maintenance_cadence_last_success_timestamp_seconds",
|
|
94
|
+
help: "Unix time of the latest maintenance cadence tick that checked for work and, when needed, started its workflow.",
|
|
95
|
+
labels: { job },
|
|
96
|
+
value: now() / 1_000,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
return outcome;
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
const run = async (cadence: MaintenanceCadence): Promise<MaintenanceTickOutcome> => {
|
|
103
|
+
let probeFailed = false;
|
|
104
|
+
try {
|
|
105
|
+
if (!(await cadence.hasWork())) return record(cadence.job, "idle");
|
|
106
|
+
} catch (error) {
|
|
107
|
+
// Fail open: a broken check must never silently stop maintenance. The
|
|
108
|
+
// workflow's own claim remains the authority on what work exists.
|
|
109
|
+
probeFailed = true;
|
|
110
|
+
input.observability.warn("maintenance cadence work check failed; starting the job anyway", {
|
|
111
|
+
job: cadence.job,
|
|
112
|
+
error: error instanceof Error ? error.message : String(error),
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
try {
|
|
116
|
+
const started = await cadence.start();
|
|
117
|
+
return record(cadence.job, probeFailed ? "probe_failed" : started);
|
|
118
|
+
} catch (error) {
|
|
119
|
+
input.observability.warn("maintenance cadence could not start its workflow", {
|
|
120
|
+
job: cadence.job,
|
|
121
|
+
error: error instanceof Error ? error.message : String(error),
|
|
122
|
+
});
|
|
123
|
+
return record(cadence.job, "start_failed");
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
|
|
127
|
+
const tick = async (job: string): Promise<MaintenanceTickOutcome> => {
|
|
128
|
+
const cadence = jobs.get(job);
|
|
129
|
+
if (!cadence) throw new Error(`unknown maintenance cadence ${job}`);
|
|
130
|
+
if (stopped) return "busy";
|
|
131
|
+
if (inFlight.has(job)) return record(job, "busy");
|
|
132
|
+
const running = run(cadence).finally(() => inFlight.delete(job));
|
|
133
|
+
inFlight.set(job, running);
|
|
134
|
+
return await running;
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
if (input.autoStart !== false) {
|
|
138
|
+
for (const cadence of jobs.values()) {
|
|
139
|
+
const fire = () => {
|
|
140
|
+
void tick(cadence.job);
|
|
141
|
+
};
|
|
142
|
+
// Start immediately so a restart never delays due work by a full period.
|
|
143
|
+
scheduler.defer(fire);
|
|
144
|
+
cancels.push(scheduler.interval(fire, cadence.periodMs));
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return {
|
|
149
|
+
tick,
|
|
150
|
+
close: async () => {
|
|
151
|
+
stopped = true;
|
|
152
|
+
for (const cancel of cancels.splice(0)) cancel();
|
|
153
|
+
await Promise.allSettled([...inFlight.values()]);
|
|
154
|
+
},
|
|
155
|
+
};
|
|
156
|
+
}
|