@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -1,17 +1,43 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
3
|
+
import {
|
|
4
|
+
BundledSkillId,
|
|
5
|
+
CREDIT_GRANT_CLASSES,
|
|
6
|
+
SandboxBackend,
|
|
7
|
+
type SessionEventType,
|
|
8
|
+
type SkillReadKind,
|
|
9
|
+
type SkillUseSource,
|
|
10
|
+
} from "@opengeni/contracts";
|
|
11
|
+
import {
|
|
12
|
+
ACTIVE_USER_WINDOWS,
|
|
13
|
+
type ActiveUserWindow,
|
|
14
|
+
type CreditGrantTotals,
|
|
15
|
+
type IdleInteractionInspection,
|
|
16
|
+
type SessionEventAppendPhaseObservation,
|
|
17
|
+
} from "@opengeni/db";
|
|
18
|
+
import {
|
|
19
|
+
natsSubscriptionTerminationCounter,
|
|
20
|
+
type EventBusOptions,
|
|
21
|
+
type EventLogger,
|
|
22
|
+
} from "@opengeni/events";
|
|
6
23
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
7
24
|
import type { CompanyBrainContributionReceipt } from "./model-context-contributions";
|
|
25
|
+
import {
|
|
26
|
+
createTelemetryClock,
|
|
27
|
+
telemetryInterval,
|
|
28
|
+
telemetryScheduler,
|
|
29
|
+
type TelemetryScheduler,
|
|
30
|
+
} from "./telemetry-scheduler";
|
|
8
31
|
import {
|
|
9
32
|
OPENSANDBOX_BATCHSANDBOX_PHASES,
|
|
10
33
|
OPENSANDBOX_WORKLOAD_POD_CONDITIONS,
|
|
11
34
|
type OpenSandboxKubernetesInventory,
|
|
12
35
|
} from "./opensandbox-kubernetes-inventory";
|
|
13
36
|
import {
|
|
37
|
+
MCP_TOOL_METRIC_EXTERNAL_LABEL,
|
|
14
38
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
39
|
+
WORKSPACE_CAPTURE_SKIP_REASONS,
|
|
40
|
+
isMcpToolMetricLabel,
|
|
15
41
|
modelUsageTokenCountOrNull,
|
|
16
42
|
type RuntimeMetricsHooks,
|
|
17
43
|
type SelfhostedOpObservation,
|
|
@@ -31,9 +57,23 @@ export type TurnTaskQueueStats = {
|
|
|
31
57
|
tasksAddRate: number;
|
|
32
58
|
tasksDispatchRate: number;
|
|
33
59
|
};
|
|
60
|
+
export type TurnTaskQueueIdentity = { temporalNamespace: string; taskQueue: string };
|
|
61
|
+
export type TurnTaskQueueReadOptions = { signal: AbortSignal; deadline: number };
|
|
62
|
+
|
|
63
|
+
export function turnTaskQueueMetricLabels(identity: TurnTaskQueueIdentity) {
|
|
64
|
+
return { temporal_namespace: identity.temporalNamespace, task_queue: identity.taskQueue };
|
|
65
|
+
}
|
|
66
|
+
export type SessionRecoveryBacklogStateSummary = {
|
|
67
|
+
/** Every durable recovering session in this state with no active attempt. */
|
|
68
|
+
count: number;
|
|
69
|
+
/** Of those, sessions still inside the backoff their recovery request recorded. */
|
|
70
|
+
scheduled: number;
|
|
71
|
+
/** Seconds the single oldest candidate has been past its recovery due time; 0 when none is. */
|
|
72
|
+
oldestOverdueSeconds: number;
|
|
73
|
+
};
|
|
34
74
|
export type SessionRecoveryBacklog = {
|
|
35
|
-
quiescence_missing:
|
|
36
|
-
projection_stale:
|
|
75
|
+
quiescence_missing: SessionRecoveryBacklogStateSummary;
|
|
76
|
+
projection_stale: SessionRecoveryBacklogStateSummary;
|
|
37
77
|
};
|
|
38
78
|
export type ContextCompactionPendingSummary = {
|
|
39
79
|
pendingCount: number;
|
|
@@ -80,6 +120,16 @@ const CONTEXT_COMPACTIONS_METRIC = {
|
|
|
80
120
|
help: "Total completed context compactions, by trigger.",
|
|
81
121
|
} as const;
|
|
82
122
|
|
|
123
|
+
/** Logger plus the closed-label subscription-termination counter for NATS connections. */
|
|
124
|
+
export function observabilityEventBusOptions(
|
|
125
|
+
observability: Observability,
|
|
126
|
+
): Pick<EventBusOptions, "logger" | "onSubscriptionTerminated"> {
|
|
127
|
+
return {
|
|
128
|
+
logger: observabilityEventLogger(observability),
|
|
129
|
+
onSubscriptionTerminated: natsSubscriptionTerminationCounter(observability),
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
|
|
83
133
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
84
134
|
return {
|
|
85
135
|
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
@@ -156,6 +206,13 @@ export function runtimeMetricsHooksForObservability(
|
|
|
156
206
|
labels: { backend, stage },
|
|
157
207
|
});
|
|
158
208
|
},
|
|
209
|
+
onSandboxReadinessReplacement: ({ backend, outcome }) => {
|
|
210
|
+
observability.incrementCounter({
|
|
211
|
+
name: "opengeni_sandbox_readiness_replacements_total",
|
|
212
|
+
help: "Fresh sandbox command-readiness replacement decisions by backend and outcome.",
|
|
213
|
+
labels: { backend, outcome },
|
|
214
|
+
});
|
|
215
|
+
},
|
|
159
216
|
onSandboxProviderApiThrottle: ({ backend, operation }) => {
|
|
160
217
|
observability.incrementCounter({
|
|
161
218
|
name: "opengeni_sandbox_provider_api_throttles_total",
|
|
@@ -191,6 +248,17 @@ export function runtimeMetricsHooksForObservability(
|
|
|
191
248
|
value: durationSeconds,
|
|
192
249
|
});
|
|
193
250
|
},
|
|
251
|
+
onWorkspaceCaptureSkipped: ({ backend, reason }) => {
|
|
252
|
+
const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
|
|
253
|
+
const safeReason = (WORKSPACE_CAPTURE_SKIP_REASONS as readonly string[]).includes(reason)
|
|
254
|
+
? reason
|
|
255
|
+
: "unknown";
|
|
256
|
+
observability.incrementCounter({
|
|
257
|
+
name: "opengeni_workspace_capture_skipped_total",
|
|
258
|
+
help: "Due warm workspace checkpoints that could not start, by backend and fixed reason. Not-yet-due (throttled) and already-complete checkpoints are excluded.",
|
|
259
|
+
labels: { backend: safeBackend, reason: safeReason },
|
|
260
|
+
});
|
|
261
|
+
},
|
|
194
262
|
onWorkspaceArchiveObject: ({ outcome, backend }) => {
|
|
195
263
|
observability.incrementCounter({
|
|
196
264
|
name: "opengeni_workspace_archive_object_total",
|
|
@@ -198,17 +266,23 @@ export function runtimeMetricsHooksForObservability(
|
|
|
198
266
|
labels: { outcome, backend },
|
|
199
267
|
});
|
|
200
268
|
},
|
|
201
|
-
onMcpToolCall: ({ outcome, durationSeconds }) => {
|
|
202
|
-
|
|
269
|
+
onMcpToolCall: ({ outcome, tool, durationSeconds }) => {
|
|
270
|
+
// Re-check the closed label set at the export boundary so a caller can
|
|
271
|
+
// never turn a raw user-defined tool name into a metric series.
|
|
272
|
+
const safeTool = isMcpToolMetricLabel(tool) ? tool : MCP_TOOL_METRIC_EXTERNAL_LABEL;
|
|
273
|
+
completedOperationSpan(observability, "worker.mcp.tool_call", durationSeconds, {
|
|
274
|
+
outcome,
|
|
275
|
+
tool: safeTool,
|
|
276
|
+
});
|
|
203
277
|
observability.incrementCounter({
|
|
204
278
|
name: "opengeni_mcp_tool_calls_total",
|
|
205
|
-
help: "Total physical MCP tool calls by bounded structural outcome.",
|
|
206
|
-
labels: { outcome },
|
|
279
|
+
help: "Total physical MCP tool calls by bounded structural outcome and first-party tool (other tools are 'external').",
|
|
280
|
+
labels: { outcome, tool: safeTool },
|
|
207
281
|
});
|
|
208
282
|
observability.observeHistogram({
|
|
209
283
|
name: "opengeni_mcp_tool_call_duration_seconds",
|
|
210
|
-
help: "MCP tool-call duration in seconds by bounded structural outcome.",
|
|
211
|
-
labels: { outcome },
|
|
284
|
+
help: "MCP tool-call duration in seconds by bounded structural outcome and first-party tool (other tools are 'external').",
|
|
285
|
+
labels: { outcome, tool: safeTool },
|
|
212
286
|
value: durationSeconds,
|
|
213
287
|
});
|
|
214
288
|
},
|
|
@@ -667,7 +741,7 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
|
|
|
667
741
|
}
|
|
668
742
|
|
|
669
743
|
/**
|
|
670
|
-
* Record the
|
|
744
|
+
* Record the approximate shared turn-worker activity queue rather than Postgres
|
|
671
745
|
* prompts. A paused human prompt is durable queue truth but is not runnable and
|
|
672
746
|
* never reaches this Temporal task queue. DescribeTaskQueue's approximate
|
|
673
747
|
* backlog count and age are explicitly documented by Temporal as autoscaling
|
|
@@ -676,26 +750,40 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
|
|
|
676
750
|
export function recordTurnTaskQueueStats(
|
|
677
751
|
observability: Observability,
|
|
678
752
|
stats: TurnTaskQueueStats,
|
|
753
|
+
identity: TurnTaskQueueIdentity,
|
|
679
754
|
): void {
|
|
755
|
+
const labels = turnTaskQueueMetricLabels(identity);
|
|
756
|
+
// Invalid observations are failures, not a fabricated fresh zero. Validate
|
|
757
|
+
// every field before publishing any portion of a new observation.
|
|
758
|
+
const values = {
|
|
759
|
+
backlog: requiredTemporalNumber(stats.eligibleBacklog, "eligibleBacklog", { integer: true }),
|
|
760
|
+
age: requiredTemporalNumber(stats.oldestBacklogAgeSeconds, "oldestBacklogAgeSeconds"),
|
|
761
|
+
add: requiredTemporalNumber(stats.tasksAddRate, "tasksAddRate"),
|
|
762
|
+
dispatch: requiredTemporalNumber(stats.tasksDispatchRate, "tasksDispatchRate"),
|
|
763
|
+
};
|
|
680
764
|
observability.setGauge({
|
|
681
765
|
name: "opengeni_turn_eligible_backlog",
|
|
682
|
-
help: "Temporal
|
|
683
|
-
|
|
766
|
+
help: "Approximate Temporal activity backlog on the shared turn-worker queue, including video and retries.",
|
|
767
|
+
labels,
|
|
768
|
+
value: values.backlog,
|
|
684
769
|
});
|
|
685
770
|
observability.setGauge({
|
|
686
771
|
name: "opengeni_turn_eligible_backlog_oldest_age_seconds",
|
|
687
|
-
help: "Approximate age of the oldest eligible
|
|
688
|
-
|
|
772
|
+
help: "Approximate age of the oldest eligible activity task on the shared turn-worker queue.",
|
|
773
|
+
labels,
|
|
774
|
+
value: values.age,
|
|
689
775
|
});
|
|
690
776
|
observability.setGauge({
|
|
691
777
|
name: "opengeni_turn_eligible_tasks_add_rate",
|
|
692
|
-
help: "Temporal
|
|
693
|
-
|
|
778
|
+
help: "Temporal activities added per second to the shared turn-worker queue over its rolling window.",
|
|
779
|
+
labels,
|
|
780
|
+
value: values.add,
|
|
694
781
|
});
|
|
695
782
|
observability.setGauge({
|
|
696
783
|
name: "opengeni_turn_eligible_tasks_dispatch_rate",
|
|
697
|
-
help: "Temporal
|
|
698
|
-
|
|
784
|
+
help: "Temporal activities dispatched per second from the shared turn-worker queue over its rolling window.",
|
|
785
|
+
labels,
|
|
786
|
+
value: values.dispatch,
|
|
699
787
|
});
|
|
700
788
|
}
|
|
701
789
|
|
|
@@ -705,13 +793,22 @@ export function normalizeTurnTaskQueueStats(
|
|
|
705
793
|
if (!stats) {
|
|
706
794
|
throw new Error("Temporal DescribeTaskQueue response omitted required stats");
|
|
707
795
|
}
|
|
796
|
+
if (typeof stats !== "object" || Array.isArray(stats)) {
|
|
797
|
+
throw new Error("Temporal DescribeTaskQueue returned malformed stats");
|
|
798
|
+
}
|
|
708
799
|
const eligibleBacklog = requiredTemporalNumber(
|
|
709
800
|
stats.approximateBacklogCount,
|
|
710
801
|
"approximateBacklogCount",
|
|
711
802
|
{ integer: true },
|
|
712
803
|
);
|
|
713
804
|
let oldestBacklogAgeSeconds = 0;
|
|
714
|
-
if (stats.approximateBacklogAge) {
|
|
805
|
+
if (stats.approximateBacklogAge !== null && stats.approximateBacklogAge !== undefined) {
|
|
806
|
+
if (
|
|
807
|
+
typeof stats.approximateBacklogAge !== "object" ||
|
|
808
|
+
Array.isArray(stats.approximateBacklogAge)
|
|
809
|
+
) {
|
|
810
|
+
throw new Error("Temporal stats returned invalid approximateBacklogAge");
|
|
811
|
+
}
|
|
715
812
|
const seconds = optionalTemporalNumber(
|
|
716
813
|
stats.approximateBacklogAge.seconds,
|
|
717
814
|
"approximateBacklogAge.seconds",
|
|
@@ -737,24 +834,62 @@ export function normalizeTurnTaskQueueStats(
|
|
|
737
834
|
};
|
|
738
835
|
}
|
|
739
836
|
|
|
837
|
+
/** Requires a fresh metric registry/schema; pre-registered legacy unlabelled
|
|
838
|
+
* queue gauges are not migrated in place. Each collector owns its native read. */
|
|
740
839
|
export function startTurnCapacityMonitor(input: {
|
|
741
840
|
observability: Observability;
|
|
742
|
-
|
|
841
|
+
identity: TurnTaskQueueIdentity;
|
|
842
|
+
read: (options: TurnTaskQueueReadOptions) => Promise<TurnTaskQueueStats>;
|
|
743
843
|
intervalMs?: number;
|
|
844
|
+
readTimeoutMs?: number;
|
|
845
|
+
closeTimeoutMs?: number;
|
|
744
846
|
now?: () => number;
|
|
847
|
+
monotonicNow?: () => number;
|
|
848
|
+
scheduler?: TelemetryScheduler;
|
|
745
849
|
}): { close: () => Promise<void> } {
|
|
746
|
-
const intervalMs = input.intervalMs ?? 15_000;
|
|
747
|
-
const
|
|
748
|
-
|
|
850
|
+
const intervalMs = telemetryInterval(input.intervalMs ?? 15_000, "queue monitor intervalMs");
|
|
851
|
+
const readTimeoutMs = telemetryInterval(
|
|
852
|
+
input.readTimeoutMs ?? 5_000,
|
|
853
|
+
"queue monitor readTimeoutMs",
|
|
854
|
+
);
|
|
855
|
+
const closeTimeoutMs = telemetryInterval(
|
|
856
|
+
input.closeTimeoutMs ?? 1_000,
|
|
857
|
+
"queue monitor closeTimeoutMs",
|
|
858
|
+
);
|
|
859
|
+
const scheduler = input.scheduler ?? telemetryScheduler;
|
|
860
|
+
const labels = turnTaskQueueMetricLabels(input.identity);
|
|
861
|
+
const wallClock = createTelemetryClock(input.now ?? Date.now);
|
|
862
|
+
const monotonicClock = createTelemetryClock(input.monotonicNow ?? (() => performance.now()));
|
|
863
|
+
let startedAt = wallClock.read();
|
|
864
|
+
const readClock = () => {
|
|
865
|
+
const wall = wallClock.read();
|
|
866
|
+
const monotonic = monotonicClock.read();
|
|
867
|
+
return wall === null || monotonic === null ? null : { wall, monotonic };
|
|
868
|
+
};
|
|
749
869
|
let lastSuccessAt: number | null = null;
|
|
750
870
|
let lastReadSucceeded = false;
|
|
751
871
|
let stopped = false;
|
|
752
|
-
let running:
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
872
|
+
let running: {
|
|
873
|
+
controller: AbortController;
|
|
874
|
+
done: Promise<void>;
|
|
875
|
+
cancelTimeout: () => void;
|
|
876
|
+
} | null = null;
|
|
877
|
+
let closePromise: Promise<void> | undefined;
|
|
878
|
+
const safely = (action: () => void) => {
|
|
879
|
+
try {
|
|
880
|
+
action();
|
|
881
|
+
} catch {
|
|
882
|
+
/* Telemetry must not break the worker. */
|
|
883
|
+
}
|
|
884
|
+
};
|
|
885
|
+
const recordStatus = (observation = readClock()) => {
|
|
886
|
+
if (observation === null) lastReadSucceeded = false;
|
|
887
|
+
const observedAt = wallClock.latest();
|
|
888
|
+
startedAt ??= observedAt;
|
|
889
|
+
const successAgeMs =
|
|
890
|
+
observedAt === null ? 0 : observedAt - (lastSuccessAt ?? startedAt ?? observedAt);
|
|
756
891
|
const set = (name: string, help: string, value: number) =>
|
|
757
|
-
input.observability.setGauge({ name, help, value });
|
|
892
|
+
safely(() => input.observability.setGauge({ name, help, labels, value }));
|
|
758
893
|
set(
|
|
759
894
|
"opengeni_turn_capacity_monitor_last_read_success",
|
|
760
895
|
"Whether the latest Temporal turn-queue capacity read completed successfully.",
|
|
@@ -772,56 +907,127 @@ export function startTurnCapacityMonitor(input: {
|
|
|
772
907
|
);
|
|
773
908
|
set(
|
|
774
909
|
"opengeni_turn_capacity_monitor_fresh",
|
|
775
|
-
"Whether
|
|
910
|
+
"Whether the latest read succeeded and the shared activity-queue gauges are within three monitor intervals.",
|
|
776
911
|
lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,
|
|
777
912
|
);
|
|
778
913
|
};
|
|
779
914
|
const refresh = () => {
|
|
780
915
|
// Advance freshness age even while a Temporal read is hung. Old backlog
|
|
781
916
|
// values remain observable for diagnosis but cease to be authoritative.
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
917
|
+
if (stopped) return;
|
|
918
|
+
const observation = readClock();
|
|
919
|
+
recordStatus(observation);
|
|
920
|
+
if (running || observation === null) return;
|
|
921
|
+
const deadline = observation.wall + readTimeoutMs;
|
|
922
|
+
if (!Number.isFinite(deadline) || deadline > Number.MAX_SAFE_INTEGER) {
|
|
923
|
+
recordStatus(null);
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
const controller = new AbortController();
|
|
927
|
+
const attempt = { controller, done: Promise.resolve(), cancelTimeout: () => {} };
|
|
928
|
+
running = attempt;
|
|
929
|
+
attempt.cancelTimeout = scheduler.timeout(() => {
|
|
930
|
+
if (stopped || running !== attempt) return;
|
|
931
|
+
lastReadSucceeded = false;
|
|
932
|
+
recordStatus();
|
|
933
|
+
controller.abort();
|
|
934
|
+
// Keep the attempt owned until its native call settles. A misbehaving
|
|
935
|
+
// reader ignoring cancellation must not cause overlapping RPCs.
|
|
936
|
+
}, readTimeoutMs);
|
|
937
|
+
let read: Promise<TurnTaskQueueStats>;
|
|
938
|
+
try {
|
|
939
|
+
read = input.read({ signal: controller.signal, deadline });
|
|
940
|
+
} catch (error) {
|
|
941
|
+
read = Promise.reject(error);
|
|
942
|
+
}
|
|
943
|
+
attempt.done = read
|
|
786
944
|
.then((stats) => {
|
|
787
|
-
|
|
788
|
-
|
|
945
|
+
if (stopped || controller.signal.aborted || running !== attempt) return;
|
|
946
|
+
const completedAt = readClock();
|
|
947
|
+
// A native RPC may settle before its deadline but its Promise callback
|
|
948
|
+
// can run after it, ahead of a delayed JS timeout. Never restamp that old
|
|
949
|
+
// response as fresh. Monotonic age also fences wall-clock adjustments.
|
|
950
|
+
if (
|
|
951
|
+
completedAt === null ||
|
|
952
|
+
completedAt.wall >= deadline ||
|
|
953
|
+
completedAt.monotonic - observation.monotonic >= readTimeoutMs
|
|
954
|
+
) {
|
|
955
|
+
lastReadSucceeded = false;
|
|
956
|
+
recordStatus(completedAt);
|
|
957
|
+
controller.abort();
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
recordTurnTaskQueueStats(input.observability, stats, input.identity);
|
|
961
|
+
lastSuccessAt = completedAt.wall;
|
|
789
962
|
lastReadSucceeded = true;
|
|
790
|
-
recordStatus();
|
|
963
|
+
recordStatus(completedAt);
|
|
791
964
|
})
|
|
792
|
-
.catch((
|
|
965
|
+
.catch(() => {
|
|
966
|
+
if (stopped || controller.signal.aborted || running !== attempt) return;
|
|
793
967
|
lastReadSucceeded = false;
|
|
794
968
|
recordStatus();
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
969
|
+
safely(() =>
|
|
970
|
+
input.observability.warn("turn capacity monitor: Temporal task-queue stats failed", {
|
|
971
|
+
// Do not export arbitrary server errors or credentials.
|
|
972
|
+
errorClass: "WorkerTelemetryReadError",
|
|
973
|
+
origin: "turn-capacity-monitor",
|
|
974
|
+
}),
|
|
975
|
+
);
|
|
798
976
|
})
|
|
799
977
|
.finally(() => {
|
|
800
|
-
|
|
978
|
+
attempt.cancelTimeout();
|
|
979
|
+
if (running === attempt) running = null;
|
|
801
980
|
});
|
|
802
981
|
};
|
|
803
982
|
refresh();
|
|
804
|
-
const
|
|
805
|
-
timer.unref?.();
|
|
983
|
+
const stopTimer = scheduler.interval(refresh, intervalMs);
|
|
806
984
|
return {
|
|
807
|
-
close:
|
|
985
|
+
close: () => {
|
|
986
|
+
if (closePromise) return closePromise;
|
|
808
987
|
stopped = true;
|
|
809
|
-
|
|
810
|
-
|
|
988
|
+
stopTimer();
|
|
989
|
+
lastReadSucceeded = false;
|
|
990
|
+
recordStatus();
|
|
991
|
+
const attempt = running;
|
|
992
|
+
attempt?.cancelTimeout();
|
|
993
|
+
attempt?.controller.abort();
|
|
994
|
+
closePromise = attempt
|
|
995
|
+
? new Promise<void>((resolve) => {
|
|
996
|
+
const cancel = scheduler.timeout(resolve, closeTimeoutMs);
|
|
997
|
+
void attempt.done.then(() => {
|
|
998
|
+
cancel();
|
|
999
|
+
resolve();
|
|
1000
|
+
});
|
|
1001
|
+
})
|
|
1002
|
+
: Promise.resolve();
|
|
1003
|
+
return closePromise;
|
|
811
1004
|
},
|
|
812
1005
|
};
|
|
813
1006
|
}
|
|
814
1007
|
|
|
815
1008
|
export function recordSessionRecoveryBacklogGauges(
|
|
816
1009
|
observability: Observability,
|
|
817
|
-
|
|
1010
|
+
backlog: SessionRecoveryBacklog,
|
|
818
1011
|
): void {
|
|
819
1012
|
for (const state of ["quiescence_missing", "projection_stale"] as const) {
|
|
1013
|
+
const summary = backlog[state];
|
|
820
1014
|
observability.setGauge({
|
|
821
1015
|
name: "opengeni_session_recovery_backlog",
|
|
822
|
-
help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state.",
|
|
1016
|
+
help: "Current durable recovering-session obligations with no active attempt, by bounded reconciliation state. Includes sessions legitimately sleeping in their recorded recovery backoff.",
|
|
1017
|
+
labels: { state },
|
|
1018
|
+
value: nonnegativeFinite(summary.count),
|
|
1019
|
+
});
|
|
1020
|
+
observability.setGauge({
|
|
1021
|
+
name: "opengeni_session_recovery_scheduled",
|
|
1022
|
+
help: "Durable recovering sessions still inside the recovery backoff (provider Retry-After or connectivity pacing) their recovery request recorded, by bounded reconciliation state.",
|
|
823
1023
|
labels: { state },
|
|
824
|
-
value: nonnegativeFinite(
|
|
1024
|
+
value: nonnegativeFinite(summary.scheduled),
|
|
1025
|
+
});
|
|
1026
|
+
observability.setGauge({
|
|
1027
|
+
name: "opengeni_session_recovery_oldest_overdue_seconds",
|
|
1028
|
+
help: "Seconds the single oldest durable recovering session with no active attempt has been past its recovery due time, by bounded reconciliation state; 0 when none is overdue.",
|
|
1029
|
+
labels: { state },
|
|
1030
|
+
value: nonnegativeFinite(summary.oldestOverdueSeconds),
|
|
825
1031
|
});
|
|
826
1032
|
}
|
|
827
1033
|
}
|
|
@@ -865,8 +1071,8 @@ export function startSessionRecoveryMonitor(input: {
|
|
|
865
1071
|
if (stopped || running) return;
|
|
866
1072
|
running = input
|
|
867
1073
|
.read()
|
|
868
|
-
.then((
|
|
869
|
-
recordSessionRecoveryBacklogGauges(input.observability,
|
|
1074
|
+
.then((backlog) => {
|
|
1075
|
+
recordSessionRecoveryBacklogGauges(input.observability, backlog);
|
|
870
1076
|
lastSuccessAt = now();
|
|
871
1077
|
lastReadSucceeded = true;
|
|
872
1078
|
recordStatus();
|
|
@@ -1033,10 +1239,14 @@ export function recordOpenSandboxKubernetesInventoryGauges(
|
|
|
1033
1239
|
export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
|
|
1034
1240
|
"leases",
|
|
1035
1241
|
"checkpoint_artifacts",
|
|
1242
|
+
"recovery_observations",
|
|
1036
1243
|
"rotation_backlog",
|
|
1037
1244
|
"retained_processes",
|
|
1038
1245
|
"expired_drains",
|
|
1039
1246
|
"opensandbox_kubernetes",
|
|
1247
|
+
"modal_provider",
|
|
1248
|
+
"interaction_idle",
|
|
1249
|
+
"checkpoint_staleness",
|
|
1040
1250
|
] as const;
|
|
1041
1251
|
|
|
1042
1252
|
export type SandboxInventoryProjectionDomain =
|
|
@@ -1094,6 +1304,151 @@ export function recordCreditBalanceGauges(
|
|
|
1094
1304
|
creditBalanceGaugeAccounts.set(observability, current);
|
|
1095
1305
|
}
|
|
1096
1306
|
|
|
1307
|
+
/**
|
|
1308
|
+
* Distinct managed people with authenticated browser activity in each fixed
|
|
1309
|
+
* window (migration 0565 presence). Every control worker publishes the same
|
|
1310
|
+
* global value, so dashboards must aggregate with `max()` across pods, never
|
|
1311
|
+
* `sum()`. API keys, services and embedded hosts are never counted.
|
|
1312
|
+
*/
|
|
1313
|
+
export function recordActiveUserGauges(
|
|
1314
|
+
observability: Observability,
|
|
1315
|
+
counts: Record<ActiveUserWindow, number>,
|
|
1316
|
+
): void {
|
|
1317
|
+
for (const window of ACTIVE_USER_WINDOWS) {
|
|
1318
|
+
observability.setGauge({
|
|
1319
|
+
name: "opengeni_active_users",
|
|
1320
|
+
help: "Distinct managed users with authenticated activity within the window (global; use max across pods).",
|
|
1321
|
+
labels: { window },
|
|
1322
|
+
value: counts[window],
|
|
1323
|
+
});
|
|
1324
|
+
}
|
|
1325
|
+
}
|
|
1326
|
+
|
|
1327
|
+
/**
|
|
1328
|
+
* Positive credit grants observed by the ledger trigger since migration 0565,
|
|
1329
|
+
* by closed class. These are cumulative database totals published as gauges by
|
|
1330
|
+
* every control worker: aggregate with `max()` across pods, then subtract the
|
|
1331
|
+
* same expression `offset` by the window. A per-pod `increase()` sees only one
|
|
1332
|
+
* pod's lifetime and undercounts across control-worker restarts.
|
|
1333
|
+
* Counting in the database covers the grants no application process writes:
|
|
1334
|
+
* the verified-signup trial grant (a setup trigger) and operator grants.
|
|
1335
|
+
*/
|
|
1336
|
+
export function recordCreditGrantGauges(
|
|
1337
|
+
observability: Observability,
|
|
1338
|
+
totals: CreditGrantTotals,
|
|
1339
|
+
): void {
|
|
1340
|
+
for (const grantClass of CREDIT_GRANT_CLASSES) {
|
|
1341
|
+
const total = totals[grantClass];
|
|
1342
|
+
observability.setGauge({
|
|
1343
|
+
name: "opengeni_credit_grants_total",
|
|
1344
|
+
help: "Positive credit grants observed in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
|
|
1345
|
+
labels: { grant_class: grantClass },
|
|
1346
|
+
value: total.count,
|
|
1347
|
+
});
|
|
1348
|
+
observability.setGauge({
|
|
1349
|
+
name: "opengeni_credit_granted_micros_total",
|
|
1350
|
+
help: "Credit micros granted in the ledger since migration 0565, by grant class (global cumulative; use max across pods).",
|
|
1351
|
+
labels: { grant_class: grantClass },
|
|
1352
|
+
value: total.micros,
|
|
1353
|
+
});
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
/**
|
|
1358
|
+
* The deployment-level runtime switch for the one-time verified signup trial
|
|
1359
|
+
* credit (migration 0521): 1 while it allows grants, 0 when an operator has
|
|
1360
|
+
* disabled it or no revision exists. A grant also needs the API's
|
|
1361
|
+
* OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in, which
|
|
1362
|
+
* {@link recordVerifiedSignupTrialDeploymentFlagGauge} reports separately.
|
|
1363
|
+
*/
|
|
1364
|
+
export function recordVerifiedSignupTrialSwitchGauge(
|
|
1365
|
+
observability: Observability,
|
|
1366
|
+
grantsEnabled: boolean,
|
|
1367
|
+
): void {
|
|
1368
|
+
observability.setGauge({
|
|
1369
|
+
name: "opengeni_verified_signup_trial_credits_runtime_enabled",
|
|
1370
|
+
help: "Whether the runtime switch allows new verified signup trial credit grants (1) or blocks them (0).",
|
|
1371
|
+
value: grantsEnabled ? 1 : 0,
|
|
1372
|
+
});
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
/**
|
|
1376
|
+
* The deployment-level runtime switch for new managed account sign-ups
|
|
1377
|
+
* (migration 0585): 1 while it allows new accounts (also when no revision
|
|
1378
|
+
* exists), 0 when an operator has paused them. The API additionally honours
|
|
1379
|
+
* its OPENGENI_MANAGED_AUTH_NEW_SIGNUPS_ENABLED ceiling, which this gauge does
|
|
1380
|
+
* not see; `GET /v1/config/client` reports the combined decision.
|
|
1381
|
+
*/
|
|
1382
|
+
export function recordManagedAuthNewSignupsSwitchGauge(
|
|
1383
|
+
observability: Observability,
|
|
1384
|
+
signupsEnabled: boolean,
|
|
1385
|
+
): void {
|
|
1386
|
+
observability.setGauge({
|
|
1387
|
+
name: "opengeni_managed_auth_new_signups_runtime_enabled",
|
|
1388
|
+
help: "Whether the runtime switch allows new managed account sign-ups (1) or has paused them (0).",
|
|
1389
|
+
value: signupsEnabled ? 1 : 0,
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
/**
|
|
1394
|
+
* The OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in as this
|
|
1395
|
+
* worker's configuration sees it. The API reads the same shared setting; new
|
|
1396
|
+
* grants happen only while this gauge and the runtime switch gauge are both 1.
|
|
1397
|
+
*/
|
|
1398
|
+
export function recordVerifiedSignupTrialDeploymentFlagGauge(
|
|
1399
|
+
observability: Observability,
|
|
1400
|
+
enabled: boolean,
|
|
1401
|
+
): void {
|
|
1402
|
+
observability.setGauge({
|
|
1403
|
+
name: "opengeni_verified_signup_trial_credits_deployment_enabled",
|
|
1404
|
+
help: "Whether the OPENGENI_VERIFIED_SIGNUP_TRIAL_CREDITS_ENABLED master opt-in is on (1) or off (0) in this deployment's configuration.",
|
|
1405
|
+
value: enabled ? 1 : 0,
|
|
1406
|
+
});
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Provider-side Modal inventory reconciled against live leases by the orphan
|
|
1411
|
+
* sweep: `running` = every running box in the app, `unleased` = running boxes
|
|
1412
|
+
* no live lease protects (orphans, counted before termination), and
|
|
1413
|
+
* `lease_missing_instance` = warm leases whose exact provider instance is no
|
|
1414
|
+
* longer running (zombies).
|
|
1415
|
+
*/
|
|
1416
|
+
export function recordModalSandboxInventoryGauges(
|
|
1417
|
+
observability: Observability,
|
|
1418
|
+
inventory: { running: number; unleased: number; liveLeaseInstancesMissing: number },
|
|
1419
|
+
): void {
|
|
1420
|
+
const states = {
|
|
1421
|
+
running: inventory.running,
|
|
1422
|
+
unleased: inventory.unleased,
|
|
1423
|
+
lease_missing_instance: inventory.liveLeaseInstancesMissing,
|
|
1424
|
+
};
|
|
1425
|
+
for (const [state, value] of Object.entries(states)) {
|
|
1426
|
+
observability.setGauge({
|
|
1427
|
+
name: "opengeni_modal_sandbox_inventory",
|
|
1428
|
+
help: "Running Modal sandboxes reconciled against live sandbox leases by the orphan sweep.",
|
|
1429
|
+
labels: { state },
|
|
1430
|
+
value: Math.max(0, value),
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1434
|
+
|
|
1435
|
+
/** Warm leases held only by Browser/Computer interaction holders, by time since
|
|
1436
|
+
* the newest interaction activity. Such boxes never drain on their own until the
|
|
1437
|
+
* interaction session ends or the provider deadline. */
|
|
1438
|
+
export function recordInteractionOnlyLeaseGauges(
|
|
1439
|
+
observability: Observability,
|
|
1440
|
+
counts: Record<string, number>,
|
|
1441
|
+
): void {
|
|
1442
|
+
for (const [idleBucket, value] of Object.entries(counts)) {
|
|
1443
|
+
observability.setGauge({
|
|
1444
|
+
name: "opengeni_sandbox_leases_interaction_only",
|
|
1445
|
+
help: "Warm sandbox leases held only by Browser/Computer sessions, by time since last interaction activity.",
|
|
1446
|
+
labels: { idle_bucket: idleBucket },
|
|
1447
|
+
value: Math.max(0, value),
|
|
1448
|
+
});
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1097
1452
|
export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
|
|
1098
1453
|
if (count <= 0) {
|
|
1099
1454
|
return;
|
|
@@ -1169,6 +1524,349 @@ export function recordSandboxDeadlineRotationsRequested(
|
|
|
1169
1524
|
});
|
|
1170
1525
|
}
|
|
1171
1526
|
|
|
1527
|
+
/** Fixed outcomes of legacy retained-command containment: per-candidate
|
|
1528
|
+
* enrollment inspection, then the enrolled drain's exact cold commit. */
|
|
1529
|
+
export const SANDBOX_COMMAND_CONTAINMENT_OUTCOMES = [
|
|
1530
|
+
"idle_enrolled",
|
|
1531
|
+
"deadline_enrolled",
|
|
1532
|
+
"forced_deadline_enrolled",
|
|
1533
|
+
"quiescence_enrolled",
|
|
1534
|
+
"resumed_enrolled",
|
|
1535
|
+
"not_eligible",
|
|
1536
|
+
"inspection_failed",
|
|
1537
|
+
"contained",
|
|
1538
|
+
"provider_missing",
|
|
1539
|
+
] as const;
|
|
1540
|
+
|
|
1541
|
+
export type SandboxCommandContainmentOutcome =
|
|
1542
|
+
(typeof SANDBOX_COMMAND_CONTAINMENT_OUTCOMES)[number];
|
|
1543
|
+
|
|
1544
|
+
/**
|
|
1545
|
+
* Shared subscription core shadow (docs/subscription-accounts.md, SUB-COMPAT-03).
|
|
1546
|
+
* Every label is a closed set: provider, security parity, placement agreement,
|
|
1547
|
+
* contract requirement, skip reason and the legacy decision inputs present.
|
|
1548
|
+
*/
|
|
1549
|
+
export const SUBSCRIPTION_CORE_SHADOW_PROVIDERS = ["codex", "xai", "claude"] as const;
|
|
1550
|
+
/** Is the account the legacy selector chose inside the core's eligible set? */
|
|
1551
|
+
export const SUBSCRIPTION_CORE_SHADOW_PARITY = [
|
|
1552
|
+
"eligible",
|
|
1553
|
+
"not_authorized",
|
|
1554
|
+
"not_servable",
|
|
1555
|
+
"unknown_connection",
|
|
1556
|
+
"no_selection",
|
|
1557
|
+
] as const;
|
|
1558
|
+
/** How the core's decision compares with the legacy decision (would-switch). */
|
|
1559
|
+
export const SUBSCRIPTION_CORE_SHADOW_PLACEMENTS = [
|
|
1560
|
+
"same_account",
|
|
1561
|
+
"different_account",
|
|
1562
|
+
"core_waits",
|
|
1563
|
+
"core_runs_legacy_waits",
|
|
1564
|
+
"both_wait",
|
|
1565
|
+
] as const;
|
|
1566
|
+
/** Requirements the reference checker can report, plus checker failure. */
|
|
1567
|
+
export const SUBSCRIPTION_CORE_SHADOW_REQUIREMENTS = [
|
|
1568
|
+
"SUB-ACCESS-06",
|
|
1569
|
+
"SUB-CREDITS-01",
|
|
1570
|
+
"SUB-ELIG-01",
|
|
1571
|
+
"SUB-ELIG-02",
|
|
1572
|
+
"SUB-ELIG-03",
|
|
1573
|
+
"SUB-ELIG-04",
|
|
1574
|
+
"SUB-ELIG-05",
|
|
1575
|
+
"SUB-FAIL-02",
|
|
1576
|
+
"SUB-FAIL-03",
|
|
1577
|
+
"SUB-FAIL-04",
|
|
1578
|
+
"SUB-OWN-05",
|
|
1579
|
+
"SUB-SEL-02",
|
|
1580
|
+
"SUB-SEL-04",
|
|
1581
|
+
"SUB-SET-06",
|
|
1582
|
+
"SUB-STICK-02",
|
|
1583
|
+
"SUB-WAIT-01",
|
|
1584
|
+
"SUB-WAIT-02",
|
|
1585
|
+
"other",
|
|
1586
|
+
"checker_error",
|
|
1587
|
+
] as const;
|
|
1588
|
+
export const SUBSCRIPTION_CORE_SHADOW_SKIPS = [
|
|
1589
|
+
"busy",
|
|
1590
|
+
"no_session_actor",
|
|
1591
|
+
"session_not_visible",
|
|
1592
|
+
"timeout",
|
|
1593
|
+
"cancelled",
|
|
1594
|
+
"error",
|
|
1595
|
+
] as const;
|
|
1596
|
+
/**
|
|
1597
|
+
* Legacy decision inputs the Codex fleet shadow does not record (inventory
|
|
1598
|
+
* Part 3, answer (c)), counted when present in a compared decision.
|
|
1599
|
+
*/
|
|
1600
|
+
export const SUBSCRIPTION_CORE_SHADOW_INPUTS = [
|
|
1601
|
+
"manual_pin",
|
|
1602
|
+
"policy_pin",
|
|
1603
|
+
"last_account",
|
|
1604
|
+
"rotation_off",
|
|
1605
|
+
"rotation_on",
|
|
1606
|
+
"organization_pool",
|
|
1607
|
+
"personal_pool",
|
|
1608
|
+
"codex_mode_override",
|
|
1609
|
+
"model_policy",
|
|
1610
|
+
"model_filtered",
|
|
1611
|
+
"plan_excluded",
|
|
1612
|
+
"model_cooldown",
|
|
1613
|
+
"exhausted",
|
|
1614
|
+
"unknown_quota",
|
|
1615
|
+
"compaction_lock",
|
|
1616
|
+
"lease_reused",
|
|
1617
|
+
"truncated",
|
|
1618
|
+
] as const;
|
|
1619
|
+
|
|
1620
|
+
export type SubscriptionCoreShadowProvider = (typeof SUBSCRIPTION_CORE_SHADOW_PROVIDERS)[number];
|
|
1621
|
+
export type SubscriptionCoreShadowParity = (typeof SUBSCRIPTION_CORE_SHADOW_PARITY)[number];
|
|
1622
|
+
export type SubscriptionCoreShadowPlacement = (typeof SUBSCRIPTION_CORE_SHADOW_PLACEMENTS)[number];
|
|
1623
|
+
export type SubscriptionCoreShadowSkip = (typeof SUBSCRIPTION_CORE_SHADOW_SKIPS)[number];
|
|
1624
|
+
export type SubscriptionCoreShadowInput = (typeof SUBSCRIPTION_CORE_SHADOW_INPUTS)[number];
|
|
1625
|
+
|
|
1626
|
+
export type SubscriptionCoreShadowObservation =
|
|
1627
|
+
| { outcome: "skipped"; reason: SubscriptionCoreShadowSkip }
|
|
1628
|
+
| {
|
|
1629
|
+
outcome: "compared";
|
|
1630
|
+
parity: SubscriptionCoreShadowParity;
|
|
1631
|
+
/** The first reason the legacy account is outside the eligible set. */
|
|
1632
|
+
parityReason: string | null;
|
|
1633
|
+
placement: SubscriptionCoreShadowPlacement;
|
|
1634
|
+
violations: readonly string[];
|
|
1635
|
+
inputs: readonly SubscriptionCoreShadowInput[];
|
|
1636
|
+
};
|
|
1637
|
+
|
|
1638
|
+
const SUBSCRIPTION_CORE_SHADOW_REQUIREMENT_SET = new Set<string>(
|
|
1639
|
+
SUBSCRIPTION_CORE_SHADOW_REQUIREMENTS,
|
|
1640
|
+
);
|
|
1641
|
+
/** First reasons a legacy account is outside the eligible set (policy ineligibility). */
|
|
1642
|
+
export const SUBSCRIPTION_CORE_SHADOW_PARITY_REASONS = [
|
|
1643
|
+
"provider_disabled",
|
|
1644
|
+
"organization_accounts_off",
|
|
1645
|
+
"out_of_scope",
|
|
1646
|
+
"personal_connections_disabled",
|
|
1647
|
+
"personal_owner_inactive",
|
|
1648
|
+
"personal_not_owners_work",
|
|
1649
|
+
"personal_authority_missing",
|
|
1650
|
+
"unhealthy",
|
|
1651
|
+
"allocator_disabled",
|
|
1652
|
+
"wrong_provider",
|
|
1653
|
+
"model_unknown",
|
|
1654
|
+
"model_not_allowed_by_workspace",
|
|
1655
|
+
"model_not_entitled",
|
|
1656
|
+
"model_not_allowed_by_connection",
|
|
1657
|
+
"model_cooling_down",
|
|
1658
|
+
"exhausted",
|
|
1659
|
+
] as const;
|
|
1660
|
+
const SUBSCRIPTION_CORE_SHADOW_PARITY_REASON_SET = new Set<string>(
|
|
1661
|
+
SUBSCRIPTION_CORE_SHADOW_PARITY_REASONS,
|
|
1662
|
+
);
|
|
1663
|
+
|
|
1664
|
+
const SUBSCRIPTION_CORE_SHADOW_DURATION_BUCKETS = [
|
|
1665
|
+
0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1,
|
|
1666
|
+
];
|
|
1667
|
+
|
|
1668
|
+
export function recordSubscriptionCoreShadow(
|
|
1669
|
+
observability: Pick<Observability, "incrementCounter" | "observeHistogram">,
|
|
1670
|
+
provider: SubscriptionCoreShadowProvider,
|
|
1671
|
+
observation: SubscriptionCoreShadowObservation,
|
|
1672
|
+
durationSeconds: number,
|
|
1673
|
+
): void {
|
|
1674
|
+
observability.observeHistogram({
|
|
1675
|
+
name: "opengeni_subscription_core_shadow_duration_seconds",
|
|
1676
|
+
help: "Shared subscription core shadow comparison duration by provider, including skips.",
|
|
1677
|
+
labels: { provider },
|
|
1678
|
+
value: durationSeconds,
|
|
1679
|
+
buckets: SUBSCRIPTION_CORE_SHADOW_DURATION_BUCKETS,
|
|
1680
|
+
});
|
|
1681
|
+
if (observation.outcome === "skipped") {
|
|
1682
|
+
observability.incrementCounter({
|
|
1683
|
+
name: "opengeni_subscription_core_shadow_skips_total",
|
|
1684
|
+
help: "Shadow comparisons not made, by fixed reason; placement is unaffected.",
|
|
1685
|
+
labels: { provider, reason: observation.reason },
|
|
1686
|
+
});
|
|
1687
|
+
return;
|
|
1688
|
+
}
|
|
1689
|
+
observability.incrementCounter({
|
|
1690
|
+
name: "opengeni_subscription_core_shadow_comparisons_total",
|
|
1691
|
+
help: "Shadow comparisons by security parity of the legacy account and core placement agreement.",
|
|
1692
|
+
labels: { provider, parity: observation.parity, placement: observation.placement },
|
|
1693
|
+
});
|
|
1694
|
+
if (observation.parityReason !== null) {
|
|
1695
|
+
observability.incrementCounter({
|
|
1696
|
+
name: "opengeni_subscription_core_shadow_parity_failures_total",
|
|
1697
|
+
help: "Legacy-selected accounts outside the core's eligible set, by first fixed reason.",
|
|
1698
|
+
labels: {
|
|
1699
|
+
provider,
|
|
1700
|
+
reason: SUBSCRIPTION_CORE_SHADOW_PARITY_REASON_SET.has(observation.parityReason)
|
|
1701
|
+
? observation.parityReason
|
|
1702
|
+
: "other",
|
|
1703
|
+
},
|
|
1704
|
+
});
|
|
1705
|
+
}
|
|
1706
|
+
for (const requirement of new Set(observation.violations)) {
|
|
1707
|
+
observability.incrementCounter({
|
|
1708
|
+
name: "opengeni_subscription_core_shadow_violations_total",
|
|
1709
|
+
help: "Contract invariant violations of the core's own shadow decision, by requirement.",
|
|
1710
|
+
labels: {
|
|
1711
|
+
provider,
|
|
1712
|
+
requirement: SUBSCRIPTION_CORE_SHADOW_REQUIREMENT_SET.has(requirement)
|
|
1713
|
+
? requirement
|
|
1714
|
+
: "other",
|
|
1715
|
+
},
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
for (const input of observation.inputs) {
|
|
1719
|
+
observability.incrementCounter({
|
|
1720
|
+
name: "opengeni_subscription_core_shadow_inputs_total",
|
|
1721
|
+
help: "Legacy decision inputs present in compared shadow decisions, by fixed input.",
|
|
1722
|
+
labels: { provider, input },
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
/** A shadow load that outlived its slot ceiling; its slot was given back. */
|
|
1728
|
+
export function recordSubscriptionCoreShadowStuckLoad(
|
|
1729
|
+
observability: Pick<Observability, "incrementCounter">,
|
|
1730
|
+
provider: SubscriptionCoreShadowProvider,
|
|
1731
|
+
): void {
|
|
1732
|
+
observability.incrementCounter({
|
|
1733
|
+
name: "opengeni_subscription_core_shadow_stuck_loads_total",
|
|
1734
|
+
help: "Shadow world loads still unsettled at the slot ceiling; the slot was released.",
|
|
1735
|
+
labels: { provider },
|
|
1736
|
+
});
|
|
1737
|
+
}
|
|
1738
|
+
|
|
1739
|
+
/** Idle browser/desktop release inspections by fixed outcome: `checkpointing`
|
|
1740
|
+
* (idle browser saves prepared), `released` (holders released without a save). */
|
|
1741
|
+
export function recordSandboxIdleInteractionRelease(
|
|
1742
|
+
observability: Observability,
|
|
1743
|
+
outcome: IdleInteractionInspection,
|
|
1744
|
+
): void {
|
|
1745
|
+
observability.incrementCounter({
|
|
1746
|
+
name: "opengeni_sandbox_idle_interaction_release_total",
|
|
1747
|
+
help: "Idle browser and desktop release inspections on warm boxes, by fixed outcome.",
|
|
1748
|
+
labels: { outcome },
|
|
1749
|
+
});
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
export function recordSandboxCommandContainment(
|
|
1753
|
+
observability: Observability,
|
|
1754
|
+
outcome: SandboxCommandContainmentOutcome,
|
|
1755
|
+
): void {
|
|
1756
|
+
observability.incrementCounter({
|
|
1757
|
+
name: "opengeni_sandbox_command_containment_total",
|
|
1758
|
+
help: "Legacy retained-command containment inspections and enrolled drain commits by fixed outcome.",
|
|
1759
|
+
labels: { outcome },
|
|
1760
|
+
});
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/** Only call after the exact draining->cold commit reports wentCold. The
|
|
1764
|
+
* backend is validated against the closed contract so provider IDs and other
|
|
1765
|
+
* per-sandbox values can never become metric labels. */
|
|
1766
|
+
export function recordSandboxProviderMissingBeforeCapture(
|
|
1767
|
+
observability: Observability,
|
|
1768
|
+
backend: string,
|
|
1769
|
+
): void {
|
|
1770
|
+
const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
|
|
1771
|
+
observability.incrementCounter({
|
|
1772
|
+
name: "opengeni_sandbox_provider_missing_before_capture_total",
|
|
1773
|
+
help: "Exact sandbox cold commits after definitive provider disappearance before workspace capture.",
|
|
1774
|
+
labels: { backend: safeBackend },
|
|
1775
|
+
});
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1778
|
+
/** Host workspace directory outcome after a Docker drain's exact-container
|
|
1779
|
+
* teardown (`retained` means the directory stays on disk). */
|
|
1780
|
+
export function recordSandboxDockerWorkspaceRelease(
|
|
1781
|
+
observability: Observability,
|
|
1782
|
+
status: "released" | "already_released" | "retained",
|
|
1783
|
+
): void {
|
|
1784
|
+
observability.incrementCounter({
|
|
1785
|
+
name: "opengeni_sandbox_docker_workspace_release_total",
|
|
1786
|
+
help: "Docker drain host workspace directories released or retained after exact-container teardown.",
|
|
1787
|
+
labels: { status },
|
|
1788
|
+
});
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
export function recordSandboxRecoveryObservationGauges(
|
|
1792
|
+
observability: Observability,
|
|
1793
|
+
observations: {
|
|
1794
|
+
providerLosses: number;
|
|
1795
|
+
fallbackSelections: number;
|
|
1796
|
+
freshWorkspaceSelections?: number;
|
|
1797
|
+
},
|
|
1798
|
+
): void {
|
|
1799
|
+
for (const [kind, value] of [
|
|
1800
|
+
["provider_missing_before_capture", observations.providerLosses],
|
|
1801
|
+
["checkpoint_fallback_selected", observations.fallbackSelections],
|
|
1802
|
+
["fresh_workspace_selected", observations.freshWorkspaceSelections ?? 0],
|
|
1803
|
+
] as const) {
|
|
1804
|
+
observability.setGauge({
|
|
1805
|
+
name: "opengeni_sandbox_recovery_observations_recent",
|
|
1806
|
+
help: "Committed sandbox recovery observations in the last 30 minutes by fixed kind.",
|
|
1807
|
+
labels: { kind },
|
|
1808
|
+
value,
|
|
1809
|
+
});
|
|
1810
|
+
}
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
/** Fixed outcomes for a committed automatic continuity decision after
|
|
1814
|
+
* definitive managed-provider loss: a singleton checkpoint (`selected`), a
|
|
1815
|
+
* shared-group checkpoint (`selected_shared`), or a new empty workspace
|
|
1816
|
+
* (`fresh_workspace`). Call only after the durable authorization committed. */
|
|
1817
|
+
export const SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES = [
|
|
1818
|
+
"selected",
|
|
1819
|
+
"selected_shared",
|
|
1820
|
+
"fresh_workspace",
|
|
1821
|
+
] as const;
|
|
1822
|
+
export type SandboxAutomaticRecoveryOutcome = (typeof SANDBOX_AUTOMATIC_RECOVERY_OUTCOMES)[number];
|
|
1823
|
+
|
|
1824
|
+
export function sandboxAutomaticRecoveryOutcome(input: {
|
|
1825
|
+
lane: "checkpoint" | "fresh_workspace";
|
|
1826
|
+
groupSessionCount: number;
|
|
1827
|
+
}): SandboxAutomaticRecoveryOutcome {
|
|
1828
|
+
if (input.lane === "fresh_workspace") return "fresh_workspace";
|
|
1829
|
+
return input.groupSessionCount > 1 ? "selected_shared" : "selected";
|
|
1830
|
+
}
|
|
1831
|
+
|
|
1832
|
+
export function recordSandboxAutomaticRecoverySelected(
|
|
1833
|
+
observability: Observability,
|
|
1834
|
+
backend: string,
|
|
1835
|
+
outcome: SandboxAutomaticRecoveryOutcome,
|
|
1836
|
+
): void {
|
|
1837
|
+
const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
|
|
1838
|
+
observability.incrementCounter({
|
|
1839
|
+
name: "opengeni_sandbox_checkpoint_fallback_total",
|
|
1840
|
+
help: "System-selected continuity after managed provider loss: verified checkpoint or empty workspace.",
|
|
1841
|
+
labels: { backend: safeBackend, outcome },
|
|
1842
|
+
});
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
/** How long live boxes have held changes no checkpoint covers yet. */
|
|
1846
|
+
export function recordSandboxCheckpointStalenessGauges(
|
|
1847
|
+
observability: Observability,
|
|
1848
|
+
staleness: { dirty: number; stale4h: number; stale12h: number; maxAgeSeconds: number },
|
|
1849
|
+
): void {
|
|
1850
|
+
const values = {
|
|
1851
|
+
dirty: staleness.dirty,
|
|
1852
|
+
stale_4h: staleness.stale4h,
|
|
1853
|
+
stale_12h: staleness.stale12h,
|
|
1854
|
+
} as const;
|
|
1855
|
+
for (const [kind, value] of Object.entries(values)) {
|
|
1856
|
+
observability.setGauge({
|
|
1857
|
+
name: "opengeni_sandbox_checkpoint_staleness",
|
|
1858
|
+
help: "Live Modal sandboxes holding a write their last checkpoint did not capture, by age of the first such write.",
|
|
1859
|
+
labels: { kind },
|
|
1860
|
+
value,
|
|
1861
|
+
});
|
|
1862
|
+
}
|
|
1863
|
+
observability.setGauge({
|
|
1864
|
+
name: "opengeni_sandbox_checkpoint_age_max_seconds",
|
|
1865
|
+
help: "Age in seconds of the oldest write a live Modal sandbox's last checkpoint did not capture.",
|
|
1866
|
+
value: Number.isFinite(staleness.maxAgeSeconds) ? Math.max(0, staleness.maxAgeSeconds) : 0,
|
|
1867
|
+
});
|
|
1868
|
+
}
|
|
1869
|
+
|
|
1172
1870
|
export function recordSandboxRotationBacklogGauges(
|
|
1173
1871
|
observability: Observability,
|
|
1174
1872
|
backlog: {
|
|
@@ -1199,6 +1897,11 @@ export function recordSandboxRotationBacklogGauges(
|
|
|
1199
1897
|
}
|
|
1200
1898
|
|
|
1201
1899
|
const RETAINED_PROCESS_OWNER_STATES = [
|
|
1900
|
+
// Session-owned background commands (migration 0637). Running is live
|
|
1901
|
+
// session work after its launch turn ended; stopping already has a stop
|
|
1902
|
+
// request, so it still counts as terminal-owner backlog.
|
|
1903
|
+
"background_running",
|
|
1904
|
+
"background_stopping",
|
|
1202
1905
|
"direct",
|
|
1203
1906
|
"queued",
|
|
1204
1907
|
"running",
|
|
@@ -1246,7 +1949,7 @@ export function recordRetainedProcessInventoryGauges(
|
|
|
1246
1949
|
});
|
|
1247
1950
|
observability.setGauge({
|
|
1248
1951
|
name: "opengeni_retained_processes_terminal_owner_backlog",
|
|
1249
|
-
help: "Current active retained processes whose
|
|
1952
|
+
help: "Current active retained processes whose owner is gone: a closed launch attempt or released direct request, or a background command already asked to stop. A running session background command is never counted.",
|
|
1250
1953
|
labels: { owner_state: ownerState },
|
|
1251
1954
|
value: count.terminal,
|
|
1252
1955
|
});
|
|
@@ -1286,11 +1989,13 @@ export function recordExpiredDrainingSandboxLeaseGauges(
|
|
|
1286
1989
|
|
|
1287
1990
|
export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
|
|
1288
1991
|
"claim_failed",
|
|
1992
|
+
"background_recovered",
|
|
1289
1993
|
"proof_exited",
|
|
1290
1994
|
"proof_lost",
|
|
1291
1995
|
"proof_checkpoint_failed",
|
|
1292
1996
|
"settled_exited",
|
|
1293
1997
|
"settled_lost",
|
|
1998
|
+
"provider_lost_whole_box",
|
|
1294
1999
|
"settlement_failed",
|
|
1295
2000
|
"identity_mismatch",
|
|
1296
2001
|
"resume_state_missing",
|
|
@@ -1339,6 +2044,147 @@ export function recordCreditMicros(
|
|
|
1339
2044
|
});
|
|
1340
2045
|
}
|
|
1341
2046
|
|
|
2047
|
+
// ── Per-model usage and money ───────────────────────────────────────────────
|
|
2048
|
+
// `model` is the deployment catalog product id (resolved by the caller), or
|
|
2049
|
+
// `custom` for a workspace-owned model. The recorder bounds it again: a value
|
|
2050
|
+
// outside a conservative id shape becomes `custom`, and after
|
|
2051
|
+
// MAX_MODEL_METRIC_LABELS distinct values per process every new value becomes
|
|
2052
|
+
// `other`, so a catalog change or a bug can never grow the series count without
|
|
2053
|
+
// bound. Account, workspace, user and session ids are never labels here.
|
|
2054
|
+
|
|
2055
|
+
/** Who pays the upstream provider for the call. */
|
|
2056
|
+
export type ModelUsagePayer = "deployment" | "external";
|
|
2057
|
+
|
|
2058
|
+
const MODEL_METRIC_LABEL_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$/;
|
|
2059
|
+
const MAX_MODEL_METRIC_LABELS = 64;
|
|
2060
|
+
const modelMetricLabels = new WeakMap<Observability, Set<string>>();
|
|
2061
|
+
|
|
2062
|
+
export function boundedModelMetricLabel(observability: Observability, model: string): string {
|
|
2063
|
+
if (model === "custom" || model === "other") return model;
|
|
2064
|
+
if (!MODEL_METRIC_LABEL_PATTERN.test(model)) return "custom";
|
|
2065
|
+
let seen = modelMetricLabels.get(observability);
|
|
2066
|
+
if (!seen) {
|
|
2067
|
+
seen = new Set<string>();
|
|
2068
|
+
modelMetricLabels.set(observability, seen);
|
|
2069
|
+
}
|
|
2070
|
+
if (seen.has(model)) return model;
|
|
2071
|
+
if (seen.size >= MAX_MODEL_METRIC_LABELS) return "other";
|
|
2072
|
+
seen.add(model);
|
|
2073
|
+
return model;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
function nonnegativeSafeInteger(value: unknown): number {
|
|
2077
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : 0;
|
|
2078
|
+
}
|
|
2079
|
+
|
|
2080
|
+
/**
|
|
2081
|
+
* One authoritative model response, by provider and catalog model. Recorded
|
|
2082
|
+
* behind the same durable usage-event fence as the provider-only cache metrics,
|
|
2083
|
+
* so a duplicate or late response is not counted twice.
|
|
2084
|
+
* - `opengeni_model_responses_total{provider,model,priced}` — responses with
|
|
2085
|
+
* reported usage; `priced="false"` means no provider cost estimate exists.
|
|
2086
|
+
* - `opengeni_model_tokens_total{provider,model,type}` — `input` (all prompt
|
|
2087
|
+
* tokens, including cached and cache-write), `cached_input` and
|
|
2088
|
+
* `cache_write` (subsets of `input`), `output`, and `reasoning` (a subset
|
|
2089
|
+
* of `output`).
|
|
2090
|
+
* - `opengeni_model_provider_cost_micros_total{provider,model,payer,pricing_source}`
|
|
2091
|
+
* — estimated upstream cost in USD micros: configured list price, or the
|
|
2092
|
+
* gateway-reported cost. `payer="external"` is a customer-paid subscription
|
|
2093
|
+
* or key, so that cost is a list-price equivalent Opengeni does not pay.
|
|
2094
|
+
*/
|
|
2095
|
+
export function recordModelResponseUsage(
|
|
2096
|
+
observability: Observability,
|
|
2097
|
+
input: {
|
|
2098
|
+
provider: string;
|
|
2099
|
+
model: string;
|
|
2100
|
+
payer: ModelUsagePayer;
|
|
2101
|
+
tokens: {
|
|
2102
|
+
inputTokens: number | null;
|
|
2103
|
+
cachedTokens: number | null;
|
|
2104
|
+
cacheWriteTokens: number | null;
|
|
2105
|
+
outputTokens: number | null;
|
|
2106
|
+
reasoningTokens: number | null;
|
|
2107
|
+
};
|
|
2108
|
+
estimatedProviderCostMicros: number | null;
|
|
2109
|
+
pricingSource: "configured_list_price" | "gateway_reported" | null;
|
|
2110
|
+
webSearchRequests?: number;
|
|
2111
|
+
},
|
|
2112
|
+
): void {
|
|
2113
|
+
const model = boundedModelMetricLabel(observability, input.model);
|
|
2114
|
+
const base = { provider: input.provider, model };
|
|
2115
|
+
const priced = input.pricingSource !== null && input.estimatedProviderCostMicros !== null;
|
|
2116
|
+
observability.incrementCounter({
|
|
2117
|
+
name: "opengeni_model_responses_total",
|
|
2118
|
+
help: "Authoritative model responses with reported usage, by provider, catalog model and whether a cost estimate exists.",
|
|
2119
|
+
labels: { ...base, priced: priced ? "true" : "false" },
|
|
2120
|
+
});
|
|
2121
|
+
const tokenTypes = [
|
|
2122
|
+
["input", input.tokens.inputTokens],
|
|
2123
|
+
["cached_input", input.tokens.cachedTokens],
|
|
2124
|
+
["cache_write", input.tokens.cacheWriteTokens],
|
|
2125
|
+
["output", input.tokens.outputTokens],
|
|
2126
|
+
["reasoning", input.tokens.reasoningTokens],
|
|
2127
|
+
] as const;
|
|
2128
|
+
for (const [type, value] of tokenTypes) {
|
|
2129
|
+
const amount = modelUsageTokenCountOrNull(value);
|
|
2130
|
+
if (amount === null || amount === 0) continue;
|
|
2131
|
+
incrementBoundedModelCacheCounter(observability, {
|
|
2132
|
+
name: "opengeni_model_tokens_total",
|
|
2133
|
+
help: "Tokens of authoritative model responses by provider, catalog model and token type (cached_input and cache_write are subsets of input; reasoning is a subset of output).",
|
|
2134
|
+
provider: input.provider,
|
|
2135
|
+
labels: { ...base, type },
|
|
2136
|
+
amount,
|
|
2137
|
+
});
|
|
2138
|
+
}
|
|
2139
|
+
const searches = nonnegativeSafeInteger(input.webSearchRequests ?? 0);
|
|
2140
|
+
if (searches > 0)
|
|
2141
|
+
observability.incrementCounter({
|
|
2142
|
+
name: "opengeni_model_web_search_requests_total",
|
|
2143
|
+
help: "Provider-run web searches billed with authoritative model responses, by provider, catalog model and payer.",
|
|
2144
|
+
labels: { ...base, payer: input.payer },
|
|
2145
|
+
amount: searches,
|
|
2146
|
+
});
|
|
2147
|
+
const cost = nonnegativeSafeInteger(input.estimatedProviderCostMicros);
|
|
2148
|
+
if (priced && input.pricingSource !== null && cost > 0) {
|
|
2149
|
+
observability.incrementCounter({
|
|
2150
|
+
name: "opengeni_model_provider_cost_micros_total",
|
|
2151
|
+
help: "Estimated upstream provider cost in USD micros by provider, catalog model, payer and pricing source.",
|
|
2152
|
+
labels: { ...base, payer: input.payer, pricing_source: input.pricingSource },
|
|
2153
|
+
amount: cost,
|
|
2154
|
+
});
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
2157
|
+
|
|
2158
|
+
/**
|
|
2159
|
+
* Credits actually debited for one model response, recorded once when the
|
|
2160
|
+
* ledger row is inserted (an idempotent replay debits 0 and records nothing).
|
|
2161
|
+
* `funding="promotional"` is the part paid by scoped promotional grants
|
|
2162
|
+
* (verified-signup trial, scoped coupon offers); `funding="general"` is the
|
|
2163
|
+
* rest, paid from general credit: purchased credits plus any unscoped grant.
|
|
2164
|
+
*/
|
|
2165
|
+
export function recordModelCreditsCharged(
|
|
2166
|
+
observability: Observability | undefined,
|
|
2167
|
+
input: { provider: string; model: string; debitedMicros: number; grantDebitedMicros: number },
|
|
2168
|
+
): void {
|
|
2169
|
+
if (!observability) return;
|
|
2170
|
+
const debited = nonnegativeSafeInteger(input.debitedMicros);
|
|
2171
|
+
if (debited === 0) return;
|
|
2172
|
+
const promotional = Math.min(debited, nonnegativeSafeInteger(input.grantDebitedMicros));
|
|
2173
|
+
const model = boundedModelMetricLabel(observability, input.model);
|
|
2174
|
+
for (const [funding, amount] of [
|
|
2175
|
+
["promotional", promotional],
|
|
2176
|
+
["general", debited - promotional],
|
|
2177
|
+
] as const) {
|
|
2178
|
+
if (amount === 0) continue;
|
|
2179
|
+
observability.incrementCounter({
|
|
2180
|
+
name: "opengeni_model_credits_charged_micros_total",
|
|
2181
|
+
help: "Credit micros debited for model usage by provider, catalog model and funding (promotional grant or general credit).",
|
|
2182
|
+
labels: { provider: input.provider, model, funding },
|
|
2183
|
+
amount,
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
}
|
|
2187
|
+
|
|
1342
2188
|
const MODEL_REQUEST_PHASE_BUCKETS = [
|
|
1343
2189
|
0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5, 10, 30, 60, 300,
|
|
1344
2190
|
];
|
|
@@ -1373,10 +2219,18 @@ export function recordModelRequestPhase(
|
|
|
1373
2219
|
}
|
|
1374
2220
|
|
|
1375
2221
|
export type TurnStartupPhase =
|
|
2222
|
+
| "services_initialization"
|
|
2223
|
+
| "claim_catalog_read"
|
|
2224
|
+
| "claim_atomic"
|
|
2225
|
+
| "claim_session_read"
|
|
2226
|
+
| "claim_capability_settings"
|
|
2227
|
+
| "learning_policy_freeze"
|
|
2228
|
+
| "attachment_authority_projection"
|
|
1376
2229
|
| "claim_and_policy"
|
|
1377
2230
|
| "turn_start_settlement"
|
|
1378
2231
|
| "credential_selection"
|
|
1379
2232
|
| "runtime_preparation"
|
|
2233
|
+
| "session_mcp_settings"
|
|
1380
2234
|
| "sandbox_establish"
|
|
1381
2235
|
| "file_resolution"
|
|
1382
2236
|
| "tool_context_preparation"
|
|
@@ -1556,6 +2410,32 @@ export function recordTurnStartupPhase(
|
|
|
1556
2410
|
});
|
|
1557
2411
|
}
|
|
1558
2412
|
|
|
2413
|
+
/** Observe an existing dependency only. Child intervals can overlap and are
|
|
2414
|
+
* not additive; observer failure must never change authority or error identity. */
|
|
2415
|
+
export async function measureTurnStartupPhase<T>(
|
|
2416
|
+
observability: Observability,
|
|
2417
|
+
input: Omit<Parameters<typeof recordTurnStartupPhase>[1], "durationSeconds" | "outcome">,
|
|
2418
|
+
work: () => Promise<T>,
|
|
2419
|
+
): Promise<T> {
|
|
2420
|
+
const started = performance.now();
|
|
2421
|
+
let outcome: TurnStartupOutcome = "failed";
|
|
2422
|
+
try {
|
|
2423
|
+
const result = await work();
|
|
2424
|
+
outcome = "completed";
|
|
2425
|
+
return result;
|
|
2426
|
+
} finally {
|
|
2427
|
+
try {
|
|
2428
|
+
recordTurnStartupPhase(observability, {
|
|
2429
|
+
...input,
|
|
2430
|
+
outcome,
|
|
2431
|
+
durationSeconds: (performance.now() - started) / 1_000,
|
|
2432
|
+
});
|
|
2433
|
+
} catch {
|
|
2434
|
+
// Telemetry is not an execution gate.
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
}
|
|
2438
|
+
|
|
1559
2439
|
/** Background MCP preparation must not inflate startup phase distributions. */
|
|
1560
2440
|
export function recordToolPreparationPhase(
|
|
1561
2441
|
observability: Observability,
|
|
@@ -1602,6 +2482,7 @@ function completedOperationSpan(
|
|
|
1602
2482
|
outcome: string;
|
|
1603
2483
|
provider?: string;
|
|
1604
2484
|
backend?: string;
|
|
2485
|
+
tool?: string;
|
|
1605
2486
|
correlationId?: string | undefined;
|
|
1606
2487
|
},
|
|
1607
2488
|
): void {
|
|
@@ -1838,34 +2719,61 @@ function contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {
|
|
|
1838
2719
|
return null;
|
|
1839
2720
|
}
|
|
1840
2721
|
|
|
1841
|
-
// TTFT
|
|
1842
|
-
//
|
|
1843
|
-
//
|
|
1844
|
-
|
|
2722
|
+
// TTFT lives on a human-perceptible scale, but reasoning models legitimately
|
|
2723
|
+
// think for tens of seconds before their first streamed token. The finite
|
|
2724
|
+
// buckets therefore run well past 10s: a top bucket at 10s made every slower
|
|
2725
|
+
// first token land in +Inf, so p50/p99 saturated at exactly "10" and hid how
|
|
2726
|
+
// slow the tail really was.
|
|
2727
|
+
const STREAM_TTFT_BUCKETS = [
|
|
2728
|
+
0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 300,
|
|
2729
|
+
];
|
|
1845
2730
|
const STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
|
|
2731
|
+
// Opengeni's own per-request work before the provider sees bytes: admission,
|
|
2732
|
+
// durable history/audit checkpoints, request build. Normally milliseconds to a
|
|
2733
|
+
// second; tens of seconds means our database or consumer is the bottleneck.
|
|
2734
|
+
const MODEL_REQUEST_PRE_DISPATCH_BUCKETS = [
|
|
2735
|
+
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60, 120,
|
|
2736
|
+
];
|
|
2737
|
+
|
|
2738
|
+
/** First streamed content of a provider response. `any` is the first reasoning
|
|
2739
|
+
* or answer delta; `text` is the first answer-text delta. They overlap — never
|
|
2740
|
+
* sum across `content`. */
|
|
2741
|
+
export type ProviderFirstContent = "any" | "text";
|
|
1846
2742
|
|
|
1847
2743
|
/**
|
|
1848
|
-
* Per-
|
|
1849
|
-
*
|
|
2744
|
+
* Per-stream timing tracker fed every normalized runtime event in push order,
|
|
2745
|
+
* plus two producer-side hooks. It separates OUR latency from the PROVIDER's:
|
|
1850
2746
|
*
|
|
1851
|
-
* - `opengeni_stream_ttft_seconds{provider}` —
|
|
1852
|
-
*
|
|
1853
|
-
*
|
|
1854
|
-
*
|
|
1855
|
-
*
|
|
1856
|
-
*
|
|
2747
|
+
* - `opengeni_stream_ttft_seconds{provider}` — user-perceived (re)start latency:
|
|
2748
|
+
* time from a stream start / structural boundary (tool call, tool result,
|
|
2749
|
+
* completed message, usage frame) to the first streamed content delta. It
|
|
2750
|
+
* mixes our between-call work with provider time; keep it for the absolute
|
|
2751
|
+
* dashboard view, not for provider alerting.
|
|
2752
|
+
* - `opengeni_model_request_pre_dispatch_seconds{provider}` — OUR per-request
|
|
2753
|
+
* latency: SDK model entry (the first admission check) to the literal
|
|
2754
|
+
* provider dispatch. Covers admission, the durable history checkpoint,
|
|
2755
|
+
* credit revalidation, the durable request audit and request build.
|
|
2756
|
+
* - `opengeni_model_provider_ttft_seconds{provider,content}` — the PROVIDER's
|
|
2757
|
+
* latency: literal dispatch to the first streamed `any` (reasoning or text)
|
|
2758
|
+
* delta and to the first answer `text` delta. Includes network, provider
|
|
2759
|
+
* queueing, prompt prefill and reasoning before the first streamed token.
|
|
1857
2760
|
* - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
|
|
1858
|
-
* consecutive content deltas of the SAME class
|
|
1859
|
-
*
|
|
1860
|
-
* measures only the choppiness of a live token stream.
|
|
2761
|
+
* consecutive content deltas of the SAME class, reset at every structural
|
|
2762
|
+
* event so a gap never spans a tool call or a model boundary.
|
|
1861
2763
|
*
|
|
1862
|
-
*
|
|
2764
|
+
* Dispatch timing deliberately survives structural events: the consumer may
|
|
2765
|
+
* still be persisting the previous response's tool results when the producer
|
|
2766
|
+
* dispatches the next request, but the next request's first delta always
|
|
2767
|
+
* follows its own dispatch. Purely observational and clock-injectable.
|
|
1863
2768
|
*/
|
|
1864
2769
|
export class StreamTimingMetrics {
|
|
1865
2770
|
private readonly now: () => number;
|
|
1866
2771
|
private ttftAnchor: number;
|
|
1867
2772
|
private ttftArmed = true;
|
|
1868
2773
|
private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();
|
|
2774
|
+
private modelEntryAt: number | null = null;
|
|
2775
|
+
private dispatchedAt: number | null = null;
|
|
2776
|
+
private readonly providerFirstRecorded = new Set<ProviderFirstContent>();
|
|
1869
2777
|
|
|
1870
2778
|
constructor(
|
|
1871
2779
|
private readonly observability: Observability,
|
|
@@ -1875,6 +2783,31 @@ export class StreamTimingMetrics {
|
|
|
1875
2783
|
this.ttftAnchor = this.now();
|
|
1876
2784
|
}
|
|
1877
2785
|
|
|
2786
|
+
/** Producer side: the SDK entered a model request (first admission check).
|
|
2787
|
+
* Admission can be re-entered for the same request; keep the earliest. */
|
|
2788
|
+
onModelRequestEntry(): void {
|
|
2789
|
+
if (this.modelEntryAt === null) this.modelEntryAt = this.now();
|
|
2790
|
+
}
|
|
2791
|
+
|
|
2792
|
+
/** Producer side: request bytes are about to leave this process. */
|
|
2793
|
+
onProviderDispatch(): void {
|
|
2794
|
+
const at = this.now();
|
|
2795
|
+
if (this.modelEntryAt !== null) {
|
|
2796
|
+
this.observability.observeHistogram({
|
|
2797
|
+
name: "opengeni_model_request_pre_dispatch_seconds",
|
|
2798
|
+
help: "Seconds of Opengeni work from SDK model-request entry to literal provider dispatch.",
|
|
2799
|
+
buckets: MODEL_REQUEST_PRE_DISPATCH_BUCKETS,
|
|
2800
|
+
labels: { provider: this.options.provider },
|
|
2801
|
+
value: Math.max(0, (at - this.modelEntryAt) / 1000),
|
|
2802
|
+
});
|
|
2803
|
+
this.modelEntryAt = null;
|
|
2804
|
+
}
|
|
2805
|
+
// A transport retry re-dispatches the same request; the first delta then
|
|
2806
|
+
// belongs to the latest dispatch.
|
|
2807
|
+
this.dispatchedAt = at;
|
|
2808
|
+
this.providerFirstRecorded.clear();
|
|
2809
|
+
}
|
|
2810
|
+
|
|
1878
2811
|
onEvent(type: SessionEventType): void {
|
|
1879
2812
|
const deltaClass = contentDeltaClass(type);
|
|
1880
2813
|
if (deltaClass === null) {
|
|
@@ -1890,13 +2823,17 @@ export class StreamTimingMetrics {
|
|
|
1890
2823
|
if (this.ttftArmed) {
|
|
1891
2824
|
this.observability.observeHistogram({
|
|
1892
2825
|
name: "opengeni_stream_ttft_seconds",
|
|
1893
|
-
help: "Seconds from a
|
|
2826
|
+
help: "Seconds from a stream start or structural boundary to the next streamed content delta (user-perceived; includes Opengeni between-call work).",
|
|
1894
2827
|
buckets: STREAM_TTFT_BUCKETS,
|
|
1895
2828
|
labels: { provider: this.options.provider },
|
|
1896
2829
|
value: Math.max(0, (at - this.ttftAnchor) / 1000),
|
|
1897
2830
|
});
|
|
1898
2831
|
this.ttftArmed = false;
|
|
1899
2832
|
}
|
|
2833
|
+
if (this.dispatchedAt !== null) {
|
|
2834
|
+
this.observeProviderFirst("any", at);
|
|
2835
|
+
if (deltaClass === "message") this.observeProviderFirst("text", at);
|
|
2836
|
+
}
|
|
1900
2837
|
const last = this.lastDeltaAt.get(deltaClass);
|
|
1901
2838
|
if (last !== undefined) {
|
|
1902
2839
|
this.observability.observeHistogram({
|
|
@@ -1909,6 +2846,18 @@ export class StreamTimingMetrics {
|
|
|
1909
2846
|
}
|
|
1910
2847
|
this.lastDeltaAt.set(deltaClass, at);
|
|
1911
2848
|
}
|
|
2849
|
+
|
|
2850
|
+
private observeProviderFirst(content: ProviderFirstContent, at: number): void {
|
|
2851
|
+
if (this.dispatchedAt === null || this.providerFirstRecorded.has(content)) return;
|
|
2852
|
+
this.providerFirstRecorded.add(content);
|
|
2853
|
+
this.observability.observeHistogram({
|
|
2854
|
+
name: "opengeni_model_provider_ttft_seconds",
|
|
2855
|
+
help: "Seconds from literal provider dispatch to the first streamed content delta (any = reasoning or text, text = answer text).",
|
|
2856
|
+
buckets: STREAM_TTFT_BUCKETS,
|
|
2857
|
+
labels: { provider: this.options.provider, content },
|
|
2858
|
+
value: Math.max(0, (at - this.dispatchedAt) / 1000),
|
|
2859
|
+
});
|
|
2860
|
+
}
|
|
1912
2861
|
}
|
|
1913
2862
|
|
|
1914
2863
|
// Batch shapes: sizes are small integers; durations are the append+publish round
|
|
@@ -2206,13 +3155,26 @@ function requiredTemporalNumber(
|
|
|
2206
3155
|
if (value === null || value === undefined) {
|
|
2207
3156
|
throw new Error(`Temporal stats omitted required ${field}`);
|
|
2208
3157
|
}
|
|
2209
|
-
|
|
3158
|
+
// protobuf Long is supported through its string representation. Refuse JS
|
|
3159
|
+
// coercions (booleans, arrays, blanks) and hostile toString implementations.
|
|
3160
|
+
let raw: unknown = value;
|
|
3161
|
+
try {
|
|
3162
|
+
if (typeof raw === "object" && raw !== null && !Array.isArray(raw)) raw = raw.toString();
|
|
3163
|
+
} catch {
|
|
3164
|
+
throw new Error(`Temporal stats returned invalid ${field}`);
|
|
3165
|
+
}
|
|
3166
|
+
const parsed =
|
|
3167
|
+
typeof raw === "number"
|
|
3168
|
+
? raw
|
|
3169
|
+
: typeof raw === "string" && /^\d+(?:\.\d+)?(?:[eE][+-]?\d+)?$/.test(raw)
|
|
3170
|
+
? Number(raw)
|
|
3171
|
+
: Number.NaN;
|
|
2210
3172
|
if (
|
|
2211
3173
|
!Number.isFinite(parsed) ||
|
|
2212
3174
|
parsed < 0 ||
|
|
2213
3175
|
(options.integer && !Number.isSafeInteger(parsed))
|
|
2214
3176
|
) {
|
|
2215
|
-
throw new Error(`Temporal stats returned invalid ${field}
|
|
3177
|
+
throw new Error(`Temporal stats returned invalid ${field}`);
|
|
2216
3178
|
}
|
|
2217
3179
|
return parsed;
|
|
2218
3180
|
}
|
|
@@ -2264,3 +3226,141 @@ export function modelCallAccountContext(input: {
|
|
|
2264
3226
|
accountChangedFromPrevCall,
|
|
2265
3227
|
};
|
|
2266
3228
|
}
|
|
3229
|
+
|
|
3230
|
+
export type CodeSearchCallOutcome =
|
|
3231
|
+
| "completed"
|
|
3232
|
+
| "jev_unavailable"
|
|
3233
|
+
| "jev_rejected"
|
|
3234
|
+
| "workspace_unavailable"
|
|
3235
|
+
| "invalid_arguments"
|
|
3236
|
+
| "breaker_open"
|
|
3237
|
+
| "cancelled"
|
|
3238
|
+
| "failed";
|
|
3239
|
+
|
|
3240
|
+
/** One `code_search` tool call: outcome, wall time and the Jev work it used. */
|
|
3241
|
+
export function recordCodeSearchCall(
|
|
3242
|
+
observability: Observability,
|
|
3243
|
+
input: {
|
|
3244
|
+
outcome: CodeSearchCallOutcome;
|
|
3245
|
+
durationSeconds: number;
|
|
3246
|
+
jevRequests: number;
|
|
3247
|
+
jevCostUsd: number;
|
|
3248
|
+
},
|
|
3249
|
+
): void {
|
|
3250
|
+
observability.incrementCounter({
|
|
3251
|
+
name: "opengeni_code_search_calls_total",
|
|
3252
|
+
help: "Jev-backed code_search tool calls by outcome.",
|
|
3253
|
+
labels: { outcome: input.outcome },
|
|
3254
|
+
});
|
|
3255
|
+
observability.observeHistogram({
|
|
3256
|
+
name: "opengeni_code_search_duration_seconds",
|
|
3257
|
+
help: "Wall time of one code_search tool call.",
|
|
3258
|
+
buckets: [0.5, 1, 2, 4, 8, 15, 30, 60],
|
|
3259
|
+
labels: { outcome: input.outcome },
|
|
3260
|
+
value: Math.max(0, input.durationSeconds),
|
|
3261
|
+
});
|
|
3262
|
+
if (input.jevRequests > 0) {
|
|
3263
|
+
observability.incrementCounter({
|
|
3264
|
+
name: "opengeni_code_search_jev_requests_total",
|
|
3265
|
+
help: "Jev requests made by code_search.",
|
|
3266
|
+
amount: input.jevRequests,
|
|
3267
|
+
});
|
|
3268
|
+
}
|
|
3269
|
+
if (input.jevCostUsd > 0) {
|
|
3270
|
+
observability.incrementCounter({
|
|
3271
|
+
name: "opengeni_code_search_jev_cost_micro_usd_total",
|
|
3272
|
+
help: "Estimated Jev list-price cost of code_search, in micro-USD.",
|
|
3273
|
+
amount: Math.round(input.jevCostUsd * 1_000_000),
|
|
3274
|
+
});
|
|
3275
|
+
}
|
|
3276
|
+
}
|
|
3277
|
+
|
|
3278
|
+
/**
|
|
3279
|
+
* One skill_read call. Labels are closed sets: `skill` is a built-in id or
|
|
3280
|
+
* `custom` for every other Skill, so tenant Skill ids, names, and requested
|
|
3281
|
+
* identifiers never become labels. `source` is `unknown` when the read was
|
|
3282
|
+
* refused before a Skill resolved.
|
|
3283
|
+
*/
|
|
3284
|
+
export function recordSkillRead(
|
|
3285
|
+
observability: Observability,
|
|
3286
|
+
read: {
|
|
3287
|
+
caller: "model" | "codemode";
|
|
3288
|
+
kind: SkillReadKind;
|
|
3289
|
+
source: SkillUseSource | null;
|
|
3290
|
+
/** The resolved Skill id, or the requested identifier when none resolved. */
|
|
3291
|
+
skill: string;
|
|
3292
|
+
},
|
|
3293
|
+
): void {
|
|
3294
|
+
observability.incrementCounter({
|
|
3295
|
+
name: "opengeni_skill_reads_total",
|
|
3296
|
+
help: "skill_read calls by Skill source, built-in Skill id (custom for any other Skill), result kind, and caller.",
|
|
3297
|
+
labels: {
|
|
3298
|
+
source: read.source ?? "unknown",
|
|
3299
|
+
skill: skillReadMetricLabel(read.source, read.skill),
|
|
3300
|
+
kind: read.kind,
|
|
3301
|
+
caller: read.caller,
|
|
3302
|
+
},
|
|
3303
|
+
});
|
|
3304
|
+
}
|
|
3305
|
+
|
|
3306
|
+
/**
|
|
3307
|
+
* One skill_checkout call. Labels are closed sets and never carry Skill ids,
|
|
3308
|
+
* names, or paths. Phases that did not run are not observed.
|
|
3309
|
+
*/
|
|
3310
|
+
export function recordSkillCheckout(
|
|
3311
|
+
observability: Observability,
|
|
3312
|
+
checkout: {
|
|
3313
|
+
outcome: "written" | "unchanged" | "refused" | "failed";
|
|
3314
|
+
selection: "all" | "paths";
|
|
3315
|
+
written: number;
|
|
3316
|
+
unchanged: number;
|
|
3317
|
+
resolveSeconds: number | null;
|
|
3318
|
+
sandboxSeconds: number | null;
|
|
3319
|
+
writeSeconds: number | null;
|
|
3320
|
+
totalSeconds: number;
|
|
3321
|
+
},
|
|
3322
|
+
): void {
|
|
3323
|
+
observability.incrementCounter({
|
|
3324
|
+
name: "opengeni_skill_checkouts_total",
|
|
3325
|
+
help: "skill_checkout calls by outcome and whether they selected specific paths.",
|
|
3326
|
+
labels: { outcome: checkout.outcome, selection: checkout.selection },
|
|
3327
|
+
});
|
|
3328
|
+
const phases = [
|
|
3329
|
+
["resolve", checkout.resolveSeconds],
|
|
3330
|
+
["sandbox", checkout.sandboxSeconds],
|
|
3331
|
+
["write", checkout.writeSeconds],
|
|
3332
|
+
["total", checkout.totalSeconds],
|
|
3333
|
+
] as const;
|
|
3334
|
+
for (const [phase, seconds] of phases) {
|
|
3335
|
+
if (seconds === null) continue;
|
|
3336
|
+
observability.observeHistogram({
|
|
3337
|
+
name: "opengeni_skill_checkout_duration_seconds",
|
|
3338
|
+
help: "Wall time of skill_checkout phases: resolve (authority and Skill read), sandbox (filesystem handle, including lazy box start), write (batched file write), and total.",
|
|
3339
|
+
buckets: [0.05, 0.1, 0.25, 0.5, 1, 2, 4, 8, 15, 30, 60],
|
|
3340
|
+
labels: { phase, outcome: checkout.outcome },
|
|
3341
|
+
value: Math.max(0, seconds),
|
|
3342
|
+
});
|
|
3343
|
+
}
|
|
3344
|
+
for (const [result, amount] of [
|
|
3345
|
+
["written", checkout.written],
|
|
3346
|
+
["unchanged", checkout.unchanged],
|
|
3347
|
+
] as const) {
|
|
3348
|
+
if (amount <= 0) continue;
|
|
3349
|
+
observability.incrementCounter({
|
|
3350
|
+
name: "opengeni_skill_checkout_files_total",
|
|
3351
|
+
help: "Files skill_checkout wrote or found already present with the same content.",
|
|
3352
|
+
labels: { result },
|
|
3353
|
+
amount,
|
|
3354
|
+
});
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
|
|
3358
|
+
function skillReadMetricLabel(source: SkillUseSource | null, skill: string): string {
|
|
3359
|
+
if (source !== null && source !== "builtin") return "custom";
|
|
3360
|
+
// A refused read may name a built-in by its plain name.
|
|
3361
|
+
for (const candidate of source === null ? [skill, `builtin:${skill}`] : [skill]) {
|
|
3362
|
+
const parsed = BundledSkillId.safeParse(candidate);
|
|
3363
|
+
if (parsed.success) return parsed.data;
|
|
3364
|
+
}
|
|
3365
|
+
return "custom";
|
|
3366
|
+
}
|