@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38686408778001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +172 -28
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +3 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +30 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-NTXZHYET.js} +1552 -2876
- package/dist/activities-control-NTXZHYET.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-RKRI5GY4.js} +9238 -4392
- package/dist/activities-turn-RKRI5GY4.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +104 -35
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-7BIHFUGB.js} +2293 -3956
- package/dist/chunk-7BIHFUGB.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1134 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +26 -4
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +599 -110
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +162 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +116 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +758 -141
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +79 -38
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -1,35 +1,40 @@
|
|
|
1
|
+
import { recordSessionUsageLimitEvent } from "@opengeni/core";
|
|
1
2
|
import {
|
|
2
3
|
requestSessionTurnRecovery,
|
|
3
|
-
getSessionGoal,
|
|
4
|
-
armXaiCapacityWait,
|
|
5
|
-
reconcileXaiCapacityWait,
|
|
6
|
-
listCodexAccountStatuses,
|
|
7
|
-
quarantineCodexCredentialForLease,
|
|
8
4
|
recordUsageEvent,
|
|
9
5
|
getActiveSessionHistoryItemsPaged,
|
|
10
|
-
settleCodexCredentialLeaseLoss,
|
|
11
|
-
settleCodexCredentialFailover,
|
|
12
6
|
readLease,
|
|
13
7
|
SandboxLeaseSupersededError,
|
|
14
8
|
isSessionEventPersistenceError,
|
|
15
|
-
|
|
9
|
+
SANDBOX_SETUP_RECOVERY_LIMIT,
|
|
10
|
+
SubscriptionCoreCodexAccessLostError,
|
|
11
|
+
SubscriptionCoreCodexSourceDisconnectedError,
|
|
12
|
+
SubscriptionCoreCodexRequestOutcomeUnknownError,
|
|
13
|
+
isSubscriptionCoreCodexSourceDisconnected,
|
|
14
|
+
SubscriptionCoreCodexLeaseLostError,
|
|
15
|
+
readSessionTurnToolProgressRecoveryState,
|
|
16
16
|
} from "@opengeni/db";
|
|
17
17
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
18
|
-
import { maxTurnsExceededRunState } from "@opengeni/runtime";
|
|
19
|
-
import { CancelledFailure } from "@temporalio/activity";
|
|
20
18
|
import {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
19
|
+
maxTurnsExceededRunState,
|
|
20
|
+
isModalTaskExecStartPreDispatchUnavailableError,
|
|
21
|
+
isModalCommandStartOutcomeUnknownError,
|
|
22
|
+
isProviderCommandObservationUnavailableError,
|
|
23
|
+
isRoutingMutationOutcomeUnknownError,
|
|
24
|
+
toolCallCommandNotStartedCallIds,
|
|
25
|
+
} from "@opengeni/runtime";
|
|
26
|
+
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
27
|
+
|
|
28
|
+
import { TurnExecutionPolicyDefinitionMismatchError, type Settings } from "@opengeni/config";
|
|
27
29
|
import {
|
|
30
|
+
CodexReloginRequired,
|
|
31
|
+
CodexStreamingTerminalError,
|
|
28
32
|
classifyCodexEncryptedArtifactRejection,
|
|
33
|
+
classifyCodexEntitlementRejection,
|
|
29
34
|
classifyCodexUsageLimitError,
|
|
30
35
|
isCodexTransportError,
|
|
31
|
-
type CodexUsageHeaderSnapshot,
|
|
32
36
|
} from "@opengeni/codex";
|
|
37
|
+
|
|
33
38
|
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
34
39
|
import { deliverFailedChildTurnToParent } from "../parent-wake";
|
|
35
40
|
import type {
|
|
@@ -38,13 +43,32 @@ import type {
|
|
|
38
43
|
RunAgentTurnResult,
|
|
39
44
|
} from "../types";
|
|
40
45
|
import { CodexCredentialLeaseLostError, createTurnCredentialLeases } from "./credential-leases";
|
|
46
|
+
import {
|
|
47
|
+
SubscriptionCoreCodexTurnError,
|
|
48
|
+
subscriptionCoreAccountRefusedFailure,
|
|
49
|
+
subscriptionCoreLeaseBusyChain,
|
|
50
|
+
subscriptionCoreLeaseBusyDelayMs,
|
|
51
|
+
subscriptionCoreLeaseBusyExhaustedFailure,
|
|
52
|
+
} from "./codex-core-errors";
|
|
53
|
+
import { recordCoreCodexRefusal } from "./codex-core-settlement";
|
|
54
|
+
import { failOverCoreCodexTurn } from "./codex-core-failover";
|
|
41
55
|
import { createTurnHistorySink } from "./history-sink";
|
|
42
56
|
|
|
43
|
-
import { BudgetExhaustedError } from "./admission";
|
|
57
|
+
import { BudgetExhaustedError, sessionUsageLimitStopEvents } from "./admission";
|
|
58
|
+
import {
|
|
59
|
+
SESSION_USAGE_CUSTODY_PENDING,
|
|
60
|
+
SESSION_USAGE_CUSTODY_RECHECK_MS,
|
|
61
|
+
sessionUsageCustodyPending,
|
|
62
|
+
} from "./session-usage-custody";
|
|
44
63
|
import {
|
|
64
|
+
automaticRecoveryUnavailableFailure,
|
|
45
65
|
providerRecoveryExhaustedFailure,
|
|
66
|
+
withModelRoutePresentation,
|
|
46
67
|
postClaimDatabaseRecoveryFailure,
|
|
47
68
|
providerRecoveryResult,
|
|
69
|
+
providerRecoveryCode,
|
|
70
|
+
providerRecoveryLimit,
|
|
71
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
48
72
|
providerRetryAfterMs,
|
|
49
73
|
escapedMcpTimeoutRecoveryFailure,
|
|
50
74
|
preClaimAdmissionFailure,
|
|
@@ -52,9 +76,8 @@ import {
|
|
|
52
76
|
sandboxLifecycleTransitionDiagnostic,
|
|
53
77
|
sandboxRouteTransitionCode,
|
|
54
78
|
safeErrorDiagnostic,
|
|
55
|
-
classifyXaiCredentialFailure,
|
|
56
79
|
agentRunFailurePayload,
|
|
57
|
-
|
|
80
|
+
agentRunRecoveryFailurePayload,
|
|
58
81
|
classifyCodexCredentialFailure,
|
|
59
82
|
codexUsageLimitFailurePayload,
|
|
60
83
|
} from "./errors";
|
|
@@ -68,12 +91,22 @@ import {
|
|
|
68
91
|
import type {
|
|
69
92
|
AttemptIdentityState,
|
|
70
93
|
BillingState,
|
|
94
|
+
CodexSubscriptionCoreTurn,
|
|
71
95
|
EventingState,
|
|
72
96
|
ProviderTurnState,
|
|
73
97
|
TurnControlState,
|
|
74
98
|
} from "./turn-context";
|
|
75
|
-
import type {
|
|
76
|
-
import {
|
|
99
|
+
import type { createCoreCodexRequests } from "./codex-core-requests";
|
|
100
|
+
import {
|
|
101
|
+
XAI_CORE_REFUSAL,
|
|
102
|
+
xaiCoreLanesRecoverAfterCheckpoint,
|
|
103
|
+
xaiCoreRequestOutcomeUnknown,
|
|
104
|
+
} from "./xai-core";
|
|
105
|
+
import { settleCoreSubscriptionRefusal } from "./subscription-core-settlement";
|
|
106
|
+
import { claudeCoreRefusal, claudeCoreRequestOutcomeUnknown } from "./claude-core";
|
|
107
|
+
import { coreLanesRecoverAfterCheckpoint } from "./subscription-core-requests";
|
|
108
|
+
|
|
109
|
+
import { providerRecoveryCause, recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
|
|
77
110
|
|
|
78
111
|
export type TurnFailureDeps = {
|
|
79
112
|
error: unknown;
|
|
@@ -83,6 +116,8 @@ export type TurnFailureDeps = {
|
|
|
83
116
|
bus: ActivityServices["bus"];
|
|
84
117
|
observability: ActivityServices["observability"];
|
|
85
118
|
wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
|
|
119
|
+
/** Typed capacity-wake signals of core waiters (a refusal's joined observations may wake them). */
|
|
120
|
+
signalCodexCapacityWorkflow?: ActivityServices["signalCodexCapacityWorkflow"];
|
|
86
121
|
cancellationSignal: AbortSignal | undefined;
|
|
87
122
|
sandboxRotationController: AbortController;
|
|
88
123
|
noteCancellationRequested: () => void;
|
|
@@ -107,124 +142,41 @@ export type TurnFailureDeps = {
|
|
|
107
142
|
|
|
108
143
|
export type CodexDefinitiveFailureDisposition = "failover" | "wait" | "terminal";
|
|
109
144
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
input.failureKind === "rate_limit" ||
|
|
142
|
-
input.decisionKind === "allCapped" ||
|
|
143
|
-
!input.rotationEnabled ||
|
|
144
|
-
input.pinDisposition === "manual"
|
|
145
|
-
) {
|
|
146
|
-
return "wait";
|
|
147
|
-
}
|
|
148
|
-
return "terminal";
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
/**
|
|
152
|
-
* Bound one turn to the alternate credentials that policy actually permits.
|
|
153
|
-
* The effective account list is already workspace/organization scoped;
|
|
154
|
-
* allocator-disabled rows must not enlarge the retry budget. The database
|
|
155
|
-
* requires a positive bound even though one-account paths never fail over.
|
|
156
|
-
*/
|
|
157
|
-
export function codexCredentialFailoverLimit(
|
|
158
|
-
accounts: ReadonlyArray<{ id: string; allocatorEnabled: boolean }>,
|
|
159
|
-
servingCredentialId: string,
|
|
160
|
-
): number {
|
|
161
|
-
const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled).length;
|
|
162
|
-
const servingIsAllocatable = accounts.some(
|
|
163
|
-
(account) => account.id === servingCredentialId && account.allocatorEnabled,
|
|
164
|
-
);
|
|
165
|
-
return Math.max(1, allocatableAccounts - (servingIsAllocatable ? 1 : 0));
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
/** Build the durable waiter payload without collapsing quota refusals into 403. */
|
|
169
|
-
export function codexCapacityWaitFailurePayload(input: {
|
|
170
|
-
failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
|
|
171
|
-
usageLimit: { resetsInSeconds: number | null } | null;
|
|
172
|
-
cooldownSeconds: number | null;
|
|
173
|
-
detail: string;
|
|
174
|
-
allAccounts: boolean;
|
|
175
|
-
}): CodexCapacityWaitFailurePayload {
|
|
176
|
-
if (input.failureKind === "quota") {
|
|
177
|
-
return codexUsageLimitFailurePayload(
|
|
178
|
-
input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
|
|
179
|
-
input.detail,
|
|
180
|
-
input.allAccounts ? { allAccounts: true } : undefined,
|
|
181
|
-
);
|
|
182
|
-
}
|
|
183
|
-
if (input.failureKind === "rate_limit") {
|
|
184
|
-
return {
|
|
185
|
-
error: "The serving Codex subscription is temporarily rate limited.",
|
|
186
|
-
code: "codex_account_rate_limited",
|
|
187
|
-
detail: input.detail,
|
|
188
|
-
retryable: false,
|
|
189
|
-
};
|
|
190
|
-
}
|
|
191
|
-
if (input.failureKind === "auth") {
|
|
192
|
-
return {
|
|
193
|
-
error: "The serving Codex account requires reconnection.",
|
|
194
|
-
code: "codex_relogin_required",
|
|
195
|
-
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
196
|
-
retryable: false,
|
|
197
|
-
};
|
|
198
|
-
}
|
|
199
|
-
return {
|
|
200
|
-
error: "The serving Codex account is not authorized for this request.",
|
|
201
|
-
code: "codex_account_forbidden",
|
|
202
|
-
detail: "the same accepted turn is waiting for the selected account to recover",
|
|
203
|
-
retryable: false,
|
|
204
|
-
};
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
function codexLeaseAccountsForSelection(
|
|
208
|
-
accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>,
|
|
209
|
-
): CodexLeaseAccountStatus[] {
|
|
210
|
-
return accounts.map((account) => ({
|
|
211
|
-
...account,
|
|
212
|
-
activeLeaseCount: 0,
|
|
213
|
-
selectionCount: 0,
|
|
214
|
-
lastSelectedAt: null,
|
|
215
|
-
}));
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
function acceptedCodexPolicySnapshot(
|
|
219
|
-
providerTurn: ProviderTurnState,
|
|
220
|
-
): CodexCredentialPolicySnapshotV1 {
|
|
221
|
-
if (!providerTurn.codexPolicySnapshot) {
|
|
222
|
-
throw new Error("Codex accepted policy snapshot is missing after durable lease acquisition");
|
|
145
|
+
export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
|
|
146
|
+
try {
|
|
147
|
+
return await settleTurnFailureInAttempt(deps);
|
|
148
|
+
} catch (error) {
|
|
149
|
+
if (
|
|
150
|
+
deps.control.activityStatus === "recovering" &&
|
|
151
|
+
error instanceof ApplicationFailure &&
|
|
152
|
+
error.type === "OpenGeniPostClaimDatabaseRecovery"
|
|
153
|
+
)
|
|
154
|
+
throw error;
|
|
155
|
+
// Connectivity can disappear while settling an unrelated run error too.
|
|
156
|
+
// Do not overwrite a possibly committed settlement; the control lane
|
|
157
|
+
// re-reads exact ownership and becomes a stale no-op if it already closed.
|
|
158
|
+
if (deps.attempt.turnId && deps.attempt.triggerEventId) {
|
|
159
|
+
const recovery = postClaimDatabaseRecoveryFailure({
|
|
160
|
+
// A failed rollback/terminal write must not erase no-replay evidence
|
|
161
|
+
// from the failure we were settling (notably unknown tool effects).
|
|
162
|
+
error: new AggregateError([error, deps.error], "Turn failure settlement failed"),
|
|
163
|
+
turnId: deps.attempt.turnId,
|
|
164
|
+
triggerEventId: deps.attempt.triggerEventId,
|
|
165
|
+
executionGeneration: deps.attempt.executionGeneration,
|
|
166
|
+
requireDatabaseProvenance: true,
|
|
167
|
+
});
|
|
168
|
+
if (recovery) {
|
|
169
|
+
deps.control.activityStatus = "recovering";
|
|
170
|
+
deps.control.turnMetricOutcome = "recovering";
|
|
171
|
+
deps.control.activityError = error;
|
|
172
|
+
throw recovery;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
throw error;
|
|
223
176
|
}
|
|
224
|
-
return providerTurn.codexPolicySnapshot;
|
|
225
177
|
}
|
|
226
178
|
|
|
227
|
-
|
|
179
|
+
async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
|
|
228
180
|
if (deps.settings.environment === "local" && deps.error instanceof Error) {
|
|
229
181
|
// Keep local startup failures diagnosable without logging error messages,
|
|
230
182
|
// absolute host paths, prompts, credentials, or provider response bodies.
|
|
@@ -302,6 +254,69 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
302
254
|
// back to the workflow-claimed turn when the local lookup had not
|
|
303
255
|
// finished yet.
|
|
304
256
|
const recoveryTurnId = attempt.turnId;
|
|
257
|
+
// Unlike proven pre-dispatch failure, a genuine SDK Start/Wait uncertainty
|
|
258
|
+
// cannot reconstruct setup on a replacement attempt. The exact command and
|
|
259
|
+
// writer remain retained by sandbox-runtime; the logical turn is parked as
|
|
260
|
+
// recovering with a durable no-replay marker, not failed or completed.
|
|
261
|
+
const observationUnavailable = isProviderCommandObservationUnavailableError(error);
|
|
262
|
+
if (
|
|
263
|
+
(isModalCommandStartOutcomeUnknownError(error) || observationUnavailable) &&
|
|
264
|
+
recoveryTurnId &&
|
|
265
|
+
attempt.triggerEventId &&
|
|
266
|
+
attempt.executionGeneration > 0
|
|
267
|
+
) {
|
|
268
|
+
let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
|
|
269
|
+
try {
|
|
270
|
+
if (eventing.turnStartedPublished) {
|
|
271
|
+
await flushRuntimeBatcher();
|
|
272
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
273
|
+
}
|
|
274
|
+
recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
275
|
+
sessionId: input.sessionId,
|
|
276
|
+
turnId: recoveryTurnId,
|
|
277
|
+
triggerEventId: attempt.triggerEventId,
|
|
278
|
+
attemptId: input.attemptId,
|
|
279
|
+
reason: observationUnavailable
|
|
280
|
+
? "sandbox_command_observation_unavailable"
|
|
281
|
+
: "sandbox_command_start_outcome_unknown",
|
|
282
|
+
sandboxSetupOutcomeUnknown: true,
|
|
283
|
+
detail: {
|
|
284
|
+
code: observationUnavailable
|
|
285
|
+
? "sandbox_command_observation_unavailable"
|
|
286
|
+
: "sandbox_command_start_outcome_unknown",
|
|
287
|
+
retryable: false,
|
|
288
|
+
setupOutcome: "unknown",
|
|
289
|
+
replay: "blocked",
|
|
290
|
+
providerRecoveryCount: attempt.providerRecoveryCount,
|
|
291
|
+
},
|
|
292
|
+
});
|
|
293
|
+
} catch (checkpointError) {
|
|
294
|
+
const databaseRecovery = postClaimDatabaseRecoveryFailure({
|
|
295
|
+
error: checkpointError,
|
|
296
|
+
turnId: recoveryTurnId,
|
|
297
|
+
triggerEventId: attempt.triggerEventId,
|
|
298
|
+
executionGeneration: attempt.executionGeneration,
|
|
299
|
+
sandboxSetupOutcomeUnknown: true,
|
|
300
|
+
});
|
|
301
|
+
if (!databaseRecovery) throw checkpointError;
|
|
302
|
+
control.activityStatus = "recovering";
|
|
303
|
+
control.turnMetricOutcome = "recovering";
|
|
304
|
+
control.activityError = error;
|
|
305
|
+
throw databaseRecovery;
|
|
306
|
+
}
|
|
307
|
+
if (recovery.action === "stale") {
|
|
308
|
+
acknowledgeLostAttemptOwnership();
|
|
309
|
+
control.activityStatus = "cancelled";
|
|
310
|
+
control.turnMetricOutcome = "cancelled";
|
|
311
|
+
return claimedResult({ status: "cancelled" });
|
|
312
|
+
}
|
|
313
|
+
acknowledgeRecoveryQuiescence();
|
|
314
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
315
|
+
control.activityStatus = "recovering";
|
|
316
|
+
control.turnMetricOutcome = "recovering";
|
|
317
|
+
control.activityError = error;
|
|
318
|
+
return claimedResult({ status: "recovering", deferredUntilWake: true });
|
|
319
|
+
}
|
|
305
320
|
// A true epoch supersession and a provider lifecycle transition are both
|
|
306
321
|
// recoverable control-plane states, never session failures. A rotation
|
|
307
322
|
// persists an exact group/epoch wait marker so the workflow parks before
|
|
@@ -380,14 +395,76 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
380
395
|
throw recoveryError;
|
|
381
396
|
}
|
|
382
397
|
}
|
|
383
|
-
// A
|
|
384
|
-
//
|
|
385
|
-
// clears the durable pointer to home, the commit is valid but this exact
|
|
386
|
-
// attempt cannot serve a later home operation. Preserve the completed attach
|
|
398
|
+
// A route change can require a different home, filesystem root, or native
|
|
399
|
+
// capability set than this attempt established. Preserve the completed attach
|
|
387
400
|
// and every preceding model/tool receipt, close only the unresolved suffix,
|
|
388
401
|
// and continue the SAME logical turn in a fresh attempt. That next attempt
|
|
389
|
-
// starts from the
|
|
402
|
+
// starts from the committed pointer and establishes its route normally.
|
|
390
403
|
const routeTransitionCode = sandboxRouteTransitionCode(error);
|
|
404
|
+
// An earlier single model call in this task still owes its usage receipt
|
|
405
|
+
// (a fenced custody marker, or an open one past the overdue window). This
|
|
406
|
+
// is a wait for the owner to fill it or for maintenance to disclose it as
|
|
407
|
+
// unpriced, never a turn failure: keep the turn and its durable history,
|
|
408
|
+
// park it as recovering, and let the replacement attempt re-check
|
|
409
|
+
// admission (database reads only) with the workflow's bounded backoff.
|
|
410
|
+
// The blocked request was never sent, so nothing is replayed.
|
|
411
|
+
if (
|
|
412
|
+
sessionUsageCustodyPending(error) &&
|
|
413
|
+
recoveryTurnId &&
|
|
414
|
+
attempt.triggerEventId &&
|
|
415
|
+
eventing.publish &&
|
|
416
|
+
eventing.turnStartedPublished
|
|
417
|
+
) {
|
|
418
|
+
recordSessionUsageLimitEvent(observability, "between_calls", "unavailable");
|
|
419
|
+
let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
|
|
420
|
+
try {
|
|
421
|
+
await flushRuntimeBatcher();
|
|
422
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
423
|
+
recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
424
|
+
sessionId: input.sessionId,
|
|
425
|
+
turnId: recoveryTurnId,
|
|
426
|
+
triggerEventId: attempt.triggerEventId,
|
|
427
|
+
attemptId: input.attemptId,
|
|
428
|
+
reason: SESSION_USAGE_CUSTODY_PENDING,
|
|
429
|
+
detail: {
|
|
430
|
+
code: SESSION_USAGE_CUSTODY_PENDING,
|
|
431
|
+
retryable: true,
|
|
432
|
+
continueDelayMs: SESSION_USAGE_CUSTODY_RECHECK_MS,
|
|
433
|
+
},
|
|
434
|
+
});
|
|
435
|
+
} catch (recoveryError) {
|
|
436
|
+
const postClaimRecovery = postClaimDatabaseRecoveryFailure({
|
|
437
|
+
error: recoveryError,
|
|
438
|
+
turnId: recoveryTurnId,
|
|
439
|
+
triggerEventId: attempt.triggerEventId,
|
|
440
|
+
executionGeneration: attempt.executionGeneration,
|
|
441
|
+
});
|
|
442
|
+
if (postClaimRecovery) {
|
|
443
|
+
control.activityStatus = "recovering";
|
|
444
|
+
control.turnMetricOutcome = "recovering";
|
|
445
|
+
control.activityError = error;
|
|
446
|
+
throw postClaimRecovery;
|
|
447
|
+
}
|
|
448
|
+
throw recoveryError;
|
|
449
|
+
}
|
|
450
|
+
if (recovery.action === "stale") {
|
|
451
|
+
acknowledgeLostAttemptOwnership();
|
|
452
|
+
control.activityStatus = "cancelled";
|
|
453
|
+
control.turnMetricOutcome = "cancelled";
|
|
454
|
+
return claimedResult({ status: "cancelled" });
|
|
455
|
+
}
|
|
456
|
+
if (recovery.action === "recovering") {
|
|
457
|
+
acknowledgeRecoveryQuiescence();
|
|
458
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
459
|
+
control.activityStatus = "recovering";
|
|
460
|
+
control.turnMetricOutcome = "recovering";
|
|
461
|
+
control.activityError = error;
|
|
462
|
+
return claimedResult({
|
|
463
|
+
status: "recovering",
|
|
464
|
+
continueDelayMs: SESSION_USAGE_CUSTODY_RECHECK_MS,
|
|
465
|
+
});
|
|
466
|
+
}
|
|
467
|
+
}
|
|
391
468
|
if (routeTransitionCode && recoveryTurnId && eventing.publish && eventing.turnStartedPublished) {
|
|
392
469
|
try {
|
|
393
470
|
await flushRuntimeBatcher();
|
|
@@ -410,7 +487,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
410
487
|
return claimedResult({ status: "cancelled" });
|
|
411
488
|
}
|
|
412
489
|
if (recovery.action !== "recovering") {
|
|
413
|
-
throw new Error("
|
|
490
|
+
throw new Error("Sandbox route transition could not recover the current turn");
|
|
414
491
|
}
|
|
415
492
|
acknowledgeRecoveryQuiescence();
|
|
416
493
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
@@ -544,6 +621,27 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
544
621
|
control.turnMetricOutcome = "cancelled";
|
|
545
622
|
throw cancellationFailure;
|
|
546
623
|
}
|
|
624
|
+
if (attempt.turnId && attempt.triggerEventId && attempt.executionGeneration > 0) {
|
|
625
|
+
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
626
|
+
error,
|
|
627
|
+
turnId: attempt.turnId,
|
|
628
|
+
triggerEventId: attempt.triggerEventId,
|
|
629
|
+
executionGeneration: attempt.executionGeneration,
|
|
630
|
+
requireDatabaseProvenance: eventing.turnStartedPublished || attempt.modelRequestStarted,
|
|
631
|
+
});
|
|
632
|
+
if (recoveryFailure) {
|
|
633
|
+
// Stop this non-retryable activity without terminal logical settlement.
|
|
634
|
+
// Do not retry a delta/tool-ledger mutation with an unknown commit, nor
|
|
635
|
+
// claim that DB-down cleanup proved writer exit. Finally still drains
|
|
636
|
+
// the physical writers; the existing DB-only workflow lane closes this
|
|
637
|
+
// exact owner after connectivity returns and admission waits for its
|
|
638
|
+
// ordinary durable quiescence/writer fences.
|
|
639
|
+
control.activityStatus = "recovering";
|
|
640
|
+
control.turnMetricOutcome = "recovering";
|
|
641
|
+
control.activityError = error;
|
|
642
|
+
throw recoveryFailure;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
547
645
|
// The SDK's per-segment turn cap is a pacing valve, not a failure: end
|
|
548
646
|
// the turn gracefully and idle the session so an active goal continues
|
|
549
647
|
// via a synthesized continuation turn (or a user message resumes work).
|
|
@@ -592,112 +690,132 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
592
690
|
control.activityStatus = "idle";
|
|
593
691
|
return claimedResult({ status: "idle" });
|
|
594
692
|
}
|
|
595
|
-
const settleLostCodexAttempt = async (
|
|
596
|
-
lostTurnId: string,
|
|
597
|
-
holderId: string,
|
|
598
|
-
generation: number,
|
|
599
|
-
historyCheckpointDurable = false,
|
|
600
|
-
): Promise<RunAgentTurnResult> => {
|
|
601
|
-
let checkpointDurable = historyCheckpointDurable;
|
|
602
|
-
try {
|
|
603
|
-
if (!historyCheckpointDurable) {
|
|
604
|
-
await flushRuntimeBatcher();
|
|
605
|
-
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
606
|
-
}
|
|
607
|
-
checkpointDurable = true;
|
|
608
|
-
} catch {
|
|
609
|
-
observability.warn("Codex lease-loss checkpoint failed; refusing automatic turn replay", {
|
|
610
|
-
errorClass: "CodexCheckpointOperationError",
|
|
611
|
-
errorCode: "codex_lease_loss_checkpoint_failed",
|
|
612
|
-
origin: "worker",
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
|
|
616
|
-
const settlement = await settleCodexCredentialLeaseLoss(db, {
|
|
617
|
-
accountId: input.accountId,
|
|
618
|
-
workspaceId: input.workspaceId,
|
|
619
|
-
sessionId: input.sessionId,
|
|
620
|
-
turnId: lostTurnId,
|
|
621
|
-
attemptId: input.attemptId,
|
|
622
|
-
holderId,
|
|
623
|
-
generation,
|
|
624
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
625
|
-
checkpointDurable,
|
|
626
|
-
recoveryPayload: {
|
|
627
|
-
triggerEventId: attempt.triggerEventId!,
|
|
628
|
-
reason: "codex_lease_lost",
|
|
629
|
-
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
630
|
-
},
|
|
631
|
-
failedPayload: {
|
|
632
|
-
error:
|
|
633
|
-
"The Codex credential lease was lost and the latest conversation checkpoint could not be persisted. Automatic replay was refused.",
|
|
634
|
-
code: "codex_lease_checkpoint_failed",
|
|
635
|
-
retryable: false,
|
|
636
|
-
},
|
|
637
|
-
});
|
|
638
|
-
leases.codex.held = false;
|
|
639
|
-
observability.incrementCounter({
|
|
640
|
-
name: "opengeni_codex_lease_loss_settlements_total",
|
|
641
|
-
help: "Fenced Codex lease-loss settlements by outcome.",
|
|
642
|
-
labels: {
|
|
643
|
-
workspace_key: codexWorkspaceKey,
|
|
644
|
-
outcome: settlement.action,
|
|
645
|
-
},
|
|
646
|
-
});
|
|
647
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, settlement.events);
|
|
648
|
-
control.activityError = error;
|
|
649
|
-
if (settlement.action === "failed") {
|
|
650
|
-
control.activityStatus = "failed";
|
|
651
|
-
control.turnMetricOutcome = "failed";
|
|
652
|
-
await deliverFailedChildTurnToParent(
|
|
653
|
-
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
654
|
-
input.workspaceId,
|
|
655
|
-
input.sessionId,
|
|
656
|
-
lostTurnId,
|
|
657
|
-
);
|
|
658
|
-
return claimedResult({ status: "failed" });
|
|
659
|
-
}
|
|
660
|
-
control.activityStatus = "recovering";
|
|
661
|
-
control.turnMetricOutcome = "recovering";
|
|
662
|
-
return claimedResult({ status: "recovering" });
|
|
663
|
-
};
|
|
664
|
-
|
|
665
693
|
// A missing/expired/superseded lease is an execution-ownership failure,
|
|
666
694
|
// not a provider failure. Settle it before credential quarantine or the
|
|
667
695
|
// generic terminal path: the DB transaction marks a still-current turn
|
|
668
696
|
// recoverable, but a successor attempt or worker recovery makes this activity
|
|
669
697
|
// stale and unable to clobber the shared turn/session.
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
)
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
698
|
+
// Codex refusal, quota attribution and failover belong exclusively to the
|
|
699
|
+
// exact core placement and its generation fence.
|
|
700
|
+
const coreCodex = billingState.isCodexTurn ? (providerTurn.codexSubscriptionCore ?? null) : null;
|
|
701
|
+
// A SuperGrok turn on the shared core: unknown request outcomes are never
|
|
702
|
+
// replayed, and refusals settle through the shared core (X1b) instead of
|
|
703
|
+
// the legacy SuperGrok wait below.
|
|
704
|
+
const coreXai = billingState.isXaiTurn ? (providerTurn.xaiSubscriptionCore ?? null) : null;
|
|
705
|
+
// A Claude turn runs only on the shared core (the legacy Claude runtime is
|
|
706
|
+
// deleted): its refusals, unknown outcomes and lease loss settle there.
|
|
707
|
+
const coreClaude = billingState.isClaudeTurn
|
|
708
|
+
? (providerTurn.claudeSubscriptionCore ?? null)
|
|
709
|
+
: null;
|
|
710
|
+
// The transport may settle an ambiguous request before the SDK throws a plain
|
|
711
|
+
// HTTP error. That custody evidence must survive classification.
|
|
712
|
+
//
|
|
713
|
+
// When the ambiguity is this attempt's own interrupted request (a stream
|
|
714
|
+
// that dropped or stalled before its terminal event, or a gateway timeout),
|
|
715
|
+
// the same turn recovers from its durable checkpoint within the bounded
|
|
716
|
+
// provider-recovery budget. Codex requests are stateless (`store: false`)
|
|
717
|
+
// and function tools only run after a complete response, so a replay can
|
|
718
|
+
// at most repeat provider-hosted work and spend more quota. A refusal by
|
|
719
|
+
// the durable admission fence, an unsettled request, or a deterministic
|
|
720
|
+
// provider terminal stays a terminal unknown-outcome failure, as does an
|
|
721
|
+
// interruption combined with a lost lease or a disconnected source.
|
|
722
|
+
//
|
|
723
|
+
// The parallel session-title request never fences replay: it is a plain,
|
|
724
|
+
// tool-less request whose text only ever becomes the session title, and it
|
|
725
|
+
// is cancelled and joined before settlement. Its custody row keeps its
|
|
726
|
+
// outcome; the replacement attempt's new generation is admitted past it.
|
|
727
|
+
const coreUnknownRecoveryFailure =
|
|
728
|
+
coreCodex &&
|
|
729
|
+
!leases.codex.lost &&
|
|
730
|
+
!hasErrorInCauseChain(error, CodexCredentialLeaseLostError) &&
|
|
731
|
+
!hasErrorInCauseChain(error, SubscriptionCoreCodexLeaseLostError) &&
|
|
732
|
+
!hasErrorInCauseChain(error, SubscriptionCoreCodexSourceDisconnectedError) &&
|
|
733
|
+
!hasErrorInCauseChain(error, SubscriptionCoreCodexAccessLostError)
|
|
734
|
+
? ownInterruptedCodexRequestRecovery(error, coreCodex, billingState.isCodexTurn)
|
|
735
|
+
: null;
|
|
736
|
+
const coreRequestOutcomeUnknown =
|
|
737
|
+
!coreUnknownRecoveryFailure &&
|
|
738
|
+
(hasErrorInCauseChain(error, SubscriptionCoreCodexRequestOutcomeUnknownError) ||
|
|
739
|
+
coreCodex?.requests?.hasUnknownOutcome())
|
|
740
|
+
? new SubscriptionCoreCodexRequestOutcomeUnknownError()
|
|
741
|
+
: coreXai
|
|
742
|
+
? xaiCoreRequestOutcomeUnknown(error, coreXai)
|
|
743
|
+
: coreClaude
|
|
744
|
+
? claudeCoreRequestOutcomeUnknown(error, coreClaude)
|
|
745
|
+
: null;
|
|
746
|
+
const coreCodexLeaseLost =
|
|
747
|
+
coreCodex !== null &&
|
|
748
|
+
!coreRequestOutcomeUnknown &&
|
|
749
|
+
(leases.codex.lost ||
|
|
750
|
+
error instanceof CodexCredentialLeaseLostError ||
|
|
751
|
+
error instanceof SubscriptionCoreCodexLeaseLostError);
|
|
752
|
+
let coreCodexLeaseRecoverable = false;
|
|
753
|
+
let coreLeaseCheckpointFailed = false;
|
|
754
|
+
if (coreCodexLeaseLost && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
755
|
+
try {
|
|
756
|
+
// A completed response still owns a pending reservation until its
|
|
757
|
+
// history commits. Test replay safety after that rendezvous, not before.
|
|
758
|
+
await flushRuntimeBatcher();
|
|
759
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
760
|
+
await coreCodex.requests?.checkpoint();
|
|
761
|
+
coreCodexLeaseRecoverable = !coreCodex.requests || coreCodex.requests.canRecover();
|
|
762
|
+
} catch {
|
|
763
|
+
coreLeaseCheckpointFailed = true;
|
|
764
|
+
observability.incrementCounter({
|
|
765
|
+
name: "opengeni_codex_failover_checkpoints_total",
|
|
766
|
+
help: "Durable Codex failover checkpoint attempts by outcome.",
|
|
767
|
+
labels: { workspace_key: codexWorkspaceKey, outcome: "failed" },
|
|
768
|
+
});
|
|
769
|
+
}
|
|
684
770
|
}
|
|
685
|
-
|
|
686
|
-
|
|
771
|
+
// A core SuperGrok turn recovers a lost lease only once every request it
|
|
772
|
+
// dispatched is settled and its committed responses are checkpointed.
|
|
773
|
+
const xaiLeaseRecoverable =
|
|
687
774
|
billingState.isXaiTurn &&
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
await
|
|
693
|
-
|
|
775
|
+
leases.xai.lost &&
|
|
776
|
+
coreXai !== null &&
|
|
777
|
+
!coreRequestOutcomeUnknown &&
|
|
778
|
+
eventing.turnStartedPublished &&
|
|
779
|
+
(await xaiCoreLanesRecoverAfterCheckpoint(coreXai, async () => {
|
|
780
|
+
await flushRuntimeBatcher();
|
|
781
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
782
|
+
}));
|
|
783
|
+
// The same for a core Claude turn.
|
|
784
|
+
const claudeLeaseRecoverable =
|
|
785
|
+
coreClaude !== null &&
|
|
786
|
+
leases.claude.lost &&
|
|
787
|
+
!coreRequestOutcomeUnknown &&
|
|
788
|
+
eventing.turnStartedPublished &&
|
|
789
|
+
(await coreLanesRecoverAfterCheckpoint(coreClaude, async () => {
|
|
790
|
+
await flushRuntimeBatcher();
|
|
791
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
792
|
+
}));
|
|
793
|
+
const scopedLeaseLost = claudeLeaseRecoverable
|
|
794
|
+
? "claude"
|
|
795
|
+
: xaiLeaseRecoverable
|
|
796
|
+
? "xai"
|
|
797
|
+
: coreCodexLeaseRecoverable
|
|
798
|
+
? "codex"
|
|
799
|
+
: null;
|
|
800
|
+
if (scopedLeaseLost && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
801
|
+
if (scopedLeaseLost !== "codex") {
|
|
802
|
+
await flushRuntimeBatcher();
|
|
803
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
804
|
+
}
|
|
694
805
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
695
806
|
sessionId: input.sessionId,
|
|
696
807
|
turnId: attempt.turnId,
|
|
697
808
|
triggerEventId: attempt.triggerEventId!,
|
|
698
809
|
attemptId: input.attemptId,
|
|
699
|
-
reason: "
|
|
700
|
-
detail: {
|
|
810
|
+
reason: scopedLeaseLost + "_lease_lost",
|
|
811
|
+
detail: {
|
|
812
|
+
provider:
|
|
813
|
+
scopedLeaseLost === "claude"
|
|
814
|
+
? "claude-subscription"
|
|
815
|
+
: scopedLeaseLost === "codex"
|
|
816
|
+
? "codex-subscription"
|
|
817
|
+
: "supergrok-subscription",
|
|
818
|
+
},
|
|
701
819
|
});
|
|
702
820
|
if (recovery.action === "stale") {
|
|
703
821
|
acknowledgeLostAttemptOwnership();
|
|
@@ -718,631 +836,220 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
718
836
|
// 5xx does not classify here and therefore cannot consume another
|
|
719
837
|
// subscription or duplicate a side effect.
|
|
720
838
|
const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
839
|
+
// A definitive account refusal on a core turn becomes a typed terminal
|
|
840
|
+
// failure that idles the session (settled below).
|
|
841
|
+
let coreAccountRefusal: SubscriptionCoreCodexTurnError | null = null;
|
|
842
|
+
if (coreCodex && !coreRequestOutcomeUnknown && !coreLeaseCheckpointFailed) {
|
|
843
|
+
// Core turns record the refusal against the leased connection (failure
|
|
844
|
+
// receipt, plus the quota, health or model-cooldown state that keeps
|
|
845
|
+
// placement away from it) and re-place the same turn within the per-turn
|
|
846
|
+
// failover bound (M3 PR 2a). Without a recorded refusal or a durable
|
|
847
|
+
// checkpoint they settle through the typed paths below instead.
|
|
848
|
+
const coreFailure = classifyCodexCredentialFailure(error);
|
|
849
|
+
// Only explicit plan evidence is an entitlement refusal; an unexplained
|
|
850
|
+
// 400 keeps the generic typed "request rejected" copy.
|
|
851
|
+
const planEntitlement =
|
|
852
|
+
!coreFailure && classifyCodexEntitlementRejection(error)?.evidence === "plan_entitlement";
|
|
853
|
+
const refusal =
|
|
854
|
+
coreFailure ??
|
|
855
|
+
(planEntitlement ? { kind: "plan_entitlement" as const, cooldownSeconds: null } : null);
|
|
856
|
+
// Only a pre-dispatch lifecycle fence or a definite refusal can enter
|
|
857
|
+
// graceful recovery. A concurrent disconnect never upgrades a timeout,
|
|
858
|
+
// partial stream, missing response, or failed checkpoint into replay proof.
|
|
859
|
+
const disconnected =
|
|
860
|
+
hasErrorInCauseChain(error, SubscriptionCoreCodexSourceDisconnectedError) ||
|
|
861
|
+
(!!refusal &&
|
|
862
|
+
(await isSubscriptionCoreCodexSourceDisconnected(
|
|
863
|
+
db,
|
|
864
|
+
coreCodex.identity,
|
|
865
|
+
coreCodex.connectionId,
|
|
866
|
+
)));
|
|
867
|
+
if (disconnected) {
|
|
868
|
+
const settled = await failOverCoreCodexTurn(deps, coreCodex, "source_disconnected");
|
|
869
|
+
if (settled) return settled;
|
|
870
|
+
}
|
|
871
|
+
if (refusal && !disconnected) {
|
|
872
|
+
const recorded = await recordCoreCodexRefusal({
|
|
873
|
+
db,
|
|
874
|
+
core: coreCodex,
|
|
875
|
+
lease: leases.codex,
|
|
876
|
+
failure: refusal,
|
|
877
|
+
credentialVersion: providerTurn.effectiveCodexCredentialVersion,
|
|
878
|
+
modelId: providerTurn.codexProductModelId ?? null,
|
|
879
|
+
}).catch(() => ({ receipt: false, health: false }));
|
|
880
|
+
if (recorded.receipt && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
881
|
+
const settled = await failOverCoreCodexTurn(deps, coreCodex, refusal);
|
|
882
|
+
if (settled) return settled;
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
coreAccountRefusal =
|
|
886
|
+
disconnected || hasErrorInCauseChain(error, SubscriptionCoreCodexAccessLostError)
|
|
887
|
+
? subscriptionCoreAccountRefusedFailure("access_lost")
|
|
888
|
+
: refusal?.kind === "auth" || hasErrorInCauseChain(error, CodexReloginRequired)
|
|
889
|
+
? subscriptionCoreAccountRefusedFailure("sign_in")
|
|
890
|
+
: refusal?.kind === "forbidden"
|
|
891
|
+
? subscriptionCoreAccountRefusedFailure("forbidden")
|
|
892
|
+
: refusal?.kind === "plan_entitlement"
|
|
893
|
+
? subscriptionCoreAccountRefusedFailure("entitlement")
|
|
894
|
+
: null;
|
|
895
|
+
}
|
|
896
|
+
// A definitive SuperGrok account refusal on a core turn: recorded against
|
|
897
|
+
// the leased connection, then the same turn re-places within the shared
|
|
898
|
+
// bound, or fails with typed copy that idles the session (settled below).
|
|
899
|
+
let coreXaiRefusal: SubscriptionCoreCodexTurnError | null = null;
|
|
900
|
+
if (coreXai && !coreRequestOutcomeUnknown && !leases.xai.lost) {
|
|
901
|
+
const settled = await settleCoreSubscriptionRefusal(deps, coreXai, XAI_CORE_REFUSAL);
|
|
902
|
+
if (settled.result) return settled.result;
|
|
903
|
+
coreXaiRefusal = settled.failure;
|
|
904
|
+
}
|
|
905
|
+
// The same for a Claude core turn: a sign-in refusal (after the transport's
|
|
906
|
+
// one forced refresh) or a model rate limit re-places the same turn; 529
|
|
907
|
+
// stays in the overload lane below and 403 fails the turn, neither moving
|
|
908
|
+
// it to another account. The bound counts the core's turn-failure
|
|
909
|
+
// receipts (the legacy `claudeAuthRecovery` marker is never read here).
|
|
910
|
+
let coreClaudeRefusal: SubscriptionCoreCodexTurnError | null = null;
|
|
911
|
+
if (coreClaude && !coreRequestOutcomeUnknown && !leases.claude.lost) {
|
|
912
|
+
const settled = await settleCoreSubscriptionRefusal(
|
|
913
|
+
deps,
|
|
914
|
+
coreClaude,
|
|
915
|
+
claudeCoreRefusal(coreClaude.productModelId),
|
|
916
|
+
);
|
|
917
|
+
if (settled.result) return settled.result;
|
|
918
|
+
coreClaudeRefusal = settled.failure;
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// The leased credential path above normally quarantines quota state and
|
|
922
|
+
// either recovers the same turn or arms a durable capacity wait. This narrow
|
|
923
|
+
// fallback covers failures before a credential lease existed, or a failed
|
|
924
|
+
// durable checkpoint where replay would be unsafe. Keep the session usable,
|
|
925
|
+
// but never synthesize another turn or walk an unfenced legacy pointer.
|
|
926
|
+
// A lease-busy chain that outlived its bound becomes a terminal failure.
|
|
927
|
+
const leaseBusyChain =
|
|
928
|
+
error instanceof SubscriptionCoreCodexTurnError &&
|
|
929
|
+
error.payload.code === "subscription_lease_busy" &&
|
|
930
|
+
error.payload.retryable
|
|
931
|
+
? subscriptionCoreLeaseBusyChain(
|
|
932
|
+
attempt.subscriptionLeaseBusy,
|
|
933
|
+
attempt.executionGeneration,
|
|
934
|
+
Date.now(),
|
|
935
|
+
)
|
|
724
936
|
: null;
|
|
937
|
+
// An older attempt of this turn still holds its core lease. Retry right
|
|
938
|
+
// after that lease can expire, outside the provider recovery budget: this
|
|
939
|
+
// is ownership handover, not provider backpressure. The chain is bounded
|
|
940
|
+
// above (SUBSCRIPTION_CORE_LEASE_BUSY_MAX_MS).
|
|
941
|
+
let leaseBusyNotRecoverable = false;
|
|
725
942
|
if (
|
|
726
|
-
|
|
727
|
-
|
|
943
|
+
leaseBusyChain &&
|
|
944
|
+
!leaseBusyChain.exhausted &&
|
|
945
|
+
error instanceof SubscriptionCoreCodexTurnError &&
|
|
728
946
|
eventing.publish &&
|
|
729
947
|
attempt.turnId &&
|
|
730
948
|
eventing.turnStartedPublished
|
|
731
949
|
) {
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
outcome: "classified",
|
|
739
|
-
},
|
|
740
|
-
});
|
|
741
|
-
const failoverStartedAt = performance.now();
|
|
742
|
-
let checkpointDurable = false;
|
|
950
|
+
const continueDelayMs = subscriptionCoreLeaseBusyDelayMs(
|
|
951
|
+
error.payload.resetsAt,
|
|
952
|
+
Date.now(),
|
|
953
|
+
Math.random(),
|
|
954
|
+
);
|
|
955
|
+
let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
|
|
743
956
|
try {
|
|
744
957
|
await flushRuntimeBatcher();
|
|
745
958
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
959
|
+
recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
960
|
+
sessionId: input.sessionId,
|
|
961
|
+
turnId: attempt.turnId,
|
|
962
|
+
triggerEventId: attempt.triggerEventId!,
|
|
963
|
+
attemptId: input.attemptId,
|
|
964
|
+
reason: "subscription_lease_busy",
|
|
965
|
+
subscriptionLeaseBusy: {
|
|
966
|
+
startedAt: new Date(leaseBusyChain.startedAt).toISOString(),
|
|
967
|
+
executionGeneration: leaseBusyChain.executionGeneration,
|
|
968
|
+
},
|
|
969
|
+
detail: { ...error.payload, continueDelayMs },
|
|
757
970
|
});
|
|
758
|
-
}
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
971
|
+
} catch (recoveryError) {
|
|
972
|
+
// Keep the turn recoverable across a transient database failure, as
|
|
973
|
+
// the provider recovery path does.
|
|
974
|
+
const postClaimRecovery = postClaimDatabaseRecoveryFailure({
|
|
975
|
+
error: recoveryError,
|
|
976
|
+
turnId: attempt.turnId,
|
|
977
|
+
triggerEventId: attempt.triggerEventId!,
|
|
978
|
+
executionGeneration: attempt.executionGeneration,
|
|
765
979
|
});
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
(account) => account.id === providerTurn.effectiveCodexCredentialId,
|
|
772
|
-
);
|
|
773
|
-
const usageSnapshot = providerTurn.latestCodexUsage as CodexUsageHeaderSnapshot | null;
|
|
774
|
-
const serving = servingCached
|
|
775
|
-
? {
|
|
776
|
-
...servingCached,
|
|
777
|
-
...(usageSnapshot
|
|
778
|
-
? {
|
|
779
|
-
primaryUsedPercent: usageSnapshot.primaryUsedPercent,
|
|
780
|
-
primaryResetAt: usageSnapshot.primaryResetAt,
|
|
781
|
-
secondaryUsedPercent: usageSnapshot.secondaryUsedPercent,
|
|
782
|
-
secondaryResetAt: usageSnapshot.secondaryResetAt,
|
|
783
|
-
}
|
|
784
|
-
: {}),
|
|
785
|
-
}
|
|
786
|
-
: null;
|
|
787
|
-
const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
|
|
788
|
-
const quarantineResult =
|
|
789
|
-
leases.codex.holderId &&
|
|
790
|
-
leases.codex.generation !== null &&
|
|
791
|
-
providerTurn.effectiveCodexCredentialVersion !== null
|
|
792
|
-
? await quarantineCodexCredentialForLease(db, {
|
|
793
|
-
accountId: input.accountId,
|
|
794
|
-
workspaceId: input.workspaceId,
|
|
795
|
-
sessionId: input.sessionId,
|
|
796
|
-
turnId: attempt.turnId,
|
|
797
|
-
attemptId: input.attemptId,
|
|
798
|
-
executionGeneration: attempt.executionGeneration,
|
|
799
|
-
workflowId: input.workflowId,
|
|
800
|
-
workflowRunId: input.workflowRunId,
|
|
801
|
-
dispatchId: attempt.dispatchId,
|
|
802
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
803
|
-
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
804
|
-
credentialVersion: providerTurn.effectiveCodexCredentialVersion,
|
|
805
|
-
holderId: leases.codex.holderId,
|
|
806
|
-
generation: leases.codex.generation,
|
|
807
|
-
maxFailovers: providerTurn.codexCredentialFailoverLimit,
|
|
808
|
-
quarantine:
|
|
809
|
-
codexCredentialFailure.kind === "auth"
|
|
810
|
-
? {
|
|
811
|
-
kind: "status",
|
|
812
|
-
status: "needs_relogin",
|
|
813
|
-
lastError: "model request remained unauthorized after refresh",
|
|
814
|
-
}
|
|
815
|
-
: codexCredentialFailure.kind === "forbidden"
|
|
816
|
-
? {
|
|
817
|
-
kind: "status",
|
|
818
|
-
status: "error",
|
|
819
|
-
lastError: "model request was forbidden for this credential",
|
|
820
|
-
}
|
|
821
|
-
: {
|
|
822
|
-
kind: "cooldown",
|
|
823
|
-
until: cooldownUntil!,
|
|
824
|
-
cooldownKind: codexCredentialFailure.kind,
|
|
825
|
-
},
|
|
826
|
-
})
|
|
827
|
-
: null;
|
|
828
|
-
if (
|
|
829
|
-
quarantineResult?.action === "credential_changed" &&
|
|
830
|
-
leases.codex.holderId &&
|
|
831
|
-
leases.codex.generation !== null
|
|
832
|
-
) {
|
|
833
|
-
const recovery = await settleCodexCredentialLeaseLoss(db, {
|
|
834
|
-
accountId: input.accountId,
|
|
835
|
-
workspaceId: input.workspaceId,
|
|
836
|
-
sessionId: input.sessionId,
|
|
837
|
-
turnId: attempt.turnId,
|
|
838
|
-
attemptId: input.attemptId,
|
|
839
|
-
holderId: leases.codex.holderId,
|
|
840
|
-
generation: leases.codex.generation,
|
|
841
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
842
|
-
checkpointDurable: true,
|
|
843
|
-
recoveryPayload: {
|
|
844
|
-
triggerEventId: attempt.triggerEventId!,
|
|
845
|
-
reason: "codex_credential_version_changed",
|
|
846
|
-
},
|
|
847
|
-
failedPayload: {},
|
|
848
|
-
});
|
|
849
|
-
if (recovery.action === "recovering") {
|
|
850
|
-
leases.codex.held = false;
|
|
851
|
-
await publishDurableSessionEvents(
|
|
852
|
-
bus,
|
|
853
|
-
input.workspaceId,
|
|
854
|
-
input.sessionId,
|
|
855
|
-
recovery.events,
|
|
856
|
-
);
|
|
857
|
-
control.activityStatus = "recovering";
|
|
858
|
-
control.turnMetricOutcome = "recovering";
|
|
859
|
-
return claimedResult({ status: "recovering" });
|
|
860
|
-
}
|
|
861
|
-
acknowledgeLostAttemptOwnership();
|
|
862
|
-
control.activityStatus = "cancelled";
|
|
863
|
-
control.turnMetricOutcome = "cancelled";
|
|
864
|
-
return claimedResult({ status: "cancelled" });
|
|
865
|
-
}
|
|
866
|
-
const statePersisted = quarantineResult?.action === "recorded";
|
|
867
|
-
if (!statePersisted && leases.codex.holderId && leases.codex.generation !== null) {
|
|
868
|
-
leases.codex.lost = true;
|
|
869
|
-
return await settleLostCodexAttempt(
|
|
870
|
-
attempt.turnId,
|
|
871
|
-
leases.codex.holderId,
|
|
872
|
-
leases.codex.generation,
|
|
873
|
-
true,
|
|
874
|
-
);
|
|
875
|
-
}
|
|
876
|
-
if (
|
|
877
|
-
quarantineResult?.action === "recorded" &&
|
|
878
|
-
quarantineResult.exhausted &&
|
|
879
|
-
leases.codex.holderId &&
|
|
880
|
-
leases.codex.generation !== null
|
|
881
|
-
) {
|
|
882
|
-
const settlement = await settleCodexCredentialFailover(db, {
|
|
883
|
-
accountId: input.accountId,
|
|
884
|
-
workspaceId: input.workspaceId,
|
|
885
|
-
sessionId: input.sessionId,
|
|
886
|
-
turnId: attempt.turnId,
|
|
887
|
-
attemptId: input.attemptId,
|
|
888
|
-
holderId: leases.codex.holderId,
|
|
889
|
-
generation: leases.codex.generation,
|
|
890
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
891
|
-
maxFailovers: quarantineResult.maxFailovers,
|
|
892
|
-
recoveryPayload: {
|
|
893
|
-
triggerEventId: attempt.triggerEventId!,
|
|
894
|
-
reason: "codex_credential_failover",
|
|
895
|
-
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
896
|
-
failureKind: codexCredentialFailure.kind,
|
|
897
|
-
},
|
|
898
|
-
failedPayload: {
|
|
899
|
-
error:
|
|
900
|
-
"Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
901
|
-
code: "codex_credential_failover_exhausted",
|
|
902
|
-
retryable: false,
|
|
903
|
-
recovery: "user_message",
|
|
904
|
-
failoverCount: quarantineResult.failoverCount,
|
|
905
|
-
maxFailovers: quarantineResult.maxFailovers,
|
|
906
|
-
},
|
|
907
|
-
});
|
|
908
|
-
if (settlement.action === "limit_exceeded") {
|
|
909
|
-
leases.codex.held = false;
|
|
910
|
-
await publishDurableSessionEvents(
|
|
911
|
-
bus,
|
|
912
|
-
input.workspaceId,
|
|
913
|
-
input.sessionId,
|
|
914
|
-
settlement.events,
|
|
915
|
-
);
|
|
916
|
-
control.activityError = error;
|
|
917
|
-
control.activityStatus = "idle";
|
|
918
|
-
control.turnMetricOutcome = "failed";
|
|
919
|
-
await deliverFailedChildTurnToParent(
|
|
920
|
-
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
921
|
-
input.workspaceId,
|
|
922
|
-
input.sessionId,
|
|
923
|
-
attempt.turnId,
|
|
924
|
-
);
|
|
925
|
-
return claimedResult({ status: "idle" });
|
|
926
|
-
}
|
|
927
|
-
if (settlement.action === "stale") {
|
|
928
|
-
acknowledgeLostAttemptOwnership();
|
|
929
|
-
control.activityStatus = "cancelled";
|
|
930
|
-
control.turnMetricOutcome = "cancelled";
|
|
931
|
-
return claimedResult({ status: "cancelled" });
|
|
932
|
-
}
|
|
933
|
-
throw new Error("Exhausted Codex failover receipt unexpectedly recovered");
|
|
934
|
-
}
|
|
935
|
-
let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
|
|
936
|
-
try {
|
|
937
|
-
accounts = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId);
|
|
938
|
-
} catch (metadataError) {
|
|
939
|
-
// Current account health/cooldown metadata is still required after
|
|
940
|
-
// quarantine. Operational database failures re-enter the existing
|
|
941
|
-
// exact-attempt recovery lane; no policy choice is made from partial
|
|
942
|
-
// account state.
|
|
943
|
-
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
944
|
-
error: metadataError,
|
|
945
|
-
turnId: attempt.turnId,
|
|
946
|
-
triggerEventId: attempt.triggerEventId!,
|
|
947
|
-
executionGeneration: attempt.executionGeneration,
|
|
948
|
-
});
|
|
949
|
-
if (recoveryFailure) {
|
|
950
|
-
control.activityStatus = "recovering";
|
|
951
|
-
control.turnMetricOutcome = "recovering";
|
|
952
|
-
control.activityError = metadataError;
|
|
953
|
-
throw recoveryFailure;
|
|
954
|
-
}
|
|
955
|
-
throw metadataError;
|
|
956
|
-
}
|
|
957
|
-
const acceptedPolicy = acceptedCodexPolicySnapshot(providerTurn);
|
|
958
|
-
const selected = selectCodexCredentialLeaseForTurn({
|
|
959
|
-
context: {
|
|
960
|
-
accounts: codexLeaseAccountsForSelection(accounts),
|
|
961
|
-
activeCredentialId: acceptedPolicy.activeCredentialId,
|
|
962
|
-
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
963
|
-
rotationStrategy: acceptedPolicy.rotationStrategy,
|
|
964
|
-
existingCredentialId: null,
|
|
965
|
-
failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
|
|
966
|
-
policyScope: null,
|
|
967
|
-
unavailableDiagnostics: [],
|
|
968
|
-
},
|
|
969
|
-
sessionId: input.sessionId,
|
|
970
|
-
sessionPinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
971
|
-
sessionPinSource: acceptedPolicy.pinSource,
|
|
972
|
-
sessionLastCredentialId: acceptedPolicy.lastCredentialId,
|
|
973
|
-
now,
|
|
974
|
-
});
|
|
975
|
-
const decisionKind =
|
|
976
|
-
selected.decision.kind === "allocatorDisabled" ? "allCapped" : selected.decision.kind;
|
|
977
|
-
const pinDisposition = classifyCodexPin({
|
|
978
|
-
pinnedCredentialId: acceptedPolicy.pinnedCredentialId,
|
|
979
|
-
pinSource: acceptedPolicy.pinSource,
|
|
980
|
-
strategy: acceptedPolicy.rotationStrategy as CodexRotationStrategy,
|
|
981
|
-
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
982
|
-
});
|
|
983
|
-
const failureDisposition = codexDefinitiveFailureDisposition({
|
|
984
|
-
failureKind: codexCredentialFailure.kind,
|
|
985
|
-
rotationEnabled: acceptedPolicy.rotationEnabled,
|
|
986
|
-
pinDisposition,
|
|
987
|
-
decisionKind,
|
|
988
|
-
decisionCredentialId: selected.decision.kind === "active" ? selected.credentialId : null,
|
|
989
|
-
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
990
|
-
});
|
|
991
|
-
const maxFailovers = providerTurn.codexCredentialFailoverLimit;
|
|
992
|
-
|
|
993
|
-
if (
|
|
994
|
-
statePersisted &&
|
|
995
|
-
failureDisposition === "failover" &&
|
|
996
|
-
leases.codex.holderId &&
|
|
997
|
-
leases.codex.generation !== null
|
|
998
|
-
) {
|
|
999
|
-
const settlement = await settleCodexCredentialFailover(db, {
|
|
1000
|
-
accountId: input.accountId,
|
|
1001
|
-
workspaceId: input.workspaceId,
|
|
1002
|
-
sessionId: input.sessionId,
|
|
1003
|
-
turnId: attempt.turnId,
|
|
1004
|
-
attemptId: input.attemptId,
|
|
1005
|
-
holderId: leases.codex.holderId,
|
|
1006
|
-
generation: leases.codex.generation,
|
|
1007
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
1008
|
-
maxFailovers,
|
|
1009
|
-
recoveryPayload: {
|
|
1010
|
-
triggerEventId: attempt.triggerEventId!,
|
|
1011
|
-
reason: "codex_credential_failover",
|
|
1012
|
-
credentialId: providerTurn.effectiveCodexCredentialId,
|
|
1013
|
-
failureKind: codexCredentialFailure.kind,
|
|
1014
|
-
...(cooldownUntil ? { cooldownUntil: cooldownUntil.toISOString() } : {}),
|
|
1015
|
-
},
|
|
1016
|
-
failedPayload: {
|
|
1017
|
-
error:
|
|
1018
|
-
"Automatic Codex credential failover stopped after every bounded account attempt was consumed. Send a new message after checking account health or capacity.",
|
|
1019
|
-
code: "codex_credential_failover_exhausted",
|
|
1020
|
-
retryable: false,
|
|
1021
|
-
recovery: "user_message",
|
|
1022
|
-
failoverCount: quarantineResult.failoverCount,
|
|
1023
|
-
maxFailovers: quarantineResult.maxFailovers,
|
|
1024
|
-
},
|
|
1025
|
-
});
|
|
1026
|
-
observability.incrementCounter({
|
|
1027
|
-
name: "opengeni_codex_failover_settlements_total",
|
|
1028
|
-
help: "Atomic Codex failover settlements by outcome.",
|
|
1029
|
-
labels: {
|
|
1030
|
-
workspace_key: codexWorkspaceKey,
|
|
1031
|
-
outcome: settlement.action,
|
|
1032
|
-
},
|
|
1033
|
-
});
|
|
1034
|
-
if (settlement.action === "recovering") {
|
|
1035
|
-
leases.codex.held = false;
|
|
1036
|
-
await publishDurableSessionEvents(
|
|
1037
|
-
bus,
|
|
1038
|
-
input.workspaceId,
|
|
1039
|
-
input.sessionId,
|
|
1040
|
-
settlement.events,
|
|
1041
|
-
);
|
|
1042
|
-
observability.observeHistogram({
|
|
1043
|
-
name: "opengeni_codex_failover_recovery_seconds",
|
|
1044
|
-
help: "Time from credential refusal to durable same-turn recovery.",
|
|
1045
|
-
labels: {
|
|
1046
|
-
workspace_key: codexWorkspaceKey,
|
|
1047
|
-
kind: codexCredentialFailure.kind,
|
|
1048
|
-
},
|
|
1049
|
-
value: Math.max(0, (performance.now() - failoverStartedAt) / 1000),
|
|
1050
|
-
});
|
|
1051
|
-
control.activityStatus = "recovering";
|
|
1052
|
-
control.turnMetricOutcome = "recovering";
|
|
1053
|
-
return claimedResult({ status: "recovering" });
|
|
1054
|
-
}
|
|
1055
|
-
if (settlement.action === "stale") {
|
|
1056
|
-
// One transaction proves both exact-holder recovery (including a
|
|
1057
|
-
// just-expired or reaped lease row) and successor/control-gate
|
|
1058
|
-
// rejection. Cross the hard tool fence so a control-gate loss can
|
|
1059
|
-
// write its quiescence receipt; a successor-only loss is a no-op.
|
|
1060
|
-
acknowledgeLostAttemptOwnership();
|
|
1061
|
-
control.activityStatus = "cancelled";
|
|
1062
|
-
control.turnMetricOutcome = "cancelled";
|
|
1063
|
-
return claimedResult({ status: "cancelled" });
|
|
1064
|
-
}
|
|
1065
|
-
if (settlement.action === "limit_exceeded") {
|
|
1066
|
-
leases.codex.held = false;
|
|
1067
|
-
await publishDurableSessionEvents(
|
|
1068
|
-
bus,
|
|
1069
|
-
input.workspaceId,
|
|
1070
|
-
input.sessionId,
|
|
1071
|
-
settlement.events,
|
|
1072
|
-
);
|
|
1073
|
-
control.activityError = error;
|
|
1074
|
-
control.activityStatus = "idle";
|
|
1075
|
-
control.turnMetricOutcome = "failed";
|
|
1076
|
-
await deliverFailedChildTurnToParent(
|
|
1077
|
-
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
1078
|
-
input.workspaceId,
|
|
1079
|
-
input.sessionId,
|
|
1080
|
-
attempt.turnId,
|
|
1081
|
-
);
|
|
1082
|
-
return claimedResult({ status: "idle" });
|
|
1083
|
-
}
|
|
1084
|
-
}
|
|
1085
|
-
|
|
1086
|
-
if (
|
|
1087
|
-
statePersisted &&
|
|
1088
|
-
failureDisposition === "wait" &&
|
|
1089
|
-
leases.codex.holderId &&
|
|
1090
|
-
leases.codex.generation !== null
|
|
1091
|
-
) {
|
|
1092
|
-
let goal: Awaited<ReturnType<typeof getSessionGoal>>;
|
|
1093
|
-
try {
|
|
1094
|
-
goal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
1095
|
-
} catch (metadataError) {
|
|
1096
|
-
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
1097
|
-
error: metadataError,
|
|
1098
|
-
turnId: attempt.turnId,
|
|
1099
|
-
triggerEventId: attempt.triggerEventId!,
|
|
1100
|
-
executionGeneration: attempt.executionGeneration,
|
|
1101
|
-
});
|
|
1102
|
-
if (recoveryFailure) {
|
|
1103
|
-
control.activityStatus = "recovering";
|
|
1104
|
-
control.turnMetricOutcome = "recovering";
|
|
1105
|
-
control.activityError = metadataError;
|
|
1106
|
-
throw recoveryFailure;
|
|
1107
|
-
}
|
|
1108
|
-
throw metadataError;
|
|
1109
|
-
}
|
|
1110
|
-
const activeGoal = goal?.status === "active" ? goal : null;
|
|
1111
|
-
const exactProviderReset =
|
|
1112
|
-
codexCredentialFailure.cooldownSeconds !== null &&
|
|
1113
|
-
Number.isFinite(codexCredentialFailure.cooldownSeconds) &&
|
|
1114
|
-
codexCredentialFailure.cooldownSeconds > 0;
|
|
1115
|
-
const policyCredentialId =
|
|
1116
|
-
pinDisposition === "manual" && acceptedPolicy.pinnedCredentialId
|
|
1117
|
-
? acceptedPolicy.pinnedCredentialId
|
|
1118
|
-
: !acceptedPolicy.rotationEnabled
|
|
1119
|
-
? acceptedPolicy.activeCredentialId
|
|
1120
|
-
: null;
|
|
1121
|
-
const capacityAccounts = policyCredentialId
|
|
1122
|
-
? accounts.filter((account) => account.id === policyCredentialId)
|
|
1123
|
-
: accounts;
|
|
1124
|
-
const authoritativeResetAt = exactProviderReset
|
|
1125
|
-
? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
|
|
1126
|
-
: null;
|
|
1127
|
-
const allAccounts =
|
|
1128
|
-
acceptedPolicy.rotationEnabled &&
|
|
1129
|
-
pinDisposition !== "manual" &&
|
|
1130
|
-
decisionKind === "allCapped";
|
|
1131
|
-
const failurePayload = codexCapacityWaitFailurePayload({
|
|
1132
|
-
failureKind: codexCredentialFailure.kind,
|
|
1133
|
-
usageLimit,
|
|
1134
|
-
cooldownSeconds: codexCredentialFailure.cooldownSeconds,
|
|
1135
|
-
detail:
|
|
1136
|
-
codexCredentialFailure.kind === "quota"
|
|
1137
|
-
? error instanceof Error
|
|
1138
|
-
? error.message
|
|
1139
|
-
: String(error)
|
|
1140
|
-
: "the same accepted turn is waiting for eligible credential capacity",
|
|
1141
|
-
allAccounts,
|
|
1142
|
-
});
|
|
1143
|
-
const evaluated = await armAndReconcileCodexCapacityWait(
|
|
1144
|
-
{ db, bus },
|
|
1145
|
-
{
|
|
1146
|
-
accountId: input.accountId,
|
|
1147
|
-
workspaceId: input.workspaceId,
|
|
1148
|
-
sessionId: input.sessionId,
|
|
1149
|
-
turnId: attempt.turnId,
|
|
1150
|
-
attemptId: input.attemptId,
|
|
1151
|
-
workflowId: input.workflowId,
|
|
1152
|
-
goalId: activeGoal?.id ?? null,
|
|
1153
|
-
goalVersion: activeGoal?.version ?? null,
|
|
1154
|
-
earliestResetAt: authoritativeResetAt,
|
|
1155
|
-
resetKind: authoritativeResetAt
|
|
1156
|
-
? "authoritative"
|
|
1157
|
-
: codexCredentialFailure.kind === "auth" ||
|
|
1158
|
-
codexCredentialFailure.kind === "forbidden"
|
|
1159
|
-
? "mutation_only"
|
|
1160
|
-
: "bounded_refresh",
|
|
1161
|
-
failurePayload,
|
|
1162
|
-
leaseFence: {
|
|
1163
|
-
holderId: leases.codex.holderId,
|
|
1164
|
-
generation: leases.codex.generation,
|
|
1165
|
-
},
|
|
1166
|
-
expectedRedispatches: attempt.redispatchesAtDispatch,
|
|
1167
|
-
},
|
|
1168
|
-
{ onArmed: () => (leases.codex.held = false) },
|
|
1169
|
-
);
|
|
1170
|
-
control.activityError = error;
|
|
1171
|
-
if (evaluated.action === "stopped") {
|
|
1172
|
-
control.activityStatus = evaluated.sessionStatus === "queued" ? "idle" : "failed";
|
|
1173
|
-
control.turnMetricOutcome = "failed";
|
|
1174
|
-
return claimedResult({ status: control.activityStatus });
|
|
1175
|
-
}
|
|
1176
|
-
if (evaluated.action === "resumed") {
|
|
1177
|
-
control.activityStatus = "recovering";
|
|
1178
|
-
control.turnMetricOutcome = "recovering";
|
|
1179
|
-
return claimedResult({ status: "recovering" });
|
|
1180
|
-
}
|
|
1181
|
-
if (evaluated.action === "waiting") {
|
|
1182
|
-
control.activityError = error;
|
|
1183
|
-
control.activityStatus = "waiting_capacity";
|
|
1184
|
-
control.turnMetricOutcome = "recovering";
|
|
1185
|
-
return claimedResult({
|
|
1186
|
-
status: "waiting_capacity",
|
|
1187
|
-
capacityWait: {
|
|
1188
|
-
waiterId: evaluated.waiter.id,
|
|
1189
|
-
generation: evaluated.waiter.generation,
|
|
1190
|
-
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
1191
|
-
wakeRevision: evaluated.waiter.wakeRevision,
|
|
1192
|
-
},
|
|
1193
|
-
});
|
|
1194
|
-
}
|
|
1195
|
-
acknowledgeLostAttemptOwnership();
|
|
1196
|
-
control.activityStatus = "cancelled";
|
|
1197
|
-
control.turnMetricOutcome = "cancelled";
|
|
1198
|
-
return claimedResult({ status: "cancelled" });
|
|
980
|
+
if (postClaimRecovery) {
|
|
981
|
+
control.activityStatus = "recovering";
|
|
982
|
+
control.turnMetricOutcome = "recovering";
|
|
983
|
+
control.activityError = error;
|
|
984
|
+
throw postClaimRecovery;
|
|
1199
985
|
}
|
|
986
|
+
throw recoveryError;
|
|
987
|
+
}
|
|
988
|
+
if (recovery.action === "stale") {
|
|
989
|
+
acknowledgeLostAttemptOwnership();
|
|
990
|
+
control.activityStatus = "cancelled";
|
|
991
|
+
control.turnMetricOutcome = "cancelled";
|
|
992
|
+
return claimedResult({ status: "cancelled" });
|
|
1200
993
|
}
|
|
994
|
+
if (recovery.action === "recovering") {
|
|
995
|
+
acknowledgeRecoveryQuiescence();
|
|
996
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
997
|
+
control.turnMetricOutcome = "recovering";
|
|
998
|
+
control.activityStatus = "recovering";
|
|
999
|
+
control.activityError = error;
|
|
1000
|
+
return claimedResult({ status: "recovering", continueDelayMs });
|
|
1001
|
+
}
|
|
1002
|
+
leaseBusyNotRecoverable = true;
|
|
1201
1003
|
}
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1004
|
+
// A shared-core placement outcome (no capacity, a waiting explicit choice,
|
|
1005
|
+
// a paused cutover, or a consumer not yet on the core) or a definitive
|
|
1006
|
+
// refusal from the leased account fails this turn with its typed copy and
|
|
1007
|
+
// keeps the session usable for the next message.
|
|
1008
|
+
const coreTurnFailure =
|
|
1009
|
+
leaseBusyChain?.exhausted || leaseBusyNotRecoverable
|
|
1010
|
+
? // The chain exists only for a lease-busy core error: keep its provider.
|
|
1011
|
+
subscriptionCoreLeaseBusyExhaustedFailure(
|
|
1012
|
+
error instanceof SubscriptionCoreCodexTurnError ? error.displayName : undefined,
|
|
1013
|
+
)
|
|
1014
|
+
: error instanceof SubscriptionCoreCodexTurnError
|
|
1015
|
+
? error
|
|
1016
|
+
: (coreAccountRefusal ?? coreXaiRefusal ?? coreClaudeRefusal);
|
|
1206
1017
|
if (
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
providerTurn.xaiAuthoritySnapshot &&
|
|
1210
|
-
leases.xai.subjectId &&
|
|
1211
|
-
leases.xai.holderId &&
|
|
1212
|
-
leases.xai.generation !== null &&
|
|
1018
|
+
coreTurnFailure &&
|
|
1019
|
+
!coreTurnFailure.payload.retryable &&
|
|
1213
1020
|
eventing.publish &&
|
|
1214
1021
|
attempt.turnId &&
|
|
1215
1022
|
eventing.turnStartedPublished
|
|
1216
1023
|
) {
|
|
1217
1024
|
await flushRuntimeBatcher();
|
|
1218
|
-
await historySink.reconcileConversationTruth(
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
: null;
|
|
1226
|
-
const failurePayload = {
|
|
1227
|
-
error:
|
|
1228
|
-
xaiCredentialFailure.kind === "auth"
|
|
1229
|
-
? "The serving SuperGrok account requires reconnection"
|
|
1230
|
-
: xaiCredentialFailure.kind === "forbidden"
|
|
1231
|
-
? "The serving SuperGrok account is not authorized for this request"
|
|
1232
|
-
: "The serving SuperGrok account is temporarily rate limited",
|
|
1233
|
-
code:
|
|
1234
|
-
xaiCredentialFailure.kind === "auth"
|
|
1235
|
-
? "xai_relogin_required"
|
|
1236
|
-
: xaiCredentialFailure.kind === "forbidden"
|
|
1237
|
-
? "xai_account_forbidden"
|
|
1238
|
-
: "xai_account_rate_limited",
|
|
1239
|
-
detail: "the same accepted turn is waiting for another eligible account",
|
|
1240
|
-
};
|
|
1241
|
-
const armed = await armXaiCapacityWait(db, {
|
|
1242
|
-
accountId: input.accountId,
|
|
1243
|
-
workspaceId: input.workspaceId,
|
|
1244
|
-
subjectId: leases.xai.subjectId,
|
|
1245
|
-
sessionId: input.sessionId,
|
|
1246
|
-
turnId: attempt.turnId,
|
|
1247
|
-
attemptId: input.attemptId,
|
|
1248
|
-
workflowId: input.workflowId,
|
|
1249
|
-
authoritySnapshot: providerTurn.xaiAuthoritySnapshot,
|
|
1250
|
-
goalId: activeGoal?.id ?? null,
|
|
1251
|
-
goalVersion: activeGoal?.version ?? null,
|
|
1252
|
-
earliestResetAt: cooldownUntil,
|
|
1253
|
-
failurePayload,
|
|
1254
|
-
leaseFence: {
|
|
1255
|
-
holderId: leases.xai.holderId,
|
|
1256
|
-
generation: leases.xai.generation,
|
|
1257
|
-
},
|
|
1258
|
-
credentialQuarantine:
|
|
1259
|
-
xaiCredentialFailure.kind === "auth"
|
|
1260
|
-
? {
|
|
1261
|
-
kind: "status",
|
|
1262
|
-
status: "needs_relogin",
|
|
1263
|
-
lastError: "model request remained unauthorized after refresh",
|
|
1264
|
-
}
|
|
1265
|
-
: xaiCredentialFailure.kind === "forbidden"
|
|
1266
|
-
? {
|
|
1267
|
-
kind: "status",
|
|
1268
|
-
status: "error",
|
|
1269
|
-
lastError: "model request was forbidden for this credential",
|
|
1270
|
-
}
|
|
1271
|
-
: { kind: "cooldown", until: cooldownUntil! },
|
|
1272
|
-
now,
|
|
1273
|
-
});
|
|
1274
|
-
if (armed.action === "waiting") {
|
|
1275
|
-
leases.xai.held = false;
|
|
1276
|
-
providerTurn.xaiCredentialQuarantined = true;
|
|
1277
|
-
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
|
|
1278
|
-
const evaluated = await reconcileXaiCapacityWait(db, {
|
|
1279
|
-
accountId: input.accountId,
|
|
1280
|
-
workspaceId: input.workspaceId,
|
|
1281
|
-
sessionId: input.sessionId,
|
|
1282
|
-
waiterId: armed.waiter.id,
|
|
1283
|
-
generation: armed.waiter.generation,
|
|
1284
|
-
now,
|
|
1285
|
-
});
|
|
1286
|
-
if (evaluated.events.length > 0) {
|
|
1287
|
-
await publishDurableSessionEvents(
|
|
1288
|
-
bus,
|
|
1289
|
-
input.workspaceId,
|
|
1290
|
-
input.sessionId,
|
|
1291
|
-
evaluated.events,
|
|
1292
|
-
);
|
|
1293
|
-
}
|
|
1294
|
-
control.activityError = error;
|
|
1295
|
-
if (evaluated.action === "resumed") {
|
|
1296
|
-
control.activityStatus = "recovering";
|
|
1297
|
-
control.turnMetricOutcome = "recovering";
|
|
1298
|
-
return claimedResult({ status: "recovering" });
|
|
1299
|
-
}
|
|
1300
|
-
if (evaluated.action === "waiting") {
|
|
1301
|
-
control.activityStatus = "waiting_capacity";
|
|
1302
|
-
control.turnMetricOutcome = "recovering";
|
|
1303
|
-
return claimedResult({
|
|
1304
|
-
status: "waiting_capacity",
|
|
1305
|
-
capacityWait: {
|
|
1306
|
-
provider: "xai",
|
|
1307
|
-
waiterId: evaluated.waiter.id,
|
|
1308
|
-
generation: evaluated.waiter.generation,
|
|
1309
|
-
nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
|
|
1310
|
-
wakeRevision: evaluated.waiter.wakeRevision,
|
|
1025
|
+
await historySink.reconcileConversationTruth();
|
|
1026
|
+
if (
|
|
1027
|
+
!(await eventing.settle!({
|
|
1028
|
+
events: [
|
|
1029
|
+
{
|
|
1030
|
+
type: "turn.failed",
|
|
1031
|
+
payload: { ...coreTurnFailure.payload, recovery: "user_message" },
|
|
1311
1032
|
},
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
1321
|
-
sessionId: input.sessionId,
|
|
1322
|
-
turnId: attempt.turnId,
|
|
1323
|
-
triggerEventId: attempt.triggerEventId!,
|
|
1324
|
-
attemptId: input.attemptId,
|
|
1325
|
-
reason: "xai_credential_recheck",
|
|
1326
|
-
detail: failurePayload,
|
|
1327
|
-
});
|
|
1328
|
-
if (recovery.action === "stale") {
|
|
1329
|
-
acknowledgeLostAttemptOwnership();
|
|
1330
|
-
control.activityStatus = "cancelled";
|
|
1331
|
-
control.turnMetricOutcome = "cancelled";
|
|
1033
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1034
|
+
],
|
|
1035
|
+
turnStatus: "failed",
|
|
1036
|
+
sessionStatus: "idle",
|
|
1037
|
+
activeTurnId: null,
|
|
1038
|
+
}))
|
|
1039
|
+
) {
|
|
1332
1040
|
return claimedResult({ status: "cancelled" });
|
|
1333
1041
|
}
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
control.activityStatus = "recovering";
|
|
1337
|
-
control.turnMetricOutcome = "recovering";
|
|
1042
|
+
control.turnMetricOutcome = "failed";
|
|
1043
|
+
control.activityStatus = "idle";
|
|
1338
1044
|
control.activityError = error;
|
|
1339
|
-
|
|
1045
|
+
await deliverFailedChildTurnToParent(
|
|
1046
|
+
{ db, bus, settings, observability, wakeSessionWorkflow },
|
|
1047
|
+
input.workspaceId,
|
|
1048
|
+
input.sessionId,
|
|
1049
|
+
attempt.turnId,
|
|
1050
|
+
);
|
|
1051
|
+
return claimedResult({ status: "idle" });
|
|
1340
1052
|
}
|
|
1341
|
-
// The leased credential path above normally quarantines quota state and
|
|
1342
|
-
// either recovers the same turn or arms a durable capacity wait. This narrow
|
|
1343
|
-
// fallback covers failures before a credential lease existed, or a failed
|
|
1344
|
-
// durable checkpoint where replay would be unsafe. Keep the session usable,
|
|
1345
|
-
// but never synthesize another turn or walk an unfenced legacy pointer.
|
|
1346
1053
|
if (usageLimit && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
|
|
1347
1054
|
await flushRuntimeBatcher();
|
|
1348
1055
|
await historySink.reconcileConversationTruth();
|
|
@@ -1387,23 +1094,48 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1387
1094
|
) {
|
|
1388
1095
|
await flushRuntimeBatcher();
|
|
1389
1096
|
await historySink.reconcileConversationTruth();
|
|
1097
|
+
if (error.sessionUsageLimit) {
|
|
1098
|
+
recordSessionUsageLimitEvent(observability, "between_calls", "reached");
|
|
1099
|
+
}
|
|
1390
1100
|
if (
|
|
1391
|
-
!(await eventing.settle!(
|
|
1392
|
-
|
|
1393
|
-
{
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1101
|
+
!(await eventing.settle!(
|
|
1102
|
+
error.sessionUsageLimit
|
|
1103
|
+
? {
|
|
1104
|
+
// the task's usage limit was reached between model
|
|
1105
|
+
// calls. Already admitted usage is recorded; history is kept.
|
|
1106
|
+
events: sessionUsageLimitStopEvents(error.sessionUsageLimit),
|
|
1107
|
+
turnStatus: "completed",
|
|
1108
|
+
sessionStatus: "idle",
|
|
1109
|
+
activeTurnId: null,
|
|
1110
|
+
allowanceGoalPause: {
|
|
1111
|
+
rationale: error.sessionUsageLimit.message,
|
|
1112
|
+
reason: "session_usage_limit",
|
|
1113
|
+
},
|
|
1114
|
+
}
|
|
1115
|
+
: {
|
|
1116
|
+
events: [
|
|
1117
|
+
...(error.allowance
|
|
1118
|
+
? [{ type: "usage.exhausted" as const, payload: error.allowance }]
|
|
1119
|
+
: []),
|
|
1120
|
+
{
|
|
1121
|
+
type: "turn.completed",
|
|
1122
|
+
payload: {
|
|
1123
|
+
output: "",
|
|
1124
|
+
segmentLimit: "budget_exhausted",
|
|
1125
|
+
detail: error.message,
|
|
1126
|
+
...(error.allowance ?? {}),
|
|
1127
|
+
},
|
|
1128
|
+
},
|
|
1129
|
+
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1130
|
+
],
|
|
1131
|
+
turnStatus: "completed",
|
|
1132
|
+
sessionStatus: "idle",
|
|
1133
|
+
activeTurnId: null,
|
|
1134
|
+
...(error.allowance
|
|
1135
|
+
? { allowanceGoalPause: { rationale: error.allowance.message } }
|
|
1136
|
+
: {}),
|
|
1399
1137
|
},
|
|
1400
|
-
|
|
1401
|
-
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1402
|
-
],
|
|
1403
|
-
turnStatus: "completed",
|
|
1404
|
-
sessionStatus: "idle",
|
|
1405
|
-
activeTurnId: null,
|
|
1406
|
-
}))
|
|
1138
|
+
))
|
|
1407
1139
|
) {
|
|
1408
1140
|
return claimedResult({ status: "cancelled" });
|
|
1409
1141
|
}
|
|
@@ -1435,7 +1167,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1435
1167
|
// fall through to the terminal path rather than resend an equivalent
|
|
1436
1168
|
// request forever.
|
|
1437
1169
|
const encryptedArtifactRejection =
|
|
1438
|
-
|
|
1170
|
+
!coreRequestOutcomeUnknown &&
|
|
1171
|
+
!coreLeaseCheckpointFailed &&
|
|
1172
|
+
billingState.isCodexTurn &&
|
|
1173
|
+
providerTurn.effectiveCodexCredentialId
|
|
1439
1174
|
? classifyCodexEncryptedArtifactRejection(error)
|
|
1440
1175
|
: null;
|
|
1441
1176
|
if (
|
|
@@ -1499,31 +1234,180 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1499
1234
|
// truth, recover this SAME accepted turn, then let the workflow re-claim
|
|
1500
1235
|
// it after a pacing delay. This is independent of goal state and never
|
|
1501
1236
|
// relies on a synthetic continuation prompt.
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1237
|
+
// A rolling-deployment definition mismatch is a separate configuration
|
|
1238
|
+
// class: only the exact typed setup error can use this checkpoint before
|
|
1239
|
+
// eventing exists. No generic setup/credential failure gains retry authority.
|
|
1240
|
+
const earlyDefinitionMismatch =
|
|
1241
|
+
error instanceof TurnExecutionPolicyDefinitionMismatchError &&
|
|
1242
|
+
!attempt.modelRequestStarted &&
|
|
1243
|
+
!eventing.turnStartedPublished &&
|
|
1244
|
+
!!attempt.turnId &&
|
|
1245
|
+
!!attempt.triggerEventId &&
|
|
1246
|
+
attempt.executionGeneration > 0;
|
|
1247
|
+
const earlyCommandStartUnavailable =
|
|
1248
|
+
isModalTaskExecStartPreDispatchUnavailableError(error) &&
|
|
1249
|
+
!attempt.modelRequestStarted &&
|
|
1250
|
+
!eventing.turnStartedPublished &&
|
|
1251
|
+
!!attempt.turnId &&
|
|
1252
|
+
!!attempt.triggerEventId &&
|
|
1253
|
+
attempt.executionGeneration > 0;
|
|
1254
|
+
const earlyRecoverableSetup = earlyDefinitionMismatch || earlyCommandStartUnavailable;
|
|
1255
|
+
// Call-scoped proof that an in-flight exec_command never dispatched its
|
|
1256
|
+
// command. Those calls settle as not started (replay safe), never unknown,
|
|
1257
|
+
// whether this turn recovers or fails.
|
|
1258
|
+
const notStartedToolCallIds = isRoutingMutationOutcomeUnknownError(error)
|
|
1259
|
+
? []
|
|
1260
|
+
: toolCallCommandNotStartedCallIds(error);
|
|
1261
|
+
// Custody decides whether replay is safe, not what caused the failure. Keep
|
|
1262
|
+
// the existing diagnostic in the permission-controlled terminal event;
|
|
1263
|
+
// never copy it into logs or use its retry classification for admission.
|
|
1264
|
+
let unknownOutcomeDetail: string | undefined;
|
|
1265
|
+
if (coreRequestOutcomeUnknown) {
|
|
1266
|
+
try {
|
|
1267
|
+
const sourceFailure = agentRunFailurePayload(error, {
|
|
1268
|
+
isCodexTurn: billingState.isCodexTurn,
|
|
1269
|
+
});
|
|
1270
|
+
unknownOutcomeDetail = sourceFailure.detail ?? sourceFailure.error;
|
|
1271
|
+
} catch {
|
|
1272
|
+
// Diagnostic extraction cannot prevent durable failure settlement.
|
|
1273
|
+
}
|
|
1274
|
+
}
|
|
1275
|
+
let failure = withModelRoutePresentation(
|
|
1276
|
+
(coreRequestOutcomeUnknown
|
|
1277
|
+
? {
|
|
1278
|
+
error: coreRequestOutcomeUnknown.message,
|
|
1279
|
+
code: coreRequestOutcomeUnknown.code,
|
|
1280
|
+
retryable: false,
|
|
1281
|
+
...(unknownOutcomeDetail && unknownOutcomeDetail !== coreRequestOutcomeUnknown.message
|
|
1282
|
+
? { detail: unknownOutcomeDetail }
|
|
1283
|
+
: {}),
|
|
1284
|
+
}
|
|
1285
|
+
: earlyDefinitionMismatch
|
|
1286
|
+
? { error: error.message, code: error.code, retryable: true }
|
|
1287
|
+
: (coreUnknownRecoveryFailure ??
|
|
1288
|
+
agentRunFailurePayload(error, {
|
|
1289
|
+
isCodexTurn: billingState.isCodexTurn,
|
|
1290
|
+
}))) as ReturnType<typeof agentRunFailurePayload>,
|
|
1291
|
+
attempt.modelRoutePresentation,
|
|
1292
|
+
);
|
|
1293
|
+
const coreLaneRecoverable = (lane: ReturnType<typeof createCoreCodexRequests> | undefined) =>
|
|
1294
|
+
!lane || lane.canRecover() || (!!coreUnknownRecoveryFailure && lane.unknownIsOwnAndSettled());
|
|
1295
|
+
// A core lane holds the answered request that asked for a tool until the
|
|
1296
|
+
// next model call checkpoints it, so a retryable failure thrown while a
|
|
1297
|
+
// tool runs (a command that never started, an MCP transport failure) or a
|
|
1298
|
+
// paused search's refused continuation finds an answered, unsettled
|
|
1299
|
+
// request. Commit the history durably and checkpoint first, as the
|
|
1300
|
+
// lease-lost and failover paths do, so the failure can recover. Nothing is
|
|
1301
|
+
// replayed: a lane without every response stays unrecoverable. This holds
|
|
1302
|
+
// for every core provider: a lane's answered request is complete regardless
|
|
1303
|
+
// of who serves it.
|
|
1304
|
+
const placedCore = coreClaude ?? coreCodex ?? coreXai;
|
|
1305
|
+
if (
|
|
1306
|
+
placedCore &&
|
|
1307
|
+
attempt.turnId &&
|
|
1308
|
+
!coreRequestOutcomeUnknown &&
|
|
1309
|
+
!coreLeaseCheckpointFailed &&
|
|
1310
|
+
failure.retryable &&
|
|
1311
|
+
eventing.publish &&
|
|
1312
|
+
eventing.turnStartedPublished &&
|
|
1313
|
+
!coreLaneRecoverable(placedCore.requests)
|
|
1314
|
+
) {
|
|
1315
|
+
await coreLanesRecoverAfterCheckpoint(placedCore, async () => {
|
|
1316
|
+
await flushRuntimeBatcher();
|
|
1317
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1318
|
+
});
|
|
1319
|
+
}
|
|
1320
|
+
if (
|
|
1321
|
+
attempt.turnId &&
|
|
1322
|
+
!coreRequestOutcomeUnknown &&
|
|
1323
|
+
!coreLeaseCheckpointFailed &&
|
|
1324
|
+
coreLaneRecoverable(coreCodex?.requests) &&
|
|
1325
|
+
coreLaneRecoverable(coreXai?.requests) &&
|
|
1326
|
+
coreLaneRecoverable(coreClaude?.requests) &&
|
|
1327
|
+
(earlyRecoverableSetup ||
|
|
1328
|
+
(failure.retryable && eventing.publish && eventing.turnStartedPublished))
|
|
1329
|
+
) {
|
|
1330
|
+
const recoveryCode = providerRecoveryCode(error, failure);
|
|
1331
|
+
// A recovery that interrupts a running tool is followed by a successful
|
|
1332
|
+
// model response before the tool runs again, which resets the in-memory
|
|
1333
|
+
// budget. Count those recoveries since the turn's last tool progress so
|
|
1334
|
+
// the same bounded backoff and limit still apply.
|
|
1335
|
+
const toolProgress =
|
|
1336
|
+
earlyRecoverableSetup || recoveryCode === PROVIDER_OVERLOAD_RECOVERY_CODE
|
|
1337
|
+
? null
|
|
1338
|
+
: await readSessionTurnToolProgressRecoveryState(db, input.workspaceId, {
|
|
1339
|
+
sessionId: input.sessionId,
|
|
1340
|
+
turnId: attempt.turnId,
|
|
1341
|
+
}).catch(() => null);
|
|
1342
|
+
const nextProviderRecoveryCount =
|
|
1343
|
+
(toolProgress?.toolCallInFlight
|
|
1344
|
+
? Math.max(attempt.providerRecoveryCount, toolProgress.toolInterruptedRecoveries)
|
|
1345
|
+
: attempt.providerRecoveryCount) + 1;
|
|
1507
1346
|
const recoveryResult = providerRecoveryResult({
|
|
1508
|
-
failureCode:
|
|
1347
|
+
failureCode: recoveryCode,
|
|
1509
1348
|
attemptNumber: nextProviderRecoveryCount,
|
|
1349
|
+
recoveryStartedAt:
|
|
1350
|
+
attempt.providerRecoveryCount > 0 ? attempt.providerRecoveryStartedAt : undefined,
|
|
1510
1351
|
retryAfterMs: providerRetryAfterMs(error),
|
|
1352
|
+
jitterSample: Math.random(),
|
|
1511
1353
|
});
|
|
1354
|
+
const setupRecoveryExhausted =
|
|
1355
|
+
earlyCommandStartUnavailable &&
|
|
1356
|
+
recoveryResult.status === "exhausted" &&
|
|
1357
|
+
attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
|
|
1512
1358
|
try {
|
|
1359
|
+
if (setupRecoveryExhausted) {
|
|
1360
|
+
// This is positive pre-dispatch proof, not an ambiguous command. Park
|
|
1361
|
+
// the SAME accepted turn without resetting or advancing its budget.
|
|
1362
|
+
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
1363
|
+
sessionId: input.sessionId,
|
|
1364
|
+
turnId: attempt.turnId,
|
|
1365
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1366
|
+
attemptId: input.attemptId,
|
|
1367
|
+
reason: "sandbox_command_start_recovery_exhausted",
|
|
1368
|
+
sandboxSetupRecoveryExhausted: true,
|
|
1369
|
+
detail: {
|
|
1370
|
+
code: failure.code,
|
|
1371
|
+
error:
|
|
1372
|
+
"Automatic sandbox setup recovery exhausted after five retries; the accepted turn remains parked without starting a command.",
|
|
1373
|
+
retryable: false,
|
|
1374
|
+
setupOutcome: "not_started",
|
|
1375
|
+
replay: "blocked",
|
|
1376
|
+
recoveryExhausted: true,
|
|
1377
|
+
providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
|
|
1378
|
+
},
|
|
1379
|
+
});
|
|
1380
|
+
if (recovery.action === "stale") {
|
|
1381
|
+
acknowledgeLostAttemptOwnership();
|
|
1382
|
+
control.activityStatus = "cancelled";
|
|
1383
|
+
control.turnMetricOutcome = "cancelled";
|
|
1384
|
+
return claimedResult({ status: "cancelled" });
|
|
1385
|
+
}
|
|
1386
|
+
acknowledgeRecoveryQuiescence();
|
|
1387
|
+
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
1388
|
+
control.turnMetricOutcome = "recovering";
|
|
1389
|
+
control.activityStatus = "recovering";
|
|
1390
|
+
control.activityError = error;
|
|
1391
|
+
return claimedResult({ status: "recovering" });
|
|
1392
|
+
}
|
|
1513
1393
|
if (recoveryResult.status === "recovering") {
|
|
1514
|
-
|
|
1515
|
-
|
|
1394
|
+
if (!earlyRecoverableSetup) {
|
|
1395
|
+
await flushRuntimeBatcher();
|
|
1396
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1397
|
+
}
|
|
1516
1398
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
1517
1399
|
sessionId: input.sessionId,
|
|
1518
1400
|
turnId: attempt.turnId,
|
|
1519
1401
|
triggerEventId: attempt.triggerEventId!,
|
|
1520
1402
|
attemptId: input.attemptId,
|
|
1521
|
-
reason:
|
|
1403
|
+
reason: recoveryCode ?? "provider_unavailable",
|
|
1522
1404
|
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1405
|
+
...(notStartedToolCallIds.length > 0 ? { notStartedToolCallIds } : {}),
|
|
1523
1406
|
detail: {
|
|
1524
|
-
...failure,
|
|
1407
|
+
...agentRunRecoveryFailurePayload(error, failure),
|
|
1525
1408
|
continueDelayMs: recoveryResult.continueDelayMs,
|
|
1526
1409
|
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1410
|
+
maxProviderRecoveryCount: providerRecoveryLimit(recoveryCode),
|
|
1527
1411
|
},
|
|
1528
1412
|
});
|
|
1529
1413
|
if (recovery.action === "stale") {
|
|
@@ -1537,9 +1421,45 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1537
1421
|
control.turnMetricOutcome = "recovering";
|
|
1538
1422
|
control.activityStatus = "recovering";
|
|
1539
1423
|
control.activityError = error;
|
|
1424
|
+
const recoveryCause = providerRecoveryCause(failure.code);
|
|
1425
|
+
if (recoveryCause) {
|
|
1426
|
+
recordProviderRecoveryOutcome(observability, {
|
|
1427
|
+
route: attempt.modelMetricRoute,
|
|
1428
|
+
cause: recoveryCause,
|
|
1429
|
+
outcome: "scheduled",
|
|
1430
|
+
delayMs: recoveryResult.continueDelayMs,
|
|
1431
|
+
});
|
|
1432
|
+
}
|
|
1540
1433
|
return claimedResult(recoveryResult);
|
|
1541
1434
|
}
|
|
1542
1435
|
failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
|
|
1436
|
+
const recoveryCause = providerRecoveryCause(failure.code);
|
|
1437
|
+
if (recoveryCause) {
|
|
1438
|
+
recordProviderRecoveryOutcome(observability, {
|
|
1439
|
+
route: attempt.modelMetricRoute,
|
|
1440
|
+
cause: recoveryCause,
|
|
1441
|
+
outcome: "exhausted",
|
|
1442
|
+
...(attempt.providerRecoveryObservation
|
|
1443
|
+
? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt }
|
|
1444
|
+
: {}),
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
if (earlyRecoverableSetup) {
|
|
1448
|
+
// Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
|
|
1449
|
+
// through Temporal into exact-attempt workflow failure settlement.
|
|
1450
|
+
control.activityStatus = "failed";
|
|
1451
|
+
control.turnMetricOutcome = "failed";
|
|
1452
|
+
control.activityError = error;
|
|
1453
|
+
throw ApplicationFailure.create({
|
|
1454
|
+
message: earlyDefinitionMismatch
|
|
1455
|
+
? `${error.message}. Automatic same-turn configuration recovery exhausted after ${recoveryResult.providerRecoveryCount} retries.`
|
|
1456
|
+
: failure.error,
|
|
1457
|
+
type: earlyDefinitionMismatch
|
|
1458
|
+
? "TurnExecutionPolicyDefinitionMismatchError"
|
|
1459
|
+
: "SandboxCommandStartUnavailableError",
|
|
1460
|
+
nonRetryable: true,
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1543
1463
|
} catch (recoveryError) {
|
|
1544
1464
|
const escaped =
|
|
1545
1465
|
recoveryResult.status === "recovering"
|
|
@@ -1569,11 +1489,22 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1569
1489
|
triggerEventId: attempt.triggerEventId!,
|
|
1570
1490
|
executionGeneration: attempt.executionGeneration,
|
|
1571
1491
|
providerRecovery: {
|
|
1572
|
-
failureCode:
|
|
1492
|
+
failureCode: recoveryCode ?? "provider_unavailable",
|
|
1573
1493
|
providerRecoveryCount: nextProviderRecoveryCount,
|
|
1494
|
+
...(recoveryCode === PROVIDER_OVERLOAD_RECOVERY_CODE
|
|
1495
|
+
? { continueDelayMs: recoveryResult.continueDelayMs }
|
|
1496
|
+
: {}),
|
|
1574
1497
|
},
|
|
1575
1498
|
})
|
|
1576
|
-
:
|
|
1499
|
+
: setupRecoveryExhausted
|
|
1500
|
+
? postClaimDatabaseRecoveryFailure({
|
|
1501
|
+
error: recoveryError,
|
|
1502
|
+
turnId: attempt.turnId,
|
|
1503
|
+
triggerEventId: attempt.triggerEventId!,
|
|
1504
|
+
executionGeneration: attempt.executionGeneration,
|
|
1505
|
+
sandboxSetupRecoveryExhausted: true,
|
|
1506
|
+
})
|
|
1507
|
+
: null;
|
|
1577
1508
|
if (postClaimRecovery) {
|
|
1578
1509
|
control.activityStatus = "recovering";
|
|
1579
1510
|
control.turnMetricOutcome = "recovering";
|
|
@@ -1583,26 +1514,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1583
1514
|
throw recoveryError;
|
|
1584
1515
|
}
|
|
1585
1516
|
}
|
|
1586
|
-
if (
|
|
1587
|
-
attempt.turnId &&
|
|
1588
|
-
attempt.triggerEventId &&
|
|
1589
|
-
attempt.executionGeneration > 0 &&
|
|
1590
|
-
!eventing.turnStartedPublished &&
|
|
1591
|
-
!attempt.modelRequestStarted
|
|
1592
|
-
) {
|
|
1593
|
-
const recoveryFailure = postClaimDatabaseRecoveryFailure({
|
|
1594
|
-
error,
|
|
1595
|
-
turnId: attempt.turnId,
|
|
1596
|
-
triggerEventId: attempt.triggerEventId,
|
|
1597
|
-
executionGeneration: attempt.executionGeneration,
|
|
1598
|
-
});
|
|
1599
|
-
if (recoveryFailure) {
|
|
1600
|
-
control.activityStatus = "recovering";
|
|
1601
|
-
control.turnMetricOutcome = "recovering";
|
|
1602
|
-
control.activityError = error;
|
|
1603
|
-
throw recoveryFailure;
|
|
1604
|
-
}
|
|
1605
|
-
}
|
|
1606
1517
|
control.activityStatus = "failed";
|
|
1607
1518
|
control.activityError = error;
|
|
1608
1519
|
if (!attempt.turnId) {
|
|
@@ -1611,13 +1522,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1611
1522
|
if (!eventing.publish || !eventing.turnStartedPublished) {
|
|
1612
1523
|
throw error;
|
|
1613
1524
|
}
|
|
1525
|
+
// Automatic recovery was not authorized for this attempt. A terminal record
|
|
1526
|
+
// never promises a retry that will not happen.
|
|
1527
|
+
if (failure.retryable === true) failure = automaticRecoveryUnavailableFailure(failure);
|
|
1614
1528
|
// A partial/malformed stream may have emitted assistant/tool items (and
|
|
1615
1529
|
// external side effects) before its terminal error. Persist every item the
|
|
1616
1530
|
// SDK state observed before marking the turn failed so a later user revive
|
|
1617
1531
|
// never replays work from an incomplete history. This does not retry or
|
|
1618
1532
|
// rotate the ambiguous request.
|
|
1619
1533
|
await flushRuntimeBatcher();
|
|
1620
|
-
|
|
1534
|
+
if (coreRequestOutcomeUnknown || coreUnknownRecoveryFailure) {
|
|
1535
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1536
|
+
} else {
|
|
1537
|
+
await historySink.reconcileConversationTruth();
|
|
1538
|
+
}
|
|
1621
1539
|
if (
|
|
1622
1540
|
!(await eventing.settle!({
|
|
1623
1541
|
events: [
|
|
@@ -1627,6 +1545,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1627
1545
|
turnStatus: "failed",
|
|
1628
1546
|
sessionStatus: "failed",
|
|
1629
1547
|
activeTurnId: null,
|
|
1548
|
+
...(notStartedToolCallIds.length > 0 ? { notStartedToolCallIds } : {}),
|
|
1630
1549
|
}))
|
|
1631
1550
|
) {
|
|
1632
1551
|
return claimedResult({ status: "cancelled" });
|
|
@@ -1647,3 +1566,77 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
1647
1566
|
);
|
|
1648
1567
|
return claimedResult({ status: "failed" });
|
|
1649
1568
|
}
|
|
1569
|
+
|
|
1570
|
+
/** An error of the given class, directly or anywhere in the cause chain. */
|
|
1571
|
+
function hasErrorInCauseChain(
|
|
1572
|
+
error: unknown,
|
|
1573
|
+
type: abstract new (...args: never[]) => Error,
|
|
1574
|
+
): boolean {
|
|
1575
|
+
let current = error;
|
|
1576
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1577
|
+
if (current instanceof type) return true;
|
|
1578
|
+
current = (current as { cause?: unknown }).cause;
|
|
1579
|
+
}
|
|
1580
|
+
return false;
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
function findInCauseChain<T extends Error>(
|
|
1584
|
+
error: unknown,
|
|
1585
|
+
type: abstract new (...args: never[]) => T,
|
|
1586
|
+
): T | null {
|
|
1587
|
+
let current = error;
|
|
1588
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1589
|
+
if (current instanceof type) return current;
|
|
1590
|
+
current = (current as { cause?: unknown }).cause;
|
|
1591
|
+
}
|
|
1592
|
+
return null;
|
|
1593
|
+
}
|
|
1594
|
+
|
|
1595
|
+
const INTERRUPTED_CODEX_RESPONSE_MESSAGE = "The Codex response was interrupted before it finished";
|
|
1596
|
+
|
|
1597
|
+
/**
|
|
1598
|
+
* Classify a Codex failure whose only replay obstacle is this attempt's own
|
|
1599
|
+
* interrupted request. Returns the retryable failure the bounded provider
|
|
1600
|
+
* recovery should use, or null when the unknown outcome must stay terminal.
|
|
1601
|
+
*/
|
|
1602
|
+
export function ownInterruptedCodexRequestRecovery(
|
|
1603
|
+
error: unknown,
|
|
1604
|
+
core: Pick<CodexSubscriptionCoreTurn, "requests">,
|
|
1605
|
+
isCodexTurn: boolean,
|
|
1606
|
+
): ReturnType<typeof agentRunFailurePayload> | null {
|
|
1607
|
+
const lane = core.requests;
|
|
1608
|
+
if (!lane?.hasUnknownOutcome()) return null;
|
|
1609
|
+
if (lane.admissionRefused()) return null;
|
|
1610
|
+
if (!lane.unknownIsOwnAndSettled()) return null;
|
|
1611
|
+
// A complete provider terminal that is incomplete (for example the output
|
|
1612
|
+
// token limit) would end the same way again.
|
|
1613
|
+
if (findInCauseChain(error, CodexStreamingTerminalError)?.code === "response_incomplete") {
|
|
1614
|
+
return null;
|
|
1615
|
+
}
|
|
1616
|
+
// After an interrupted request the tracker refuses the SDK's own retry with
|
|
1617
|
+
// the native unknown-outcome error; the SDK retried, so the cause was
|
|
1618
|
+
// transient.
|
|
1619
|
+
if (hasErrorInCauseChain(error, SubscriptionCoreCodexRequestOutcomeUnknownError)) {
|
|
1620
|
+
return {
|
|
1621
|
+
error: INTERRUPTED_CODEX_RESPONSE_MESSAGE,
|
|
1622
|
+
code: "provider_unavailable",
|
|
1623
|
+
retryable: true,
|
|
1624
|
+
};
|
|
1625
|
+
}
|
|
1626
|
+
let failure: ReturnType<typeof agentRunFailurePayload>;
|
|
1627
|
+
try {
|
|
1628
|
+
failure = agentRunFailurePayload(error, { isCodexTurn });
|
|
1629
|
+
} catch {
|
|
1630
|
+
return null;
|
|
1631
|
+
}
|
|
1632
|
+
// A stream that ended without a terminal event, or stalled after it began,
|
|
1633
|
+
// is a broken connection rather than a provider answer.
|
|
1634
|
+
if (failure.code === "invalid_sse_terminal" || failure.code === "codex_response_timeout") {
|
|
1635
|
+
return {
|
|
1636
|
+
error: INTERRUPTED_CODEX_RESPONSE_MESSAGE,
|
|
1637
|
+
code: "provider_unavailable",
|
|
1638
|
+
retryable: true,
|
|
1639
|
+
};
|
|
1640
|
+
}
|
|
1641
|
+
return failure.retryable === true ? failure : null;
|
|
1642
|
+
}
|