@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38686408778001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +172 -28
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +3 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +30 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-NTXZHYET.js} +1552 -2876
- package/dist/activities-control-NTXZHYET.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-RKRI5GY4.js} +9238 -4392
- package/dist/activities-turn-RKRI5GY4.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +104 -35
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-7BIHFUGB.js} +2293 -3956
- package/dist/chunk-7BIHFUGB.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1134 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +26 -4
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +94 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +599 -110
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +162 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +116 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +758 -141
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +79 -38
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -1,14 +1,22 @@
|
|
|
1
|
+
import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
|
|
2
|
+
import { DrizzleQueryError } from "drizzle-orm";
|
|
3
|
+
import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
|
|
4
|
+
import { heldSessionUsageReceiptsForTurn } from "./session-usage-custody";
|
|
1
5
|
import {
|
|
2
6
|
ActiveSessionHistoryLimitExceededError,
|
|
3
7
|
ApprovalRunStateLimitExceededError,
|
|
4
8
|
nestedPostgresSqlState,
|
|
9
|
+
databaseFailureCode,
|
|
10
|
+
isRetryablePersistenceSqlState,
|
|
5
11
|
safeDatabaseErrorFacts,
|
|
6
12
|
isRetryableDatabaseTransportFailure,
|
|
7
13
|
isSessionEventPersistenceError,
|
|
14
|
+
DatabaseTransactionError,
|
|
8
15
|
SandboxLeaseTransitionError,
|
|
9
16
|
} from "@opengeni/db";
|
|
10
17
|
import {
|
|
11
18
|
ActiveBackendUnresolvableError,
|
|
19
|
+
AnthropicProviderRejection,
|
|
12
20
|
CompactionProviderResponseError,
|
|
13
21
|
EmptyCompactionSummaryError,
|
|
14
22
|
compactionProviderRejection,
|
|
@@ -16,29 +24,61 @@ import {
|
|
|
16
24
|
type CompactionProviderRejection,
|
|
17
25
|
isMcpRequestTimeoutError,
|
|
18
26
|
isMcpTransportConnectivityError,
|
|
19
|
-
|
|
27
|
+
isModalTaskExecStartPreDispatchUnavailableError,
|
|
28
|
+
isModalCommandStartOutcomeUnknownError,
|
|
29
|
+
isProviderCommandObservationUnavailableError,
|
|
30
|
+
ProviderCommandInputOutcomeUnknownError,
|
|
31
|
+
ProviderCommandStartOutcomeUnknownError,
|
|
32
|
+
isRoutingMutationOutcomeUnknownError,
|
|
33
|
+
isRoutingMutationOutputRejectedError,
|
|
20
34
|
RoutingWorkspaceRootChangedError,
|
|
35
|
+
RoutingBackendRecoveryRequiredError,
|
|
36
|
+
ResponsesStreamingTerminalError,
|
|
37
|
+
classifyModelStreamIdleTimeoutError,
|
|
21
38
|
SandboxMaterializationVerificationError,
|
|
22
39
|
materializationVerificationDiagnostic,
|
|
23
40
|
type MaterializationVerificationDiagnostic,
|
|
41
|
+
PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
42
|
+
type ProviderQuotaExhaustion,
|
|
43
|
+
type ProviderQuotaScope,
|
|
44
|
+
classifyProviderQuotaError,
|
|
45
|
+
providerQuotaExhaustedMessage,
|
|
24
46
|
SelfhostedWorkspaceRootChangedError,
|
|
25
47
|
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
48
|
+
AnthropicRequestError,
|
|
49
|
+
findAnthropicRequestSizeError,
|
|
50
|
+
AnthropicSizeRecoveryExhaustedError,
|
|
51
|
+
type AnthropicRequestSize,
|
|
26
52
|
} from "@opengeni/runtime";
|
|
27
53
|
import {
|
|
28
54
|
mcpTransportRequestFailureDiagnostic,
|
|
29
55
|
type McpTransportRequestFailureDiagnostic,
|
|
30
56
|
} from "@opengeni/runtime/mcp-network";
|
|
31
57
|
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
32
|
-
import {
|
|
58
|
+
import type { ProviderErrorOutcome } from "@opengeni/subscriptions";
|
|
59
|
+
import { CODEX_USAGE_EXHAUSTED_PCT } from "./codex-usage-policy";
|
|
33
60
|
import { RetainedAttachmentTransportLimitError } from "../run-input";
|
|
34
61
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
35
62
|
import {
|
|
63
|
+
CODEX_USAGE_LIMIT_ERROR_TYPE,
|
|
36
64
|
CodexReloginRequired,
|
|
37
65
|
classifyCodexEncryptedArtifactRejection,
|
|
66
|
+
classifyCodexEntitlementRejection,
|
|
38
67
|
classifyCodexResponseTimeoutError,
|
|
39
68
|
classifyCodexUsageLimitError,
|
|
40
69
|
isCodexTransportError,
|
|
41
70
|
} from "@opengeni/codex";
|
|
71
|
+
import {
|
|
72
|
+
CodexPlanEntitlementError,
|
|
73
|
+
codexPlanEntitlementFailurePayload,
|
|
74
|
+
codexRequestRejectedFailurePayload,
|
|
75
|
+
} from "./codex-plan-entitlement";
|
|
76
|
+
import { SubscriptionCoreCodexTurnError } from "./codex-core-errors";
|
|
77
|
+
import {
|
|
78
|
+
CodexIncludedUsageExhaustedError,
|
|
79
|
+
CodexIncludedUsageUnknownError,
|
|
80
|
+
findCodexCreditPolicyError,
|
|
81
|
+
} from "./codex-credit-policy";
|
|
42
82
|
import {
|
|
43
83
|
classifyXaiSubscriptionStreamingTerminalError,
|
|
44
84
|
classifyXaiSubscriptionStreamIdleTimeoutError,
|
|
@@ -64,12 +104,45 @@ import {
|
|
|
64
104
|
MandatoryHistoryPersistenceError,
|
|
65
105
|
type MandatoryHistoryPersistenceStage,
|
|
66
106
|
} from "./quiescence";
|
|
107
|
+
import {
|
|
108
|
+
MODEL_PROVIDER_RECOVERY_CODES,
|
|
109
|
+
providerRecoveryExhaustedMessage,
|
|
110
|
+
providerRecoverySubject,
|
|
111
|
+
type ProviderCondition,
|
|
112
|
+
} from "./provider-recovery-copy";
|
|
113
|
+
import {
|
|
114
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
115
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
116
|
+
PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
|
|
117
|
+
validProviderOverloadRecoveryDelay,
|
|
118
|
+
} from "./provider-recovery-policy";
|
|
119
|
+
export {
|
|
120
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
121
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
122
|
+
PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
|
|
123
|
+
} from "./provider-recovery-policy";
|
|
67
124
|
|
|
68
125
|
// Retryable provider connectivity/5xx failures start quickly and back off to
|
|
69
126
|
// this ceiling. Explicit rate limits retain the minute-granular fallback.
|
|
70
127
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
71
128
|
export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
72
129
|
export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
130
|
+
export function providerRecoveryLimit(failureCode: unknown): number {
|
|
131
|
+
return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE
|
|
132
|
+
? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES
|
|
133
|
+
: MAX_AUTOMATIC_PROVIDER_RECOVERIES;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
|
|
137
|
+
* answer a per-minute token limit with a `retry-after` of about a second, which
|
|
138
|
+
* alone would spend every automatic recovery before the window resets.
|
|
139
|
+
*/
|
|
140
|
+
export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
|
|
141
|
+
/** Positive-only spread: never shorten the provider's minimum delay. */
|
|
142
|
+
export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
|
|
143
|
+
const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
|
|
144
|
+
return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
|
|
145
|
+
}
|
|
73
146
|
export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
74
147
|
|
|
75
148
|
export class PostCompactionContinuationEmptyError extends Error {
|
|
@@ -85,25 +158,47 @@ export type ProviderRecoveryResult =
|
|
|
85
158
|
| {
|
|
86
159
|
status: "recovering";
|
|
87
160
|
continueDelayMs: number;
|
|
161
|
+
maxProviderRecoveryCount?: number;
|
|
88
162
|
}
|
|
89
163
|
| {
|
|
90
164
|
status: "exhausted";
|
|
91
165
|
providerRecoveryCount: number;
|
|
92
166
|
maxProviderRecoveryCount: number;
|
|
167
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
93
168
|
};
|
|
94
169
|
|
|
95
170
|
export function providerRecoveryResult(input: {
|
|
96
171
|
failureCode: string | undefined;
|
|
97
172
|
attemptNumber: number;
|
|
98
173
|
retryAfterMs?: number | null;
|
|
174
|
+
jitterSample?: number;
|
|
175
|
+
/** Durable first-failure clock, never a new clock on each replacement attempt. */
|
|
176
|
+
recoveryStartedAt?: number | undefined;
|
|
177
|
+
now?: number;
|
|
99
178
|
}): ProviderRecoveryResult {
|
|
100
|
-
|
|
179
|
+
const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
180
|
+
const limit = providerRecoveryLimit(input.failureCode);
|
|
181
|
+
const exhausted = (
|
|
182
|
+
reason: "deadline" | "retry_limit" | "invalid_clock",
|
|
183
|
+
): Extract<ProviderRecoveryResult, { status: "exhausted" }> => ({
|
|
184
|
+
status: "exhausted",
|
|
185
|
+
providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
|
|
186
|
+
maxProviderRecoveryCount: limit,
|
|
187
|
+
providerRecoveryExhaustedReason: reason,
|
|
188
|
+
});
|
|
189
|
+
if (input.attemptNumber > limit) {
|
|
190
|
+
if (overload) return exhausted("retry_limit");
|
|
101
191
|
return {
|
|
102
192
|
status: "exhausted",
|
|
103
193
|
providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
104
194
|
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
105
195
|
};
|
|
106
196
|
}
|
|
197
|
+
const now = input.now ?? Date.now();
|
|
198
|
+
const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
|
|
199
|
+
if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
|
|
200
|
+
return exhausted("invalid_clock");
|
|
201
|
+
}
|
|
107
202
|
const providerDelay =
|
|
108
203
|
input.retryAfterMs !== null &&
|
|
109
204
|
input.retryAfterMs !== undefined &&
|
|
@@ -113,12 +208,22 @@ export function providerRecoveryResult(input: {
|
|
|
113
208
|
: null;
|
|
114
209
|
const continueDelayMs =
|
|
115
210
|
input.failureCode === "provider_rate_limited"
|
|
116
|
-
? (
|
|
117
|
-
|
|
211
|
+
? Math.max(
|
|
212
|
+
providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
213
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS[
|
|
214
|
+
Math.min(
|
|
215
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
216
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1,
|
|
217
|
+
)
|
|
218
|
+
]!,
|
|
219
|
+
)
|
|
220
|
+
: overload ||
|
|
221
|
+
input.failureCode === "provider_unavailable" ||
|
|
118
222
|
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
119
223
|
input.failureCode === "sandbox_command_start_unavailable" ||
|
|
120
224
|
input.failureCode === "mcp_transport_timeout" ||
|
|
121
225
|
input.failureCode === "mcp_transport_unavailable" ||
|
|
226
|
+
input.failureCode === "turn_execution_policy_definition_mismatch" ||
|
|
122
227
|
input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
|
|
123
228
|
? Math.max(
|
|
124
229
|
providerDelay ?? 0,
|
|
@@ -130,12 +235,79 @@ export function providerRecoveryResult(input: {
|
|
|
130
235
|
]!,
|
|
131
236
|
)
|
|
132
237
|
: PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
238
|
+
const delayMs =
|
|
239
|
+
continueDelayMs +
|
|
240
|
+
(overload ||
|
|
241
|
+
input.failureCode === "provider_rate_limited" ||
|
|
242
|
+
input.failureCode === "provider_unavailable" ||
|
|
243
|
+
input.failureCode === "upstream_connectivity_unavailable"
|
|
244
|
+
? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
|
|
245
|
+
: 0);
|
|
246
|
+
// Do not shorten Retry-After, or schedule a replacement outside the window.
|
|
247
|
+
if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
248
|
+
return exhausted("deadline");
|
|
249
|
+
}
|
|
133
250
|
return {
|
|
134
251
|
status: "recovering",
|
|
135
|
-
continueDelayMs,
|
|
252
|
+
continueDelayMs: delayMs,
|
|
253
|
+
...(overload ? { maxProviderRecoveryCount: limit } : {}),
|
|
136
254
|
};
|
|
137
255
|
}
|
|
138
256
|
|
|
257
|
+
/** A timer/queue delay cannot authorize a provider call after the recovery window. */
|
|
258
|
+
export class ProviderOverloadRecoveryExpiredError extends Error {
|
|
259
|
+
readonly failure;
|
|
260
|
+
|
|
261
|
+
constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock") {
|
|
262
|
+
super("Confirmed provider overload recovery window expired before dispatch");
|
|
263
|
+
this.name = "ProviderOverloadRecoveryExpiredError";
|
|
264
|
+
this.failure = providerRecoveryExhaustedFailure(
|
|
265
|
+
{
|
|
266
|
+
error: this.message,
|
|
267
|
+
code: "provider_unavailable",
|
|
268
|
+
providerCondition: "overloaded" as const,
|
|
269
|
+
},
|
|
270
|
+
{
|
|
271
|
+
status: "exhausted",
|
|
272
|
+
// The queued replacement has not dispatched; count only prior retries.
|
|
273
|
+
providerRecoveryCount: Math.min(
|
|
274
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
275
|
+
Math.max(0, count - 1),
|
|
276
|
+
),
|
|
277
|
+
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
278
|
+
providerRecoveryExhaustedReason: reason,
|
|
279
|
+
},
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export function assertProviderOverloadRecoveryActive(input: {
|
|
285
|
+
failureCode: unknown;
|
|
286
|
+
providerRecoveryCount: number;
|
|
287
|
+
recoveryStartedAt?: number | undefined;
|
|
288
|
+
now?: number;
|
|
289
|
+
}): void {
|
|
290
|
+
if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
|
|
294
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
|
|
295
|
+
}
|
|
296
|
+
const now = input.now ?? Date.now();
|
|
297
|
+
const startedAt = input.recoveryStartedAt;
|
|
298
|
+
if (
|
|
299
|
+
startedAt === undefined ||
|
|
300
|
+
!Number.isFinite(startedAt) ||
|
|
301
|
+
!Number.isFinite(now) ||
|
|
302
|
+
startedAt > now
|
|
303
|
+
) {
|
|
304
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
|
|
305
|
+
}
|
|
306
|
+
if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
307
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
139
311
|
export function providerRecoveryExhaustedFailure<
|
|
140
312
|
T extends Record<string, unknown> & { error: string },
|
|
141
313
|
>(
|
|
@@ -147,18 +319,89 @@ export function providerRecoveryExhaustedFailure<
|
|
|
147
319
|
providerRecoveryCount: number;
|
|
148
320
|
maxProviderRecoveryCount: number;
|
|
149
321
|
lastRetryableError: string;
|
|
322
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
150
323
|
} {
|
|
151
324
|
return {
|
|
152
325
|
...failure,
|
|
153
|
-
error:
|
|
326
|
+
error: providerRecoveryExhaustedMessage({
|
|
327
|
+
code: typeof failure.code === "string" ? failure.code : null,
|
|
328
|
+
providerCondition: isProviderCondition(failure.providerCondition)
|
|
329
|
+
? failure.providerCondition
|
|
330
|
+
: null,
|
|
331
|
+
modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
|
|
332
|
+
providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
|
|
333
|
+
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
334
|
+
}),
|
|
154
335
|
retryable: false,
|
|
155
336
|
recoveryExhausted: true,
|
|
156
337
|
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
157
338
|
maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
|
|
339
|
+
...(recovery.providerRecoveryExhaustedReason
|
|
340
|
+
? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason }
|
|
341
|
+
: {}),
|
|
158
342
|
lastRetryableError: failure.error,
|
|
159
343
|
};
|
|
160
344
|
}
|
|
161
345
|
|
|
346
|
+
function isProviderCondition(value: unknown): value is ProviderCondition {
|
|
347
|
+
return (
|
|
348
|
+
value === "overloaded" ||
|
|
349
|
+
value === "unavailable" ||
|
|
350
|
+
value === "unresponsive" ||
|
|
351
|
+
value === "rate_limited"
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* A retryable failure whose automatic same-turn recovery was not authorized
|
|
357
|
+
* (for example, request custody could not be checkpointed). The classifier's
|
|
358
|
+
* "will retry" sentence would be false on a terminal record, so name the
|
|
359
|
+
* condition and the person's options instead.
|
|
360
|
+
*/
|
|
361
|
+
export function automaticRecoveryUnavailableFailure<
|
|
362
|
+
T extends Record<string, unknown> & { error: string },
|
|
363
|
+
>(
|
|
364
|
+
failure: T,
|
|
365
|
+
): T & {
|
|
366
|
+
retryable: false;
|
|
367
|
+
automaticRecovery: "unavailable";
|
|
368
|
+
lastRetryableError: string;
|
|
369
|
+
} {
|
|
370
|
+
const subject = providerRecoverySubject({
|
|
371
|
+
code: typeof failure.code === "string" ? failure.code : null,
|
|
372
|
+
providerCondition: isProviderCondition(failure.providerCondition)
|
|
373
|
+
? failure.providerCondition
|
|
374
|
+
: null,
|
|
375
|
+
modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
|
|
376
|
+
providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
|
|
377
|
+
});
|
|
378
|
+
return {
|
|
379
|
+
...failure,
|
|
380
|
+
error: `${subject}. Opengeni couldn't retry this turn automatically. Try again, or send a message to continue.`,
|
|
381
|
+
retryable: false,
|
|
382
|
+
automaticRecovery: "unavailable",
|
|
383
|
+
lastRetryableError: failure.error,
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* Name the exact accepted model route on model-provider recovery evidence so a
|
|
389
|
+
* person learns which model is affected. Display labels only: the route's
|
|
390
|
+
* credentials, base URL and upstream ids never enter the event.
|
|
391
|
+
*/
|
|
392
|
+
export function withModelRoutePresentation<T extends { code?: string | undefined }>(
|
|
393
|
+
failure: T,
|
|
394
|
+
route: { model: string; modelLabel: string; providerLabel: string } | undefined,
|
|
395
|
+
): T & { model?: string; modelLabel?: string; providerLabel?: string } {
|
|
396
|
+
if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
|
|
397
|
+
return {
|
|
398
|
+
...failure,
|
|
399
|
+
model: route.model,
|
|
400
|
+
modelLabel: route.modelLabel,
|
|
401
|
+
providerLabel: route.providerLabel,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
162
405
|
export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
|
|
163
406
|
const value = metadata.providerRecoveryCount;
|
|
164
407
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
@@ -194,6 +437,13 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
|
|
|
194
437
|
value.error && typeof value.error === "object"
|
|
195
438
|
? (value.error as Record<string, unknown>)
|
|
196
439
|
: null;
|
|
440
|
+
const milliseconds = Number(
|
|
441
|
+
headerValue(value.headers, "retry-after-ms") ??
|
|
442
|
+
headerValue(value.responseHeaders, "retry-after-ms") ??
|
|
443
|
+
headerValue(body?.headers, "retry-after-ms") ??
|
|
444
|
+
undefined,
|
|
445
|
+
);
|
|
446
|
+
if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
|
|
197
447
|
const directSeconds = Number(
|
|
198
448
|
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
199
449
|
);
|
|
@@ -317,20 +567,199 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
|
317
567
|
});
|
|
318
568
|
}
|
|
319
569
|
|
|
570
|
+
/** Complete structured-cause graph or no recovery authority. Budget distinct
|
|
571
|
+
* objects (including array containers), not queue positions/duplicate refs.
|
|
572
|
+
* The separate link ceiling bounds huge duplicate arrays without truncating
|
|
573
|
+
* them into permission. Cycles are harmless; overflow/unreadable edges hold. */
|
|
574
|
+
function structuredRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
|
|
575
|
+
const graph = new Map<object, Set<object>>();
|
|
576
|
+
const queue: object[] = [];
|
|
577
|
+
let links = 0;
|
|
578
|
+
const add = (value: unknown): boolean => {
|
|
579
|
+
if (!value || typeof value !== "object" || graph.has(value)) return true;
|
|
580
|
+
if (graph.size >= 64) return false;
|
|
581
|
+
graph.set(value, new Set());
|
|
582
|
+
queue.push(value);
|
|
583
|
+
return true;
|
|
584
|
+
};
|
|
585
|
+
if (!add(error)) return null;
|
|
586
|
+
try {
|
|
587
|
+
for (const current of queue) {
|
|
588
|
+
const record = current as Record<string, unknown>;
|
|
589
|
+
const children = Array.isArray(current)
|
|
590
|
+
? current
|
|
591
|
+
: ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
|
|
592
|
+
for (const child of children) {
|
|
593
|
+
if (++links > 4096 || !add(child)) return null;
|
|
594
|
+
if (child && typeof child === "object") graph.get(current)!.add(child);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
} catch {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
return graph;
|
|
601
|
+
}
|
|
602
|
+
|
|
320
603
|
function retryableDatabaseFailureCode(
|
|
321
604
|
error: unknown,
|
|
605
|
+
requireDatabaseProvenance = false,
|
|
322
606
|
): PostClaimDatabaseRecoveryDetail["code"] | null {
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
607
|
+
try {
|
|
608
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
609
|
+
if (!graph) return null;
|
|
610
|
+
const transports = new Set<object>();
|
|
611
|
+
const boundaries = new Set<object>();
|
|
612
|
+
const ownDatabaseNodes = new Set<object>();
|
|
613
|
+
const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
|
|
614
|
+
const ownChildren = (node: object): object[] =>
|
|
615
|
+
node instanceof DatabaseTransactionError
|
|
616
|
+
? // Callback failures retained beside a failed rollback supply vetoes,
|
|
617
|
+
// never driver provenance for a provider's connection-looking error.
|
|
618
|
+
[...graph.get(node)!].filter((child) => child === node.cause)
|
|
619
|
+
: [...graph.get(node)!];
|
|
620
|
+
// Ask the canonical transport predicate about ONLY this node's facts. Its
|
|
621
|
+
// recursive search must not pair a DB sibling with an unrelated provider.
|
|
622
|
+
for (const node of graph.keys()) {
|
|
623
|
+
if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
|
|
624
|
+
return null;
|
|
625
|
+
const record = node as Record<string, unknown>;
|
|
626
|
+
if (
|
|
627
|
+
requireDatabaseProvenance
|
|
628
|
+
? isRunningTurnDatabaseTransportFailure(record)
|
|
629
|
+
: isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno })
|
|
630
|
+
)
|
|
631
|
+
transports.add(node);
|
|
632
|
+
const sqlState = isSessionEventPersistenceError(node)
|
|
633
|
+
? node.details.sqlState
|
|
634
|
+
: record.name === "PostgresError" && typeof record.code === "string"
|
|
635
|
+
? record.code
|
|
636
|
+
: null;
|
|
637
|
+
if (
|
|
638
|
+
requireDatabaseProvenance
|
|
639
|
+
? isRunningTurnDatabaseConnectionSqlState(sqlState)
|
|
640
|
+
: isDatabaseConnectionSqlState(sqlState)
|
|
641
|
+
)
|
|
642
|
+
transports.add(node);
|
|
643
|
+
if (
|
|
644
|
+
node instanceof DrizzleQueryError ||
|
|
645
|
+
node instanceof DatabaseTransactionError ||
|
|
646
|
+
isSessionEventPersistenceError(node)
|
|
647
|
+
) {
|
|
648
|
+
// Only actual errors raised at our ORM/typed persistence boundary own
|
|
649
|
+
// their driver subtree. A PostgresError name, SDK wrapper or provider
|
|
650
|
+
// socket by itself is never own-client provenance for a running turn.
|
|
651
|
+
const queue: object[] = [node];
|
|
652
|
+
for (const source of queue) {
|
|
653
|
+
if (ownDatabaseNodes.has(source)) continue;
|
|
654
|
+
ownDatabaseNodes.add(source);
|
|
655
|
+
queue.push(...ownChildren(source));
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const hasOwnTransport = (boundary: object): boolean => {
|
|
660
|
+
const seen = new Set<object>();
|
|
661
|
+
const queue = [boundary];
|
|
662
|
+
for (const node of queue) {
|
|
663
|
+
if (seen.has(node)) continue;
|
|
664
|
+
seen.add(node);
|
|
665
|
+
if (transports.has(node)) return true;
|
|
666
|
+
queue.push(...ownChildren(node));
|
|
667
|
+
}
|
|
668
|
+
return false;
|
|
669
|
+
};
|
|
670
|
+
// Inspect ALL trusted DB evidence before returning a permit. A permanent,
|
|
671
|
+
// auth or uncertain SQLSTATE vetoes every sibling regardless of traversal
|
|
672
|
+
// order; a deeper transport/reset cannot override it either.
|
|
673
|
+
for (const node of graph.keys()) {
|
|
674
|
+
const record = node as Record<string, unknown>;
|
|
675
|
+
if (node instanceof DrizzleQueryError || node instanceof DatabaseTransactionError) {
|
|
676
|
+
boundaries.add(node);
|
|
677
|
+
if (hasOwnTransport(node)) codes.add("db_failure");
|
|
678
|
+
}
|
|
679
|
+
const typed = isSessionEventPersistenceError(node);
|
|
680
|
+
if (!typed && record.name !== "PostgresError") continue;
|
|
681
|
+
boundaries.add(node);
|
|
682
|
+
const sqlState = typed
|
|
683
|
+
? node.details.sqlState
|
|
684
|
+
: typeof record.code === "string"
|
|
685
|
+
? record.code
|
|
686
|
+
: null;
|
|
687
|
+
const connectionOutage = requireDatabaseProvenance
|
|
688
|
+
? isRunningTurnDatabaseConnectionSqlState(sqlState)
|
|
689
|
+
: isDatabaseConnectionSqlState(sqlState);
|
|
690
|
+
// PostgreSQL aborts a deadlock or serialization victim's whole
|
|
691
|
+
// transaction: unlike a lost connection, that write certainly did not
|
|
692
|
+
// commit, so it is at least as safe for exact-attempt recovery.
|
|
693
|
+
const rolledBack = isRetryablePersistenceSqlState(sqlState);
|
|
694
|
+
const code = typed
|
|
695
|
+
? retryablePersistenceFailureCode(sqlState)
|
|
696
|
+
: connectionOutage || rolledBack
|
|
697
|
+
? databaseFailureCode(sqlState)
|
|
698
|
+
: null;
|
|
699
|
+
if (
|
|
700
|
+
!code ||
|
|
701
|
+
(requireDatabaseProvenance &&
|
|
702
|
+
!connectionOutage &&
|
|
703
|
+
!rolledBack &&
|
|
704
|
+
!(sqlState === null && hasOwnTransport(node)))
|
|
705
|
+
)
|
|
706
|
+
return null;
|
|
707
|
+
if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
|
|
708
|
+
}
|
|
709
|
+
// Preserve the legacy pre-execution transport-only allowance, but never
|
|
710
|
+
// borrow it across an explicit DB boundary with no eligible own evidence.
|
|
711
|
+
if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
|
|
712
|
+
codes.add("db_failure");
|
|
713
|
+
// Stable classification for multiple positive DB siblings too.
|
|
714
|
+
for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"] as const)
|
|
715
|
+
if (codes.has(code)) return code;
|
|
716
|
+
return null;
|
|
717
|
+
} catch {
|
|
718
|
+
// Unreadable structured facts are no more authority than unreadable edges.
|
|
719
|
+
return null;
|
|
327
720
|
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
|
|
724
|
+
// postgres.js reports these for a physically lost connection, including
|
|
725
|
+
// transaction cleanup after its socket has already closed. Own-client
|
|
726
|
+
// provenance and the unknown-outcome veto remain mandatory above.
|
|
727
|
+
"CONNECTION_CLOSED",
|
|
728
|
+
"CONNECTION_DESTROYED",
|
|
729
|
+
"CONNECTION_ENDED",
|
|
730
|
+
"ECONNREFUSED",
|
|
731
|
+
"ECONNRESET",
|
|
732
|
+
"CONNECT_TIMEOUT",
|
|
733
|
+
]);
|
|
734
|
+
|
|
735
|
+
function isRunningTurnDatabaseTransportFailure(record: Record<string, unknown>): boolean {
|
|
736
|
+
return [record.code, record.errno].some(
|
|
737
|
+
(code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code),
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function isRunningTurnDatabaseConnectionSqlState(sqlState: string | null): boolean {
|
|
742
|
+
return (
|
|
743
|
+
sqlState !== null &&
|
|
744
|
+
(/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState))
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
function isDatabaseConnectionSqlState(sqlState: string | null): boolean {
|
|
749
|
+
return (
|
|
750
|
+
sqlState !== null &&
|
|
751
|
+
(sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState))
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
function retryablePersistenceFailureCode(
|
|
756
|
+
sqlState: string | null,
|
|
757
|
+
): PostClaimDatabaseRecoveryDetail["code"] | null {
|
|
328
758
|
if (
|
|
329
|
-
!persistenceFailure ||
|
|
330
759
|
!(
|
|
331
760
|
sqlState === null ||
|
|
332
761
|
sqlState.startsWith("08") ||
|
|
333
|
-
sqlState
|
|
762
|
+
isRetryablePersistenceSqlState(sqlState) ||
|
|
334
763
|
sqlState.startsWith("53") ||
|
|
335
764
|
sqlState === "55P03" ||
|
|
336
765
|
sqlState === "57014" ||
|
|
@@ -342,32 +771,59 @@ function retryableDatabaseFailureCode(
|
|
|
342
771
|
) {
|
|
343
772
|
return null;
|
|
344
773
|
}
|
|
345
|
-
return
|
|
774
|
+
return databaseFailureCode(sqlState);
|
|
346
775
|
}
|
|
347
776
|
|
|
348
777
|
/**
|
|
349
|
-
*
|
|
350
|
-
* recovery
|
|
351
|
-
*
|
|
778
|
+
* True when an error is a structured database failure that the exact-attempt
|
|
779
|
+
* database recovery path owns. Callers that translate other failures into
|
|
780
|
+
* user-visible states must rethrow these unchanged.
|
|
781
|
+
*/
|
|
782
|
+
export function isPostClaimDatabaseRecoveryCandidate(error: unknown): boolean {
|
|
783
|
+
return retryableDatabaseFailureCode(error, true) !== null;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Carry one exact claimed attempt into the workflow's DB-only
|
|
788
|
+
* recovery lane. Permanent database/state failures remain terminal. The
|
|
789
|
+
* running-turn lane additionally requires a closed own-client outage class;
|
|
790
|
+
* existing pre-execution recovery classifications are unchanged.
|
|
352
791
|
*/
|
|
353
792
|
export function postClaimDatabaseRecoveryFailure(input: {
|
|
354
793
|
error: unknown;
|
|
355
794
|
turnId: string;
|
|
356
795
|
triggerEventId: string;
|
|
357
796
|
executionGeneration: number;
|
|
797
|
+
/** Require an actual own-client boundary and the closed running-turn allowlist. */
|
|
798
|
+
requireDatabaseProvenance?: boolean;
|
|
799
|
+
sandboxSetupOutcomeUnknown?: true;
|
|
800
|
+
sandboxSetupRecoveryExhausted?: true;
|
|
358
801
|
providerRecovery?: {
|
|
359
802
|
failureCode: string;
|
|
360
803
|
providerRecoveryCount: number;
|
|
804
|
+
continueDelayMs?: number;
|
|
361
805
|
};
|
|
362
806
|
}): ApplicationFailure | null {
|
|
363
|
-
const code = retryableDatabaseFailureCode(input.error);
|
|
807
|
+
const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
|
|
364
808
|
if (!code || input.executionGeneration < 1) return null;
|
|
809
|
+
if (
|
|
810
|
+
(input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
|
|
811
|
+
((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
|
|
812
|
+
input.providerRecovery)
|
|
813
|
+
) {
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
365
816
|
if (
|
|
366
817
|
input.providerRecovery &&
|
|
367
818
|
(!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
|
|
368
819
|
input.providerRecovery.providerRecoveryCount <= 0 ||
|
|
369
|
-
input.providerRecovery.providerRecoveryCount >
|
|
370
|
-
|
|
820
|
+
input.providerRecovery.providerRecoveryCount >
|
|
821
|
+
providerRecoveryLimit(input.providerRecovery.failureCode) ||
|
|
822
|
+
!/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) ||
|
|
823
|
+
!validProviderOverloadRecoveryDelay(
|
|
824
|
+
input.providerRecovery.failureCode,
|
|
825
|
+
input.providerRecovery.continueDelayMs,
|
|
826
|
+
))
|
|
371
827
|
) {
|
|
372
828
|
return null;
|
|
373
829
|
}
|
|
@@ -376,13 +832,21 @@ export function postClaimDatabaseRecoveryFailure(input: {
|
|
|
376
832
|
triggerEventId: input.triggerEventId,
|
|
377
833
|
executionGeneration: input.executionGeneration,
|
|
378
834
|
code,
|
|
835
|
+
...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
|
|
836
|
+
...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
|
|
379
837
|
...(input.providerRecovery
|
|
380
838
|
? {
|
|
381
839
|
providerFailureCode: input.providerRecovery.failureCode,
|
|
382
840
|
providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
|
|
841
|
+
...(input.providerRecovery.continueDelayMs !== undefined
|
|
842
|
+
? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs }
|
|
843
|
+
: {}),
|
|
383
844
|
}
|
|
384
845
|
: {}),
|
|
385
846
|
};
|
|
847
|
+
// Known usage whose receipt write failed travels with the exact attempt.
|
|
848
|
+
const sessionUsageReceipts = heldSessionUsageReceiptsForTurn(input.turnId);
|
|
849
|
+
if (sessionUsageReceipts.length > 0) detail.sessionUsageReceipts = sessionUsageReceipts;
|
|
386
850
|
return ApplicationFailure.create({
|
|
387
851
|
message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
|
|
388
852
|
type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
|
|
@@ -462,8 +926,25 @@ export function sandboxLifecycleTransitionDiagnostic(
|
|
|
462
926
|
return null;
|
|
463
927
|
}
|
|
464
928
|
|
|
929
|
+
export function modelPreparationFailureEventPayload(error: unknown, durationMs: number) {
|
|
930
|
+
const transition = sandboxLifecycleTransitionDiagnostic(error);
|
|
931
|
+
return {
|
|
932
|
+
phase: "model_preparation",
|
|
933
|
+
durationMs: Math.max(0, Math.round(durationMs)),
|
|
934
|
+
expectedTransition: transition !== null,
|
|
935
|
+
...(transition
|
|
936
|
+
? {
|
|
937
|
+
failureCategory: "drain_capture_wait",
|
|
938
|
+
failureStage: "lifecycle_wait",
|
|
939
|
+
failureCode: transition.reason,
|
|
940
|
+
retryable: true,
|
|
941
|
+
}
|
|
942
|
+
: {}),
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
|
|
465
946
|
/**
|
|
466
|
-
* Recognize
|
|
947
|
+
* Recognize active-route transitions that cannot finish inside their
|
|
467
948
|
* originating attempt. A Modal-home session may start on a Connected Machine
|
|
468
949
|
* without creating or leasing its managed home box. When an explicit attach
|
|
469
950
|
* clears the active pointer back to home, the pointer commit is authoritative,
|
|
@@ -474,9 +955,56 @@ export function sandboxLifecycleTransitionDiagnostic(
|
|
|
474
955
|
* only those structural error links with a strict bound; never classify from
|
|
475
956
|
* message text, which could originate in model or tool content.
|
|
476
957
|
*/
|
|
958
|
+
/**
|
|
959
|
+
* Only the home resolver's positive pre-dispatch signal permits a fresh attempt.
|
|
960
|
+
* The same error class can also escape AFTER a provider mutation, so neither its
|
|
961
|
+
* retryable property nor message alone establishes replay safety. Inspect the
|
|
962
|
+
* complete bounded SDK cause graph and let uncertain peer outcomes veto this
|
|
963
|
+
* narrow recovery; unreadable or overflowing graphs fail closed.
|
|
964
|
+
*/
|
|
965
|
+
function isPreDispatchHomeBackendRecoveryRequired(error: unknown): boolean {
|
|
966
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
967
|
+
if (!graph) return false;
|
|
968
|
+
let found = false;
|
|
969
|
+
try {
|
|
970
|
+
for (const node of graph.keys()) {
|
|
971
|
+
if (
|
|
972
|
+
isRoutingMutationOutcomeUnknownError(node) ||
|
|
973
|
+
isRoutingMutationOutputRejectedError(node) ||
|
|
974
|
+
isRawProviderCommandOutcomeUnknown(node) ||
|
|
975
|
+
isModalCommandStartOutcomeUnknownError(node) ||
|
|
976
|
+
isProviderCommandObservationUnavailableError(node)
|
|
977
|
+
)
|
|
978
|
+
return false;
|
|
979
|
+
if (node instanceof RoutingBackendRecoveryRequiredError) {
|
|
980
|
+
if (
|
|
981
|
+
node.op !== "resolve_home_backend" ||
|
|
982
|
+
!Number.isSafeInteger(node.leaseEpoch) ||
|
|
983
|
+
node.leaseEpoch < 0 ||
|
|
984
|
+
!node.retryable ||
|
|
985
|
+
(node.recovery !== "pending" && node.recovery !== "superseded")
|
|
986
|
+
)
|
|
987
|
+
return false;
|
|
988
|
+
found = true;
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
} catch {
|
|
992
|
+
return false;
|
|
993
|
+
}
|
|
994
|
+
return found;
|
|
995
|
+
}
|
|
996
|
+
|
|
477
997
|
export function sandboxRouteTransitionCode(
|
|
478
998
|
error: unknown,
|
|
479
|
-
):
|
|
999
|
+
):
|
|
1000
|
+
| "home_unavailable_this_turn"
|
|
1001
|
+
| "workspace_root_changed_this_turn"
|
|
1002
|
+
| "native_capabilities_changed_this_attempt"
|
|
1003
|
+
| "home_backend_recovery_pending"
|
|
1004
|
+
| null {
|
|
1005
|
+
if (isPreDispatchHomeBackendRecoveryRequired(error)) {
|
|
1006
|
+
return "home_backend_recovery_pending";
|
|
1007
|
+
}
|
|
480
1008
|
const pending: unknown[] = [error];
|
|
481
1009
|
const seen = new WeakSet<object>();
|
|
482
1010
|
let inspected = 0;
|
|
@@ -489,6 +1017,13 @@ export function sandboxRouteTransitionCode(
|
|
|
489
1017
|
|
|
490
1018
|
try {
|
|
491
1019
|
const record = current as Record<string, unknown>;
|
|
1020
|
+
if (
|
|
1021
|
+
(current instanceof SandboxCapabilitiesChangedError ||
|
|
1022
|
+
record.name === "SandboxCapabilitiesChangedError") &&
|
|
1023
|
+
record.code === "native_capabilities_changed_this_attempt"
|
|
1024
|
+
) {
|
|
1025
|
+
return "native_capabilities_changed_this_attempt";
|
|
1026
|
+
}
|
|
492
1027
|
if (
|
|
493
1028
|
(current instanceof ActiveBackendUnresolvableError ||
|
|
494
1029
|
record.name === "ActiveBackendUnresolvableError") &&
|
|
@@ -612,6 +1147,14 @@ export function compactionFailureReasonFromError(error: unknown): string {
|
|
|
612
1147
|
`the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
|
|
613
1148
|
);
|
|
614
1149
|
}
|
|
1150
|
+
// An exhausted provider quota is not retried (see agentRunFailurePayload),
|
|
1151
|
+
// so name the refusal plainly instead of the raw diagnostic envelope.
|
|
1152
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
1153
|
+
if (quota) {
|
|
1154
|
+
return compactionFailureReason(
|
|
1155
|
+
`${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`,
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
615
1158
|
if (
|
|
616
1159
|
error instanceof CompactionProviderResponseError ||
|
|
617
1160
|
error instanceof EmptyCompactionSummaryError
|
|
@@ -638,8 +1181,12 @@ export function compactionFailureTurnEventPayload(
|
|
|
638
1181
|
recovery: "user_message";
|
|
639
1182
|
compacted: false;
|
|
640
1183
|
providerRejection?: CompactionProviderRejection;
|
|
1184
|
+
quotaScope?: ProviderQuotaScope;
|
|
641
1185
|
} {
|
|
642
1186
|
const rejection = compactionProviderRejection(error);
|
|
1187
|
+
// The same closed marker as a `provider_quota_exhausted` turn failure, so
|
|
1188
|
+
// clients can name the exhausted limit and offer another model here too.
|
|
1189
|
+
const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
|
|
643
1190
|
return {
|
|
644
1191
|
error: overrides.error ?? compactionFailureReasonFromError(error),
|
|
645
1192
|
code: "context_compaction_failed",
|
|
@@ -647,6 +1194,7 @@ export function compactionFailureTurnEventPayload(
|
|
|
647
1194
|
recovery: "user_message",
|
|
648
1195
|
compacted: false,
|
|
649
1196
|
...(rejection ? { providerRejection: rejection } : {}),
|
|
1197
|
+
...(quota ? { quotaScope: quota.scope } : {}),
|
|
650
1198
|
};
|
|
651
1199
|
}
|
|
652
1200
|
|
|
@@ -658,18 +1206,38 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
|
|
|
658
1206
|
|
|
659
1207
|
export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
|
|
660
1208
|
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
1209
|
+
// The worker's own included-usage refusal (usage exhausted or unverifiable)
|
|
1210
|
+
// happens before dispatch, so the checkpoint request was never sent. It
|
|
1211
|
+
// settles exactly like the same refusal on an ordinary request: recorded
|
|
1212
|
+
// against the account, then the same turn re-places or waits for capacity.
|
|
1213
|
+
if (findCodexCreditPolicyError(error)) return true;
|
|
661
1214
|
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
1215
|
+
// A Claude subscription refusal (rate limit or reconnect) on the checkpoint
|
|
1216
|
+
// request recovers exactly like one on an ordinary request: failure
|
|
1217
|
+
// settlement rotates to another account or waits for capacity on the same
|
|
1218
|
+
// logical turn. Its wording or retry hint never makes it a terminal quota.
|
|
1219
|
+
const claudeRefusal = isClaudeSubscriptionTransportError(error)
|
|
1220
|
+
? classifyClaudeProviderError(error)
|
|
1221
|
+
: null;
|
|
1222
|
+
if (claudeRefusal?.kind === "unauthorized" || claudeRefusal?.kind === "rate_limited") return true;
|
|
662
1223
|
// Codex may reject an opaque artifact it minted itself on the compaction
|
|
663
1224
|
// request exactly as it can on an ordinary request. Failure settlement
|
|
664
1225
|
// invalidates only the exact participating artifacts and recovers the same
|
|
665
1226
|
// logical turn; when nothing can be invalidated it fails closed there.
|
|
666
1227
|
if (classifyCodexEncryptedArtifactRejection(error)) return true;
|
|
1228
|
+
// A ChatGPT plan that no longer includes the model refuses the compaction
|
|
1229
|
+
// request exactly as it refuses an ordinary one (often with an empty 400).
|
|
1230
|
+
// Failure settlement re-checks the plan and, when it proves the loss,
|
|
1231
|
+
// excludes that account for the model and fails the same turn over; an
|
|
1232
|
+
// unexplained rejection still fails there with typed copy.
|
|
1233
|
+
if (classifyCodexEntitlementRejection(error)) return true;
|
|
667
1234
|
return agentRunFailurePayload(error).retryable === true;
|
|
668
1235
|
}
|
|
669
1236
|
|
|
670
1237
|
export function classifyContextWindowOverflowError(
|
|
671
1238
|
error: unknown,
|
|
672
1239
|
): { message: string; code?: string; detail?: string } | null {
|
|
1240
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
673
1241
|
const fields = collectErrorStrings(error);
|
|
674
1242
|
const matched = fields.find(
|
|
675
1243
|
(value) =>
|
|
@@ -736,6 +1304,9 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
|
|
|
736
1304
|
}
|
|
737
1305
|
seen.add(value);
|
|
738
1306
|
const out: string[] = [];
|
|
1307
|
+
// This diagnostic is already provider-owned and byte-bounded. Do not widen
|
|
1308
|
+
// generic detail traversal to arbitrary application payloads.
|
|
1309
|
+
if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
|
|
739
1310
|
const record = value as Record<string, unknown>;
|
|
740
1311
|
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
741
1312
|
const field = record[key];
|
|
@@ -792,15 +1363,47 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
|
|
|
792
1363
|
*/
|
|
793
1364
|
export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
|
|
794
1365
|
"unable to connect. is the computer able to access the url?";
|
|
1366
|
+
/** Bun's exact fetch text when an upstream drops an open connection. */
|
|
1367
|
+
export const STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE =
|
|
1368
|
+
"the socket connection was closed unexpectedly. for more information, pass `verbose: true` in the second argument to fetch()";
|
|
1369
|
+
const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES = new Set([
|
|
1370
|
+
STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE,
|
|
1371
|
+
STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE,
|
|
1372
|
+
]);
|
|
795
1373
|
|
|
796
1374
|
export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
|
|
797
|
-
return message.trim().toLowerCase()
|
|
1375
|
+
return STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES.has(message.trim().toLowerCase());
|
|
1376
|
+
}
|
|
1377
|
+
|
|
1378
|
+
const TRANSIENT_TRANSPORT_CODE = /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i;
|
|
1379
|
+
|
|
1380
|
+
/**
|
|
1381
|
+
* A network fault code anywhere on the Error.cause chain. SDK and runner
|
|
1382
|
+
* wrappers often keep the transport error only as `cause`, so the outermost
|
|
1383
|
+
* error's own `code` is not enough.
|
|
1384
|
+
*/
|
|
1385
|
+
function transientTransportCode(error: unknown): string | undefined {
|
|
1386
|
+
let current = error;
|
|
1387
|
+
const seen = new Set<unknown>();
|
|
1388
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1389
|
+
if (seen.has(current)) break;
|
|
1390
|
+
seen.add(current);
|
|
1391
|
+
const value = current as { code?: unknown; cause?: unknown };
|
|
1392
|
+
if (typeof value.code === "string" && TRANSIENT_TRANSPORT_CODE.test(value.code)) {
|
|
1393
|
+
return value.code;
|
|
1394
|
+
}
|
|
1395
|
+
current = value.cause;
|
|
1396
|
+
}
|
|
1397
|
+
return undefined;
|
|
798
1398
|
}
|
|
799
1399
|
|
|
800
1400
|
function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
|
|
1401
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1402
|
+
return error.category === "safety" ? error.detail : undefined;
|
|
1403
|
+
}
|
|
801
1404
|
return collectErrorStrings(error).find(
|
|
802
1405
|
(value) =>
|
|
803
|
-
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
|
|
1406
|
+
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
|
|
804
1407
|
value,
|
|
805
1408
|
) || /\bthis request was blocked by our safety systems\b/i.test(value),
|
|
806
1409
|
);
|
|
@@ -810,16 +1413,48 @@ function isProviderSafetyRefusal(error: unknown): boolean {
|
|
|
810
1413
|
return providerSafetyRefusalDiagnostic(error) !== undefined;
|
|
811
1414
|
}
|
|
812
1415
|
|
|
1416
|
+
/** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
|
|
1417
|
+
function providerHttpStatus(error: unknown): number | undefined {
|
|
1418
|
+
let current = error;
|
|
1419
|
+
const seen = new Set<unknown>();
|
|
1420
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1421
|
+
if (seen.has(current)) break;
|
|
1422
|
+
seen.add(current);
|
|
1423
|
+
const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
|
|
1424
|
+
const status = Number(value.status ?? value.statusCode);
|
|
1425
|
+
if (Number.isInteger(status) && status >= 100 && status < 600) return status;
|
|
1426
|
+
current = value.cause;
|
|
1427
|
+
}
|
|
1428
|
+
return undefined;
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
/**
|
|
1432
|
+
* A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
|
|
1433
|
+
* "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
|
|
1434
|
+
* Explicit cancellation (`AbortError`) is deliberately not matched.
|
|
1435
|
+
*/
|
|
1436
|
+
export function isTransportTimeoutError(error: unknown): boolean {
|
|
1437
|
+
let current: unknown = error;
|
|
1438
|
+
const seen = new Set<unknown>();
|
|
1439
|
+
for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
|
|
1440
|
+
if (seen.has(current)) break;
|
|
1441
|
+
seen.add(current);
|
|
1442
|
+
const value = current as { name?: unknown; message?: unknown; cause?: unknown };
|
|
1443
|
+
if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
|
|
1444
|
+
return true;
|
|
1445
|
+
}
|
|
1446
|
+
current = value.cause;
|
|
1447
|
+
}
|
|
1448
|
+
return false;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
813
1451
|
export function isTransientProviderError(error: unknown): boolean {
|
|
1452
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1453
|
+
return error.category === "unavailable";
|
|
1454
|
+
}
|
|
814
1455
|
// A semantic refusal can arrive inside a 5xx transport envelope.
|
|
815
1456
|
if (isProviderSafetyRefusal(error)) return false;
|
|
816
|
-
const status =
|
|
817
|
-
typeof error === "object" && error !== null
|
|
818
|
-
? Number(
|
|
819
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
820
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
821
|
-
)
|
|
822
|
-
: undefined;
|
|
1457
|
+
const status = providerHttpStatus(error);
|
|
823
1458
|
// A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
|
|
824
1459
|
// (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
|
|
825
1460
|
// a request fault that must NOT auto-retry — even if its body happens to read like
|
|
@@ -828,13 +1463,7 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
828
1463
|
if (status !== undefined && Number.isFinite(status)) {
|
|
829
1464
|
return status >= 500 && status < 600;
|
|
830
1465
|
}
|
|
831
|
-
|
|
832
|
-
typeof error === "object" && error !== null && "code" in error
|
|
833
|
-
? String((error as { code?: unknown }).code)
|
|
834
|
-
: undefined;
|
|
835
|
-
if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
|
|
836
|
-
return true;
|
|
837
|
-
}
|
|
1466
|
+
if (transientTransportCode(error)) return true;
|
|
838
1467
|
const message = error instanceof Error ? error.message : String(error);
|
|
839
1468
|
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
840
1469
|
return true;
|
|
@@ -844,23 +1473,58 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
844
1473
|
);
|
|
845
1474
|
}
|
|
846
1475
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
}
|
|
1476
|
+
/** An explicit `usage_limit_reached` type or code string anywhere on the error chain. */
|
|
1477
|
+
function hasCodexUsageLimitType(error: unknown): boolean {
|
|
1478
|
+
return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
|
|
1479
|
+
}
|
|
851
1480
|
|
|
852
1481
|
/**
|
|
853
|
-
*
|
|
854
|
-
*
|
|
855
|
-
*
|
|
856
|
-
*
|
|
1482
|
+
* Recognize an exhausted API-key provider quota (a daily or monthly allowance,
|
|
1483
|
+
* a free-tier day cap, or an account out of credits) as distinct from an
|
|
1484
|
+
* ordinary per-minute rate limit. Retrying within the bounded same-turn budget
|
|
1485
|
+
* cannot succeed, so the turn fails promptly instead. Subscription transports
|
|
1486
|
+
* own their quota semantics through credential rotation and durable capacity
|
|
1487
|
+
* waits, so a Codex, SuperGrok or Claude subscription transport error never
|
|
1488
|
+
* classifies here.
|
|
857
1489
|
*/
|
|
858
|
-
export function
|
|
1490
|
+
export function classifyProviderQuotaExhaustionError(
|
|
1491
|
+
error: unknown,
|
|
1492
|
+
): ProviderQuotaExhaustion | null {
|
|
1493
|
+
if (
|
|
1494
|
+
isCodexTransportError(error) ||
|
|
1495
|
+
isXaiSubscriptionTransportError(error) ||
|
|
1496
|
+
isClaudeSubscriptionTransportError(error)
|
|
1497
|
+
)
|
|
1498
|
+
return null;
|
|
1499
|
+
// The same reader the OpenAI SDK retry veto uses, so the two never disagree.
|
|
1500
|
+
return classifyProviderQuotaError(error);
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
/** A refusal raised by a Claude subscription (OAuth) transport, at any wrapper depth. */
|
|
1504
|
+
export function isClaudeSubscriptionTransportError(error: unknown): boolean {
|
|
1505
|
+
let current: unknown = error;
|
|
1506
|
+
for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
|
|
1507
|
+
if (current instanceof AnthropicRequestError && current.subscription) return true;
|
|
1508
|
+
current = current.cause;
|
|
1509
|
+
}
|
|
1510
|
+
return false;
|
|
1511
|
+
}
|
|
1512
|
+
|
|
1513
|
+
/**
|
|
1514
|
+
* SuperGrok's classification of a failed model request as a shared provider
|
|
1515
|
+
* outcome (the adapter's `classifyError`, SuperGrok track X1b): a refresh
|
|
1516
|
+
* relogin or a marked 401 is `unauthorized`, a marked 403 `forbidden`, and a
|
|
1517
|
+
* marked 429 or HTTP 200 SSE capacity terminal `rate_limited` with the
|
|
1518
|
+
* provider's retry time (null when it gave none: the adapter's fallback
|
|
1519
|
+
* applies). Anything else, a provider safety refusal included, is null: not
|
|
1520
|
+
* a refusal of the account.
|
|
1521
|
+
*/
|
|
1522
|
+
export function classifyXaiProviderError(error: unknown): ProviderErrorOutcome | null {
|
|
859
1523
|
if (isProviderSafetyRefusal(error)) return null;
|
|
860
1524
|
let relogin: unknown = error;
|
|
861
1525
|
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
862
1526
|
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
863
|
-
return { kind: "
|
|
1527
|
+
return { kind: "unauthorized" };
|
|
864
1528
|
}
|
|
865
1529
|
relogin = (relogin as Record<string, unknown>).cause;
|
|
866
1530
|
}
|
|
@@ -875,10 +1539,10 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
875
1539
|
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
876
1540
|
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
877
1541
|
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
878
|
-
return { kind: "
|
|
1542
|
+
return { kind: "unauthorized" };
|
|
879
1543
|
}
|
|
880
1544
|
if (status === 403) {
|
|
881
|
-
return { kind: "forbidden"
|
|
1545
|
+
return { kind: "forbidden" };
|
|
882
1546
|
}
|
|
883
1547
|
const message = String(value.message ?? body?.message ?? "");
|
|
884
1548
|
if (
|
|
@@ -888,10 +1552,7 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
888
1552
|
status: Number.isInteger(status) ? status : null,
|
|
889
1553
|
})
|
|
890
1554
|
) {
|
|
891
|
-
return {
|
|
892
|
-
kind: "rate_limit",
|
|
893
|
-
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
894
|
-
};
|
|
1555
|
+
return { kind: "rate_limited", retryAfterMs: providerRetryAfterMs(error) };
|
|
895
1556
|
}
|
|
896
1557
|
current = value.cause;
|
|
897
1558
|
}
|
|
@@ -918,15 +1579,167 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
|
|
|
918
1579
|
return null;
|
|
919
1580
|
}
|
|
920
1581
|
|
|
1582
|
+
function isRawDatabaseQueryError(error: unknown): boolean {
|
|
1583
|
+
return (
|
|
1584
|
+
error instanceof Error &&
|
|
1585
|
+
"query" in error &&
|
|
1586
|
+
typeof error.query === "string" &&
|
|
1587
|
+
"params" in error &&
|
|
1588
|
+
Array.isArray(error.params)
|
|
1589
|
+
);
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
|
|
1593
|
+
if (error instanceof AnthropicRequestError) return error;
|
|
1594
|
+
return error instanceof Error && error.cause instanceof AnthropicRequestError
|
|
1595
|
+
? error.cause
|
|
1596
|
+
: undefined;
|
|
1597
|
+
}
|
|
1598
|
+
|
|
1599
|
+
function isRawProviderCommandOutcomeUnknown(error: unknown): boolean {
|
|
1600
|
+
try {
|
|
1601
|
+
return (
|
|
1602
|
+
error instanceof ProviderCommandInputOutcomeUnknownError ||
|
|
1603
|
+
error instanceof ProviderCommandStartOutcomeUnknownError
|
|
1604
|
+
);
|
|
1605
|
+
} catch {
|
|
1606
|
+
return false;
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
|
|
921
1610
|
export function agentRunFailurePayload(
|
|
922
1611
|
error: unknown,
|
|
923
1612
|
options: { isCodexTurn?: boolean } = {},
|
|
924
1613
|
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1614
|
+
return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
/**
|
|
1618
|
+
* Record the closed provider condition on a retryable model-provider failure so
|
|
1619
|
+
* clients can say "overloaded" rather than a generic outage. Presentation
|
|
1620
|
+
* evidence only: retry authority and pacing are decided by `code`.
|
|
1621
|
+
*/
|
|
1622
|
+
function withProviderCondition(
|
|
1623
|
+
error: unknown,
|
|
1624
|
+
failure: ReturnType<typeof baseAgentRunFailurePayload>,
|
|
1625
|
+
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1626
|
+
if (failure.code === "provider_rate_limited") {
|
|
1627
|
+
return { ...failure, providerCondition: "rate_limited" };
|
|
1628
|
+
}
|
|
1629
|
+
if (failure.code !== "provider_unavailable") return failure;
|
|
1630
|
+
if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
|
|
1631
|
+
return {
|
|
1632
|
+
...failure,
|
|
1633
|
+
providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable",
|
|
1634
|
+
};
|
|
1635
|
+
}
|
|
1636
|
+
|
|
1637
|
+
/** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
|
|
1638
|
+
export function isProviderOverloadError(
|
|
1639
|
+
error: unknown,
|
|
1640
|
+
failure: { error?: string; detail?: string } = {},
|
|
1641
|
+
): boolean {
|
|
1642
|
+
if (providerHttpStatus(error) === 529) return true;
|
|
1643
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
1644
|
+
return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
|
|
1645
|
+
(value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value),
|
|
1646
|
+
);
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
/** Retry authority must not depend on the looser presentation-only overload label. */
|
|
1650
|
+
export function providerRecoveryCode(
|
|
1651
|
+
error: unknown,
|
|
1652
|
+
failure: { code?: string; retryable?: boolean },
|
|
1653
|
+
): string | undefined {
|
|
1654
|
+
if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
|
|
1655
|
+
const status = providerHttpStatus(error);
|
|
1656
|
+
let current = error;
|
|
1657
|
+
const seen = new Set<unknown>();
|
|
1658
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1659
|
+
if (seen.has(current)) break;
|
|
1660
|
+
seen.add(current);
|
|
1661
|
+
if (
|
|
1662
|
+
current instanceof AnthropicRequestError &&
|
|
1663
|
+
status !== undefined &&
|
|
1664
|
+
status >= 500 &&
|
|
1665
|
+
status < 600 &&
|
|
1666
|
+
(status === 529 || current.errorType === "overloaded_error")
|
|
1667
|
+
) {
|
|
1668
|
+
return PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
1669
|
+
}
|
|
1670
|
+
current = (current as { cause?: unknown }).cause;
|
|
1671
|
+
}
|
|
1672
|
+
return failure.code;
|
|
1673
|
+
}
|
|
1674
|
+
|
|
1675
|
+
function classifyAgentRunFailurePayload(
|
|
1676
|
+
error: unknown,
|
|
1677
|
+
options: { isCodexTurn?: boolean } = {},
|
|
1678
|
+
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1679
|
+
const sizeError = findAnthropicRequestSizeError(error);
|
|
1680
|
+
if (sizeError)
|
|
1681
|
+
return {
|
|
1682
|
+
error: sizeError.message,
|
|
1683
|
+
code: sizeError.code,
|
|
1684
|
+
retryable: false,
|
|
1685
|
+
requestSize: sizeError.requestSize,
|
|
1686
|
+
...(sizeError.request_id ? { requestId: sizeError.request_id } : {}),
|
|
1687
|
+
};
|
|
1688
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
1689
|
+
const nodes = graph ? [...graph.keys()] : [];
|
|
1690
|
+
const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
|
|
1691
|
+
if (outputRejected) {
|
|
1692
|
+
// A known receipt applies only to its own mutation. Uncertain peers retain
|
|
1693
|
+
// their established terminal classification, regardless of SDK graph order.
|
|
1694
|
+
const unknown = nodes.find(
|
|
1695
|
+
(node) =>
|
|
1696
|
+
isRoutingMutationOutcomeUnknownError(node) ||
|
|
1697
|
+
isModalCommandStartOutcomeUnknownError(node) ||
|
|
1698
|
+
isProviderCommandObservationUnavailableError(node) ||
|
|
1699
|
+
isRawProviderCommandOutcomeUnknown(node),
|
|
1700
|
+
);
|
|
1701
|
+
if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
|
|
1702
|
+
return {
|
|
1703
|
+
error: outputRejected.message,
|
|
1704
|
+
code: outputRejected.code,
|
|
1705
|
+
retryable: false,
|
|
1706
|
+
};
|
|
1707
|
+
}
|
|
1708
|
+
const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
|
|
1709
|
+
if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
|
|
925
1710
|
const failure = baseAgentRunFailurePayload(error, options);
|
|
926
1711
|
const diagnostic = materializationVerificationDiagnostic(error);
|
|
1712
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
1713
|
+
if (anthropic) {
|
|
1714
|
+
const authenticationRejected =
|
|
1715
|
+
anthropic.status === 401 &&
|
|
1716
|
+
(error === anthropic ||
|
|
1717
|
+
(error instanceof Error && (error as Error & { status?: unknown }).status === 401));
|
|
1718
|
+
return {
|
|
1719
|
+
...failure,
|
|
1720
|
+
code:
|
|
1721
|
+
failure.code ??
|
|
1722
|
+
(authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
|
|
1723
|
+
retryable: failure.retryable ?? false,
|
|
1724
|
+
...(anthropic.detail ? { detail: anthropic.detail } : {}),
|
|
1725
|
+
...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
927
1728
|
return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
|
|
928
1729
|
}
|
|
929
1730
|
|
|
1731
|
+
/** Keep Anthropic provider text on terminal failures, never recovery events. */
|
|
1732
|
+
export function agentRunRecoveryFailurePayload(
|
|
1733
|
+
error: unknown,
|
|
1734
|
+
failure: ReturnType<typeof agentRunFailurePayload>,
|
|
1735
|
+
): ReturnType<typeof agentRunFailurePayload> {
|
|
1736
|
+
if (!anthropicRequestDiagnostic(error)) return failure;
|
|
1737
|
+
// Project a copy: retry exhaustion still needs the terminal diagnostic.
|
|
1738
|
+
const recovery = { ...failure };
|
|
1739
|
+
delete recovery.detail;
|
|
1740
|
+
return recovery;
|
|
1741
|
+
}
|
|
1742
|
+
|
|
930
1743
|
function baseAgentRunFailurePayload(
|
|
931
1744
|
error: unknown,
|
|
932
1745
|
options: { isCodexTurn?: boolean } = {},
|
|
@@ -950,6 +1763,22 @@ function baseAgentRunFailurePayload(
|
|
|
950
1763
|
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
951
1764
|
mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
|
|
952
1765
|
materializationDiagnostic?: MaterializationVerificationDiagnostic;
|
|
1766
|
+
requestSize?: AnthropicRequestSize;
|
|
1767
|
+
quotaScope?: ProviderQuotaScope;
|
|
1768
|
+
/** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
|
|
1769
|
+
planType?: string | null;
|
|
1770
|
+
/** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
|
|
1771
|
+
model?: string | null;
|
|
1772
|
+
/** Display label of the turn's model on model-provider recovery evidence. */
|
|
1773
|
+
modelLabel?: string;
|
|
1774
|
+
/** Display label of the serving provider on model-provider recovery evidence. */
|
|
1775
|
+
providerLabel?: string;
|
|
1776
|
+
/** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
|
|
1777
|
+
providerCondition?: ProviderCondition;
|
|
1778
|
+
recoveryExhausted?: boolean;
|
|
1779
|
+
providerRecoveryCount?: number;
|
|
1780
|
+
maxProviderRecoveryCount?: number;
|
|
1781
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
953
1782
|
} {
|
|
954
1783
|
if (error instanceof SandboxMaterializationVerificationError) {
|
|
955
1784
|
return {
|
|
@@ -963,7 +1792,11 @@ function baseAgentRunFailurePayload(
|
|
|
963
1792
|
return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
|
|
964
1793
|
}
|
|
965
1794
|
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
966
|
-
const
|
|
1795
|
+
const databaseFailure =
|
|
1796
|
+
isSessionEventPersistenceError(error.cause) ||
|
|
1797
|
+
findPostgresDriverError(error.cause) !== null ||
|
|
1798
|
+
isRawDatabaseQueryError(error.cause);
|
|
1799
|
+
const underlying = databaseFailure
|
|
967
1800
|
? agentRunFailurePayload(error.cause, options)
|
|
968
1801
|
: {
|
|
969
1802
|
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
@@ -973,6 +1806,32 @@ function baseAgentRunFailurePayload(
|
|
|
973
1806
|
historyPersistenceStage: error.stage,
|
|
974
1807
|
};
|
|
975
1808
|
}
|
|
1809
|
+
// Raw ORM wrappers can contain the full SQL and its parameters. Classify a
|
|
1810
|
+
// real driver before provider message heuristics; the original cause stays
|
|
1811
|
+
// available to internal diagnostics. This payload grants no replay authority.
|
|
1812
|
+
const postgresDriverError = findPostgresDriverError(error);
|
|
1813
|
+
const rawOrmFailure = isRawDatabaseQueryError(error);
|
|
1814
|
+
if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
|
|
1815
|
+
const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
|
|
1816
|
+
const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
|
|
1817
|
+
return {
|
|
1818
|
+
error: "Opengeni encountered a database error.",
|
|
1819
|
+
code: databaseFailureCode(sqlState),
|
|
1820
|
+
sqlState,
|
|
1821
|
+
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1822
|
+
};
|
|
1823
|
+
}
|
|
1824
|
+
if (error instanceof AnthropicSizeRecoveryExhaustedError) {
|
|
1825
|
+
return { error: error.message, code: error.code, retryable: false };
|
|
1826
|
+
}
|
|
1827
|
+
if (error instanceof AnthropicProviderRejection) {
|
|
1828
|
+
return {
|
|
1829
|
+
error: error.message,
|
|
1830
|
+
code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
|
|
1831
|
+
retryable: false,
|
|
1832
|
+
...(error.request_id ? { requestId: error.request_id } : {}),
|
|
1833
|
+
};
|
|
1834
|
+
}
|
|
976
1835
|
const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
|
|
977
1836
|
if (safetyRefusalDiagnostic !== undefined) {
|
|
978
1837
|
return {
|
|
@@ -983,14 +1842,43 @@ function baseAgentRunFailurePayload(
|
|
|
983
1842
|
detail: safetyRefusalDiagnostic,
|
|
984
1843
|
};
|
|
985
1844
|
}
|
|
1845
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1846
|
+
const quota =
|
|
1847
|
+
error.category === "unknown" || error.category === "rate_limit"
|
|
1848
|
+
? classifyProviderQuotaExhaustionError({
|
|
1849
|
+
code: error.code,
|
|
1850
|
+
error: { code: error.code, type: error.type, message: error.detail },
|
|
1851
|
+
retryAfterSeconds: error.retryAfterSeconds,
|
|
1852
|
+
})
|
|
1853
|
+
: null;
|
|
1854
|
+
if (quota) {
|
|
1855
|
+
return {
|
|
1856
|
+
error: providerQuotaExhaustedMessage(quota.scope),
|
|
1857
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
1858
|
+
retryable: false,
|
|
1859
|
+
quotaScope: quota.scope,
|
|
1860
|
+
detail: error.detail,
|
|
1861
|
+
};
|
|
1862
|
+
}
|
|
1863
|
+
return {
|
|
1864
|
+
error:
|
|
1865
|
+
error.category === "rate_limit"
|
|
1866
|
+
? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort."
|
|
1867
|
+
: error.category === "unavailable"
|
|
1868
|
+
? "The model provider is temporarily unavailable. The same turn will retry after a short delay."
|
|
1869
|
+
: "The model provider rejected the response. Automatic retries stopped.",
|
|
1870
|
+
code:
|
|
1871
|
+
error.category === "rate_limit"
|
|
1872
|
+
? "provider_rate_limited"
|
|
1873
|
+
: error.category === "unavailable"
|
|
1874
|
+
? "provider_unavailable"
|
|
1875
|
+
: "provider_request_rejected",
|
|
1876
|
+
retryable: error.category === "rate_limit" || error.category === "unavailable",
|
|
1877
|
+
detail: error.detail,
|
|
1878
|
+
};
|
|
1879
|
+
}
|
|
986
1880
|
const message = error instanceof Error ? error.message : String(error);
|
|
987
|
-
const status =
|
|
988
|
-
typeof error === "object" && error !== null
|
|
989
|
-
? Number(
|
|
990
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
991
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
992
|
-
)
|
|
993
|
-
: undefined;
|
|
1881
|
+
const status = providerHttpStatus(error);
|
|
994
1882
|
const code =
|
|
995
1883
|
typeof error === "object" && error !== null && "code" in error
|
|
996
1884
|
? String((error as { code?: unknown }).code)
|
|
@@ -1021,10 +1909,29 @@ function baseAgentRunFailurePayload(
|
|
|
1021
1909
|
retryable: true,
|
|
1022
1910
|
};
|
|
1023
1911
|
}
|
|
1024
|
-
if (
|
|
1912
|
+
if (isProviderCommandObservationUnavailableError(error)) {
|
|
1913
|
+
return {
|
|
1914
|
+
error:
|
|
1915
|
+
"A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
1916
|
+
code: "sandbox_command_observation_unavailable",
|
|
1917
|
+
retryable: false,
|
|
1918
|
+
};
|
|
1919
|
+
}
|
|
1920
|
+
if (isModalCommandStartOutcomeUnknownError(error)) {
|
|
1921
|
+
return {
|
|
1922
|
+
error:
|
|
1923
|
+
"A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
1924
|
+
code: "sandbox_command_start_outcome_unknown",
|
|
1925
|
+
retryable: false,
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
if (
|
|
1929
|
+
!isRoutingMutationOutcomeUnknownError(error) &&
|
|
1930
|
+
isModalTaskExecStartPreDispatchUnavailableError(error)
|
|
1931
|
+
) {
|
|
1025
1932
|
return {
|
|
1026
1933
|
error:
|
|
1027
|
-
"The managed sandbox command
|
|
1934
|
+
"The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
|
|
1028
1935
|
code: "sandbox_command_start_unavailable",
|
|
1029
1936
|
retryable: true,
|
|
1030
1937
|
};
|
|
@@ -1089,6 +1996,33 @@ function baseAgentRunFailurePayload(
|
|
|
1089
1996
|
...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
|
|
1090
1997
|
};
|
|
1091
1998
|
}
|
|
1999
|
+
// Codex plan entitlement: the settlement path normally re-checks the plan
|
|
2000
|
+
// and records a precise payload. These branches keep any other path from
|
|
2001
|
+
// surfacing the SDK's raw "400 status code (no body)" text.
|
|
2002
|
+
if (error instanceof CodexPlanEntitlementError) {
|
|
2003
|
+
return error.payload;
|
|
2004
|
+
}
|
|
2005
|
+
// Shared subscription core placement outcomes carry their own typed copy.
|
|
2006
|
+
if (error instanceof SubscriptionCoreCodexTurnError) {
|
|
2007
|
+
return error.payload;
|
|
2008
|
+
}
|
|
2009
|
+
const entitlementRejection = classifyCodexEntitlementRejection(error);
|
|
2010
|
+
if (entitlementRejection) {
|
|
2011
|
+
return entitlementRejection.evidence === "plan_entitlement"
|
|
2012
|
+
? codexPlanEntitlementFailurePayload({
|
|
2013
|
+
accountLabel: null,
|
|
2014
|
+
planType: null,
|
|
2015
|
+
planChanged: false,
|
|
2016
|
+
modelId: null,
|
|
2017
|
+
rejection: entitlementRejection,
|
|
2018
|
+
})
|
|
2019
|
+
: codexRequestRejectedFailurePayload({
|
|
2020
|
+
accountLabel: null,
|
|
2021
|
+
planType: null,
|
|
2022
|
+
rejection: entitlementRejection,
|
|
2023
|
+
planChecked: false,
|
|
2024
|
+
});
|
|
2025
|
+
}
|
|
1092
2026
|
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
1093
2027
|
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
1094
2028
|
// goal against a capped backend). Surface a precise, actionable message with
|
|
@@ -1099,8 +2033,15 @@ function baseAgentRunFailurePayload(
|
|
|
1099
2033
|
// `usage_limit_reached` shape must still outrank generic 429 retryability.
|
|
1100
2034
|
// Credential quarantine/failover remains separately provenance-gated by
|
|
1101
2035
|
// `isCodexTransportError`; this branch only chooses the truthful user payload.
|
|
2036
|
+
// The looser "429 ... usage limit" wording counts only on a Codex transport
|
|
2037
|
+
// error: an API-key provider's 429 that says "usage limit" is provider quota
|
|
2038
|
+
// evidence, not a ChatGPT/Codex subscription cap.
|
|
2039
|
+
const creditPolicy = findCodexCreditPolicyError(error);
|
|
2040
|
+
if (creditPolicy) {
|
|
2041
|
+
return { error: creditPolicy.message, code: creditPolicy.code, retryable: false };
|
|
2042
|
+
}
|
|
1102
2043
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1103
|
-
if (usageLimit) {
|
|
2044
|
+
if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
|
|
1104
2045
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
1105
2046
|
}
|
|
1106
2047
|
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
@@ -1144,6 +2085,39 @@ function baseAgentRunFailurePayload(
|
|
|
1144
2085
|
...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
|
|
1145
2086
|
};
|
|
1146
2087
|
}
|
|
2088
|
+
// A generic model stream that went silent mid-response (typed idle bound) or
|
|
2089
|
+
// a transport timeout ("The operation timed out." DOMException TimeoutError
|
|
2090
|
+
// from the fetch layer) is a dead upstream connection, not a request fault:
|
|
2091
|
+
// checkpoint durable truth and recover the same turn within the finite
|
|
2092
|
+
// provider recovery budget. Codex/SuperGrok typed timeouts are classified
|
|
2093
|
+
// above under their own transport policy.
|
|
2094
|
+
const streamIdle = classifyModelStreamIdleTimeoutError(error);
|
|
2095
|
+
if (streamIdle) {
|
|
2096
|
+
const seconds =
|
|
2097
|
+
streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1_000) : null;
|
|
2098
|
+
const window = seconds !== null ? ` for ${seconds}s` : "";
|
|
2099
|
+
return {
|
|
2100
|
+
error:
|
|
2101
|
+
streamIdle.kind === "progress"
|
|
2102
|
+
? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.`
|
|
2103
|
+
: `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
|
|
2104
|
+
code: "provider_unavailable",
|
|
2105
|
+
retryable: true,
|
|
2106
|
+
timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
|
|
2107
|
+
responseObserved: true,
|
|
2108
|
+
detail: streamIdle.message,
|
|
2109
|
+
};
|
|
2110
|
+
}
|
|
2111
|
+
if (status === undefined && isTransportTimeoutError(error)) {
|
|
2112
|
+
return {
|
|
2113
|
+
error:
|
|
2114
|
+
"The model provider connection timed out. The same turn will retry after a short delay.",
|
|
2115
|
+
code: "provider_unavailable",
|
|
2116
|
+
retryable: true,
|
|
2117
|
+
timeoutClass: "transport",
|
|
2118
|
+
detail: message,
|
|
2119
|
+
};
|
|
2120
|
+
}
|
|
1147
2121
|
if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
|
|
1148
2122
|
return {
|
|
1149
2123
|
error:
|
|
@@ -1152,10 +2126,33 @@ function baseAgentRunFailurePayload(
|
|
|
1152
2126
|
retryable: true,
|
|
1153
2127
|
};
|
|
1154
2128
|
}
|
|
2129
|
+
// An exhausted quota also arrives as HTTP 429 (or 402), but no retry within
|
|
2130
|
+
// the finite same-turn budget can succeed. Fail the turn promptly with a
|
|
2131
|
+
// distinct code so the client can offer another model; ordinary short rate
|
|
2132
|
+
// limits fall through to the retryable branch below.
|
|
2133
|
+
if (status === 402 && code === "anthropic_billing_error") {
|
|
2134
|
+
return {
|
|
2135
|
+
error: "Claude could not bill this request. Check the account's billing and payment details.",
|
|
2136
|
+
code: "provider_billing_error",
|
|
2137
|
+
retryable: false,
|
|
2138
|
+
...(message ? { detail: message } : {}),
|
|
2139
|
+
};
|
|
2140
|
+
}
|
|
2141
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
2142
|
+
if (quota) {
|
|
2143
|
+
return {
|
|
2144
|
+
error: providerQuotaExhaustedMessage(quota.scope),
|
|
2145
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
2146
|
+
retryable: false,
|
|
2147
|
+
quotaScope: quota.scope,
|
|
2148
|
+
...(message ? { detail: message } : {}),
|
|
2149
|
+
};
|
|
2150
|
+
}
|
|
1155
2151
|
if (
|
|
1156
2152
|
status === 429 ||
|
|
1157
|
-
|
|
1158
|
-
|
|
2153
|
+
((status === undefined || !Number.isFinite(status)) &&
|
|
2154
|
+
(code === "rate_limit_exceeded" ||
|
|
2155
|
+
/(?:too many requests|rate.?limit|\b429\b)/i.test(message)))
|
|
1159
2156
|
) {
|
|
1160
2157
|
return {
|
|
1161
2158
|
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
@@ -1172,31 +2169,24 @@ function baseAgentRunFailurePayload(
|
|
|
1172
2169
|
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
1173
2170
|
return {
|
|
1174
2171
|
error:
|
|
1175
|
-
"
|
|
2172
|
+
"Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
1176
2173
|
code: "upstream_connectivity_unavailable",
|
|
1177
2174
|
retryable: true,
|
|
1178
2175
|
};
|
|
1179
2176
|
}
|
|
1180
2177
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
1181
2178
|
}
|
|
1182
|
-
const postgresDriverError = findPostgresDriverError(error);
|
|
1183
|
-
if (postgresDriverError) {
|
|
1184
|
-
const database = safeDatabaseErrorFacts(postgresDriverError);
|
|
1185
|
-
const sqlState = nestedPostgresSqlState(postgresDriverError);
|
|
1186
|
-
if (sqlState !== null || Object.keys(database).length > 0) {
|
|
1187
|
-
return {
|
|
1188
|
-
error: message,
|
|
1189
|
-
sqlState,
|
|
1190
|
-
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1191
|
-
};
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
2179
|
return { error: message };
|
|
1195
2180
|
}
|
|
1196
2181
|
|
|
1197
2182
|
export type CodexCredentialFailure = {
|
|
1198
|
-
|
|
2183
|
+
/**
|
|
2184
|
+
* `plan_entitlement` is produced only after a plan re-check proves the
|
|
2185
|
+
* serving account's current plan does not include the requested model.
|
|
2186
|
+
*/
|
|
2187
|
+
kind: "auth" | "forbidden" | "rate_limit" | "quota" | "plan_entitlement";
|
|
1199
2188
|
cooldownSeconds: number | null;
|
|
2189
|
+
origin?: "included_usage_policy" | "usage_verification_policy";
|
|
1200
2190
|
};
|
|
1201
2191
|
|
|
1202
2192
|
export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
|
|
@@ -1216,7 +2206,11 @@ export function codexCredentialCooldownUntil(
|
|
|
1216
2206
|
> | null,
|
|
1217
2207
|
now: Date,
|
|
1218
2208
|
): Date | null {
|
|
1219
|
-
if (
|
|
2209
|
+
if (
|
|
2210
|
+
failure.kind === "auth" ||
|
|
2211
|
+
failure.kind === "forbidden" ||
|
|
2212
|
+
failure.kind === "plan_entitlement"
|
|
2213
|
+
) {
|
|
1220
2214
|
return null;
|
|
1221
2215
|
}
|
|
1222
2216
|
const providerReset =
|
|
@@ -1257,6 +2251,19 @@ export function codexCredentialCooldownUntil(
|
|
|
1257
2251
|
* progress and therefore MUST NOT walk the credential pool automatically.
|
|
1258
2252
|
*/
|
|
1259
2253
|
export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
|
|
2254
|
+
const creditPolicy = findCodexCreditPolicyError(error);
|
|
2255
|
+
if (creditPolicy instanceof CodexIncludedUsageUnknownError) {
|
|
2256
|
+
// Verified pre-dispatch refusal. A short hard hold cannot authorize credit
|
|
2257
|
+
// fallback and does not claim that the provider exhausted its allowance.
|
|
2258
|
+
return { kind: "rate_limit", cooldownSeconds: 60, origin: "usage_verification_policy" };
|
|
2259
|
+
}
|
|
2260
|
+
if (creditPolicy instanceof CodexIncludedUsageExhaustedError) {
|
|
2261
|
+
return {
|
|
2262
|
+
kind: "quota",
|
|
2263
|
+
cooldownSeconds: creditPolicy.resetsInSeconds,
|
|
2264
|
+
origin: "included_usage_policy",
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
1260
2267
|
// A request safety refusal is not evidence that another account should run it.
|
|
1261
2268
|
if (isProviderSafetyRefusal(error)) return null;
|
|
1262
2269
|
// A permanent OAuth refresh failure is definitive and the shared resolver has
|
|
@@ -1276,6 +2283,12 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
|
|
|
1276
2283
|
if (!isCodexTransportError(error)) {
|
|
1277
2284
|
return null;
|
|
1278
2285
|
}
|
|
2286
|
+
// Plan/model entitlement evidence (an explicit plan refusal, or an empty
|
|
2287
|
+
// HTTP 400) is not account health or quota. The worker re-checks the plan
|
|
2288
|
+
// first; only a proven entitlement loss walks the pool, as `plan_entitlement`.
|
|
2289
|
+
if (classifyCodexEntitlementRejection(error)) {
|
|
2290
|
+
return null;
|
|
2291
|
+
}
|
|
1279
2292
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1280
2293
|
if (usageLimit) {
|
|
1281
2294
|
return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
|
|
@@ -1380,3 +2393,42 @@ export function codexUsageLimitFailurePayload(
|
|
|
1380
2393
|
// open indefinitely for a goal-bearing session; cap the continuation hold so the
|
|
1381
2394
|
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
1382
2395
|
export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
2396
|
+
|
|
2397
|
+
/**
|
|
2398
|
+
* Claude's classification of a failed model request as a shared provider
|
|
2399
|
+
* outcome (the adapter's `classifyError`, Claude track C1b), for a request
|
|
2400
|
+
* of the Claude subscription transport only: a reconnect-required refresh or
|
|
2401
|
+
* HTTP 401 is `unauthorized` (the transport already ran the turn's one
|
|
2402
|
+
* forced refresh); 429 is `rate_limited` for the turn's model `modelId`, with
|
|
2403
|
+
* the provider's retry time (null: the adapter's fallback applies); 529 or
|
|
2404
|
+
* `overloaded_error` is `overloaded`, which never moves the turn to another
|
|
2405
|
+
* account; 403 is `fatal`, which never rotates or quarantines the account.
|
|
2406
|
+
* Anything else, a provider safety refusal included, is null: not a refusal
|
|
2407
|
+
* of the account.
|
|
2408
|
+
*/
|
|
2409
|
+
export function classifyClaudeProviderError(
|
|
2410
|
+
error: unknown,
|
|
2411
|
+
modelId?: string,
|
|
2412
|
+
): ProviderErrorOutcome | null {
|
|
2413
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
2414
|
+
let current: unknown = error;
|
|
2415
|
+
for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
|
|
2416
|
+
if (current instanceof ClaudeSubscriptionReconnectRequired) return { kind: "unauthorized" };
|
|
2417
|
+
if (current instanceof AnthropicRequestError) {
|
|
2418
|
+
if (!current.subscription) return null;
|
|
2419
|
+
if (current.status === 401) return { kind: "unauthorized" };
|
|
2420
|
+
if (current.status === 429)
|
|
2421
|
+
return {
|
|
2422
|
+
kind: "rate_limited",
|
|
2423
|
+
retryAfterMs: providerRetryAfterMs(current),
|
|
2424
|
+
...(modelId === undefined ? {} : { modelId }),
|
|
2425
|
+
};
|
|
2426
|
+
if (current.status === 529 || current.errorType === "overloaded_error")
|
|
2427
|
+
return { kind: "overloaded" };
|
|
2428
|
+
if (current.status === 403) return { kind: "fatal" };
|
|
2429
|
+
return null;
|
|
2430
|
+
}
|
|
2431
|
+
current = current.cause;
|
|
2432
|
+
}
|
|
2433
|
+
return null;
|
|
2434
|
+
}
|