@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -1,14 +1,22 @@
|
|
|
1
|
+
import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
|
|
2
|
+
import { DrizzleQueryError } from "drizzle-orm";
|
|
3
|
+
import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
|
|
4
|
+
import { heldSessionUsageReceiptsForTurn } from "./session-usage-custody";
|
|
1
5
|
import {
|
|
2
6
|
ActiveSessionHistoryLimitExceededError,
|
|
3
7
|
ApprovalRunStateLimitExceededError,
|
|
4
8
|
nestedPostgresSqlState,
|
|
9
|
+
databaseFailureCode,
|
|
10
|
+
isRetryablePersistenceSqlState,
|
|
5
11
|
safeDatabaseErrorFacts,
|
|
6
12
|
isRetryableDatabaseTransportFailure,
|
|
7
13
|
isSessionEventPersistenceError,
|
|
14
|
+
DatabaseTransactionError,
|
|
8
15
|
SandboxLeaseTransitionError,
|
|
9
16
|
} from "@opengeni/db";
|
|
10
17
|
import {
|
|
11
18
|
ActiveBackendUnresolvableError,
|
|
19
|
+
AnthropicProviderRejection,
|
|
12
20
|
CompactionProviderResponseError,
|
|
13
21
|
EmptyCompactionSummaryError,
|
|
14
22
|
compactionProviderRejection,
|
|
@@ -16,29 +24,61 @@ import {
|
|
|
16
24
|
type CompactionProviderRejection,
|
|
17
25
|
isMcpRequestTimeoutError,
|
|
18
26
|
isMcpTransportConnectivityError,
|
|
19
|
-
|
|
27
|
+
isModalTaskExecStartPreDispatchUnavailableError,
|
|
28
|
+
isModalCommandStartOutcomeUnknownError,
|
|
29
|
+
isProviderCommandObservationUnavailableError,
|
|
30
|
+
ProviderCommandInputOutcomeUnknownError,
|
|
31
|
+
ProviderCommandStartOutcomeUnknownError,
|
|
32
|
+
isRoutingMutationOutcomeUnknownError,
|
|
33
|
+
isRoutingMutationOutputRejectedError,
|
|
20
34
|
RoutingWorkspaceRootChangedError,
|
|
35
|
+
RoutingBackendRecoveryRequiredError,
|
|
36
|
+
ResponsesStreamingTerminalError,
|
|
37
|
+
classifyModelStreamIdleTimeoutError,
|
|
21
38
|
SandboxMaterializationVerificationError,
|
|
22
39
|
materializationVerificationDiagnostic,
|
|
23
40
|
type MaterializationVerificationDiagnostic,
|
|
41
|
+
PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
42
|
+
type ProviderQuotaExhaustion,
|
|
43
|
+
type ProviderQuotaScope,
|
|
44
|
+
classifyProviderQuotaError,
|
|
45
|
+
providerQuotaExhaustedMessage,
|
|
24
46
|
SelfhostedWorkspaceRootChangedError,
|
|
25
47
|
UNKNOWN_MODEL_FINISH_REASON_CODE,
|
|
48
|
+
AnthropicRequestError,
|
|
49
|
+
findAnthropicRequestSizeError,
|
|
50
|
+
AnthropicSizeRecoveryExhaustedError,
|
|
51
|
+
type AnthropicRequestSize,
|
|
26
52
|
} from "@opengeni/runtime";
|
|
27
53
|
import {
|
|
28
54
|
mcpTransportRequestFailureDiagnostic,
|
|
29
55
|
type McpTransportRequestFailureDiagnostic,
|
|
30
56
|
} from "@opengeni/runtime/mcp-network";
|
|
31
57
|
import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
|
|
32
|
-
import {
|
|
58
|
+
import type { ProviderErrorOutcome } from "@opengeni/subscriptions";
|
|
59
|
+
import { CODEX_USAGE_EXHAUSTED_PCT } from "./codex-usage-policy";
|
|
33
60
|
import { RetainedAttachmentTransportLimitError } from "../run-input";
|
|
34
61
|
import type { CodexAccountStatus } from "@opengeni/db";
|
|
35
62
|
import {
|
|
63
|
+
CODEX_USAGE_LIMIT_ERROR_TYPE,
|
|
36
64
|
CodexReloginRequired,
|
|
37
65
|
classifyCodexEncryptedArtifactRejection,
|
|
66
|
+
classifyCodexEntitlementRejection,
|
|
38
67
|
classifyCodexResponseTimeoutError,
|
|
39
68
|
classifyCodexUsageLimitError,
|
|
40
69
|
isCodexTransportError,
|
|
41
70
|
} from "@opengeni/codex";
|
|
71
|
+
import {
|
|
72
|
+
CodexPlanEntitlementError,
|
|
73
|
+
codexPlanEntitlementFailurePayload,
|
|
74
|
+
codexRequestRejectedFailurePayload,
|
|
75
|
+
} from "./codex-plan-entitlement";
|
|
76
|
+
import { SubscriptionCoreCodexTurnError } from "./codex-core-errors";
|
|
77
|
+
import {
|
|
78
|
+
CodexIncludedUsageExhaustedError,
|
|
79
|
+
CodexIncludedUsageUnknownError,
|
|
80
|
+
findCodexCreditPolicyError,
|
|
81
|
+
} from "./codex-credit-policy";
|
|
42
82
|
import {
|
|
43
83
|
classifyXaiSubscriptionStreamingTerminalError,
|
|
44
84
|
classifyXaiSubscriptionStreamIdleTimeoutError,
|
|
@@ -64,12 +104,45 @@ import {
|
|
|
64
104
|
MandatoryHistoryPersistenceError,
|
|
65
105
|
type MandatoryHistoryPersistenceStage,
|
|
66
106
|
} from "./quiescence";
|
|
107
|
+
import {
|
|
108
|
+
MODEL_PROVIDER_RECOVERY_CODES,
|
|
109
|
+
providerRecoveryExhaustedMessage,
|
|
110
|
+
providerRecoverySubject,
|
|
111
|
+
type ProviderCondition,
|
|
112
|
+
} from "./provider-recovery-copy";
|
|
113
|
+
import {
|
|
114
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
115
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
116
|
+
PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
|
|
117
|
+
validProviderOverloadRecoveryDelay,
|
|
118
|
+
} from "./provider-recovery-policy";
|
|
119
|
+
export {
|
|
120
|
+
PROVIDER_OVERLOAD_RECOVERY_CODE,
|
|
121
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
122
|
+
PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS,
|
|
123
|
+
} from "./provider-recovery-policy";
|
|
67
124
|
|
|
68
125
|
// Retryable provider connectivity/5xx failures start quickly and back off to
|
|
69
126
|
// this ceiling. Explicit rate limits retain the minute-granular fallback.
|
|
70
127
|
export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
|
|
71
128
|
export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
|
|
72
129
|
export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
|
|
130
|
+
export function providerRecoveryLimit(failureCode: unknown): number {
|
|
131
|
+
return failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE
|
|
132
|
+
? MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES
|
|
133
|
+
: MAX_AUTOMATIC_PROVIDER_RECOVERIES;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
|
|
137
|
+
* answer a per-minute token limit with a `retry-after` of about a second, which
|
|
138
|
+
* alone would spend every automatic recovery before the window resets.
|
|
139
|
+
*/
|
|
140
|
+
export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
|
|
141
|
+
/** Positive-only spread: never shorten the provider's minimum delay. */
|
|
142
|
+
export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
|
|
143
|
+
const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
|
|
144
|
+
return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
|
|
145
|
+
}
|
|
73
146
|
export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
|
|
74
147
|
|
|
75
148
|
export class PostCompactionContinuationEmptyError extends Error {
|
|
@@ -85,25 +158,47 @@ export type ProviderRecoveryResult =
|
|
|
85
158
|
| {
|
|
86
159
|
status: "recovering";
|
|
87
160
|
continueDelayMs: number;
|
|
161
|
+
maxProviderRecoveryCount?: number;
|
|
88
162
|
}
|
|
89
163
|
| {
|
|
90
164
|
status: "exhausted";
|
|
91
165
|
providerRecoveryCount: number;
|
|
92
166
|
maxProviderRecoveryCount: number;
|
|
167
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
93
168
|
};
|
|
94
169
|
|
|
95
170
|
export function providerRecoveryResult(input: {
|
|
96
171
|
failureCode: string | undefined;
|
|
97
172
|
attemptNumber: number;
|
|
98
173
|
retryAfterMs?: number | null;
|
|
174
|
+
jitterSample?: number;
|
|
175
|
+
/** Durable first-failure clock, never a new clock on each replacement attempt. */
|
|
176
|
+
recoveryStartedAt?: number | undefined;
|
|
177
|
+
now?: number;
|
|
99
178
|
}): ProviderRecoveryResult {
|
|
100
|
-
|
|
179
|
+
const overload = input.failureCode === PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
180
|
+
const limit = providerRecoveryLimit(input.failureCode);
|
|
181
|
+
const exhausted = (
|
|
182
|
+
reason: "deadline" | "retry_limit" | "invalid_clock",
|
|
183
|
+
): Extract<ProviderRecoveryResult, { status: "exhausted" }> => ({
|
|
184
|
+
status: "exhausted",
|
|
185
|
+
providerRecoveryCount: Math.min(Math.max(input.attemptNumber - 1, 0), limit),
|
|
186
|
+
maxProviderRecoveryCount: limit,
|
|
187
|
+
providerRecoveryExhaustedReason: reason,
|
|
188
|
+
});
|
|
189
|
+
if (input.attemptNumber > limit) {
|
|
190
|
+
if (overload) return exhausted("retry_limit");
|
|
101
191
|
return {
|
|
102
192
|
status: "exhausted",
|
|
103
193
|
providerRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
104
194
|
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_RECOVERIES,
|
|
105
195
|
};
|
|
106
196
|
}
|
|
197
|
+
const now = input.now ?? Date.now();
|
|
198
|
+
const startedAt = input.recoveryStartedAt ?? (input.attemptNumber === 1 ? now : NaN);
|
|
199
|
+
if (overload && (!Number.isFinite(startedAt) || !Number.isFinite(now) || startedAt > now)) {
|
|
200
|
+
return exhausted("invalid_clock");
|
|
201
|
+
}
|
|
107
202
|
const providerDelay =
|
|
108
203
|
input.retryAfterMs !== null &&
|
|
109
204
|
input.retryAfterMs !== undefined &&
|
|
@@ -113,12 +208,22 @@ export function providerRecoveryResult(input: {
|
|
|
113
208
|
: null;
|
|
114
209
|
const continueDelayMs =
|
|
115
210
|
input.failureCode === "provider_rate_limited"
|
|
116
|
-
? (
|
|
117
|
-
|
|
211
|
+
? Math.max(
|
|
212
|
+
providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
213
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS[
|
|
214
|
+
Math.min(
|
|
215
|
+
Math.max(Math.trunc(input.attemptNumber) - 1, 0),
|
|
216
|
+
PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1,
|
|
217
|
+
)
|
|
218
|
+
]!,
|
|
219
|
+
)
|
|
220
|
+
: overload ||
|
|
221
|
+
input.failureCode === "provider_unavailable" ||
|
|
118
222
|
input.failureCode === "upstream_connectivity_unavailable" ||
|
|
119
223
|
input.failureCode === "sandbox_command_start_unavailable" ||
|
|
120
224
|
input.failureCode === "mcp_transport_timeout" ||
|
|
121
225
|
input.failureCode === "mcp_transport_unavailable" ||
|
|
226
|
+
input.failureCode === "turn_execution_policy_definition_mismatch" ||
|
|
122
227
|
input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
|
|
123
228
|
? Math.max(
|
|
124
229
|
providerDelay ?? 0,
|
|
@@ -130,12 +235,79 @@ export function providerRecoveryResult(input: {
|
|
|
130
235
|
]!,
|
|
131
236
|
)
|
|
132
237
|
: PROVIDER_BACKPRESSURE_DELAY_MS;
|
|
238
|
+
const delayMs =
|
|
239
|
+
continueDelayMs +
|
|
240
|
+
(overload ||
|
|
241
|
+
input.failureCode === "provider_rate_limited" ||
|
|
242
|
+
input.failureCode === "provider_unavailable" ||
|
|
243
|
+
input.failureCode === "upstream_connectivity_unavailable"
|
|
244
|
+
? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
|
|
245
|
+
: 0);
|
|
246
|
+
// Do not shorten Retry-After, or schedule a replacement outside the window.
|
|
247
|
+
if (overload && now + delayMs >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
248
|
+
return exhausted("deadline");
|
|
249
|
+
}
|
|
133
250
|
return {
|
|
134
251
|
status: "recovering",
|
|
135
|
-
continueDelayMs,
|
|
252
|
+
continueDelayMs: delayMs,
|
|
253
|
+
...(overload ? { maxProviderRecoveryCount: limit } : {}),
|
|
136
254
|
};
|
|
137
255
|
}
|
|
138
256
|
|
|
257
|
+
/** A timer/queue delay cannot authorize a provider call after the recovery window. */
|
|
258
|
+
export class ProviderOverloadRecoveryExpiredError extends Error {
|
|
259
|
+
readonly failure;
|
|
260
|
+
|
|
261
|
+
constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock") {
|
|
262
|
+
super("Confirmed provider overload recovery window expired before dispatch");
|
|
263
|
+
this.name = "ProviderOverloadRecoveryExpiredError";
|
|
264
|
+
this.failure = providerRecoveryExhaustedFailure(
|
|
265
|
+
{
|
|
266
|
+
error: this.message,
|
|
267
|
+
code: "provider_unavailable",
|
|
268
|
+
providerCondition: "overloaded" as const,
|
|
269
|
+
},
|
|
270
|
+
{
|
|
271
|
+
status: "exhausted",
|
|
272
|
+
// The queued replacement has not dispatched; count only prior retries.
|
|
273
|
+
providerRecoveryCount: Math.min(
|
|
274
|
+
MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
275
|
+
Math.max(0, count - 1),
|
|
276
|
+
),
|
|
277
|
+
maxProviderRecoveryCount: MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES,
|
|
278
|
+
providerRecoveryExhaustedReason: reason,
|
|
279
|
+
},
|
|
280
|
+
);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
export function assertProviderOverloadRecoveryActive(input: {
|
|
285
|
+
failureCode: unknown;
|
|
286
|
+
providerRecoveryCount: number;
|
|
287
|
+
recoveryStartedAt?: number | undefined;
|
|
288
|
+
now?: number;
|
|
289
|
+
}): void {
|
|
290
|
+
if (input.failureCode !== PROVIDER_OVERLOAD_RECOVERY_CODE || input.providerRecoveryCount === 0) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (input.providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES) {
|
|
294
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "retry_limit");
|
|
295
|
+
}
|
|
296
|
+
const now = input.now ?? Date.now();
|
|
297
|
+
const startedAt = input.recoveryStartedAt;
|
|
298
|
+
if (
|
|
299
|
+
startedAt === undefined ||
|
|
300
|
+
!Number.isFinite(startedAt) ||
|
|
301
|
+
!Number.isFinite(now) ||
|
|
302
|
+
startedAt > now
|
|
303
|
+
) {
|
|
304
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "invalid_clock");
|
|
305
|
+
}
|
|
306
|
+
if (now >= startedAt + PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS) {
|
|
307
|
+
throw new ProviderOverloadRecoveryExpiredError(input.providerRecoveryCount, "deadline");
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
139
311
|
export function providerRecoveryExhaustedFailure<
|
|
140
312
|
T extends Record<string, unknown> & { error: string },
|
|
141
313
|
>(
|
|
@@ -147,18 +319,89 @@ export function providerRecoveryExhaustedFailure<
|
|
|
147
319
|
providerRecoveryCount: number;
|
|
148
320
|
maxProviderRecoveryCount: number;
|
|
149
321
|
lastRetryableError: string;
|
|
322
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
150
323
|
} {
|
|
151
324
|
return {
|
|
152
325
|
...failure,
|
|
153
|
-
error:
|
|
326
|
+
error: providerRecoveryExhaustedMessage({
|
|
327
|
+
code: typeof failure.code === "string" ? failure.code : null,
|
|
328
|
+
providerCondition: isProviderCondition(failure.providerCondition)
|
|
329
|
+
? failure.providerCondition
|
|
330
|
+
: null,
|
|
331
|
+
modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
|
|
332
|
+
providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
|
|
333
|
+
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
334
|
+
}),
|
|
154
335
|
retryable: false,
|
|
155
336
|
recoveryExhausted: true,
|
|
156
337
|
providerRecoveryCount: recovery.providerRecoveryCount,
|
|
157
338
|
maxProviderRecoveryCount: recovery.maxProviderRecoveryCount,
|
|
339
|
+
...(recovery.providerRecoveryExhaustedReason
|
|
340
|
+
? { providerRecoveryExhaustedReason: recovery.providerRecoveryExhaustedReason }
|
|
341
|
+
: {}),
|
|
342
|
+
lastRetryableError: failure.error,
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function isProviderCondition(value: unknown): value is ProviderCondition {
|
|
347
|
+
return (
|
|
348
|
+
value === "overloaded" ||
|
|
349
|
+
value === "unavailable" ||
|
|
350
|
+
value === "unresponsive" ||
|
|
351
|
+
value === "rate_limited"
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
/**
|
|
356
|
+
* A retryable failure whose automatic same-turn recovery was not authorized
|
|
357
|
+
* (for example, request custody could not be checkpointed). The classifier's
|
|
358
|
+
* "will retry" sentence would be false on a terminal record, so name the
|
|
359
|
+
* condition and the person's options instead.
|
|
360
|
+
*/
|
|
361
|
+
export function automaticRecoveryUnavailableFailure<
|
|
362
|
+
T extends Record<string, unknown> & { error: string },
|
|
363
|
+
>(
|
|
364
|
+
failure: T,
|
|
365
|
+
): T & {
|
|
366
|
+
retryable: false;
|
|
367
|
+
automaticRecovery: "unavailable";
|
|
368
|
+
lastRetryableError: string;
|
|
369
|
+
} {
|
|
370
|
+
const subject = providerRecoverySubject({
|
|
371
|
+
code: typeof failure.code === "string" ? failure.code : null,
|
|
372
|
+
providerCondition: isProviderCondition(failure.providerCondition)
|
|
373
|
+
? failure.providerCondition
|
|
374
|
+
: null,
|
|
375
|
+
modelLabel: typeof failure.modelLabel === "string" ? failure.modelLabel : null,
|
|
376
|
+
providerLabel: typeof failure.providerLabel === "string" ? failure.providerLabel : null,
|
|
377
|
+
});
|
|
378
|
+
return {
|
|
379
|
+
...failure,
|
|
380
|
+
error: `${subject}. Opengeni couldn't retry this turn automatically. Try again, or send a message to continue.`,
|
|
381
|
+
retryable: false,
|
|
382
|
+
automaticRecovery: "unavailable",
|
|
158
383
|
lastRetryableError: failure.error,
|
|
159
384
|
};
|
|
160
385
|
}
|
|
161
386
|
|
|
387
|
+
/**
|
|
388
|
+
* Name the exact accepted model route on model-provider recovery evidence so a
|
|
389
|
+
* person learns which model is affected. Display labels only: the route's
|
|
390
|
+
* credentials, base URL and upstream ids never enter the event.
|
|
391
|
+
*/
|
|
392
|
+
export function withModelRoutePresentation<T extends { code?: string | undefined }>(
|
|
393
|
+
failure: T,
|
|
394
|
+
route: { model: string; modelLabel: string; providerLabel: string } | undefined,
|
|
395
|
+
): T & { model?: string; modelLabel?: string; providerLabel?: string } {
|
|
396
|
+
if (!route || !failure.code || !MODEL_PROVIDER_RECOVERY_CODES.has(failure.code)) return failure;
|
|
397
|
+
return {
|
|
398
|
+
...failure,
|
|
399
|
+
model: route.model,
|
|
400
|
+
modelLabel: route.modelLabel,
|
|
401
|
+
providerLabel: route.providerLabel,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
162
405
|
export function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number {
|
|
163
406
|
const value = metadata.providerRecoveryCount;
|
|
164
407
|
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : 0;
|
|
@@ -194,6 +437,13 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
|
|
|
194
437
|
value.error && typeof value.error === "object"
|
|
195
438
|
? (value.error as Record<string, unknown>)
|
|
196
439
|
: null;
|
|
440
|
+
const milliseconds = Number(
|
|
441
|
+
headerValue(value.headers, "retry-after-ms") ??
|
|
442
|
+
headerValue(value.responseHeaders, "retry-after-ms") ??
|
|
443
|
+
headerValue(body?.headers, "retry-after-ms") ??
|
|
444
|
+
undefined,
|
|
445
|
+
);
|
|
446
|
+
if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
|
|
197
447
|
const directSeconds = Number(
|
|
198
448
|
value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
|
|
199
449
|
);
|
|
@@ -317,20 +567,199 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
|
317
567
|
});
|
|
318
568
|
}
|
|
319
569
|
|
|
570
|
+
/** Complete structured-cause graph or no recovery authority. Budget distinct
|
|
571
|
+
* objects (including array containers), not queue positions/duplicate refs.
|
|
572
|
+
* The separate link ceiling bounds huge duplicate arrays without truncating
|
|
573
|
+
* them into permission. Cycles are harmless; overflow/unreadable edges hold. */
|
|
574
|
+
function structuredRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
|
|
575
|
+
const graph = new Map<object, Set<object>>();
|
|
576
|
+
const queue: object[] = [];
|
|
577
|
+
let links = 0;
|
|
578
|
+
const add = (value: unknown): boolean => {
|
|
579
|
+
if (!value || typeof value !== "object" || graph.has(value)) return true;
|
|
580
|
+
if (graph.size >= 64) return false;
|
|
581
|
+
graph.set(value, new Set());
|
|
582
|
+
queue.push(value);
|
|
583
|
+
return true;
|
|
584
|
+
};
|
|
585
|
+
if (!add(error)) return null;
|
|
586
|
+
try {
|
|
587
|
+
for (const current of queue) {
|
|
588
|
+
const record = current as Record<string, unknown>;
|
|
589
|
+
const children = Array.isArray(current)
|
|
590
|
+
? current
|
|
591
|
+
: ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
|
|
592
|
+
for (const child of children) {
|
|
593
|
+
if (++links > 4096 || !add(child)) return null;
|
|
594
|
+
if (child && typeof child === "object") graph.get(current)!.add(child);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
} catch {
|
|
598
|
+
return null;
|
|
599
|
+
}
|
|
600
|
+
return graph;
|
|
601
|
+
}
|
|
602
|
+
|
|
320
603
|
function retryableDatabaseFailureCode(
|
|
321
604
|
error: unknown,
|
|
605
|
+
requireDatabaseProvenance = false,
|
|
322
606
|
): PostClaimDatabaseRecoveryDetail["code"] | null {
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
607
|
+
try {
|
|
608
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
609
|
+
if (!graph) return null;
|
|
610
|
+
const transports = new Set<object>();
|
|
611
|
+
const boundaries = new Set<object>();
|
|
612
|
+
const ownDatabaseNodes = new Set<object>();
|
|
613
|
+
const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
|
|
614
|
+
const ownChildren = (node: object): object[] =>
|
|
615
|
+
node instanceof DatabaseTransactionError
|
|
616
|
+
? // Callback failures retained beside a failed rollback supply vetoes,
|
|
617
|
+
// never driver provenance for a provider's connection-looking error.
|
|
618
|
+
[...graph.get(node)!].filter((child) => child === node.cause)
|
|
619
|
+
: [...graph.get(node)!];
|
|
620
|
+
// Ask the canonical transport predicate about ONLY this node's facts. Its
|
|
621
|
+
// recursive search must not pair a DB sibling with an unrelated provider.
|
|
622
|
+
for (const node of graph.keys()) {
|
|
623
|
+
if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
|
|
624
|
+
return null;
|
|
625
|
+
const record = node as Record<string, unknown>;
|
|
626
|
+
if (
|
|
627
|
+
requireDatabaseProvenance
|
|
628
|
+
? isRunningTurnDatabaseTransportFailure(record)
|
|
629
|
+
: isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno })
|
|
630
|
+
)
|
|
631
|
+
transports.add(node);
|
|
632
|
+
const sqlState = isSessionEventPersistenceError(node)
|
|
633
|
+
? node.details.sqlState
|
|
634
|
+
: record.name === "PostgresError" && typeof record.code === "string"
|
|
635
|
+
? record.code
|
|
636
|
+
: null;
|
|
637
|
+
if (
|
|
638
|
+
requireDatabaseProvenance
|
|
639
|
+
? isRunningTurnDatabaseConnectionSqlState(sqlState)
|
|
640
|
+
: isDatabaseConnectionSqlState(sqlState)
|
|
641
|
+
)
|
|
642
|
+
transports.add(node);
|
|
643
|
+
if (
|
|
644
|
+
node instanceof DrizzleQueryError ||
|
|
645
|
+
node instanceof DatabaseTransactionError ||
|
|
646
|
+
isSessionEventPersistenceError(node)
|
|
647
|
+
) {
|
|
648
|
+
// Only actual errors raised at our ORM/typed persistence boundary own
|
|
649
|
+
// their driver subtree. A PostgresError name, SDK wrapper or provider
|
|
650
|
+
// socket by itself is never own-client provenance for a running turn.
|
|
651
|
+
const queue: object[] = [node];
|
|
652
|
+
for (const source of queue) {
|
|
653
|
+
if (ownDatabaseNodes.has(source)) continue;
|
|
654
|
+
ownDatabaseNodes.add(source);
|
|
655
|
+
queue.push(...ownChildren(source));
|
|
656
|
+
}
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const hasOwnTransport = (boundary: object): boolean => {
|
|
660
|
+
const seen = new Set<object>();
|
|
661
|
+
const queue = [boundary];
|
|
662
|
+
for (const node of queue) {
|
|
663
|
+
if (seen.has(node)) continue;
|
|
664
|
+
seen.add(node);
|
|
665
|
+
if (transports.has(node)) return true;
|
|
666
|
+
queue.push(...ownChildren(node));
|
|
667
|
+
}
|
|
668
|
+
return false;
|
|
669
|
+
};
|
|
670
|
+
// Inspect ALL trusted DB evidence before returning a permit. A permanent,
|
|
671
|
+
// auth or uncertain SQLSTATE vetoes every sibling regardless of traversal
|
|
672
|
+
// order; a deeper transport/reset cannot override it either.
|
|
673
|
+
for (const node of graph.keys()) {
|
|
674
|
+
const record = node as Record<string, unknown>;
|
|
675
|
+
if (node instanceof DrizzleQueryError || node instanceof DatabaseTransactionError) {
|
|
676
|
+
boundaries.add(node);
|
|
677
|
+
if (hasOwnTransport(node)) codes.add("db_failure");
|
|
678
|
+
}
|
|
679
|
+
const typed = isSessionEventPersistenceError(node);
|
|
680
|
+
if (!typed && record.name !== "PostgresError") continue;
|
|
681
|
+
boundaries.add(node);
|
|
682
|
+
const sqlState = typed
|
|
683
|
+
? node.details.sqlState
|
|
684
|
+
: typeof record.code === "string"
|
|
685
|
+
? record.code
|
|
686
|
+
: null;
|
|
687
|
+
const connectionOutage = requireDatabaseProvenance
|
|
688
|
+
? isRunningTurnDatabaseConnectionSqlState(sqlState)
|
|
689
|
+
: isDatabaseConnectionSqlState(sqlState);
|
|
690
|
+
// PostgreSQL aborts a deadlock or serialization victim's whole
|
|
691
|
+
// transaction: unlike a lost connection, that write certainly did not
|
|
692
|
+
// commit, so it is at least as safe for exact-attempt recovery.
|
|
693
|
+
const rolledBack = isRetryablePersistenceSqlState(sqlState);
|
|
694
|
+
const code = typed
|
|
695
|
+
? retryablePersistenceFailureCode(sqlState)
|
|
696
|
+
: connectionOutage || rolledBack
|
|
697
|
+
? databaseFailureCode(sqlState)
|
|
698
|
+
: null;
|
|
699
|
+
if (
|
|
700
|
+
!code ||
|
|
701
|
+
(requireDatabaseProvenance &&
|
|
702
|
+
!connectionOutage &&
|
|
703
|
+
!rolledBack &&
|
|
704
|
+
!(sqlState === null && hasOwnTransport(node)))
|
|
705
|
+
)
|
|
706
|
+
return null;
|
|
707
|
+
if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
|
|
708
|
+
}
|
|
709
|
+
// Preserve the legacy pre-execution transport-only allowance, but never
|
|
710
|
+
// borrow it across an explicit DB boundary with no eligible own evidence.
|
|
711
|
+
if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
|
|
712
|
+
codes.add("db_failure");
|
|
713
|
+
// Stable classification for multiple positive DB siblings too.
|
|
714
|
+
for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"] as const)
|
|
715
|
+
if (codes.has(code)) return code;
|
|
716
|
+
return null;
|
|
717
|
+
} catch {
|
|
718
|
+
// Unreadable structured facts are no more authority than unreadable edges.
|
|
719
|
+
return null;
|
|
327
720
|
}
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
|
|
724
|
+
// postgres.js reports these for a physically lost connection, including
|
|
725
|
+
// transaction cleanup after its socket has already closed. Own-client
|
|
726
|
+
// provenance and the unknown-outcome veto remain mandatory above.
|
|
727
|
+
"CONNECTION_CLOSED",
|
|
728
|
+
"CONNECTION_DESTROYED",
|
|
729
|
+
"CONNECTION_ENDED",
|
|
730
|
+
"ECONNREFUSED",
|
|
731
|
+
"ECONNRESET",
|
|
732
|
+
"CONNECT_TIMEOUT",
|
|
733
|
+
]);
|
|
734
|
+
|
|
735
|
+
function isRunningTurnDatabaseTransportFailure(record: Record<string, unknown>): boolean {
|
|
736
|
+
return [record.code, record.errno].some(
|
|
737
|
+
(code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code),
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function isRunningTurnDatabaseConnectionSqlState(sqlState: string | null): boolean {
|
|
742
|
+
return (
|
|
743
|
+
sqlState !== null &&
|
|
744
|
+
(/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState))
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
function isDatabaseConnectionSqlState(sqlState: string | null): boolean {
|
|
749
|
+
return (
|
|
750
|
+
sqlState !== null &&
|
|
751
|
+
(sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState))
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
function retryablePersistenceFailureCode(
|
|
756
|
+
sqlState: string | null,
|
|
757
|
+
): PostClaimDatabaseRecoveryDetail["code"] | null {
|
|
328
758
|
if (
|
|
329
|
-
!persistenceFailure ||
|
|
330
759
|
!(
|
|
331
760
|
sqlState === null ||
|
|
332
761
|
sqlState.startsWith("08") ||
|
|
333
|
-
sqlState
|
|
762
|
+
isRetryablePersistenceSqlState(sqlState) ||
|
|
334
763
|
sqlState.startsWith("53") ||
|
|
335
764
|
sqlState === "55P03" ||
|
|
336
765
|
sqlState === "57014" ||
|
|
@@ -342,32 +771,59 @@ function retryableDatabaseFailureCode(
|
|
|
342
771
|
) {
|
|
343
772
|
return null;
|
|
344
773
|
}
|
|
345
|
-
return
|
|
774
|
+
return databaseFailureCode(sqlState);
|
|
346
775
|
}
|
|
347
776
|
|
|
348
777
|
/**
|
|
349
|
-
*
|
|
350
|
-
* recovery
|
|
351
|
-
*
|
|
778
|
+
* True when an error is a structured database failure that the exact-attempt
|
|
779
|
+
* database recovery path owns. Callers that translate other failures into
|
|
780
|
+
* user-visible states must rethrow these unchanged.
|
|
781
|
+
*/
|
|
782
|
+
export function isPostClaimDatabaseRecoveryCandidate(error: unknown): boolean {
|
|
783
|
+
return retryableDatabaseFailureCode(error, true) !== null;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Carry one exact claimed attempt into the workflow's DB-only
|
|
788
|
+
* recovery lane. Permanent database/state failures remain terminal. The
|
|
789
|
+
* running-turn lane additionally requires a closed own-client outage class;
|
|
790
|
+
* existing pre-execution recovery classifications are unchanged.
|
|
352
791
|
*/
|
|
353
792
|
export function postClaimDatabaseRecoveryFailure(input: {
|
|
354
793
|
error: unknown;
|
|
355
794
|
turnId: string;
|
|
356
795
|
triggerEventId: string;
|
|
357
796
|
executionGeneration: number;
|
|
797
|
+
/** Require an actual own-client boundary and the closed running-turn allowlist. */
|
|
798
|
+
requireDatabaseProvenance?: boolean;
|
|
799
|
+
sandboxSetupOutcomeUnknown?: true;
|
|
800
|
+
sandboxSetupRecoveryExhausted?: true;
|
|
358
801
|
providerRecovery?: {
|
|
359
802
|
failureCode: string;
|
|
360
803
|
providerRecoveryCount: number;
|
|
804
|
+
continueDelayMs?: number;
|
|
361
805
|
};
|
|
362
806
|
}): ApplicationFailure | null {
|
|
363
|
-
const code = retryableDatabaseFailureCode(input.error);
|
|
807
|
+
const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
|
|
364
808
|
if (!code || input.executionGeneration < 1) return null;
|
|
809
|
+
if (
|
|
810
|
+
(input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
|
|
811
|
+
((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
|
|
812
|
+
input.providerRecovery)
|
|
813
|
+
) {
|
|
814
|
+
return null;
|
|
815
|
+
}
|
|
365
816
|
if (
|
|
366
817
|
input.providerRecovery &&
|
|
367
818
|
(!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
|
|
368
819
|
input.providerRecovery.providerRecoveryCount <= 0 ||
|
|
369
|
-
input.providerRecovery.providerRecoveryCount >
|
|
370
|
-
|
|
820
|
+
input.providerRecovery.providerRecoveryCount >
|
|
821
|
+
providerRecoveryLimit(input.providerRecovery.failureCode) ||
|
|
822
|
+
!/^[a-z][a-z0-9_]{0,63}$/.test(input.providerRecovery.failureCode) ||
|
|
823
|
+
!validProviderOverloadRecoveryDelay(
|
|
824
|
+
input.providerRecovery.failureCode,
|
|
825
|
+
input.providerRecovery.continueDelayMs,
|
|
826
|
+
))
|
|
371
827
|
) {
|
|
372
828
|
return null;
|
|
373
829
|
}
|
|
@@ -376,13 +832,21 @@ export function postClaimDatabaseRecoveryFailure(input: {
|
|
|
376
832
|
triggerEventId: input.triggerEventId,
|
|
377
833
|
executionGeneration: input.executionGeneration,
|
|
378
834
|
code,
|
|
835
|
+
...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
|
|
836
|
+
...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
|
|
379
837
|
...(input.providerRecovery
|
|
380
838
|
? {
|
|
381
839
|
providerFailureCode: input.providerRecovery.failureCode,
|
|
382
840
|
providerRecoveryCount: input.providerRecovery.providerRecoveryCount,
|
|
841
|
+
...(input.providerRecovery.continueDelayMs !== undefined
|
|
842
|
+
? { providerRecoveryContinueDelayMs: input.providerRecovery.continueDelayMs }
|
|
843
|
+
: {}),
|
|
383
844
|
}
|
|
384
845
|
: {}),
|
|
385
846
|
};
|
|
847
|
+
// Known usage whose receipt write failed travels with the exact attempt.
|
|
848
|
+
const sessionUsageReceipts = heldSessionUsageReceiptsForTurn(input.turnId);
|
|
849
|
+
if (sessionUsageReceipts.length > 0) detail.sessionUsageReceipts = sessionUsageReceipts;
|
|
386
850
|
return ApplicationFailure.create({
|
|
387
851
|
message: POST_CLAIM_DATABASE_RECOVERY_FAILURE_MESSAGE,
|
|
388
852
|
type: POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE,
|
|
@@ -462,8 +926,43 @@ export function sandboxLifecycleTransitionDiagnostic(
|
|
|
462
926
|
return null;
|
|
463
927
|
}
|
|
464
928
|
|
|
929
|
+
export function modelPreparationFailureEventPayload(error: unknown, durationMs: number) {
|
|
930
|
+
const transition = sandboxLifecycleTransitionDiagnostic(error);
|
|
931
|
+
return {
|
|
932
|
+
phase: "model_preparation",
|
|
933
|
+
durationMs: Math.max(0, Math.round(durationMs)),
|
|
934
|
+
expectedTransition: transition !== null,
|
|
935
|
+
...(transition
|
|
936
|
+
? {
|
|
937
|
+
failureCategory: "drain_capture_wait",
|
|
938
|
+
failureStage: "lifecycle_wait",
|
|
939
|
+
failureCode: transition.reason,
|
|
940
|
+
retryable: true,
|
|
941
|
+
}
|
|
942
|
+
: {}),
|
|
943
|
+
};
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
/**
|
|
947
|
+
* The turn's first model request could not be dispatched until the context
|
|
948
|
+
* was compacted (proactive threshold or an operator-requested Compact). That
|
|
949
|
+
* is an expected step of preparation, not a failure: the same activity
|
|
950
|
+
* compacts and then reopens preparation for the retried request.
|
|
951
|
+
*/
|
|
952
|
+
export function compactionFirstPreparationEventPayload(durationMs: number) {
|
|
953
|
+
return {
|
|
954
|
+
phase: "model_preparation",
|
|
955
|
+
durationMs: Math.max(0, Math.round(durationMs)),
|
|
956
|
+
expectedTransition: true,
|
|
957
|
+
failureCategory: "context_compaction",
|
|
958
|
+
failureStage: "context_compaction",
|
|
959
|
+
failureCode: "compaction_required",
|
|
960
|
+
retryable: true,
|
|
961
|
+
} as const;
|
|
962
|
+
}
|
|
963
|
+
|
|
465
964
|
/**
|
|
466
|
-
* Recognize
|
|
965
|
+
* Recognize active-route transitions that cannot finish inside their
|
|
467
966
|
* originating attempt. A Modal-home session may start on a Connected Machine
|
|
468
967
|
* without creating or leasing its managed home box. When an explicit attach
|
|
469
968
|
* clears the active pointer back to home, the pointer commit is authoritative,
|
|
@@ -474,9 +973,56 @@ export function sandboxLifecycleTransitionDiagnostic(
|
|
|
474
973
|
* only those structural error links with a strict bound; never classify from
|
|
475
974
|
* message text, which could originate in model or tool content.
|
|
476
975
|
*/
|
|
976
|
+
/**
|
|
977
|
+
* Only the home resolver's positive pre-dispatch signal permits a fresh attempt.
|
|
978
|
+
* The same error class can also escape AFTER a provider mutation, so neither its
|
|
979
|
+
* retryable property nor message alone establishes replay safety. Inspect the
|
|
980
|
+
* complete bounded SDK cause graph and let uncertain peer outcomes veto this
|
|
981
|
+
* narrow recovery; unreadable or overflowing graphs fail closed.
|
|
982
|
+
*/
|
|
983
|
+
function isPreDispatchHomeBackendRecoveryRequired(error: unknown): boolean {
|
|
984
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
985
|
+
if (!graph) return false;
|
|
986
|
+
let found = false;
|
|
987
|
+
try {
|
|
988
|
+
for (const node of graph.keys()) {
|
|
989
|
+
if (
|
|
990
|
+
isRoutingMutationOutcomeUnknownError(node) ||
|
|
991
|
+
isRoutingMutationOutputRejectedError(node) ||
|
|
992
|
+
isRawProviderCommandOutcomeUnknown(node) ||
|
|
993
|
+
isModalCommandStartOutcomeUnknownError(node) ||
|
|
994
|
+
isProviderCommandObservationUnavailableError(node)
|
|
995
|
+
)
|
|
996
|
+
return false;
|
|
997
|
+
if (node instanceof RoutingBackendRecoveryRequiredError) {
|
|
998
|
+
if (
|
|
999
|
+
node.op !== "resolve_home_backend" ||
|
|
1000
|
+
!Number.isSafeInteger(node.leaseEpoch) ||
|
|
1001
|
+
node.leaseEpoch < 0 ||
|
|
1002
|
+
!node.retryable ||
|
|
1003
|
+
(node.recovery !== "pending" && node.recovery !== "superseded")
|
|
1004
|
+
)
|
|
1005
|
+
return false;
|
|
1006
|
+
found = true;
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
} catch {
|
|
1010
|
+
return false;
|
|
1011
|
+
}
|
|
1012
|
+
return found;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
477
1015
|
export function sandboxRouteTransitionCode(
|
|
478
1016
|
error: unknown,
|
|
479
|
-
):
|
|
1017
|
+
):
|
|
1018
|
+
| "home_unavailable_this_turn"
|
|
1019
|
+
| "workspace_root_changed_this_turn"
|
|
1020
|
+
| "native_capabilities_changed_this_attempt"
|
|
1021
|
+
| "home_backend_recovery_pending"
|
|
1022
|
+
| null {
|
|
1023
|
+
if (isPreDispatchHomeBackendRecoveryRequired(error)) {
|
|
1024
|
+
return "home_backend_recovery_pending";
|
|
1025
|
+
}
|
|
480
1026
|
const pending: unknown[] = [error];
|
|
481
1027
|
const seen = new WeakSet<object>();
|
|
482
1028
|
let inspected = 0;
|
|
@@ -489,6 +1035,13 @@ export function sandboxRouteTransitionCode(
|
|
|
489
1035
|
|
|
490
1036
|
try {
|
|
491
1037
|
const record = current as Record<string, unknown>;
|
|
1038
|
+
if (
|
|
1039
|
+
(current instanceof SandboxCapabilitiesChangedError ||
|
|
1040
|
+
record.name === "SandboxCapabilitiesChangedError") &&
|
|
1041
|
+
record.code === "native_capabilities_changed_this_attempt"
|
|
1042
|
+
) {
|
|
1043
|
+
return "native_capabilities_changed_this_attempt";
|
|
1044
|
+
}
|
|
492
1045
|
if (
|
|
493
1046
|
(current instanceof ActiveBackendUnresolvableError ||
|
|
494
1047
|
record.name === "ActiveBackendUnresolvableError") &&
|
|
@@ -612,6 +1165,14 @@ export function compactionFailureReasonFromError(error: unknown): string {
|
|
|
612
1165
|
`the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
|
|
613
1166
|
);
|
|
614
1167
|
}
|
|
1168
|
+
// An exhausted provider quota is not retried (see agentRunFailurePayload),
|
|
1169
|
+
// so name the refusal plainly instead of the raw diagnostic envelope.
|
|
1170
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
1171
|
+
if (quota) {
|
|
1172
|
+
return compactionFailureReason(
|
|
1173
|
+
`${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`,
|
|
1174
|
+
);
|
|
1175
|
+
}
|
|
615
1176
|
if (
|
|
616
1177
|
error instanceof CompactionProviderResponseError ||
|
|
617
1178
|
error instanceof EmptyCompactionSummaryError
|
|
@@ -638,8 +1199,12 @@ export function compactionFailureTurnEventPayload(
|
|
|
638
1199
|
recovery: "user_message";
|
|
639
1200
|
compacted: false;
|
|
640
1201
|
providerRejection?: CompactionProviderRejection;
|
|
1202
|
+
quotaScope?: ProviderQuotaScope;
|
|
641
1203
|
} {
|
|
642
1204
|
const rejection = compactionProviderRejection(error);
|
|
1205
|
+
// The same closed marker as a `provider_quota_exhausted` turn failure, so
|
|
1206
|
+
// clients can name the exhausted limit and offer another model here too.
|
|
1207
|
+
const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
|
|
643
1208
|
return {
|
|
644
1209
|
error: overrides.error ?? compactionFailureReasonFromError(error),
|
|
645
1210
|
code: "context_compaction_failed",
|
|
@@ -647,6 +1212,7 @@ export function compactionFailureTurnEventPayload(
|
|
|
647
1212
|
recovery: "user_message",
|
|
648
1213
|
compacted: false,
|
|
649
1214
|
...(rejection ? { providerRejection: rejection } : {}),
|
|
1215
|
+
...(quota ? { quotaScope: quota.scope } : {}),
|
|
650
1216
|
};
|
|
651
1217
|
}
|
|
652
1218
|
|
|
@@ -658,18 +1224,38 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
|
|
|
658
1224
|
|
|
659
1225
|
export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
|
|
660
1226
|
if (!(error instanceof CompactionProviderResponseError)) return false;
|
|
1227
|
+
// The worker's own included-usage refusal (usage exhausted or unverifiable)
|
|
1228
|
+
// happens before dispatch, so the checkpoint request was never sent. It
|
|
1229
|
+
// settles exactly like the same refusal on an ordinary request: recorded
|
|
1230
|
+
// against the account, then the same turn re-places or waits for capacity.
|
|
1231
|
+
if (findCodexCreditPolicyError(error)) return true;
|
|
661
1232
|
if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
|
|
1233
|
+
// A Claude subscription refusal (rate limit or reconnect) on the checkpoint
|
|
1234
|
+
// request recovers exactly like one on an ordinary request: failure
|
|
1235
|
+
// settlement rotates to another account or waits for capacity on the same
|
|
1236
|
+
// logical turn. Its wording or retry hint never makes it a terminal quota.
|
|
1237
|
+
const claudeRefusal = isClaudeSubscriptionTransportError(error)
|
|
1238
|
+
? classifyClaudeProviderError(error)
|
|
1239
|
+
: null;
|
|
1240
|
+
if (claudeRefusal?.kind === "unauthorized" || claudeRefusal?.kind === "rate_limited") return true;
|
|
662
1241
|
// Codex may reject an opaque artifact it minted itself on the compaction
|
|
663
1242
|
// request exactly as it can on an ordinary request. Failure settlement
|
|
664
1243
|
// invalidates only the exact participating artifacts and recovers the same
|
|
665
1244
|
// logical turn; when nothing can be invalidated it fails closed there.
|
|
666
1245
|
if (classifyCodexEncryptedArtifactRejection(error)) return true;
|
|
1246
|
+
// A ChatGPT plan that no longer includes the model refuses the compaction
|
|
1247
|
+
// request exactly as it refuses an ordinary one (often with an empty 400).
|
|
1248
|
+
// Failure settlement re-checks the plan and, when it proves the loss,
|
|
1249
|
+
// excludes that account for the model and fails the same turn over; an
|
|
1250
|
+
// unexplained rejection still fails there with typed copy.
|
|
1251
|
+
if (classifyCodexEntitlementRejection(error)) return true;
|
|
667
1252
|
return agentRunFailurePayload(error).retryable === true;
|
|
668
1253
|
}
|
|
669
1254
|
|
|
670
1255
|
export function classifyContextWindowOverflowError(
|
|
671
1256
|
error: unknown,
|
|
672
1257
|
): { message: string; code?: string; detail?: string } | null {
|
|
1258
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
673
1259
|
const fields = collectErrorStrings(error);
|
|
674
1260
|
const matched = fields.find(
|
|
675
1261
|
(value) =>
|
|
@@ -736,6 +1322,9 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
|
|
|
736
1322
|
}
|
|
737
1323
|
seen.add(value);
|
|
738
1324
|
const out: string[] = [];
|
|
1325
|
+
// This diagnostic is already provider-owned and byte-bounded. Do not widen
|
|
1326
|
+
// generic detail traversal to arbitrary application payloads.
|
|
1327
|
+
if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
|
|
739
1328
|
const record = value as Record<string, unknown>;
|
|
740
1329
|
for (const key of ["message", "code", "type", "name", "param"]) {
|
|
741
1330
|
const field = record[key];
|
|
@@ -792,15 +1381,47 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
|
|
|
792
1381
|
*/
|
|
793
1382
|
export const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE =
|
|
794
1383
|
"unable to connect. is the computer able to access the url?";
|
|
1384
|
+
/** Bun's exact fetch text when an upstream drops an open connection. */
|
|
1385
|
+
export const STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE =
|
|
1386
|
+
"the socket connection was closed unexpectedly. for more information, pass `verbose: true` in the second argument to fetch()";
|
|
1387
|
+
const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES = new Set([
|
|
1388
|
+
STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE,
|
|
1389
|
+
STATUSLESS_UPSTREAM_SOCKET_CLOSED_MESSAGE,
|
|
1390
|
+
]);
|
|
795
1391
|
|
|
796
1392
|
export function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean {
|
|
797
|
-
return message.trim().toLowerCase()
|
|
1393
|
+
return STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGES.has(message.trim().toLowerCase());
|
|
1394
|
+
}
|
|
1395
|
+
|
|
1396
|
+
const TRANSIENT_TRANSPORT_CODE = /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i;
|
|
1397
|
+
|
|
1398
|
+
/**
|
|
1399
|
+
* A network fault code anywhere on the Error.cause chain. SDK and runner
|
|
1400
|
+
* wrappers often keep the transport error only as `cause`, so the outermost
|
|
1401
|
+
* error's own `code` is not enough.
|
|
1402
|
+
*/
|
|
1403
|
+
function transientTransportCode(error: unknown): string | undefined {
|
|
1404
|
+
let current = error;
|
|
1405
|
+
const seen = new Set<unknown>();
|
|
1406
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1407
|
+
if (seen.has(current)) break;
|
|
1408
|
+
seen.add(current);
|
|
1409
|
+
const value = current as { code?: unknown; cause?: unknown };
|
|
1410
|
+
if (typeof value.code === "string" && TRANSIENT_TRANSPORT_CODE.test(value.code)) {
|
|
1411
|
+
return value.code;
|
|
1412
|
+
}
|
|
1413
|
+
current = value.cause;
|
|
1414
|
+
}
|
|
1415
|
+
return undefined;
|
|
798
1416
|
}
|
|
799
1417
|
|
|
800
1418
|
function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
|
|
1419
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1420
|
+
return error.category === "safety" ? error.detail : undefined;
|
|
1421
|
+
}
|
|
801
1422
|
return collectErrorStrings(error).find(
|
|
802
1423
|
(value) =>
|
|
803
|
-
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
|
|
1424
|
+
/^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
|
|
804
1425
|
value,
|
|
805
1426
|
) || /\bthis request was blocked by our safety systems\b/i.test(value),
|
|
806
1427
|
);
|
|
@@ -810,16 +1431,48 @@ function isProviderSafetyRefusal(error: unknown): boolean {
|
|
|
810
1431
|
return providerSafetyRefusalDiagnostic(error) !== undefined;
|
|
811
1432
|
}
|
|
812
1433
|
|
|
1434
|
+
/** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
|
|
1435
|
+
function providerHttpStatus(error: unknown): number | undefined {
|
|
1436
|
+
let current = error;
|
|
1437
|
+
const seen = new Set<unknown>();
|
|
1438
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1439
|
+
if (seen.has(current)) break;
|
|
1440
|
+
seen.add(current);
|
|
1441
|
+
const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
|
|
1442
|
+
const status = Number(value.status ?? value.statusCode);
|
|
1443
|
+
if (Number.isInteger(status) && status >= 100 && status < 600) return status;
|
|
1444
|
+
current = value.cause;
|
|
1445
|
+
}
|
|
1446
|
+
return undefined;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
/**
|
|
1450
|
+
* A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
|
|
1451
|
+
* "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
|
|
1452
|
+
* Explicit cancellation (`AbortError`) is deliberately not matched.
|
|
1453
|
+
*/
|
|
1454
|
+
export function isTransportTimeoutError(error: unknown): boolean {
|
|
1455
|
+
let current: unknown = error;
|
|
1456
|
+
const seen = new Set<unknown>();
|
|
1457
|
+
for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
|
|
1458
|
+
if (seen.has(current)) break;
|
|
1459
|
+
seen.add(current);
|
|
1460
|
+
const value = current as { name?: unknown; message?: unknown; cause?: unknown };
|
|
1461
|
+
if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
|
|
1462
|
+
return true;
|
|
1463
|
+
}
|
|
1464
|
+
current = value.cause;
|
|
1465
|
+
}
|
|
1466
|
+
return false;
|
|
1467
|
+
}
|
|
1468
|
+
|
|
813
1469
|
export function isTransientProviderError(error: unknown): boolean {
|
|
1470
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1471
|
+
return error.category === "unavailable";
|
|
1472
|
+
}
|
|
814
1473
|
// A semantic refusal can arrive inside a 5xx transport envelope.
|
|
815
1474
|
if (isProviderSafetyRefusal(error)) return false;
|
|
816
|
-
const status =
|
|
817
|
-
typeof error === "object" && error !== null
|
|
818
|
-
? Number(
|
|
819
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
820
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
821
|
-
)
|
|
822
|
-
: undefined;
|
|
1475
|
+
const status = providerHttpStatus(error);
|
|
823
1476
|
// A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
|
|
824
1477
|
// (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
|
|
825
1478
|
// a request fault that must NOT auto-retry — even if its body happens to read like
|
|
@@ -828,13 +1481,7 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
828
1481
|
if (status !== undefined && Number.isFinite(status)) {
|
|
829
1482
|
return status >= 500 && status < 600;
|
|
830
1483
|
}
|
|
831
|
-
|
|
832
|
-
typeof error === "object" && error !== null && "code" in error
|
|
833
|
-
? String((error as { code?: unknown }).code)
|
|
834
|
-
: undefined;
|
|
835
|
-
if (code && /^(?:ECONNRESET|ETIMEDOUT|EAI_AGAIN|ECONNREFUSED|EPIPE)$/i.test(code)) {
|
|
836
|
-
return true;
|
|
837
|
-
}
|
|
1484
|
+
if (transientTransportCode(error)) return true;
|
|
838
1485
|
const message = error instanceof Error ? error.message : String(error);
|
|
839
1486
|
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
840
1487
|
return true;
|
|
@@ -844,23 +1491,58 @@ export function isTransientProviderError(error: unknown): boolean {
|
|
|
844
1491
|
);
|
|
845
1492
|
}
|
|
846
1493
|
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
}
|
|
1494
|
+
/** An explicit `usage_limit_reached` type or code string anywhere on the error chain. */
|
|
1495
|
+
function hasCodexUsageLimitType(error: unknown): boolean {
|
|
1496
|
+
return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
|
|
1497
|
+
}
|
|
851
1498
|
|
|
852
1499
|
/**
|
|
853
|
-
*
|
|
854
|
-
*
|
|
855
|
-
*
|
|
856
|
-
*
|
|
1500
|
+
* Recognize an exhausted API-key provider quota (a daily or monthly allowance,
|
|
1501
|
+
* a free-tier day cap, or an account out of credits) as distinct from an
|
|
1502
|
+
* ordinary per-minute rate limit. Retrying within the bounded same-turn budget
|
|
1503
|
+
* cannot succeed, so the turn fails promptly instead. Subscription transports
|
|
1504
|
+
* own their quota semantics through credential rotation and durable capacity
|
|
1505
|
+
* waits, so a Codex, SuperGrok or Claude subscription transport error never
|
|
1506
|
+
* classifies here.
|
|
857
1507
|
*/
|
|
858
|
-
export function
|
|
1508
|
+
export function classifyProviderQuotaExhaustionError(
|
|
1509
|
+
error: unknown,
|
|
1510
|
+
): ProviderQuotaExhaustion | null {
|
|
1511
|
+
if (
|
|
1512
|
+
isCodexTransportError(error) ||
|
|
1513
|
+
isXaiSubscriptionTransportError(error) ||
|
|
1514
|
+
isClaudeSubscriptionTransportError(error)
|
|
1515
|
+
)
|
|
1516
|
+
return null;
|
|
1517
|
+
// The same reader the OpenAI SDK retry veto uses, so the two never disagree.
|
|
1518
|
+
return classifyProviderQuotaError(error);
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
/** A refusal raised by a Claude subscription (OAuth) transport, at any wrapper depth. */
|
|
1522
|
+
export function isClaudeSubscriptionTransportError(error: unknown): boolean {
|
|
1523
|
+
let current: unknown = error;
|
|
1524
|
+
for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
|
|
1525
|
+
if (current instanceof AnthropicRequestError && current.subscription) return true;
|
|
1526
|
+
current = current.cause;
|
|
1527
|
+
}
|
|
1528
|
+
return false;
|
|
1529
|
+
}
|
|
1530
|
+
|
|
1531
|
+
/**
|
|
1532
|
+
* SuperGrok's classification of a failed model request as a shared provider
|
|
1533
|
+
* outcome (the adapter's `classifyError`, SuperGrok track X1b): a refresh
|
|
1534
|
+
* relogin or a marked 401 is `unauthorized`, a marked 403 `forbidden`, and a
|
|
1535
|
+
* marked 429 or HTTP 200 SSE capacity terminal `rate_limited` with the
|
|
1536
|
+
* provider's retry time (null when it gave none: the adapter's fallback
|
|
1537
|
+
* applies). Anything else, a provider safety refusal included, is null: not
|
|
1538
|
+
* a refusal of the account.
|
|
1539
|
+
*/
|
|
1540
|
+
export function classifyXaiProviderError(error: unknown): ProviderErrorOutcome | null {
|
|
859
1541
|
if (isProviderSafetyRefusal(error)) return null;
|
|
860
1542
|
let relogin: unknown = error;
|
|
861
1543
|
for (let depth = 0; depth < 6 && relogin && typeof relogin === "object"; depth += 1) {
|
|
862
1544
|
if (relogin instanceof XaiSubscriptionReloginRequired) {
|
|
863
|
-
return { kind: "
|
|
1545
|
+
return { kind: "unauthorized" };
|
|
864
1546
|
}
|
|
865
1547
|
relogin = (relogin as Record<string, unknown>).cause;
|
|
866
1548
|
}
|
|
@@ -875,10 +1557,10 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
875
1557
|
const status = Number(value.status ?? value.statusCode ?? body?.status ?? body?.statusCode);
|
|
876
1558
|
const code = String(value.code ?? body?.code ?? "").toLowerCase();
|
|
877
1559
|
if (status === 401 || code === "unauthorized" || code === "invalid_token") {
|
|
878
|
-
return { kind: "
|
|
1560
|
+
return { kind: "unauthorized" };
|
|
879
1561
|
}
|
|
880
1562
|
if (status === 403) {
|
|
881
|
-
return { kind: "forbidden"
|
|
1563
|
+
return { kind: "forbidden" };
|
|
882
1564
|
}
|
|
883
1565
|
const message = String(value.message ?? body?.message ?? "");
|
|
884
1566
|
if (
|
|
@@ -888,10 +1570,7 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
888
1570
|
status: Number.isInteger(status) ? status : null,
|
|
889
1571
|
})
|
|
890
1572
|
) {
|
|
891
|
-
return {
|
|
892
|
-
kind: "rate_limit",
|
|
893
|
-
cooldownMs: providerRetryAfterMs(error) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
|
|
894
|
-
};
|
|
1573
|
+
return { kind: "rate_limited", retryAfterMs: providerRetryAfterMs(error) };
|
|
895
1574
|
}
|
|
896
1575
|
current = value.cause;
|
|
897
1576
|
}
|
|
@@ -918,15 +1597,167 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
|
|
|
918
1597
|
return null;
|
|
919
1598
|
}
|
|
920
1599
|
|
|
1600
|
+
function isRawDatabaseQueryError(error: unknown): boolean {
|
|
1601
|
+
return (
|
|
1602
|
+
error instanceof Error &&
|
|
1603
|
+
"query" in error &&
|
|
1604
|
+
typeof error.query === "string" &&
|
|
1605
|
+
"params" in error &&
|
|
1606
|
+
Array.isArray(error.params)
|
|
1607
|
+
);
|
|
1608
|
+
}
|
|
1609
|
+
|
|
1610
|
+
function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
|
|
1611
|
+
if (error instanceof AnthropicRequestError) return error;
|
|
1612
|
+
return error instanceof Error && error.cause instanceof AnthropicRequestError
|
|
1613
|
+
? error.cause
|
|
1614
|
+
: undefined;
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
function isRawProviderCommandOutcomeUnknown(error: unknown): boolean {
|
|
1618
|
+
try {
|
|
1619
|
+
return (
|
|
1620
|
+
error instanceof ProviderCommandInputOutcomeUnknownError ||
|
|
1621
|
+
error instanceof ProviderCommandStartOutcomeUnknownError
|
|
1622
|
+
);
|
|
1623
|
+
} catch {
|
|
1624
|
+
return false;
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
|
|
921
1628
|
export function agentRunFailurePayload(
|
|
922
1629
|
error: unknown,
|
|
923
1630
|
options: { isCodexTurn?: boolean } = {},
|
|
924
1631
|
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1632
|
+
return withProviderCondition(error, classifyAgentRunFailurePayload(error, options));
|
|
1633
|
+
}
|
|
1634
|
+
|
|
1635
|
+
/**
|
|
1636
|
+
* Record the closed provider condition on a retryable model-provider failure so
|
|
1637
|
+
* clients can say "overloaded" rather than a generic outage. Presentation
|
|
1638
|
+
* evidence only: retry authority and pacing are decided by `code`.
|
|
1639
|
+
*/
|
|
1640
|
+
function withProviderCondition(
|
|
1641
|
+
error: unknown,
|
|
1642
|
+
failure: ReturnType<typeof baseAgentRunFailurePayload>,
|
|
1643
|
+
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1644
|
+
if (failure.code === "provider_rate_limited") {
|
|
1645
|
+
return { ...failure, providerCondition: "rate_limited" };
|
|
1646
|
+
}
|
|
1647
|
+
if (failure.code !== "provider_unavailable") return failure;
|
|
1648
|
+
if (failure.timeoutClass) return { ...failure, providerCondition: "unresponsive" };
|
|
1649
|
+
return {
|
|
1650
|
+
...failure,
|
|
1651
|
+
providerCondition: isProviderOverloadError(error, failure) ? "overloaded" : "unavailable",
|
|
1652
|
+
};
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
/** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
|
|
1656
|
+
export function isProviderOverloadError(
|
|
1657
|
+
error: unknown,
|
|
1658
|
+
failure: { error?: string; detail?: string } = {},
|
|
1659
|
+
): boolean {
|
|
1660
|
+
if (providerHttpStatus(error) === 529) return true;
|
|
1661
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
1662
|
+
return [...collectErrorStrings(error), anthropic?.detail, failure.error, failure.detail].some(
|
|
1663
|
+
(value) => typeof value === "string" && /\boverload(?:ed)?(?:_error)?\b/i.test(value),
|
|
1664
|
+
);
|
|
1665
|
+
}
|
|
1666
|
+
|
|
1667
|
+
/** Retry authority must not depend on the looser presentation-only overload label. */
|
|
1668
|
+
export function providerRecoveryCode(
|
|
1669
|
+
error: unknown,
|
|
1670
|
+
failure: { code?: string; retryable?: boolean },
|
|
1671
|
+
): string | undefined {
|
|
1672
|
+
if (failure.code !== "provider_unavailable" || !failure.retryable) return failure.code;
|
|
1673
|
+
const status = providerHttpStatus(error);
|
|
1674
|
+
let current = error;
|
|
1675
|
+
const seen = new Set<unknown>();
|
|
1676
|
+
for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
|
|
1677
|
+
if (seen.has(current)) break;
|
|
1678
|
+
seen.add(current);
|
|
1679
|
+
if (
|
|
1680
|
+
current instanceof AnthropicRequestError &&
|
|
1681
|
+
status !== undefined &&
|
|
1682
|
+
status >= 500 &&
|
|
1683
|
+
status < 600 &&
|
|
1684
|
+
(status === 529 || current.errorType === "overloaded_error")
|
|
1685
|
+
) {
|
|
1686
|
+
return PROVIDER_OVERLOAD_RECOVERY_CODE;
|
|
1687
|
+
}
|
|
1688
|
+
current = (current as { cause?: unknown }).cause;
|
|
1689
|
+
}
|
|
1690
|
+
return failure.code;
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
function classifyAgentRunFailurePayload(
|
|
1694
|
+
error: unknown,
|
|
1695
|
+
options: { isCodexTurn?: boolean } = {},
|
|
1696
|
+
): ReturnType<typeof baseAgentRunFailurePayload> {
|
|
1697
|
+
const sizeError = findAnthropicRequestSizeError(error);
|
|
1698
|
+
if (sizeError)
|
|
1699
|
+
return {
|
|
1700
|
+
error: sizeError.message,
|
|
1701
|
+
code: sizeError.code,
|
|
1702
|
+
retryable: false,
|
|
1703
|
+
requestSize: sizeError.requestSize,
|
|
1704
|
+
...(sizeError.request_id ? { requestId: sizeError.request_id } : {}),
|
|
1705
|
+
};
|
|
1706
|
+
const graph = structuredRecoveryCauseGraph(error);
|
|
1707
|
+
const nodes = graph ? [...graph.keys()] : [];
|
|
1708
|
+
const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
|
|
1709
|
+
if (outputRejected) {
|
|
1710
|
+
// A known receipt applies only to its own mutation. Uncertain peers retain
|
|
1711
|
+
// their established terminal classification, regardless of SDK graph order.
|
|
1712
|
+
const unknown = nodes.find(
|
|
1713
|
+
(node) =>
|
|
1714
|
+
isRoutingMutationOutcomeUnknownError(node) ||
|
|
1715
|
+
isModalCommandStartOutcomeUnknownError(node) ||
|
|
1716
|
+
isProviderCommandObservationUnavailableError(node) ||
|
|
1717
|
+
isRawProviderCommandOutcomeUnknown(node),
|
|
1718
|
+
);
|
|
1719
|
+
if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
|
|
1720
|
+
return {
|
|
1721
|
+
error: outputRejected.message,
|
|
1722
|
+
code: outputRejected.code,
|
|
1723
|
+
retryable: false,
|
|
1724
|
+
};
|
|
1725
|
+
}
|
|
1726
|
+
const expired = nodes.find((node) => node instanceof ProviderOverloadRecoveryExpiredError);
|
|
1727
|
+
if (expired instanceof ProviderOverloadRecoveryExpiredError) return expired.failure;
|
|
925
1728
|
const failure = baseAgentRunFailurePayload(error, options);
|
|
926
1729
|
const diagnostic = materializationVerificationDiagnostic(error);
|
|
1730
|
+
const anthropic = anthropicRequestDiagnostic(error);
|
|
1731
|
+
if (anthropic) {
|
|
1732
|
+
const authenticationRejected =
|
|
1733
|
+
anthropic.status === 401 &&
|
|
1734
|
+
(error === anthropic ||
|
|
1735
|
+
(error instanceof Error && (error as Error & { status?: unknown }).status === 401));
|
|
1736
|
+
return {
|
|
1737
|
+
...failure,
|
|
1738
|
+
code:
|
|
1739
|
+
failure.code ??
|
|
1740
|
+
(authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
|
|
1741
|
+
retryable: failure.retryable ?? false,
|
|
1742
|
+
...(anthropic.detail ? { detail: anthropic.detail } : {}),
|
|
1743
|
+
...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
927
1746
|
return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
|
|
928
1747
|
}
|
|
929
1748
|
|
|
1749
|
+
/** Keep Anthropic provider text on terminal failures, never recovery events. */
|
|
1750
|
+
export function agentRunRecoveryFailurePayload(
|
|
1751
|
+
error: unknown,
|
|
1752
|
+
failure: ReturnType<typeof agentRunFailurePayload>,
|
|
1753
|
+
): ReturnType<typeof agentRunFailurePayload> {
|
|
1754
|
+
if (!anthropicRequestDiagnostic(error)) return failure;
|
|
1755
|
+
// Project a copy: retry exhaustion still needs the terminal diagnostic.
|
|
1756
|
+
const recovery = { ...failure };
|
|
1757
|
+
delete recovery.detail;
|
|
1758
|
+
return recovery;
|
|
1759
|
+
}
|
|
1760
|
+
|
|
930
1761
|
function baseAgentRunFailurePayload(
|
|
931
1762
|
error: unknown,
|
|
932
1763
|
options: { isCodexTurn?: boolean } = {},
|
|
@@ -950,6 +1781,22 @@ function baseAgentRunFailurePayload(
|
|
|
950
1781
|
historyPersistenceStage?: MandatoryHistoryPersistenceStage;
|
|
951
1782
|
mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
|
|
952
1783
|
materializationDiagnostic?: MaterializationVerificationDiagnostic;
|
|
1784
|
+
requestSize?: AnthropicRequestSize;
|
|
1785
|
+
quotaScope?: ProviderQuotaScope;
|
|
1786
|
+
/** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
|
|
1787
|
+
planType?: string | null;
|
|
1788
|
+
/** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
|
|
1789
|
+
model?: string | null;
|
|
1790
|
+
/** Display label of the turn's model on model-provider recovery evidence. */
|
|
1791
|
+
modelLabel?: string;
|
|
1792
|
+
/** Display label of the serving provider on model-provider recovery evidence. */
|
|
1793
|
+
providerLabel?: string;
|
|
1794
|
+
/** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
|
|
1795
|
+
providerCondition?: ProviderCondition;
|
|
1796
|
+
recoveryExhausted?: boolean;
|
|
1797
|
+
providerRecoveryCount?: number;
|
|
1798
|
+
maxProviderRecoveryCount?: number;
|
|
1799
|
+
providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
|
|
953
1800
|
} {
|
|
954
1801
|
if (error instanceof SandboxMaterializationVerificationError) {
|
|
955
1802
|
return {
|
|
@@ -963,7 +1810,11 @@ function baseAgentRunFailurePayload(
|
|
|
963
1810
|
return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
|
|
964
1811
|
}
|
|
965
1812
|
if (error instanceof MandatoryHistoryPersistenceError) {
|
|
966
|
-
const
|
|
1813
|
+
const databaseFailure =
|
|
1814
|
+
isSessionEventPersistenceError(error.cause) ||
|
|
1815
|
+
findPostgresDriverError(error.cause) !== null ||
|
|
1816
|
+
isRawDatabaseQueryError(error.cause);
|
|
1817
|
+
const underlying = databaseFailure
|
|
967
1818
|
? agentRunFailurePayload(error.cause, options)
|
|
968
1819
|
: {
|
|
969
1820
|
error: error.cause instanceof Error ? error.cause.message : String(error.cause),
|
|
@@ -973,6 +1824,32 @@ function baseAgentRunFailurePayload(
|
|
|
973
1824
|
historyPersistenceStage: error.stage,
|
|
974
1825
|
};
|
|
975
1826
|
}
|
|
1827
|
+
// Raw ORM wrappers can contain the full SQL and its parameters. Classify a
|
|
1828
|
+
// real driver before provider message heuristics; the original cause stays
|
|
1829
|
+
// available to internal diagnostics. This payload grants no replay authority.
|
|
1830
|
+
const postgresDriverError = findPostgresDriverError(error);
|
|
1831
|
+
const rawOrmFailure = isRawDatabaseQueryError(error);
|
|
1832
|
+
if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
|
|
1833
|
+
const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
|
|
1834
|
+
const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
|
|
1835
|
+
return {
|
|
1836
|
+
error: "Opengeni encountered a database error.",
|
|
1837
|
+
code: databaseFailureCode(sqlState),
|
|
1838
|
+
sqlState,
|
|
1839
|
+
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1840
|
+
};
|
|
1841
|
+
}
|
|
1842
|
+
if (error instanceof AnthropicSizeRecoveryExhaustedError) {
|
|
1843
|
+
return { error: error.message, code: error.code, retryable: false };
|
|
1844
|
+
}
|
|
1845
|
+
if (error instanceof AnthropicProviderRejection) {
|
|
1846
|
+
return {
|
|
1847
|
+
error: error.message,
|
|
1848
|
+
code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
|
|
1849
|
+
retryable: false,
|
|
1850
|
+
...(error.request_id ? { requestId: error.request_id } : {}),
|
|
1851
|
+
};
|
|
1852
|
+
}
|
|
976
1853
|
const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
|
|
977
1854
|
if (safetyRefusalDiagnostic !== undefined) {
|
|
978
1855
|
return {
|
|
@@ -983,14 +1860,43 @@ function baseAgentRunFailurePayload(
|
|
|
983
1860
|
detail: safetyRefusalDiagnostic,
|
|
984
1861
|
};
|
|
985
1862
|
}
|
|
1863
|
+
if (error instanceof ResponsesStreamingTerminalError) {
|
|
1864
|
+
const quota =
|
|
1865
|
+
error.category === "unknown" || error.category === "rate_limit"
|
|
1866
|
+
? classifyProviderQuotaExhaustionError({
|
|
1867
|
+
code: error.code,
|
|
1868
|
+
error: { code: error.code, type: error.type, message: error.detail },
|
|
1869
|
+
retryAfterSeconds: error.retryAfterSeconds,
|
|
1870
|
+
})
|
|
1871
|
+
: null;
|
|
1872
|
+
if (quota) {
|
|
1873
|
+
return {
|
|
1874
|
+
error: providerQuotaExhaustedMessage(quota.scope),
|
|
1875
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
1876
|
+
retryable: false,
|
|
1877
|
+
quotaScope: quota.scope,
|
|
1878
|
+
detail: error.detail,
|
|
1879
|
+
};
|
|
1880
|
+
}
|
|
1881
|
+
return {
|
|
1882
|
+
error:
|
|
1883
|
+
error.category === "rate_limit"
|
|
1884
|
+
? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort."
|
|
1885
|
+
: error.category === "unavailable"
|
|
1886
|
+
? "The model provider is temporarily unavailable. The same turn will retry after a short delay."
|
|
1887
|
+
: "The model provider rejected the response. Automatic retries stopped.",
|
|
1888
|
+
code:
|
|
1889
|
+
error.category === "rate_limit"
|
|
1890
|
+
? "provider_rate_limited"
|
|
1891
|
+
: error.category === "unavailable"
|
|
1892
|
+
? "provider_unavailable"
|
|
1893
|
+
: "provider_request_rejected",
|
|
1894
|
+
retryable: error.category === "rate_limit" || error.category === "unavailable",
|
|
1895
|
+
detail: error.detail,
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
986
1898
|
const message = error instanceof Error ? error.message : String(error);
|
|
987
|
-
const status =
|
|
988
|
-
typeof error === "object" && error !== null
|
|
989
|
-
? Number(
|
|
990
|
-
(error as { status?: unknown; statusCode?: unknown }).status ??
|
|
991
|
-
(error as { statusCode?: unknown }).statusCode,
|
|
992
|
-
)
|
|
993
|
-
: undefined;
|
|
1899
|
+
const status = providerHttpStatus(error);
|
|
994
1900
|
const code =
|
|
995
1901
|
typeof error === "object" && error !== null && "code" in error
|
|
996
1902
|
? String((error as { code?: unknown }).code)
|
|
@@ -1021,10 +1927,29 @@ function baseAgentRunFailurePayload(
|
|
|
1021
1927
|
retryable: true,
|
|
1022
1928
|
};
|
|
1023
1929
|
}
|
|
1024
|
-
if (
|
|
1930
|
+
if (isProviderCommandObservationUnavailableError(error)) {
|
|
1025
1931
|
return {
|
|
1026
1932
|
error:
|
|
1027
|
-
"
|
|
1933
|
+
"A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
1934
|
+
code: "sandbox_command_observation_unavailable",
|
|
1935
|
+
retryable: false,
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
if (isModalCommandStartOutcomeUnknownError(error)) {
|
|
1939
|
+
return {
|
|
1940
|
+
error:
|
|
1941
|
+
"A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
|
|
1942
|
+
code: "sandbox_command_start_outcome_unknown",
|
|
1943
|
+
retryable: false,
|
|
1944
|
+
};
|
|
1945
|
+
}
|
|
1946
|
+
if (
|
|
1947
|
+
!isRoutingMutationOutcomeUnknownError(error) &&
|
|
1948
|
+
isModalTaskExecStartPreDispatchUnavailableError(error)
|
|
1949
|
+
) {
|
|
1950
|
+
return {
|
|
1951
|
+
error:
|
|
1952
|
+
"The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
|
|
1028
1953
|
code: "sandbox_command_start_unavailable",
|
|
1029
1954
|
retryable: true,
|
|
1030
1955
|
};
|
|
@@ -1089,6 +2014,33 @@ function baseAgentRunFailurePayload(
|
|
|
1089
2014
|
...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
|
|
1090
2015
|
};
|
|
1091
2016
|
}
|
|
2017
|
+
// Codex plan entitlement: the settlement path normally re-checks the plan
|
|
2018
|
+
// and records a precise payload. These branches keep any other path from
|
|
2019
|
+
// surfacing the SDK's raw "400 status code (no body)" text.
|
|
2020
|
+
if (error instanceof CodexPlanEntitlementError) {
|
|
2021
|
+
return error.payload;
|
|
2022
|
+
}
|
|
2023
|
+
// Shared subscription core placement outcomes carry their own typed copy.
|
|
2024
|
+
if (error instanceof SubscriptionCoreCodexTurnError) {
|
|
2025
|
+
return error.payload;
|
|
2026
|
+
}
|
|
2027
|
+
const entitlementRejection = classifyCodexEntitlementRejection(error);
|
|
2028
|
+
if (entitlementRejection) {
|
|
2029
|
+
return entitlementRejection.evidence === "plan_entitlement"
|
|
2030
|
+
? codexPlanEntitlementFailurePayload({
|
|
2031
|
+
accountLabel: null,
|
|
2032
|
+
planType: null,
|
|
2033
|
+
planChanged: false,
|
|
2034
|
+
modelId: null,
|
|
2035
|
+
rejection: entitlementRejection,
|
|
2036
|
+
})
|
|
2037
|
+
: codexRequestRejectedFailurePayload({
|
|
2038
|
+
accountLabel: null,
|
|
2039
|
+
planType: null,
|
|
2040
|
+
rejection: entitlementRejection,
|
|
2041
|
+
planChecked: false,
|
|
2042
|
+
});
|
|
2043
|
+
}
|
|
1092
2044
|
// A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
|
|
1093
2045
|
// must NOT be reported as a generic, retryable rate-limit (which would loop a
|
|
1094
2046
|
// goal against a capped backend). Surface a precise, actionable message with
|
|
@@ -1099,8 +2051,15 @@ function baseAgentRunFailurePayload(
|
|
|
1099
2051
|
// `usage_limit_reached` shape must still outrank generic 429 retryability.
|
|
1100
2052
|
// Credential quarantine/failover remains separately provenance-gated by
|
|
1101
2053
|
// `isCodexTransportError`; this branch only chooses the truthful user payload.
|
|
2054
|
+
// The looser "429 ... usage limit" wording counts only on a Codex transport
|
|
2055
|
+
// error: an API-key provider's 429 that says "usage limit" is provider quota
|
|
2056
|
+
// evidence, not a ChatGPT/Codex subscription cap.
|
|
2057
|
+
const creditPolicy = findCodexCreditPolicyError(error);
|
|
2058
|
+
if (creditPolicy) {
|
|
2059
|
+
return { error: creditPolicy.message, code: creditPolicy.code, retryable: false };
|
|
2060
|
+
}
|
|
1102
2061
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1103
|
-
if (usageLimit) {
|
|
2062
|
+
if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
|
|
1104
2063
|
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
1105
2064
|
}
|
|
1106
2065
|
const codexTimeout = classifyCodexResponseTimeoutError(error, {
|
|
@@ -1144,6 +2103,39 @@ function baseAgentRunFailurePayload(
|
|
|
1144
2103
|
...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
|
|
1145
2104
|
};
|
|
1146
2105
|
}
|
|
2106
|
+
// A generic model stream that went silent mid-response (typed idle bound) or
|
|
2107
|
+
// a transport timeout ("The operation timed out." DOMException TimeoutError
|
|
2108
|
+
// from the fetch layer) is a dead upstream connection, not a request fault:
|
|
2109
|
+
// checkpoint durable truth and recover the same turn within the finite
|
|
2110
|
+
// provider recovery budget. Codex/SuperGrok typed timeouts are classified
|
|
2111
|
+
// above under their own transport policy.
|
|
2112
|
+
const streamIdle = classifyModelStreamIdleTimeoutError(error);
|
|
2113
|
+
if (streamIdle) {
|
|
2114
|
+
const seconds =
|
|
2115
|
+
streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1_000) : null;
|
|
2116
|
+
const window = seconds !== null ? ` for ${seconds}s` : "";
|
|
2117
|
+
return {
|
|
2118
|
+
error:
|
|
2119
|
+
streamIdle.kind === "progress"
|
|
2120
|
+
? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.`
|
|
2121
|
+
: `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
|
|
2122
|
+
code: "provider_unavailable",
|
|
2123
|
+
retryable: true,
|
|
2124
|
+
timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
|
|
2125
|
+
responseObserved: true,
|
|
2126
|
+
detail: streamIdle.message,
|
|
2127
|
+
};
|
|
2128
|
+
}
|
|
2129
|
+
if (status === undefined && isTransportTimeoutError(error)) {
|
|
2130
|
+
return {
|
|
2131
|
+
error:
|
|
2132
|
+
"The model provider connection timed out. The same turn will retry after a short delay.",
|
|
2133
|
+
code: "provider_unavailable",
|
|
2134
|
+
retryable: true,
|
|
2135
|
+
timeoutClass: "transport",
|
|
2136
|
+
detail: message,
|
|
2137
|
+
};
|
|
2138
|
+
}
|
|
1147
2139
|
if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
|
|
1148
2140
|
return {
|
|
1149
2141
|
error:
|
|
@@ -1152,10 +2144,33 @@ function baseAgentRunFailurePayload(
|
|
|
1152
2144
|
retryable: true,
|
|
1153
2145
|
};
|
|
1154
2146
|
}
|
|
2147
|
+
// An exhausted quota also arrives as HTTP 429 (or 402), but no retry within
|
|
2148
|
+
// the finite same-turn budget can succeed. Fail the turn promptly with a
|
|
2149
|
+
// distinct code so the client can offer another model; ordinary short rate
|
|
2150
|
+
// limits fall through to the retryable branch below.
|
|
2151
|
+
if (status === 402 && code === "anthropic_billing_error") {
|
|
2152
|
+
return {
|
|
2153
|
+
error: "Claude could not bill this request. Check the account's billing and payment details.",
|
|
2154
|
+
code: "provider_billing_error",
|
|
2155
|
+
retryable: false,
|
|
2156
|
+
...(message ? { detail: message } : {}),
|
|
2157
|
+
};
|
|
2158
|
+
}
|
|
2159
|
+
const quota = classifyProviderQuotaExhaustionError(error);
|
|
2160
|
+
if (quota) {
|
|
2161
|
+
return {
|
|
2162
|
+
error: providerQuotaExhaustedMessage(quota.scope),
|
|
2163
|
+
code: PROVIDER_QUOTA_EXHAUSTED_CODE,
|
|
2164
|
+
retryable: false,
|
|
2165
|
+
quotaScope: quota.scope,
|
|
2166
|
+
...(message ? { detail: message } : {}),
|
|
2167
|
+
};
|
|
2168
|
+
}
|
|
1155
2169
|
if (
|
|
1156
2170
|
status === 429 ||
|
|
1157
|
-
|
|
1158
|
-
|
|
2171
|
+
((status === undefined || !Number.isFinite(status)) &&
|
|
2172
|
+
(code === "rate_limit_exceeded" ||
|
|
2173
|
+
/(?:too many requests|rate.?limit|\b429\b)/i.test(message)))
|
|
1159
2174
|
) {
|
|
1160
2175
|
return {
|
|
1161
2176
|
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
@@ -1172,31 +2187,24 @@ function baseAgentRunFailurePayload(
|
|
|
1172
2187
|
if (isExactStatuslessUpstreamConnectivityMessage(message)) {
|
|
1173
2188
|
return {
|
|
1174
2189
|
error:
|
|
1175
|
-
"
|
|
2190
|
+
"Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
|
|
1176
2191
|
code: "upstream_connectivity_unavailable",
|
|
1177
2192
|
retryable: true,
|
|
1178
2193
|
};
|
|
1179
2194
|
}
|
|
1180
2195
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
1181
2196
|
}
|
|
1182
|
-
const postgresDriverError = findPostgresDriverError(error);
|
|
1183
|
-
if (postgresDriverError) {
|
|
1184
|
-
const database = safeDatabaseErrorFacts(postgresDriverError);
|
|
1185
|
-
const sqlState = nestedPostgresSqlState(postgresDriverError);
|
|
1186
|
-
if (sqlState !== null || Object.keys(database).length > 0) {
|
|
1187
|
-
return {
|
|
1188
|
-
error: message,
|
|
1189
|
-
sqlState,
|
|
1190
|
-
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1191
|
-
};
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
2197
|
return { error: message };
|
|
1195
2198
|
}
|
|
1196
2199
|
|
|
1197
2200
|
export type CodexCredentialFailure = {
|
|
1198
|
-
|
|
2201
|
+
/**
|
|
2202
|
+
* `plan_entitlement` is produced only after a plan re-check proves the
|
|
2203
|
+
* serving account's current plan does not include the requested model.
|
|
2204
|
+
*/
|
|
2205
|
+
kind: "auth" | "forbidden" | "rate_limit" | "quota" | "plan_entitlement";
|
|
1199
2206
|
cooldownSeconds: number | null;
|
|
2207
|
+
origin?: "included_usage_policy" | "usage_verification_policy";
|
|
1200
2208
|
};
|
|
1201
2209
|
|
|
1202
2210
|
export const CODEX_ALLOWANCE_FALLBACK_MS = 5 * 60 * 60_000;
|
|
@@ -1216,7 +2224,11 @@ export function codexCredentialCooldownUntil(
|
|
|
1216
2224
|
> | null,
|
|
1217
2225
|
now: Date,
|
|
1218
2226
|
): Date | null {
|
|
1219
|
-
if (
|
|
2227
|
+
if (
|
|
2228
|
+
failure.kind === "auth" ||
|
|
2229
|
+
failure.kind === "forbidden" ||
|
|
2230
|
+
failure.kind === "plan_entitlement"
|
|
2231
|
+
) {
|
|
1220
2232
|
return null;
|
|
1221
2233
|
}
|
|
1222
2234
|
const providerReset =
|
|
@@ -1257,6 +2269,19 @@ export function codexCredentialCooldownUntil(
|
|
|
1257
2269
|
* progress and therefore MUST NOT walk the credential pool automatically.
|
|
1258
2270
|
*/
|
|
1259
2271
|
export function classifyCodexCredentialFailure(error: unknown): CodexCredentialFailure | null {
|
|
2272
|
+
const creditPolicy = findCodexCreditPolicyError(error);
|
|
2273
|
+
if (creditPolicy instanceof CodexIncludedUsageUnknownError) {
|
|
2274
|
+
// Verified pre-dispatch refusal. A short hard hold cannot authorize credit
|
|
2275
|
+
// fallback and does not claim that the provider exhausted its allowance.
|
|
2276
|
+
return { kind: "rate_limit", cooldownSeconds: 60, origin: "usage_verification_policy" };
|
|
2277
|
+
}
|
|
2278
|
+
if (creditPolicy instanceof CodexIncludedUsageExhaustedError) {
|
|
2279
|
+
return {
|
|
2280
|
+
kind: "quota",
|
|
2281
|
+
cooldownSeconds: creditPolicy.resetsInSeconds,
|
|
2282
|
+
origin: "included_usage_policy",
|
|
2283
|
+
};
|
|
2284
|
+
}
|
|
1260
2285
|
// A request safety refusal is not evidence that another account should run it.
|
|
1261
2286
|
if (isProviderSafetyRefusal(error)) return null;
|
|
1262
2287
|
// A permanent OAuth refresh failure is definitive and the shared resolver has
|
|
@@ -1276,6 +2301,12 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
|
|
|
1276
2301
|
if (!isCodexTransportError(error)) {
|
|
1277
2302
|
return null;
|
|
1278
2303
|
}
|
|
2304
|
+
// Plan/model entitlement evidence (an explicit plan refusal, or an empty
|
|
2305
|
+
// HTTP 400) is not account health or quota. The worker re-checks the plan
|
|
2306
|
+
// first; only a proven entitlement loss walks the pool, as `plan_entitlement`.
|
|
2307
|
+
if (classifyCodexEntitlementRejection(error)) {
|
|
2308
|
+
return null;
|
|
2309
|
+
}
|
|
1279
2310
|
const usageLimit = classifyCodexUsageLimitError(error);
|
|
1280
2311
|
if (usageLimit) {
|
|
1281
2312
|
return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
|
|
@@ -1380,3 +2411,42 @@ export function codexUsageLimitFailurePayload(
|
|
|
1380
2411
|
// open indefinitely for a goal-bearing session; cap the continuation hold so the
|
|
1381
2412
|
// goal re-evaluates at most this far out (it will re-pause if still capped).
|
|
1382
2413
|
export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
|
|
2414
|
+
|
|
2415
|
+
/**
|
|
2416
|
+
* Claude's classification of a failed model request as a shared provider
|
|
2417
|
+
* outcome (the adapter's `classifyError`, Claude track C1b), for a request
|
|
2418
|
+
* of the Claude subscription transport only: a reconnect-required refresh or
|
|
2419
|
+
* HTTP 401 is `unauthorized` (the transport already ran the turn's one
|
|
2420
|
+
* forced refresh); 429 is `rate_limited` for the turn's model `modelId`, with
|
|
2421
|
+
* the provider's retry time (null: the adapter's fallback applies); 529 or
|
|
2422
|
+
* `overloaded_error` is `overloaded`, which never moves the turn to another
|
|
2423
|
+
* account; 403 is `fatal`, which never rotates or quarantines the account.
|
|
2424
|
+
* Anything else, a provider safety refusal included, is null: not a refusal
|
|
2425
|
+
* of the account.
|
|
2426
|
+
*/
|
|
2427
|
+
export function classifyClaudeProviderError(
|
|
2428
|
+
error: unknown,
|
|
2429
|
+
modelId?: string,
|
|
2430
|
+
): ProviderErrorOutcome | null {
|
|
2431
|
+
if (isProviderSafetyRefusal(error)) return null;
|
|
2432
|
+
let current: unknown = error;
|
|
2433
|
+
for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
|
|
2434
|
+
if (current instanceof ClaudeSubscriptionReconnectRequired) return { kind: "unauthorized" };
|
|
2435
|
+
if (current instanceof AnthropicRequestError) {
|
|
2436
|
+
if (!current.subscription) return null;
|
|
2437
|
+
if (current.status === 401) return { kind: "unauthorized" };
|
|
2438
|
+
if (current.status === 429)
|
|
2439
|
+
return {
|
|
2440
|
+
kind: "rate_limited",
|
|
2441
|
+
retryAfterMs: providerRetryAfterMs(current),
|
|
2442
|
+
...(modelId === undefined ? {} : { modelId }),
|
|
2443
|
+
};
|
|
2444
|
+
if (current.status === 529 || current.errorType === "overloaded_error")
|
|
2445
|
+
return { kind: "overloaded" };
|
|
2446
|
+
if (current.status === 403) return { kind: "fatal" };
|
|
2447
|
+
return null;
|
|
2448
|
+
}
|
|
2449
|
+
current = current.cause;
|
|
2450
|
+
}
|
|
2451
|
+
return null;
|
|
2452
|
+
}
|