mixdog 0.9.51 → 0.9.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -3
- package/scripts/abort-recovery-test.mjs +17 -1
- package/scripts/agent-model-liveness-test.mjs +79 -2
- package/scripts/anthropic-admission-retry-integration-test.mjs +119 -0
- package/scripts/anthropic-transport-policy-test.mjs +466 -0
- package/scripts/atomic-lock-tryonce-test.mjs +60 -1
- package/scripts/bench-run.mjs +2 -2
- package/scripts/build-tui.mjs +13 -1
- package/scripts/channel-daemon-smoke.mjs +630 -10
- package/scripts/code-graph-aggregate-cwd-test.mjs +41 -37
- package/scripts/code-graph-disk-hit-test.mjs +11 -0
- package/scripts/code-graph-root-federation-test.mjs +273 -0
- package/scripts/compact-pressure-test.mjs +159 -1
- package/scripts/compact-smoke.mjs +80 -0
- package/scripts/context-mcp-metering-test.mjs +1350 -19
- package/scripts/deferred-tool-loading-test.mjs +17 -0
- package/scripts/desktop-session-bridge-test.mjs +704 -0
- package/scripts/freevar-smoke.mjs +7 -4
- package/scripts/gemini-provider-test.mjs +1053 -0
- package/scripts/interrupted-turn-history-test.mjs +371 -0
- package/scripts/lifecycle-api-test.mjs +137 -0
- package/scripts/max-output-recovery-test.mjs +86 -0
- package/scripts/mcp-grace-deferred-test.mjs +89 -13
- package/scripts/memory-core-input-test.mjs +10 -0
- package/scripts/memory-pg-recovery-test.mjs +59 -0
- package/scripts/openai-oauth-ws-1006-retry-test.mjs +387 -6
- package/scripts/openai-ws-early-settle-test.mjs +40 -0
- package/scripts/parent-abort-link-test.mjs +24 -0
- package/scripts/process-lifecycle-test.mjs +447 -0
- package/scripts/provider-admission-scheduler-test.mjs +582 -0
- package/scripts/provider-contract-test.mjs +525 -0
- package/scripts/provider-toolcall-test.mjs +492 -11
- package/scripts/reactive-compact-persist-smoke.mjs +59 -0
- package/scripts/resource-admission-test.mjs +789 -0
- package/scripts/session-orphan-sweep-test.mjs +27 -1
- package/scripts/shell-jobs-windows-hide-test.mjs +1 -1
- package/scripts/steering-drain-buckets-test.mjs +18 -0
- package/scripts/toolcall-args-test.mjs +14 -6
- package/scripts/tui-transcript-perf-test.mjs +5 -7
- package/src/cli.mjs +15 -2
- package/src/lib/keychain-cjs.cjs +36 -23
- package/src/runtime/agent/orchestrator/agent-runtime/agent-dispatch.mjs +27 -13
- package/src/runtime/agent/orchestrator/agent-runtime/agent-progress-watchdog.mjs +4 -1
- package/src/runtime/agent/orchestrator/agent-runtime/cache-strategy.mjs +7 -8
- package/src/runtime/agent/orchestrator/agent-trace.mjs +33 -9
- package/src/runtime/agent/orchestrator/providers/admission-scheduler.mjs +331 -0
- package/src/runtime/agent/orchestrator/providers/anthropic-effort.mjs +7 -2
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +114 -308
- package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs +31 -21
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +136 -286
- package/src/runtime/agent/orchestrator/providers/gemini-cache.mjs +42 -5
- package/src/runtime/agent/orchestrator/providers/gemini-schema.mjs +554 -42
- package/src/runtime/agent/orchestrator/providers/gemini-stream.mjs +67 -18
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +84 -150
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +17 -119
- package/src/runtime/agent/orchestrator/providers/lib/anthropic-request-utils.mjs +224 -0
- package/src/runtime/agent/orchestrator/providers/lib/env-utils.mjs +6 -0
- package/src/runtime/agent/orchestrator/providers/lib/gemini-model-catalog.mjs +119 -0
- package/src/runtime/agent/orchestrator/providers/lib/grok-tool-schema.mjs +109 -0
- package/src/runtime/agent/orchestrator/providers/lib/openai-tool-args.mjs +70 -0
- package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +54 -14
- package/src/runtime/agent/orchestrator/providers/openai-compat-presets.mjs +1 -1
- package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +28 -74
- package/src/runtime/agent/orchestrator/providers/openai-compat-xai.mjs +10 -80
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +136 -20
- package/src/runtime/agent/orchestrator/providers/openai-oauth-http-sse.mjs +94 -33
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +201 -123
- package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +37 -28
- package/src/runtime/agent/orchestrator/providers/openai-ws-delta.mjs +10 -10
- package/src/runtime/agent/orchestrator/providers/openai-ws-events.mjs +30 -3
- package/src/runtime/agent/orchestrator/providers/openai-ws-pool.mjs +36 -3
- package/src/runtime/agent/orchestrator/providers/openai-ws-stream.mjs +50 -45
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +78 -12
- package/src/runtime/agent/orchestrator/providers/opencode-go.mjs +44 -5
- package/src/runtime/agent/orchestrator/providers/registry.mjs +49 -8
- package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs +229 -111
- package/src/runtime/agent/orchestrator/session/agent-loop.mjs +130 -63
- package/src/runtime/agent/orchestrator/session/compact/engine.mjs +99 -32
- package/src/runtime/agent/orchestrator/session/context-compaction-policy.mjs +170 -0
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +37 -224
- package/src/runtime/agent/orchestrator/session/loop/compact-policy.mjs +11 -17
- package/src/runtime/agent/orchestrator/session/manager/ask-session.mjs +81 -42
- package/src/runtime/agent/orchestrator/session/manager/compaction-runner.mjs +10 -1
- package/src/runtime/agent/orchestrator/session/manager/idle-cleanup.mjs +21 -5
- package/src/runtime/agent/orchestrator/session/manager/message-sanitize.mjs +8 -28
- package/src/runtime/agent/orchestrator/session/manager/prefetch-bridge.mjs +3 -58
- package/src/runtime/agent/orchestrator/session/manager/runtime-liveness.mjs +30 -7
- package/src/runtime/agent/orchestrator/session/manager/session-close.mjs +2 -0
- package/src/runtime/agent/orchestrator/session/manager/session-crud.mjs +10 -1
- package/src/runtime/agent/orchestrator/session/manager/session-lifecycle.mjs +42 -1
- package/src/runtime/agent/orchestrator/session/manager/turn-interruption.mjs +220 -0
- package/src/runtime/agent/orchestrator/session/manager/usage-metrics.mjs +57 -16
- package/src/runtime/agent/orchestrator/session/pre-send-compact.mjs +20 -4
- package/src/runtime/agent/orchestrator/session/save-session-worker.mjs +2 -2
- package/src/runtime/agent/orchestrator/session/send-with-recovery.mjs +12 -1
- package/src/runtime/agent/orchestrator/session/store/paths-heartbeat.mjs +52 -0
- package/src/runtime/agent/orchestrator/session/store/write-guards.mjs +62 -0
- package/src/runtime/agent/orchestrator/session/store-summary-index.mjs +17 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +353 -108
- package/src/runtime/agent/orchestrator/stall-policy.mjs +2 -12
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +42 -4
- package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +74 -37
- package/src/runtime/agent/orchestrator/tools/builtin/lib/list-helpers.mjs +46 -0
- package/src/runtime/agent/orchestrator/tools/builtin/lib/search-grep-chunks.mjs +173 -0
- package/src/runtime/agent/orchestrator/tools/builtin/lib/search-input-helpers.mjs +117 -0
- package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-job-insights.mjs +199 -0
- package/src/runtime/agent/orchestrator/tools/builtin/lib/shell-spawn-helpers.mjs +107 -0
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +1 -40
- package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +19 -277
- package/src/runtime/agent/orchestrator/tools/builtin/shell-job-process.mjs +223 -2
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +317 -250
- package/src/runtime/agent/orchestrator/tools/code-graph/build.mjs +176 -21
- package/src/runtime/agent/orchestrator/tools/code-graph/disk-cache.mjs +14 -0
- package/src/runtime/agent/orchestrator/tools/code-graph/dispatch.mjs +108 -2
- package/src/runtime/agent/orchestrator/tools/code-graph/trusted-roots.mjs +93 -0
- package/src/runtime/agent/orchestrator/tools/code-graph-prewarm-worker.mjs +12 -3
- package/src/runtime/agent/orchestrator/tools/lib/shell-spawn-retry.mjs +67 -0
- package/src/runtime/agent/orchestrator/tools/shell-command.mjs +125 -85
- package/src/runtime/channels/backends/discord-gateway.mjs +1 -1
- package/src/runtime/channels/backends/discord.mjs +6 -6
- package/src/runtime/channels/lib/config.mjs +15 -2
- package/src/runtime/channels/lib/memory-client.mjs +20 -3
- package/src/runtime/channels/lib/owned-runtime.mjs +19 -12
- package/src/runtime/channels/lib/status-snapshot.mjs +9 -0
- package/src/runtime/channels/lib/tool-dispatch.mjs +9 -5
- package/src/runtime/channels/lib/worker-main.mjs +16 -5
- package/src/runtime/memory/index.mjs +46 -202
- package/src/runtime/memory/lib/memory-action-handlers.mjs +2 -53
- package/src/runtime/memory/lib/memory-daemon-lifecycle.mjs +115 -0
- package/src/runtime/memory/lib/memory-port-advertiser.mjs +105 -0
- package/src/runtime/memory/lib/pg/adapter.mjs +84 -10
- package/src/runtime/memory/lib/pg/process.mjs +91 -47
- package/src/runtime/memory/lib/pg/supervisor.mjs +50 -17
- package/src/runtime/memory/lib/query-handlers.mjs +2 -122
- package/src/runtime/memory/lib/query-maintenance-handlers.mjs +126 -0
- package/src/runtime/memory/lib/tool-call-handler.mjs +57 -0
- package/src/runtime/search/lib/http-fetch.mjs +1 -1
- package/src/runtime/shared/atomic-file.mjs +28 -150
- package/src/runtime/shared/config.mjs +58 -13
- package/src/runtime/shared/llm/cost.mjs +14 -4
- package/src/runtime/shared/memory-snapshot.mjs +236 -0
- package/src/runtime/shared/process-lifecycle.mjs +436 -0
- package/src/runtime/shared/process-shutdown.mjs +33 -4
- package/src/runtime/shared/resource-admission.mjs +364 -0
- package/src/runtime/shared/staged-child-result.mjs +19 -0
- package/src/session-runtime/channel-config-api.mjs +7 -7
- package/src/session-runtime/config-lifecycle.mjs +20 -17
- package/src/session-runtime/context-status.mjs +38 -27
- package/src/session-runtime/cwd-plugins.mjs +9 -7
- package/src/session-runtime/env.mjs +1 -2
- package/src/session-runtime/hitch-profile.mjs +45 -0
- package/src/session-runtime/lifecycle-api.mjs +60 -13
- package/src/session-runtime/mcp-glue.mjs +6 -11
- package/src/session-runtime/provider-init-key.mjs +17 -0
- package/src/session-runtime/provider-request-tools.mjs +72 -0
- package/src/session-runtime/runtime-core.mjs +49 -105
- package/src/session-runtime/runtime-paths.mjs +20 -0
- package/src/session-runtime/runtime-tool-routing.mjs +55 -0
- package/src/session-runtime/session-turn-api.mjs +5 -4
- package/src/session-runtime/tool-catalog-data.mjs +51 -0
- package/src/session-runtime/tool-catalog.mjs +390 -104
- package/src/standalone/agent-tool/spawn-preset.mjs +73 -0
- package/src/standalone/agent-tool/worker-rows.mjs +93 -0
- package/src/standalone/agent-tool.mjs +29 -200
- package/src/standalone/channel-admin.mjs +29 -0
- package/src/standalone/channel-daemon-client.mjs +200 -38
- package/src/standalone/channel-daemon-transport.mjs +136 -9
- package/src/standalone/channel-worker.mjs +79 -12
- package/src/tui/App.jsx +11 -5
- package/src/tui/dist/index.mjs +275 -343
- package/src/tui/engine/session-api-ext.mjs +50 -6
- package/src/tui/engine/session-api.mjs +1 -1
- package/src/tui/engine/turn.mjs +10 -6
- package/src/tui/engine.mjs +13 -4
- package/src/tui/index.jsx +5 -1
- package/src/tui/lib/voice-setup.mjs +21 -5
- package/scripts/_devtools-stub.mjs +0 -1
- package/src/runtime/lib/keychain-cjs.cjs +0 -288
|
@@ -28,17 +28,9 @@ import {
|
|
|
28
28
|
// HTTP statuses considered transient — safe to retry with backoff.
|
|
29
29
|
// 408 — request timeout
|
|
30
30
|
// 500/502/503/504 — server errors (overload / bad gateway / timeout)
|
|
31
|
-
// 429 is
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
// brief throttle windows retry once the server-stated wait elapses; long or
|
|
35
|
-
// headerless 429s still surface immediately (quota windows are
|
|
36
|
-
// deterministic — sleeping into an outer watchdog just relabels the error).
|
|
37
|
-
const TRANSIENT_STATUSES = new Set([408, 500, 502, 503, 504])
|
|
38
|
-
|
|
39
|
-
// Max server-stated Retry-After we are willing to sleep on a 429 before
|
|
40
|
-
// surfacing it. Mirrors the first-party client's 20s short-retry threshold.
|
|
41
|
-
const SHORT_RETRY_AFTER_MAX_MS = 20_000
|
|
31
|
+
// 429 is handled separately by withRetry(): only the affected request waits
|
|
32
|
+
// with jitter; provider/account admission concurrency remains fixed.
|
|
33
|
+
const TRANSIENT_STATUSES = new Set([408, 409])
|
|
42
34
|
|
|
43
35
|
// HTTP statuses that mean "permanent: stop retrying, surface to caller".
|
|
44
36
|
// 401/403 — auth issue
|
|
@@ -109,38 +101,76 @@ export function classifyError(err) {
|
|
|
109
101
|
// Once a streamed tool call has been surfaced to the loop, retrying the same
|
|
110
102
|
// provider turn can double-execute that tool. Providers mark these stream
|
|
111
103
|
// failures as unsafe so the shared retry wrapper fails fast.
|
|
112
|
-
if (err.emittedToolCall === true || err.toolCallEmitted === true
|
|
104
|
+
if (err.emittedToolCall === true || err.toolCallEmitted === true
|
|
105
|
+
|| err.partialToolCall === true || err.emittedThinking === true
|
|
106
|
+
|| err.unsafeToRetry === true) return 'permanent'
|
|
107
|
+
// Cancellation is a caller decision, never a transport symptom. Anthropic's
|
|
108
|
+
// APIUserAbortError inherits Error without overriding `name`, so recognize
|
|
109
|
+
// only exact SDK constructor/type markers (plus standard AbortError markers)
|
|
110
|
+
// across the bounded chain before considering stale connection causes.
|
|
111
|
+
const chain = boundedCauseChain(err)
|
|
112
|
+
if (chain.some(isExplicitUserAbortError)) return 'permanent'
|
|
113
|
+
|
|
114
|
+
// Current typed HTTP status outranks stale stream/connection annotations.
|
|
115
|
+
const status = Number(err.httpStatus || err.status || err.response?.status || 0) || populateHttpStatusFromMessage(err)
|
|
116
|
+
if (AUTH_STATUSES.has(status)) return 'auth'
|
|
117
|
+
if (status === 429) return 'permanent'
|
|
118
|
+
if (PERMANENT_STATUSES.has(status)
|
|
119
|
+
|| (status >= 400 && status < 500 && !TRANSIENT_STATUSES.has(status))) return 'permanent'
|
|
113
120
|
// Truncated SSE stream (message_start without message_stop). These are
|
|
114
121
|
// idempotent to retry: the partial result is discarded, and a pendingToolUse
|
|
115
122
|
// means the tool_use input JSON never completed, so re-requesting is safe.
|
|
116
|
-
//
|
|
117
|
-
// carries a 4xx/429 status still classifies transient per the contract.
|
|
118
|
-
// Treating this as transient is what lets every withRetry / mid-stream-loop
|
|
119
|
-
// consumer recover a cut-off stream uniformly.
|
|
123
|
+
// A current permanent/auth status and cancellation were checked above.
|
|
120
124
|
if (err.truncatedStream === true || err.code === 'TRUNCATED_STREAM') return 'transient'
|
|
121
125
|
|
|
122
|
-
|
|
123
|
-
const status = Number(err.httpStatus || err.status || err.response?.status || 0) || populateHttpStatusFromMessage(err)
|
|
124
|
-
if (AUTH_STATUSES.has(status)) return 'auth'
|
|
125
|
-
if (status === 429) return 'permanent'
|
|
126
|
-
if (TRANSIENT_STATUSES.has(status)) return 'transient'
|
|
127
|
-
if (PERMANENT_STATUSES.has(status)) return 'permanent'
|
|
126
|
+
if (TRANSIENT_STATUSES.has(status) || (status >= 500 && status < 600)) return 'transient'
|
|
128
127
|
|
|
129
128
|
// Socket-level codes (Node errno) — DNS / reset / refused / timeout are all
|
|
130
129
|
// transient: we can retry the same request and may succeed.
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
130
|
+
if (chain.some((item) => TRANSIENT_ERROR_CODES.has(String(item?.code || '')))) return 'transient'
|
|
131
|
+
// The Anthropic SDK uses APIConnectionError for transport failures which
|
|
132
|
+
// may not carry a Node errno. Native fetch commonly wraps the errno in
|
|
133
|
+
// cause.code, or exposes only TypeError("fetch failed").
|
|
134
|
+
const name = String(err.name || '')
|
|
135
|
+
const message = String(err.message || '')
|
|
136
|
+
if (name === 'APIConnectionError'
|
|
137
|
+
|| (name === 'TypeError' && /fetch failed|network error/i.test(message))) {
|
|
138
138
|
return 'transient'
|
|
139
139
|
}
|
|
140
140
|
|
|
141
141
|
return 'unknown'
|
|
142
142
|
}
|
|
143
143
|
|
|
144
|
+
const MAX_CAUSE_CHAIN_DEPTH = 8
|
|
145
|
+
const TRANSIENT_ERROR_CODES = new Set([
|
|
146
|
+
'ECONNRESET', 'ETIMEDOUT', 'ESOCKETTIMEDOUT', 'EAI_AGAIN', 'ENOTFOUND',
|
|
147
|
+
'EAI_NODATA', 'ECONNREFUSED', 'ENETUNREACH', 'EHOSTUNREACH', 'EPIPE',
|
|
148
|
+
'EPROVIDERTIMEOUT', 'EGEMINITIMEOUT', 'ESTREAMSTALL', 'EWSACQUIRETIMEOUT',
|
|
149
|
+
])
|
|
150
|
+
|
|
151
|
+
function boundedCauseChain(err) {
|
|
152
|
+
const chain = []
|
|
153
|
+
const seen = new Set()
|
|
154
|
+
let cursor = err
|
|
155
|
+
while (cursor && chain.length < MAX_CAUSE_CHAIN_DEPTH && !seen.has(cursor)) {
|
|
156
|
+
chain.push(cursor)
|
|
157
|
+
seen.add(cursor)
|
|
158
|
+
cursor = cursor?.cause
|
|
159
|
+
}
|
|
160
|
+
return chain
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function isExplicitUserAbortError(err) {
|
|
164
|
+
if (!err || (typeof err !== 'object' && typeof err !== 'function')) return false
|
|
165
|
+
if (err.name === 'AbortError' || err.name === 'APIUserAbortError' || err.code === 'ABORT_ERR') return true
|
|
166
|
+
if (err.type === 'APIUserAbortError' || err.type === 'api_user_abort_error') return true
|
|
167
|
+
try {
|
|
168
|
+
return err.constructor?.name === 'APIUserAbortError'
|
|
169
|
+
} catch {
|
|
170
|
+
return false
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
144
174
|
// Provider error-text signatures for a context-window / input-too-large
|
|
145
175
|
// rejection. These are DETERMINISTIC refusals (the request is simply too big)
|
|
146
176
|
// — not transient faults — so they must never be routed through the
|
|
@@ -202,6 +232,17 @@ export function retryAfterMsFromError(err) {
|
|
|
202
232
|
return null
|
|
203
233
|
}
|
|
204
234
|
|
|
235
|
+
function isPermanentQuotaError(err) {
|
|
236
|
+
const permanentCodes = new Set(['insufficient_quota', 'quota_exceeded', 'resource_exhausted'])
|
|
237
|
+
for (const item of boundedCauseChain(err)) {
|
|
238
|
+
const codes = [item?.code, item?.error?.code]
|
|
239
|
+
if (codes.some((code) => permanentCodes.has(String(code || '').toLowerCase()))) return true
|
|
240
|
+
const text = `${String(item?.message || '')} ${String(item?.error?.message || '')}`
|
|
241
|
+
if (/insufficient_quota|quota[_ -]?exceeded|resource exhausted/i.test(text)) return true
|
|
242
|
+
}
|
|
243
|
+
return false
|
|
244
|
+
}
|
|
245
|
+
|
|
205
246
|
/**
|
|
206
247
|
* Convenience predicate: should this error be retried at the request level?
|
|
207
248
|
* Wraps classifyError() with the standard "transient = retry, otherwise no"
|
|
@@ -213,6 +254,43 @@ export function isRetryable(err) {
|
|
|
213
254
|
return classifyError(err) === 'transient'
|
|
214
255
|
}
|
|
215
256
|
|
|
257
|
+
/** Claude Code compatible Anthropic request budget: 10 retries (11 attempts).
|
|
258
|
+
* CLAUDE_CODE_MAX_RETRIES is intentionally read per request for reload/tests.
|
|
259
|
+
* The upper bound prevents an accidental unbounded retry loop. */
|
|
260
|
+
export function anthropicMaxAttempts() {
|
|
261
|
+
const raw = process.env.CLAUDE_CODE_MAX_RETRIES
|
|
262
|
+
const parsed = raw == null || raw === '' ? 10 : Number.parseInt(raw, 10)
|
|
263
|
+
const retries = Number.isFinite(parsed) ? Math.min(Math.max(parsed, 0), 100) : 10
|
|
264
|
+
return retries + 1
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
// Claude Code request defaults (withRetry.ts): 500ms exponential backoff,
|
|
268
|
+
// capped at 32s, with positive-only jitter up to 25% of the base delay.
|
|
269
|
+
// The leading duplicate accounts for withRetry's sleep-before-attempt index:
|
|
270
|
+
// retry attempt 2 reads index 1.
|
|
271
|
+
export const ANTHROPIC_RETRY_BACKOFF_MS = Object.freeze([
|
|
272
|
+
500, 500, 1000, 2000, 4000, 8000, 16000, 32000, 32000, 32000, 32000,
|
|
273
|
+
])
|
|
274
|
+
export const ANTHROPIC_RETRY_JITTER_RATIO = 0.25
|
|
275
|
+
|
|
276
|
+
// Claude Code's Anthropic SDK client defaults API_TIMEOUT_MS to ten minutes.
|
|
277
|
+
// Read per request, like CLAUDE_CODE_MAX_RETRIES, so env reload/tests work.
|
|
278
|
+
export function anthropicRequestTimeoutMs() {
|
|
279
|
+
const parsed = Number.parseInt(process.env.API_TIMEOUT_MS || '', 10)
|
|
280
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : 600_000
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
export const ANTHROPIC_MAX_CONSECUTIVE_529 = 3
|
|
284
|
+
|
|
285
|
+
export class AnthropicFallbackTriggeredError extends Error {
|
|
286
|
+
constructor(originalModel, fallbackModel, cause) {
|
|
287
|
+
super(`Anthropic model fallback triggered: ${originalModel} -> ${fallbackModel}`, { cause })
|
|
288
|
+
this.name = 'AnthropicFallbackTriggeredError'
|
|
289
|
+
this.originalModel = originalModel
|
|
290
|
+
this.fallbackModel = fallbackModel
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
216
294
|
// Default backoff schedule used by withRetry when caller does not override.
|
|
217
295
|
// Mirrors anthropic-oauth's 5-attempt curve (immediate + 1s/2s/4s/8s) so the
|
|
218
296
|
// total cap stays under 15s. Total upper bound = sum = 15s.
|
|
@@ -246,11 +324,10 @@ export function jitterDelayMs(ms, ratio = PROVIDER_RETRY_JITTER_RATIO, mode = 's
|
|
|
246
324
|
// branched on a hardcoded provider name.
|
|
247
325
|
|
|
248
326
|
// F) Retry-budget profiles as DATA. The numbers live ONLY here now.
|
|
249
|
-
// ws
|
|
250
|
-
// ws.defaultRetries (2) — every other WS mid-stream bucket.
|
|
327
|
+
// ws.*Retries (5) — one Codex Responses stream retry budget.
|
|
251
328
|
// sse.defaultRetries (3) — anthropic single-shot SSE mid-stream budget.
|
|
252
329
|
export const MIDSTREAM_RETRY_POLICY = {
|
|
253
|
-
ws: { transientCloseRetries:
|
|
330
|
+
ws: { transientCloseRetries: 5, defaultRetries: 5, backoff: [250, 1000, 2000, 4000, 5000] },
|
|
254
331
|
sse: { defaultRetries: 3, backoff: [250, 1000, 2000, 4000] },
|
|
255
332
|
}
|
|
256
333
|
|
|
@@ -286,10 +363,7 @@ export function classifyMidstreamError(err, signals, policy = {}) {
|
|
|
286
363
|
return _classifyMidstreamWs(err, signals, attemptIndex, policy)
|
|
287
364
|
}
|
|
288
365
|
|
|
289
|
-
//
|
|
290
|
-
// carries sawResponseCreated / sawCompleted / emittedText / emittedToolCall /
|
|
291
|
-
// wsCloseCode / firstByteTimeout / wsSendFailed /
|
|
292
|
-
// userAbort / watchdogAbort / responseFailedPayload exactly as before.
|
|
366
|
+
// WebSocket classification consumes the provider's stream-state signals.
|
|
293
367
|
function _classifyMidstreamWs(err, state, attemptIndex, policy) {
|
|
294
368
|
if (state.sawCompleted) return null
|
|
295
369
|
// Once a tool call has been dispatched, no transport outcome is replay-safe.
|
|
@@ -297,6 +371,9 @@ function _classifyMidstreamWs(err, state, attemptIndex, policy) {
|
|
|
297
371
|
// already be executing, so retry/fallback could duplicate its side effect.
|
|
298
372
|
if (state.emittedToolCall) return null
|
|
299
373
|
if (state.emittedText || err?.liveTextEmitted) return null
|
|
374
|
+
if (err?.wsFrameTooLarge || state.wsFrameTooLarge) {
|
|
375
|
+
return _allowMidstream('ws_frame_too_large', attemptIndex, policy)
|
|
376
|
+
}
|
|
300
377
|
if (state.firstByteTimeout || err?.firstByteTimeout) {
|
|
301
378
|
return _allowMidstream('first_byte_timeout', attemptIndex, policy)
|
|
302
379
|
}
|
|
@@ -373,19 +450,18 @@ function _classifyMidstreamWs(err, state, attemptIndex, policy) {
|
|
|
373
450
|
return null
|
|
374
451
|
}
|
|
375
452
|
|
|
376
|
-
//
|
|
377
|
-
// carries sawMessageStart / sawCompleted / emittedToolCall / userAbort /
|
|
378
|
-
// watchdogAbort exactly as before.
|
|
453
|
+
// SSE classification consumes the provider's stream-state signals.
|
|
379
454
|
function _classifyMidstreamSse(err, state, attemptIndex, policy) {
|
|
380
455
|
if (attemptIndex >= policy.defaultRetries) return null
|
|
381
456
|
if (state.sawCompleted) return null
|
|
382
|
-
if (!state.sawMessageStart) return null
|
|
383
457
|
if (state.userAbort) return null
|
|
384
|
-
if (state.emittedToolCall) return null
|
|
458
|
+
if (state.emittedText || state.emittedToolCall || state.partialToolCall || state.emittedThinking) return null
|
|
385
459
|
|
|
386
460
|
if (!err) return null
|
|
387
|
-
const status = Number(err?.httpStatus || 0)
|
|
388
|
-
if (status === 401 || status === 403
|
|
461
|
+
const status = Number(err?.httpStatus || err?.status || err?.response?.status || 0)
|
|
462
|
+
if (status === 401 || status === 403) return null
|
|
463
|
+
if (status === 429) return 'http_429'
|
|
464
|
+
if (status >= 500 && status < 600) return `http_${status}`
|
|
389
465
|
|
|
390
466
|
const name = err?.name || ''
|
|
391
467
|
if (name === 'AgentStallAbortError') return 'agent_stall'
|
|
@@ -407,6 +483,7 @@ function _classifyMidstreamSse(err, state, attemptIndex, policy) {
|
|
|
407
483
|
if (msg.includes('stream timed out after') && msg.includes('of inactivity')) return 'sse_idle_timeout'
|
|
408
484
|
if (msg.includes('body stream') && msg.includes('terminated')) return 'stream_terminated'
|
|
409
485
|
if (msg.includes('fetch failed')) return 'fetch_failed'
|
|
486
|
+
if (classifyError(err) === 'transient') return 'connection'
|
|
410
487
|
|
|
411
488
|
return null
|
|
412
489
|
}
|
|
@@ -432,8 +509,12 @@ export function shouldFallbackTransport(err, { signal, enabled = true } = {}) {
|
|
|
432
509
|
if (err?.liveTextEmitted === true) return false
|
|
433
510
|
if (err?.emittedToolCall === true || err?.unsafeToRetry === true) return false
|
|
434
511
|
const status = Number(err?.httpStatus || err?.status || 0)
|
|
435
|
-
|
|
436
|
-
|
|
512
|
+
// 401 is auth recovery, never transport fallback. Codex treats other
|
|
513
|
+
// unexpected handshake statuses as retryable transport errors; 426 is the
|
|
514
|
+
// explicit immediate WS→HTTPS switch.
|
|
515
|
+
if (status === 401) return false
|
|
516
|
+
if (status === 426) return true
|
|
517
|
+
if (status > 0) return true
|
|
437
518
|
const code = String(err?.code || '')
|
|
438
519
|
if (TRANSPORT_FALLBACK_ERRNO.has(code)) return true
|
|
439
520
|
const classifier = String(err?.retryClassifier || err?.midstreamClassifier || '')
|
|
@@ -469,46 +550,81 @@ export function createStreamSafetyStamps() {
|
|
|
469
550
|
}
|
|
470
551
|
|
|
471
552
|
const _defaultAbortSleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
553
|
+
export const MAX_SAFE_TIMEOUT_MS = 2_147_483_647
|
|
472
554
|
|
|
473
555
|
// D) Abort-aware sleep (single copy). Resolves after `ms`, or rejects with the
|
|
474
556
|
// signal's reason (or `abortMessage`) the moment the signal aborts. `sleepFn`
|
|
475
|
-
// is
|
|
476
|
-
//
|
|
557
|
+
// is injectable for deterministic tests. Oversized deadlines are chunked so
|
|
558
|
+
// Node never clamps setTimeout(>2^31-1) to approximately 1ms.
|
|
477
559
|
export async function sleepWithAbort(ms, signal, sleepFn = _defaultAbortSleep, abortMessage = 'sleep aborted') {
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
560
|
+
let remaining = Math.max(0, Number(ms) || 0)
|
|
561
|
+
const sleeper = sleepFn || _defaultAbortSleep
|
|
562
|
+
while (remaining > 0) {
|
|
563
|
+
if (signal?.aborted) {
|
|
564
|
+
const reason = signal.reason
|
|
565
|
+
throw reason instanceof Error ? reason : new Error(abortMessage)
|
|
566
|
+
}
|
|
567
|
+
const chunk = Math.min(remaining, MAX_SAFE_TIMEOUT_MS)
|
|
568
|
+
await _sleepChunkWithAbort(chunk, signal, sleeper, abortMessage)
|
|
569
|
+
remaining -= chunk
|
|
482
570
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function _sleepChunkWithAbort(ms, signal, sleepFn, abortMessage) {
|
|
574
|
+
if (!signal) return Promise.resolve().then(() => sleepFn(ms))
|
|
575
|
+
if (sleepFn === _defaultAbortSleep) {
|
|
576
|
+
return new Promise((resolve, reject) => {
|
|
577
|
+
const timer = setTimeout(() => {
|
|
578
|
+
try { signal.removeEventListener('abort', onAbort) } catch {}
|
|
579
|
+
resolve()
|
|
580
|
+
}, ms)
|
|
581
|
+
const onAbort = () => {
|
|
582
|
+
clearTimeout(timer)
|
|
583
|
+
const reason = signal.reason
|
|
584
|
+
reject(reason instanceof Error ? reason : new Error(abortMessage))
|
|
585
|
+
}
|
|
586
|
+
if (signal.aborted) { onAbort(); return }
|
|
587
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
588
|
+
})
|
|
589
|
+
}
|
|
590
|
+
return new Promise((resolve, reject) => {
|
|
591
|
+
let settled = false
|
|
488
592
|
const onAbort = () => {
|
|
489
|
-
|
|
593
|
+
if (settled) return
|
|
594
|
+
settled = true
|
|
490
595
|
const reason = signal.reason
|
|
491
596
|
reject(reason instanceof Error ? reason : new Error(abortMessage))
|
|
492
597
|
}
|
|
493
598
|
if (signal.aborted) { onAbort(); return }
|
|
494
599
|
signal.addEventListener('abort', onAbort, { once: true })
|
|
600
|
+
Promise.resolve()
|
|
601
|
+
.then(() => sleepFn(ms))
|
|
602
|
+
.then(() => {
|
|
603
|
+
if (settled) return
|
|
604
|
+
settled = true
|
|
605
|
+
try { signal.removeEventListener('abort', onAbort) } catch {}
|
|
606
|
+
resolve()
|
|
607
|
+
}, (err) => {
|
|
608
|
+
if (settled) return
|
|
609
|
+
settled = true
|
|
610
|
+
try { signal.removeEventListener('abort', onAbort) } catch {}
|
|
611
|
+
reject(err)
|
|
612
|
+
})
|
|
495
613
|
})
|
|
496
614
|
}
|
|
497
615
|
|
|
498
616
|
// E) Handshake classifier (moved here from openai-oauth-ws). Default-deny:
|
|
499
|
-
// anything not recognized as transient returns null. 401
|
|
500
|
-
//
|
|
501
|
-
// '
|
|
617
|
+
// anything not recognized as transient returns null. HTTP 401 is reserved
|
|
618
|
+
// for auth recovery and 426 for immediate HTTPS fallback; every other HTTP
|
|
619
|
+
// handshake status follows Codex's retryable UnexpectedStatus policy.
|
|
502
620
|
export function classifyHandshakeError(err) {
|
|
503
621
|
if (!err) return null
|
|
504
622
|
const code = err.code || ''
|
|
505
623
|
const msg = String(err.message || '')
|
|
506
624
|
const status = Number(err.httpStatus || 0)
|
|
507
625
|
|
|
508
|
-
if (status === 401 || status ===
|
|
509
|
-
|
|
510
|
-
}
|
|
511
|
-
if (status >= 500 && status < 600) {
|
|
626
|
+
if (status === 401 || status === 426) return null
|
|
627
|
+
if (status > 0) {
|
|
512
628
|
return `http_${status}`
|
|
513
629
|
}
|
|
514
630
|
|
|
@@ -550,12 +666,14 @@ export async function withRetry(fn, opts = {}) {
|
|
|
550
666
|
const onRetry = typeof opts.onRetry === 'function' ? opts.onRetry : null
|
|
551
667
|
const perAttemptTimeoutMs = Number(opts.perAttemptTimeoutMs || 0)
|
|
552
668
|
const perAttemptLabel = opts.perAttemptLabel || 'provider request'
|
|
553
|
-
const maxRetryAfterMs = Number(opts.maxRetryAfterMs ?? PROVIDER_MAX_BEFORE_WARN_MS)
|
|
554
669
|
const retryJitterRatio = Number(opts.retryJitterRatio ?? PROVIDER_RETRY_JITTER_RATIO)
|
|
670
|
+
const retryJitterMode = opts.retryJitterMode === 'positive' ? 'positive' : 'symmetric'
|
|
671
|
+
const sleepFn = typeof opts.sleepFn === 'function' ? opts.sleepFn : undefined
|
|
555
672
|
|
|
556
673
|
let lastErr = null
|
|
557
674
|
let nextDelayMs = null
|
|
558
675
|
let nextDelayReason = null
|
|
676
|
+
let consecutive529Errors = Math.max(0, Number(opts.initialConsecutive529Errors) || 0)
|
|
559
677
|
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
560
678
|
if (signal?.aborted) {
|
|
561
679
|
const reason = signal.reason
|
|
@@ -563,16 +681,13 @@ export async function withRetry(fn, opts = {}) {
|
|
|
563
681
|
}
|
|
564
682
|
if (attempt > 0) {
|
|
565
683
|
const rawWait = nextDelayMs ?? backoffMs[Math.min(attempt, backoffMs.length - 1)] ?? 0
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
: wait
|
|
574
|
-
onRetry?.({ attempt, lastErr, delayMs: boundedWait, delayReason: nextDelayReason })
|
|
575
|
-
if (boundedWait > 0) await _sleepWithAbort(boundedWait, signal)
|
|
684
|
+
// Retry-After is a server-mandated minimum. Do not cap, shorten, or
|
|
685
|
+
// jitter it; cancellation remains active throughout the full wait.
|
|
686
|
+
const wait = nextDelayReason === 'retry-after'
|
|
687
|
+
? Math.max(0, rawWait)
|
|
688
|
+
: jitterDelayMs(rawWait, retryJitterRatio, retryJitterMode)
|
|
689
|
+
onRetry?.({ attempt, lastErr, delayMs: wait, delayReason: nextDelayReason })
|
|
690
|
+
if (wait > 0) await sleepWithAbort(wait, signal, sleepFn, 'withRetry: sleep aborted')
|
|
576
691
|
if (signal?.aborted) {
|
|
577
692
|
const reason = signal.reason
|
|
578
693
|
throw reason instanceof Error ? reason : new Error('withRetry: aborted')
|
|
@@ -599,18 +714,46 @@ export async function withRetry(fn, opts = {}) {
|
|
|
599
714
|
populateHttpStatusFromMessage(caught)
|
|
600
715
|
const status = Number(caught?.httpStatus || caught?.status || caught?.response?.status || 0)
|
|
601
716
|
const kind = classifyError(caught)
|
|
602
|
-
const
|
|
603
|
-
|| caught?.
|
|
604
|
-
|| caught?.
|
|
605
|
-
|
|
717
|
+
const outputWasExposed = caught?.liveTextEmitted === true
|
|
718
|
+
|| caught?.emittedText === true
|
|
719
|
+
|| caught?.emittedToolCall === true
|
|
720
|
+
|| caught?.toolCallEmitted === true
|
|
721
|
+
|| caught?.partialToolCall === true
|
|
722
|
+
|| caught?.emittedThinking === true
|
|
723
|
+
if (outputWasExposed || caught?.unsafeToRetry === true) throw caught
|
|
724
|
+
// Claude Code treats x-should-retry:false as an explicit server veto
|
|
725
|
+
// (except an internal-only 5xx override that Mixdog does not have).
|
|
726
|
+
// Keep this ahead of status defaults, including the request-local 429 path.
|
|
727
|
+
const shouldRetryHeader = _headerValue(
|
|
728
|
+
caught?.headers || caught?.response?.headers || caught?.data?.responseHeaders,
|
|
729
|
+
'x-should-retry',
|
|
730
|
+
)
|
|
731
|
+
if (String(shouldRetryHeader || '').toLowerCase() === 'false') throw caught
|
|
732
|
+
// Claude Code's optional model fallback fires on the third 529. This
|
|
733
|
+
// remains opt-in: providers pass fallbackModel only when the caller set
|
|
734
|
+
// one. The hard progress veto above must run first so fallback can never
|
|
735
|
+
// replay partial thinking/tool output.
|
|
736
|
+
if (status === 529 && opts.fallbackModel && opts.fallbackModel !== opts.model) {
|
|
737
|
+
consecutive529Errors += 1
|
|
738
|
+
if (consecutive529Errors >= ANTHROPIC_MAX_CONSECUTIVE_529) {
|
|
739
|
+
throw new AnthropicFallbackTriggeredError(opts.model, opts.fallbackModel, caught)
|
|
740
|
+
}
|
|
741
|
+
}
|
|
606
742
|
if (status === 429) {
|
|
607
|
-
//
|
|
608
|
-
//
|
|
609
|
-
//
|
|
743
|
+
// A deterministic quota refusal cannot recover by replaying the same
|
|
744
|
+
// request. This check must precede generic HTTP 429 handling so shared
|
|
745
|
+
// OpenAI-compatible/Gemini message-only quota errors remain terminal.
|
|
746
|
+
if (isPermanentQuotaError(caught)) throw caught
|
|
747
|
+
// Retry only this request. Admission concurrency is fixed and is never
|
|
748
|
+
// reduced by rate limits. Respect Retry-After when present; otherwise
|
|
749
|
+
// use the ordinary jittered backoff. Output/tool stamps above remain a
|
|
750
|
+
// hard replay boundary.
|
|
610
751
|
const ra = retryAfterMsFromError(caught)
|
|
611
|
-
if (
|
|
612
|
-
|
|
613
|
-
|
|
752
|
+
if (attempt === maxAttempts - 1) throw caught
|
|
753
|
+
if (ra != null) {
|
|
754
|
+
nextDelayMs = Math.max(0, ra)
|
|
755
|
+
nextDelayReason = 'retry-after'
|
|
756
|
+
}
|
|
614
757
|
continue
|
|
615
758
|
}
|
|
616
759
|
if (kind !== 'transient') throw caught
|
|
@@ -618,7 +761,7 @@ export async function withRetry(fn, opts = {}) {
|
|
|
618
761
|
if (attempt === maxAttempts - 1) throw caught
|
|
619
762
|
const retryAfterMs = retryAfterMsFromError(caught)
|
|
620
763
|
if (retryAfterMs != null) {
|
|
621
|
-
nextDelayMs = Math.max(0,
|
|
764
|
+
nextDelayMs = Math.max(0, retryAfterMs)
|
|
622
765
|
nextDelayReason = 'retry-after'
|
|
623
766
|
}
|
|
624
767
|
} finally {
|
|
@@ -628,28 +771,3 @@ export async function withRetry(fn, opts = {}) {
|
|
|
628
771
|
// Defensive — loop above always returns or throws.
|
|
629
772
|
throw lastErr || new Error('withRetry: exhausted with no error captured')
|
|
630
773
|
}
|
|
631
|
-
|
|
632
|
-
function _sleepWithAbort(ms, signal) {
|
|
633
|
-
return new Promise((resolve, reject) => {
|
|
634
|
-
let onAbort = null
|
|
635
|
-
const t = setTimeout(() => {
|
|
636
|
-
if (signal && onAbort) {
|
|
637
|
-
try { signal.removeEventListener('abort', onAbort) } catch {}
|
|
638
|
-
}
|
|
639
|
-
resolve()
|
|
640
|
-
}, ms)
|
|
641
|
-
if (!signal) return
|
|
642
|
-
if (signal.aborted) {
|
|
643
|
-
clearTimeout(t)
|
|
644
|
-
const reason = signal.reason
|
|
645
|
-
reject(reason instanceof Error ? reason : new Error('sleep aborted'))
|
|
646
|
-
return
|
|
647
|
-
}
|
|
648
|
-
onAbort = () => {
|
|
649
|
-
clearTimeout(t)
|
|
650
|
-
const reason = signal.reason
|
|
651
|
-
reject(reason instanceof Error ? reason : new Error('sleep aborted'))
|
|
652
|
-
}
|
|
653
|
-
signal.addEventListener('abort', onAbort, { once: true })
|
|
654
|
-
})
|
|
655
|
-
}
|