mixdog 0.9.87 → 0.9.89

package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs

@@ -4,17 +4,17 @@
  * Single source of truth across every provider (openai-oauth-ws, openai-oauth,
  * anthropic-oauth, anthropic, gemini, openai-ws, openai-compat).
  *
- * Goal: when a provider returns a transient server-side condition we should
- * retry; when it returns a deterministic refusal (auth, permission, quota)
- * we should fail fast. Mid-stream WS events (server-supplied error / response
- * .failed messages) historically lost their HTTP status because the message
- * was wrapped without classification — that left "Our servers are currently
- * overloaded" indistinguishable from a permanent failure to the retry layer.
+ * Goal: when a provider returns a TYPED transient server-side condition we
+ * should retry; when it returns a deterministic refusal (auth, permission,
+ * quota) we should fail fast. Evidence is structural only — HTTP status,
+ * Node errno, SDK error type, WS close code, or a typed field on a wire event.
+ * Error MESSAGE TEXT is never parsed into a status/transience/auth verdict:
+ * an untyped failure stays 'unknown' and is surfaced, not retried.
  *
  * Usage:
- *   import { classifyError, populateHttpStatusFromMessage } from './retry-classifier.mjs'
+ *   import { classifyError, typedStatusFrom } from './retry-classifier.mjs'
  *   const kind = classifyError(err)               // 'auth' | 'permanent' | 'transient' | 'unknown'
- *   populateHttpStatusFromMessage(err)            // mutates err.httpStatus if message hints at one
+ *   typedStatusFrom(event.response?.error, event) // structured status, or 0
  */
 
 import {
@@ -24,6 +24,9 @@ import {
   PROVIDER_RETRY_MAX_ATTEMPTS,
   createTimeoutSignal,
 } from '../stall-policy.mjs'
+import { readStreamOutcome } from './lib/stream-outcome.mjs'
+
+export { readStreamOutcome, stampStreamOutcome, isReplaySafe, isReplayUnsafe, canPromoteToSuccess, hasObservedOutput, hasDispatchedToolCalls, STREAM_TRANSPORTS } from './lib/stream-outcome.mjs'
 
 // HTTP statuses considered transient — safe to retry with backoff.
 //   408 — request timeout
@@ -39,47 +42,23 @@ const TRANSIENT_STATUSES = new Set([408, 409])
 const AUTH_STATUSES = new Set([401, 403])
 const PERMANENT_STATUSES = new Set([400, 404, 405, 410, 415, 422])
 
-// Server-message text patterns. Used when a WS / SSE event carries an error
-// payload but no explicit status code — we sniff the text and assign the most
-// likely HTTP equivalent so the retry layer can use the same rules.
-const MESSAGE_PATTERNS = [
-  // OpenAI OAuth/API sometimes surfaces generic backend failures only as
-  // message text plus a request ID, with no HTTP status on the WS event.
-  { regex: /(?:an error occurred while processing your request|please include the request id)/i, status: 503 },
-  // Overload / transient 5xx — server is asking us to back off. The `\b`
-  // anchor is intentionally OMITTED on the trailing side of `overload` so
-  // "overloaded" / "overloading" both match (inflected forms are common in
-  // server error text).
-  { regex: /(?:overload(?:ed|ing)?|temporarily unavailable|try again later|service unavailable|bad gateway|gateway timeout)/i, status: 503 },
-  // Explicit 5xx mention.
-  { regex: /\b(?:5\d\d|http 5\d\d)\b/i, status: 503 },
-  // Rate limit / quota — same retry posture as 429 but treat as permanent
-  // by classifyError because per-call retry won't change the answer (the
-  // window must elapse). Providers that want time-bounded retry should
-  // honor Retry-After on the original response, not loop here.
-  { regex: /(?:rate[_ -]?limit|quota|too many requests|resource exhausted|insufficient_quota|quota_exceeded)/i, status: 429 },
-  // Auth — never retryable from our side.
-  { regex: /\b(?:unauthorized|unauthorised|authentication|not authenticated|token expired|access token|invalid api key)\b/i, status: 401 },
-  { regex: /\b(?:forbidden|permission denied|policy violation)\b/i, status: 403 },
-]
+// Structured status fields a provider error / wire event may carry. A value is
+// accepted only when it is a real numeric HTTP status; string codes
+// ('server_error', 'forbidden', ...) and free text are ignored, so nothing is
+// ever synthesized from a message.
+const TYPED_STATUS_KEYS = ['httpStatus', 'http_status', 'status', 'statusCode', 'status_code', 'code']
 
 /**
- * Inspect `err.message` (or the explicit `msg` argument) and, if the text
- * matches one of the known transient/permanent patterns, set `err.httpStatus`
- * to the corresponding code. No-op when httpStatus is already set.
- *
- * Returns the resolved httpStatus (existing or newly assigned), or 0 when
- * nothing matched.
+ * Read the first TYPED HTTP status carried by any of `sources` (error object,
+ * wire event, event.response.error payload, ...). Returns 0 when none of them
+ * declares one. Never inspects message text.
  */
-export function populateHttpStatusFromMessage(err, msg = null) {
-  if (!err || typeof err !== 'object') return 0
-  if (Number(err.httpStatus) > 0) return Number(err.httpStatus)
-  const text = String(msg ?? err.message ?? '')
-  if (!text) return 0
-  for (const { regex, status } of MESSAGE_PATTERNS) {
-    if (regex.test(text)) {
-      err.httpStatus = status
-      return status
+export function typedStatusFrom(...sources) {
+  for (const source of sources) {
+    if (!source || typeof source !== 'object') continue
+    for (const key of TYPED_STATUS_KEYS) {
+      const n = Number(source[key])
+      if (Number.isFinite(n) && n >= 100 && n <= 599) return Math.floor(n)
     }
   }
   return 0
@@ -98,12 +77,13 @@ export function populateHttpStatusFromMessage(err, msg = null) {
  */
 export function classifyError(err) {
   if (!err) return 'unknown'
-  // Once a streamed tool call has been surfaced to the loop, retrying the same
-  // provider turn can double-execute that tool. Providers mark these stream
-  // failures as unsafe so the shared retry wrapper fails fast.
-  if (err.emittedToolCall === true || err.toolCallEmitted === true
-    || err.partialToolCall === true || err.emittedThinking === true
-    || err.unsafeToRetry === true) return 'permanent'
+  // Canonical stream-outcome contract owns replay safety: exposed
+  // text/reasoning or a dispatched/ambiguous complete tool call makes
+  // re-issuing the same turn unsafe (double side effect / output
+  // concatenation) → 'permanent'. Everything else stays classifiable by the
+  // typed rules below; an unknown/untyped failure ends as 'unknown', never a
+  // blanket retry.
+  if (readStreamOutcome(err).replayUnsafe === true) return 'permanent'
   // Cancellation is a caller decision, never a transport symptom. Anthropic's
   // APIUserAbortError inherits Error without overriding `name`, so recognize
   // only exact SDK constructor/type markers (plus standard AbortError markers)
@@ -112,7 +92,7 @@ export function classifyError(err) {
   if (chain.some(isExplicitUserAbortError)) return 'permanent'
 
   // Current typed HTTP status outranks stale stream/connection annotations.
-  const status = Number(err.httpStatus || err.status || err.response?.status || 0) || populateHttpStatusFromMessage(err)
+  const status = Number(err.httpStatus || err.status || err.response?.status || 0) || 0
   if (AUTH_STATUSES.has(status)) return 'auth'
   if (status === 429) return 'permanent'
   if (PERMANENT_STATUSES.has(status)
@@ -129,14 +109,10 @@ export function classifyError(err) {
   // transient: we can retry the same request and may succeed.
   if (chain.some((item) => TRANSIENT_ERROR_CODES.has(String(item?.code || '')))) return 'transient'
   // The Anthropic SDK uses APIConnectionError for transport failures which
-  // may not carry a Node errno. Native fetch commonly wraps the errno in
-  // cause.code, or exposes only TypeError("fetch failed").
-  const name = String(err.name || '')
-  const message = String(err.message || '')
-  if (name === 'APIConnectionError'
-    || (name === 'TypeError' && /fetch failed|network error/i.test(message))) {
-    return 'transient'
-  }
+  // may not carry a Node errno. Native fetch wraps its errno in cause.code,
+  // which the bounded chain check above already covers — a bare
+  // TypeError('fetch failed') with no errno stays 'unknown'.
+  if (String(err.name || '') === 'APIConnectionError') return 'transient'
 
   return 'unknown'
 }
@@ -261,8 +237,6 @@ function isPermanentQuotaError(err) {
   for (const item of boundedCauseChain(err)) {
     const codes = [item?.code, item?.error?.code]
     if (codes.some((code) => permanentCodes.has(String(code || '').toLowerCase()))) return true
-    const text = `${String(item?.message || '')} ${String(item?.error?.message || '')}`
-    if (/insufficient_quota|quota[_ -]?exceeded|resource exhausted/i.test(text)) return true
   }
   return false
 }
@@ -389,12 +363,13 @@ export function classifyMidstreamError(err, signals, policy = {}) {
 
 // WebSocket classification consumes the provider's stream-state signals.
 function _classifyMidstreamWs(err, state, attemptIndex, policy) {
-  if (state.sawCompleted) return null
-  // Once a tool call has been dispatched, no transport outcome is replay-safe.
-  // This includes a nominal close-1000 before response.completed: the tool may
-  // already be executing, so retry/fallback could duplicate its side effect.
-  if (state.emittedToolCall) return null
-  if (state.emittedText || err?.liveTextEmitted) return null
+  // Terminal + replay gate. A mid-stream retry re-issues the turn, so it is
+  // denied once visible output was relayed or a tool call was dispatched
+  // (including a nominal close-1000 before response.completed: the tool may be
+  // executing). Everything below is the typed transient classification.
+  const outcome = readStreamOutcome(err, state)
+  if (outcome.terminalObserved) return null
+  if (outcome.replaySafe !== true) return null
   if (err?.wsFrameTooLarge || state.wsFrameTooLarge) {
     return _allowMidstream('ws_frame_too_large', attemptIndex, policy)
   }
@@ -418,8 +393,7 @@ function _classifyMidstreamWs(err, state, attemptIndex, policy) {
     if (name === 'StreamStalledAbortError' || name === 'StreamStalledError'
       || err?.code === 'ESTREAMSTALL' || err?.streamStalled === true
       || state.watchdogAbort === 'StreamStalledAbortError') {
-      // A stall AFTER a tool emit is unsafe to replay (double side-effect).
-      if (err?.unsafeToRetry === true) return null
+      // A stall after any exposure was already denied by the outcome gate.
       return _allowMidstream('stream_stalled', attemptIndex, policy)
     }
     if (closeCode === 4000) return _allowMidstream('ws_4000', attemptIndex, policy)
@@ -444,9 +418,6 @@ function _classifyMidstreamWs(err, state, attemptIndex, policy) {
   const name = err?.name || ''
   if (name === 'AgentStallAbortError') return _allowMidstream('agent_stall', attemptIndex, policy)
   if (name === 'StreamStalledAbortError' || name === 'StreamStalledError' || err?.code === 'ESTREAMSTALL' || err?.streamStalled === true) {
-    // A stall that fired AFTER a tool call was emitted is unsafe to replay
-    // (double side-effect); surface it as terminal so the turn is not retried.
-    if (err?.unsafeToRetry === true) return null
     return _allowMidstream('stream_stalled', attemptIndex, policy)
   }
 
@@ -463,23 +434,42 @@ function _classifyMidstreamWs(err, state, attemptIndex, policy) {
 
   const failed = err?.responseFailed || state.responseFailedPayload
   if (failed) {
-    try {
-      const blob = JSON.stringify(failed).toLowerCase()
-      if (blob.includes('stream_disconnected')) return _allowMidstream('response_failed_disconnected', attemptIndex, policy)
-      if (blob.includes('network_error')) return _allowMidstream('response_failed_network', attemptIndex, policy)
-      if (blob.includes('auth context expired')) return _allowMidstream('response_failed_auth_expired', attemptIndex, policy)
-    } catch {}
+    // STRUCTURED evidence only: the failure's own numeric status and its
+    // explicit error code/type field. The payload is never stringified and
+    // searched — a message/body that merely CONTAINS "network_error",
+    // "stream_disconnected" or "auth context expired" is not a typed retry
+    // signal and leaves the turn terminal.
+    const detail = failed?.response?.error || failed?.error || failed
+    const failedStatus = typedStatusFrom(detail, failed?.response, failed)
+    if (failedStatus >= 500 && failedStatus < 600) {
+      return _allowMidstream(`http_${failedStatus}`, attemptIndex, policy)
+    }
+    for (const field of [detail?.code, detail?.type]) {
+      const key = typeof field === 'string' ? field.trim().toLowerCase() : ''
+      const classifier = RESPONSE_FAILED_CODE_CLASSIFIERS.get(key)
+      if (classifier) return _allowMidstream(classifier, attemptIndex, policy)
+    }
   }
 
   return null
 }
 
+// Explicit `response.failed` error codes/types that describe a transport-level
+// interruption (Codex retries these); every other code is terminal.
+const RESPONSE_FAILED_CODE_CLASSIFIERS = new Map([
+  ['stream_disconnected', 'response_failed_disconnected'],
+  ['network_error', 'response_failed_network'],
+  ['auth_context_expired', 'response_failed_auth_expired'],
+  ['auth_expired', 'response_failed_auth_expired'],
+])
+
 // SSE classification consumes the provider's stream-state signals.
 function _classifyMidstreamSse(err, state, attemptIndex, policy) {
   if (attemptIndex >= policy.defaultRetries) return null
-  if (state.sawCompleted) return null
+  const outcome = readStreamOutcome(err, state)
+  if (outcome.terminalObserved) return null
   if (state.userAbort) return null
-  if (state.emittedText || state.emittedToolCall || state.partialToolCall || state.emittedThinking) return null
+  if (outcome.replaySafe !== true) return null
 
   if (!err) return null
   const status = Number(err?.httpStatus || err?.status || err?.response?.status || 0)
@@ -490,9 +480,6 @@ function _classifyMidstreamSse(err, state, attemptIndex, policy) {
   const name = err?.name || ''
   if (name === 'AgentStallAbortError') return 'agent_stall'
   if (name === 'StreamStalledAbortError' || name === 'StreamStalledError' || err?.code === 'ESTREAMSTALL' || err?.streamStalled === true) {
-    // A stall AFTER a tool emit is unsafe to replay (double side-effect) →
-    // terminal (null), no mid-stream retry. Otherwise route to stream_stalled.
-    if (err?.unsafeToRetry === true) return null
     return 'stream_stalled'
   }
   if (state.watchdogAbort === 'AgentStallAbortError') return 'agent_stall'
@@ -503,10 +490,6 @@ function _classifyMidstreamSse(err, state, attemptIndex, policy) {
   if (code === 'ETIMEDOUT' || code === 'ESOCKETTIMEDOUT') return 'timeout'
   if (code === 'ENOTFOUND' || code === 'EAI_AGAIN' || code === 'EAI_NODATA') return 'dns'
 
-  const msg = String(err?.message || '').toLowerCase()
-  if (msg.includes('stream timed out after') && msg.includes('of inactivity')) return 'sse_idle_timeout'
-  if (msg.includes('body stream') && msg.includes('terminated')) return 'stream_terminated'
-  if (msg.includes('fetch failed')) return 'fetch_failed'
   if (classifyError(err) === 'transient') return 'connection'
 
   return null
@@ -530,23 +513,25 @@ const TRANSPORT_FALLBACK_ERRNO = new Set([
 export function shouldFallbackTransport(err, { signal, enabled = true } = {}) {
   if (!enabled) return false
   if (signal?.aborted) return false
-  if (err?.liveTextEmitted === true) return false
-  if (err?.emittedToolCall === true || err?.unsafeToRetry === true) return false
+  // Transport fallback re-issues the request on another transport: it is a
+  // replay, so the exposure deny applies. Eligibility itself stays typed
+  // (status / errno / classifier), matching Codex's WS→HTTPS switch.
+  if (readStreamOutcome(err).replaySafe !== true) return false
   const status = Number(err?.httpStatus || err?.status || 0)
-  // 401 is auth recovery, never transport fallback. Codex treats other
-  // unexpected handshake statuses as retryable transport errors; 426 is the
-  // explicit immediate WS→HTTPS switch.
+  // 401 is auth recovery, never transport fallback. 426 is the explicit
+  // immediate WS→HTTPS switch; every other status must be TYPED transient
+  // (408/409/5xx) — an arbitrary nonzero status is not fallback evidence.
   if (status === 401) return false
   if (status === 426) return true
-  if (status > 0) return true
+  if (TRANSIENT_STATUSES.has(status) || (status >= 500 && status < 600)) return true
+  if (status > 0) return false
   const code = String(err?.code || '')
   if (TRANSPORT_FALLBACK_ERRNO.has(code)) return true
   const classifier = String(err?.retryClassifier || err?.midstreamClassifier || '')
   if (TRANSPORT_FALLBACK_CLASSIFIERS.has(classifier)) return true
   if (/^http_5\d\d$/.test(classifier)) return true
   if (err?.firstByteTimeout) return true
-  const msg = String(err?.message || '')
-  return /opening handshake has timed out|socket hang up|acquire timed out|no first server event|no meaningful output/i.test(msg)
+  return false
 }
 
 // C) Stream-safety stamp latches. Mirrors openai-oauth-ws's _stampLiveText /
@@ -645,12 +630,17 @@ function _sleepChunkWithAbort(ms, signal, sleepFn, abortMessage) {
 export function classifyHandshakeError(err, { retry429 = true } = {}) {
   if (!err) return null
   const code = err.code || ''
-  const msg = String(err.message || '')
   const status = Number(err.httpStatus || 0)
 
   if (status === 401 || status === 426 || (status === 429 && !retry429)) return null
   if (status > 0) {
-    return `http_${status}`
+    // Typed transient handshake statuses only. A 403/404/4xx upgrade refusal
+    // is a deterministic decision: spending the retry budget on it cannot
+    // change the answer.
+    if (status === 429 || TRANSIENT_STATUSES.has(status) || (status >= 500 && status < 600)) {
+      return `http_${status}`
+    }
+    return null
   }
 
   if (code === 'ECONNRESET') return 'reset'
@@ -660,9 +650,6 @@ export function classifyHandshakeError(err, { retry429 = true } = {}) {
   if (code === 'EWSACQUIRETIMEOUT') return 'acquire_timeout'
   if (code === 'ENETUNREACH' || code === 'EHOSTUNREACH' || code === 'EPIPE') return 'network'
 
-  if (/opening handshake has timed out/i.test(msg)) return 'timeout'
-  if (/socket hang up/i.test(msg)) return 'reset'
-
   return null
 }
 
@@ -677,9 +664,8 @@ export function classifyHandshakeError(err, { retry429 = true } = {}) {
  *     the provider's own responsibility via its native abort plumbing.
  *   - Uses classifyError() to decide retry. 'transient' → retry,
  *     'auth' / 'permanent' / 'unknown' → throw immediately.
- *   - populateHttpStatusFromMessage(err) is called on every caught error so
- *     server-text errors (e.g. "Our servers are currently overloaded")
- *     resolve to httpStatus before classification.
+ *   - Classification is typed-only: an error with no status/errno/SDK type is
+ *     'unknown' and is surfaced immediately instead of being replayed.
  *
  * Returns whatever `fn()` resolves to. Throws the last error if every retry
  * is exhausted, or the first error if it's classified non-transient.
@@ -736,16 +722,13 @@ export async function withRetry(fn, opts = {}) {
         throw reason instanceof Error ? reason : new Error('withRetry: aborted')
       }
       lastErr = caught
-      populateHttpStatusFromMessage(caught)
       const status = Number(caught?.httpStatus || caught?.status || caught?.response?.status || 0)
       const kind = classifyError(caught)
-      const outputWasExposed = caught?.liveTextEmitted === true
-        || caught?.emittedText === true
-        || caught?.emittedToolCall === true
-        || caught?.toolCallEmitted === true
-        || caught?.partialToolCall === true
-        || caught?.emittedThinking === true
-      if (outputWasExposed || caught?.unsafeToRetry === true) throw caught
+      // Hard replay boundary: a retry RE-ISSUES the request, so it is denied
+      // once visible output was relayed or a tool call was dispatched. Whether
+      // an eligible failure is actually retried remains the typed question
+      // resolved by classifyError()/status below.
+      if (readStreamOutcome(caught).replaySafe !== true) throw caught
       // Claude Code treats x-should-retry:false as an explicit server veto
       // (except an internal-only 5xx override that Mixdog does not have).
       // Keep this ahead of status defaults, including the request-local 429 path.