mixdog 1.3.4 → 1.4.0

package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs

@@ -43,16 +43,17 @@ import {
 } from './anthropic-oauth-credentials.mjs';
 import { claudeCliUserAgent, learnRequiredCliVersion, warmCliVersion } from './anthropic-oauth-client-version.mjs';
 import { createPassthroughSignal } from '../stall-policy.mjs';
-import { AnthropicFallbackTriggeredError } from './retry-classifier.mjs';
 import { ANTHROPIC_MAX_MIDSTREAM_RETRIES, parseSSEStream, _classifyMidstreamError } from './anthropic-sse.mjs';
 import { buildAnthropicBetaHeaders, supportsAnthropicFastMode } from './anthropic-betas.mjs';
 import { applyAnthropicServerFallback } from './anthropic-server-fallback.mjs';
 import { fastModeAvailable, fastModeDowngraded, noteFastModeCapacityError } from './anthropic-fast-mode.mjs';
 import { ANTHROPIC_VERSION, anthropicQuotaError, createAnthropicOAuthRequest } from './anthropic-oauth-request.mjs';
 import { isRefreshableAuthRejection } from './anthropic-oauth-request/initial-status.mjs';
+import { postMessages } from './anthropic-oauth-request/gzip-post.mjs';
+import { sendAnthropicCacheWarm } from './anthropic-cache-warm.mjs';
 import { applyAnthropicErrorContract } from './anthropic-error-contract.mjs';
 import { adjustMaxTokensForContextOverflow } from './anthropic-max-tokens-recovery.mjs';
-import { announceAnthropicModelFallback, createAnthropicOverloadTracker } from './anthropic-overload-fallback.mjs';
+import { createAnthropicOverloadTracker, takeAnthropicModelFallback } from './anthropic-overload-fallback.mjs';
 import { createAnthropicOAuthRecovery } from './anthropic-oauth-recovery.mjs';
 import { createMidState, createMidstreamRecovery } from './anthropic-oauth-midstream.mjs';
 import { assertAnthropicStreamNotEmpty } from './anthropic-midstream-recovery.mjs';
@@ -61,10 +62,13 @@ import { ANTHROPIC_EFFORT_LEVELS, withEffortRejectionFallback } from './effort-r
 import { getLlmDispatcher, preconnect } from '../../../shared/llm/http-agent.mjs';
 import {
   anthropicThinkingActive,
+  anthropicTurnScopedReminders,
   appendAnthropicTailReminder,
+  appendAnthropicTurnReminder,
   applyAnthropicCacheMarkers,
   clampAnthropicThinkingBudget as clampThinkingBudgetTokens,
   deferredAnthropicTools as sharedDeferredAnthropicTools,
+  hasAnthropicTurnScopedMessages,
   requestAnthropicTools as sharedRequestAnthropicTools,
   resolveAnthropicCacheTtls as resolveCacheTtls,
   resolveAnthropicMessageCacheSlots,
@@ -72,6 +76,7 @@ import {
   sharedCachePrefixBoundary,
   toAnthropicMessages,
   toAnthropicToolChoice,
+  withAnthropicTurnReminders,
 } from './lib/anthropic-request-utils.mjs';
 import { splitTailReminder } from './lib/tail-reminder-split.mjs';
 import {
@@ -111,6 +116,7 @@ function buildOAuthBetaHeaders(body, { fastMode = false, toolSearch = false, mod
     effort: shouldIncludeEffortBeta(model, opts),
     serverFallback: body?.fallbacks === 'default',
     thinkingDisplayUpdates: body?.thinking?.display === 'updates',
+    turnScopedSystem: hasAnthropicTurnScopedMessages(body?.messages),
   });
 }
 
@@ -223,12 +229,17 @@ function buildRequestBody(requestMessages, model, tools, sendOpts) {
   // applied to the FINAL sanitized array by invariant, so block drops /
   // inserts / reorders performed by the sanitizer can never move or delete a
   // marked block. NEVER sanitize again after this (see send path).
-  const lowerHistory = (history, projection) =>
-    lowerAnthropicEffortHistory(
+  // Turn-scoped tail reminders (tail-reminder-split.mjs): stored reminders are
+  // re-sent before their assistant turns, the current one ends the request.
+  const turnScoped = anthropicTurnScopedReminders('anthropic-oauth', model);
+  const lowerHistory = (history, projection) => {
+    const lowered = lowerAnthropicEffortHistory(
       history.filter((m) => m.role !== 'system'),
       (segment) => toAnthropicMessages(segment, requestTools),
       projection
     );
+    return turnScoped ? withAnthropicTurnReminders(lowered) : lowered;
+  };
   const anthropicMessages = applyAnthropicCacheMarkers(
     lowerHistory(chatMsgs, effortProjection),
     messageCacheSlots,
@@ -240,7 +251,9 @@ function buildRequestBody(requestMessages, model, tools, sendOpts) {
   const body = {
     model,
     max_tokens: maxTokens,
-    messages: appendAnthropicTailReminder(anthropicMessages, reminder),
+    messages: turnScoped
+      ? appendAnthropicTurnReminder(anthropicMessages, reminder)
+      : appendAnthropicTailReminder(anthropicMessages, reminder),
     stream: true,
   };
   applyAnthropicServerFallback(body, model, {
@@ -290,6 +303,28 @@ function buildRequestBody(requestMessages, model, tools, sendOpts) {
   return markAnthropicEffortBody(body, effortProjection);
 }
 
+// Known tool names for the leaked-tool-call guard in parseSSEStream:
+// recovered leaked calls are only synthesized when they name a tool
+// actually offered to this request (native + lowered). Derived from the
+// final request body so it matches exactly what the model was given.
+function requestToolNames(body) {
+  return new Set(
+    (Array.isArray(body.tools) ? body.tools : [])
+      .map((t) => (t && typeof t.name === 'string' ? t.name : null))
+      .filter(Boolean)
+  );
+}
+
+// The number of mid-stream retries a streamed turn took, as a hidden field of
+// the result.
+function tagMidstreamRetries(result, attemptIndex) {
+  try {
+    Object.defineProperty(result, '__midstreamRetries', { value: attemptIndex, enumerable: false });
+  } catch {
+    /* ignore non-extensible result */
+  }
+}
+
 export function _buildRequestBodyForCacheSmoke(messages, model, tools = [], sendOpts = {}) {
   return buildRequestBody(messages, model, tools, sendOpts);
 }
@@ -454,6 +489,41 @@ export class AnthropicOAuthProvider {
     );
   }
 
+  /**
+   * Cache-warm mode (anthropic-cache-warm.mjs): the body `send` would build
+   * for these arguments, POSTed once — no retry, credential replay, fallback
+   * model or re-send — and cut at the first content block. `onUsage` receives
+   * the reported usage as it arrives; resolves the final usage.
+   */
+  async sendCacheWarm(messages, model, tools, sendOpts = {}, { onUsage, accountId = '' } = {}) {
+    await warmCliVersion();
+    const opts = sendOpts || {};
+    const creds = await this.ensureAuth();
+    const body = buildRequestBody(sanitizeToolPairs(messages), model, tools, opts);
+    body.messages = await prepareAnthropicImages(body.messages, { signal: opts.signal });
+    return sendAnthropicCacheWarm({
+      providerName: this.name,
+      instance: this,
+      accountId,
+      model,
+      opts,
+      onUsage,
+      post: (signal) =>
+        postMessages({
+          accessToken: creds.accessToken,
+          requestBody: body,
+          betaHeaders: buildOAuthBetaHeaders(body, {
+            fastMode: this.fastModeBetaHeaderLatched,
+            toolSearch: Array.isArray(body.tools) && body.tools.some((t) => t?.defer_loading === true),
+            model,
+            opts,
+          }),
+          signal,
+          plain: true,
+        }),
+    });
+  }
+
   async _sendOnce(messages, model, tools, sendOpts) {
     await warmCliVersion();
     // Re-warm the kept-alive socket before the turn. preconnect() is a
@@ -493,15 +563,7 @@ export class AnthropicOAuthProvider {
     // actually carries deferred (defer_loading) tools — gate the header on
     // that instead of sending it unconditionally on every request.
     const hasDeferredTools = Array.isArray(body.tools) && body.tools.some((t) => t && t.defer_loading === true);
-    // Known tool names for the leaked-tool-call guard in parseSSEStream:
-    // recovered leaked calls are only synthesized when they name a tool
-    // actually offered to this request (native + lowered). Derived from the
-    // final request body so it matches exactly what the model was given.
-    const knownToolNames = new Set(
-      (Array.isArray(body.tools) ? body.tools : [])
-        .map((t) => (t && typeof t.name === 'string' ? t.name : null))
-        .filter(Boolean)
-    );
+    const knownToolNames = requestToolNames(body);
     const sessionId = opts.sessionId || null;
     const iteration = Number.isFinite(Number(opts.iteration)) ? Number(opts.iteration) : null;
     // Option A: no absolute wall-clock cap on streaming generation. A stream
@@ -518,51 +580,17 @@ export class AnthropicOAuthProvider {
     // totalSignal is therefore a pure pass-through of externalSignal with no timer.
     const totalTimeout = createPassthroughSignal(externalSignal);
     const totalSignal = totalTimeout.signal;
-    // Request-loop 529s and SSE overload events share one counter; at the
-    // limit the configured fallback model takes over the whole send.
-    const overload = createAnthropicOverloadTracker({
-      model: useModel,
-      fallbackModel: opts._fallbackTriggered ? undefined : opts.fallbackModel,
-    });
-    const { requestWithRetry, cleanupCancelHandler } = createAnthropicOAuthRequest({
-      provider: this,
+    const { requestWithRetry, cleanupCancelHandler, midstream } = this._createSendRecovery({
       opts,
       body,
       useModel,
       sessionId,
-      totalSignal,
-      overload,
-      betaHeadersFor: (requestBody) =>
-        buildOAuthBetaHeaders(requestBody, {
-          fastMode: this.fastModeBetaHeaderLatched,
-          toolSearch: hasDeferredTools,
-          model: useModel,
-          opts,
-        }),
-      onStageChange,
-    });
-    const recovery = createAnthropicOAuthRecovery({
-      provider: this,
-      opts,
-      body,
-      useModel,
       auth,
       totalSignal,
-      requestWithRetry,
-      cleanupCancelHandler,
+      hasDeferredTools,
       onStageChange,
       onTextReset,
     });
-    // Bounded mid-stream retries for transient stream loss; jittered backoff
-    // between attempts (anthropic-oauth-midstream.mjs).
-    const midstream = createMidstreamRecovery({
-      maxRetries: ANTHROPIC_MAX_MIDSTREAM_RETRIES,
-      totalSignal,
-      recovery,
-      onStageChange,
-      retry529: opts.retry529 !== false,
-      overload,
-    });
 
     let maxTokensAdjusted = false;
     try {
@@ -593,7 +621,9 @@ export class AnthropicOAuthProvider {
         if (SSE_VERBOSE) process.stderr.write(`[anthropic-oauth] Response ${response.status}, parsing SSE...\n`);
         try {
           onStageChange?.('streaming');
-        } catch {}
+        } catch {
+          // the stage observer is advisory
+        }
 
         const midState = createMidState(attemptIndex);
         try {
@@ -611,7 +641,9 @@ export class AnthropicOAuthProvider {
           );
           try {
             controller?.abort?.('Anthropic SSE complete');
-          } catch {}
+          } catch {
+            // the stream is complete; the controller may already be aborted
+          }
           this._settleStreamedTurn({
             result,
             midState,
@@ -621,11 +653,7 @@ export class AnthropicOAuthProvider {
             useModel,
             requestKind: opts.requestKind || null,
           });
-          try {
-            Object.defineProperty(result, '__midstreamRetries', { value: attemptIndex, enumerable: false });
-          } catch {
-            /* ignore non-extensible result */
-          }
+          tagMidstreamRetries(result, attemptIndex);
           return result;
         } catch (err) {
           const decision = await midstream.onStreamError({ err, midState, controller, response, attemptIndex });
@@ -637,22 +665,81 @@ export class AnthropicOAuthProvider {
       }
       throw midstream.exhaustedError();
     } catch (err) {
-      // The fallback trigger can surface from the request loop, a mid-stream
-      // overload, or the non-streaming recovery: handle it around the whole
-      // operation, not just the first request.
-      if (!(err instanceof AnthropicFallbackTriggeredError)) throw err;
-      process.stderr.write(`[anthropic-oauth] ${err.message}\n`);
-      announceAnthropicModelFallback(onStageChange, _displayModel(useModel), _displayModel(err.fallbackModel));
-      return await this.send(messages, err.fallbackModel, tools, {
-        ...opts,
-        fallbackModel: undefined,
-        _fallbackTriggered: true,
+      const fallbackOpts = takeAnthropicModelFallback(err, {
+        tag: 'anthropic-oauth',
+        model: useModel,
+        opts,
+        onStageChange,
+        displayModel: _displayModel,
       });
+      return await this.send(messages, err.fallbackModel, tools, fallbackOpts);
     } finally {
       totalTimeout.cleanup();
     }
   }
 
+  // The request and recovery pipeline of one send over its fixed body: the
+  // shared overload counter, the retried request with its per-request beta
+  // headers, the non-streaming recovery and the bounded mid-stream retries.
+  _createSendRecovery({
+    opts,
+    body,
+    useModel,
+    sessionId,
+    auth,
+    totalSignal,
+    hasDeferredTools,
+    onStageChange,
+    onTextReset,
+  }) {
+    // Request-loop 529s and SSE overload events share one counter; at the
+    // limit the configured fallback model takes over the whole send.
+    const overload = createAnthropicOverloadTracker({
+      model: useModel,
+      fallbackModel: opts._fallbackTriggered ? undefined : opts.fallbackModel,
+    });
+    const { requestWithRetry, cleanupCancelHandler } = createAnthropicOAuthRequest({
+      provider: this,
+      opts,
+      body,
+      useModel,
+      sessionId,
+      totalSignal,
+      overload,
+      betaHeadersFor: (requestBody) =>
+        buildOAuthBetaHeaders(requestBody, {
+          fastMode: this.fastModeBetaHeaderLatched,
+          toolSearch: hasDeferredTools,
+          model: useModel,
+          opts,
+        }),
+      onStageChange,
+    });
+    const recovery = createAnthropicOAuthRecovery({
+      provider: this,
+      opts,
+      body,
+      useModel,
+      auth,
+      totalSignal,
+      requestWithRetry,
+      cleanupCancelHandler,
+      onStageChange,
+      onTextReset,
+    });
+    // Bounded mid-stream retries for transient stream loss; jittered backoff
+    // between attempts (anthropic-oauth-midstream.mjs).
+    const midstream = createMidstreamRecovery({
+      maxRetries: ANTHROPIC_MAX_MIDSTREAM_RETRIES,
+      totalSignal,
+      recovery,
+      onStageChange,
+      retry529: opts.retry529 !== false,
+      overload,
+    });
+    return { requestWithRetry, cleanupCancelHandler, midstream };
+  }
+
   // One attempt's initial response: a 401 (or Anthropic's exact revoked-token
   // 403) refreshes the credentials in `auth` and replays once; a context
   // overflow 400 shrinks `body.max_tokens` and replays once per send
@@ -675,7 +762,9 @@ export class AnthropicOAuthProvider {
       // resources before the credential-refresh replay.
       try {
         controller?.abort?.();
-      } catch {}
+      } catch {
+        // the controller may already be aborted before the credential-refresh replay
+      }
       auth.creds = await this.ensureAuth({ forceRefresh: true, reason: String(response.status) });
       ({ response, controller, cancelHandler } = await requestWithRetry(auth.creds.accessToken));
       rejectedAuthBody = null;
@@ -690,7 +779,9 @@ export class AnthropicOAuthProvider {
         cleanupCancelHandler(cancelHandler);
         try {
           controller?.abort?.();
-        } catch {}
+        } catch {
+          // the controller may already be aborted before the max-tokens replay
+        }
         ({ response, controller, cancelHandler } = await requestWithRetry(auth.creds.accessToken));
         rejectedAuthBody = null;
       }

package/src/runtime/agent/orchestrator/providers/anthropic-overload-fallback.mjs

@@ -12,11 +12,15 @@
 import { ANTHROPIC_MAX_CONSECUTIVE_529 } from './retry-backoff.mjs';
 import { AnthropicFallbackTriggeredError } from './retry-classifier.mjs';
 
-export const OVERLOADED_ERROR_CODE = 'overloaded_error';
+const OVERLOADED_ERROR_CODE = 'overloaded_error';
 
 /** Consecutive 529s end the request only for API-key (non-subscriber) first-party Opus. */
 export function isTerminalOverloadModel(model) {
-  return /^claude-opus-/.test(String(model || '').trim().toLowerCase());
+  return /^claude-opus-/.test(
+    String(model || '')
+      .trim()
+      .toLowerCase()
+  );
 }
 
 export function isAnthropicOverloadError(err) {
@@ -56,8 +60,26 @@ export function createAnthropicOverloadTracker({ model, fallbackModel, terminalA
   };
 }
 
+/**
+ * The catch around a whole Anthropic send. The fallback trigger can surface
+ * from the request loop, a mid-stream overload, or the non-streaming
+ * recovery; any other error is rethrown unchanged. For the trigger it logs,
+ * announces the model switch, and returns the options of the single re-send
+ * on `err.fallbackModel` (no second hop).
+ */
+export function takeAnthropicModelFallback(err, { tag, model, opts, onStageChange, displayModel = (id) => id }) {
+  if (!(err instanceof AnthropicFallbackTriggeredError)) throw err;
+  process.stderr.write(`[${tag}] ${err.message}\n`);
+  announceAnthropicModelFallback(onStageChange, displayModel(model), displayModel(err.fallbackModel));
+  return {
+    ...opts,
+    fallbackModel: undefined,
+    _fallbackTriggered: true,
+  };
+}
+
 /** Visible model-switch notice on the existing display-only stage channel. */
-export function announceAnthropicModelFallback(onStageChange, fromLabel, toLabel) {
+function announceAnthropicModelFallback(onStageChange, fromLabel, toLabel) {
   if (typeof onStageChange !== 'function') return;
   try {
     onStageChange('reconnecting', {

package/src/runtime/agent/orchestrator/providers/anthropic-sse.mjs

@@ -97,7 +97,7 @@ function _statusForAnthropicSseError(event, payload) {
   return 0;
 }
 
-function _anthropicSseError(event, labels) {
+export function _anthropicSseError(event, labels) {
   const payload = event?.error && typeof event.error === 'object' ? event.error : event;
   const type = payload?.type || event?.type || 'error';
   const message = payload?.message || 'Anthropic SSE error';
@@ -198,7 +198,9 @@ export async function parseSSEStream(
   const progress = (kind) => {
     try {
       onStreamDelta?.(kind);
-    } catch {}
+    } catch {
+      // the stream-delta observer is advisory
+    }
   };
   const turn = createAnthropicSseTurn({
     state,
@@ -222,12 +224,7 @@ export async function parseSSEStream(
     _captureMidstreamAbort(state, signal.reason);
     return signal.reason instanceof Error ? signal.reason : new Error(`${labels.display} SSE stream aborted`);
   };
-  const onAbort = () => {
-    try {
-      const _c = reader.cancel('SSE aborted');
-      if (_c && typeof _c.catch === 'function') _c.catch(() => {});
-    } catch {}
-  };
+  const onAbort = () => cancelReaderOnAbort(reader);
   let buffer = '';
   let currentEvent = '';
   retainProviderSseStream(streamKey);
@@ -239,24 +236,12 @@ export async function parseSSEStream(
       signal.addEventListener('abort', onAbort, { once: true });
     }
     watchdogs.armFirstMessageTimer();
-    streamLoop: while (true) {
+    while (true) {
       let chunk;
       try {
         chunk = await watchdogs.read();
       } catch (err) {
-        const stopped = watchdogs.readFailure() ?? (signal?.aborted ? abortedError() : null);
-        if (stopped) {
-          // The caller's cancellation reason may be shared: never stamp this
-          // request's usage on it; the usage reported so far goes to this send.
-          if (signal?.aborted && state?.sawMessageStart) {
-            const reported = turn.reportedUsage();
-            noteAbandonedUsage(reported.usage, reported.model);
-          }
-          throw stopped;
-        }
-        // The connection dropped mid-body: carry what this turn completed so
-        // the loop can continue from finished tool calls instead of failing.
-        throw state?.sawMessageStart ? turn.attachTransportPartial(err) : err;
+        throw readFailureError(err, { watchdogs, signal, state, turn, abortedError });
       }
       const { done, value } = chunk;
       if (done) break;
@@ -277,10 +262,7 @@ export async function parseSSEStream(
       const framed = frameProviderSseChunk(region, { currentEvent, streamKey });
       const framedChunk = typeof framed?.then === 'function' ? await framed : framed;
       currentEvent = framedChunk.currentEvent;
-      for (const framedEvent of framedChunk.events) {
-        if (framedEvent.error) continue;
-        if (applyAnthropicSseEvent(framedEvent, { turn, watchdogs, state, progress, labels })) break streamLoop;
-      }
+      if (applyAnthropicSseChunk(framedChunk.events, { turn, watchdogs, state, progress, labels })) break;
     }
     turn.flushLeak();
     if (state?.sawMessageStart && !state?.sawCompleted) throw turn.truncatedError();
@@ -289,20 +271,69 @@ export async function parseSSEStream(
     watchdogs.dispose();
     try {
       releaseProviderSseStream(streamKey);
-    } catch {}
+    } catch {
+      // releasing the stream registration is bookkeeping only
+    }
     if (signal) signal.removeEventListener('abort', onAbort);
     // message_stop deliberately exits before EOF because Anthropic may keep
     // sending pings. Cancel the reader so the successful response body and
     // underlying keep-alive connection are not stranded.
     try {
       await reader.cancel('Anthropic SSE complete');
-    } catch {}
+    } catch {
+      // the reader may already be closed after the stream ended
+    }
+    releaseReaderLock(reader, labels);
+  }
+}
+
+// Caller cancellation of a live stream: cancel the reader without awaiting it.
+function cancelReaderOnAbort(reader) {
+  try {
+    const _c = reader.cancel('SSE aborted');
+    if (_c && typeof _c.catch === 'function') _c.catch(() => {});
+  } catch {
+    // the reader may already be closed or released
+  }
+}
+
+// The error a failed reader.read() ends the stream with: a watchdog stop or the
+// caller's abort as-is, otherwise the transport failure carrying the partial
+// turn once the message started.
+function readFailureError(err, { watchdogs, signal, state, turn, abortedError }) {
+  const stopped = watchdogs.readFailure() ?? (signal?.aborted ? abortedError() : null);
+  if (stopped) {
+    // The caller's cancellation reason may be shared: never stamp this
+    // request's usage on it; the usage reported so far goes to this send.
+    if (signal?.aborted && state?.sawMessageStart) {
+      const reported = turn.reportedUsage();
+      noteAbandonedUsage(reported.usage, reported.model);
+    }
+    return stopped;
+  }
+  // The connection dropped mid-body: carry what this turn completed so
+  // the loop can continue from finished tool calls instead of failing.
+  return state?.sawMessageStart ? turn.attachTransportPartial(err) : err;
+}
+
+// Fold one network chunk's framed records into the turn, in order. Returns true
+// once a terminal record ends the stream; a framing error skips its record.
+function applyAnthropicSseChunk(events, context) {
+  for (const framedEvent of events) {
+    if (framedEvent.error) continue;
+    if (applyAnthropicSseEvent(framedEvent, context)) return true;
+  }
+  return false;
+}
+
+function releaseReaderLock(reader, labels) {
+  try {
+    reader.releaseLock();
+  } catch (err) {
     try {
-      reader.releaseLock();
-    } catch (err) {
-      try {
-        process.stderr.write(`[${labels.tag}] reader releaseLock failed: ${err?.message ?? String(err)}\n`);
-      } catch {}
+      process.stderr.write(`[${labels.tag}] reader releaseLock failed: ${err?.message ?? String(err)}\n`);
+    } catch {
+      // stderr may be closed; the releaseLock failure is only reported
     }
   }
 }