mixdog 1.3.4 → 1.4.0

package/src/runtime/agent/orchestrator/providers/admission-scheduler.mjs

@@ -2,6 +2,13 @@ import { createHash } from 'node:crypto';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { PRIORITY_RANK, normalizePriority, resourceAdmission } from '../../../shared/resource-admission.mjs';
 import { accountProviderSend } from '../../../shared/llm/usage-accounting.mjs';
+import { sanitizeSurrogates } from '../../../shared/llm/well-formed.mjs';
+import { providerAvailable, providerUnavailableError } from './provider-availability.mjs';
+import {
+  finalizeProviderRequestTools,
+  isFinalizedProviderRequestTools,
+  providerNativeToolPrefixCount,
+} from '../runtime-core/provider-request-tools.mjs';
 
 // Normal provider traffic is not concurrency-gated. Independent sessions and
 // accounts start immediately; a finite limit exists only when an operator
@@ -257,7 +264,9 @@ export class ProviderAdmissionScheduler {
         running.parentAbort = () => {
           try {
             controller.abort(item.signal.reason);
-          } catch {}
+          } catch {
+            // the controller may already be aborted
+          }
         };
         item.signal.addEventListener('abort', running.parentAbort, { once: true });
       }
@@ -294,7 +303,9 @@ export class ProviderAdmissionScheduler {
           if (running.parentAbort && running.parentSignal) {
             try {
               running.parentSignal.removeEventListener('abort', running.parentAbort);
-            } catch {}
+            } catch {
+              // the abort listener may already be gone
+            }
           }
           this.running.delete(running);
           lane.active -= 1;
@@ -305,6 +316,32 @@ export class ProviderAdmissionScheduler {
     if (this._canDeleteLane(lane)) this._deleteLane(key, lane);
   }
 
+  /**
+   * Why a request that must not wait (a cache warm) cannot start on lane
+   * `key` now, or null: the lane's rate-limit cooldown, or a lane throttled
+   * after a 429 that would queue it.
+   */
+  immediateBlock(key) {
+    if (this.closedReason) return 'provider scheduler closed';
+    const lane = this.lanes.get(String(key || ''));
+    if (!lane) return null;
+    if (lane.adaptive && lane.cooldownUntil > this.now()) return 'rate-limit cooldown';
+    if (lane.queue.length > 0 || lane.active >= lane.limit) return 'account throttled';
+    return null;
+  }
+
+  /** Record a rate limit a request outside `run` (a cache warm) received on
+   *  lane `key`, into the same cooldown state (and broadcast) as a send's. */
+  recordRateLimit(key, error) {
+    const laneKey = String(key || '');
+    if (!laneKey || this.closedReason) return false;
+    const lane = this.lanes.get(laneKey) || createAdmissionLane(laneKey, this.concurrency);
+    this.lanes.set(laneKey, lane);
+    const recorded = this._recordFailure(laneKey, lane, error);
+    if (this._canDeleteLane(lane)) this._deleteLane(laneKey, lane);
+    return recorded;
+  }
+
   snapshot() {
     const now = this.now();
     return {
@@ -421,7 +458,9 @@ export class ProviderAdmissionScheduler {
     if (item.onAbort && item.signal) {
       try {
         item.signal.removeEventListener('abort', item.onAbort);
-      } catch {}
+      } catch {
+        // the abort listener may already be gone
+      }
       item.onAbort = null;
     }
   }
@@ -446,7 +485,9 @@ export class ProviderAdmissionScheduler {
     for (const running of this.running) {
       try {
         running.controller.abort(this.closedReason);
-      } catch {}
+      } catch {
+        // the controller may already be aborted
+      }
     }
   }
 }
@@ -469,7 +510,7 @@ function digest(value) {
     .digest('hex');
 }
 
-function providerAdmissionKey(providerName, provider) {
+export function providerAdmissionKey(providerName, provider) {
   const name = String(providerName || provider?.name || 'provider');
   const stableAnthropicOAuthPath =
     name.toLowerCase() === 'anthropic-oauth'
@@ -518,7 +559,18 @@ export function wrapProviderAdmission(provider, providerName, scheduler = provid
   if (!provider || typeof provider.send !== 'function' || provider[WRAPPED]) return provider;
   const originalSend = provider.send;
   Object.defineProperty(provider, WRAPPED, { value: true });
-  provider.send = function admittedProviderSend(messages, model, tools, sendOpts) {
+  provider.send = function admittedProviderSend(rawMessages, model, rawTools, sendOpts) {
+    // Every registered provider (and its account-pool / proxy / custom variants)
+    // is wrapped here, so this is the one point all model requests pass before
+    // serialization. A lone surrogate would make the API reject the body on
+    // every send; only the outgoing request is rewritten, never the transcript.
+    const messages = sanitizeSurrogates(rawMessages);
+    let tools = sanitizeSurrogates(rawTools);
+    // A finalized tool list is recognised by identity; its sanitized copy must
+    // stay finalized with the same native prefix (the model proxy reads both).
+    if (tools !== rawTools && isFinalizedProviderRequestTools(rawTools)) {
+      tools = finalizeProviderRequestTools(tools, providerNativeToolPrefixCount(rawTools));
+    }
     const opts = sendOpts || {};
     const signal = opts.signal || null;
     const key = providerAdmissionKey(providerName, this);
@@ -530,12 +582,17 @@ export function wrapProviderAdmission(provider, providerName, scheduler = provid
         scheduler.run(
           key,
           (admissionSignal, admissionMetrics) => {
+            // The Developer option may have been switched off after lookup or
+            // while this request waited: the gate is rechecked right before send.
+            if (!providerAvailable(providerName)) throw providerUnavailableError(providerName);
             // Admission is the common request-clock boundary for WS/SSE/HTTP.
             // Queue wait therefore cannot consume first-byte or agent-watchdog
             // time. Provider-local retry remains the sole retry owner.
             try {
               opts.onStageChange?.('requesting');
-            } catch {}
+            } catch {
+              // the stage observer is advisory
+            }
             const admittedOpts = {
               ...opts,
               signal: admissionSignal,

package/src/runtime/agent/orchestrator/providers/anthropic-api-request.mjs

@@ -18,13 +18,17 @@ import { applyAnthropicEffortToBody, shouldIncludeEffortBeta } from './anthropic
 import { isCustomProviderName, resolveCustomReasoning } from './custom-reasoning.mjs';
 import {
   anthropicThinkingActive,
+  anthropicTurnScopedReminders,
   appendAnthropicTailReminder,
+  appendAnthropicTurnReminder,
   applyAnthropicCacheMarkers,
   clampAnthropicThinkingBudget as clampThinkingBudgetTokens,
+  hasAnthropicTurnScopedMessages,
   resolveAnthropicCacheTtls as resolveCacheTtls,
   resolveAnthropicMessageCacheSlots,
   sharedCachePrefixBoundary,
   toAnthropicToolChoice,
+  withAnthropicTurnReminders,
 } from './lib/anthropic-request-utils.mjs';
 import { splitTailReminder } from './lib/tail-reminder-split.mjs';
 import {
@@ -97,12 +101,17 @@ export async function buildAnthropicApiRequest({
   // are applied to the FINAL sanitized array by invariant, so block
   // drops / inserts / reorders performed by the sanitizer can never move
   // or delete a marked block. NEVER sanitize again after this.
-  const lowerHistory = (history, projection) =>
-    lowerAnthropicEffortHistory(
+  // Turn-scoped tail reminders (tail-reminder-split.mjs): stored reminders are
+  // re-sent before their assistant turns, the current one ends the request.
+  const turnScoped = anthropicTurnScopedReminders(name, useModel, config);
+  const lowerHistory = (history, projection) => {
+    const lowered = lowerAnthropicEffortHistory(
       history.filter((m) => m.role !== 'system'),
       (segment) => toAnthropicMessages(segment, requestTools, name),
       projection
     );
+    return turnScoped ? withAnthropicTurnReminders(lowered) : lowered;
+  };
   const anthropicMessages = applyAnthropicCacheMarkers(
     lowerHistory(chatMsgs, effortProjection),
     messageCacheSlots,
@@ -115,9 +124,12 @@ export async function buildAnthropicApiRequest({
     model: useModel,
     max_tokens: maxTokens,
     system: systemBlocks.length ? systemBlocks : undefined,
-    messages: await prepareAnthropicImages(appendAnthropicTailReminder(anthropicMessages, reminder), {
-      signal: opts.signal,
-    }),
+    messages: await prepareAnthropicImages(
+      turnScoped
+        ? appendAnthropicTurnReminder(anthropicMessages, reminder)
+        : appendAnthropicTailReminder(anthropicMessages, reminder),
+      { signal: opts.signal }
+    ),
   };
   // Server-side fallback and fast mode are decided from Claude model ids:
   // first-party knowledge, never applied to a custom endpoint.
@@ -255,6 +267,7 @@ function requestHeadersFor(
         effort: isCustom ? Boolean(params.output_config?.effort) : shouldIncludeEffortBeta(capabilityModel, opts),
         serverFallback: params.fallbacks === 'default',
         thinkingDisplayUpdates: params.thinking?.display === 'updates',
+        turnScopedSystem: hasAnthropicTurnScopedMessages(params.messages),
       }),
       ...(effortProjection ? [EFFORT_CONFIGURATION_BETA] : []),
     ].filter(Boolean);

package/src/runtime/agent/orchestrator/providers/anthropic-cache-warm.mjs

@@ -0,0 +1,165 @@
+// anthropic-cache-warm.mjs — the cache-warm mode both Anthropic providers
+// expose as `sendCacheWarm` (session/cache-warm.mjs): one POST of the
+// request the provider's own builder produces, no retry, fallback, re-send,
+// wait or account rotation, and the stream cut at the first content block
+// once the usage (cache read / write) has been reported. Right before the
+// POST the account gate (cache-warm-admission.mjs) may skip it; a refusal is
+// fed into the shared rate-limit and roster state. The request is bounded by
+// the providers' own watchdogs: the first-byte timeout until the usage
+// arrives, then the SSE idle timeout between events. Usage is handed to
+// `onUsage` as it arrives, so a warm that fails or is aborted after the
+// provider accepted it is still accounted. Every exit cancels the stream and
+// aborts the transport.
+import { PROVIDER_FIRST_BYTE_TIMEOUT_MS, PROVIDER_SSE_IDLE_TIMEOUT_MS } from '../stall-policy.mjs';
+import { updateTurnUsage } from './anthropic-sse-turn/turn-state.mjs';
+import { _anthropicSseError } from './anthropic-sse.mjs';
+import { anthropicQuotaError } from './anthropic-oauth-request/initial-status.mjs';
+import { cacheWarmAccountBlock, cacheWarmBlockedError, recordCacheWarmRefusal } from './cache-warm-admission.mjs';
+import { scrubOAuthSecrets } from './lib/oauth-token-utils.mjs';
+
+const SSE_LABELS = Object.freeze({ display: 'Anthropic cache warm' });
+
+function usageSnapshot(usage) {
+  const { raw: _raw, ...rest } = usage;
+  return { ...rest };
+}
+
+// An abort deadline: `arm(ms)` (re)starts it; it fires `controller.abort`.
+function watchdog(controller) {
+  let timer = null;
+  return {
+    arm(ms, what) {
+      if (timer) clearTimeout(timer);
+      timer = setTimeout(() => controller.abort(new Error(`cache warm ${what} timed out after ${ms}ms`)), ms);
+      timer.unref?.();
+    },
+    clear() {
+      if (timer) clearTimeout(timer);
+      timer = null;
+    },
+  };
+}
+
+/**
+ * Read a warm's SSE stream until usage is known and the first content block
+ * starts, or the message ends. Resolves the usage; a stream error rejects
+ * with `error.usage` set to what was reported before it. `onChunk` hears
+ * every chunk (the idle watchdog). The stream is cancelled on every exit.
+ */
+export async function readCacheWarmStream(response, { onUsage = () => {}, onChunk = () => {} } = {}) {
+  const usage = { inputTokens: 0, outputTokens: 0, cachedTokens: 0, cacheWriteTokens: 0, raw: null };
+  let reported = false;
+  const report = (raw) => {
+    if (!raw || typeof raw !== 'object') return;
+    updateTurnUsage(usage, raw);
+    reported = true;
+    onUsage(usageSnapshot(usage));
+  };
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = '';
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      let cut = buffer.indexOf('\n\n');
+      while (cut >= 0) {
+        const frame = buffer.slice(0, cut);
+        buffer = buffer.slice(cut + 2);
+        cut = buffer.indexOf('\n\n');
+        const data = frame
+          .split('\n')
+          .filter((line) => line.startsWith('data:'))
+          .map((line) => line.slice(5).trimStart())
+          .join('\n');
+        if (!data) continue;
+        let event;
+        try {
+          event = JSON.parse(data);
+        } catch {
+          continue;
+        }
+        if (event?.type === 'error' || event?.error) throw _anthropicSseError(event, SSE_LABELS);
+        if (event.type === 'message_start') report(event.message?.usage);
+        else if (event.type === 'message_delta') report(event.usage);
+        else if (event.type === 'content_block_start' && reported) {
+          // The prefix was read (and the cache refreshed) before any content:
+          // nothing generated after this point is needed.
+          return usageSnapshot(usage);
+        } else if (event.type === 'message_stop') return usageSnapshot(usage);
+      }
+      onChunk(reported);
+    }
+  } catch (error) {
+    if (error && typeof error === 'object' && error.usage === undefined) {
+      error.usage = reported ? usageSnapshot(usage) : null;
+    }
+    throw error;
+  } finally {
+    await reader.cancel().catch(() => {});
+  }
+  return usageSnapshot(usage);
+}
+
+/**
+ * One warm on account instance `instance` (roster account `accountId`, empty
+ * for a provider without a roster) of `providerName`: the account gate, then
+ * `post(signal)` (the provider's single POST of its built body), then the
+ * stream. A refusal reaches the shared rate-limit and roster state; the
+ * transport is aborted on every exit.
+ */
+export async function sendAnthropicCacheWarm({
+  providerName,
+  instance,
+  accountId = '',
+  model,
+  opts = {},
+  post,
+  onUsage,
+}) {
+  const blocked = cacheWarmAccountBlock(providerName, instance, {
+    accountId,
+    model,
+    sessionId: opts.session?.id || opts.sessionId || '',
+  });
+  if (blocked) throw cacheWarmBlockedError(blocked);
+  const controller = new AbortController();
+  const callerSignal = opts.signal;
+  const onCallerAbort = () => controller.abort(callerSignal.reason);
+  if (callerSignal?.aborted) controller.abort(callerSignal.reason);
+  else callerSignal?.addEventListener?.('abort', onCallerAbort, { once: true });
+  const deadline = watchdog(controller);
+  deadline.arm(PROVIDER_FIRST_BYTE_TIMEOUT_MS, 'first byte');
+  try {
+    const response = await post(controller.signal);
+    if (!response.ok) {
+      const bodyText = await response.text().catch(() => '');
+      throw anthropicQuotaError(response.status, response.headers, bodyText);
+    }
+    return await readCacheWarmStream(response, {
+      onUsage,
+      onChunk: (reported) => {
+        if (reported) deadline.arm(PROVIDER_SSE_IDLE_TIMEOUT_MS, 'stream');
+      },
+    });
+  } catch (error) {
+    // The warm's own error is what the caller needs; a failure to persist the
+    // refusal (quota / roster state) is reported, scrubbed, like the other
+    // credential and account-state write failures of the Anthropic providers.
+    await recordCacheWarmRefusal(providerName, instance, error, {
+      accountId,
+      model,
+      aborted: Boolean(callerSignal?.aborted),
+    }).catch((recordError) => {
+      process.stderr.write(
+        `[anthropic-cache-warm] refusal record failed: ${scrubOAuthSecrets(recordError?.message || String(recordError))}\n`
+      );
+    });
+    throw error;
+  } finally {
+    callerSignal?.removeEventListener?.('abort', onCallerAbort);
+    deadline.clear();
+    controller.abort(new Error('cache warm finished'));
+  }
+}

package/src/runtime/agent/orchestrator/providers/anthropic-oauth-client-version.mjs

@@ -10,7 +10,7 @@ import { createNpmVersionSource, maxSemver } from './npm-cli-version.mjs';
 // source release just to advance the user-agent version.
 // Claude Opus 5.5 requires 2.1.280 or newer. The release workflow raises this
 // to the published CLI version (scripts/sync-client-version-floors.mjs).
-export const DEFAULT_CLI_VERSION = '2.1.309';
+export const DEFAULT_CLI_VERSION = '2.1.311';
 export const CLAUDE_CLI_NPM_PACKAGE = '@anthropic-ai/claude-code';
 
 const CACHE_SCHEMA_VERSION = 1;

package/src/runtime/agent/orchestrator/providers/anthropic-oauth-credentials.mjs

@@ -15,6 +15,7 @@ import { boundProviderAuthPath } from '../../../shared/provider-auth-binding.mjs
 import { resolvePluginData } from '../../../shared/plugin-paths.mjs';
 import { modelProxyClientMode, serverModelCredentialsDisabledError } from '../../../shared/model-proxy-mode.mjs';
 import { getLlmDispatcher } from '../../../shared/llm/http-agent.mjs';
+import { TIMEOUT_PHASE } from '../../../shared/provider-timeout-phase.mjs';
 import { claudeCliUserAgent, resolveCliVersion } from './anthropic-oauth-client-version.mjs';
 import {
   accountIdentityFields,
@@ -375,7 +376,9 @@ async function _refreshOAuthCredentialsUnlocked(creds) {
     return refreshed;
   } catch (err) {
     if (err?.name === 'AbortError') {
-      throw new Error('Anthropic OAuth token refresh timed out after 30000ms');
+      throw Object.assign(new Error('Anthropic OAuth token refresh timed out after 30000ms'), {
+        timeoutPhase: TIMEOUT_PHASE.AUTH,
+      });
     }
     throw err;
   } finally {

package/src/runtime/agent/orchestrator/providers/anthropic-oauth-request/gzip-post.mjs

@@ -12,7 +12,9 @@ import { sendGzipCompressed } from '../lib/request-compression.mjs';
 const API_URL = 'https://api.anthropic.com/v1/messages';
 export { ANTHROPIC_VERSION };
 
-export async function postMessages({ accessToken, requestBody, betaHeaders, signal, compress }) {
+// `plain`: exactly one uncompressed POST (a cache warm), so no compression
+// rejection resend can follow.
+export async function postMessages({ accessToken, requestBody, betaHeaders, signal, compress, plain = false }) {
   const headers = {
     Authorization: `Bearer ${accessToken}`,
     'anthropic-version': ANTHROPIC_VERSION,
@@ -22,17 +24,15 @@ export async function postMessages({ accessToken, requestBody, betaHeaders, sign
     'x-app': 'cli',
     'Content-Type': 'application/json',
   };
-  return sendGzipCompressed({
-    raw: Buffer.from(JSON.stringify(requestBody)),
-    headers,
-    compress,
-    send: (body, sendHeaders) =>
-      fetch(API_URL, {
-        method: 'POST',
-        headers: sendHeaders,
-        body,
-        signal,
-        dispatcher: getLlmDispatcher(),
-      }),
-  });
+  const raw = Buffer.from(JSON.stringify(requestBody));
+  const send = (body, sendHeaders) =>
+    fetch(API_URL, {
+      method: 'POST',
+      headers: sendHeaders,
+      body,
+      signal,
+      dispatcher: getLlmDispatcher(),
+    });
+  if (plain) return send(raw, headers);
+  return sendGzipCompressed({ raw, headers, compress, send });
 }