@animalabs/membrane 0.5.75 → 0.5.77

package/src/membrane.ts

@@ -31,6 +31,7 @@ import {
   DEFAULT_RETRY_CONFIG,
   MembraneError,
   classifyError,
+  isOverloadedError,
   isTextContent,
   isAbortedResponse,
 } from './types/index.js';
@@ -80,7 +81,11 @@ export class Membrane {
   ) {
     this.adapter = adapter;
     this.registry = config.registry;
-    this.retryConfig = { ...DEFAULT_RETRY_CONFIG, ...config.retry };
+    this.retryConfig = {
+      ...DEFAULT_RETRY_CONFIG,
+      ...config.retry,
+      overloaded: { ...DEFAULT_RETRY_CONFIG.overloaded, ...config.retry?.overloaded },
+    };
     this.config = config;
     // Use provided formatter or default to AnthropicXmlFormatter
     this.formatter = config.formatter ?? new AnthropicXmlFormatter();
@@ -146,12 +151,23 @@ export class Membrane {
         const errorInfo = classifyError(error);
         errorInfo.rawRequest = rawRequest;
 
-        // Rate limits (429) always retry up to 5 attempts regardless of config.
-        // Other retryable errors only retry when maxRetries > 0.
+        // Rate limits (429) always retry up to 5 attempts regardless of
+        // config, and overloaded (529) always retries on its own longer
+        // schedule — both are transient by definition, and the default
+        // maxRetries of 0 would otherwise turn a capacity blip into a dead
+        // turn. Other retryable errors only retry when maxRetries > 0.
+        // overloaded.maxRetries: 0 disables the dedicated policy entirely;
+        // the 529 then follows the base config like any retryable server
+        // error (exactly the pre-policy behavior), rather than being
+        // silently re-promoted to the long schedule by a positive base limit.
         const isRateLimit = errorInfo.type === 'rate_limit';
+        const isOverloaded =
+          isOverloadedError(errorInfo) && this.retryConfig.overloaded.maxRetries > 0;
         const effectiveMax = isRateLimit
           ? Math.max(this.retryConfig.maxRetries, 5)
-          : this.retryConfig.maxRetries;
+          : isOverloaded
+            ? Math.max(this.retryConfig.maxRetries, this.retryConfig.overloaded.maxRetries)
+            : this.retryConfig.maxRetries;
 
         if (errorInfo.retryable && attempts < effectiveMax) {
           // Check hook for retry decision
@@ -163,7 +179,7 @@ export class Membrane {
           }
 
           // Wait before retry (abort-aware)
-          const delay = this.calculateRetryDelay(attempts);
+          const delay = this.calculateRetryDelay(attempts, isOverloaded);
           await this.sleep(delay, options.signal);
           continue;
         }
@@ -217,11 +233,74 @@ export class Membrane {
 
     // Determine tool mode
     const toolMode = this.resolveToolMode(request);
+    const useNative = toolMode === 'native' && !!request.tools && request.tools.length > 0;
+
+    // Overloaded (529) pre-emission retry. The streaming paths have no retry
+    // loop of their own, so a capacity error used to kill the turn outright —
+    // and 529s most often arrive INSTEAD of a stream, before anything reaches
+    // the caller, where retrying is transparent. Once any callback has
+    // delivered output (tokens, blocks, usage), retrying would replay content
+    // the caller already consumed, so mid-stream errors still throw.
+    let attempts = 0;
+    const retryDelaysMs: number[] = [];
+    while (true) {
+      attempts++;
+      let emitted = false;
+      const mark = <A extends unknown[], R>(fn?: (...args: A) => R) =>
+        fn && ((...args: A): R => { emitted = true; return fn(...args); });
+      const tracked: StreamOptions = {
+        ...options,
+        onChunk: mark(options.onChunk),
+        onContentBlockUpdate: mark(options.onContentBlockUpdate),
+        onToolCalls: mark(options.onToolCalls),
+        onPreToolContent: mark(options.onPreToolContent),
+        onUsage: mark(options.onUsage),
+        onBlock: mark(options.onBlock),
+        onResponse: mark(options.onResponse),
+        // onRequest fires before the send — it is not an emission.
+      };
 
-    if (toolMode === 'native' && request.tools && request.tools.length > 0) {
-      return this.streamWithNativeTools(request, options);
-    } else {
-      return this.streamWithXmlTools(request, options);
+      try {
+        const result = useNative
+          ? await this.streamWithNativeTools(request, tracked)
+          : await this.streamWithXmlTools(request, tracked);
+        // The inner paths report attempts: 1 — they can't see this wrapper.
+        // A call that succeeded after N overloaded retries must not look like
+        // a first-attempt success in durable logs, so patch the real count
+        // (and the waits) into the response telemetry.
+        if (attempts > 1 && 'details' in result) {
+          result.details.timing.attempts = attempts;
+          result.details.timing.retryDelaysMs = retryDelaysMs;
+        }
+        return result;
+      } catch (error) {
+        const errorInfo = classifyError(error);
+        // Same semantics as complete(): maxRetries bounds total attempts,
+        // the overloaded floor applies over the base config, and
+        // overloaded.maxRetries: 0 opts out of stream retries entirely
+        // (streaming had no retry before this policy existed).
+        const overloadedEnabled = this.retryConfig.overloaded.maxRetries > 0;
+        const maxOverloaded = Math.max(
+          this.retryConfig.maxRetries,
+          this.retryConfig.overloaded.maxRetries
+        );
+        if (!emitted && overloadedEnabled && isOverloadedError(errorInfo) && attempts < maxOverloaded) {
+          // Honor the same pre-retry hook contract as complete(): hosts use
+          // onError for circuit-breaking, and its 'abort' decision must work
+          // on the streaming path too.
+          if (this.config.hooks?.onError) {
+            const decision = await this.config.hooks.onError(errorInfo, attempts);
+            if (decision === 'abort') {
+              throw error;
+            }
+          }
+          const delay = this.calculateRetryDelay(attempts, true);
+          retryDelaysMs.push(delay);
+          await this.sleep(delay, options.signal);
+          continue;
+        }
+        throw error;
+      }
     }
   }
 
@@ -330,6 +409,52 @@ export class Membrane {
     // from blocks the model itself opened during generation
     const prefillDepths = parser.getDepths();
 
+    // Resumption spin guards (issue #39). Observed live on Ash 2026-07-26:
+    // each automatic resumption re-sent ~172k input tokens, streamed ~6
+    // output tokens, and stopped on the same (dropped) stop sequence — 43
+    // rounds, ~7M input tokens, zero progress, found only because a human
+    // noticed. Two guards, both scoped to AUTOMATIC false-positive
+    // resumptions — tool rounds are real caller-governed work (maxToolDepth
+    // / the yielding API's uncapped contract) and are never counted here:
+    //   - stall guard: several CONSECUTIVE resumptions that each stream
+    //     almost nothing and stop identically end the turn ('no_progress').
+    //     One short repeated round is low progress, not proof of none — a
+    //     stop sequence inside legitimate tool-argument text can cause a
+    //     couple of short resumptions on the way to completing.
+    //   - round cap: a hard bound on resumptions per turn ('round_limit'),
+    //     the backstop for a spin that keeps technically progressing.
+    const MIN_ROUND_PROGRESS_CHARS = 16;
+    const MAX_CONSECUTIVE_STALLED_RESUMPTIONS = 3;
+    const RESUMPTION_WARN_ROUNDS = 5;
+    const maxResumptionRounds = options.maxResumptionRounds ?? 24;
+    let resumptionRounds = 0;
+    let consecutiveStalledResumptions = 0;
+    let enteredViaResumption = false;
+    let prevRoundStopSequence: string | undefined;
+    const warnLog = this.config.logger ?? console;
+
+    /** Count an automatic resumption; emits the visibility warning at the
+     *  threshold and returns false when the cap says the turn should end. */
+    const registerResumptionRound = (): boolean => {
+      resumptionRounds++;
+      if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
+        warnLog.warn(
+          `[membrane] automatic resumption at round ${resumptionRounds} ` +
+          `(${totalUsage.inputTokens} input tokens so far this turn) — ` +
+          `a spin shows up here before it shows up on the bill`
+        );
+      }
+      if (resumptionRounds > maxResumptionRounds) {
+        warnLog.warn(
+          `[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
+          `ending turn with stopReason 'round_limit'. ` +
+          `${totalUsage.inputTokens} input tokens spent this turn.`
+        );
+        return false;
+      }
+      return true;
+    };
+
     try {
       // Tool execution loop
       while (toolDepth <= maxToolDepth) {
@@ -338,7 +463,10 @@ export class Membrane {
         let detectedStopSequence: string | null = null;
         let truncatedAccumulated: string | null = null;
 
-        // Track where to start checking for stop sequences (skip already-processed content)
+        // Track where to start checking for stop sequences (skip already-processed content).
+        // Also the round's progress baseline: XML we pushed ourselves at the
+        // end of the previous round (tool results, closing tags) sits below
+        // this index and doesn't count as model progress.
         const checkFromIndex = parser.getAccumulated().length;
 
         // Stream from provider
@@ -465,6 +593,38 @@ export class Membrane {
         // Get accumulated text from parser
         const accumulated = parser.getAccumulated();
 
+        // Stall accounting (issue #39): only rounds ENTERED via automatic
+        // resumption can stall — tool rounds are caller-governed work and a
+        // real tool call is longer than the threshold anyway. A stall is a
+        // resumption that streamed almost nothing and stopped identically to
+        // the previous round; the turn ends only after several IN A ROW
+        // (one short repeated round is low progress, not proof of none —
+        // a stop sequence inside legitimate tool-argument text can cause a
+        // couple of short resumptions on the way to completing).
+        const streamedThisRound = accumulated.length - checkFromIndex;
+        if (
+          enteredViaResumption &&
+          lastStopReason === 'stop_sequence' &&
+          streamedThisRound < MIN_ROUND_PROGRESS_CHARS &&
+          lastStopSequence === prevRoundStopSequence
+        ) {
+          consecutiveStalledResumptions++;
+          if (consecutiveStalledResumptions >= MAX_CONSECUTIVE_STALLED_RESUMPTIONS) {
+            warnLog.warn(
+              `[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
+              `made no progress (${streamedThisRound} chars this round, stop ` +
+              `${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
+              `stopReason 'no_progress'. ${totalUsage.inputTokens} input tokens spent this turn.`
+            );
+            lastStopReason = 'no_progress';
+            break;
+          }
+        } else {
+          consecutiveStalledResumptions = 0;
+        }
+        prevRoundStopSequence = lastStopSequence;
+        enteredViaResumption = false;
+
         // Check for tool calls (if handler provided)
         if (onToolCalls && streamResult.stopSequence === '</function_calls>') {
           // Append the closing tag (we truncated before it, or API stopped before it)
@@ -476,8 +636,11 @@ export class Membrane {
 
           if (parsed && parsed.calls.length > 0) {
             // Notify about pre-tool content
-            if (onPreToolContent && parsed.beforeText.trim()) {
-              await onPreToolContent(parsed.beforeText);
+            // Slice the seeded prefill off: beforeText starts with the whole
+            // flattened document in XML mode (see ToolContext note below).
+            const preToolNew = parsed.beforeText.slice(initialPrefillLength);
+            if (onPreToolContent && preToolNew.trim()) {
+              await onPreToolContent(preToolNew);
             }
 
             // Emit block events for each tool call
@@ -504,13 +667,24 @@ export class Membrane {
             // Track the tool calls
             executedToolCalls.push(...parsed.calls);
 
-            // Execute tools
+            // Execute tools.
+            // preamble/accumulated must expose the MODEL'S text only. The
+            // parser is seeded with the entire assistant prefill (the whole
+            // flattened document in XML mode), so parsed.beforeText starts
+            // with it — consumers that persist the preamble as "what the
+            // agent said this round" would otherwise write the full document
+            // back into the store as an assistant message (observed on Ash,
+            // 2026-07-26: a died-mid-rounds turn flushed a ~720k-char
+            // document echo into her message store as 62 sharded messages).
+            // The turn-END path already slices (newContent =
+            // fullAccumulated.slice(initialPrefillLength)); the tool-round
+            // path must match it.
             const context: ToolContext = {
               rawText: parsed.fullMatch,
-              preamble: parsed.beforeText,
+              preamble: parsed.beforeText.slice(initialPrefillLength),
               depth: toolDepth,
               previousResults: executedToolResults,
-              accumulated: parser.getAccumulated(),
+              accumulated: parser.getAccumulated().slice(initialPrefillLength),
             };
 
             const results = await onToolCalls(parsed.calls, context);
@@ -641,7 +815,9 @@ export class Membrane {
               );
             }
 
-            // Reset parser state for new streaming iteration
+            // Reset parser state for new streaming iteration. Tool rounds
+            // are the caller's work — they count against maxToolDepth only,
+            // never against the resumption guards (issue #39 review).
             parser.resetForNewIteration();
             toolDepth++;
             continue;
@@ -678,6 +854,11 @@ export class Membrane {
           if (toolDepth > maxToolDepth) {
             break;
           }
+          if (!registerResumptionRound()) {
+            lastStopReason = 'round_limit';
+            break;
+          }
+          enteredViaResumption = true;
           prefillResult.assistantPrefill = parser.getAccumulated();
           providerRequest = this.buildContinuationRequest(
             request,
@@ -1033,6 +1214,13 @@ export class Membrane {
           const textBlock: Record<string, unknown> = { type: 'text', text };
           if ((block as any).cache_control) {
             textBlock.cache_control = (block as any).cache_control;
+            // A block-level passthrough occupies one of the 4 breakpoint slots
+            // exactly like a marked message — count it, so the tools/system
+            // fallback below doesn't stack more on top. (Imported/seeded
+            // conversations can carry stale request-time cache_control on
+            // stored blocks — first seen wedging Sill 2026-07-25: 3 cm markers
+            // + 2 stale Arc-export blocks = 5 → hard 400 on every inference.)
+            messageBreakpoints++;
           }
           content.push(textBlock);
         } else if (block.type === 'tool_use') {
@@ -1872,10 +2060,15 @@ export class Membrane {
     return pricing ? calculateCost(usage, pricing) : undefined;
   }
 
-  private calculateRetryDelay(attempt: number): number {
-    const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } = this.retryConfig;
-    const delay = retryDelayMs * Math.pow(backoffMultiplier, attempt - 1);
-    return Math.min(delay, maxRetryDelayMs);
+  private calculateRetryDelay(attempt: number, overloaded = false): number {
+    const { retryDelayMs, backoffMultiplier, maxRetryDelayMs } = overloaded
+      ? this.retryConfig.overloaded
+      : this.retryConfig;
+    const delay = Math.min(retryDelayMs * Math.pow(backoffMultiplier, attempt - 1), maxRetryDelayMs);
+    // Equal jitter on the overloaded schedule only: a capacity storm is
+    // exactly the case where a fleet retrying in sync re-creates the
+    // stampede it's backing off from. [delay/2, delay) keeps the wait long.
+    return overloaded ? Math.floor(delay / 2 + Math.random() * (delay / 2)) : delay;
   }
 
   private attachRawRequest(error: unknown, rawRequest: unknown): Error {
@@ -2012,6 +2205,29 @@ export class Membrane {
         ? Infinity
         : maxToolDepthOpt;
 
+    // Resumption spin guards (issue #39). This is the path the Ash spin ran
+    // on: tool depth here is unlimited BY DESIGN (the caller budgets its own
+    // tool work — that contract stands untouched), and the false-positive
+    // resumption path counted against that same unlimited bound — 43 rounds
+    // × ~172k input tokens of zero progress. Only AUTOMATIC resumptions are
+    // guarded: a stall guard (consecutive no-progress resumptions →
+    // 'no_progress') and a hard resumption cap ('round_limit'). Tool rounds
+    // are never counted. See streamWithXmlTools for the rationale details.
+    const MIN_ROUND_PROGRESS_CHARS = 16;
+    const MAX_CONSECUTIVE_STALLED_RESUMPTIONS = 3;
+    const RESUMPTION_WARN_ROUNDS = 5;
+    const maxResumptionRounds =
+      options.maxResumptionRounds === undefined
+        ? 24
+        : options.maxResumptionRounds === -1
+          ? Infinity
+          : options.maxResumptionRounds;
+    let resumptionRounds = 0;
+    let consecutiveStalledResumptions = 0;
+    let enteredViaResumption = false;
+    let prevRoundStopSequence: string | undefined;
+    const warnLog = this.config.logger ?? console;
+
     // Initialize parser from formatter for format-specific tracking
     const formatter = this.formatter;
     const parser = formatter.createStreamParser();
@@ -2057,6 +2273,28 @@ export class Membrane {
     // from blocks the model itself opened during generation
     const prefillDepths = parser.getDepths();
 
+    /** Count an automatic resumption; emits the visibility warning at the
+     *  threshold and returns false when the cap says the turn should end. */
+    const registerResumptionRound = (): boolean => {
+      resumptionRounds++;
+      if (resumptionRounds === RESUMPTION_WARN_ROUNDS) {
+        warnLog.warn(
+          `[membrane] automatic resumption at round ${resumptionRounds} ` +
+          `(${totalUsage.inputTokens} input tokens so far this turn) — ` +
+          `a spin shows up here before it shows up on the bill`
+        );
+      }
+      if (resumptionRounds > maxResumptionRounds) {
+        warnLog.warn(
+          `[membrane] automatic resumption cap (${maxResumptionRounds}) reached — ` +
+          `ending turn with stopReason 'round_limit'. ` +
+          `${totalUsage.inputTokens} input tokens spent this turn.`
+        );
+        return false;
+      }
+      return true;
+    };
+
     try {
       // Tool execution loop
       while (toolDepth <= maxToolDepth) {
@@ -2185,6 +2423,34 @@ export class Membrane {
           }
         }
 
+        // Stall accounting (issue #39): only rounds ENTERED via automatic
+        // resumption can stall; the turn ends only after several consecutive
+        // stalls. Tool rounds are never counted. Mirrors streamWithXmlTools —
+        // see the detailed rationale there.
+        const streamedThisRound = parser.getAccumulated().length - checkFromIndex;
+        if (
+          enteredViaResumption &&
+          lastStopReason === 'stop_sequence' &&
+          streamedThisRound < MIN_ROUND_PROGRESS_CHARS &&
+          lastStopSequence === prevRoundStopSequence
+        ) {
+          consecutiveStalledResumptions++;
+          if (consecutiveStalledResumptions >= MAX_CONSECUTIVE_STALLED_RESUMPTIONS) {
+            warnLog.warn(
+              `[membrane] ${consecutiveStalledResumptions} consecutive automatic resumptions ` +
+              `made no progress (${streamedThisRound} chars this round, stop ` +
+              `${JSON.stringify(lastStopSequence ?? null)} repeated) — ending turn with ` +
+              `stopReason 'no_progress'. ${totalUsage.inputTokens} input tokens spent this turn.`
+            );
+            lastStopReason = 'no_progress';
+            break;
+          }
+        } else {
+          consecutiveStalledResumptions = 0;
+        }
+        prevRoundStopSequence = lastStopSequence;
+        enteredViaResumption = false;
+
         // Check for tool calls
         if (streamResult.stopSequence === '</function_calls>') {
           const closeTag = '</function_calls>';
@@ -2225,13 +2491,24 @@ export class Membrane {
             // Track the tool calls
             executedToolCalls.push(...parsed.calls);
 
-            // Build tool context
+            // Build tool context.
+            // preamble/accumulated must expose the MODEL'S text only: the
+            // parser was seeded with the entire assistant prefill (the whole
+            // flattened document in XML mode), so parsed.beforeText starts
+            // with it. Consumers persist the preamble as "what the agent said
+            // this round" — unsliced, a turn that dies mid-rounds flushes the
+            // full document back into the agent's store as its own message
+            // (observed on Ash 2026-07-26: ~720k-char document echo persisted
+            // as 62 sharded assistant messages, doubling her store and
+            // wedging every subsequent compile). The turn-END path already
+            // slices (newContent = fullAccumulated.slice(initialPrefillLength));
+            // the tool-round path must match it.
             const context: ToolContext = {
               rawText: parsed.fullMatch,
-              preamble: parsed.beforeText,
+              preamble: parsed.beforeText.slice(initialPrefillLength),
               depth: toolDepth,
               previousResults: executedToolResults,
-              accumulated: parser.getAccumulated(),
+              accumulated: parser.getAccumulated().slice(initialPrefillLength),
             };
 
             // Yield control for tool execution
@@ -2390,6 +2667,9 @@ export class Membrane {
               );
             }
 
+            // Tool rounds are the caller's work — they count against
+            // maxToolDepth only, never against the resumption guards
+            // (issue #39 review: the uncapped tool-loop contract stands).
             parser.resetForNewIteration();
             toolDepth++;
             continue;
@@ -2421,6 +2701,11 @@ export class Membrane {
           if (toolDepth > maxToolDepth) {
             break;
           }
+          if (!registerResumptionRound()) {
+            lastStopReason = 'round_limit';
+            break;
+          }
+          enteredViaResumption = true;
           prefillResult.assistantPrefill = parser.getAccumulated();
           providerRequest = this.buildContinuationRequest(
             request,