@animalabs/membrane 0.5.79 → 0.5.80

package/src/membrane.ts

@@ -1005,7 +1005,7 @@ export class Membrane {
       // Tool execution loop
       while (toolDepth <= maxToolDepth) {
         // Build provider request with native tools
-        const providerRequest = this.buildNativeToolRequest(request, messages);
+        const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
 
         // Stream from provider
         let textAccumulated = '';
@@ -1197,12 +1197,20 @@ export class Membrane {
     }
   }
 
+  /** See the floating-cache-marker block in buildNativeToolRequest. */
+  private floatBudgetWarned = false;
+
   /**
-   * Build a provider request with native tool support
+   * Build a provider request with native tool support.
+   *
+   * `toolLoopRebuild` is true when this build is a tool-loop continuation
+   * (toolDepth > 0) rather than the turn's first request — the only case
+   * where the floating cache marker applies.
    */
   private buildNativeToolRequest(
     request: NormalizedRequest,
-    messages: typeof request.messages
+    messages: typeof request.messages,
+    toolLoopRebuild = false
   ): any {
     // Provider-native formatters own their complete input-item shape. The
     // legacy implementation below is intentionally Anthropic-specific; using
@@ -1354,7 +1362,17 @@ export class Membrane {
     // tool_results to `messages`. Any unmatched tool_use that reaches
     // this splice is upstream stranding (the bug class this fix exists
     // to catch) — `[pending]` is exactly the right synthesis.
-    const normalized = normalizeToolPairs(providerMessages);
+    // A synthesized [pending] tool_result's bytes are rewritten when the
+    // real result lands — the floating-marker block below must not cache
+    // past one. `synthetic_pending_result` (not the downstream
+    // cache_suppressed_for_synthetic, which only fires when a marker was
+    // actually stripped) is the root condition.
+    let pendingResultSynthesized = false;
+    const normalized = normalizeToolPairs(providerMessages, {
+      onEvent: (e) => {
+        if (e.kind === 'synthetic_pending_result') pendingResultSynthesized = true;
+      },
+    });
     const mergedMessages = mergeConsecutiveRoles(normalized.messages);
 
     // Convert tools to provider format.
@@ -1387,6 +1405,90 @@ export class Membrane {
       );
     }
 
+    // ------------------------------------------------------------------
+    // Floating cache marker: incremental prompt caching inside the native
+    // tool loop. Message breakpoints are placed by the context strategy at
+    // compile time — once per turn — but this builder re-runs on every
+    // tool round with that round's messages appended, so the deepest
+    // upstream marker stays glued to the turn-start snapshot and each
+    // rebuild re-pays the entire appended suffix at full input price
+    // (qa-ops incident, 2026-08-20: two subagents re-sent a suffix growing
+    // to ~118k tokens ~30 times each — ~5.3M uncached tokens in 18 min —
+    // with their one marker sitting on message 2 of 61).
+    //
+    // The tool loop only ever appends, so a marker riding the newest
+    // message yields the intended incremental pattern: each round writes
+    // its delta and cache-reads everything before it.
+    //
+    // Authority contract: the float spends only the RESIDUAL breakpoint
+    // budget (Anthropic allows 4 cache_control including tools/system).
+    // Upstream markers are never displaced or stripped — if they fill all
+    // 4 slots the float is withheld (with a warning) and behavior is
+    // exactly pre-float. With 2+ slots free, the previous round's
+    // endpoint is marked too: a wide parallel-tool round can append more
+    // blocks than the provider's ~20-block backward search covers, which
+    // would orphan the previous round's cache entry behind an unmarked
+    // boundary.
+    //
+    // Skipped when the normalizer synthesized a [pending] tool_result:
+    // those bytes are rewritten when the real result lands, and caching
+    // past them poisons the prefix — the same rationale as the
+    // normalizer's phase 5.5 cache suppression.
+    // ------------------------------------------------------------------
+    const floatingEnabled =
+      request.floatingCacheMarker ?? this.config.defaultFloatingCacheMarker ?? true;
+    if (toolLoopRebuild && floatingEnabled && cacheControl && !pendingResultSynthesized) {
+      // Residuum from a RECOUNT of the constructed wire artifacts, not the
+      // running messageBreakpoints tally — the tally diverges from the wire
+      // in both directions (mirrors NativeFormatter's recount, same bug
+      // class as the Sill 2026-07-25 wedge): a message-level breakpoint
+      // landing on a block already carrying stale cache_control is one
+      // physical marker counted twice, and a pre-marked system block is a
+      // real wire marker the tally never sees. Counted post-fallback and
+      // post-normalize, so fallback spend and phase-5.5 suppression are
+      // both reflected.
+      let wireMarkers = 0;
+      for (const m of mergedMessages) {
+        if (!Array.isArray(m.content)) continue;
+        for (const b of m.content as Array<Record<string, unknown>>) {
+          if (b.cache_control) wireMarkers++;
+        }
+      }
+      if (tools) for (const t of tools) { if (t.cache_control) wireMarkers++; }
+      if (Array.isArray(system)) {
+        for (const b of system as Array<Record<string, unknown>>) {
+          if (b.cache_control) wireMarkers++;
+        }
+      }
+      let residuum = 4 - wireMarkers;
+      if (residuum <= 0) {
+        if (!this.floatBudgetWarned) {
+          this.floatBudgetWarned = true;
+          console.warn(
+            `[membrane] floating cache marker withheld: upstream markers already ` +
+            `occupy all 4 cache_control slots (${wireMarkers} on the wire). ` +
+            `Tool-round suffixes will not cache incrementally.`
+          );
+        }
+      } else {
+        // Newest message first; then the previous round's endpoint (two
+        // wire messages back: [..., prevResults, assistant, results]).
+        const targets = [mergedMessages.length - 1, mergedMessages.length - 3];
+        for (const mi of targets) {
+          if (residuum <= 0 || mi < 0) continue;
+          const content = mergedMessages[mi]?.content;
+          if (!Array.isArray(content) || content.length === 0) continue;
+          const bpIdx = lastCacheableBlockIndex(content as Array<Record<string, unknown>>);
+          if (bpIdx < 0) continue;
+          // Already a breakpoint here (e.g. the strategy's own end marker
+          // on the turn's first rebuild) — nothing to add.
+          if ((content[bpIdx] as Record<string, unknown>).cache_control) continue;
+          (content[bpIdx] as Record<string, unknown>).cache_control = cacheControl;
+          residuum--;
+        }
+      }
+    }
+
     // Build thinking config for native extended thinking (budget clamped to max_tokens)
     // Fable/Mythos models: thinking is always on and unconfigurable; sampling params are removed.
     // Sending thinking config or temperature returns a 400 — omit both entirely.
@@ -2913,7 +3015,7 @@ export class Membrane {
         }
 
         // Build provider request with native tools
-        const providerRequest = this.buildNativeToolRequest(request, messages);
+        const providerRequest = this.buildNativeToolRequest(request, messages, toolDepth > 0);
 
         // Stream from provider
         let textAccumulated = '';

package/src/providers/anthropic.ts

@@ -22,6 +22,7 @@ import {
   abortError,
 } from '../types/index.js';
 import { flattenRootSchemaUnion } from './anthropic-tool-schema.js';
+import { CacheKeepalive, type CacheKeepaliveConfig } from '../cache-keepalive.js';
 
 // ============================================================================
 // Model capability gates
@@ -146,6 +147,15 @@ export interface AnthropicAdapterConfig {
   
   /** Default max tokens */
   defaultMaxTokens?: number;
+
+  /**
+   * Prompt-cache keepalive: hold this agent's cached prefix warm across idle
+   * gaps by replaying the last request with `max_tokens: 0`, which refreshes
+   * the entry's TTL at cache-READ price instead of letting it expire into a
+   * 2x cache write on the next wake. See `../cache-keepalive.ts`.
+   * Pass `{ enabled: false }` to turn off.
+   */
+  cacheKeepalive?: CacheKeepaliveConfig;
 }
 
 // ============================================================================
@@ -161,6 +171,8 @@ export class AnthropicAdapter implements ProviderAdapter {
    *  the SDK rather than merging, so when we add a per-request beta we must
    *  re-carry this one alongside it or auth breaks. */
   private defaultBeta: string | undefined;
+  /** Holds idle agents' cached prefixes warm; undefined when disabled. */
+  readonly cacheKeepalive: CacheKeepalive | undefined;
 
   constructor(config: AnthropicAdapterConfig = {}) {
     const clientOptions: ClientOptions = {
@@ -178,6 +190,19 @@ export class AnthropicAdapter implements ProviderAdapter {
 
     this.client = new Anthropic(clientOptions);
     this.defaultMaxTokens = config.defaultMaxTokens ?? 4096;
+
+    this.cacheKeepalive = config.cacheKeepalive?.enabled === false
+      ? undefined
+      : new CacheKeepalive(
+          // Replay path. Deliberately bypasses buildRequest(): the payload is
+          // the already-built wire request from a real call, and rebuilding it
+          // risks a byte diff that silently converts a 0.1x read into a 2x write.
+          async (wire, headers) => await this.client.messages.create(
+            wire as unknown as Anthropic.MessageCreateParamsNonStreaming,
+            headers ? { headers } : undefined,
+          ),
+          config.cacheKeepalive ?? {},
+        );
   }
 
   supportsModel(modelId: string): boolean {
@@ -192,10 +217,15 @@ export class AnthropicAdapter implements ProviderAdapter {
     const fullRequest = { ...anthropicRequest, stream: false as const };
     options?.onRequest?.(fullRequest);
 
+    const headers = this.betaHeaders(request);
+    this.cacheKeepalive?.record(
+      fullRequest as unknown as Record<string, unknown>, headers, 'complete',
+    );
+
     try {
       const response = await this.client.messages.create(fullRequest, {
         signal: options?.signal,
-        headers: this.betaHeaders(request),
+        headers,
       });
 
       return this.parseResponse(response, fullRequest);
@@ -214,6 +244,14 @@ export class AnthropicAdapter implements ProviderAdapter {
     const fullRequest = { ...anthropicRequest, stream: true };
     options?.onRequest?.(fullRequest);
 
+    // Snapshot the primary lane's prefix so it can be held warm across idle
+    // gaps. `stream: true` is dropped at replay time (transport, not cache key).
+    this.cacheKeepalive?.record(
+      fullRequest as unknown as Record<string, unknown>,
+      this.betaHeaders(request),
+      'stream',
+    );
+
     // Idle timeout: abort if no SSE event arrives within the deadline.
     // The SDK's timeout only covers the initial HTTP response headers;
     // once streaming starts, a silently dropped connection waits forever.
@@ -664,7 +702,12 @@ export class AnthropicAdapter implements ProviderAdapter {
         return authError(message, error, rawRequest);
       }
 
-      if (message.includes('context') || message.includes('too long')) {
+      // Context-length is a client-side request-shape problem — it only ever
+      // arrives as a 400 (invalid_request_error). Without the status guard, a
+      // transient 5xx whose body happens to contain "context" or "too long"
+      // (e.g. "Internal error: context processing failed") was misclassified
+      // as non-retryable context_length, silently suppressing retries.
+      if (status === 400 && (message.includes('context') || message.includes('too long'))) {
         return contextLengthError(message, error, rawRequest);
       }
 

package/src/types/config.ts

@@ -180,6 +180,12 @@ export interface MembraneConfig {
    */
   defaultPromptCaching?: boolean;
 
+  /**
+   * Default for request.floatingCacheMarker when the request doesn't set it.
+   * Default: true. See NormalizedRequest.floatingCacheMarker.
+   */
+  defaultFloatingCacheMarker?: boolean;
+
   /**
    * Prefill formatter for message serialization and response parsing.
    * Controls how messages are formatted for the API and how responses are parsed.

package/src/types/request.ts

@@ -159,6 +159,20 @@ export interface NormalizedRequest {
    */
   cacheTtl?: '5m' | '1h';
 
+  /**
+   * Float a trailing cache_control marker onto the newest message when the
+   * native tool loop rebuilds the request between tool-execution rounds, so
+   * the growing tool-round suffix caches incrementally (each round writes
+   * only its delta and cache-reads everything before it). Placed only from
+   * the request's *residual* breakpoint budget — the marker is withheld when
+   * upstream markers already occupy all 4 Anthropic cache_control slots —
+   * so upstream breakpoints are never displaced or stripped.
+   * Defaults to true (when promptCaching is enabled). Set false for context
+   * strategies whose request prefix churns between rounds, where a trailing
+   * marker would be pure cache-write cost.
+   */
+  floatingCacheMarker?: boolean;
+
   /**
    * Context prefix for simulacrum seeding.
    * Injected as first assistant message (before conversation history).