@animalabs/membrane 0.5.76 → 0.5.78

package/src/providers/bedrock.ts

@@ -49,6 +49,16 @@ export interface BedrockAdapterConfig {
 
   /** Anthropic API version header (defaults to 2023-06-01) */
   anthropicVersion?: string;
+
+  /**
+   * Endpoint override (no trailing slash), e.g. an inference gateway leg
+   * like `https://gate.animalabs.ai/bedrock/apse1`. Defaults to
+   * `https://bedrock-runtime.{region}.amazonaws.com`. When pointing at a
+   * gateway, set accessKeyId to the gate token — the gateway reads it from
+   * the SigV4 Credential field, discards the client signature, and re-signs
+   * with real AWS creds that never leave the gateway box.
+   */
+  baseURL?: string;
 }
 
 // ============================================================================
@@ -120,9 +130,25 @@ interface BedrockStreamEvent {
   usage?: {
     input_tokens: number;
     output_tokens: number;
+    cache_creation_input_tokens?: number | null;
+    cache_read_input_tokens?: number | null;
   };
 }
 
+/**
+ * Bedrock accepts `cache_control: { type: 'ephemeral' }` but rejects the
+ * direct-API `ttl` extension ("cache_control.ttl: Extra inputs are not
+ * permitted"). Drop the ttl, keep the marker — the cache still works, at
+ * Bedrock's fixed default TTL.
+ */
+function stripCacheTtl<T extends Record<string, any>>(block: T): T {
+  if (block?.cache_control && typeof block.cache_control === 'object' && 'ttl' in block.cache_control) {
+    const { ttl, ...cacheControl } = block.cache_control;
+    return { ...block, cache_control: cacheControl };
+  }
+  return block;
+}
+
 // ============================================================================
 // AWS Signature V4 Implementation
 // ============================================================================
@@ -252,6 +278,7 @@ export class BedrockAdapter implements ProviderAdapter {
   private region: string;
   private defaultMaxTokens: number;
   private anthropicVersion: string;
+  private baseURL?: string;
 
   constructor(config: BedrockAdapterConfig = {}) {
     this.accessKeyId = config.accessKeyId ?? process.env.AWS_ACCESS_KEY_ID ?? '';
@@ -260,6 +287,7 @@ export class BedrockAdapter implements ProviderAdapter {
     this.region = config.region ?? process.env.AWS_REGION ?? 'us-west-2';
     this.defaultMaxTokens = config.defaultMaxTokens ?? 4096;
     this.anthropicVersion = config.anthropicVersion ?? 'bedrock-2023-05-31';
+    this.baseURL = config.baseURL?.replace(/\/$/, '');
 
     if (!this.accessKeyId || !this.secretAccessKey) {
       throw new Error('AWS credentials required: accessKeyId and secretAccessKey');
@@ -271,6 +299,18 @@ export class BedrockAdapter implements ProviderAdapter {
     return modelId.includes('claude') || modelId.startsWith('anthropic.');
   }
 
+  /**
+   * Cross-region inference-profile prefix for this adapter's region.
+   * Claude 4-era models on Bedrock reject on-demand invocation of the
+   * direct id ("Invocation ... with on-demand throughput isn't supported")
+   * and require the profile form — verified live 2026-07-31.
+   */
+  private inferenceProfilePrefix(): string {
+    if (this.region.startsWith('eu-')) return 'eu.';
+    if (this.region.startsWith('ap-')) return 'apac.';
+    return 'us.';
+  }
+
   /**
    * Convert a standard Claude model ID to Bedrock format if needed
    */
@@ -282,12 +322,19 @@ export class BedrockAdapter implements ProviderAdapter {
 
     // If already in Bedrock format, use as-is
     // Supports both direct model IDs (anthropic.claude-...) and
-    // cross-region inference profile IDs (us.anthropic.claude-..., eu.anthropic.claude-..., apac.anthropic.claude-...)
-    if (modelId.startsWith('anthropic.') || /^[a-z]{2,4}\.anthropic\./.test(modelId)) {
+    // cross-region inference profile IDs (us.anthropic.claude-..., eu.anthropic.claude-...,
+    // apac.anthropic.claude-..., global.anthropic.claude-... — `global` is 6 chars, hence {2,6}).
+    if (modelId.startsWith('anthropic.') || /^[a-z]{2,6}\.anthropic\./.test(modelId)) {
       return modelId;
     }
 
-    // Map common Claude model IDs to Bedrock format
+    const profile = this.inferenceProfilePrefix();
+
+    // Map common Claude model IDs to Bedrock format. The 3.x entries keep
+    // their historical direct-id form (those models predate inference
+    // profiles; all are EOL on Bedrock as of 2026-07 anyway, so the exact
+    // shape is moot). 4-era entries use the profile form — the direct id
+    // no longer invokes.
     const modelMap: Record<string, string> = {
       'claude-3-5-sonnet-20241022': 'anthropic.claude-3-5-sonnet-20241022-v2:0',
       'claude-3-5-sonnet-latest': 'anthropic.claude-3-5-sonnet-20241022-v2:0',
@@ -296,13 +343,15 @@ export class BedrockAdapter implements ProviderAdapter {
       'claude-3-opus-20240229': 'anthropic.claude-3-opus-20240229-v1:0',
       'claude-3-sonnet-20240229': 'anthropic.claude-3-sonnet-20240229-v1:0',
       'claude-3-haiku-20240307': 'anthropic.claude-3-haiku-20240307-v1:0',
-      'claude-sonnet-4-20250514': 'anthropic.claude-sonnet-4-20250514-v1:0',
-      'claude-opus-4-20250514': 'anthropic.claude-opus-4-20250514-v1:0',
-      // Haiku 4.5 aliases
-      'claude-haiku-4-5-20251001': 'anthropic.claude-3-5-haiku-20241022-v1:0',
+      'claude-sonnet-4-20250514': `${profile}anthropic.claude-sonnet-4-20250514-v1:0`,
+      'claude-opus-4-20250514': `${profile}anthropic.claude-opus-4-20250514-v1:0`,
+      // Haiku 4.5 previously aliased to 3.5 Haiku (a stand-in from before
+      // Haiku 4.5 reached Bedrock). 3.5 Haiku is EOL on Bedrock now, so the
+      // alias routed every plain-id caller to a guaranteed error.
+      'claude-haiku-4-5-20251001': `${profile}anthropic.claude-haiku-4-5-20251001-v1:0`,
     };
 
-    return modelMap[modelId] ?? `anthropic.${modelId}-v1:0`;
+    return modelMap[modelId] ?? `${profile}anthropic.${modelId}-v1:0`;
   }
 
   async complete(
@@ -347,7 +396,14 @@ export class BedrockAdapter implements ProviderAdapter {
 
   private buildRequest(request: ProviderRequest, bedrockModelId?: string): BedrockMessageRequest {
     // Strip provider-specific fields (e.g., sourceUrl for Gemini) from image blocks
-    // before sending to Bedrock/Anthropic, which rejects extra inputs
+    // before sending to Bedrock/Anthropic, which rejects extra inputs.
+    //
+    // Same treatment for cache_control.ttl: Bedrock's prompt cache runs at the
+    // fixed default (5m) TTL — the ttl field is a direct-API extension and
+    // Bedrock rejects it as an extra input. The marker itself is fine and
+    // caching works without the field, so strip just the ttl and keep the
+    // breakpoint. Transport quirks belong to the transport, not to every
+    // caller that sets cacheTtl. (Connectome issue #35.)
     const sanitizedMessages = (request.messages as any[]).map((msg: any) => {
       if (!Array.isArray(msg.content)) return msg;
       return {
@@ -355,9 +411,9 @@ export class BedrockAdapter implements ProviderAdapter {
         content: msg.content.map((block: any) => {
           if (block.type === 'image' && block.sourceUrl !== undefined) {
             const { sourceUrl, ...rest } = block;
-            return rest;
+            return stripCacheTtl(rest);
           }
-          return block;
+          return stripCacheTtl(block);
         }),
       };
     });
@@ -377,6 +433,10 @@ export class BedrockAdapter implements ProviderAdapter {
       if (needsFlatten && Array.isArray(request.system)) {
         const blocks = request.system as Array<{ type: string; text: string }>;
         params.system = blocks.map(b => b.text).join('\n\n');
+      } else if (Array.isArray(request.system)) {
+        params.system = (request.system as Array<Record<string, any>>).map(
+          b => stripCacheTtl(b)
+        ) as BedrockMessageRequest['system'];
       } else {
         params.system = request.system as BedrockMessageRequest['system'];
       }
@@ -400,7 +460,7 @@ export class BedrockAdapter implements ProviderAdapter {
     }
 
     if (request.tools && request.tools.length > 0) {
-      params.tools = request.tools;
+      params.tools = (request.tools as Array<Record<string, any>>).map(t => stripCacheTtl(t));
     }
 
     // Handle extended thinking
@@ -434,7 +494,7 @@ export class BedrockAdapter implements ProviderAdapter {
     signal?: AbortSignal
   ): Promise<BedrockMessageResponse> {
     const url = new URL(
-      `https://bedrock-runtime.${this.region}.amazonaws.com/model/${encodeURIComponent(modelId)}/invoke`
+      `${this.baseURL ?? `https://bedrock-runtime.${this.region}.amazonaws.com`}/model/${encodeURIComponent(modelId)}/invoke`
     );
 
     const body = JSON.stringify(request);
@@ -477,7 +537,7 @@ export class BedrockAdapter implements ProviderAdapter {
     signal?: AbortSignal
   ): Promise<ProviderResponse> {
     const url = new URL(
-      `https://bedrock-runtime.${this.region}.amazonaws.com/model/${encodeURIComponent(modelId)}/invoke-with-response-stream`
+      `${this.baseURL ?? `https://bedrock-runtime.${this.region}.amazonaws.com`}/model/${encodeURIComponent(modelId)}/invoke-with-response-stream`
     );
 
     const body = JSON.stringify(request);
@@ -516,6 +576,8 @@ export class BedrockAdapter implements ProviderAdapter {
     let finalMessage: BedrockMessageResponse | undefined;
     let inputTokens = 0;
     let outputTokens = 0;
+    let cacheCreationTokens: number | undefined;
+    let cacheReadTokens: number | undefined;
     let stopReason: string = 'end_turn';
     let stopSequence: string | undefined;
     let fullText = '';
@@ -630,7 +692,19 @@ export class BedrockAdapter implements ProviderAdapter {
                 }
 
                 if (eventData.type === 'message_start' && eventData.message) {
-                  inputTokens = eventData.message.usage?.input_tokens ?? 0;
+                  // Cache metrics ride the same usage objects as on the direct
+                  // API. Dropping them (pre-2026-07-31) made caching look
+                  // permanently inert on Bedrock streams: complete() surfaced
+                  // them, stream() zeroed them, and every ledger/pricing
+                  // consumer downstream saw zeros. (Connectome issue #35.)
+                  const startUsage = eventData.message.usage;
+                  inputTokens = startUsage?.input_tokens ?? 0;
+                  if (startUsage?.cache_creation_input_tokens != null) {
+                    cacheCreationTokens = startUsage.cache_creation_input_tokens;
+                  }
+                  if (startUsage?.cache_read_input_tokens != null) {
+                    cacheReadTokens = startUsage.cache_read_input_tokens;
+                  }
                 } else if (eventData.type === 'content_block_start') {
                   currentBlockIndex = eventData.index ?? 0;
                   contentBlocks[currentBlockIndex] = eventData.content_block as { type: string };
@@ -679,6 +753,15 @@ export class BedrockAdapter implements ProviderAdapter {
                 } else if (eventData.type === 'message_delta') {
                   if (eventData.usage) {
                     outputTokens = eventData.usage.output_tokens;
+                    // message_delta carries cumulative cache metrics — use as
+                    // authoritative when present (same contract as the
+                    // Anthropic adapter).
+                    if (eventData.usage.cache_creation_input_tokens != null) {
+                      cacheCreationTokens = eventData.usage.cache_creation_input_tokens;
+                    }
+                    if (eventData.usage.cache_read_input_tokens != null) {
+                      cacheReadTokens = eventData.usage.cache_read_input_tokens;
+                    }
                   }
                   if (eventData.delta?.stop_reason) {
                     stopReason = eventData.delta.stop_reason;
@@ -751,6 +834,8 @@ export class BedrockAdapter implements ProviderAdapter {
       usage: {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
+        ...(cacheCreationTokens != null ? { cache_creation_input_tokens: cacheCreationTokens } : {}),
+        ...(cacheReadTokens != null ? { cache_read_input_tokens: cacheReadTokens } : {}),
       },
     };
 

package/src/types/config.ts

@@ -15,17 +15,53 @@ import type { PrefillFormatter } from '../formatters/types.js';
 export interface RetryConfig {
   /** Maximum number of retry attempts (default: 3) */
   maxRetries: number;
-  
+
   /** Initial retry delay in milliseconds (default: 1000) */
   retryDelayMs: number;
-  
+
   /** Backoff multiplier (default: 2) */
   backoffMultiplier: number;
-  
+
   /** Maximum retry delay (default: 30000) */
   maxRetryDelayMs: number;
+
+  /**
+   * Separate, longer schedule for provider capacity errors (529
+   * overloaded_error). Capacity storms last minutes, not seconds — the
+   * standard schedule's 30s ceiling turns one into a dead turn. Overloaded
+   * retries are always attempted (mirroring the forced 429 retries), with
+   * jitter so a fleet backing off doesn't re-create the stampede in sync.
+   *
+   * maxRetries: 0 here disables this dedicated policy entirely: 529s then
+   * follow the base retry config like any other retryable server error
+   * (no forced retries, base schedule, no stream-path retry) — the exact
+   * pre-policy behavior.
+   */
+  overloaded: OverloadedRetryConfig;
 }
 
+export interface OverloadedRetryConfig {
+  /** Attempt bound for overloaded errors, applied even when the base
+   *  maxRetries is 0. Like the base maxRetries (and the forced 429 path),
+   *  this bounds TOTAL attempts, not retries-after-the-first (default: 7) */
+  maxRetries: number;
+
+  /** Initial overloaded retry delay in milliseconds (default: 10000) */
+  retryDelayMs: number;
+
+  /** Backoff multiplier (default: 2) */
+  backoffMultiplier: number;
+
+  /** Maximum overloaded retry delay (default: 300000 — 5 minutes) */
+  maxRetryDelayMs: number;
+}
+
+/** Shape accepted by MembraneConfig.retry — every field optional, including
+ *  inside the nested overloaded schedule. */
+export type RetryConfigInput = Partial<Omit<RetryConfig, 'overloaded'>> & {
+  overloaded?: Partial<OverloadedRetryConfig>;
+};
+
 // ============================================================================
 // Media Processing Config
 // ============================================================================
@@ -152,7 +188,7 @@ export interface MembraneConfig {
   formatter?: PrefillFormatter;
 
   /** Retry configuration */
-  retry?: Partial<RetryConfig>;
+  retry?: RetryConfigInput;
 
   /** Media processing configuration */
   media?: Partial<MediaConfig>;
@@ -176,6 +212,14 @@ export const DEFAULT_RETRY_CONFIG: RetryConfig = {
   retryDelayMs: 1000,
   backoffMultiplier: 2,
   maxRetryDelayMs: 30000,
+  // 7 attempts = 6 waits: 10s → 20s → 40s → 80s → 160s → 300s, ~10 minutes
+  // of patience in total — the scale capacity storms actually resolve on.
+  overloaded: {
+    maxRetries: 7,
+    retryDelayMs: 10_000,
+    backoffMultiplier: 2,
+    maxRetryDelayMs: 300_000,
+  },
 };
 
 export const DEFAULT_MEDIA_CONFIG: MediaConfig = {

package/src/types/errors.ts

@@ -239,6 +239,24 @@ export function unsupportedError(message: string, rawRequest?: unknown): Membran
 // Error Classification
 // ============================================================================
 
+/**
+ * Provider capacity exhaustion — Anthropic 529 overloaded_error, whichever
+ * path it arrived by (structured status from the provider handler, or the
+ * message-matched fallbacks in classifyError). Used only to CHOOSE the retry
+ * schedule among already-retryable errors, never to decide retryability.
+ * Matches the same deliberately narrow tokens as classifyError's fallback
+ * (status/`529`/exact `overloaded_error`) — a bare 'overloaded' in prose
+ * (e.g. "worker pool overloaded") must not put an unrelated error onto the
+ * ~10-minute schedule. The provider handlers' own bare-'overloaded' safety
+ * nets attach httpStatus 529, so those still land here via the status check.
+ */
+export function isOverloadedError(info: ErrorInfo): boolean {
+  if (!info.retryable) return false;
+  if (info.httpStatus === 529) return true;
+  const m = info.message.toLowerCase();
+  return m.includes('529') || m.includes('overloaded_error');
+}
+
 export function classifyError(error: unknown): ErrorInfo {
   if (error instanceof MembraneError) {
     return error.toErrorInfo();

package/src/types/index.ts

@@ -165,6 +165,7 @@ export {
   safetyError,
   unsupportedError,
   classifyError,
+  isOverloadedError,
 } from './errors.js';
 
 // Config

package/src/types/response.ts

@@ -15,7 +15,11 @@ export type StopReason =
   | 'stop_sequence'  // Hit stop sequence
   | 'tool_use'       // Stopped for tool use
   | 'refusal'        // Content refused by safety
-  | 'abort';         // Request was aborted
+  | 'abort'          // Request was aborted
+  | 'no_progress'    // Stall guard ended the turn (issue #39): consecutive
+                     // automatic resumptions re-sent context without advancing
+  | 'round_limit';   // Resumption round cap ended the turn: the turn kept
+                     // resuming (with progress) past maxResumptionRounds
 
 // ============================================================================
 // Usage Information

package/src/types/streaming.ts

@@ -231,6 +231,17 @@ export interface StreamOptions {
   /** Maximum tool execution depth (default: 10) */
   maxToolDepth?: number;
 
+  /**
+   * Cap on AUTOMATIC false-positive stop-sequence resumptions per turn —
+   * membrane's own re-streams, not the caller's work. Tool rounds are
+   * deliberately not counted: they are governed by maxToolDepth and caller
+   * policy. Distinct from maxToolDepth on purpose: raising the tool budget
+   * for deep chains must not also raise how often a turn may re-send its
+   * full context on membrane's own initiative (issue #39). Exceeding it
+   * ends the turn with stopReason 'round_limit'. Default: 24.
+   */
+  maxResumptionRounds?: number;
+
   /** Timeout for each tool execution */
   toolTimeoutMs?: number;
 
@@ -251,6 +262,21 @@ export interface CompleteOptions {
   /** Abort signal for cancellation */
   signal?: AbortSignal;
 
+  /**
+   * Re-issue the request when it ends with `stop_reason: 'refusal'`, up to
+   * this many times. Default 0 (off).
+   *
+   * Safe and invisible on this path: nothing has been emitted to the caller
+   * yet, so a discarded attempt leaves no trace beyond its output tokens
+   * (the replay is cache-warm on input). The streaming equivalent needs the
+   * caller to handle `RetryingEvent` — see YieldingStreamOptions.
+   *
+   * Near the content-policy threshold a refusal is probabilistic rather than
+   * a property of the payload, so re-asking is the cheapest correct response
+   * — cheaper and far less invasive than rewriting the conversation.
+   */
+  refusalRetries?: number;
+
   /** Request timeout */
   timeoutMs?: number;
 

package/src/types/yielding-stream.ts

@@ -69,6 +69,35 @@ export interface ErrorEvent {
   error: Error;
 }
 
+/**
+ * Retrying event — the provider ended the attempt with
+ * `stop_reason: 'refusal'` and membrane is re-issuing it (opt-in via
+ * `refusalRetries`).
+ *
+ * **The consumer MUST discard everything this call has emitted so far**:
+ * `tokens`, `block`, and any partially built assistant content belong to an
+ * attempt that no longer exists. A fresh sequence follows. Consumers that
+ * have already shown those tokens to a human (a TUI, a chat surface) must
+ * retract or overwrite them.
+ *
+ * Why this exists: near the classifier threshold a refusal is probabilistic
+ * rather than a property of the payload — the same bytes pass and refuse
+ * minutes apart — so re-asking is the cheapest correct response. Retrying
+ * silently would corrupt any consumer that already rendered the discarded
+ * attempt, which is why it is opt-in and announced rather than invisible.
+ */
+export interface RetryingEvent {
+  type: 'retrying';
+  /** 1-based index of the retry about to be issued. */
+  attempt: number;
+  /** Configured maximum number of retries. */
+  maxAttempts: number;
+  /** Always 'refusal' today; widened only if other retryable stops appear. */
+  reason: 'refusal';
+  /** Provider's refusal category when it supplies one (e.g. 'cyber'). */
+  category?: string;
+}
+
 /**
  * Aborted event - stream was cancelled.
  */
@@ -87,6 +116,7 @@ export interface AbortedEvent {
 export type StreamEvent =
   | TokensEvent
   | StreamBlockEvent
+  | RetryingEvent
   | ToolCallsEvent
   | UsageEvent
   | CompleteEvent
@@ -225,6 +255,23 @@ export interface YieldingStreamOptions {
   /** Request ID for correlation/logging */
   requestId?: string;
 
+  /**
+   * Re-issue an attempt that ends with `stop_reason: 'refusal'`, up to this
+   * many times. Default 0 (off).
+   *
+   * Enabling it means the stream can emit `RetryingEvent` — **consumers MUST
+   * handle it and discard what they have received for the call**, or two
+   * attempts will be concatenated. That is why it is off by default and why
+   * turning it on is a per-call decision by a consumer that has been updated.
+   *
+   * Rationale: near the content-policy threshold a refusal is probabilistic,
+   * not a property of the payload — identical bytes pass and refuse minutes
+   * apart. Re-asking is cheaper and less invasive than rewriting the
+   * conversation, and the replay is cache-warm, so only the discarded output
+   * tokens are real spend.
+   */
+  refusalRetries?: number;
+
   /**
    * Maximum tool execution depth. Default: unlimited.
    *
@@ -237,6 +284,19 @@ export interface YieldingStreamOptions {
    */
   maxToolDepth?: number;
 
+  /**
+   * Cap on AUTOMATIC false-positive stop-sequence resumptions per turn —
+   * membrane's own re-streams, not the caller's tool work. Tool rounds are
+   * deliberately NOT counted: this path's uncapped-by-default tool-loop
+   * contract stands (the caller budgets its own work via maxToolDepth).
+   * What this bounds is membrane's own failure surface — how many times a
+   * turn may re-send its full context on membrane's initiative; an
+   * unlimited resumption bound is how the 43-round Ash spin happened
+   * (issue #39). Exceeding it ends the turn with stopReason 'round_limit'.
+   * Default: 24. `-1` for unlimited, at your own risk.
+   */
+  maxResumptionRounds?: number;
+
   /**
    * Whether to emit 'tokens' events.
    * Set to false if you only care about tool calls and final response.