@animalabs/membrane 0.5.67 → 0.5.69

package/src/providers/openrouter.ts

@@ -31,7 +31,45 @@ import { safeParseJson, createCombinedSignal, SSELineParser } from './utils.js';
 /** Content block for messages through OpenRouter */
 type OpenRouterContentBlock =
   | { type: 'text'; text: string; cache_control?: { type: 'ephemeral' } }
-  | { type: 'image_url'; image_url: { url: string; detail?: string } };
+  | { type: 'image_url'; image_url: { url: string; detail?: string } }
+  | { type: 'input_audio'; input_audio: { data: string; format: string } };
+
+/**
+ * Map a media MIME type to the `format` string OpenRouter expects on an
+ * `input_audio` part. OpenRouter follows the OpenAI Chat Completions shape and
+ * documents these format values: wav, mp3, aiff, aac, ogg, flac, m4a, pcm16,
+ * pcm24. Returns undefined for MIME types we don't have a mapping for, so the
+ * caller can skip the block rather than send a malformed part (which model
+ * actually accepts which format is the model's concern — this is just plumbing).
+ */
+function audioMimeToFormat(mimeType: string): string | undefined {
+  // Strip MIME parameters ("audio/mpeg; rate=16000") before matching.
+  switch (mimeType.split(';')[0]!.trim().toLowerCase()) {
+    case 'audio/mpeg':
+    case 'audio/mp3':
+    case 'audio/mpeg3':      // legacy MP3 aliases some clients (e.g. Discord) report
+    case 'audio/x-mpeg-3':
+      return 'mp3';
+    case 'audio/wav':
+    case 'audio/x-wav':
+    case 'audio/wave':
+    case 'audio/vnd.wave':
+      return 'wav';
+    case 'audio/ogg':
+      return 'ogg';
+    case 'audio/flac':
+    case 'audio/x-flac':
+      return 'flac';
+    case 'audio/aac':
+      return 'aac';
+    case 'audio/mp4':
+    case 'audio/m4a':
+    case 'audio/x-m4a':
+      return 'm4a';
+    default:
+      return undefined;
+  }
+}
 
 interface OpenRouterMessage {
   role: 'user' | 'assistant' | 'system' | 'tool';
@@ -193,8 +231,27 @@ export class OpenRouterAdapter implements ProviderAdapter {
         for (const data of dataLines) {
           if (data === '[DONE]') continue;
 
+          // Parse first; only JSON noise is ignorable. Everything after the
+          // parse must NOT be swallowed by the catch below.
+          let parsed: Record<string, any>;
+          try {
+            parsed = JSON.parse(data);
+          } catch {
+            continue; // Ignore parse errors (partial/keep-alive lines)
+          }
+
+          // OpenRouter delivers mid-stream failures (e.g. upstream 429s) as an
+          // SSE data line with an `error` payload. Silently ignoring it would
+          // yield a fake-successful empty completion — surface it instead so
+          // retry logic can handle it.
+          if (typeof parsed === 'object' && parsed !== null && parsed.error) {
+            const err = parsed.error as { code?: number | string; message?: string };
+            throw new Error(
+              `OpenRouter stream error${err.code !== undefined ? ` (${err.code})` : ''}: ${err.message ?? JSON.stringify(err)}`
+            );
+          }
+
           try {
-            const parsed = JSON.parse(data);
             const delta = parsed.choices?.[0]?.delta;
 
             if (delta?.content) {
@@ -370,10 +427,11 @@ export class OpenRouterAdapter implements ProviderAdapter {
         const textParts: string[] = [];
         const toolResults: OpenRouterMessage[] = [];
         let hasImages = false;
+        let hasAudio = false;
 
         for (const block of msg.content) {
           if (block.type === 'text') {
-            if (hasCache || hasImages) {
+            if (hasCache || hasImages || hasAudio) {
               const contentBlock: OpenRouterContentBlock = {
                 type: 'text',
                 text: block.text,
@@ -406,6 +464,37 @@ export class OpenRouterAdapter implements ProviderAdapter {
                 image_url: { url: block.source.url },
               });
             }
+          } else if (block.type === 'audio') {
+            // Audio input → OpenAI-style `input_audio` content part (sibling to
+            // the Gemini inlineData path). Like images, this forces the array
+            // content format. MIME (canonical `mediaType`, falling back to
+            // snake_case `media_type`, then `audio/mpeg`) is mapped to
+            // OpenRouter's `format`; unmappable types are skipped with a warning
+            // rather than sending a malformed part. Data is raw base64 (no
+            // data-uri prefix), matching the Gemini audio path.
+            hasAudio = true;
+            // Migrate any already-collected textParts into contentBlocks
+            // (we didn't know we'd need array format until hitting audio).
+            for (const text of textParts) {
+              contentBlocks.push({ type: 'text', text });
+            }
+            textParts.length = 0;
+
+            if (block.source?.type === 'base64' && block.source.data) {
+              const mediaType = block.source.mediaType ?? block.source.media_type ?? 'audio/mpeg';
+              const format = audioMimeToFormat(mediaType);
+              if (format) {
+                contentBlocks.push({
+                  type: 'input_audio',
+                  input_audio: { data: block.source.data, format },
+                });
+              } else {
+                console.warn(
+                  `[membrane:openrouter] Skipping audio block with unsupported MIME type "${mediaType}" ` +
+                  `(no OpenRouter input_audio format mapping)`
+                );
+              }
+            }
           } else if (block.type === 'tool_use') {
             toolCalls.push({
               id: block.id,
@@ -435,8 +524,8 @@ export class OpenRouterAdapter implements ProviderAdapter {
           return [];
         }
 
-        // Use content blocks array if caching or images require it, otherwise concatenate text
-        const useArray = hasCache || hasImages;
+        // Use content blocks array if caching, images, or audio require it, otherwise concatenate text
+        const useArray = hasCache || hasImages || hasAudio;
         const result: OpenRouterMessage = {
           role: msg.role,
           content: useArray ? contentBlocks : (textParts.length > 0 ? textParts.join('\n') : null),

package/src/types/provider.ts

@@ -223,7 +223,13 @@ export interface ProviderRequest {
   
   /** Tools in provider format */
   tools?: unknown[];
-  
+
+  /**
+   * Extended-thinking config (Anthropic). Presence with a `type` other than
+   * `'disabled'` enables thinking, which strips custom sampling parameters.
+   */
+  thinking?: { type?: string; [key: string]: unknown };
+
   /** Additional provider-specific params */
   extra?: Record<string, unknown>;
 }

package/src/types/request.ts

@@ -112,6 +112,14 @@ export type ToolMode =
 // ============================================================================
 
 export interface NormalizedRequest {
+  /**
+   * Explicitly own the loss of old inline images when the serialized request
+   * exceeds the API byte cap: oldest images are replaced with loud
+   * placeholders (error-logged). Without this flag an oversize request FAILS
+   * LOUDLY before the API call (2026-07-12 — no silent transport-layer
+   * mutation). Intended for summarizer/compression callers.
+   */
+  shedOversizeImages?: boolean;
   /** Conversation messages */
   messages: NormalizedMessage[];
   

package/src/utils/image-media.ts

@@ -0,0 +1,150 @@
+/**
+ * Image media-type sanitation shared by every request-build path.
+ *
+ * sharp-readable but API-rejected formats (image/svg, image/tiff, ...) can
+ * enter an agent's event store through permissive ingest surfaces; one such
+ * block then 400s EVERY subsequent compile (invalid_request) and hard-downs
+ * the agent (LabClaude 2026-07-11: an SVG attachment). Every place that turns
+ * a normalized image block into a provider image block must either emit an
+ * accepted media type or degrade to a loud text placeholder the agent can see.
+ */
+
+export const API_ACCEPTED_IMAGE_MEDIA_TYPES: ReadonlySet<string> = new Set([
+  'image/jpeg',
+  'image/png',
+  'image/gif',
+  'image/webp',
+]);
+
+export function isAcceptedImageMediaType(mediaType: string | null | undefined): boolean {
+  return API_ACCEPTED_IMAGE_MEDIA_TYPES.has((mediaType ?? '').toLowerCase());
+}
+
+/** Requests are also capped by SIZE (~32MB serialized) independent of token
+ *  count — token math is blind to base64 bulk, so an image-heavy window can
+ *  pass the context budget yet draw 413 request_too_large (Mythos 2026-07-12:
+ *  47 inline images = 34.3MB). Default cap leaves headroom for system/tools
+ *  and JSON overhead; override via MEMBRANE_MAX_REQUEST_BYTES. */
+export const DEFAULT_MAX_REQUEST_BYTES = 28 * 1024 * 1024;
+
+/** The effective request byte cap. */
+export function requestByteCap(capBytes?: number): number {
+  return capBytes ?? (Number(process.env.MEMBRANE_MAX_REQUEST_BYTES) || DEFAULT_MAX_REQUEST_BYTES);
+}
+
+/** Serialized byte size of the messages array (JSON length ≈ wire size). */
+export function serializedMessageBytes(messages: Array<{ content?: unknown }>): number {
+  return JSON.stringify(messages).length;
+}
+
+/**
+ * FAIL LOUDLY when the serialized messages exceed the byte cap (2026-07-12).
+ * Silent content mutation at the transport layer is a diagnosis trap: every
+ * layer above believes it sent a different context than the wire carried.
+ * Callers that can genuinely tolerate losing old images must OWN that policy
+ * by setting `shedOversizeImages` on the request — everything else fails
+ * here, before the API round-trip, with the full breakdown.
+ */
+export function assertWithinByteBudget(
+  messages: Array<{ content?: unknown }>,
+  capBytes: number | undefined,
+  site: string,
+): void {
+  const cap = requestByteCap(capBytes);
+  const size = serializedMessageBytes(messages);
+  if (size <= cap) return;
+  let images = 0;
+  let imageBytes = 0;
+  const walk = (content: unknown[]): void => {
+    for (const b of content) {
+      if (!b || typeof b !== 'object') continue;
+      const typed = b as { type?: string; content?: unknown };
+      if (typed.type === 'image') {
+        images++;
+        imageBytes += JSON.stringify(b).length;
+      } else if (typed.type === 'tool_result' && Array.isArray(typed.content)) {
+        walk(typed.content);
+      }
+    }
+  };
+  for (const m of messages) if (Array.isArray(m.content)) walk(m.content);
+  throw new Error(
+    `[membrane] request exceeds the byte cap at ${site}: ${Math.round(size / 1e6)}MB > ` +
+      `cap ${Math.round(cap / 1e6)}MB (${images} inline image(s), ~${Math.round(imageBytes / 1e6)}MB of them). ` +
+      `Refusing to silently drop content. Either the compile must respect the byte wall ` +
+      `(context-manager maxLiveImageBytes), or the caller must explicitly own image loss ` +
+      `by setting shedOversizeImages on the request.`,
+  );
+}
+
+/** Shed inline images, OLDEST first, until the serialized messages fit the
+ *  byte cap. Mutates content arrays in place, replacing shed image blocks
+ *  (including ones nested in tool_result content) with loud agent-facing
+ *  placeholders. Returns the number of images shed.
+ *
+ *  ONLY runs for callers that explicitly opted in (`shedOversizeImages`) —
+ *  and even then it reports at error grade: an exercised opt-in is a signal
+ *  the upstream byte wall is misconfigured. */
+export function shedImagesToFitByteBudget(
+  messages: Array<{ content?: unknown }>,
+  capBytes?: number,
+  site = 'unknown-site',
+): number {
+  const cap = requestByteCap(capBytes);
+  let size = JSON.stringify(messages).length;
+  if (size <= cap) return 0;
+
+  const placeholder = (): { type: 'text'; text: string } => ({
+    type: 'text',
+    text:
+      `[system: an image that belongs here was dropped from THIS request only — the ` +
+      `request exceeded the API's total size limit, and older images are dropped first. ` +
+      `You are not seeing this image right now; it remains in your history and recent ` +
+      `images are kept.]`,
+  });
+
+  let shed = 0;
+  const shedInArray = (content: unknown[]): boolean => {
+    for (let i = 0; i < content.length; i++) {
+      const b = content[i];
+      if (!b || typeof b !== 'object') continue;
+      const typed = b as { type?: string; content?: unknown };
+      if (typed.type === 'image') {
+        const before = JSON.stringify(b).length;
+        content[i] = placeholder();
+        size -= before - JSON.stringify(content[i]).length;
+        shed++;
+        if (size <= cap) return true;
+      } else if (typed.type === 'tool_result' && Array.isArray(typed.content)) {
+        if (shedInArray(typed.content)) return true;
+      }
+    }
+    return false;
+  };
+
+  for (const msg of messages) {
+    if (Array.isArray(msg.content) && shedInArray(msg.content)) break;
+  }
+  console.error(
+    `[membrane-oversize] shed ${shed} inline image(s) at ${site} to fit the request byte cap ` +
+      `(${Math.round(size / 1e6)}MB / cap ${Math.round(cap / 1e6)}MB). This opt-in firing means ` +
+      `the upstream compile exceeded the byte wall — check maxLiveImageBytes / image policy.`,
+  );
+  return shed;
+}
+
+/** Loud agent-facing stand-in for an image the API would reject. The agent
+ *  must be clearly aware an image was stripped — silence here reads as
+ *  "there was no image". Also warns on stderr for ops visibility. */
+export function strippedImagePlaceholder(mediaType: unknown): { type: 'text'; text: string } {
+  console.warn(
+    `[membrane] image block stripped: unsupported media type "${String(mediaType)}"`,
+  );
+  return {
+    type: 'text',
+    text:
+      `[system: an image that belongs here was NOT shown to you — its media type ` +
+      `"${String(mediaType)}" is not accepted by the model API (only jpeg/png/gif/webp are). ` +
+      `You are not seeing this image. If it matters, ask for it in a supported format.]`,
+  };
+}

package/src/utils/tool-parser.ts

@@ -11,6 +11,7 @@
  */
 
 import type { ToolCall, ToolResult, ParsedToolCalls, ContentBlock, ToolResultContentBlock } from '../types/index.js';
+import { isAcceptedImageMediaType, strippedImagePlaceholder } from './image-media.js';
 
 // ============================================================================
 // Helper Functions
@@ -630,15 +631,19 @@ export function formatToolResultsForSplitTurn(results: ToolResult[]): SplitTurnC
         if (block.type === 'text') {
           textParts.push(escapeXml(block.text));
         } else if (block.type === 'image') {
-          resultHasImages = true;
-          resultImages.push({
-            type: 'image',
-            source: {
-              type: 'base64',
-              media_type: block.source.mediaType,
-              data: block.source.data,
-            },
-          });
+          if (!isAcceptedImageMediaType(block.source.mediaType)) {
+            textParts.push(escapeXml(strippedImagePlaceholder(block.source.mediaType).text));
+          } else {
+            resultHasImages = true;
+            resultImages.push({
+              type: 'image',
+              source: {
+                type: 'base64',
+                media_type: block.source.mediaType,
+                data: block.source.data,
+              },
+            });
+          }
         }
       }
     }