@animalabs/membrane 0.5.79 → 0.5.81

package/src/providers/openrouter.ts

@@ -12,6 +12,7 @@ import type {
   StreamCallbacks,
   ContentBlock,
   ToolDefinition,
+  UsageCacheConvention,
 } from '../types/index.js';
 import {
   MembraneError,
@@ -22,7 +23,7 @@ import {
   abortError,
   networkError,
 } from '../types/index.js';
-import { safeParseJson, createCombinedSignal, SSELineParser } from './utils.js';
+import { safeParseJson, createCombinedSignal, SSELineParser, isDeadlineAbort, deadlineTimeoutError, throwOnStreamErrorFrame, assertTerminalEventObserved } from './utils.js';
 
 // ============================================================================
 // Types
@@ -144,8 +145,33 @@ export interface OpenRouterAdapterConfig {
 // OpenRouter Adapter
 // ============================================================================
 
+/**
+ * Which convention the tokens OpenRouter just handed back are in. OpenRouter
+ * passes the routed provider's usage payload through, so an Anthropic-routed
+ * call reports `cache_read_input_tokens` (disjoint from prompt_tokens) while an
+ * OpenAI-routed one reports `prompt_tokens_details.cached_tokens` (a subset of
+ * it). The field that carried the number therefore identifies the convention.
+ */
+function resolveRoutedCacheConvention(
+  anthropicShapedCacheRead: number | undefined,
+  resolvedCacheRead: number | undefined,
+): UsageCacheConvention | undefined {
+  if (resolvedCacheRead == null) return undefined;
+  return anthropicShapedCacheRead != null ? 'cache-excluded' : 'cache-inclusive';
+}
+
 export class OpenRouterAdapter implements ProviderAdapter {
   readonly name = 'openrouter';
+
+  /**
+   * OpenRouter fronts BOTH conventions: it passes through Anthropic's
+   * `cache_read_input_tokens` (cache-excluded) or OpenAI's
+   * `prompt_tokens_details.cached_tokens` (cache-inclusive) depending on which
+   * provider it routed to. The convention is therefore a per-response fact —
+   * each parse sets `usage.cacheConvention` from the field it actually read,
+   * and this adapter-level value is only the no-cache-tokens fallback.
+   */
+  readonly usageCacheConvention = 'unknown' as const;
   private apiKey: string;
   private baseURL: string;
   private httpReferer: string;
@@ -218,80 +244,102 @@ export class OpenRouterAdapter implements ProviderAdapter {
       const sseParser = new SSELineParser();
       let accumulated = '';
       let finishReason = 'stop';
+      let sawTerminalEvent = false;
       let toolCalls: OpenRouterToolCall[] = [];
       let streamUsage: OpenRouterResponse['usage'] | undefined;
+      // The model the provider actually served, echoed on every SSE
+      // chunk. Reporting the requested id instead hides alias
+      // resolution (and, on OpenRouter, which provider it routed to).
+      let servedModel: string | undefined;
+
+      // One frame handler for both the streamed lines and the EOF flush — the
+      // trailing buffer carries real terminal frames, not leftovers.
+      const processDataLine = (data: string): void => {
+        if (data === '[DONE]') {
+          sawTerminalEvent = true;
+          return;
+        }
 
-      while (true) {
-        const { done, value } = await reader.read();
-        if (done) break;
-
-        const chunk = decoder.decode(value, { stream: true });
-        const dataLines = sseParser.feed(chunk);
-
-        for (const data of dataLines) {
-          if (data === '[DONE]') continue;
-
-          // Parse first; only JSON noise is ignorable. Everything after the
-          // parse must NOT be swallowed by the catch below.
-          let parsed: Record<string, any>;
-          try {
-            parsed = JSON.parse(data);
-          } catch {
-            continue; // Ignore parse errors (partial/keep-alive lines)
-          }
+        // Parse first; only JSON noise is ignorable. Everything after the
+        // parse must NOT be swallowed by the catch below.
+        let parsed: Record<string, any>;
+        try {
+          parsed = JSON.parse(data);
+        } catch {
+          return; // Ignore parse errors (partial/keep-alive lines)
+        }
 
-          // OpenRouter delivers mid-stream failures (e.g. upstream 429s) as an
-          // SSE data line with an `error` payload. Silently ignoring it would
-          // yield a fake-successful empty completion — surface it instead so
-          // retry logic can handle it.
-          if (typeof parsed === 'object' && parsed !== null && parsed.error) {
-            const err = parsed.error as { code?: number | string; message?: string };
-            throw new Error(
-              `OpenRouter stream error${err.code !== undefined ? ` (${err.code})` : ''}: ${err.message ?? JSON.stringify(err)}`
-            );
-          }
+        // OpenRouter delivers mid-stream failures (e.g. upstream 429s) as an
+        // SSE data line with an `error` payload. Silently ignoring it would
+        // yield a fake-successful empty completion — surface it instead so
+        // retry logic can handle it. Shared with every other SSE adapter.
+        throwOnStreamErrorFrame(parsed, 'OpenRouter', openRouterRequest);
 
-          try {
-            const delta = parsed.choices?.[0]?.delta;
+        try {
+          const delta = parsed.choices?.[0]?.delta;
 
-            if (delta?.content) {
-              accumulated += delta.content;
-              callbacks.onChunk(delta.content);
-            }
+          if (delta?.content) {
+            accumulated += delta.content;
+            callbacks.onChunk(delta.content);
+          }
 
-            // Handle streaming tool calls
-            if (delta?.tool_calls) {
-              for (const tc of delta.tool_calls) {
-                const index = tc.index ?? 0;
-                if (!toolCalls[index]) {
-                  toolCalls[index] = {
-                    id: tc.id ?? '',
-                    type: 'function',
-                    function: { name: '', arguments: '' },
-                  };
-                }
-                if (tc.id) toolCalls[index].id = tc.id;
-                if (tc.function?.name) toolCalls[index].function.name = tc.function.name;
-                if (tc.function?.arguments) {
-                  toolCalls[index].function.arguments += tc.function.arguments;
-                }
+          // Handle streaming tool calls
+          if (delta?.tool_calls) {
+            for (const tc of delta.tool_calls) {
+              const index = tc.index ?? 0;
+              if (!toolCalls[index]) {
+                toolCalls[index] = {
+                  id: tc.id ?? '',
+                  type: 'function',
+                  function: { name: '', arguments: '' },
+                };
+              }
+              if (tc.id) toolCalls[index].id = tc.id;
+              if (tc.function?.name) toolCalls[index].function.name = tc.function.name;
+              if (tc.function?.arguments) {
+                toolCalls[index].function.arguments += tc.function.arguments;
               }
             }
+          }
 
-            if (parsed.choices?.[0]?.finish_reason) {
-              finishReason = parsed.choices[0].finish_reason;
-            }
+          if (parsed.choices?.[0]?.finish_reason) {
+            finishReason = parsed.choices[0].finish_reason;
+            sawTerminalEvent = true;
+          }
 
-            // Capture usage data (comes in final chunk when stream_options.include_usage is set)
-            if (parsed.usage) {
-              streamUsage = parsed.usage;
-            }
-          } catch (e) {
-            // Ignore parse errors
+          // Capture usage data (comes in final chunk when stream_options.include_usage is set)
+          if (parsed.usage) {
+            streamUsage = parsed.usage;
+          }
+
+          if (parsed.model) {
+            servedModel = parsed.model;
           }
+        } catch (e) {
+          // Ignore parse errors
+        }
+      };
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        const chunk = decoder.decode(value, { stream: true });
+        for (const data of sseParser.feed(chunk)) {
+          processDataLine(data);
         }
       }
 
+      // A final `data:` line that arrived without its trailing newline is still
+      // buffered here. Servers and proxies do close right after writing the
+      // last event, so dropping it would report a finished turn as a dropped
+      // connection at the guard below.
+      for (const data of sseParser.flush()) {
+        processDataLine(data);
+      }
+
+      assertTerminalEventObserved(sawTerminalEvent, 'OpenRouter', openRouterRequest);
+
       // Build response with accumulated data
       const message: OpenRouterMessage = {
         role: 'assistant',
@@ -302,7 +350,7 @@ export class OpenRouterAdapter implements ProviderAdapter {
         message.tool_calls = toolCalls;
       }
 
-      return this.parseStreamedResponse(message, finishReason, request.model, streamUsage, openRouterRequest);
+      return this.parseStreamedResponse(message, finishReason, servedModel ?? request.model, streamUsage, openRouterRequest);
 
     } catch (error) {
       throw this.handleError(error, openRouterRequest);
@@ -602,8 +650,10 @@ export class OpenRouterAdapter implements ProviderAdapter {
     // Anthropic: cache_creation_input_tokens, cache_read_input_tokens
     // OpenAI: prompt_tokens_details.cached_tokens
     const cacheCreationTokens = response.usage?.cache_creation_input_tokens;
-    const cacheReadTokens = response.usage?.cache_read_input_tokens
+    const anthropicShapedCacheRead = response.usage?.cache_read_input_tokens;
+    const cacheReadTokens = anthropicShapedCacheRead
       ?? response.usage?.prompt_tokens_details?.cached_tokens;
+    const cacheConvention = resolveRoutedCacheConvention(anthropicShapedCacheRead, cacheReadTokens);
 
     return {
       content: this.messageToContent(message),
@@ -614,6 +664,7 @@ export class OpenRouterAdapter implements ProviderAdapter {
         outputTokens: response.usage?.completion_tokens ?? 0,
         cacheCreationTokens: cacheCreationTokens ?? undefined,
         cacheReadTokens: cacheReadTokens ?? undefined,
+        cacheConvention,
       },
       model: response.model ?? requestedModel,
       rawRequest,
@@ -630,8 +681,10 @@ export class OpenRouterAdapter implements ProviderAdapter {
   ): ProviderResponse {
     // Extract cache tokens if available from stream usage
     const cacheCreationTokens = streamUsage?.cache_creation_input_tokens;
-    const cacheReadTokens = streamUsage?.cache_read_input_tokens
+    const anthropicShapedCacheRead = streamUsage?.cache_read_input_tokens;
+    const cacheReadTokens = anthropicShapedCacheRead
       ?? streamUsage?.prompt_tokens_details?.cached_tokens;
+    const cacheConvention = resolveRoutedCacheConvention(anthropicShapedCacheRead, cacheReadTokens);
 
     return {
       content: this.messageToContent(message),
@@ -642,6 +695,7 @@ export class OpenRouterAdapter implements ProviderAdapter {
         outputTokens: streamUsage?.completion_tokens ?? 0,
         cacheCreationTokens: cacheCreationTokens ?? undefined,
         cacheReadTokens: cacheReadTokens ?? undefined,
+        cacheConvention,
       },
       model: requestedModel,
       rawRequest,
@@ -696,6 +750,14 @@ export class OpenRouterAdapter implements ProviderAdapter {
   }
 
   private handleError(error: unknown, rawRequest?: unknown): MembraneError {
+    // A deadline abort is a timeout and stays one. Collapsing it into a bare
+    // abortError() here is what erased the identity before Membrane's
+    // caller-signal > timeout > error ladder could read it.
+    if (isDeadlineAbort(error)) return deadlineTimeoutError(error, rawRequest);
+    // Already-classified failures (e.g. the stream-integrity guards) keep
+    // their type and retryability instead of being re-derived from a string.
+    if (error instanceof MembraneError) return error;
+
     if (error instanceof Error) {
       const message = error.message;
 

package/src/providers/utils.ts

@@ -1,3 +1,180 @@
+import { TimeoutAbortError, authError, networkError, rateLimitError, serverError } from '../types/errors.js';
+
+interface StreamErrorFrameFields {
+  code?: unknown;
+  status?: unknown;
+  type?: unknown;
+  message?: unknown;
+  retry_after?: unknown;
+  retryAfter?: unknown;
+  retry_after_ms?: unknown;
+  retryAfterMs?: unknown;
+  retryDelay?: unknown;
+  retry_delay?: unknown;
+}
+
+const RATE_LIMIT_FRAME_TOKENS = ['rate_limit', 'rate-limit', 'ratelimit', 'too_many_requests', 'resource_exhausted'];
+const OVERLOADED_FRAME_TOKENS = ['overloaded'];
+const SERVER_FRAME_TOKENS = ['server_error', 'internal', 'unavailable'];
+const AUTH_FRAME_TOKENS = [
+  'invalid_api_key',
+  'api_key_invalid',
+  'authentication_error',
+  'unauthenticated',
+  'permission_denied',
+  'permission_error',
+];
+
+function readNumericField(...candidates: unknown[]): number | undefined {
+  for (const candidate of candidates) {
+    if (typeof candidate === 'number' && Number.isFinite(candidate)) return candidate;
+    if (typeof candidate === 'string' && /^\d+$/.test(candidate.trim())) return Number(candidate.trim());
+  }
+  return undefined;
+}
+
+function readDurationSeconds(...candidates: unknown[]): number | undefined {
+  for (const candidate of candidates) {
+    if (typeof candidate === 'number' && Number.isFinite(candidate)) return candidate;
+    if (typeof candidate === 'string') {
+      const duration = candidate.trim().match(/^(\d+(?:\.\d+)?)s?$/);
+      if (duration?.[1]) return Number(duration[1]);
+    }
+  }
+  return undefined;
+}
+
+function readFrameRetryAfterMs(fields: StreamErrorFrameFields): number | undefined {
+  const explicitMilliseconds = readNumericField(fields.retry_after_ms, fields.retryAfterMs);
+  if (explicitMilliseconds !== undefined) return explicitMilliseconds;
+
+  const seconds = readDurationSeconds(fields.retry_after, fields.retryAfter, fields.retryDelay, fields.retry_delay);
+  return seconds === undefined ? undefined : Math.round(seconds * 1000);
+}
+
+/**
+ * The frame's own non-numeric `type`/`status` values — the provider's
+ * structured classification tokens (`rate_limit_error`, `RESOURCE_EXHAUSTED`,
+ * `overloaded_error`), never free prose. Matching a substring against these is
+ * safe in a way that matching the same substring against a human-readable
+ * message is not.
+ */
+function readFrameClassificationTokens(fields: StreamErrorFrameFields): string[] {
+  return [fields.type, fields.status, fields.code]
+    .filter((value): value is string => typeof value === 'string' && value.trim() !== '')
+    .map((value) => value.trim())
+    .filter((token) => !/^\d+$/.test(token));
+}
+
+function frameTokensMatch(tokens: string[], needles: string[]): boolean {
+  return tokens.some((token) => needles.some((needle) => token.toLowerCase().includes(needle)));
+}
+
+/**
+ * Throw when an SSE data frame carries a provider `error` payload.
+ *
+ * Every provider delivers mid-stream failures (upstream 429s, capacity loss,
+ * safety aborts) as a data line with an `error` object inside an HTTP-200
+ * stream. A loop that only looks for `choices`/`candidates` drops that frame,
+ * reaches EOF, and builds a well-formed success out of whatever arrived first:
+ * partial content, the initialised default finish reason, zero usage. Nothing
+ * distinguishes it from the model choosing to stop, so the truncated turn is
+ * persisted as a real one.
+ *
+ * The house ruled this a bug twice before it was hoisted here — once in
+ * openrouter.ts (tests/unit/openrouter-stream-error.test.ts) and once in
+ * openai-responses-api.ts — and both times the fix stayed in the one adapter
+ * that was being touched. This is the single site every SSE adapter calls.
+ *
+ * The frame is CLASSIFIED here rather than downstream. Throwing a bare
+ * `Error(string)` left `code`/`type`/`status` as prose, and each adapter's
+ * `handleError` then re-derived a category by substring-matching that prose —
+ * so a mid-stream 429 arriving as `status: 429` (no literal "429" in the
+ * message) normalized to `unknown, retryable: false` and suppressed the retry
+ * the provider was explicitly asking for. Classified failures reach the caller
+ * intact because every adapter's `handleError` returns a MembraneError
+ * unchanged. Mapping, using the structured fields only:
+ *   - 429, or a rate-limit-shaped token  -> rate_limit (retryable), carrying
+ *     the frame's retry hint when it has one
+ *   - 5xx, or an overloaded/server-shaped token -> server (retryable);
+ *     overloaded-without-a-status takes 529 so it lands on the capacity
+ *     backoff schedule, matching how anthropic.ts recovers the same shape
+ *   - 401/403, or an auth-shaped token   -> auth (non-retryable)
+ *   - anything else -> the previous bare Error, so the adapter's own
+ *     provider-specific fallbacks still get their swing at it
+ * The provider's message text is never dropped, and the raw frame plus the
+ * request ride along on the classified error.
+ *
+ * `errorNoun` names what carried the payload. It defaults to the SSE case, and
+ * exists because the same `{ error: { code, message } }` object also arrives on
+ * a non-streaming 200 body, where calling the failure a stream error would be
+ * false. Classification reads the payload's own fields either way — the
+ * transport was never part of the rule.
+ */
+export function throwOnStreamErrorFrame(
+  parsed: unknown,
+  providerLabel: string,
+  rawRequest?: unknown,
+  errorNoun: string = 'stream error'
+): void {
+  if (typeof parsed !== 'object' || parsed === null) return;
+  const streamError = (parsed as { error?: unknown }).error;
+  if (!streamError) return;
+
+  const fields: StreamErrorFrameFields =
+    typeof streamError === 'object' ? (streamError as StreamErrorFrameFields) : {};
+  const httpStatus = readNumericField(fields.code, fields.status);
+  const tokens = readFrameClassificationTokens(fields);
+  const providerMessage =
+    typeof fields.message === 'string' && fields.message !== ''
+      ? fields.message
+      : JSON.stringify(streamError);
+
+  const description =
+    `${providerLabel} ${errorNoun}` +
+    `${httpStatus !== undefined ? ` (${httpStatus})` : ''}` +
+    `${tokens.length > 0 ? ` [${tokens.join(' ')}]` : ''}: ${providerMessage}`;
+
+  if (httpStatus === 429 || frameTokensMatch(tokens, RATE_LIMIT_FRAME_TOKENS)) {
+    throw rateLimitError(description, readFrameRetryAfterMs(fields), parsed, rawRequest);
+  }
+
+  if (httpStatus === 401 || httpStatus === 403 || frameTokensMatch(tokens, AUTH_FRAME_TOKENS)) {
+    throw authError(description, parsed, rawRequest);
+  }
+
+  const overloadedShaped = frameTokensMatch(tokens, OVERLOADED_FRAME_TOKENS);
+  if ((httpStatus !== undefined && httpStatus >= 500) || overloadedShaped || frameTokensMatch(tokens, SERVER_FRAME_TOKENS)) {
+    throw serverError(description, httpStatus ?? (overloadedShaped ? 529 : undefined), parsed, rawRequest);
+  }
+
+  throw new Error(description);
+}
+
+/**
+ * Throw when a stream reached EOF without ever observing a terminal event.
+ *
+ * The terminal signal (`finish_reason`, `[DONE]`, Anthropic's `message_delta`,
+ * Gemini's `finishReason`) must be an OBSERVATION, not a default. A graceful
+ * upstream close with no terminal frame — proxy/LB idle timeout, early FIN,
+ * a gateway truncating the body — otherwise yields a clean-looking
+ * `end_turn` over partial content plus a fabricated `finish_reason` that never
+ * came off the wire. Abrupt resets already reject the read; graceful ones did
+ * not. Retryable by construction: the request was never answered in full.
+ */
+export function assertTerminalEventObserved(
+  sawTerminalEvent: boolean,
+  providerLabel: string,
+  rawRequest?: unknown
+): void {
+  if (sawTerminalEvent) return;
+  throw networkError(
+    `${providerLabel} stream ended before a terminal event (connection dropped mid-stream)`,
+    undefined,
+    rawRequest
+  );
+}
+
 /**
  * Safely parse a JSON string, returning an empty object on failure.
  * Used for tool call arguments which may be malformed from streaming.
@@ -11,6 +188,52 @@ export function safeParseJson(str: string | undefined): Record<string, unknown>
   }
 }
 
+/**
+ * Marks the abort reason raised by an adapter's own `timeoutMs` deadline, so
+ * the error that comes back out of `fetch` can be told apart from a caller's
+ * cancellation by PROVENANCE rather than by matching its message text.
+ *
+ * `fetch` rejects with the signal's own `reason` object — identity and extra
+ * properties intact, on both the pre-headers and the body-read paths — so the
+ * mark survives the round trip through the platform. When some layer does
+ * replace the error, `isDeadlineAbort` simply reports false and the abort
+ * classifies as it did before: a missing mark degrades to the old answer, it
+ * never invents a timeout.
+ */
+const DEADLINE_ABORT = Symbol.for('membrane.deadlineAbort');
+
+function deadlineAbortReason(): DOMException {
+  const reason = new DOMException('Request timed out', 'AbortError');
+  Object.defineProperty(reason, DEADLINE_ABORT, { value: true, enumerable: false });
+  return reason;
+}
+
+/**
+ * True when this error is the abort raised by an adapter's own deadline.
+ *
+ * A caller's cancellation never carries the mark (the combined controller is
+ * aborted with the caller's own reason), and if both race, whichever fired
+ * first is the reason the platform rejects with — so the mark answers "was
+ * this OUR deadline?" without a separate tie-break.
+ */
+export function isDeadlineAbort(error: unknown): boolean {
+  return (
+    typeof error === 'object' &&
+    error !== null &&
+    (error as Record<symbol, unknown>)[DEADLINE_ABORT] === true
+  );
+}
+
+/**
+ * The typed error a deadline abort should become: a timeout by classification,
+ * an abort by provenance. Adapters return this from `handleError` instead of a
+ * bare `abortError()`, which used to erase the distinction before Membrane's
+ * caller-signal > timeout > error ladder could read it.
+ */
+export function deadlineTimeoutError(error: unknown, rawRequest?: unknown): TimeoutAbortError {
+  return new TimeoutAbortError('Request timed out', error, rawRequest);
+}
+
 /**
  * Create a combined AbortSignal that fires on either the caller's signal
  * or a timeout (whichever comes first).
@@ -18,8 +241,9 @@ export function safeParseJson(str: string | undefined): Record<string, unknown>
  * The returned `cleanup` function MUST be called in a `finally` block to
  * clear the timeout and remove the event listener, preventing leaks.
  *
- * Timeout aborts with `DOMException('Request timed out', 'AbortError')`
- * so it classifies identically to user-initiated aborts.
+ * Timeout aborts with a marked `DOMException('Request timed out',
+ * 'AbortError')`: abort-shaped like any cancellation, and identifiable as the
+ * deadline's doing via `isDeadlineAbort`.
  */
 export function createCombinedSignal(
   signal?: AbortSignal,
@@ -32,10 +256,7 @@ export function createCombinedSignal(
   let timeoutId: ReturnType<typeof setTimeout> | undefined;
 
   if (timeoutMs) {
-    timeoutId = setTimeout(
-      () => controller.abort(new DOMException('Request timed out', 'AbortError')),
-      timeoutMs
-    );
+    timeoutId = setTimeout(() => controller.abort(deadlineAbortReason()), timeoutMs);
   }
 
   const onAbort = () => controller.abort(signal!.reason);