@animalabs/membrane 0.5.83 → 0.5.85

package/src/providers/openai-responses.ts

@@ -12,7 +12,8 @@
  * - If text-only → uses /generations
  *
  * Both endpoints:
- * - Take a single `prompt` string (not conversation messages)
+ * - Take a single `prompt` string (not conversation messages) — see
+ *   `promptFormat` for how the conversation is flattened into it
  * - Return base64-encoded images in `data[].b64_json`
  * - No streaming support (returns complete image)
  * - Support `size`, `quality`, `n`, `background`, `output_format`
@@ -68,6 +69,13 @@ interface ImagesEditRequest {
 
 type ImagesRequest = ImagesGenerateRequest | ImagesEditRequest;
 
+/** A base64 image found in the conversation, with its position for transcript references. */
+interface ImageRef {
+  dataUrl: string;
+  msgIndex: number;
+  blockIndex: number;
+}
+
 interface ImagesResponseData {
   b64_json?: string;
   url?: string;
@@ -115,8 +123,49 @@ export interface OpenAIResponsesAdapterConfig {
    * When false, always uses /v1/images/generations (text-only).
    */
   allowImageEditing?: boolean;
+
+  /**
+   * How the conversation is flattened into the single Images API `prompt`.
+   *
+   * - `'legacy'` (default): system prompt followed by `User:` / `Assistant:`
+   *   text lines. Images are attached in conversation order but never
+   *   mentioned in the prompt, and an image-only assistant message (the
+   *   typical shape of a previous generation) leaves no trace in the text.
+   * - `'transcript'`: role-delimited turns (`### User` / `### Assistant`).
+   *   Every attached image — user-provided or generated by the assistant in
+   *   an earlier turn — is numbered in attachment order and referenced
+   *   inline as `[Image N]` inside the turn it belongs to, so previous
+   *   generations read as the assistant's own prior turns. The prompt ends
+   *   with an open `### Assistant` header for the turn being generated.
+   */
+  promptFormat?: 'legacy' | 'transcript';
+
+  /**
+   * Maximum images attached to /v1/images/edits (API cap: 16). `'legacy'`
+   * keeps the first N in conversation order; `'transcript'` keeps the most
+   * recent N and marks the rest `[Image omitted]`.
+   */
+  maxInputImages?: number;
+
+  /**
+   * Text placed between the system prompt and the transcript in
+   * `'transcript'` mode, explaining the turn delimiters and image numbering
+   * to the model. Defaults to {@link DEFAULT_TRANSCRIPT_PREAMBLE}.
+   */
+  transcriptPreamble?: string;
 }
 
+/** API cap on images per /v1/images/edits request. */
+export const IMAGES_EDIT_MAX_INPUT_IMAGES = 16;
+
+export const DEFAULT_TRANSCRIPT_PREAMBLE =
+  'Below is a multi-turn conversation between users and you, the Assistant. ' +
+  'Turns are delimited by "### User" and "### Assistant" headers. ' +
+  'Attached images are numbered in attachment order; "[Image N]" marks where image N appeared. ' +
+  'Images inside Assistant turns are images you generated earlier in this conversation; ' +
+  'images inside User turns were provided by users. ' +
+  'Generate the image for the next Assistant turn.';
+
 // ============================================================================
 // OpenAI Images Adapter
 // ============================================================================
@@ -130,12 +179,21 @@ export class OpenAIResponsesAdapter implements ProviderAdapter {
   private baseURL: string;
   private organization?: string;
   private allowImageEditing: boolean;
+  private promptFormat: 'legacy' | 'transcript';
+  private maxInputImages: number;
+  private transcriptPreamble: string;
 
   constructor(config: OpenAIResponsesAdapterConfig = {}) {
     this.apiKey = config.apiKey ?? process.env.OPENAI_API_KEY ?? '';
     this.baseURL = (config.baseURL ?? 'https://api.openai.com/v1').replace(/\/$/, '');
     this.organization = config.organization;
     this.allowImageEditing = config.allowImageEditing ?? true;
+    this.promptFormat = config.promptFormat ?? 'legacy';
+    this.maxInputImages = Math.max(
+      1,
+      Math.min(config.maxInputImages ?? IMAGES_EDIT_MAX_INPUT_IMAGES, IMAGES_EDIT_MAX_INPUT_IMAGES)
+    );
+    this.transcriptPreamble = config.transcriptPreamble ?? DEFAULT_TRANSCRIPT_PREAMBLE;
 
     if (!this.apiKey) {
       throw new Error('OpenAI API key not provided');
@@ -150,10 +208,12 @@ export class OpenAIResponsesAdapter implements ProviderAdapter {
     request: ProviderRequest,
     options?: ProviderRequestOptions
   ): Promise<ProviderResponse> {
-    const inputImages = this.allowImageEditing ? this.extractImages(request) : [];
+    const allImages = this.allowImageEditing ? this.collectImages(request) : [];
+    const selected = this.selectImages(allImages);
+    const inputImages = selected.map(ref => ref.dataUrl);
     const isEdit = inputImages.length > 0;
     const endpoint = isEdit ? 'images/edits' : 'images/generations';
-    const imagesRequest = this.buildRequest(request, inputImages);
+    const imagesRequest = this.buildRequest(request, selected);
     options?.onRequest?.(imagesRequest);
 
     const { signal: combinedSignal, cleanup } = createCombinedSignal(options?.signal, options?.timeoutMs);
@@ -245,33 +305,47 @@ export class OpenAIResponsesAdapter implements ProviderAdapter {
   }
 
   /**
-   * Extract base64 images from conversation messages as data URLs.
-   * Used to determine whether to use /edits (with images) or /generations.
-   * Returns up to 16 images (OpenAI limit for /v1/images/edits).
+   * Collect every base64 image in the conversation, in order, remembering
+   * which message/block it came from so the transcript can reference it.
+   * Both `image` blocks and `generated_image` blocks (this adapter's own
+   * earlier outputs, when a consumer carries them forward as-is) count.
    */
-  private extractImages(request: ProviderRequest): string[] {
-    const dataUrls: string[] = [];
-    const MAX_IMAGES = 16;
-
-    if (!request.messages) return dataUrls;
-
-    for (const msg of request.messages as any[]) {
-      if (!Array.isArray(msg.content)) continue;
-
-      for (const block of msg.content) {
-        if (dataUrls.length >= MAX_IMAGES) break;
-
-        if (block.type === 'image') {
+  private collectImages(request: ProviderRequest): ImageRef[] {
+    const refs: ImageRef[] = [];
+    if (!request.messages) return refs;
+
+    (request.messages as any[]).forEach((msg, msgIndex) => {
+      if (!Array.isArray(msg.content)) return;
+      msg.content.forEach((block: any, blockIndex: number) => {
+        if (block?.type === 'image') {
           const source = block.source;
           if (source?.type === 'base64' && source.data) {
             const mimeType = source.media_type ?? source.mediaType ?? 'image/png';
-            dataUrls.push(`data:${mimeType};base64,${source.data}`);
+            refs.push({ dataUrl: `data:${mimeType};base64,${source.data}`, msgIndex, blockIndex });
           }
+        } else if (block?.type === 'generated_image' && typeof block.data === 'string' && block.data) {
+          // A previous output of this adapter carried forward verbatim in
+          // history (consumers that keep ProviderResponse content rather
+          // than rebuilding from a channel). It is an image like any other.
+          const mimeType = block.mimeType ?? 'image/png';
+          refs.push({ dataUrl: `data:${mimeType};base64,${block.data}`, msgIndex, blockIndex });
         }
-      }
-    }
+      });
+    });
 
-    return dataUrls;
+    return refs;
+  }
+
+  /**
+   * Apply the input-image cap. Legacy keeps the FIRST N (its historical
+   * behaviour); transcript keeps the MOST RECENT N, since the latest turns
+   * are what an edit request is about.
+   */
+  private selectImages(refs: ImageRef[]): ImageRef[] {
+    if (refs.length <= this.maxInputImages) return refs;
+    return this.promptFormat === 'transcript'
+      ? refs.slice(refs.length - this.maxInputImages)
+      : refs.slice(0, this.maxInputImages);
   }
 
   /**
@@ -288,8 +362,11 @@ export class OpenAIResponsesAdapter implements ProviderAdapter {
     };
   }
 
-  private buildRequest(request: ProviderRequest, inputImages: string[]): ImagesRequest {
-    const prompt = this.flattenToPrompt(request);
+  private buildRequest(request: ProviderRequest, selectedImages: ImageRef[]): ImagesRequest {
+    const inputImages = selectedImages.map(ref => ref.dataUrl);
+    const prompt = this.promptFormat === 'transcript'
+      ? this.flattenToTranscript(request, selectedImages)
+      : this.flattenToPrompt(request);
 
     const imagesRequest: ImagesRequest = {
       model: request.model,
@@ -362,6 +439,78 @@ export class OpenAIResponsesAdapter implements ProviderAdapter {
     return parts.join('\n\n');
   }
 
+  private systemText(request: ProviderRequest): string {
+    if (!request.system) return '';
+    if (typeof request.system === 'string') return request.system;
+    return (request.system as any[])
+      .filter((b: any) => b.type === 'text')
+      .map((b: any) => b.text)
+      .join('\n');
+  }
+
+  /**
+   * Flatten the conversation into a role-delimited transcript.
+   *
+   * Each message becomes (part of) a `### User` / `### Assistant` turn;
+   * consecutive same-role messages share one header. Text blocks are kept
+   * verbatim. Image blocks become `[Image N]` (N = 1-based position in the
+   * attached `image[]` list) when the image is attached, or
+   * `[Image omitted]` when it fell outside the input-image cap. An
+   * assistant message that only carried an image therefore still appears
+   * as an assistant turn — that is what lets a previous generation read as
+   * the assistant's own prior output. The prompt ends with an open
+   * `### Assistant` header for the turn being generated.
+   */
+  private flattenToTranscript(request: ProviderRequest, selectedImages: ImageRef[]): string {
+    const attachedIndex = new Map<string, number>();
+    selectedImages.forEach((ref, i) => attachedIndex.set(`${ref.msgIndex}:${ref.blockIndex}`, i + 1));
+
+    const sections: string[] = [];
+    const systemText = this.systemText(request);
+    if (systemText) sections.push(systemText);
+    if (this.transcriptPreamble) sections.push(this.transcriptPreamble);
+
+    const turns: string[] = [];
+    let currentRole: 'User' | 'Assistant' | undefined;
+    let currentLines: string[] = [];
+    const flushTurn = () => {
+      if (!currentRole) return;
+      const body = currentLines.join('\n').trim();
+      if (body) turns.push(`### ${currentRole}\n${body}`);
+      currentLines = [];
+    };
+
+    (request.messages as any[] | undefined)?.forEach((msg, msgIndex) => {
+      const role: 'User' | 'Assistant' = msg.role === 'assistant' ? 'Assistant' : 'User';
+      const lines: string[] = [];
+
+      if (typeof msg.content === 'string') {
+        if (msg.content) lines.push(msg.content);
+      } else if (Array.isArray(msg.content)) {
+        msg.content.forEach((block: any, blockIndex: number) => {
+          if (block?.type === 'text' && block.text) {
+            lines.push(block.text);
+          } else if (block?.type === 'image' || block?.type === 'generated_image') {
+            const n = attachedIndex.get(`${msgIndex}:${blockIndex}`);
+            lines.push(n !== undefined ? `[Image ${n}]` : '[Image omitted]');
+          }
+        });
+      }
+      if (lines.length === 0) return;
+
+      if (role !== currentRole) {
+        flushTurn();
+        currentRole = role;
+      }
+      currentLines.push(...lines);
+    });
+    flushTurn();
+
+    sections.push(turns.join('\n\n'));
+    sections.push('### Assistant');
+    return sections.filter(Boolean).join('\n\n');
+  }
+
   // --------------------------------------------------------------------------
   // Response Parsing
   // --------------------------------------------------------------------------

package/src/providers/responses-input.ts

@@ -0,0 +1,131 @@
+import type { ProviderRequest } from '../types/index.js';
+import type { OpenAIResponsesInputItem } from './openai-responses-api.js';
+
+type JsonObject = Record<string, unknown>;
+
+/**
+ * Most agent turns arrive already formatted as provider-native Responses
+ * items. Internal maintenance calls, however, can bypass that formatter and
+ * carry Membrane's normalized `text`/`image`/tool blocks. Normalize at the
+ * final transport boundary so every call shape accepted by ProviderAdapter is
+ * valid on the Codex Responses endpoint.
+ */
+export function normalizeResponsesInput(messages: ProviderRequest['messages']): OpenAIResponsesInputItem[] {
+  const output: unknown[] = [];
+
+  for (const rawMessage of messages as unknown[]) {
+    if (!isObject(rawMessage)) {
+      output.push(rawMessage);
+      continue;
+    }
+    if (rawMessage.type !== 'message' && rawMessage.role === undefined) {
+      output.push(normalizeStandaloneItem(rawMessage));
+      continue;
+    }
+
+    // Native messages (including phase, status and developer/system roles)
+    // must survive replay verbatim. Only translate normalized content blocks.
+    if (Array.isArray(rawMessage.content) && !rawMessage.content.some((block) =>
+      isObject(block) && ['text', 'image', 'tool_use', 'tool_result', 'redacted_thinking'].includes(asString(block.type))
+    )) {
+      output.push(rawMessage);
+      continue;
+    }
+    const role = typeof rawMessage.role === 'string' ? rawMessage.role : 'user';
+    const blocks = Array.isArray(rawMessage.content)
+      ? rawMessage.content
+      : typeof rawMessage.content === 'string'
+        ? [{ type: 'text', text: rawMessage.content }]
+        : [];
+    let parts: unknown[] = [];
+    const flush = () => {
+      if (parts.length === 0) return;
+      output.push({
+        type: 'message',
+        ...rawMessage,
+        role,
+        content: parts,
+      });
+      parts = [];
+    };
+
+    for (const rawBlock of blocks) {
+      if (!isObject(rawBlock)) continue;
+      if (rawBlock.type === 'text') {
+        parts.push({ type: role === 'assistant' ? 'output_text' : 'input_text', text: asString(rawBlock.text) });
+      } else if (rawBlock.type === 'image') {
+        const imageUrl = responsesImageUrl(rawBlock);
+        if (imageUrl && role !== 'assistant') parts.push({ type: 'input_image', image_url: imageUrl });
+      } else if (rawBlock.type === 'tool_use') {
+        flush();
+        output.push(normalizeStandaloneItem(rawBlock));
+      } else if (rawBlock.type === 'tool_result') {
+        flush();
+        output.push(normalizeStandaloneItem(rawBlock));
+      } else if (rawBlock.type === 'redacted_thinking') {
+        flush();
+        output.push(reasoningInputItem(rawBlock));
+      } else {
+        // Already-native input_text/output_text/input_image/refusal parts.
+        parts.push(rawBlock);
+      }
+    }
+    flush();
+  }
+
+  return output as OpenAIResponsesInputItem[];
+}
+
+function normalizeStandaloneItem(item: JsonObject): unknown {
+  if (item.type === 'tool_use') {
+    return {
+      type: 'function_call',
+      call_id: asString(item.id),
+      name: asString(item.name),
+      arguments: JSON.stringify(isObject(item.input) ? item.input : {}),
+    };
+  }
+  if (item.type === 'tool_result') {
+    const content = item.content;
+    return {
+      type: 'function_call_output',
+      call_id: asString(item.toolUseId) || asString(item.tool_use_id),
+      output: typeof content === 'string' ? content : JSON.stringify(content ?? null),
+    };
+  }
+  if (item.type === 'redacted_thinking') {
+    return reasoningInputItem(item);
+  }
+  return item;
+}
+
+/** Replay a captured reasoning carrier as a Responses input item.
+ *
+ * Prefer the provider-native item verbatim when the block still carries it
+ * (`rawItem` from response parsing). Otherwise reconstruct the minimum the
+ * Responses API accepts: `summary` is a REQUIRED field on reasoning input
+ * items (empty array = "no summaries") — omitting it 400s with
+ * "Missing required parameter: 'input[N].summary'". */
+function reasoningInputItem(block: JsonObject): unknown {
+  const raw = block.rawItem;
+  if (isObject(raw) && raw.type === 'reasoning') return raw;
+  return { type: 'reasoning', summary: [], encrypted_content: asString(block.data) };
+}
+
+function responsesImageUrl(block: JsonObject): string | undefined {
+  const source = isObject(block.source) ? block.source : undefined;
+  if (!source) return typeof block.image_url === 'string' ? block.image_url : undefined;
+  if (source.type === 'url') return asString(source.url) || undefined;
+  if (source.type !== 'base64') return undefined;
+  const mediaType = asString(source.mediaType) || asString(source.media_type) || 'image/png';
+  const data = asString(source.data);
+  return data ? `data:${mediaType};base64,${data}` : undefined;
+}
+
+function asString(value: unknown): string {
+  return typeof value === 'string' ? value : '';
+}
+
+function isObject(value: unknown): value is JsonObject {
+  return Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+}

package/src/providers/utils.ts

@@ -291,6 +291,9 @@ export function createCombinedSignal(
  */
 export class SSELineParser {
   private buffer: string = '';
+  private data: string[] = [];
+
+  constructor(private readonly options: { multiline?: boolean } = {}) {}
 
   /**
    * Feed a raw chunk from the stream reader and get back complete SSE data lines.
@@ -305,6 +308,10 @@ export class SSELineParser {
     this.buffer = lines.pop() || '';
 
     for (const line of lines) {
+      if (this.options.multiline) {
+        this.processEventLine(line.replace(/\r$/, ''), results);
+        continue;
+      }
       const trimmed = line.trim();
       if (trimmed.startsWith('data: ')) {
         results.push(trimmed.slice(6));
@@ -315,10 +322,24 @@ export class SSELineParser {
     return results;
   }
 
-  /**
-   * Flush any remaining buffered content (call when stream ends).
-   */
+  private processEventLine(line: string, results: string[]): void {
+    if (line === '') {
+      if (this.data.length) results.push(this.data.join('\n'));
+      this.data = [];
+    } else if (line.startsWith('data:')) {
+      this.data.push(line.slice(5).replace(/^ /, ''));
+    }
+  }
+
+  /** Flush any remaining event/tail when the stream ends. */
   flush(): string[] {
+    if (this.options.multiline) {
+      const results: string[] = [];
+      if (this.buffer) this.processEventLine(this.buffer.replace(/\r$/, ''), results);
+      this.buffer = '';
+      this.processEventLine('', results);
+      return results;
+    }
     if (!this.buffer.trim()) return [];
     const trimmed = this.buffer.trim();
     this.buffer = '';

package/src/types/provider.ts

@@ -207,6 +207,11 @@ export interface ProviderAdapter {
    */
   usageCacheConvention?: UsageCacheConvention;
 
+  /** Whether this transport requires the configured Responses formatter.
+   * False permits generic per-request formatter overrides (e.g. named
+   * maintenance messages). Wrappers must forward this capability. */
+  readonly requiresNativeResponsesInput?: boolean;
+
   /** Check if this adapter handles a model */
   supportsModel(modelId: string): boolean;