llm-messages 0.2.0 → 0.4.0

package/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project are documented here. The format is based on
 [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and this project adheres
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - 2026-06-01
+
+### Added
+
+- Audio and document content parts. Audio (OpenAI `input_audio`) converts between
+  OpenAI and Gemini; Anthropic has no audio input, so audio is dropped with an
+  `unsupported-modality` warning. Documents (OpenAI `file`, Anthropic `document`,
+  Gemini `inlineData` / `fileData`) convert across all three, base64 losslessly.
+  Adds the `MediaPart` type and `unsupported-modality` / `gemini-url-media`
+  warning codes. (#5)
+
+## [0.3.0] - 2026-06-01
+
+### Added
+
+- Response normalization. `responseFromOpenAI`, `responseFromAnthropic`,
+  `responseFromGemini` and `normalizeResponse(body, { from })` parse a provider
+  response body into a canonical OpenAI assistant message plus a neutral
+  `finishReason` and `usage` (`inputTokens` / `outputTokens`). Tool-call arguments
+  are serialized to JSON strings; `finishReason` becomes `tool_calls` whenever a
+  tool was called (including Gemini, which reports `STOP`); Gemini tool calls
+  without an id get a deterministic one. (#4)
+
 ## [0.2.0] - 2026-06-01
 
 ### Added

package/README.md

@@ -102,6 +102,27 @@ toGemini(messages, {
 Warning codes: `generated-id`, `unmapped-tool-result`, `merged-role`,
 `dropped-content`, `invalid-json-arguments`, `system-midstream`.
 
+## Reading responses
+
+The same idea applies to the read side. Normalize a provider's response body into
+a canonical OpenAI assistant message, plus a neutral finish reason and token usage:
+
+```ts
+import { responseFromAnthropic, normalizeResponse } from 'llm-messages';
+
+const { message, finishReason, usage } = responseFromAnthropic(anthropicResponseBody);
+// message     -> { role: 'assistant', content, tool_calls? }  (tool input re-serialized to a JSON string)
+// finishReason -> 'stop' | 'tool_calls' | 'length' | 'content_filter' | 'unknown'
+// usage       -> { inputTokens, outputTokens }
+
+// Or dispatch by provider:
+normalizeResponse(geminiResponseBody, { from: 'gemini' });
+```
+
+`finishReason` is normalized to `tool_calls` whenever the model called a tool, even
+for Gemini (which reports `STOP`). Gemini tool calls without an id get a
+deterministic one.
+
 ## Format cheatsheet
 
 |                  | OpenAI                   | Anthropic                        | Gemini                          |
@@ -114,7 +135,7 @@ Warning codes: `generated-id`, `unmapped-tool-result`, `merged-role`,
 | Match key        | `tool_call_id`           | `tool_use_id`                    | function `name` (id optional)   |
 | Role alternation | not required             | strict                           | strict                          |
 
-## Images
+## Images, audio and documents
 
 Image parts convert across all three providers:
 
@@ -138,11 +159,16 @@ Base64 data URLs round trip losslessly. A remote `https` URL maps to an Anthropi
 `gemini-url-image` warning, since Gemini may require the Files API for non-Google
 URIs.
 
+**Audio** (`input_audio`) and **documents** (`file`, e.g. PDF) convert too. Audio
+moves between OpenAI and Gemini; Anthropic has no audio input, so an audio part is
+dropped with an `unsupported-modality` warning. Documents convert across all three
+(OpenAI `file`, Anthropic `document`, Gemini `inlineData`).
+
 ## Scope
 
-Version 0.x covers text, system prompts, tool calls/results and images, which is
-the core of every agent loop. Other modalities (audio, files) are passed through
-where possible and reported as `dropped-content` otherwise.
+Version 0.x covers text, system prompts, tool calls/results, images, audio and
+documents, which is the core of every agent loop. Unsupported parts are reported
+via `dropped-content` rather than failing.
 
 ## Part of a set
 

package/dist/index.cjs

@@ -23,7 +23,11 @@ __export(index_exports, {
   convert: () => convert,
   fromAnthropic: () => fromAnthropic,
   fromGemini: () => fromGemini,
+  normalizeResponse: () => normalizeResponse,
   parseDataUrl: () => parseDataUrl,
+  responseFromAnthropic: () => responseFromAnthropic,
+  responseFromGemini: () => responseFromGemini,
+  responseFromOpenAI: () => responseFromOpenAI,
   toAnthropic: () => toAnthropic,
   toDataUrl: () => toDataUrl,
   toGemini: () => toGemini
@@ -124,13 +128,16 @@ function imageToAnthropic(image) {
 function imageFromGemini(part) {
   if (isRecord(part) && isRecord(part.inlineData)) {
     const data = part.inlineData;
-    if (typeof data.mimeType === "string" && typeof data.data === "string") {
+    if (typeof data.mimeType === "string" && typeof data.data === "string" && data.mimeType.startsWith("image/")) {
       return { kind: "base64", mediaType: data.mimeType, data: data.data };
     }
   }
   if (isRecord(part) && isRecord(part.fileData)) {
     const data = part.fileData;
-    if (typeof data.fileUri === "string") return { kind: "url", url: data.fileUri };
+    const mime = typeof data.mimeType === "string" ? data.mimeType : "";
+    if (typeof data.fileUri === "string" && (mime === "" || mime.startsWith("image/"))) {
+      return { kind: "url", url: data.fileUri };
+    }
   }
   return null;
 }
@@ -145,6 +152,124 @@ function imageToGemini(image, reporter) {
   return { fileData: { fileUri: image.url } };
 }
 
+// src/media.ts
+function modalityFromMime(mediaType) {
+  return mediaType.startsWith("audio/") ? "audio" : "document";
+}
+function mediaFromOpenAI(part) {
+  if (!isRecord(part)) return null;
+  if (part.type === "input_audio" && isRecord(part.input_audio)) {
+    const audio = part.input_audio;
+    if (typeof audio.data === "string") {
+      const format = typeof audio.format === "string" ? audio.format : "wav";
+      return { modality: "audio", source: { kind: "base64", mediaType: `audio/${format}`, data: audio.data } };
+    }
+  }
+  if (part.type === "file" && isRecord(part.file)) {
+    const file = part.file;
+    const filename = typeof file.filename === "string" ? file.filename : void 0;
+    if (typeof file.file_data === "string") {
+      const parsed = parseDataUrl(file.file_data);
+      if (parsed) return { modality: "document", source: { kind: "base64", ...parsed }, filename };
+    }
+    if (typeof file.file_id === "string") {
+      return { modality: "document", source: { kind: "file_id", id: file.file_id }, filename };
+    }
+  }
+  return null;
+}
+function mediaToOpenAI(media) {
+  const { modality, source } = media;
+  if (modality === "audio") {
+    if (source.kind !== "base64") return null;
+    const audio = {
+      type: "input_audio",
+      input_audio: { data: source.data, format: source.mediaType.replace(/^audio\//, "") }
+    };
+    return audio;
+  }
+  if (source.kind === "base64") {
+    const file = {
+      type: "file",
+      file: {
+        file_data: toDataUrl(source.mediaType, source.data),
+        ...media.filename ? { filename: media.filename } : {}
+      }
+    };
+    return file;
+  }
+  if (source.kind === "file_id") {
+    const file = {
+      type: "file",
+      file: { file_id: source.id, ...media.filename ? { filename: media.filename } : {} }
+    };
+    return file;
+  }
+  return null;
+}
+function mediaFromAnthropic(block) {
+  if (!isRecord(block) || block.type !== "document" || !isRecord(block.source)) return null;
+  const source = block.source;
+  if (source.type === "base64" && typeof source.media_type === "string" && typeof source.data === "string") {
+    return { modality: "document", source: { kind: "base64", mediaType: source.media_type, data: source.data } };
+  }
+  if (source.type === "url" && typeof source.url === "string") {
+    return { modality: "document", source: { kind: "url", url: source.url } };
+  }
+  if (source.type === "file" && typeof source.file_id === "string") {
+    return { modality: "document", source: { kind: "file_id", id: source.file_id } };
+  }
+  return null;
+}
+function mediaToAnthropic(media, reporter) {
+  if (media.modality === "audio") {
+    reporter.warn("unsupported-modality", "Anthropic has no audio input; dropped an audio part.");
+    return null;
+  }
+  const { source } = media;
+  if (source.kind === "base64") {
+    return { type: "document", source: { type: "base64", media_type: source.mediaType, data: source.data } };
+  }
+  if (source.kind === "url") {
+    return { type: "document", source: { type: "url", url: source.url } };
+  }
+  return { type: "document", source: { type: "file", file_id: source.id } };
+}
+function mediaFromGemini(part) {
+  if (isRecord(part) && isRecord(part.inlineData)) {
+    const data = part.inlineData;
+    if (typeof data.mimeType === "string" && typeof data.data === "string" && !data.mimeType.startsWith("image/")) {
+      return {
+        modality: modalityFromMime(data.mimeType),
+        source: { kind: "base64", mediaType: data.mimeType, data: data.data }
+      };
+    }
+  }
+  if (isRecord(part) && isRecord(part.fileData)) {
+    const data = part.fileData;
+    const mime = typeof data.mimeType === "string" ? data.mimeType : "";
+    if (typeof data.fileUri === "string" && mime !== "" && !mime.startsWith("image/")) {
+      return { modality: modalityFromMime(mime), source: { kind: "url", url: data.fileUri } };
+    }
+  }
+  return null;
+}
+function mediaToGemini(media, reporter) {
+  const { source } = media;
+  if (source.kind === "base64") {
+    return { inlineData: { mimeType: source.mediaType, data: source.data } };
+  }
+  if (source.kind === "url") {
+    reporter.warn(
+      "gemini-url-media",
+      "A media URL was emitted as Gemini fileData.fileUri; Gemini may require the Files API for non-Google URIs."
+    );
+    return { fileData: { fileUri: source.url } };
+  }
+  reporter.warn("unsupported-modality", "Gemini has no file-id media reference; dropped a file_id part.");
+  return null;
+}
+
 // src/providers/openai.ts
 function isSystem(message) {
   return message.role === "system" || message.role === "developer";
@@ -213,6 +338,12 @@ function userContent(content, reporter) {
       blocks.push(imageToAnthropic(image));
       continue;
     }
+    const media = mediaFromOpenAI(part);
+    if (media) {
+      const block = mediaToAnthropic(media, reporter);
+      if (block) blocks.push(block);
+      continue;
+    }
     reporter.warn("dropped-content", "Dropped an unsupported user content part.");
   }
   return blocks;
@@ -254,7 +385,7 @@ function asBlocks(content) {
   return content ? [{ type: "text", text: content }] : [];
 }
 function fromAnthropic(conversation, options = {}) {
-  void options;
+  const reporter = new Reporter(options);
   const out = [];
   if (conversation.system) {
     out.push({ role: "system", content: textOf(conversation.system) });
@@ -272,7 +403,7 @@ function fromAnthropic(conversation, options = {}) {
         });
       }
       if (contentBlocks.length > 0) {
-        out.push({ role: "user", content: userContentToOpenAI(contentBlocks) });
+        out.push({ role: "user", content: userContentToOpenAI(contentBlocks, reporter) });
       }
       continue;
     }
@@ -289,15 +420,24 @@ function fromAnthropic(conversation, options = {}) {
   }
   return out;
 }
-function userContentToOpenAI(blocks) {
-  const hasImage = blocks.some((block) => imageFromAnthropic(block) !== null);
-  if (!hasImage) return textOf(blocks);
+function userContentToOpenAI(blocks, reporter) {
+  const hasMedia = blocks.some((block) => imageFromAnthropic(block) !== null || mediaFromAnthropic(block) !== null);
+  if (!hasMedia) return textOf(blocks);
   const parts = [];
   for (const block of blocks) {
     const image = imageFromAnthropic(block);
     if (image) {
       parts.push(imageToOpenAI(image));
-    } else if (isRecord(block) && block.type === "text" && typeof block.text === "string") {
+      continue;
+    }
+    const media = mediaFromAnthropic(block);
+    if (media) {
+      const part = mediaToOpenAI(media);
+      if (part) parts.push(part);
+      else reporter.warn("dropped-content", "A document URL has no OpenAI Chat Completions equivalent; dropped.");
+      continue;
+    }
+    if (isRecord(block) && block.type === "text" && typeof block.text === "string") {
       parts.push({ type: "text", text: block.text });
     }
   }
@@ -364,6 +504,12 @@ function userParts(content, reporter) {
       parts.push(imageToGemini(image, reporter));
       continue;
     }
+    const media = mediaFromOpenAI(part);
+    if (media) {
+      const geminiPart = mediaToGemini(media, reporter);
+      if (geminiPart) parts.push(geminiPart);
+      continue;
+    }
     reporter.warn("dropped-content", "Dropped an unsupported user content part.");
   }
   return parts.length > 0 ? parts : [{ text: "" }];
@@ -433,7 +579,7 @@ function fromGemini(conversation, options = {}) {
       continue;
     }
     const contentParts = [];
-    let hasImage = false;
+    let hasMedia = false;
     for (const part of parts) {
       if (isRecord(part) && isRecord(part.functionResponse)) {
         const fr = part.functionResponse;
@@ -444,7 +590,16 @@ function fromGemini(conversation, options = {}) {
       const image = imageFromGemini(part);
       if (image) {
         contentParts.push(imageToOpenAI(image));
-        hasImage = true;
+        hasMedia = true;
+        continue;
+      }
+      const media = mediaFromGemini(part);
+      if (media) {
+        const openaiPart = mediaToOpenAI(media);
+        if (openaiPart) {
+          contentParts.push(openaiPart);
+          hasMedia = true;
+        }
         continue;
       }
       if (isRecord(part) && typeof part.text === "string") {
@@ -452,7 +607,7 @@ function fromGemini(conversation, options = {}) {
       }
     }
     if (contentParts.length > 0) {
-      if (hasImage) {
+      if (hasMedia) {
         out.push({ role: "user", content: contentParts });
       } else {
         const text = textOf(contentParts);
@@ -511,12 +666,129 @@ function fromCanonical(canonical, to, options) {
       throw new Error(`Unknown target provider: ${String(to)}`);
   }
 }
+
+// src/response.ts
+var num = (value) => typeof value === "number" ? value : 0;
+function buildMessage(text, toolCalls) {
+  const message = { role: "assistant", content: text ? text : null };
+  if (toolCalls.length > 0) message.tool_calls = toolCalls;
+  return message;
+}
+function finalReason(mapped, toolCalls) {
+  return toolCalls.length > 0 ? "tool_calls" : mapped;
+}
+var OPENAI_FINISH = {
+  stop: "stop",
+  length: "length",
+  tool_calls: "tool_calls",
+  content_filter: "content_filter",
+  function_call: "tool_calls"
+};
+function responseFromOpenAI(body) {
+  const root = isRecord(body) ? body : {};
+  const choice = Array.isArray(root.choices) && isRecord(root.choices[0]) ? root.choices[0] : {};
+  const message = isRecord(choice.message) ? choice.message : {};
+  const text = typeof message.content === "string" ? message.content : textOf(message.content);
+  const toolCalls = Array.isArray(message.tool_calls) ? message.tool_calls : [];
+  const usage = isRecord(root.usage) ? root.usage : {};
+  return {
+    message: buildMessage(text, toolCalls),
+    finishReason: finalReason(OPENAI_FINISH[String(choice.finish_reason)] ?? "unknown", toolCalls),
+    usage: { inputTokens: num(usage.prompt_tokens), outputTokens: num(usage.completion_tokens) }
+  };
+}
+var ANTHROPIC_FINISH = {
+  end_turn: "stop",
+  stop_sequence: "stop",
+  tool_use: "tool_calls",
+  max_tokens: "length",
+  refusal: "content_filter",
+  pause_turn: "unknown"
+};
+function responseFromAnthropic(body) {
+  const root = isRecord(body) ? body : {};
+  const blocks = Array.isArray(root.content) ? root.content : [];
+  const textPieces = [];
+  const toolCalls = [];
+  for (const block of blocks) {
+    if (!isRecord(block)) continue;
+    if (block.type === "text" && typeof block.text === "string") {
+      textPieces.push(block.text);
+    } else if (block.type === "tool_use" && typeof block.name === "string") {
+      toolCalls.push({
+        id: typeof block.id === "string" ? block.id : "",
+        type: "function",
+        function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) }
+      });
+    }
+  }
+  const usage = isRecord(root.usage) ? root.usage : {};
+  return {
+    message: buildMessage(textPieces.join(""), toolCalls),
+    finishReason: finalReason(ANTHROPIC_FINISH[String(root.stop_reason)] ?? "unknown", toolCalls),
+    usage: { inputTokens: num(usage.input_tokens), outputTokens: num(usage.output_tokens) }
+  };
+}
+var GEMINI_FINISH = {
+  STOP: "stop",
+  MAX_TOKENS: "length",
+  SAFETY: "content_filter",
+  RECITATION: "content_filter",
+  MALFORMED_FUNCTION_CALL: "content_filter"
+};
+function responseFromGemini(body, options = {}) {
+  const reporter = new Reporter(options);
+  const root = isRecord(body) ? body : {};
+  const candidate = Array.isArray(root.candidates) && isRecord(root.candidates[0]) ? root.candidates[0] : {};
+  const content = isRecord(candidate.content) ? candidate.content : {};
+  const parts = Array.isArray(content.parts) ? content.parts : [];
+  const textPieces = [];
+  const toolCalls = [];
+  let counter = 0;
+  for (const part of parts) {
+    if (!isRecord(part)) continue;
+    if (isRecord(part.functionCall)) {
+      const call = part.functionCall;
+      const name = typeof call.name === "string" ? call.name : "function";
+      let id = call.id;
+      if (!id) {
+        id = `call_${name.replace(/[^a-zA-Z0-9_-]/g, "_")}_${counter++}`;
+        reporter.warn("generated-id", `Gemini functionCall '${name}' had no id; generated '${id}'.`);
+      }
+      toolCalls.push({ id, type: "function", function: { name, arguments: JSON.stringify(call.args ?? {}) } });
+    } else if (typeof part.text === "string") {
+      textPieces.push(part.text);
+    }
+  }
+  const usage = isRecord(root.usageMetadata) ? root.usageMetadata : {};
+  return {
+    message: buildMessage(textPieces.join(""), toolCalls),
+    finishReason: finalReason(GEMINI_FINISH[String(candidate.finishReason)] ?? "unknown", toolCalls),
+    usage: { inputTokens: num(usage.promptTokenCount), outputTokens: num(usage.candidatesTokenCount) }
+  };
+}
+function normalizeResponse(body, route, options = {}) {
+  switch (route.from) {
+    case "openai":
+      return responseFromOpenAI(body);
+    case "anthropic":
+      return responseFromAnthropic(body);
+    case "gemini":
+      return responseFromGemini(body, options);
+    default:
+      throw new Error(`Unknown source provider: ${String(route.from)}`);
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   convert,
   fromAnthropic,
   fromGemini,
+  normalizeResponse,
   parseDataUrl,
+  responseFromAnthropic,
+  responseFromGemini,
+  responseFromOpenAI,
   toAnthropic,
   toDataUrl,
   toGemini