@langchain/openai 0.3.9 → 0.3.11

package/dist/chat_models.cjs

@@ -78,9 +78,11 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
-            const content = message.audio ? [message.audio] : message.content;
+            if (message.audio) {
+                additional_kwargs.audio = message.audio;
+            }
             return new messages_1.AIMessage({
-                content: content || "",
+                content: message.content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -96,18 +98,7 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    let content;
-    if (delta.audio) {
-        content = [
-            {
-                ...delta.audio,
-                index: rawResponse.choices[0].index,
-            },
-        ];
-    }
-    else {
-        content = delta.content ?? "";
-    }
+    const content = delta.content ?? "";
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -125,6 +116,12 @@ delta, rawResponse, defaultRole, includeRawResponse) {
     if (includeRawResponse) {
         additional_kwargs.__raw_response = rawResponse;
     }
+    if (delta.audio) {
+        additional_kwargs.audio = {
+            ...delta.audio,
+            index: rawResponse.choices[0].index,
+        };
+    }
     const response_metadata = { usage: { ...rawResponse.usage } };
     if (role === "user") {
         return new messages_1.HumanMessageChunk({ content, response_metadata });
@@ -176,7 +173,7 @@ delta, rawResponse, defaultRole, includeRawResponse) {
 // Used in LangSmith, export is important here
 function _convertMessagesToOpenAIParams(messages) {
     // TODO: Function messages do not support array content, fix cast
-    return messages.map((message) => {
+    return messages.flatMap((message) => {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role: messageToOpenAIRole(message),
@@ -201,6 +198,17 @@ function _convertMessagesToOpenAIParams(messages) {
                 completionParam.tool_call_id = message.tool_call_id;
             }
         }
+        if (message.additional_kwargs.audio &&
+            typeof message.additional_kwargs.audio === "object" &&
+            "id" in message.additional_kwargs.audio) {
+            const audioMessage = {
+                role: "assistant",
+                audio: {
+                    id: message.additional_kwargs.audio.id,
+                },
+            };
+            return [completionParam, audioMessage];
+        }
         return completionParam;
     });
 }
@@ -718,6 +726,43 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 class ChatOpenAI extends chat_models_1.BaseChatModel {
     static lc_name() {
@@ -1202,6 +1247,10 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
+            if (typeof chunk.content !== "string") {
+                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
+                continue;
+            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1215,7 +1264,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generationChunk = new outputs_1.ChatGenerationChunk({
                 message: chunk,
-                text: typeof chunk.content === "string" ? chunk.content : "",
+                text: chunk.content,
                 generationInfo,
             });
             yield generationChunk;
@@ -1374,8 +1423,9 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
+                const text = part.message?.content ?? "";
                 const generation = {
-                    text: part.message?.content ?? "",
+                    text,
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/dist/chat_models.d.ts

@@ -18,8 +18,9 @@ interface OpenAILLMOutput {
     tokenUsage: TokenUsage;
 }
 type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool";
+type OpenAICompletionParam = OpenAIClient.Chat.Completions.ChatCompletionMessageParam;
 export declare function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum;
-export declare function _convertMessagesToOpenAIParams(messages: BaseMessage[]): OpenAIClient.Chat.Completions.ChatCompletionMessageParam[];
+export declare function _convertMessagesToOpenAIParams(messages: BaseMessage[]): OpenAICompletionParam[];
 type ChatOpenAIToolType = BindToolsInput | OpenAIClient.ChatCompletionTool;
 export interface ChatOpenAIStructuredOutputMethodOptions<IncludeRaw extends boolean> extends StructuredOutputMethodOptions<IncludeRaw> {
     /**
@@ -596,6 +597,43 @@ export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<Azur
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = ChatOpenAICallOptions> extends BaseChatModel<CallOptions, AIMessageChunk> implements OpenAIChatInput, AzureOpenAIInput {
     static lc_name(): string;

package/dist/chat_models.js

@@ -74,9 +74,11 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
-            const content = message.audio ? [message.audio] : message.content;
+            if (message.audio) {
+                additional_kwargs.audio = message.audio;
+            }
             return new AIMessage({
-                content: content || "",
+                content: message.content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -92,18 +94,7 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    let content;
-    if (delta.audio) {
-        content = [
-            {
-                ...delta.audio,
-                index: rawResponse.choices[0].index,
-            },
-        ];
-    }
-    else {
-        content = delta.content ?? "";
-    }
+    const content = delta.content ?? "";
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -121,6 +112,12 @@ delta, rawResponse, defaultRole, includeRawResponse) {
     if (includeRawResponse) {
         additional_kwargs.__raw_response = rawResponse;
     }
+    if (delta.audio) {
+        additional_kwargs.audio = {
+            ...delta.audio,
+            index: rawResponse.choices[0].index,
+        };
+    }
     const response_metadata = { usage: { ...rawResponse.usage } };
     if (role === "user") {
         return new HumanMessageChunk({ content, response_metadata });
@@ -172,7 +169,7 @@ delta, rawResponse, defaultRole, includeRawResponse) {
 // Used in LangSmith, export is important here
 export function _convertMessagesToOpenAIParams(messages) {
     // TODO: Function messages do not support array content, fix cast
-    return messages.map((message) => {
+    return messages.flatMap((message) => {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role: messageToOpenAIRole(message),
@@ -197,6 +194,17 @@ export function _convertMessagesToOpenAIParams(messages) {
                 completionParam.tool_call_id = message.tool_call_id;
             }
         }
+        if (message.additional_kwargs.audio &&
+            typeof message.additional_kwargs.audio === "object" &&
+            "id" in message.additional_kwargs.audio) {
+            const audioMessage = {
+                role: "assistant",
+                audio: {
+                    id: message.additional_kwargs.audio.id,
+                },
+            };
+            return [completionParam, audioMessage];
+        }
         return completionParam;
     });
 }
@@ -713,6 +721,43 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export class ChatOpenAI extends BaseChatModel {
     static lc_name() {
@@ -1197,6 +1242,10 @@ export class ChatOpenAI extends BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
+            if (typeof chunk.content !== "string") {
+                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
+                continue;
+            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1210,7 +1259,7 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generationChunk = new ChatGenerationChunk({
                 message: chunk,
-                text: typeof chunk.content === "string" ? chunk.content : "",
+                text: chunk.content,
                 generationInfo,
             });
             yield generationChunk;
@@ -1369,8 +1418,9 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
+                const text = part.message?.content ?? "";
                 const generation = {
-                    text: part.message?.content ?? "",
+                    text,
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.3.9",
+  "version": "0.3.11",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {