@langchain/openai 0.3.10 → 0.3.11

package/dist/chat_models.cjs

@@ -78,6 +78,9 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
+            if (message.audio) {
+                additional_kwargs.audio = message.audio;
+            }
             return new messages_1.AIMessage({
                 content: message.content || "",
                 tool_calls: toolCalls,
@@ -113,6 +116,12 @@ delta, rawResponse, defaultRole, includeRawResponse) {
     if (includeRawResponse) {
         additional_kwargs.__raw_response = rawResponse;
     }
+    if (delta.audio) {
+        additional_kwargs.audio = {
+            ...delta.audio,
+            index: rawResponse.choices[0].index,
+        };
+    }
     const response_metadata = { usage: { ...rawResponse.usage } };
     if (role === "user") {
         return new messages_1.HumanMessageChunk({ content, response_metadata });
@@ -164,7 +173,7 @@ delta, rawResponse, defaultRole, includeRawResponse) {
 // Used in LangSmith, export is important here
 function _convertMessagesToOpenAIParams(messages) {
     // TODO: Function messages do not support array content, fix cast
-    return messages.map((message) => {
+    return messages.flatMap((message) => {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role: messageToOpenAIRole(message),
@@ -189,6 +198,17 @@ function _convertMessagesToOpenAIParams(messages) {
                 completionParam.tool_call_id = message.tool_call_id;
             }
         }
+        if (message.additional_kwargs.audio &&
+            typeof message.additional_kwargs.audio === "object" &&
+            "id" in message.additional_kwargs.audio) {
+            const audioMessage = {
+                role: "assistant",
+                audio: {
+                    id: message.additional_kwargs.audio.id,
+                },
+            };
+            return [completionParam, audioMessage];
+        }
         return completionParam;
     });
 }
@@ -669,6 +689,80 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 class ChatOpenAI extends chat_models_1.BaseChatModel {
     static lc_name() {
@@ -912,6 +1006,18 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "audio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "modalities", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -960,6 +1066,8 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
+        this.audio = fields?.audio;
+        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1081,6 +1189,12 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
+            ...(this.audio || options?.audio
+                ? { audio: this.audio || options?.audio }
+                : {}),
+            ...(this.modalities || options?.modalities
+                ? { modalities: this.modalities || options?.modalities }
+                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1116,7 +1230,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices[0];
+            const choice = data?.choices?.[0];
             if (data.usage) {
                 usage = data.usage;
             }

package/dist/chat_models.d.ts

@@ -18,8 +18,9 @@ interface OpenAILLMOutput {
     tokenUsage: TokenUsage;
 }
 type OpenAIRoleEnum = "system" | "assistant" | "user" | "function" | "tool";
+type OpenAICompletionParam = OpenAIClient.Chat.Completions.ChatCompletionMessageParam;
 export declare function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum;
-export declare function _convertMessagesToOpenAIParams(messages: BaseMessage[]): OpenAIClient.Chat.Completions.ChatCompletionMessageParam[];
+export declare function _convertMessagesToOpenAIParams(messages: BaseMessage[]): OpenAICompletionParam[];
 type ChatOpenAIToolType = BindToolsInput | OpenAIClient.ChatCompletionTool;
 export interface ChatOpenAIStructuredOutputMethodOptions<IncludeRaw extends boolean> extends StructuredOutputMethodOptions<IncludeRaw> {
     /**
@@ -75,6 +76,25 @@ export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCa
      * @version 0.2.6
      */
     strict?: boolean;
+    /**
+     * Output types that you would like the model to generate for this request. Most
+     * models are capable of generating text, which is the default:
+     *
+     * `["text"]`
+     *
+     * The `gpt-4o-audio-preview` model can also be used to
+     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
+     * this model generate both text and audio responses, you can use:
+     *
+     * `["text", "audio"]`
+     */
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
+    /**
+     * Parameters for audio output. Required when audio output is requested with
+     * `modalities: ["audio"]`.
+     * [Learn more](https://platform.openai.com/docs/guides/audio).
+     */
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<AzureOpenAIInput>, BaseChatModelParams {
     configuration?: ClientOptions & LegacyOpenAIInput;
@@ -540,6 +560,80 @@ export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<Azur
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = ChatOpenAICallOptions> extends BaseChatModel<CallOptions, AIMessageChunk> implements OpenAIChatInput, AzureOpenAIInput {
     static lc_name(): string;
@@ -585,6 +679,8 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
     constructor(fields?: ChatOpenAIFields, 
     /** @deprecated */
     configuration?: ClientOptions & LegacyOpenAIInput);

package/dist/chat_models.js

@@ -74,6 +74,9 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
+            if (message.audio) {
+                additional_kwargs.audio = message.audio;
+            }
             return new AIMessage({
                 content: message.content || "",
                 tool_calls: toolCalls,
@@ -109,6 +112,12 @@ delta, rawResponse, defaultRole, includeRawResponse) {
     if (includeRawResponse) {
         additional_kwargs.__raw_response = rawResponse;
     }
+    if (delta.audio) {
+        additional_kwargs.audio = {
+            ...delta.audio,
+            index: rawResponse.choices[0].index,
+        };
+    }
     const response_metadata = { usage: { ...rawResponse.usage } };
     if (role === "user") {
         return new HumanMessageChunk({ content, response_metadata });
@@ -160,7 +169,7 @@ delta, rawResponse, defaultRole, includeRawResponse) {
 // Used in LangSmith, export is important here
 export function _convertMessagesToOpenAIParams(messages) {
     // TODO: Function messages do not support array content, fix cast
-    return messages.map((message) => {
+    return messages.flatMap((message) => {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role: messageToOpenAIRole(message),
@@ -185,6 +194,17 @@ export function _convertMessagesToOpenAIParams(messages) {
                 completionParam.tool_call_id = message.tool_call_id;
             }
         }
+        if (message.additional_kwargs.audio &&
+            typeof message.additional_kwargs.audio === "object" &&
+            "id" in message.additional_kwargs.audio) {
+            const audioMessage = {
+                role: "assistant",
+                audio: {
+                    id: message.additional_kwargs.audio.id,
+                },
+            };
+            return [completionParam, audioMessage];
+        }
         return completionParam;
     });
 }
@@ -664,6 +684,80 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castAudioContent = audioOutputResult.additional_kwargs.audio as Record<string, any>;
+ *
+ * console.log({
+ *   ...castAudioContent,
+ *   data: castAudioContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export class ChatOpenAI extends BaseChatModel {
     static lc_name() {
@@ -907,6 +1001,18 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "audio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "modalities", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -955,6 +1061,8 @@ export class ChatOpenAI extends BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
+        this.audio = fields?.audio;
+        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1076,6 +1184,12 @@ export class ChatOpenAI extends BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
+            ...(this.audio || options?.audio
+                ? { audio: this.audio || options?.audio }
+                : {}),
+            ...(this.modalities || options?.modalities
+                ? { modalities: this.modalities || options?.modalities }
+                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1111,7 +1225,7 @@ export class ChatOpenAI extends BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices[0];
+            const choice = data?.choices?.[0];
             if (data.usage) {
                 usage = data.usage;
             }

package/dist/types.d.ts

@@ -128,6 +128,25 @@ export interface OpenAIChatInput extends OpenAIBaseInput {
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
+    /**
+     * Output types that you would like the model to generate for this request. Most
+     * models are capable of generating text, which is the default:
+     *
+     * `["text"]`
+     *
+     * The `gpt-4o-audio-preview` model can also be used to
+     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
+     * this model generate both text and audio responses, you can use:
+     *
+     * `["text", "audio"]`
+     */
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
+    /**
+     * Parameters for audio output. Required when audio output is requested with
+     * `modalities: ["audio"]`.
+     * [Learn more](https://platform.openai.com/docs/guides/audio).
+     */
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export declare interface AzureOpenAIInput {
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.3.10",
+  "version": "0.3.11",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {
@@ -36,7 +36,7 @@
   "license": "MIT",
   "dependencies": {
     "js-tiktoken": "^1.0.12",
-    "openai": "^4.67.2",
+    "openai": "^4.68.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
   },