@langchain/openai 0.3.8 → 0.3.9

package/dist/chat_models.cjs

@@ -78,8 +78,9 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
+            const content = message.audio ? [message.audio] : message.content;
             return new messages_1.AIMessage({
-                content: message.content || "",
+                content: content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -95,7 +96,18 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    const content = delta.content ?? "";
+    let content;
+    if (delta.audio) {
+        content = [
+            {
+                ...delta.audio,
+                index: rawResponse.choices[0].index,
+            },
+        ];
+    }
+    else {
+        content = delta.content ?? "";
+    }
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -669,6 +681,43 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 class ChatOpenAI extends chat_models_1.BaseChatModel {
     static lc_name() {
@@ -912,6 +961,18 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "audio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "modalities", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -960,6 +1021,8 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
+        this.audio = fields?.audio;
+        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1081,6 +1144,12 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
+            ...(this.audio || options?.audio
+                ? { audio: this.audio || options?.audio }
+                : {}),
+            ...(this.modalities || options?.modalities
+                ? { modalities: this.modalities || options?.modalities }
+                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1116,7 +1185,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices[0];
+            const choice = data?.choices?.[0];
             if (data.usage) {
                 usage = data.usage;
             }
@@ -1133,10 +1202,6 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
-            if (typeof chunk.content !== "string") {
-                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
-                continue;
-            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1150,7 +1215,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generationChunk = new outputs_1.ChatGenerationChunk({
                 message: chunk,
-                text: chunk.content,
+                text: typeof chunk.content === "string" ? chunk.content : "",
                 generationInfo,
             });
             yield generationChunk;
@@ -1309,9 +1374,8 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
-                const text = part.message?.content ?? "";
                 const generation = {
-                    text,
+                    text: part.message?.content ?? "",
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/dist/chat_models.d.ts

@@ -75,6 +75,25 @@ export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCa
      * @version 0.2.6
      */
     strict?: boolean;
+    /**
+     * Output types that you would like the model to generate for this request. Most
+     * models are capable of generating text, which is the default:
+     *
+     * `["text"]`
+     *
+     * The `gpt-4o-audio-preview` model can also be used to
+     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
+     * this model generate both text and audio responses, you can use:
+     *
+     * `["text", "audio"]`
+     */
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
+    /**
+     * Parameters for audio output. Required when audio output is requested with
+     * `modalities: ["audio"]`.
+     * [Learn more](https://platform.openai.com/docs/guides/audio).
+     */
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<AzureOpenAIInput>, BaseChatModelParams {
     configuration?: ClientOptions & LegacyOpenAIInput;
@@ -540,6 +559,43 @@ export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<Azur
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = ChatOpenAICallOptions> extends BaseChatModel<CallOptions, AIMessageChunk> implements OpenAIChatInput, AzureOpenAIInput {
     static lc_name(): string;
@@ -585,6 +641,8 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
     constructor(fields?: ChatOpenAIFields, 
     /** @deprecated */
     configuration?: ClientOptions & LegacyOpenAIInput);

package/dist/chat_models.js

@@ -74,8 +74,9 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
+            const content = message.audio ? [message.audio] : message.content;
             return new AIMessage({
-                content: message.content || "",
+                content: content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -91,7 +92,18 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    const content = delta.content ?? "";
+    let content;
+    if (delta.audio) {
+        content = [
+            {
+                ...delta.audio,
+                index: rawResponse.choices[0].index,
+            },
+        ];
+    }
+    else {
+        content = delta.content ?? "";
+    }
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -664,6 +676,43 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
+ *
+ * <details>
+ * <summary><strong>Audio Outputs</strong></summary>
+ *
+ * ```typescript
+ * import { ChatOpenAI } from "@langchain/openai";
+ *
+ * const modelWithAudioOutput = new ChatOpenAI({
+ *   model: "gpt-4o-audio-preview",
+ *   // You may also pass these fields to `.bind` as a call argument.
+ *   modalities: ["text", "audio"], // Specifies that the model should output audio.
+ *   audio: {
+ *     voice: "alloy",
+ *     format: "wav",
+ *   },
+ * });
+ *
+ * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
+ * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
+ *
+ * console.log({
+ *   ...castMessageContent,
+ *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
+ * })
+ * ```
+ *
+ * ```txt
+ * {
+ *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
+ *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
+ *   expires_at: 1729201448,
+ *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
+ * }
+ * ```
+ * </details>
+ *
+ * <br />
  */
 export class ChatOpenAI extends BaseChatModel {
     static lc_name() {
@@ -907,6 +956,18 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "audio", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "modalities", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -955,6 +1016,8 @@ export class ChatOpenAI extends BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
+        this.audio = fields?.audio;
+        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1076,6 +1139,12 @@ export class ChatOpenAI extends BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
+            ...(this.audio || options?.audio
+                ? { audio: this.audio || options?.audio }
+                : {}),
+            ...(this.modalities || options?.modalities
+                ? { modalities: this.modalities || options?.modalities }
+                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1111,7 +1180,7 @@ export class ChatOpenAI extends BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices[0];
+            const choice = data?.choices?.[0];
             if (data.usage) {
                 usage = data.usage;
             }
@@ -1128,10 +1197,6 @@ export class ChatOpenAI extends BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
-            if (typeof chunk.content !== "string") {
-                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
-                continue;
-            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1145,7 +1210,7 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generationChunk = new ChatGenerationChunk({
                 message: chunk,
-                text: chunk.content,
+                text: typeof chunk.content === "string" ? chunk.content : "",
                 generationInfo,
             });
             yield generationChunk;
@@ -1304,9 +1369,8 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
-                const text = part.message?.content ?? "";
                 const generation = {
-                    text,
+                    text: part.message?.content ?? "",
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/dist/types.d.ts

@@ -128,6 +128,25 @@ export interface OpenAIChatInput extends OpenAIBaseInput {
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
+    /**
+     * Output types that you would like the model to generate for this request. Most
+     * models are capable of generating text, which is the default:
+     *
+     * `["text"]`
+     *
+     * The `gpt-4o-audio-preview` model can also be used to
+     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
+     * this model generate both text and audio responses, you can use:
+     *
+     * `["text", "audio"]`
+     */
+    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
+    /**
+     * Parameters for audio output. Required when audio output is requested with
+     * `modalities: ["audio"]`.
+     * [Learn more](https://platform.openai.com/docs/guides/audio).
+     */
+    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export declare interface AzureOpenAIInput {
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.3.8",
+  "version": "0.3.9",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {
@@ -36,7 +36,7 @@
   "license": "MIT",
   "dependencies": {
     "js-tiktoken": "^1.0.12",
-    "openai": "^4.67.2",
+    "openai": "^4.68.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
   },