@langchain/openai 0.3.9 → 0.3.10

package/dist/chat_models.cjs

@@ -78,9 +78,8 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
-            const content = message.audio ? [message.audio] : message.content;
             return new messages_1.AIMessage({
-                content: content || "",
+                content: message.content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -96,18 +95,7 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    let content;
-    if (delta.audio) {
-        content = [
-            {
-                ...delta.audio,
-                index: rawResponse.choices[0].index,
-            },
-        ];
-    }
-    else {
-        content = delta.content ?? "";
-    }
+    const content = delta.content ?? "";
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -681,43 +669,6 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
- *
- * <details>
- * <summary><strong>Audio Outputs</strong></summary>
- *
- * ```typescript
- * import { ChatOpenAI } from "@langchain/openai";
- *
- * const modelWithAudioOutput = new ChatOpenAI({
- *   model: "gpt-4o-audio-preview",
- *   // You may also pass these fields to `.bind` as a call argument.
- *   modalities: ["text", "audio"], // Specifies that the model should output audio.
- *   audio: {
- *     voice: "alloy",
- *     format: "wav",
- *   },
- * });
- *
- * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
- * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
- *
- * console.log({
- *   ...castMessageContent,
- *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
- * })
- * ```
- *
- * ```txt
- * {
- *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
- *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
- *   expires_at: 1729201448,
- *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
- * }
- * ```
- * </details>
- *
- * <br />
  */
 class ChatOpenAI extends chat_models_1.BaseChatModel {
     static lc_name() {
@@ -961,18 +912,6 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "audio", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "modalities", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -1021,8 +960,6 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
-        this.audio = fields?.audio;
-        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1144,12 +1081,6 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
-            ...(this.audio || options?.audio
-                ? { audio: this.audio || options?.audio }
-                : {}),
-            ...(this.modalities || options?.modalities
-                ? { modalities: this.modalities || options?.modalities }
-                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1185,7 +1116,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices?.[0];
+            const choice = data?.choices[0];
             if (data.usage) {
                 usage = data.usage;
             }
@@ -1202,6 +1133,10 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
+            if (typeof chunk.content !== "string") {
+                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
+                continue;
+            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1215,7 +1150,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generationChunk = new outputs_1.ChatGenerationChunk({
                 message: chunk,
-                text: typeof chunk.content === "string" ? chunk.content : "",
+                text: chunk.content,
                 generationInfo,
             });
             yield generationChunk;
@@ -1374,8 +1309,9 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
+                const text = part.message?.content ?? "";
                 const generation = {
-                    text: part.message?.content ?? "",
+                    text,
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/dist/chat_models.d.ts

@@ -75,25 +75,6 @@ export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCa
      * @version 0.2.6
      */
     strict?: boolean;
-    /**
-     * Output types that you would like the model to generate for this request. Most
-     * models are capable of generating text, which is the default:
-     *
-     * `["text"]`
-     *
-     * The `gpt-4o-audio-preview` model can also be used to
-     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
-     * this model generate both text and audio responses, you can use:
-     *
-     * `["text", "audio"]`
-     */
-    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
-    /**
-     * Parameters for audio output. Required when audio output is requested with
-     * `modalities: ["audio"]`.
-     * [Learn more](https://platform.openai.com/docs/guides/audio).
-     */
-    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<AzureOpenAIInput>, BaseChatModelParams {
     configuration?: ClientOptions & LegacyOpenAIInput;
@@ -559,43 +540,6 @@ export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, Partial<Azur
  * </details>
  *
  * <br />
- *
- * <details>
- * <summary><strong>Audio Outputs</strong></summary>
- *
- * ```typescript
- * import { ChatOpenAI } from "@langchain/openai";
- *
- * const modelWithAudioOutput = new ChatOpenAI({
- *   model: "gpt-4o-audio-preview",
- *   // You may also pass these fields to `.bind` as a call argument.
- *   modalities: ["text", "audio"], // Specifies that the model should output audio.
- *   audio: {
- *     voice: "alloy",
- *     format: "wav",
- *   },
- * });
- *
- * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
- * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
- *
- * console.log({
- *   ...castMessageContent,
- *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
- * })
- * ```
- *
- * ```txt
- * {
- *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
- *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
- *   expires_at: 1729201448,
- *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
- * }
- * ```
- * </details>
- *
- * <br />
  */
 export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = ChatOpenAICallOptions> extends BaseChatModel<CallOptions, AIMessageChunk> implements OpenAIChatInput, AzureOpenAIInput {
     static lc_name(): string;
@@ -641,8 +585,6 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
-    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
-    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
     constructor(fields?: ChatOpenAIFields, 
     /** @deprecated */
     configuration?: ClientOptions & LegacyOpenAIInput);

package/dist/chat_models.js

@@ -74,9 +74,8 @@ function openAIResponseToChatMessage(message, rawResponse, includeRawResponse) {
                     system_fingerprint: rawResponse.system_fingerprint,
                 };
             }
-            const content = message.audio ? [message.audio] : message.content;
             return new AIMessage({
-                content: content || "",
+                content: message.content || "",
                 tool_calls: toolCalls,
                 invalid_tool_calls: invalidToolCalls,
                 additional_kwargs,
@@ -92,18 +91,7 @@ function _convertDeltaToMessageChunk(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 delta, rawResponse, defaultRole, includeRawResponse) {
     const role = delta.role ?? defaultRole;
-    let content;
-    if (delta.audio) {
-        content = [
-            {
-                ...delta.audio,
-                index: rawResponse.choices[0].index,
-            },
-        ];
-    }
-    else {
-        content = delta.content ?? "";
-    }
+    const content = delta.content ?? "";
     let additional_kwargs;
     if (delta.function_call) {
         additional_kwargs = {
@@ -676,43 +664,6 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
  * </details>
  *
  * <br />
- *
- * <details>
- * <summary><strong>Audio Outputs</strong></summary>
- *
- * ```typescript
- * import { ChatOpenAI } from "@langchain/openai";
- *
- * const modelWithAudioOutput = new ChatOpenAI({
- *   model: "gpt-4o-audio-preview",
- *   // You may also pass these fields to `.bind` as a call argument.
- *   modalities: ["text", "audio"], // Specifies that the model should output audio.
- *   audio: {
- *     voice: "alloy",
- *     format: "wav",
- *   },
- * });
- *
- * const audioOutputResult = await modelWithAudioOutput.invoke("Tell me a joke about cats.");
- * const castMessageContent = audioOutputResult.content[0] as Record<string, any>;
- *
- * console.log({
- *   ...castMessageContent,
- *   data: castMessageContent.data.slice(0, 100) // Sliced for brevity
- * })
- * ```
- *
- * ```txt
- * {
- *   id: 'audio_67117718c6008190a3afad3e3054b9b6',
- *   data: 'UklGRqYwBgBXQVZFZm10IBAAAAABAAEAwF0AAIC7AAACABAATElTVBoAAABJTkZPSVNGVA4AAABMYXZmNTguMjkuMTAwAGRhdGFg',
- *   expires_at: 1729201448,
- *   transcript: 'Sure! Why did the cat sit on the computer? Because it wanted to keep an eye on the mouse!'
- * }
- * ```
- * </details>
- *
- * <br />
  */
 export class ChatOpenAI extends BaseChatModel {
     static lc_name() {
@@ -956,18 +907,6 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "audio", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "modalities", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -1016,8 +955,6 @@ export class ChatOpenAI extends BaseChatModel {
         this.stopSequences = this?.stop;
         this.user = fields?.user;
         this.__includeRawResponse = fields?.__includeRawResponse;
-        this.audio = fields?.audio;
-        this.modalities = fields?.modalities;
         if (this.azureOpenAIApiKey || this.azureADTokenProvider) {
             if (!this.azureOpenAIApiInstanceName &&
                 !this.azureOpenAIBasePath &&
@@ -1139,12 +1076,6 @@ export class ChatOpenAI extends BaseChatModel {
             seed: options?.seed,
             ...streamOptionsConfig,
             parallel_tool_calls: options?.parallel_tool_calls,
-            ...(this.audio || options?.audio
-                ? { audio: this.audio || options?.audio }
-                : {}),
-            ...(this.modalities || options?.modalities
-                ? { modalities: this.modalities || options?.modalities }
-                : {}),
             ...this.modelKwargs,
         };
         return params;
@@ -1180,7 +1111,7 @@ export class ChatOpenAI extends BaseChatModel {
         const streamIterable = await this.completionWithRetry(params, options);
         let usage;
         for await (const data of streamIterable) {
-            const choice = data?.choices?.[0];
+            const choice = data?.choices[0];
             if (data.usage) {
                 usage = data.usage;
             }
@@ -1197,6 +1128,10 @@ export class ChatOpenAI extends BaseChatModel {
                 prompt: options.promptIndex ?? 0,
                 completion: choice.index ?? 0,
             };
+            if (typeof chunk.content !== "string") {
+                console.log("[WARNING]: Received non-string content from OpenAI. This is currently not supported.");
+                continue;
+            }
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const generationInfo = { ...newTokenIndices };
             if (choice.finish_reason != null) {
@@ -1210,7 +1145,7 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generationChunk = new ChatGenerationChunk({
                 message: chunk,
-                text: typeof chunk.content === "string" ? chunk.content : "",
+                text: chunk.content,
                 generationInfo,
             });
             yield generationChunk;
@@ -1369,8 +1304,9 @@ export class ChatOpenAI extends BaseChatModel {
             }
             const generations = [];
             for (const part of data?.choices ?? []) {
+                const text = part.message?.content ?? "";
                 const generation = {
-                    text: part.message?.content ?? "",
+                    text,
                     message: openAIResponseToChatMessage(part.message ?? { role: "assistant" }, data, this.__includeRawResponse),
                 };
                 generation.generationInfo = {

package/dist/types.d.ts

@@ -128,25 +128,6 @@ export interface OpenAIChatInput extends OpenAIBaseInput {
      * If `undefined` the `strict` argument will not be passed to OpenAI.
      */
     supportsStrictToolCalling?: boolean;
-    /**
-     * Output types that you would like the model to generate for this request. Most
-     * models are capable of generating text, which is the default:
-     *
-     * `["text"]`
-     *
-     * The `gpt-4o-audio-preview` model can also be used to
-     * [generate audio](https://platform.openai.com/docs/guides/audio). To request that
-     * this model generate both text and audio responses, you can use:
-     *
-     * `["text", "audio"]`
-     */
-    modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
-    /**
-     * Parameters for audio output. Required when audio output is requested with
-     * `modalities: ["audio"]`.
-     * [Learn more](https://platform.openai.com/docs/guides/audio).
-     */
-    audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
 }
 export declare interface AzureOpenAIInput {
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.3.9",
+  "version": "0.3.10",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {
@@ -36,7 +36,7 @@
   "license": "MIT",
   "dependencies": {
     "js-tiktoken": "^1.0.12",
-    "openai": "^4.68.0",
+    "openai": "^4.67.2",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
   },