@langchain/openai 0.5.5 → 0.5.7

package/dist/azure/embeddings.cjs

@@ -116,7 +116,12 @@ class AzureOpenAIEmbeddings extends embeddings_js_1.OpenAIEmbeddings {
         }
         return this.caller.call(async () => {
             try {
-                const res = await this.client.embeddings.create(request, requestOptions);
+                const res = await this.client.embeddings.create(request, 
+                // This unknown cast is required because OpenAI types seem to be incorrect here
+                // without it, we can't pass arbitrary keys via the `query` field in the options,
+                // which means we can't specify the `api-version` as required by Azure OpenAI.
+                // See https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning
+                requestOptions);
                 return res;
             }
             catch (e) {

package/dist/azure/embeddings.js

@@ -113,7 +113,12 @@ export class AzureOpenAIEmbeddings extends OpenAIEmbeddings {
         }
         return this.caller.call(async () => {
             try {
-                const res = await this.client.embeddings.create(request, requestOptions);
+                const res = await this.client.embeddings.create(request, 
+                // This unknown cast is required because OpenAI types seem to be incorrect here
+                // without it, we can't pass arbitrary keys via the `query` field in the options,
+                // which means we can't specify the `api-version` as required by Azure OpenAI.
+                // See https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#rest-api-versioning
+                requestOptions);
                 return res;
             }
             catch (e) {

package/dist/chat_models.cjs

@@ -50,6 +50,132 @@ function messageToOpenAIRole(message) {
     }
 }
 exports.messageToOpenAIRole = messageToOpenAIRole;
+const completionsApiContentBlockConverter = {
+    providerName: "ChatOpenAI",
+    fromStandardTextBlock(block) {
+        return { type: "text", text: block.text };
+    },
+    fromStandardImageBlock(block) {
+        if (block.source_type === "url") {
+            return {
+                type: "image_url",
+                image_url: {
+                    url: block.url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
+            return {
+                type: "image_url",
+                image_url: {
+                    url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardAudioBlock(block) {
+        if (block.source_type === "url") {
+            const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            const rawMimeType = data.mime_type || block.mime_type || "";
+            let mimeType;
+            try {
+                mimeType = (0, messages_1.parseMimeType)(rawMimeType);
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== "audio" ||
+                (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: "input_audio",
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: data.data,
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            let mimeType;
+            try {
+                mimeType = (0, messages_1.parseMimeType)(block.mime_type ?? "");
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== "audio" ||
+                (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: "input_audio",
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: block.data,
+                },
+            };
+        }
+        throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardFileBlock(block) {
+        if (block.source_type === "url") {
+            const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            return {
+                type: "file",
+                file: {
+                    file_data: block.url,
+                    ...(block.metadata?.filename || block.metadata?.name
+                        ? {
+                            filename: (block.metadata?.filename ||
+                                block.metadata?.name),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            return {
+                type: "file",
+                file: {
+                    file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
+                    ...(block.metadata?.filename ||
+                        block.metadata?.name ||
+                        block.metadata?.title
+                        ? {
+                            filename: (block.metadata?.filename ||
+                                block.metadata?.name ||
+                                block.metadata?.title),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "id") {
+            return {
+                type: "file",
+                file: {
+                    file_id: block.id,
+                },
+            };
+        }
+        throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+};
 // Used in LangSmith, export is important here
 function _convertMessagesToOpenAIParams(messages, model) {
     // TODO: Function messages do not support array content, fix cast
@@ -58,10 +184,18 @@ function _convertMessagesToOpenAIParams(messages, model) {
         if (role === "system" && isReasoningModel(model)) {
             role = "developer";
         }
+        const content = typeof message.content === "string"
+            ? message.content
+            : message.content.map((m) => {
+                if ((0, messages_1.isDataContentBlock)(m)) {
+                    return (0, messages_1.convertToProviderContentBlock)(m, completionsApiContentBlockConverter);
+                }
+                return m;
+            });
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role,
-            content: message.content,
+            content,
         };
         if (message.name != null) {
             completionParam.name = message.name;
@@ -201,7 +335,6 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     name: toolCall.name,
                     arguments: JSON.stringify(toolCall.args),
                     call_id: toolCall.id,
-                    // @ts-expect-error Might come from a non-Responses API message
                     id: functionCallIds?.[toolCall.id],
                 })));
             }
@@ -210,7 +343,6 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     type: "function_call",
                     name: toolCall.function.name,
                     call_id: toolCall.id,
-                    // @ts-expect-error Might come from a non-Responses API message
                     id: functionCallIds?.[toolCall.id],
                     arguments: toolCall.function.arguments,
                 })));
@@ -234,6 +366,9 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
         const content = typeof lcMsg.content === "string"
             ? lcMsg.content
             : lcMsg.content.flatMap((item) => {
+                if ((0, messages_1.isDataContentBlock)(item)) {
+                    return (0, messages_1.convertToProviderContentBlock)(item, completionsApiContentBlockConverter);
+                }
                 if (item.type === "text") {
                     return { type: "input_text", text: item.text };
                 }
@@ -322,7 +457,9 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
                 invalid_tool_calls.push((0, openai_tools_1.makeInvalidToolCall)(fnAdapter, errMessage));
             }
             additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
-            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+            if (item.id) {
+                additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+            }
         }
         else if (item.type === "reasoning") {
             additional_kwargs.reasoning = item;
@@ -466,7 +603,9 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
     return (0, tools_js_1._convertToOpenAITool)(tool, fields);
 }
 function isReasoningModel(model) {
-    return model?.startsWith("o1") || model?.startsWith("o3");
+    return (model?.startsWith("o1") ||
+        model?.startsWith("o3") ||
+        model?.startsWith("o4"));
 }
 /**
  * OpenAI chat model integration.

package/dist/chat_models.js

@@ -1,5 +1,5 @@
 import { OpenAI as OpenAIClient } from "openai";
-import { AIMessage, AIMessageChunk, ChatMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk, isAIMessage, } from "@langchain/core/messages";
+import { AIMessage, AIMessageChunk, ChatMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk, isAIMessage, parseBase64DataUrl, parseMimeType, convertToProviderContentBlock, isDataContentBlock, } from "@langchain/core/messages";
 import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
@@ -46,6 +46,132 @@ export function messageToOpenAIRole(message) {
             throw new Error(`Unknown message type: ${type}`);
     }
 }
+const completionsApiContentBlockConverter = {
+    providerName: "ChatOpenAI",
+    fromStandardTextBlock(block) {
+        return { type: "text", text: block.text };
+    },
+    fromStandardImageBlock(block) {
+        if (block.source_type === "url") {
+            return {
+                type: "image_url",
+                image_url: {
+                    url: block.url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
+            return {
+                type: "image_url",
+                image_url: {
+                    url,
+                    ...(block.metadata?.detail
+                        ? { detail: block.metadata.detail }
+                        : {}),
+                },
+            };
+        }
+        throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardAudioBlock(block) {
+        if (block.source_type === "url") {
+            const data = parseBase64DataUrl({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            const rawMimeType = data.mime_type || block.mime_type || "";
+            let mimeType;
+            try {
+                mimeType = parseMimeType(rawMimeType);
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== "audio" ||
+                (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: "input_audio",
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: data.data,
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            let mimeType;
+            try {
+                mimeType = parseMimeType(block.mime_type ?? "");
+            }
+            catch {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            if (mimeType.type !== "audio" ||
+                (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
+                throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
+            }
+            return {
+                type: "input_audio",
+                input_audio: {
+                    format: mimeType.subtype,
+                    data: block.data,
+                },
+            };
+        }
+        throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+    fromStandardFileBlock(block) {
+        if (block.source_type === "url") {
+            const data = parseBase64DataUrl({ dataUrl: block.url });
+            if (!data) {
+                throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
+            }
+            return {
+                type: "file",
+                file: {
+                    file_data: block.url,
+                    ...(block.metadata?.filename || block.metadata?.name
+                        ? {
+                            filename: (block.metadata?.filename ||
+                                block.metadata?.name),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "base64") {
+            return {
+                type: "file",
+                file: {
+                    file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
+                    ...(block.metadata?.filename ||
+                        block.metadata?.name ||
+                        block.metadata?.title
+                        ? {
+                            filename: (block.metadata?.filename ||
+                                block.metadata?.name ||
+                                block.metadata?.title),
+                        }
+                        : {}),
+                },
+            };
+        }
+        if (block.source_type === "id") {
+            return {
+                type: "file",
+                file: {
+                    file_id: block.id,
+                },
+            };
+        }
+        throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
+    },
+};
 // Used in LangSmith, export is important here
 export function _convertMessagesToOpenAIParams(messages, model) {
     // TODO: Function messages do not support array content, fix cast
@@ -54,10 +180,18 @@ export function _convertMessagesToOpenAIParams(messages, model) {
         if (role === "system" && isReasoningModel(model)) {
             role = "developer";
         }
+        const content = typeof message.content === "string"
+            ? message.content
+            : message.content.map((m) => {
+                if (isDataContentBlock(m)) {
+                    return convertToProviderContentBlock(m, completionsApiContentBlockConverter);
+                }
+                return m;
+            });
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const completionParam = {
             role,
-            content: message.content,
+            content,
         };
         if (message.name != null) {
             completionParam.name = message.name;
@@ -196,7 +330,6 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     name: toolCall.name,
                     arguments: JSON.stringify(toolCall.args),
                     call_id: toolCall.id,
-                    // @ts-expect-error Might come from a non-Responses API message
                     id: functionCallIds?.[toolCall.id],
                 })));
             }
@@ -205,7 +338,6 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     type: "function_call",
                     name: toolCall.function.name,
                     call_id: toolCall.id,
-                    // @ts-expect-error Might come from a non-Responses API message
                     id: functionCallIds?.[toolCall.id],
                     arguments: toolCall.function.arguments,
                 })));
@@ -229,6 +361,9 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
         const content = typeof lcMsg.content === "string"
             ? lcMsg.content
             : lcMsg.content.flatMap((item) => {
+                if (isDataContentBlock(item)) {
+                    return convertToProviderContentBlock(item, completionsApiContentBlockConverter);
+                }
                 if (item.type === "text") {
                     return { type: "input_text", text: item.text };
                 }
@@ -317,7 +452,9 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
                 invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
             }
             additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
-            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+            if (item.id) {
+                additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+            }
         }
         else if (item.type === "reasoning") {
             additional_kwargs.reasoning = item;
@@ -461,7 +598,9 @@ function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
     return _convertToOpenAITool(tool, fields);
 }
 function isReasoningModel(model) {
-    return model?.startsWith("o1") || model?.startsWith("o3");
+    return (model?.startsWith("o1") ||
+        model?.startsWith("o3") ||
+        model?.startsWith("o4"));
 }
 /**
  * OpenAI chat model integration.

package/dist/types.d.ts

@@ -1,4 +1,5 @@
 import type { OpenAI as OpenAIClient } from "openai";
+import type { RequestOptions as _OpenAICoreRequestOptions } from "openai/core";
 import type { ResponseFormatText, ResponseFormatJSONObject, ResponseFormatJSONSchema } from "openai/resources/shared";
 import { TiktokenModel } from "js-tiktoken/lite";
 import type { BaseLanguageModelCallOptions } from "@langchain/core/language_models/base";
@@ -73,18 +74,7 @@ export declare interface OpenAIBaseInput {
      */
     apiKey?: string;
 }
-export type OpenAICoreRequestOptions<Req extends object = Record<string, unknown>> = {
-    path?: string;
-    query?: Req | undefined;
-    body?: Req | undefined;
-    headers?: Record<string, string | null | undefined> | undefined;
-    maxRetries?: number;
-    stream?: boolean | undefined;
-    timeout?: number;
-    httpAgent?: any;
-    signal?: AbortSignal | undefined | null;
-    idempotencyKey?: string;
-};
+export type OpenAICoreRequestOptions<Req = Record<string, unknown>> = _OpenAICoreRequestOptions<Req>;
 export interface OpenAICallOptions extends BaseLanguageModelCallOptions {
     /**
      * Additional options to pass to the underlying axios request.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.5.5",
+  "version": "0.5.7",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {
@@ -36,12 +36,12 @@
   "license": "MIT",
   "dependencies": {
     "js-tiktoken": "^1.0.12",
-    "openai": "^4.87.3",
+    "openai": "^4.93.0",
     "zod": "^3.22.4",
     "zod-to-json-schema": "^3.22.3"
   },
   "peerDependencies": {
-    "@langchain/core": ">=0.3.39 <0.4.0"
+    "@langchain/core": ">=0.3.48 <0.4.0"
   },
   "devDependencies": {
     "@azure/identity": "^4.2.1",
@@ -62,7 +62,7 @@
     "jest": "^29.5.0",
     "jest-environment-node": "^29.6.4",
     "prettier": "^2.8.3",
-    "release-it": "^17.6.0",
+    "release-it": "^18.1.2",
     "rimraf": "^5.0.1",
     "ts-jest": "^29.1.0",
     "typescript": "~5.1.6"