@langchain/openai 0.4.5 → 0.4.6

package/dist/chat_models.cjs

@@ -97,6 +97,319 @@ function _convertMessagesToOpenAIParams(messages, model) {
     });
 }
 exports._convertMessagesToOpenAIParams = _convertMessagesToOpenAIParams;
+const _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__";
+function _convertMessagesToOpenAIResponsesParams(messages, model) {
+    return messages.flatMap((lcMsg) => {
+        let role = messageToOpenAIRole(lcMsg);
+        if (role === "system" && isReasoningModel(model))
+            role = "developer";
+        if (role === "function") {
+            throw new Error("Function messages are not supported in Responses API");
+        }
+        if (role === "tool") {
+            const toolMessage = lcMsg;
+            // Handle computer call output
+            if (toolMessage.additional_kwargs?.type === "computer_call_output") {
+                const output = (() => {
+                    if (typeof toolMessage.content === "string") {
+                        return {
+                            type: "computer_screenshot",
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        return toolMessage.content.find((i) => i.type === "computer_screenshot");
+                    }
+                    return toolMessage.content;
+                })();
+                return {
+                    type: "computer_call_output",
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                    id: toolMessage.id,
+                };
+            }
+            return {
+                type: "function_call_output",
+                call_id: toolMessage.tool_call_id,
+                id: toolMessage.id,
+                output: typeof toolMessage.content !== "string"
+                    ? JSON.stringify(toolMessage.content)
+                    : toolMessage.content,
+            };
+        }
+        if (role === "assistant") {
+            const functionCallIds = 
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
+            if ((0, messages_1.isAIMessage)(lcMsg) && !!lcMsg.tool_calls?.length) {
+                return lcMsg.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.name,
+                    arguments: JSON.stringify(toolCall.args),
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                }));
+            }
+            if (lcMsg.additional_kwargs.tool_calls != null) {
+                return lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.function.name,
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                    arguments: toolCall.function.arguments,
+                }));
+            }
+            let { content } = lcMsg;
+            if (lcMsg.additional_kwargs.refusal != null) {
+                if (typeof content === "string") {
+                    content = [{ type: "output_text", text: content, annotations: [] }];
+                }
+                content = [
+                    ...content,
+                    { type: "refusal", refusal: lcMsg.additional_kwargs.refusal },
+                ];
+            }
+            return {
+                type: "message",
+                role: "assistant",
+                content: typeof content === "string"
+                    ? content
+                    : content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return {
+                                type: "output_text",
+                                text: item.text,
+                                // @ts-expect-error TODO: add types for `annotations`
+                                annotations: item.annotations ?? [],
+                            };
+                        }
+                        if (item.type === "output_text" || item.type === "refusal") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            };
+        }
+        if (role === "user") {
+            return {
+                type: "message",
+                role: "user",
+                content: typeof lcMsg.content === "string"
+                    ? lcMsg.content
+                    : lcMsg.content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return { type: "input_text", text: item.text };
+                        }
+                        if (item.type === "image_url") {
+                            const image_url = typeof item.image_url === "string"
+                                ? item.image_url
+                                : item.image_url.url;
+                            const detail = typeof item.image_url === "string"
+                                ? "auto"
+                                : item.image_url.detail;
+                            return { type: "input_image", image_url, detail };
+                        }
+                        if (item.type === "input_text" ||
+                            item.type === "input_image" ||
+                            item.type === "input_file") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            };
+        }
+        return [];
+    });
+}
+function _convertOpenAIResponsesMessageToBaseMessage(response) {
+    if (response.error) {
+        // TODO: add support for `addLangChainErrorFields`
+        const error = new Error(response.error.message);
+        error.name = response.error.code;
+        throw error;
+    }
+    const content = [];
+    const tool_calls = [];
+    const invalid_tool_calls = [];
+    const response_metadata = {
+        model: response.model,
+        created_at: response.created_at,
+        id: response.id,
+        incomplete_details: response.incomplete_details,
+        metadata: response.metadata,
+        object: response.object,
+        status: response.status,
+        user: response.user,
+        // for compatibility with chat completion calls.
+        model_name: response.model,
+    };
+    const additional_kwargs = {};
+    for (const item of response.output) {
+        if (item.type === "message") {
+            // TODO: how to handle refusals?
+            content.push(...item.content.map((part) => {
+                if (part.type === "output_text") {
+                    if ("parsed" in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
+                    return {
+                        type: "text",
+                        text: part.text,
+                        annotations: part.annotations,
+                    };
+                }
+                return part;
+            }));
+        }
+        else if (item.type === "function_call") {
+            const fnAdapter = {
+                function: { name: item.name, arguments: item.arguments },
+                id: item.call_id,
+            };
+            try {
+                tool_calls.push((0, openai_tools_1.parseToolCall)(fnAdapter, { returnId: true }));
+            }
+            catch (e) {
+                let errMessage;
+                if (typeof e === "object" &&
+                    e != null &&
+                    "message" in e &&
+                    typeof e.message === "string") {
+                    errMessage = e.message;
+                }
+                invalid_tool_calls.push((0, openai_tools_1.makeInvalidToolCall)(fnAdapter, errMessage));
+            }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+        }
+        else if (item.type === "reasoning") {
+            additional_kwargs.reasoning = item;
+        }
+        else {
+            additional_kwargs.tool_outputs ??= [];
+            additional_kwargs.tool_outputs.push(item);
+        }
+    }
+    return new messages_1.AIMessage({
+        id: response.id,
+        content,
+        tool_calls,
+        invalid_tool_calls,
+        usage_metadata: response.usage,
+        additional_kwargs,
+        response_metadata,
+    });
+}
+function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
+    const content = [];
+    let generationInfo = {};
+    let usage_metadata;
+    const tool_call_chunks = [];
+    const response_metadata = {};
+    const additional_kwargs = {};
+    let id;
+    if (chunk.type === "response.output_text.delta") {
+        content.push({
+            type: "text",
+            text: chunk.delta,
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_text.annotation.added") {
+        content.push({
+            type: "text",
+            text: "",
+            annotations: [chunk.annotation],
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "message") {
+        id = chunk.item.id;
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "function_call") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            name: chunk.item.name,
+            args: chunk.item.arguments,
+            id: chunk.item.id,
+            index: chunk.output_index,
+        });
+        additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {
+            [chunk.item.call_id]: chunk.item.id,
+        };
+    }
+    else if (chunk.type === "response.output_item.done" &&
+        (chunk.item.type === "web_search_call" ||
+            chunk.item.type === "file_search_call")) {
+        additional_kwargs.tool_outputs = [chunk.item];
+    }
+    else if (chunk.type === "response.created") {
+        response_metadata.id = chunk.response.id;
+        response_metadata.model_name = chunk.response.model;
+        response_metadata.model = chunk.response.model;
+    }
+    else if (chunk.type === "response.completed") {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
+        usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === "json_schema") {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== "id")
+                response_metadata[key] = value;
+        }
+    }
+    else if (chunk.type === "response.function_call_arguments.delta") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            args: chunk.delta,
+            index: chunk.output_index,
+        });
+    }
+    else if (chunk.type === "response.web_search_call.completed" ||
+        chunk.type === "response.file_search_call.completed") {
+        generationInfo = {
+            tool_outputs: {
+                id: chunk.item_id,
+                type: chunk.type.replace("response.", "").replace(".completed", ""),
+                status: "completed",
+            },
+        };
+    }
+    else if (chunk.type === "response.refusal.done") {
+        additional_kwargs.refusal = chunk.refusal;
+    }
+    else {
+        return null;
+    }
+    return new outputs_1.ChatGenerationChunk({
+        // Legacy reasons, `onLLMNewToken` should pulls this out
+        text: content.map((part) => part.text).join(""),
+        message: new messages_1.AIMessageChunk({
+            id,
+            content,
+            tool_call_chunks,
+            usage_metadata,
+            additional_kwargs,
+            response_metadata,
+        }),
+        generationInfo,
+    });
+}
+function isBuiltInTool(tool) {
+    return "type" in tool && tool.type !== "function";
+}
+function isBuiltInToolChoice(tool_choice) {
+    return (tool_choice != null &&
+        typeof tool_choice === "object" &&
+        "type" in tool_choice &&
+        tool_choice.type !== "function");
+}
 function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
     if ((0, base_1.isOpenAITool)(tool)) {
         if (fields?.strict !== undefined) {
@@ -904,6 +1217,12 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "useResponsesApi", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -933,6 +1252,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         this.modalities = fields?.modalities;
         this.reasoningEffort = fields?.reasoningEffort;
         this.maxTokens = fields?.maxCompletionTokens ?? fields?.maxTokens;
+        this.useResponsesApi = fields?.useResponsesApi ?? this.useResponsesApi;
         if (this.model === "o1") {
             this.disableStreaming = true;
         }
@@ -970,7 +1290,9 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             strict = this.supportsStrictToolCalling;
         }
         return this.bind({
-            tools: tools.map((tool) => _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
+            tools: tools.map((tool) => isBuiltInTool(tool)
+                ? tool
+                : _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
             ...kwargs,
         });
     }
@@ -996,6 +1318,80 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         else if (this.supportsStrictToolCalling !== undefined) {
             strict = this.supportsStrictToolCalling;
         }
+        if (this._useResponseApi(options)) {
+            const params = {
+                model: this.model,
+                temperature: this.temperature,
+                top_p: this.topP,
+                user: this.user,
+                // if include_usage is set or streamUsage then stream must be set to true.
+                stream: this.streaming,
+                previous_response_id: options?.previous_response_id,
+                truncation: options?.truncation,
+                include: options?.include,
+                tools: options?.tools?.length
+                    ? options.tools
+                        .map((tool) => {
+                        if (isBuiltInTool(tool)) {
+                            return tool;
+                        }
+                        else if ((0, base_1.isOpenAITool)(tool)) {
+                            return {
+                                type: "function",
+                                name: tool.function.name,
+                                parameters: tool.function.parameters,
+                                description: tool.function.description,
+                                strict,
+                            };
+                        }
+                        else {
+                            return null;
+                        }
+                    })
+                        .filter((tool) => tool !== null)
+                    : undefined,
+                tool_choice: isBuiltInToolChoice(options?.tool_choice)
+                    ? options?.tool_choice
+                    : (() => {
+                        const formatted = (0, openai_js_1.formatToOpenAIToolChoice)(options?.tool_choice);
+                        if (typeof formatted === "object" && "type" in formatted) {
+                            return { type: "function", name: formatted.function.name };
+                        }
+                        else {
+                            return undefined;
+                        }
+                    })(),
+                text: (() => {
+                    if (options?.text)
+                        return options.text;
+                    const format = this.createResponseFormat(options?.response_format);
+                    if (format?.type === "json_schema") {
+                        if (format.json_schema.schema != null) {
+                            return {
+                                format: {
+                                    type: "json_schema",
+                                    schema: format.json_schema.schema,
+                                    description: format.json_schema.description,
+                                    name: format.json_schema.name,
+                                    strict: format.json_schema.strict,
+                                },
+                            };
+                        }
+                        return undefined;
+                    }
+                    return { format };
+                })(),
+                parallel_tool_calls: options?.parallel_tool_calls,
+                max_output_tokens: this.maxTokens === -1 ? undefined : this.maxTokens,
+                ...this.modelKwargs,
+            };
+            const reasoningEffort = options?.reasoning_effort ?? this.reasoningEffort;
+            if (reasoningEffort !== undefined) {
+                params.reasoning = { effort: reasoningEffort };
+            }
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            return params;
+        }
         let streamOptionsConfig = {};
         if (options?.stream_options !== undefined) {
             streamOptionsConfig = { stream_options: options.stream_options };
@@ -1049,6 +1445,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         else {
             params.max_tokens = this.maxTokens === -1 ? undefined : this.maxTokens;
         }
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         return params;
     }
     _convertOpenAIChatCompletionMessageToBaseMessage(message, rawResponse) {
@@ -1192,6 +1589,20 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         };
     }
     async *_streamResponseChunks(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            const streamIterable = await this.responseApiWithRetry({
+                ...this.invocationParams(options, { streaming: true }),
+                input: _convertMessagesToOpenAIResponsesParams(messages, this.model),
+                stream: true,
+            }, options);
+            for await (const data of streamIterable) {
+                const chunk = _convertOpenAIResponsesDeltaToBaseMessageChunk(data);
+                if (chunk == null)
+                    continue;
+                yield chunk;
+            }
+            return;
+        }
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
         const params = {
             ...this.invocationParams(options, {
@@ -1296,7 +1707,60 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         return this._identifyingParams();
     }
     /** @ignore */
+    async _responseApiGenerate(messages, options, runManager) {
+        const invocationParams = this.invocationParams(options);
+        if (invocationParams.stream) {
+            const stream = this._streamResponseChunks(messages, options, runManager);
+            let finalChunk;
+            for await (const chunk of stream) {
+                chunk.message.response_metadata = {
+                    ...chunk.generationInfo,
+                    ...chunk.message.response_metadata,
+                };
+                finalChunk = finalChunk?.concat(chunk) ?? chunk;
+            }
+            return {
+                generations: finalChunk ? [finalChunk] : [],
+                llmOutput: {
+                    estimatedTokenUsage: finalChunk?.message
+                        ?.usage_metadata,
+                },
+            };
+        }
+        const input = _convertMessagesToOpenAIResponsesParams(messages, this.model);
+        const data = await this.responseApiWithRetry({ input, ...invocationParams }, { signal: options?.signal, ...options?.options });
+        return {
+            generations: [
+                {
+                    text: data.output_text,
+                    message: _convertOpenAIResponsesMessageToBaseMessage(data),
+                },
+            ],
+            llmOutput: {
+                id: data.id,
+                estimatedTokenUsage: data.usage
+                    ? {
+                        promptTokens: data.usage.input_tokens,
+                        completionTokens: data.usage.output_tokens,
+                        totalTokens: data.usage.total_tokens,
+                    }
+                    : undefined,
+            },
+        };
+    }
+    _useResponseApi(options) {
+        const usesBuiltInTools = options?.tools?.some(isBuiltInTool);
+        const hasResponsesOnlyKwargs = options?.previous_response_id != null ||
+            options?.text != null ||
+            options?.truncation != null ||
+            options?.include != null;
+        return this.useResponsesApi || usesBuiltInTools || hasResponsesOnlyKwargs;
+    }
+    /** @ignore */
     async _generate(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            return this._responseApiGenerate(messages, options, runManager);
+        }
         const usageMetadata = {};
         const params = this.invocationParams(options);
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
@@ -1534,6 +1998,22 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
             }
         });
     }
+    async responseApiWithRetry(request, options) {
+        return this.caller.call(async () => {
+            const requestOptions = this._getClientOptions(options);
+            try {
+                // use parse if dealing with json_schema
+                if (request.text?.format?.type === "json_schema" && !request.stream) {
+                    return await this.client.responses.parse(request, requestOptions);
+                }
+                return await this.client.responses.create(request, requestOptions);
+            }
+            catch (e) {
+                const error = (0, openai_js_1.wrapOpenAIClientError)(e);
+                throw error;
+            }
+        });
+    }
     /**
      * Call the beta chat completions parse endpoint. This should only be called if
      * response_format is set to "json_object".
@@ -1656,7 +2136,13 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
                 },
             });
             if (isZodSchema(schema)) {
-                outputParser = output_parsers_1.StructuredOutputParser.fromZodSchema(schema);
+                const altParser = output_parsers_1.StructuredOutputParser.fromZodSchema(schema);
+                outputParser = runnables_1.RunnableLambda.from((aiMessage) => {
+                    if ("parsed" in aiMessage.additional_kwargs) {
+                        return aiMessage.additional_kwargs.parsed;
+                    }
+                    return altParser;
+                });
             }
             else {
                 outputParser = new output_parsers_1.JsonOutputParser();
@@ -1744,12 +2230,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         const parsedWithFallback = parserAssign.withFallbacks({
             fallbacks: [parserNone],
         });
-        return runnables_1.RunnableSequence.from([
-            {
-                raw: llm,
-            },
-            parsedWithFallback,
-        ]);
+        return runnables_1.RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
     }
 }
 exports.ChatOpenAI = ChatOpenAI;

package/dist/chat_models.d.ts

@@ -2,7 +2,7 @@ import { type ClientOptions, OpenAI as OpenAIClient } from "openai";
 import { CallbackManagerForLLMRun } from "@langchain/core/callbacks/manager";
 import { AIMessageChunk, type BaseMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk } from "@langchain/core/messages";
 import { ChatGenerationChunk, type ChatResult } from "@langchain/core/outputs";
-import { BaseChatModel, BindToolsInput, LangSmithParams, type BaseChatModelParams } from "@langchain/core/language_models/chat_models";
+import { BaseChatModel, type BindToolsInput, type LangSmithParams, type BaseChatModelParams } from "@langchain/core/language_models/chat_models";
 import { type BaseFunctionCallOptions, type BaseLanguageModelInput, type StructuredOutputMethodOptions } from "@langchain/core/language_models/base";
 import { z } from "zod";
 import { Runnable } from "@langchain/core/runnables";
@@ -21,7 +21,21 @@ type OpenAIRoleEnum = "system" | "developer" | "assistant" | "user" | "function"
 type OpenAICompletionParam = OpenAIClient.Chat.Completions.ChatCompletionMessageParam;
 export declare function messageToOpenAIRole(message: BaseMessage): OpenAIRoleEnum;
 export declare function _convertMessagesToOpenAIParams(messages: BaseMessage[], model?: string): OpenAICompletionParam[];
-type ChatOpenAIToolType = BindToolsInput | OpenAIClient.ChatCompletionTool;
+type ExcludeController<T> = T extends {
+    controller: unknown;
+} ? never : T;
+type ExcludeNonController<T> = T extends {
+    controller: unknown;
+} ? T : never;
+type ResponsesCreate = OpenAIClient.Responses["create"];
+type ResponsesCreateParams = Parameters<OpenAIClient.Responses["create"]>[0];
+type ResponsesTool = Exclude<ResponsesCreateParams["tools"], undefined>[number];
+type ResponsesToolChoice = Exclude<ResponsesCreateParams["tool_choice"], undefined>;
+type ResponsesCreateInvoke = ExcludeController<Awaited<ReturnType<ResponsesCreate>>>;
+type ResponsesCreateStream = ExcludeNonController<Awaited<ReturnType<ResponsesCreate>>>;
+type ResponseInvocationParams = Omit<ResponsesCreateParams, "input">;
+type ChatCompletionInvocationParams = Omit<OpenAIClient.Chat.ChatCompletionCreateParams, "messages">;
+type ChatOpenAIToolType = BindToolsInput | OpenAIClient.ChatCompletionTool | ResponsesTool;
 export interface ChatOpenAIStructuredOutputMethodOptions<IncludeRaw extends boolean> extends StructuredOutputMethodOptions<IncludeRaw> {
     /**
      * strict: If `true` and `method` = "function_calling", model output is
@@ -41,7 +55,7 @@ export interface ChatOpenAIStructuredOutputMethodOptions<IncludeRaw extends bool
 }
 export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCallOptions {
     tools?: ChatOpenAIToolType[];
-    tool_choice?: OpenAIToolChoice;
+    tool_choice?: OpenAIToolChoice | ResponsesToolChoice;
     promptIndex?: number;
     response_format?: ChatOpenAIResponseFormat;
     seed?: number;
@@ -105,9 +119,28 @@ export interface ChatOpenAICallOptions extends OpenAICallOptions, BaseFunctionCa
      * Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning in a response.
      */
     reasoning_effort?: OpenAIClient.Chat.ChatCompletionReasoningEffort;
+    /**
+     * Configuration options for a text response from the model. Can be plain text or
+     * structured JSON data.
+     */
+    text?: ResponsesCreateParams["text"];
+    /**
+     * The truncation strategy to use for the model response.
+     */
+    truncation?: ResponsesCreateParams["truncation"];
+    /**
+     * Specify additional output data to include in the model response.
+     */
+    include?: ResponsesCreateParams["include"];
+    /**
+     * The unique ID of the previous response to the model. Use this to create
+     * multi-turn conversations.
+     */
+    previous_response_id?: ResponsesCreateParams["previous_response_id"];
 }
 export interface ChatOpenAIFields extends Partial<OpenAIChatInput>, BaseChatModelParams {
     configuration?: ClientOptions;
+    useResponsesApi?: boolean;
 }
 /**
  * OpenAI chat model integration.
@@ -689,6 +722,7 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
     audio?: OpenAIClient.Chat.ChatCompletionAudioParam;
     modalities?: Array<OpenAIClient.Chat.ChatCompletionModality>;
     reasoningEffort?: OpenAIClient.Chat.ChatCompletionReasoningEffort;
+    useResponsesApi: boolean;
     constructor(fields?: ChatOpenAIFields);
     getLsParams(options: this["ParsedCallOptions"]): LangSmithParams;
     bindTools(tools: ChatOpenAIToolType[], kwargs?: Partial<CallOptions>): Runnable<BaseLanguageModelInput, AIMessageChunk, CallOptions>;
@@ -696,9 +730,9 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
     /**
      * Get the parameters used to invoke the model
      */
-    invocationParams(options?: this["ParsedCallOptions"], extra?: {
+    invocationParams<Type extends "responses" | "completion" = "completion">(options?: this["ParsedCallOptions"], extra?: {
         streaming?: boolean;
-    }): Omit<OpenAIClient.Chat.ChatCompletionCreateParams, "messages">;
+    }): Type extends "responses" ? ResponseInvocationParams : ChatCompletionInvocationParams;
     protected _convertOpenAIChatCompletionMessageToBaseMessage(message: OpenAIClient.Chat.Completions.ChatCompletionMessage, rawResponse: OpenAIClient.Chat.Completions.ChatCompletion): BaseMessage;
     protected _convertOpenAIDeltaToBaseMessageChunk(delta: Record<string, any>, rawResponse: OpenAIClient.Chat.Completions.ChatCompletionChunk, defaultRole?: OpenAIRoleEnum): AIMessageChunk | HumanMessageChunk | SystemMessageChunk | FunctionMessageChunk | ToolMessageChunk | ChatMessageChunk;
     /** @ignore */
@@ -714,6 +748,9 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
         model_name: string;
     } & ClientOptions;
     /** @ignore */
+    _responseApiGenerate(messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<ChatResult>;
+    protected _useResponseApi(options: this["ParsedCallOptions"] | undefined): boolean;
+    /** @ignore */
     _generate(messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<ChatResult>;
     /**
      * Estimate the number of tokens a prompt will use.
@@ -736,6 +773,10 @@ export declare class ChatOpenAI<CallOptions extends ChatOpenAICallOptions = Chat
      */
     completionWithRetry(request: OpenAIClient.Chat.ChatCompletionCreateParamsStreaming, options?: OpenAICoreRequestOptions): Promise<AsyncIterable<OpenAIClient.Chat.Completions.ChatCompletionChunk>>;
     completionWithRetry(request: OpenAIClient.Chat.ChatCompletionCreateParamsNonStreaming, options?: OpenAICoreRequestOptions): Promise<OpenAIClient.Chat.Completions.ChatCompletion>;
+    protected responseApiWithRetry(request: ResponsesCreateParams & {
+        stream: true;
+    }, options?: OpenAICoreRequestOptions): Promise<ResponsesCreateStream>;
+    protected responseApiWithRetry(request: ResponsesCreateParams, options?: OpenAICoreRequestOptions): Promise<ResponsesCreateInvoke>;
     /**
      * Call the beta chat completions parse endpoint. This should only be called if
      * response_format is set to "json_object".

package/dist/chat_models.js

@@ -4,7 +4,7 @@ import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
 import { isOpenAITool, } from "@langchain/core/language_models/base";
-import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
+import { RunnableLambda, RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
 import { JsonOutputParser, StructuredOutputParser, } from "@langchain/core/output_parsers";
 import { JsonOutputKeyToolsParser, convertLangChainToolCallToOpenAI, makeInvalidToolCall, parseToolCall, } from "@langchain/core/output_parsers/openai_tools";
 import { zodToJsonSchema } from "zod-to-json-schema";
@@ -92,6 +92,319 @@ export function _convertMessagesToOpenAIParams(messages, model) {
         return completionParam;
     });
 }
+const _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__";
+function _convertMessagesToOpenAIResponsesParams(messages, model) {
+    return messages.flatMap((lcMsg) => {
+        let role = messageToOpenAIRole(lcMsg);
+        if (role === "system" && isReasoningModel(model))
+            role = "developer";
+        if (role === "function") {
+            throw new Error("Function messages are not supported in Responses API");
+        }
+        if (role === "tool") {
+            const toolMessage = lcMsg;
+            // Handle computer call output
+            if (toolMessage.additional_kwargs?.type === "computer_call_output") {
+                const output = (() => {
+                    if (typeof toolMessage.content === "string") {
+                        return {
+                            type: "computer_screenshot",
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        return toolMessage.content.find((i) => i.type === "computer_screenshot");
+                    }
+                    return toolMessage.content;
+                })();
+                return {
+                    type: "computer_call_output",
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                    id: toolMessage.id,
+                };
+            }
+            return {
+                type: "function_call_output",
+                call_id: toolMessage.tool_call_id,
+                id: toolMessage.id,
+                output: typeof toolMessage.content !== "string"
+                    ? JSON.stringify(toolMessage.content)
+                    : toolMessage.content,
+            };
+        }
+        if (role === "assistant") {
+            const functionCallIds = 
+            // eslint-disable-next-line @typescript-eslint/no-use-before-define
+            lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
+            if (isAIMessage(lcMsg) && !!lcMsg.tool_calls?.length) {
+                return lcMsg.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.name,
+                    arguments: JSON.stringify(toolCall.args),
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                }));
+            }
+            if (lcMsg.additional_kwargs.tool_calls != null) {
+                return lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
+                    type: "function_call",
+                    name: toolCall.function.name,
+                    call_id: toolCall.id,
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
+                    arguments: toolCall.function.arguments,
+                }));
+            }
+            let { content } = lcMsg;
+            if (lcMsg.additional_kwargs.refusal != null) {
+                if (typeof content === "string") {
+                    content = [{ type: "output_text", text: content, annotations: [] }];
+                }
+                content = [
+                    ...content,
+                    { type: "refusal", refusal: lcMsg.additional_kwargs.refusal },
+                ];
+            }
+            return {
+                type: "message",
+                role: "assistant",
+                content: typeof content === "string"
+                    ? content
+                    : content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return {
+                                type: "output_text",
+                                text: item.text,
+                                // @ts-expect-error TODO: add types for `annotations`
+                                annotations: item.annotations ?? [],
+                            };
+                        }
+                        if (item.type === "output_text" || item.type === "refusal") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            };
+        }
+        if (role === "user") {
+            return {
+                type: "message",
+                role: "user",
+                content: typeof lcMsg.content === "string"
+                    ? lcMsg.content
+                    : lcMsg.content.flatMap((item) => {
+                        if (item.type === "text") {
+                            return { type: "input_text", text: item.text };
+                        }
+                        if (item.type === "image_url") {
+                            const image_url = typeof item.image_url === "string"
+                                ? item.image_url
+                                : item.image_url.url;
+                            const detail = typeof item.image_url === "string"
+                                ? "auto"
+                                : item.image_url.detail;
+                            return { type: "input_image", image_url, detail };
+                        }
+                        if (item.type === "input_text" ||
+                            item.type === "input_image" ||
+                            item.type === "input_file") {
+                            return item;
+                        }
+                        return [];
+                    }),
+            };
+        }
+        return [];
+    });
+}
+function _convertOpenAIResponsesMessageToBaseMessage(response) {
+    if (response.error) {
+        // TODO: add support for `addLangChainErrorFields`
+        const error = new Error(response.error.message);
+        error.name = response.error.code;
+        throw error;
+    }
+    const content = [];
+    const tool_calls = [];
+    const invalid_tool_calls = [];
+    const response_metadata = {
+        model: response.model,
+        created_at: response.created_at,
+        id: response.id,
+        incomplete_details: response.incomplete_details,
+        metadata: response.metadata,
+        object: response.object,
+        status: response.status,
+        user: response.user,
+        // for compatibility with chat completion calls.
+        model_name: response.model,
+    };
+    const additional_kwargs = {};
+    for (const item of response.output) {
+        if (item.type === "message") {
+            // TODO: how to handle refusals?
+            content.push(...item.content.map((part) => {
+                if (part.type === "output_text") {
+                    if ("parsed" in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
+                    return {
+                        type: "text",
+                        text: part.text,
+                        annotations: part.annotations,
+                    };
+                }
+                return part;
+            }));
+        }
+        else if (item.type === "function_call") {
+            const fnAdapter = {
+                function: { name: item.name, arguments: item.arguments },
+                id: item.call_id,
+            };
+            try {
+                tool_calls.push(parseToolCall(fnAdapter, { returnId: true }));
+            }
+            catch (e) {
+                let errMessage;
+                if (typeof e === "object" &&
+                    e != null &&
+                    "message" in e &&
+                    typeof e.message === "string") {
+                    errMessage = e.message;
+                }
+                invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
+            }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
+        }
+        else if (item.type === "reasoning") {
+            additional_kwargs.reasoning = item;
+        }
+        else {
+            additional_kwargs.tool_outputs ??= [];
+            additional_kwargs.tool_outputs.push(item);
+        }
+    }
+    return new AIMessage({
+        id: response.id,
+        content,
+        tool_calls,
+        invalid_tool_calls,
+        usage_metadata: response.usage,
+        additional_kwargs,
+        response_metadata,
+    });
+}
+function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
+    const content = [];
+    let generationInfo = {};
+    let usage_metadata;
+    const tool_call_chunks = [];
+    const response_metadata = {};
+    const additional_kwargs = {};
+    let id;
+    if (chunk.type === "response.output_text.delta") {
+        content.push({
+            type: "text",
+            text: chunk.delta,
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_text.annotation.added") {
+        content.push({
+            type: "text",
+            text: "",
+            annotations: [chunk.annotation],
+            index: chunk.content_index,
+        });
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "message") {
+        id = chunk.item.id;
+    }
+    else if (chunk.type === "response.output_item.added" &&
+        chunk.item.type === "function_call") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            name: chunk.item.name,
+            args: chunk.item.arguments,
+            id: chunk.item.id,
+            index: chunk.output_index,
+        });
+        additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] = {
+            [chunk.item.call_id]: chunk.item.id,
+        };
+    }
+    else if (chunk.type === "response.output_item.done" &&
+        (chunk.item.type === "web_search_call" ||
+            chunk.item.type === "file_search_call")) {
+        additional_kwargs.tool_outputs = [chunk.item];
+    }
+    else if (chunk.type === "response.created") {
+        response_metadata.id = chunk.response.id;
+        response_metadata.model_name = chunk.response.model;
+        response_metadata.model = chunk.response.model;
+    }
+    else if (chunk.type === "response.completed") {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
+        usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === "json_schema") {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== "id")
+                response_metadata[key] = value;
+        }
+    }
+    else if (chunk.type === "response.function_call_arguments.delta") {
+        tool_call_chunks.push({
+            type: "tool_call_chunk",
+            args: chunk.delta,
+            index: chunk.output_index,
+        });
+    }
+    else if (chunk.type === "response.web_search_call.completed" ||
+        chunk.type === "response.file_search_call.completed") {
+        generationInfo = {
+            tool_outputs: {
+                id: chunk.item_id,
+                type: chunk.type.replace("response.", "").replace(".completed", ""),
+                status: "completed",
+            },
+        };
+    }
+    else if (chunk.type === "response.refusal.done") {
+        additional_kwargs.refusal = chunk.refusal;
+    }
+    else {
+        return null;
+    }
+    return new ChatGenerationChunk({
+        // Legacy reasons, `onLLMNewToken` should pulls this out
+        text: content.map((part) => part.text).join(""),
+        message: new AIMessageChunk({
+            id,
+            content,
+            tool_call_chunks,
+            usage_metadata,
+            additional_kwargs,
+            response_metadata,
+        }),
+        generationInfo,
+    });
+}
+function isBuiltInTool(tool) {
+    return "type" in tool && tool.type !== "function";
+}
+function isBuiltInToolChoice(tool_choice) {
+    return (tool_choice != null &&
+        typeof tool_choice === "object" &&
+        "type" in tool_choice &&
+        tool_choice.type !== "function");
+}
 function _convertChatOpenAIToolTypeToOpenAITool(tool, fields) {
     if (isOpenAITool(tool)) {
         if (fields?.strict !== undefined) {
@@ -899,6 +1212,12 @@ export class ChatOpenAI extends BaseChatModel {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "useResponsesApi", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         this.openAIApiKey =
             fields?.apiKey ??
                 fields?.openAIApiKey ??
@@ -928,6 +1247,7 @@ export class ChatOpenAI extends BaseChatModel {
         this.modalities = fields?.modalities;
         this.reasoningEffort = fields?.reasoningEffort;
         this.maxTokens = fields?.maxCompletionTokens ?? fields?.maxTokens;
+        this.useResponsesApi = fields?.useResponsesApi ?? this.useResponsesApi;
         if (this.model === "o1") {
             this.disableStreaming = true;
         }
@@ -965,7 +1285,9 @@ export class ChatOpenAI extends BaseChatModel {
             strict = this.supportsStrictToolCalling;
         }
         return this.bind({
-            tools: tools.map((tool) => _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
+            tools: tools.map((tool) => isBuiltInTool(tool)
+                ? tool
+                : _convertChatOpenAIToolTypeToOpenAITool(tool, { strict })),
             ...kwargs,
         });
     }
@@ -991,6 +1313,80 @@ export class ChatOpenAI extends BaseChatModel {
         else if (this.supportsStrictToolCalling !== undefined) {
             strict = this.supportsStrictToolCalling;
         }
+        if (this._useResponseApi(options)) {
+            const params = {
+                model: this.model,
+                temperature: this.temperature,
+                top_p: this.topP,
+                user: this.user,
+                // if include_usage is set or streamUsage then stream must be set to true.
+                stream: this.streaming,
+                previous_response_id: options?.previous_response_id,
+                truncation: options?.truncation,
+                include: options?.include,
+                tools: options?.tools?.length
+                    ? options.tools
+                        .map((tool) => {
+                        if (isBuiltInTool(tool)) {
+                            return tool;
+                        }
+                        else if (isOpenAITool(tool)) {
+                            return {
+                                type: "function",
+                                name: tool.function.name,
+                                parameters: tool.function.parameters,
+                                description: tool.function.description,
+                                strict,
+                            };
+                        }
+                        else {
+                            return null;
+                        }
+                    })
+                        .filter((tool) => tool !== null)
+                    : undefined,
+                tool_choice: isBuiltInToolChoice(options?.tool_choice)
+                    ? options?.tool_choice
+                    : (() => {
+                        const formatted = formatToOpenAIToolChoice(options?.tool_choice);
+                        if (typeof formatted === "object" && "type" in formatted) {
+                            return { type: "function", name: formatted.function.name };
+                        }
+                        else {
+                            return undefined;
+                        }
+                    })(),
+                text: (() => {
+                    if (options?.text)
+                        return options.text;
+                    const format = this.createResponseFormat(options?.response_format);
+                    if (format?.type === "json_schema") {
+                        if (format.json_schema.schema != null) {
+                            return {
+                                format: {
+                                    type: "json_schema",
+                                    schema: format.json_schema.schema,
+                                    description: format.json_schema.description,
+                                    name: format.json_schema.name,
+                                    strict: format.json_schema.strict,
+                                },
+                            };
+                        }
+                        return undefined;
+                    }
+                    return { format };
+                })(),
+                parallel_tool_calls: options?.parallel_tool_calls,
+                max_output_tokens: this.maxTokens === -1 ? undefined : this.maxTokens,
+                ...this.modelKwargs,
+            };
+            const reasoningEffort = options?.reasoning_effort ?? this.reasoningEffort;
+            if (reasoningEffort !== undefined) {
+                params.reasoning = { effort: reasoningEffort };
+            }
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            return params;
+        }
         let streamOptionsConfig = {};
         if (options?.stream_options !== undefined) {
             streamOptionsConfig = { stream_options: options.stream_options };
@@ -1044,6 +1440,7 @@ export class ChatOpenAI extends BaseChatModel {
         else {
             params.max_tokens = this.maxTokens === -1 ? undefined : this.maxTokens;
         }
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         return params;
     }
     _convertOpenAIChatCompletionMessageToBaseMessage(message, rawResponse) {
@@ -1187,6 +1584,20 @@ export class ChatOpenAI extends BaseChatModel {
         };
     }
     async *_streamResponseChunks(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            const streamIterable = await this.responseApiWithRetry({
+                ...this.invocationParams(options, { streaming: true }),
+                input: _convertMessagesToOpenAIResponsesParams(messages, this.model),
+                stream: true,
+            }, options);
+            for await (const data of streamIterable) {
+                const chunk = _convertOpenAIResponsesDeltaToBaseMessageChunk(data);
+                if (chunk == null)
+                    continue;
+                yield chunk;
+            }
+            return;
+        }
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
         const params = {
             ...this.invocationParams(options, {
@@ -1291,7 +1702,60 @@ export class ChatOpenAI extends BaseChatModel {
         return this._identifyingParams();
     }
     /** @ignore */
+    async _responseApiGenerate(messages, options, runManager) {
+        const invocationParams = this.invocationParams(options);
+        if (invocationParams.stream) {
+            const stream = this._streamResponseChunks(messages, options, runManager);
+            let finalChunk;
+            for await (const chunk of stream) {
+                chunk.message.response_metadata = {
+                    ...chunk.generationInfo,
+                    ...chunk.message.response_metadata,
+                };
+                finalChunk = finalChunk?.concat(chunk) ?? chunk;
+            }
+            return {
+                generations: finalChunk ? [finalChunk] : [],
+                llmOutput: {
+                    estimatedTokenUsage: finalChunk?.message
+                        ?.usage_metadata,
+                },
+            };
+        }
+        const input = _convertMessagesToOpenAIResponsesParams(messages, this.model);
+        const data = await this.responseApiWithRetry({ input, ...invocationParams }, { signal: options?.signal, ...options?.options });
+        return {
+            generations: [
+                {
+                    text: data.output_text,
+                    message: _convertOpenAIResponsesMessageToBaseMessage(data),
+                },
+            ],
+            llmOutput: {
+                id: data.id,
+                estimatedTokenUsage: data.usage
+                    ? {
+                        promptTokens: data.usage.input_tokens,
+                        completionTokens: data.usage.output_tokens,
+                        totalTokens: data.usage.total_tokens,
+                    }
+                    : undefined,
+            },
+        };
+    }
+    _useResponseApi(options) {
+        const usesBuiltInTools = options?.tools?.some(isBuiltInTool);
+        const hasResponsesOnlyKwargs = options?.previous_response_id != null ||
+            options?.text != null ||
+            options?.truncation != null ||
+            options?.include != null;
+        return this.useResponsesApi || usesBuiltInTools || hasResponsesOnlyKwargs;
+    }
+    /** @ignore */
     async _generate(messages, options, runManager) {
+        if (this._useResponseApi(options)) {
+            return this._responseApiGenerate(messages, options, runManager);
+        }
         const usageMetadata = {};
         const params = this.invocationParams(options);
         const messagesMapped = _convertMessagesToOpenAIParams(messages, this.model);
@@ -1529,6 +1993,22 @@ export class ChatOpenAI extends BaseChatModel {
             }
         });
     }
+    async responseApiWithRetry(request, options) {
+        return this.caller.call(async () => {
+            const requestOptions = this._getClientOptions(options);
+            try {
+                // use parse if dealing with json_schema
+                if (request.text?.format?.type === "json_schema" && !request.stream) {
+                    return await this.client.responses.parse(request, requestOptions);
+                }
+                return await this.client.responses.create(request, requestOptions);
+            }
+            catch (e) {
+                const error = wrapOpenAIClientError(e);
+                throw error;
+            }
+        });
+    }
     /**
      * Call the beta chat completions parse endpoint. This should only be called if
      * response_format is set to "json_object".
@@ -1651,7 +2131,13 @@ export class ChatOpenAI extends BaseChatModel {
                 },
             });
             if (isZodSchema(schema)) {
-                outputParser = StructuredOutputParser.fromZodSchema(schema);
+                const altParser = StructuredOutputParser.fromZodSchema(schema);
+                outputParser = RunnableLambda.from((aiMessage) => {
+                    if ("parsed" in aiMessage.additional_kwargs) {
+                        return aiMessage.additional_kwargs.parsed;
+                    }
+                    return altParser;
+                });
             }
             else {
                 outputParser = new JsonOutputParser();
@@ -1739,12 +2225,7 @@ export class ChatOpenAI extends BaseChatModel {
         const parsedWithFallback = parserAssign.withFallbacks({
             fallbacks: [parserNone],
         });
-        return RunnableSequence.from([
-            {
-                raw: llm,
-            },
-            parsedWithFallback,
-        ]);
+        return RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
     }
 }
 function isZodSchema(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {