@langchain/openai 0.4.5-rc.0 → 0.4.6

package/dist/chat_models.cjs

@@ -108,6 +108,27 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
         }
         if (role === "tool") {
             const toolMessage = lcMsg;
+            // Handle computer call output
+            if (toolMessage.additional_kwargs?.type === "computer_call_output") {
+                const output = (() => {
+                    if (typeof toolMessage.content === "string") {
+                        return {
+                            type: "computer_screenshot",
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        return toolMessage.content.find((i) => i.type === "computer_screenshot");
+                    }
+                    return toolMessage.content;
+                })();
+                return {
+                    type: "computer_call_output",
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                    id: toolMessage.id,
+                };
+            }
             return {
                 type: "function_call_output",
                 call_id: toolMessage.tool_call_id,
@@ -127,7 +148,8 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     name: toolCall.name,
                     arguments: JSON.stringify(toolCall.args),
                     call_id: toolCall.id,
-                    id: functionCallIds[toolCall.id],
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
                 }));
             }
             if (lcMsg.additional_kwargs.tool_calls != null) {
@@ -135,7 +157,8 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     type: "function_call",
                     name: toolCall.function.name,
                     call_id: toolCall.id,
-                    id: functionCallIds[toolCall.id],
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
                     arguments: toolCall.function.arguments,
                 }));
             }
@@ -229,6 +252,9 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
             // TODO: how to handle refusals?
             content.push(...item.content.map((part) => {
                 if (part.type === "output_text") {
+                    if ("parsed" in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
                     return {
                         type: "text",
                         text: part.text,
@@ -256,6 +282,8 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
                 }
                 invalid_tool_calls.push((0, openai_tools_1.makeInvalidToolCall)(fnAdapter, errMessage));
             }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
         }
         else if (item.type === "reasoning") {
             additional_kwargs.reasoning = item;
@@ -326,7 +354,15 @@ function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
         response_metadata.model = chunk.response.model;
     }
     else if (chunk.type === "response.completed") {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
         usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === "json_schema") {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== "id")
+                response_metadata[key] = value;
+        }
     }
     else if (chunk.type === "response.function_call_arguments.delta") {
         tool_call_chunks.push({
@@ -1966,8 +2002,11 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         return this.caller.call(async () => {
             const requestOptions = this._getClientOptions(options);
             try {
-                const res = await this.client.responses.create(request, requestOptions);
-                return res;
+                // use parse if dealing with json_schema
+                if (request.text?.format?.type === "json_schema" && !request.stream) {
+                    return await this.client.responses.parse(request, requestOptions);
+                }
+                return await this.client.responses.create(request, requestOptions);
             }
             catch (e) {
                 const error = (0, openai_js_1.wrapOpenAIClientError)(e);
@@ -2097,7 +2136,13 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
                 },
             });
             if (isZodSchema(schema)) {
-                outputParser = output_parsers_1.StructuredOutputParser.fromZodSchema(schema);
+                const altParser = output_parsers_1.StructuredOutputParser.fromZodSchema(schema);
+                outputParser = runnables_1.RunnableLambda.from((aiMessage) => {
+                    if ("parsed" in aiMessage.additional_kwargs) {
+                        return aiMessage.additional_kwargs.parsed;
+                    }
+                    return altParser;
+                });
             }
             else {
                 outputParser = new output_parsers_1.JsonOutputParser();
@@ -2185,12 +2230,7 @@ class ChatOpenAI extends chat_models_1.BaseChatModel {
         const parsedWithFallback = parserAssign.withFallbacks({
             fallbacks: [parserNone],
         });
-        return runnables_1.RunnableSequence.from([
-            {
-                raw: llm,
-            },
-            parsedWithFallback,
-        ]);
+        return runnables_1.RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
     }
 }
 exports.ChatOpenAI = ChatOpenAI;

package/dist/chat_models.js

@@ -4,7 +4,7 @@ import { ChatGenerationChunk, } from "@langchain/core/outputs";
 import { getEnvironmentVariable } from "@langchain/core/utils/env";
 import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
 import { isOpenAITool, } from "@langchain/core/language_models/base";
-import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
+import { RunnableLambda, RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
 import { JsonOutputParser, StructuredOutputParser, } from "@langchain/core/output_parsers";
 import { JsonOutputKeyToolsParser, convertLangChainToolCallToOpenAI, makeInvalidToolCall, parseToolCall, } from "@langchain/core/output_parsers/openai_tools";
 import { zodToJsonSchema } from "zod-to-json-schema";
@@ -103,6 +103,27 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
         }
         if (role === "tool") {
             const toolMessage = lcMsg;
+            // Handle computer call output
+            if (toolMessage.additional_kwargs?.type === "computer_call_output") {
+                const output = (() => {
+                    if (typeof toolMessage.content === "string") {
+                        return {
+                            type: "computer_screenshot",
+                            image_url: toolMessage.content,
+                        };
+                    }
+                    if (Array.isArray(toolMessage.content)) {
+                        return toolMessage.content.find((i) => i.type === "computer_screenshot");
+                    }
+                    return toolMessage.content;
+                })();
+                return {
+                    type: "computer_call_output",
+                    output,
+                    call_id: toolMessage.tool_call_id,
+                    id: toolMessage.id,
+                };
+            }
             return {
                 type: "function_call_output",
                 call_id: toolMessage.tool_call_id,
@@ -122,7 +143,8 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     name: toolCall.name,
                     arguments: JSON.stringify(toolCall.args),
                     call_id: toolCall.id,
-                    id: functionCallIds[toolCall.id],
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
                 }));
             }
             if (lcMsg.additional_kwargs.tool_calls != null) {
@@ -130,7 +152,8 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
                     type: "function_call",
                     name: toolCall.function.name,
                     call_id: toolCall.id,
-                    id: functionCallIds[toolCall.id],
+                    // @ts-expect-error Might come from a non-Responses API message
+                    id: functionCallIds?.[toolCall.id],
                     arguments: toolCall.function.arguments,
                 }));
             }
@@ -224,6 +247,9 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
             // TODO: how to handle refusals?
             content.push(...item.content.map((part) => {
                 if (part.type === "output_text") {
+                    if ("parsed" in part && part.parsed != null) {
+                        additional_kwargs.parsed = part.parsed;
+                    }
                     return {
                         type: "text",
                         text: part.text,
@@ -251,6 +277,8 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
                 }
                 invalid_tool_calls.push(makeInvalidToolCall(fnAdapter, errMessage));
             }
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY] ??= {};
+            additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY][item.call_id] = item.id;
         }
         else if (item.type === "reasoning") {
             additional_kwargs.reasoning = item;
@@ -321,7 +349,15 @@ function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
         response_metadata.model = chunk.response.model;
     }
     else if (chunk.type === "response.completed") {
+        const msg = _convertOpenAIResponsesMessageToBaseMessage(chunk.response);
         usage_metadata = chunk.response.usage;
+        if (chunk.response.text?.format?.type === "json_schema") {
+            additional_kwargs.parsed ??= JSON.parse(msg.text);
+        }
+        for (const [key, value] of Object.entries(chunk.response)) {
+            if (key !== "id")
+                response_metadata[key] = value;
+        }
     }
     else if (chunk.type === "response.function_call_arguments.delta") {
         tool_call_chunks.push({
@@ -1961,8 +1997,11 @@ export class ChatOpenAI extends BaseChatModel {
         return this.caller.call(async () => {
             const requestOptions = this._getClientOptions(options);
             try {
-                const res = await this.client.responses.create(request, requestOptions);
-                return res;
+                // use parse if dealing with json_schema
+                if (request.text?.format?.type === "json_schema" && !request.stream) {
+                    return await this.client.responses.parse(request, requestOptions);
+                }
+                return await this.client.responses.create(request, requestOptions);
             }
             catch (e) {
                 const error = wrapOpenAIClientError(e);
@@ -2092,7 +2131,13 @@ export class ChatOpenAI extends BaseChatModel {
                 },
             });
             if (isZodSchema(schema)) {
-                outputParser = StructuredOutputParser.fromZodSchema(schema);
+                const altParser = StructuredOutputParser.fromZodSchema(schema);
+                outputParser = RunnableLambda.from((aiMessage) => {
+                    if ("parsed" in aiMessage.additional_kwargs) {
+                        return aiMessage.additional_kwargs.parsed;
+                    }
+                    return altParser;
+                });
             }
             else {
                 outputParser = new JsonOutputParser();
@@ -2180,12 +2225,7 @@ export class ChatOpenAI extends BaseChatModel {
         const parsedWithFallback = parserAssign.withFallbacks({
             fallbacks: [parserNone],
         });
-        return RunnableSequence.from([
-            {
-                raw: llm,
-            },
-            parsedWithFallback,
-        ]);
+        return RunnableSequence.from([{ raw: llm }, parsedWithFallback]);
     }
 }
 function isZodSchema(

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/openai",
-  "version": "0.4.5-rc.0",
+  "version": "0.4.6",
   "description": "OpenAI integrations for LangChain.js",
   "type": "module",
   "engines": {