@langchain/openai 1.5.12 → 1.6.0

package/dist/converters/responses.js

@@ -1,9 +1,10 @@
 import { isComputerToolCall, isCustomToolCall, parseComputerCall, parseCustomToolCall } from "../utils/tools.js";
-import { getFilenameFromMetadata, getRequiredFilenameFromMetadata, iife as iife$1, isReasoningModel, messageToOpenAIRole } from "../utils/misc.js";
+import { applyPromptCacheBreakpoint, getFilenameFromMetadata, getRequiredFilenameFromMetadata, iife as iife$1, isReasoningModel, messageToOpenAIRole } from "../utils/misc.js";
 import { completionsApiContentBlockConverter } from "./completions.js";
 import { AIMessage, AIMessageChunk, convertToProviderContentBlock, isDataContentBlock } from "@langchain/core/messages";
 import { ChatGenerationChunk } from "@langchain/core/outputs";
 import { makeInvalidToolCall, parseToolCall } from "@langchain/core/output_parsers/openai_tools";
+import { toResponseInputItems } from "openai/lib/responses/ResponseInputItems.js";
 //#region src/converters/responses.ts
 const _FUNCTION_CALL_IDS_MAP_KEY = "__openai_function_call_ids__";
 const _CUSTOM_TOOL_CALL_IDS_MAP_KEY = "__openai_custom_tool_call_ids__";
@@ -612,6 +613,51 @@ const convertResponsesDeltaToChatGenerationChunk = (event) => {
 		generationInfo
 	});
 };
+function resolveImageItem(block) {
+	const detail = iife$1(() => {
+		const raw = block.metadata?.detail;
+		if (raw === "low" || raw === "high" || raw === "auto") return raw;
+		return "auto";
+	});
+	if (block.fileId) return {
+		type: "input_image",
+		detail,
+		file_id: block.fileId
+	};
+	if (block.url) return {
+		type: "input_image",
+		detail,
+		image_url: block.url
+	};
+	if (block.data) {
+		const base64Data = typeof block.data === "string" ? block.data : Buffer.from(block.data).toString("base64");
+		return {
+			type: "input_image",
+			detail,
+			image_url: `data:${block.mimeType ?? "image/png"};base64,${base64Data}`
+		};
+	}
+}
+/** Converts tool content with images into a native `function_call_output` list. */
+function convertToolContentToResponsesOutput(message) {
+	if (!Array.isArray(message.content)) return;
+	const blocks = message.contentBlocks;
+	if (!blocks.some((block) => block.type === "image")) return;
+	return blocks.map((block) => {
+		if (block.type === "text") return {
+			type: "input_text",
+			text: block.text
+		};
+		if (block.type === "image") {
+			const image = resolveImageItem(block);
+			if (image) return image;
+		}
+		return {
+			type: "input_text",
+			text: JSON.stringify(block)
+		};
+	});
+}
 /**
 * Converts a single LangChain BaseMessage to OpenAI Responses API input format.
 *
@@ -663,6 +709,7 @@ const convertStandardContentMessageToResponsesInput = (message) => {
 				return "assistant";
 			}
 		});
+		const withBreakpoint = (block, part) => messageRole === "assistant" ? part : applyPromptCacheBreakpoint(block, part);
 		const makeTextPart = (text) => messageRole === "assistant" ? {
 			type: "output_text",
 			text,
@@ -700,31 +747,6 @@ const convertStandardContentMessageToResponsesInput = (message) => {
 				return "{}";
 			}
 		};
-		const resolveImageItem = (block) => {
-			const detail = iife$1(() => {
-				const raw = block.metadata?.detail;
-				if (raw === "low" || raw === "high" || raw === "auto") return raw;
-				return "auto";
-			});
-			if (block.fileId) return {
-				type: "input_image",
-				detail,
-				file_id: block.fileId
-			};
-			if (block.url) return {
-				type: "input_image",
-				detail,
-				image_url: block.url
-			};
-			if (block.data) {
-				const base64Data = typeof block.data === "string" ? block.data : Buffer.from(block.data).toString("base64");
-				return {
-					type: "input_image",
-					detail,
-					image_url: `data:${block.mimeType ?? "image/png"};base64,${base64Data}`
-				};
-			}
-		};
 		const resolveFileItem = (block) => {
 			if (block.fileId) {
 				const filename = getFilenameFromMetadata(block);
@@ -767,10 +789,12 @@ const convertStandardContentMessageToResponsesInput = (message) => {
 				type: "summary_text",
 				text: ""
 			}];
+			const encryptedContent = block.encrypted_content;
 			return {
 				type: "reasoning",
 				...block.id ? { id: block.id } : {},
-				summary
+				summary,
+				...typeof encryptedContent === "string" ? { encrypted_content: encryptedContent } : {}
 			};
 		};
 		const convertFunctionCall = (block) => ({
@@ -794,7 +818,7 @@ const convertStandardContentMessageToResponsesInput = (message) => {
 				if (!("extras" in block && typeof block.extras === "object" && block.extras !== null && "phase" in block.extras)) return void 0;
 				return block.extras.phase;
 			});
-			pushMessageContent([makeTextPart(block.text)], phase);
+			pushMessageContent([withBreakpoint(block, makeTextPart(block.text))], phase);
 		} else if (block.type === "invalid_tool_call") {} else if (block.type === "reasoning") {
 			yield* flushMessage();
 			yield convertReasoningBlock(block);
@@ -839,15 +863,15 @@ const convertStandardContentMessageToResponsesInput = (message) => {
 			yield convertFunctionCallOutput(block);
 		} else if (block.type === "audio") {} else if (block.type === "file") {
 			const fileItem = resolveFileItem(block);
-			if (fileItem) pushMessageContent([fileItem]);
+			if (fileItem) pushMessageContent([withBreakpoint(block, fileItem)]);
 		} else if (block.type === "image") {
 			const imageItem = resolveImageItem(block);
-			if (imageItem) pushMessageContent([imageItem]);
+			if (imageItem) pushMessageContent([withBreakpoint(block, imageItem)]);
 		} else if (block.type === "video") {
 			const videoItem = resolveFileItem(block);
-			if (videoItem) pushMessageContent([videoItem]);
+			if (videoItem) pushMessageContent([withBreakpoint(block, videoItem)]);
 		} else if (block.type === "text-plain") {
-			if (block.text) pushMessageContent([makeTextPart(block.text)]);
+			if (block.text) pushMessageContent([withBreakpoint(block, makeTextPart(block.text))]);
 		} else if (block.type === "non_standard" && isResponsesMessage) {
 			yield* flushMessage();
 			yield block.value;
@@ -960,15 +984,19 @@ const convertMessagesToResponsesInput = ({ messages, zdrEnabled, model }) => {
 				output: toolMessage.content
 			};
 			const isProviderNativeContent = Array.isArray(toolMessage.content) && toolMessage.content.every((item) => typeof item === "object" && item !== null && "type" in item && (item.type === "input_file" || item.type === "input_image" || item.type === "input_text"));
+			const attachmentOutput = isProviderNativeContent ? void 0 : convertToolContentToResponsesOutput(toolMessage);
 			return {
 				type: "function_call_output",
 				call_id: toolMessage.tool_call_id,
 				id: toolMessage.id?.startsWith("fc_") ? toolMessage.id : void 0,
-				output: isProviderNativeContent ? toolMessage.content : typeof toolMessage.content !== "string" ? JSON.stringify(toolMessage.content) : toolMessage.content
+				output: isProviderNativeContent ? toolMessage.content : attachmentOutput ?? (typeof toolMessage.content !== "string" ? JSON.stringify(toolMessage.content) : toolMessage.content)
 			};
 		}
 		if (role === "assistant") {
-			if (!zdrEnabled && responseMetadata?.output != null && Array.isArray(responseMetadata?.output) && responseMetadata?.output.length > 0 && responseMetadata?.output.every((item) => "type" in item)) return responseMetadata?.output;
+			if (responseMetadata?.output != null && Array.isArray(responseMetadata.output) && responseMetadata.output.length > 0 && responseMetadata.output.every((item) => typeof item === "object" && item != null && "type" in item && typeof item.type === "string")) {
+				const output = responseMetadata.output;
+				return zdrEnabled ? toResponseInputItems(output) : output;
+			}
 			const input = [];
 			const reasoning = additional_kwargs?.reasoning;
 			const hasEncryptedContent = !!reasoning?.encrypted_content;
@@ -1087,39 +1115,46 @@ const convertMessagesToResponsesInput = ({ messages, zdrEnabled, model }) => {
 				if (isDataContentBlock(item)) {
 					if (item.type === "file") {
 						const filename = getFilenameFromMetadata(item);
-						if (item.source_type === "url") return {
+						if (item.source_type === "url") return applyPromptCacheBreakpoint(item, {
 							type: "input_file",
 							file_url: item.url,
 							...filename ? { filename } : {}
-						};
-						if (item.source_type === "id") return {
+						});
+						if (item.source_type === "id") return applyPromptCacheBreakpoint(item, {
 							type: "input_file",
 							file_id: item.id,
 							...filename ? { filename } : {}
-						};
-						if (item.source_type === "base64") return {
-							type: "input_file",
-							file_data: `data:${item.mime_type ?? ""};base64,${item.data}`,
-							filename: getRequiredFilenameFromMetadata(item)
-						};
+						});
+						if (item.source_type === "base64") {
+							const mimeType = item.mime_type ?? "";
+							return applyPromptCacheBreakpoint(item, {
+								type: "input_file",
+								file_data: `data:${mimeType};base64,${item.data}`,
+								filename: getRequiredFilenameFromMetadata(item)
+							});
+						}
 					}
-					return convertToProviderContentBlock(item, completionsApiContentBlockConverter);
+					return applyPromptCacheBreakpoint(item, convertToProviderContentBlock(item, completionsApiContentBlockConverter));
 				}
-				if (item.type === "text") return {
+				if (item.type === "text") return applyPromptCacheBreakpoint(item, {
 					type: "input_text",
 					text: item.text
-				};
-				if (item.type === "image_url") return {
-					type: "input_image",
-					image_url: iife$1(() => {
+				});
+				if (item.type === "image_url") {
+					const imageUrl = iife$1(() => {
 						if (typeof item.image_url === "string") return item.image_url;
 						else if (typeof item.image_url === "object" && item.image_url !== null && "url" in item.image_url) return item.image_url.url;
-					}),
-					detail: iife$1(() => {
+					});
+					const detail = iife$1(() => {
 						if (typeof item.image_url === "string") return "auto";
 						else if (typeof item.image_url === "object" && item.image_url !== null && "detail" in item.image_url) return item.image_url.detail;
-					})
-				};
+					});
+					return applyPromptCacheBreakpoint(item, {
+						type: "input_image",
+						image_url: imageUrl,
+						detail
+					});
+				}
 				if (item.type === "input_text" || item.type === "input_image" || item.type === "input_file") return item;
 				return [];
 			});