@langchain/openai 0.5.6 → 0.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/chat_models.cjs +138 -1
  2. package/dist/chat_models.js +139 -2
  3. package/package.json +2 -2
package/dist/chat_models.cjs CHANGED
@@ -50,6 +50,132 @@ function messageToOpenAIRole(message) {
50
50
  }
51
51
  }
52
52
  exports.messageToOpenAIRole = messageToOpenAIRole;
53
+ const completionsApiContentBlockConverter = {
54
+ providerName: "ChatOpenAI",
55
+ fromStandardTextBlock(block) {
56
+ return { type: "text", text: block.text };
57
+ },
58
+ fromStandardImageBlock(block) {
59
+ if (block.source_type === "url") {
60
+ return {
61
+ type: "image_url",
62
+ image_url: {
63
+ url: block.url,
64
+ ...(block.metadata?.detail
65
+ ? { detail: block.metadata.detail }
66
+ : {}),
67
+ },
68
+ };
69
+ }
70
+ if (block.source_type === "base64") {
71
+ const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
72
+ return {
73
+ type: "image_url",
74
+ image_url: {
75
+ url,
76
+ ...(block.metadata?.detail
77
+ ? { detail: block.metadata.detail }
78
+ : {}),
79
+ },
80
+ };
81
+ }
82
+ throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
83
+ },
84
+ fromStandardAudioBlock(block) {
85
+ if (block.source_type === "url") {
86
+ const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
87
+ if (!data) {
88
+ throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
89
+ }
90
+ const rawMimeType = data.mime_type || block.mime_type || "";
91
+ let mimeType;
92
+ try {
93
+ mimeType = (0, messages_1.parseMimeType)(rawMimeType);
94
+ }
95
+ catch {
96
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
97
+ }
98
+ if (mimeType.type !== "audio" ||
99
+ (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
100
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
101
+ }
102
+ return {
103
+ type: "input_audio",
104
+ input_audio: {
105
+ format: mimeType.subtype,
106
+ data: data.data,
107
+ },
108
+ };
109
+ }
110
+ if (block.source_type === "base64") {
111
+ let mimeType;
112
+ try {
113
+ mimeType = (0, messages_1.parseMimeType)(block.mime_type ?? "");
114
+ }
115
+ catch {
116
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
117
+ }
118
+ if (mimeType.type !== "audio" ||
119
+ (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
120
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
121
+ }
122
+ return {
123
+ type: "input_audio",
124
+ input_audio: {
125
+ format: mimeType.subtype,
126
+ data: block.data,
127
+ },
128
+ };
129
+ }
130
+ throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
131
+ },
132
+ fromStandardFileBlock(block) {
133
+ if (block.source_type === "url") {
134
+ const data = (0, messages_1.parseBase64DataUrl)({ dataUrl: block.url });
135
+ if (!data) {
136
+ throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
137
+ }
138
+ return {
139
+ type: "file",
140
+ file: {
141
+ file_data: block.url,
142
+ ...(block.metadata?.filename || block.metadata?.name
143
+ ? {
144
+ filename: (block.metadata?.filename ||
145
+ block.metadata?.name),
146
+ }
147
+ : {}),
148
+ },
149
+ };
150
+ }
151
+ if (block.source_type === "base64") {
152
+ return {
153
+ type: "file",
154
+ file: {
155
+ file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
156
+ ...(block.metadata?.filename ||
157
+ block.metadata?.name ||
158
+ block.metadata?.title
159
+ ? {
160
+ filename: (block.metadata?.filename ||
161
+ block.metadata?.name ||
162
+ block.metadata?.title),
163
+ }
164
+ : {}),
165
+ },
166
+ };
167
+ }
168
+ if (block.source_type === "id") {
169
+ return {
170
+ type: "file",
171
+ file: {
172
+ file_id: block.id,
173
+ },
174
+ };
175
+ }
176
+ throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
177
+ },
178
+ };
53
179
  // Used in LangSmith, export is important here
54
180
  function _convertMessagesToOpenAIParams(messages, model) {
55
181
  // TODO: Function messages do not support array content, fix cast
@@ -58,10 +184,18 @@ function _convertMessagesToOpenAIParams(messages, model) {
58
184
  if (role === "system" && isReasoningModel(model)) {
59
185
  role = "developer";
60
186
  }
187
+ const content = typeof message.content === "string"
188
+ ? message.content
189
+ : message.content.map((m) => {
190
+ if ((0, messages_1.isDataContentBlock)(m)) {
191
+ return (0, messages_1.convertToProviderContentBlock)(m, completionsApiContentBlockConverter);
192
+ }
193
+ return m;
194
+ });
61
195
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
62
196
  const completionParam = {
63
197
  role,
64
- content: message.content,
198
+ content,
65
199
  };
66
200
  if (message.name != null) {
67
201
  completionParam.name = message.name;
@@ -232,6 +366,9 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
232
366
  const content = typeof lcMsg.content === "string"
233
367
  ? lcMsg.content
234
368
  : lcMsg.content.flatMap((item) => {
369
+ if ((0, messages_1.isDataContentBlock)(item)) {
370
+ return (0, messages_1.convertToProviderContentBlock)(item, completionsApiContentBlockConverter);
371
+ }
235
372
  if (item.type === "text") {
236
373
  return { type: "input_text", text: item.text };
237
374
  }
package/dist/chat_models.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { OpenAI as OpenAIClient } from "openai";
2
- import { AIMessage, AIMessageChunk, ChatMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk, isAIMessage, } from "@langchain/core/messages";
2
+ import { AIMessage, AIMessageChunk, ChatMessage, ChatMessageChunk, FunctionMessageChunk, HumanMessageChunk, SystemMessageChunk, ToolMessageChunk, isAIMessage, parseBase64DataUrl, parseMimeType, convertToProviderContentBlock, isDataContentBlock, } from "@langchain/core/messages";
3
3
  import { ChatGenerationChunk, } from "@langchain/core/outputs";
4
4
  import { getEnvironmentVariable } from "@langchain/core/utils/env";
5
5
  import { BaseChatModel, } from "@langchain/core/language_models/chat_models";
@@ -46,6 +46,132 @@ export function messageToOpenAIRole(message) {
46
46
  throw new Error(`Unknown message type: ${type}`);
47
47
  }
48
48
  }
49
+ const completionsApiContentBlockConverter = {
50
+ providerName: "ChatOpenAI",
51
+ fromStandardTextBlock(block) {
52
+ return { type: "text", text: block.text };
53
+ },
54
+ fromStandardImageBlock(block) {
55
+ if (block.source_type === "url") {
56
+ return {
57
+ type: "image_url",
58
+ image_url: {
59
+ url: block.url,
60
+ ...(block.metadata?.detail
61
+ ? { detail: block.metadata.detail }
62
+ : {}),
63
+ },
64
+ };
65
+ }
66
+ if (block.source_type === "base64") {
67
+ const url = `data:${block.mime_type ?? ""};base64,${block.data}`;
68
+ return {
69
+ type: "image_url",
70
+ image_url: {
71
+ url,
72
+ ...(block.metadata?.detail
73
+ ? { detail: block.metadata.detail }
74
+ : {}),
75
+ },
76
+ };
77
+ }
78
+ throw new Error(`Image content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
79
+ },
80
+ fromStandardAudioBlock(block) {
81
+ if (block.source_type === "url") {
82
+ const data = parseBase64DataUrl({ dataUrl: block.url });
83
+ if (!data) {
84
+ throw new Error(`URL audio blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
85
+ }
86
+ const rawMimeType = data.mime_type || block.mime_type || "";
87
+ let mimeType;
88
+ try {
89
+ mimeType = parseMimeType(rawMimeType);
90
+ }
91
+ catch {
92
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
93
+ }
94
+ if (mimeType.type !== "audio" ||
95
+ (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
96
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
97
+ }
98
+ return {
99
+ type: "input_audio",
100
+ input_audio: {
101
+ format: mimeType.subtype,
102
+ data: data.data,
103
+ },
104
+ };
105
+ }
106
+ if (block.source_type === "base64") {
107
+ let mimeType;
108
+ try {
109
+ mimeType = parseMimeType(block.mime_type ?? "");
110
+ }
111
+ catch {
112
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
113
+ }
114
+ if (mimeType.type !== "audio" ||
115
+ (mimeType.subtype !== "wav" && mimeType.subtype !== "mp3")) {
116
+ throw new Error(`Audio blocks with source_type ${block.source_type} must have mime type of audio/wav or audio/mp3`);
117
+ }
118
+ return {
119
+ type: "input_audio",
120
+ input_audio: {
121
+ format: mimeType.subtype,
122
+ data: block.data,
123
+ },
124
+ };
125
+ }
126
+ throw new Error(`Audio content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
127
+ },
128
+ fromStandardFileBlock(block) {
129
+ if (block.source_type === "url") {
130
+ const data = parseBase64DataUrl({ dataUrl: block.url });
131
+ if (!data) {
132
+ throw new Error(`URL file blocks with source_type ${block.source_type} must be formatted as a data URL for ChatOpenAI`);
133
+ }
134
+ return {
135
+ type: "file",
136
+ file: {
137
+ file_data: block.url,
138
+ ...(block.metadata?.filename || block.metadata?.name
139
+ ? {
140
+ filename: (block.metadata?.filename ||
141
+ block.metadata?.name),
142
+ }
143
+ : {}),
144
+ },
145
+ };
146
+ }
147
+ if (block.source_type === "base64") {
148
+ return {
149
+ type: "file",
150
+ file: {
151
+ file_data: `data:${block.mime_type ?? ""};base64,${block.data}`,
152
+ ...(block.metadata?.filename ||
153
+ block.metadata?.name ||
154
+ block.metadata?.title
155
+ ? {
156
+ filename: (block.metadata?.filename ||
157
+ block.metadata?.name ||
158
+ block.metadata?.title),
159
+ }
160
+ : {}),
161
+ },
162
+ };
163
+ }
164
+ if (block.source_type === "id") {
165
+ return {
166
+ type: "file",
167
+ file: {
168
+ file_id: block.id,
169
+ },
170
+ };
171
+ }
172
+ throw new Error(`File content blocks with source_type ${block.source_type} are not supported for ChatOpenAI`);
173
+ },
174
+ };
49
175
  // Used in LangSmith, export is important here
50
176
  export function _convertMessagesToOpenAIParams(messages, model) {
51
177
  // TODO: Function messages do not support array content, fix cast
@@ -54,10 +180,18 @@ export function _convertMessagesToOpenAIParams(messages, model) {
54
180
  if (role === "system" && isReasoningModel(model)) {
55
181
  role = "developer";
56
182
  }
183
+ const content = typeof message.content === "string"
184
+ ? message.content
185
+ : message.content.map((m) => {
186
+ if (isDataContentBlock(m)) {
187
+ return convertToProviderContentBlock(m, completionsApiContentBlockConverter);
188
+ }
189
+ return m;
190
+ });
57
191
  // eslint-disable-next-line @typescript-eslint/no-explicit-any
58
192
  const completionParam = {
59
193
  role,
60
- content: message.content,
194
+ content,
61
195
  };
62
196
  if (message.name != null) {
63
197
  completionParam.name = message.name;
@@ -227,6 +361,9 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
227
361
  const content = typeof lcMsg.content === "string"
228
362
  ? lcMsg.content
229
363
  : lcMsg.content.flatMap((item) => {
364
+ if (isDataContentBlock(item)) {
365
+ return convertToProviderContentBlock(item, completionsApiContentBlockConverter);
366
+ }
230
367
  if (item.type === "text") {
231
368
  return { type: "input_text", text: item.text };
232
369
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@langchain/openai",
3
- "version": "0.5.6",
3
+ "version": "0.5.7",
4
4
  "description": "OpenAI integrations for LangChain.js",
5
5
  "type": "module",
6
6
  "engines": {
@@ -41,7 +41,7 @@
41
41
  "zod-to-json-schema": "^3.22.3"
42
42
  },
43
43
  "peerDependencies": {
44
- "@langchain/core": ">=0.3.39 <0.4.0"
44
+ "@langchain/core": ">=0.3.48 <0.4.0"
45
45
  },
46
46
  "devDependencies": {
47
47
  "@azure/identity": "^4.2.1",