@langchain/openai 0.4.8 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/chat_models.cjs +65 -29
  2. package/dist/chat_models.js +65 -29
  3. package/package.json +1 -1
package/dist/chat_models.cjs CHANGED
@@ -118,9 +118,20 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
118
118
  };
119
119
  }
120
120
  if (Array.isArray(toolMessage.content)) {
121
- return toolMessage.content.find((i) => i.type === "computer_screenshot");
121
+ const oaiScreenshot = toolMessage.content.find((i) => i.type === "computer_screenshot");
122
+ if (oaiScreenshot)
123
+ return oaiScreenshot;
124
+ const lcImage = toolMessage.content.find((i) => i.type === "image_url");
125
+ if (lcImage) {
126
+ return {
127
+ type: "computer_screenshot",
128
+ image_url: typeof lcImage.image_url === "string"
129
+ ? lcImage.image_url
130
+ : lcImage.image_url.url,
131
+ };
132
+ }
122
133
  }
123
- return toolMessage.content;
134
+ throw new Error("Invalid computer call output");
124
135
  })();
125
136
  return {
126
137
  type: "computer_call_output",
@@ -138,29 +149,18 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
138
149
  };
139
150
  }
140
151
  if (role === "assistant") {
141
- const functionCallIds =
142
- // eslint-disable-next-line @typescript-eslint/no-use-before-define
143
- lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
144
- if ((0, messages_1.isAIMessage)(lcMsg) && !!lcMsg.tool_calls?.length) {
145
- return lcMsg.tool_calls.map((toolCall) => ({
146
- type: "function_call",
147
- name: toolCall.name,
148
- arguments: JSON.stringify(toolCall.args),
149
- call_id: toolCall.id,
150
- // @ts-expect-error Might come from a non-Responses API message
151
- id: functionCallIds?.[toolCall.id],
152
- }));
153
- }
154
- if (lcMsg.additional_kwargs.tool_calls != null) {
155
- return lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
156
- type: "function_call",
157
- name: toolCall.function.name,
158
- call_id: toolCall.id,
159
- // @ts-expect-error Might come from a non-Responses API message
160
- id: functionCallIds?.[toolCall.id],
161
- arguments: toolCall.function.arguments,
162
- }));
152
+ const input = [];
153
+ // reasoning items
154
+ if (lcMsg.additional_kwargs.reasoning != null) {
155
+ const isReasoningItem = (item) => typeof item === "object" &&
156
+ item != null &&
157
+ "type" in item &&
158
+ item.type === "reasoning";
159
+ if (isReasoningItem(lcMsg.additional_kwargs.reasoning)) {
160
+ input.push(lcMsg.additional_kwargs.reasoning);
161
+ }
163
162
  }
163
+ // ai content
164
164
  let { content } = lcMsg;
165
165
  if (lcMsg.additional_kwargs.refusal != null) {
166
166
  if (typeof content === "string") {
@@ -171,7 +171,7 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
171
171
  { type: "refusal", refusal: lcMsg.additional_kwargs.refusal },
172
172
  ];
173
173
  }
174
- return {
174
+ input.push({
175
175
  type: "message",
176
176
  role: "assistant",
177
177
  content: typeof content === "string"
@@ -190,7 +190,39 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
190
190
  }
191
191
  return [];
192
192
  }),
193
- };
193
+ });
194
+ // function tool calls and computer use tool calls
195
+ const functionCallIds =
196
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
197
+ lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
198
+ if ((0, messages_1.isAIMessage)(lcMsg) && !!lcMsg.tool_calls?.length) {
199
+ input.push(...lcMsg.tool_calls.map((toolCall) => ({
200
+ type: "function_call",
201
+ name: toolCall.name,
202
+ arguments: JSON.stringify(toolCall.args),
203
+ call_id: toolCall.id,
204
+ // @ts-expect-error Might come from a non-Responses API message
205
+ id: functionCallIds?.[toolCall.id],
206
+ })));
207
+ }
208
+ else if (lcMsg.additional_kwargs.tool_calls != null) {
209
+ input.push(...lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
210
+ type: "function_call",
211
+ name: toolCall.function.name,
212
+ call_id: toolCall.id,
213
+ // @ts-expect-error Might come from a non-Responses API message
214
+ id: functionCallIds?.[toolCall.id],
215
+ arguments: toolCall.function.arguments,
216
+ })));
217
+ }
218
+ if (lcMsg.additional_kwargs.tool_outputs != null) {
219
+ const toolOutputs = lcMsg.additional_kwargs
220
+ .tool_outputs;
221
+ const computerCalls = toolOutputs?.filter((item) => item.type === "computer_call");
222
+ if (computerCalls.length > 0)
223
+ input.push(...computerCalls);
224
+ }
225
+ return input;
194
226
  }
195
227
  if (role === "user") {
196
228
  return {
@@ -248,8 +280,7 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
248
280
  const additional_kwargs = {};
249
281
  for (const item of response.output) {
250
282
  if (item.type === "message") {
251
- // TODO: how to handle refusals?
252
- content.push(...item.content.map((part) => {
283
+ content.push(...item.content.flatMap((part) => {
253
284
  if (part.type === "output_text") {
254
285
  if ("parsed" in part && part.parsed != null) {
255
286
  additional_kwargs.parsed = part.parsed;
@@ -260,6 +291,10 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
260
291
  annotations: part.annotations,
261
292
  };
262
293
  }
294
+ if (part.type === "refusal") {
295
+ additional_kwargs.refusal = part.refusal;
296
+ return [];
297
+ }
263
298
  return part;
264
299
  }));
265
300
  }
@@ -344,7 +379,8 @@ function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
344
379
  }
345
380
  else if (chunk.type === "response.output_item.done" &&
346
381
  (chunk.item.type === "web_search_call" ||
347
- chunk.item.type === "file_search_call")) {
382
+ chunk.item.type === "file_search_call" ||
383
+ chunk.item.type === "computer_call")) {
348
384
  additional_kwargs.tool_outputs = [chunk.item];
349
385
  }
350
386
  else if (chunk.type === "response.created") {
package/dist/chat_models.js CHANGED
@@ -113,9 +113,20 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
113
113
  };
114
114
  }
115
115
  if (Array.isArray(toolMessage.content)) {
116
- return toolMessage.content.find((i) => i.type === "computer_screenshot");
116
+ const oaiScreenshot = toolMessage.content.find((i) => i.type === "computer_screenshot");
117
+ if (oaiScreenshot)
118
+ return oaiScreenshot;
119
+ const lcImage = toolMessage.content.find((i) => i.type === "image_url");
120
+ if (lcImage) {
121
+ return {
122
+ type: "computer_screenshot",
123
+ image_url: typeof lcImage.image_url === "string"
124
+ ? lcImage.image_url
125
+ : lcImage.image_url.url,
126
+ };
127
+ }
117
128
  }
118
- return toolMessage.content;
129
+ throw new Error("Invalid computer call output");
119
130
  })();
120
131
  return {
121
132
  type: "computer_call_output",
@@ -133,29 +144,18 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
133
144
  };
134
145
  }
135
146
  if (role === "assistant") {
136
- const functionCallIds =
137
- // eslint-disable-next-line @typescript-eslint/no-use-before-define
138
- lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
139
- if (isAIMessage(lcMsg) && !!lcMsg.tool_calls?.length) {
140
- return lcMsg.tool_calls.map((toolCall) => ({
141
- type: "function_call",
142
- name: toolCall.name,
143
- arguments: JSON.stringify(toolCall.args),
144
- call_id: toolCall.id,
145
- // @ts-expect-error Might come from a non-Responses API message
146
- id: functionCallIds?.[toolCall.id],
147
- }));
148
- }
149
- if (lcMsg.additional_kwargs.tool_calls != null) {
150
- return lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
151
- type: "function_call",
152
- name: toolCall.function.name,
153
- call_id: toolCall.id,
154
- // @ts-expect-error Might come from a non-Responses API message
155
- id: functionCallIds?.[toolCall.id],
156
- arguments: toolCall.function.arguments,
157
- }));
147
+ const input = [];
148
+ // reasoning items
149
+ if (lcMsg.additional_kwargs.reasoning != null) {
150
+ const isReasoningItem = (item) => typeof item === "object" &&
151
+ item != null &&
152
+ "type" in item &&
153
+ item.type === "reasoning";
154
+ if (isReasoningItem(lcMsg.additional_kwargs.reasoning)) {
155
+ input.push(lcMsg.additional_kwargs.reasoning);
156
+ }
158
157
  }
158
+ // ai content
159
159
  let { content } = lcMsg;
160
160
  if (lcMsg.additional_kwargs.refusal != null) {
161
161
  if (typeof content === "string") {
@@ -166,7 +166,7 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
166
166
  { type: "refusal", refusal: lcMsg.additional_kwargs.refusal },
167
167
  ];
168
168
  }
169
- return {
169
+ input.push({
170
170
  type: "message",
171
171
  role: "assistant",
172
172
  content: typeof content === "string"
@@ -185,7 +185,39 @@ function _convertMessagesToOpenAIResponsesParams(messages, model) {
185
185
  }
186
186
  return [];
187
187
  }),
188
- };
188
+ });
189
+ // function tool calls and computer use tool calls
190
+ const functionCallIds =
191
+ // eslint-disable-next-line @typescript-eslint/no-use-before-define
192
+ lcMsg.additional_kwargs[_FUNCTION_CALL_IDS_MAP_KEY];
193
+ if (isAIMessage(lcMsg) && !!lcMsg.tool_calls?.length) {
194
+ input.push(...lcMsg.tool_calls.map((toolCall) => ({
195
+ type: "function_call",
196
+ name: toolCall.name,
197
+ arguments: JSON.stringify(toolCall.args),
198
+ call_id: toolCall.id,
199
+ // @ts-expect-error Might come from a non-Responses API message
200
+ id: functionCallIds?.[toolCall.id],
201
+ })));
202
+ }
203
+ else if (lcMsg.additional_kwargs.tool_calls != null) {
204
+ input.push(...lcMsg.additional_kwargs.tool_calls.map((toolCall) => ({
205
+ type: "function_call",
206
+ name: toolCall.function.name,
207
+ call_id: toolCall.id,
208
+ // @ts-expect-error Might come from a non-Responses API message
209
+ id: functionCallIds?.[toolCall.id],
210
+ arguments: toolCall.function.arguments,
211
+ })));
212
+ }
213
+ if (lcMsg.additional_kwargs.tool_outputs != null) {
214
+ const toolOutputs = lcMsg.additional_kwargs
215
+ .tool_outputs;
216
+ const computerCalls = toolOutputs?.filter((item) => item.type === "computer_call");
217
+ if (computerCalls.length > 0)
218
+ input.push(...computerCalls);
219
+ }
220
+ return input;
189
221
  }
190
222
  if (role === "user") {
191
223
  return {
@@ -243,8 +275,7 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
243
275
  const additional_kwargs = {};
244
276
  for (const item of response.output) {
245
277
  if (item.type === "message") {
246
- // TODO: how to handle refusals?
247
- content.push(...item.content.map((part) => {
278
+ content.push(...item.content.flatMap((part) => {
248
279
  if (part.type === "output_text") {
249
280
  if ("parsed" in part && part.parsed != null) {
250
281
  additional_kwargs.parsed = part.parsed;
@@ -255,6 +286,10 @@ function _convertOpenAIResponsesMessageToBaseMessage(response) {
255
286
  annotations: part.annotations,
256
287
  };
257
288
  }
289
+ if (part.type === "refusal") {
290
+ additional_kwargs.refusal = part.refusal;
291
+ return [];
292
+ }
258
293
  return part;
259
294
  }));
260
295
  }
@@ -339,7 +374,8 @@ function _convertOpenAIResponsesDeltaToBaseMessageChunk(chunk) {
339
374
  }
340
375
  else if (chunk.type === "response.output_item.done" &&
341
376
  (chunk.item.type === "web_search_call" ||
342
- chunk.item.type === "file_search_call")) {
377
+ chunk.item.type === "file_search_call" ||
378
+ chunk.item.type === "computer_call")) {
343
379
  additional_kwargs.tool_outputs = [chunk.item];
344
380
  }
345
381
  else if (chunk.type === "response.created") {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@langchain/openai",
3
- "version": "0.4.8",
3
+ "version": "0.4.9",
4
4
  "description": "OpenAI integrations for LangChain.js",
5
5
  "type": "module",
6
6
  "engines": {