@langchain/google-common 0.0.27 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.cjs +26 -5
- package/dist/auth.d.ts +3 -2
- package/dist/auth.js +26 -5
- package/dist/chat_models.cjs +13 -11
- package/dist/chat_models.d.ts +5 -6
- package/dist/chat_models.js +13 -12
- package/dist/connection.cjs +34 -7
- package/dist/connection.d.ts +14 -8
- package/dist/connection.js +32 -6
- package/dist/embeddings.cjs +4 -2
- package/dist/embeddings.js +4 -2
- package/dist/experimental/media.cjs +494 -0
- package/dist/experimental/media.d.ts +210 -0
- package/dist/experimental/media.js +478 -0
- package/dist/experimental/utils/media_core.cjs +524 -0
- package/dist/experimental/utils/media_core.d.ts +209 -0
- package/dist/experimental/utils/media_core.js +514 -0
- package/dist/llms.cjs +10 -5
- package/dist/llms.d.ts +4 -1
- package/dist/llms.js +11 -6
- package/dist/types.d.ts +8 -1
- package/dist/utils/gemini.cjs +493 -490
- package/dist/utils/gemini.d.ts +15 -50
- package/dist/utils/gemini.js +491 -464
- package/experimental/media.cjs +1 -0
- package/experimental/media.d.cts +1 -0
- package/experimental/media.d.ts +1 -0
- package/experimental/media.js +1 -0
- package/experimental/utils/media_core.cjs +1 -0
- package/experimental/utils/media_core.d.cts +1 -0
- package/experimental/utils/media_core.d.ts +1 -0
- package/experimental/utils/media_core.js +1 -0
- package/package.json +32 -3
package/dist/utils/gemini.js
CHANGED
|
@@ -11,73 +11,80 @@ const extractMimeType = (str) => {
|
|
|
11
11
|
}
|
|
12
12
|
return null;
|
|
13
13
|
};
|
|
14
|
-
function
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
14
|
+
export function getGeminiAPI(config) {
|
|
15
|
+
function messageContentText(content) {
|
|
16
|
+
if (content?.text && content?.text.length > 0) {
|
|
17
|
+
return {
|
|
18
|
+
text: content.text,
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
else {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
22
24
|
}
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
}
|
|
25
|
+
function messageContentImageUrl(content) {
|
|
26
|
+
const url = typeof content.image_url === "string"
|
|
27
|
+
? content.image_url
|
|
28
|
+
: content.image_url.url;
|
|
29
|
+
if (!url) {
|
|
30
|
+
throw new Error("Missing Image URL");
|
|
31
|
+
}
|
|
32
|
+
const mineTypeAndData = extractMimeType(url);
|
|
33
|
+
if (mineTypeAndData) {
|
|
34
|
+
return {
|
|
35
|
+
inlineData: mineTypeAndData,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
// FIXME - need some way to get mime type
|
|
40
|
+
return {
|
|
41
|
+
fileData: {
|
|
42
|
+
mimeType: "image/png",
|
|
43
|
+
fileUri: url,
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
36
47
|
}
|
|
37
|
-
|
|
38
|
-
// FIXME - need some way to get mime type
|
|
48
|
+
async function blobToFileData(blob) {
|
|
39
49
|
return {
|
|
40
50
|
fileData: {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
},
|
|
44
|
-
};
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
function messageContentMedia(
|
|
48
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
49
|
-
content) {
|
|
50
|
-
if ("mimeType" in content && "data" in content) {
|
|
51
|
-
return {
|
|
52
|
-
inlineData: {
|
|
53
|
-
mimeType: content.mimeType,
|
|
54
|
-
data: content.data,
|
|
51
|
+
fileUri: blob.path,
|
|
52
|
+
mimeType: blob.mimetype,
|
|
55
53
|
},
|
|
56
54
|
};
|
|
57
55
|
}
|
|
58
|
-
|
|
59
|
-
return
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
56
|
+
async function fileUriContentToBlob(uri) {
|
|
57
|
+
return config?.mediaManager?.getMediaBlob(uri);
|
|
58
|
+
}
|
|
59
|
+
async function messageContentMedia(
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
61
|
+
content) {
|
|
62
|
+
if ("mimeType" in content && "data" in content) {
|
|
63
|
+
return {
|
|
64
|
+
inlineData: {
|
|
65
|
+
mimeType: content.mimeType,
|
|
66
|
+
data: content.data,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
else if ("mimeType" in content && "fileUri" in content) {
|
|
71
|
+
return {
|
|
72
|
+
fileData: {
|
|
73
|
+
mimeType: content.mimeType,
|
|
74
|
+
fileUri: content.fileUri,
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
const uri = content.fileUri;
|
|
80
|
+
const blob = await fileUriContentToBlob(uri);
|
|
81
|
+
if (blob) {
|
|
82
|
+
return await blobToFileData(blob);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
throw new Error("Invalid media content");
|
|
65
86
|
}
|
|
66
|
-
|
|
67
|
-
}
|
|
68
|
-
export function messageContentToParts(content) {
|
|
69
|
-
// Convert a string to a text type MessageContent if needed
|
|
70
|
-
const messageContent = typeof content === "string"
|
|
71
|
-
? [
|
|
72
|
-
{
|
|
73
|
-
type: "text",
|
|
74
|
-
text: content,
|
|
75
|
-
},
|
|
76
|
-
]
|
|
77
|
-
: content;
|
|
78
|
-
// eslint-disable-next-line array-callback-return
|
|
79
|
-
const parts = messageContent
|
|
80
|
-
.map((content) => {
|
|
87
|
+
async function messageContentComplexToPart(content) {
|
|
81
88
|
switch (content.type) {
|
|
82
89
|
case "text":
|
|
83
90
|
if ("text" in content) {
|
|
@@ -91,464 +98,484 @@ export function messageContentToParts(content) {
|
|
|
91
98
|
}
|
|
92
99
|
break;
|
|
93
100
|
case "media":
|
|
94
|
-
return messageContentMedia(content);
|
|
101
|
+
return await messageContentMedia(content);
|
|
95
102
|
default:
|
|
96
103
|
throw new Error(`Unsupported type received while converting message to message parts`);
|
|
97
104
|
}
|
|
98
105
|
throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
106
|
+
}
|
|
107
|
+
async function messageContentComplexToParts(content) {
|
|
108
|
+
const contents = content.map(messageContentComplexToPart);
|
|
109
|
+
return Promise.all(contents);
|
|
110
|
+
}
|
|
111
|
+
async function messageContentToParts(content) {
|
|
112
|
+
// Convert a string to a text type MessageContent if needed
|
|
113
|
+
const messageContent = typeof content === "string"
|
|
114
|
+
? [
|
|
115
|
+
{
|
|
116
|
+
type: "text",
|
|
117
|
+
text: content,
|
|
118
|
+
},
|
|
119
|
+
]
|
|
120
|
+
: content;
|
|
121
|
+
// Get all of the parts, even those that don't correctly resolve
|
|
122
|
+
const allParts = await messageContentComplexToParts(messageContent);
|
|
123
|
+
// Remove any invalid parts
|
|
124
|
+
const parts = allParts.reduce((acc, val) => {
|
|
125
|
+
if (val) {
|
|
126
|
+
return [...acc, val];
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
return acc;
|
|
130
|
+
}
|
|
131
|
+
}, []);
|
|
132
|
+
return parts;
|
|
133
|
+
}
|
|
134
|
+
function messageToolCallsToParts(toolCalls) {
|
|
135
|
+
if (!toolCalls || toolCalls.length === 0) {
|
|
136
|
+
return [];
|
|
137
|
+
}
|
|
138
|
+
return toolCalls.map((tool) => {
|
|
139
|
+
let args = {};
|
|
140
|
+
if (tool?.function?.arguments) {
|
|
141
|
+
const argStr = tool.function.arguments;
|
|
142
|
+
args = JSON.parse(argStr);
|
|
143
|
+
}
|
|
144
|
+
return {
|
|
145
|
+
functionCall: {
|
|
146
|
+
name: tool.function.name,
|
|
147
|
+
args,
|
|
148
|
+
},
|
|
149
|
+
};
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
function messageKwargsToParts(kwargs) {
|
|
153
|
+
const ret = [];
|
|
154
|
+
if (kwargs?.tool_calls) {
|
|
155
|
+
ret.push(...messageToolCallsToParts(kwargs.tool_calls));
|
|
156
|
+
}
|
|
157
|
+
return ret;
|
|
158
|
+
}
|
|
159
|
+
async function roleMessageToContent(role, message) {
|
|
160
|
+
const contentParts = await messageContentToParts(message.content);
|
|
161
|
+
let toolParts;
|
|
162
|
+
if (isAIMessage(message) && !!message.tool_calls?.length) {
|
|
163
|
+
toolParts = message.tool_calls.map((toolCall) => ({
|
|
164
|
+
functionCall: {
|
|
165
|
+
name: toolCall.name,
|
|
166
|
+
args: toolCall.args,
|
|
167
|
+
},
|
|
168
|
+
}));
|
|
103
169
|
}
|
|
104
170
|
else {
|
|
105
|
-
|
|
171
|
+
toolParts = messageKwargsToParts(message.additional_kwargs);
|
|
172
|
+
}
|
|
173
|
+
const parts = [...contentParts, ...toolParts];
|
|
174
|
+
return [
|
|
175
|
+
{
|
|
176
|
+
role,
|
|
177
|
+
parts,
|
|
178
|
+
},
|
|
179
|
+
];
|
|
180
|
+
}
|
|
181
|
+
async function systemMessageToContent(message, useSystemInstruction) {
|
|
182
|
+
return useSystemInstruction
|
|
183
|
+
? roleMessageToContent("system", message)
|
|
184
|
+
: [
|
|
185
|
+
...(await roleMessageToContent("user", message)),
|
|
186
|
+
...(await roleMessageToContent("model", new AIMessage("Ok"))),
|
|
187
|
+
];
|
|
188
|
+
}
|
|
189
|
+
function toolMessageToContent(message, prevMessage) {
|
|
190
|
+
const contentStr = typeof message.content === "string"
|
|
191
|
+
? message.content
|
|
192
|
+
: message.content.reduce((acc, content) => {
|
|
193
|
+
if (content.type === "text") {
|
|
194
|
+
return acc + content.text;
|
|
195
|
+
}
|
|
196
|
+
else {
|
|
197
|
+
return acc;
|
|
198
|
+
}
|
|
199
|
+
}, "");
|
|
200
|
+
// Hacky :(
|
|
201
|
+
const responseName = (isAIMessage(prevMessage) && !!prevMessage.tool_calls?.length
|
|
202
|
+
? prevMessage.tool_calls[0].name
|
|
203
|
+
: prevMessage.name) ?? message.tool_call_id;
|
|
204
|
+
try {
|
|
205
|
+
const content = JSON.parse(contentStr);
|
|
206
|
+
return [
|
|
207
|
+
{
|
|
208
|
+
role: "function",
|
|
209
|
+
parts: [
|
|
210
|
+
{
|
|
211
|
+
functionResponse: {
|
|
212
|
+
name: responseName,
|
|
213
|
+
response: { content },
|
|
214
|
+
},
|
|
215
|
+
},
|
|
216
|
+
],
|
|
217
|
+
},
|
|
218
|
+
];
|
|
219
|
+
}
|
|
220
|
+
catch (_) {
|
|
221
|
+
return [
|
|
222
|
+
{
|
|
223
|
+
role: "function",
|
|
224
|
+
parts: [
|
|
225
|
+
{
|
|
226
|
+
functionResponse: {
|
|
227
|
+
name: responseName,
|
|
228
|
+
response: { content: contentStr },
|
|
229
|
+
},
|
|
230
|
+
},
|
|
231
|
+
],
|
|
232
|
+
},
|
|
233
|
+
];
|
|
106
234
|
}
|
|
107
|
-
}, []);
|
|
108
|
-
return parts;
|
|
109
|
-
}
|
|
110
|
-
function messageToolCallsToParts(toolCalls) {
|
|
111
|
-
if (!toolCalls || toolCalls.length === 0) {
|
|
112
|
-
return [];
|
|
113
235
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
236
|
+
async function baseMessageToContent(message, prevMessage, useSystemInstruction) {
|
|
237
|
+
const type = message._getType();
|
|
238
|
+
switch (type) {
|
|
239
|
+
case "system":
|
|
240
|
+
return systemMessageToContent(message, useSystemInstruction);
|
|
241
|
+
case "human":
|
|
242
|
+
return roleMessageToContent("user", message);
|
|
243
|
+
case "ai":
|
|
244
|
+
return roleMessageToContent("model", message);
|
|
245
|
+
case "tool":
|
|
246
|
+
if (!prevMessage) {
|
|
247
|
+
throw new Error("Tool messages cannot be the first message passed to the model.");
|
|
248
|
+
}
|
|
249
|
+
return toolMessageToContent(message, prevMessage);
|
|
250
|
+
default:
|
|
251
|
+
console.log(`Unsupported message type: ${type}`);
|
|
252
|
+
return [];
|
|
119
253
|
}
|
|
254
|
+
}
|
|
255
|
+
function textPartToMessageContent(part) {
|
|
120
256
|
return {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
args,
|
|
124
|
-
},
|
|
257
|
+
type: "text",
|
|
258
|
+
text: part.text,
|
|
125
259
|
};
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
function messageKwargsToParts(kwargs) {
|
|
129
|
-
const ret = [];
|
|
130
|
-
if (kwargs?.tool_calls) {
|
|
131
|
-
ret.push(...messageToolCallsToParts(kwargs.tool_calls));
|
|
132
260
|
}
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
? roleMessageToContent("system", message)
|
|
160
|
-
: [
|
|
161
|
-
...roleMessageToContent("user", message),
|
|
162
|
-
...roleMessageToContent("model", new AIMessage("Ok")),
|
|
163
|
-
];
|
|
164
|
-
}
|
|
165
|
-
function toolMessageToContent(message, prevMessage) {
|
|
166
|
-
const contentStr = typeof message.content === "string"
|
|
167
|
-
? message.content
|
|
168
|
-
: message.content.reduce((acc, content) => {
|
|
169
|
-
if (content.type === "text") {
|
|
170
|
-
return acc + content.text;
|
|
261
|
+
function inlineDataPartToMessageContent(part) {
|
|
262
|
+
return {
|
|
263
|
+
type: "image_url",
|
|
264
|
+
image_url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`,
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
function fileDataPartToMessageContent(part) {
|
|
268
|
+
return {
|
|
269
|
+
type: "image_url",
|
|
270
|
+
image_url: part.fileData.fileUri,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
function partsToMessageContent(parts) {
|
|
274
|
+
return parts
|
|
275
|
+
.map((part) => {
|
|
276
|
+
if (part === undefined || part === null) {
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
else if ("text" in part) {
|
|
280
|
+
return textPartToMessageContent(part);
|
|
281
|
+
}
|
|
282
|
+
else if ("inlineData" in part) {
|
|
283
|
+
return inlineDataPartToMessageContent(part);
|
|
284
|
+
}
|
|
285
|
+
else if ("fileData" in part) {
|
|
286
|
+
return fileDataPartToMessageContent(part);
|
|
171
287
|
}
|
|
172
288
|
else {
|
|
173
|
-
return
|
|
289
|
+
return null;
|
|
174
290
|
}
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
},
|
|
191
|
-
},
|
|
192
|
-
],
|
|
291
|
+
})
|
|
292
|
+
.reduce((acc, content) => {
|
|
293
|
+
if (content) {
|
|
294
|
+
acc.push(content);
|
|
295
|
+
}
|
|
296
|
+
return acc;
|
|
297
|
+
}, []);
|
|
298
|
+
}
|
|
299
|
+
function toolRawToTool(raw) {
|
|
300
|
+
return {
|
|
301
|
+
id: raw.id,
|
|
302
|
+
type: raw.type,
|
|
303
|
+
function: {
|
|
304
|
+
name: raw.function.name,
|
|
305
|
+
arguments: JSON.stringify(raw.function.arguments),
|
|
193
306
|
},
|
|
194
|
-
|
|
307
|
+
};
|
|
195
308
|
}
|
|
196
|
-
|
|
197
|
-
return
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
name: responseName,
|
|
204
|
-
response: { content: contentStr },
|
|
205
|
-
},
|
|
206
|
-
},
|
|
207
|
-
],
|
|
309
|
+
function functionCallPartToToolRaw(part) {
|
|
310
|
+
return {
|
|
311
|
+
id: uuidv4().replace(/-/g, ""),
|
|
312
|
+
type: "function",
|
|
313
|
+
function: {
|
|
314
|
+
name: part.functionCall.name,
|
|
315
|
+
arguments: part.functionCall.args ?? {},
|
|
208
316
|
},
|
|
209
|
-
|
|
317
|
+
};
|
|
210
318
|
}
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
return systemMessageToContent(message, useSystemInstruction);
|
|
217
|
-
case "human":
|
|
218
|
-
return roleMessageToContent("user", message);
|
|
219
|
-
case "ai":
|
|
220
|
-
return roleMessageToContent("model", message);
|
|
221
|
-
case "tool":
|
|
222
|
-
if (!prevMessage) {
|
|
223
|
-
throw new Error("Tool messages cannot be the first message passed to the model.");
|
|
319
|
+
function partsToToolsRaw(parts) {
|
|
320
|
+
return parts
|
|
321
|
+
.map((part) => {
|
|
322
|
+
if (part === undefined || part === null) {
|
|
323
|
+
return null;
|
|
224
324
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
325
|
+
else if ("functionCall" in part) {
|
|
326
|
+
return functionCallPartToToolRaw(part);
|
|
327
|
+
}
|
|
328
|
+
else {
|
|
329
|
+
return null;
|
|
330
|
+
}
|
|
331
|
+
})
|
|
332
|
+
.reduce((acc, content) => {
|
|
333
|
+
if (content) {
|
|
334
|
+
acc.push(content);
|
|
335
|
+
}
|
|
336
|
+
return acc;
|
|
337
|
+
}, []);
|
|
229
338
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
}
|
|
237
|
-
function inlineDataPartToMessageContent(part) {
|
|
238
|
-
return {
|
|
239
|
-
type: "image_url",
|
|
240
|
-
image_url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`,
|
|
241
|
-
};
|
|
242
|
-
}
|
|
243
|
-
function fileDataPartToMessageContent(part) {
|
|
244
|
-
return {
|
|
245
|
-
type: "image_url",
|
|
246
|
-
image_url: part.fileData.fileUri,
|
|
247
|
-
};
|
|
248
|
-
}
|
|
249
|
-
export function partsToMessageContent(parts) {
|
|
250
|
-
return parts
|
|
251
|
-
.map((part) => {
|
|
252
|
-
if (part === undefined || part === null) {
|
|
253
|
-
return null;
|
|
254
|
-
}
|
|
255
|
-
else if ("text" in part) {
|
|
256
|
-
return textPartToMessageContent(part);
|
|
257
|
-
}
|
|
258
|
-
else if ("inlineData" in part) {
|
|
259
|
-
return inlineDataPartToMessageContent(part);
|
|
260
|
-
}
|
|
261
|
-
else if ("fileData" in part) {
|
|
262
|
-
return fileDataPartToMessageContent(part);
|
|
263
|
-
}
|
|
264
|
-
else {
|
|
265
|
-
return null;
|
|
266
|
-
}
|
|
267
|
-
})
|
|
268
|
-
.reduce((acc, content) => {
|
|
269
|
-
if (content) {
|
|
270
|
-
acc.push(content);
|
|
271
|
-
}
|
|
272
|
-
return acc;
|
|
273
|
-
}, []);
|
|
274
|
-
}
|
|
275
|
-
function toolRawToTool(raw) {
|
|
276
|
-
return {
|
|
277
|
-
id: raw.id,
|
|
278
|
-
type: raw.type,
|
|
279
|
-
function: {
|
|
280
|
-
name: raw.function.name,
|
|
281
|
-
arguments: JSON.stringify(raw.function.arguments),
|
|
282
|
-
},
|
|
283
|
-
};
|
|
284
|
-
}
|
|
285
|
-
function functionCallPartToToolRaw(part) {
|
|
286
|
-
return {
|
|
287
|
-
id: uuidv4().replace(/-/g, ""),
|
|
288
|
-
type: "function",
|
|
289
|
-
function: {
|
|
290
|
-
name: part.functionCall.name,
|
|
291
|
-
arguments: part.functionCall.args ?? {},
|
|
292
|
-
},
|
|
293
|
-
};
|
|
294
|
-
}
|
|
295
|
-
export function partsToToolsRaw(parts) {
|
|
296
|
-
return parts
|
|
297
|
-
.map((part) => {
|
|
298
|
-
if (part === undefined || part === null) {
|
|
299
|
-
return null;
|
|
339
|
+
function toolsRawToTools(raws) {
|
|
340
|
+
return raws.map((raw) => toolRawToTool(raw));
|
|
341
|
+
}
|
|
342
|
+
function responseToGenerateContentResponseData(response) {
|
|
343
|
+
if ("nextChunk" in response.data) {
|
|
344
|
+
throw new Error("Cannot convert Stream to GenerateContentResponseData");
|
|
300
345
|
}
|
|
301
|
-
else if (
|
|
302
|
-
|
|
346
|
+
else if (Array.isArray(response.data)) {
|
|
347
|
+
// Collapse the array of response data as if it was a single one
|
|
348
|
+
return response.data.reduce((acc, val) => {
|
|
349
|
+
// Add all the parts
|
|
350
|
+
// FIXME: Handle other candidates?
|
|
351
|
+
const valParts = val?.candidates?.[0]?.content?.parts ?? [];
|
|
352
|
+
acc.candidates[0].content.parts.push(...valParts);
|
|
353
|
+
// FIXME: Merge promptFeedback and safety settings
|
|
354
|
+
acc.promptFeedback = val.promptFeedback;
|
|
355
|
+
return acc;
|
|
356
|
+
});
|
|
303
357
|
}
|
|
304
358
|
else {
|
|
305
|
-
return
|
|
306
|
-
}
|
|
307
|
-
})
|
|
308
|
-
.reduce((acc, content) => {
|
|
309
|
-
if (content) {
|
|
310
|
-
acc.push(content);
|
|
359
|
+
return response.data;
|
|
311
360
|
}
|
|
312
|
-
return acc;
|
|
313
|
-
}, []);
|
|
314
|
-
}
|
|
315
|
-
export function toolsRawToTools(raws) {
|
|
316
|
-
return raws.map((raw) => toolRawToTool(raw));
|
|
317
|
-
}
|
|
318
|
-
export function responseToGenerateContentResponseData(response) {
|
|
319
|
-
if ("nextChunk" in response.data) {
|
|
320
|
-
throw new Error("Cannot convert Stream to GenerateContentResponseData");
|
|
321
|
-
}
|
|
322
|
-
else if (Array.isArray(response.data)) {
|
|
323
|
-
// Collapse the array of response data as if it was a single one
|
|
324
|
-
return response.data.reduce((acc, val) => {
|
|
325
|
-
// Add all the parts
|
|
326
|
-
// FIXME: Handle other candidates?
|
|
327
|
-
const valParts = val?.candidates?.[0]?.content?.parts ?? [];
|
|
328
|
-
acc.candidates[0].content.parts.push(...valParts);
|
|
329
|
-
// FIXME: Merge promptFeedback and safety settings
|
|
330
|
-
acc.promptFeedback = val.promptFeedback;
|
|
331
|
-
return acc;
|
|
332
|
-
});
|
|
333
361
|
}
|
|
334
|
-
|
|
335
|
-
|
|
362
|
+
function responseToParts(response) {
|
|
363
|
+
const responseData = responseToGenerateContentResponseData(response);
|
|
364
|
+
const parts = responseData?.candidates?.[0]?.content?.parts ?? [];
|
|
365
|
+
return parts;
|
|
336
366
|
}
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
const responseData = responseToGenerateContentResponseData(response);
|
|
340
|
-
const parts = responseData?.candidates?.[0]?.content?.parts ?? [];
|
|
341
|
-
return parts;
|
|
342
|
-
}
|
|
343
|
-
export function partToText(part) {
|
|
344
|
-
return "text" in part ? part.text : "";
|
|
345
|
-
}
|
|
346
|
-
export function responseToString(response) {
|
|
347
|
-
const parts = responseToParts(response);
|
|
348
|
-
const ret = parts.reduce((acc, part) => {
|
|
349
|
-
const val = partToText(part);
|
|
350
|
-
return acc + val;
|
|
351
|
-
}, "");
|
|
352
|
-
return ret;
|
|
353
|
-
}
|
|
354
|
-
function safeResponseTo(response, safetyHandler, responseTo) {
|
|
355
|
-
try {
|
|
356
|
-
const safeResponse = safetyHandler.handle(response);
|
|
357
|
-
return responseTo(safeResponse);
|
|
367
|
+
function partToText(part) {
|
|
368
|
+
return "text" in part ? part.text : "";
|
|
358
369
|
}
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
const
|
|
363
|
-
|
|
370
|
+
function responseToString(response) {
|
|
371
|
+
const parts = responseToParts(response);
|
|
372
|
+
const ret = parts.reduce((acc, part) => {
|
|
373
|
+
const val = partToText(part);
|
|
374
|
+
return acc + val;
|
|
375
|
+
}, "");
|
|
376
|
+
return ret;
|
|
377
|
+
}
|
|
378
|
+
function safeResponseTo(response, safetyHandler, responseTo) {
|
|
379
|
+
try {
|
|
380
|
+
const safeResponse = safetyHandler.handle(response);
|
|
381
|
+
return responseTo(safeResponse);
|
|
382
|
+
}
|
|
383
|
+
catch (xx) {
|
|
384
|
+
// eslint-disable-next-line no-instanceof/no-instanceof
|
|
385
|
+
if (xx instanceof GoogleAISafetyError) {
|
|
386
|
+
const ret = responseTo(xx.response);
|
|
387
|
+
xx.reply = ret;
|
|
388
|
+
}
|
|
389
|
+
throw xx;
|
|
364
390
|
}
|
|
365
|
-
throw xx;
|
|
366
391
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
return safeResponseTo(response, safetyHandler, responseToString);
|
|
370
|
-
}
|
|
371
|
-
export function responseToGenerationInfo(response) {
|
|
372
|
-
if (!Array.isArray(response.data)) {
|
|
373
|
-
return {};
|
|
392
|
+
function safeResponseToString(response, safetyHandler) {
|
|
393
|
+
return safeResponseTo(response, safetyHandler, responseToString);
|
|
374
394
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
generationInfo: responseToGenerationInfo(response),
|
|
396
|
-
};
|
|
397
|
-
}
|
|
398
|
-
export function safeResponseToGeneration(response, safetyHandler) {
|
|
399
|
-
return safeResponseTo(response, safetyHandler, responseToGeneration);
|
|
400
|
-
}
|
|
401
|
-
export function responseToChatGeneration(response) {
|
|
402
|
-
return new ChatGenerationChunk({
|
|
403
|
-
text: responseToString(response),
|
|
404
|
-
message: partToMessageChunk(responseToParts(response)[0]),
|
|
405
|
-
generationInfo: responseToGenerationInfo(response),
|
|
406
|
-
});
|
|
407
|
-
}
|
|
408
|
-
export function safeResponseToChatGeneration(response, safetyHandler) {
|
|
409
|
-
return safeResponseTo(response, safetyHandler, responseToChatGeneration);
|
|
410
|
-
}
|
|
411
|
-
export function chunkToString(chunk) {
|
|
412
|
-
if (chunk === null) {
|
|
413
|
-
return "";
|
|
414
|
-
}
|
|
415
|
-
else if (typeof chunk.content === "string") {
|
|
416
|
-
return chunk.content;
|
|
395
|
+
function responseToGenerationInfo(response) {
|
|
396
|
+
if (!Array.isArray(response.data)) {
|
|
397
|
+
return {};
|
|
398
|
+
}
|
|
399
|
+
const data = response.data[0];
|
|
400
|
+
return {
|
|
401
|
+
usage_metadata: {
|
|
402
|
+
prompt_token_count: data.usageMetadata?.promptTokenCount,
|
|
403
|
+
candidates_token_count: data.usageMetadata?.candidatesTokenCount,
|
|
404
|
+
total_token_count: data.usageMetadata?.totalTokenCount,
|
|
405
|
+
},
|
|
406
|
+
safety_ratings: data.candidates[0]?.safetyRatings?.map((rating) => ({
|
|
407
|
+
category: rating.category,
|
|
408
|
+
probability: rating.probability,
|
|
409
|
+
probability_score: rating.probabilityScore,
|
|
410
|
+
severity: rating.severity,
|
|
411
|
+
severity_score: rating.severityScore,
|
|
412
|
+
})),
|
|
413
|
+
finish_reason: data.candidates[0]?.finishReason,
|
|
414
|
+
};
|
|
417
415
|
}
|
|
418
|
-
|
|
419
|
-
return
|
|
416
|
+
function responseToChatGeneration(response) {
|
|
417
|
+
return new ChatGenerationChunk({
|
|
418
|
+
text: responseToString(response),
|
|
419
|
+
message: partToMessageChunk(responseToParts(response)[0]),
|
|
420
|
+
generationInfo: responseToGenerationInfo(response),
|
|
421
|
+
});
|
|
420
422
|
}
|
|
421
|
-
|
|
422
|
-
return
|
|
423
|
+
function safeResponseToChatGeneration(response, safetyHandler) {
|
|
424
|
+
return safeResponseTo(response, safetyHandler, responseToChatGeneration);
|
|
423
425
|
}
|
|
424
|
-
|
|
425
|
-
|
|
426
|
+
function chunkToString(chunk) {
|
|
427
|
+
if (chunk === null) {
|
|
428
|
+
return "";
|
|
429
|
+
}
|
|
430
|
+
else if (typeof chunk.content === "string") {
|
|
431
|
+
return chunk.content;
|
|
432
|
+
}
|
|
433
|
+
else if (chunk.content.length === 0) {
|
|
434
|
+
return "";
|
|
435
|
+
}
|
|
436
|
+
else if (chunk.content[0].type === "text") {
|
|
437
|
+
return chunk.content[0].text;
|
|
438
|
+
}
|
|
439
|
+
else {
|
|
440
|
+
throw new Error(`Unexpected chunk: ${chunk}`);
|
|
441
|
+
}
|
|
426
442
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
443
|
+
function partToMessageChunk(part) {
|
|
444
|
+
const fields = partsToBaseMessageChunkFields([part]);
|
|
445
|
+
if (typeof fields.content === "string") {
|
|
446
|
+
return new AIMessageChunk(fields);
|
|
447
|
+
}
|
|
448
|
+
else if (fields.content.every((item) => item.type === "text")) {
|
|
449
|
+
const newContent = fields.content
|
|
450
|
+
.map((item) => ("text" in item ? item.text : ""))
|
|
451
|
+
.join("");
|
|
452
|
+
return new AIMessageChunk({
|
|
453
|
+
...fields,
|
|
454
|
+
content: newContent,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
431
457
|
return new AIMessageChunk(fields);
|
|
432
458
|
}
|
|
433
|
-
|
|
434
|
-
const
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
content: newContent,
|
|
459
|
+
function partToChatGeneration(part) {
|
|
460
|
+
const message = partToMessageChunk(part);
|
|
461
|
+
const text = partToText(part);
|
|
462
|
+
return new ChatGenerationChunk({
|
|
463
|
+
text,
|
|
464
|
+
message,
|
|
440
465
|
});
|
|
441
466
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
const text = partToText(part);
|
|
447
|
-
return new ChatGenerationChunk({
|
|
448
|
-
text,
|
|
449
|
-
message,
|
|
450
|
-
});
|
|
451
|
-
}
|
|
452
|
-
export function responseToChatGenerations(response) {
|
|
453
|
-
const parts = responseToParts(response);
|
|
454
|
-
let ret = parts.map((part) => partToChatGeneration(part));
|
|
455
|
-
if (ret.every((item) => typeof item.message.content === "string")) {
|
|
456
|
-
const combinedContent = ret.map((item) => item.message.content).join("");
|
|
457
|
-
const combinedText = ret.map((item) => item.text).join("");
|
|
458
|
-
const toolCallChunks = ret[ret.length - 1]?.message.additional_kwargs?.tool_calls?.map((toolCall, i) => ({
|
|
459
|
-
name: toolCall.function.name,
|
|
460
|
-
args: toolCall.function.arguments,
|
|
461
|
-
id: toolCall.id,
|
|
462
|
-
index: i,
|
|
463
|
-
type: "tool_call_chunk",
|
|
464
|
-
}));
|
|
465
|
-
let usageMetadata;
|
|
466
|
-
if ("usageMetadata" in response.data) {
|
|
467
|
-
usageMetadata = {
|
|
468
|
-
input_tokens: response.data.usageMetadata.promptTokenCount,
|
|
469
|
-
output_tokens: response.data.usageMetadata
|
|
470
|
-
.candidatesTokenCount,
|
|
471
|
-
total_tokens: response.data.usageMetadata.totalTokenCount,
|
|
472
|
-
};
|
|
467
|
+
function responseToChatGenerations(response) {
|
|
468
|
+
const parts = responseToParts(response);
|
|
469
|
+
if (parts.length === 0) {
|
|
470
|
+
return [];
|
|
473
471
|
}
|
|
474
|
-
ret =
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
generationInfo: ret[ret.length - 1].generationInfo,
|
|
484
|
-
}),
|
|
485
|
-
];
|
|
486
|
-
}
|
|
487
|
-
return ret;
|
|
488
|
-
}
|
|
489
|
-
export function responseToBaseMessageFields(response) {
|
|
490
|
-
const parts = responseToParts(response);
|
|
491
|
-
return partsToBaseMessageChunkFields(parts);
|
|
492
|
-
}
|
|
493
|
-
export function partsToBaseMessageChunkFields(parts) {
|
|
494
|
-
const fields = {
|
|
495
|
-
content: partsToMessageContent(parts),
|
|
496
|
-
tool_call_chunks: [],
|
|
497
|
-
tool_calls: [],
|
|
498
|
-
invalid_tool_calls: [],
|
|
499
|
-
};
|
|
500
|
-
const rawTools = partsToToolsRaw(parts);
|
|
501
|
-
if (rawTools.length > 0) {
|
|
502
|
-
const tools = toolsRawToTools(rawTools);
|
|
503
|
-
for (const tool of tools) {
|
|
504
|
-
fields.tool_call_chunks?.push({
|
|
505
|
-
name: tool.function.name,
|
|
506
|
-
args: tool.function.arguments,
|
|
507
|
-
id: tool.id,
|
|
472
|
+
let ret = parts.map((part) => partToChatGeneration(part));
|
|
473
|
+
if (ret.every((item) => typeof item.message.content === "string")) {
|
|
474
|
+
const combinedContent = ret.map((item) => item.message.content).join("");
|
|
475
|
+
const combinedText = ret.map((item) => item.text).join("");
|
|
476
|
+
const toolCallChunks = ret[ret.length - 1]?.message.additional_kwargs?.tool_calls?.map((toolCall, i) => ({
|
|
477
|
+
name: toolCall.function.name,
|
|
478
|
+
args: toolCall.function.arguments,
|
|
479
|
+
id: toolCall.id,
|
|
480
|
+
index: i,
|
|
508
481
|
type: "tool_call_chunk",
|
|
509
|
-
});
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
482
|
+
}));
|
|
483
|
+
let usageMetadata;
|
|
484
|
+
if ("usageMetadata" in response.data) {
|
|
485
|
+
usageMetadata = {
|
|
486
|
+
input_tokens: response.data.usageMetadata.promptTokenCount,
|
|
487
|
+
output_tokens: response.data.usageMetadata
|
|
488
|
+
.candidatesTokenCount,
|
|
489
|
+
total_tokens: response.data.usageMetadata.totalTokenCount,
|
|
490
|
+
};
|
|
518
491
|
}
|
|
519
|
-
|
|
520
|
-
|
|
492
|
+
ret = [
|
|
493
|
+
new ChatGenerationChunk({
|
|
494
|
+
message: new AIMessageChunk({
|
|
495
|
+
content: combinedContent,
|
|
496
|
+
additional_kwargs: ret[ret.length - 1]?.message.additional_kwargs,
|
|
497
|
+
tool_call_chunks: toolCallChunks,
|
|
498
|
+
usage_metadata: usageMetadata,
|
|
499
|
+
}),
|
|
500
|
+
text: combinedText,
|
|
501
|
+
generationInfo: ret[ret.length - 1].generationInfo,
|
|
502
|
+
}),
|
|
503
|
+
];
|
|
504
|
+
}
|
|
505
|
+
return ret;
|
|
506
|
+
}
|
|
507
|
+
function responseToBaseMessageFields(response) {
|
|
508
|
+
const parts = responseToParts(response);
|
|
509
|
+
return partsToBaseMessageChunkFields(parts);
|
|
510
|
+
}
|
|
511
|
+
function partsToBaseMessageChunkFields(parts) {
|
|
512
|
+
const fields = {
|
|
513
|
+
content: partsToMessageContent(parts),
|
|
514
|
+
tool_call_chunks: [],
|
|
515
|
+
tool_calls: [],
|
|
516
|
+
invalid_tool_calls: [],
|
|
517
|
+
};
|
|
518
|
+
const rawTools = partsToToolsRaw(parts);
|
|
519
|
+
if (rawTools.length > 0) {
|
|
520
|
+
const tools = toolsRawToTools(rawTools);
|
|
521
|
+
for (const tool of tools) {
|
|
522
|
+
fields.tool_call_chunks?.push({
|
|
521
523
|
name: tool.function.name,
|
|
522
524
|
args: tool.function.arguments,
|
|
523
525
|
id: tool.id,
|
|
524
|
-
|
|
525
|
-
type: "invalid_tool_call",
|
|
526
|
+
type: "tool_call_chunk",
|
|
526
527
|
});
|
|
528
|
+
try {
|
|
529
|
+
fields.tool_calls?.push({
|
|
530
|
+
name: tool.function.name,
|
|
531
|
+
args: JSON.parse(tool.function.arguments),
|
|
532
|
+
id: tool.id,
|
|
533
|
+
});
|
|
534
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
535
|
+
}
|
|
536
|
+
catch (e) {
|
|
537
|
+
fields.invalid_tool_calls?.push({
|
|
538
|
+
name: tool.function.name,
|
|
539
|
+
args: tool.function.arguments,
|
|
540
|
+
id: tool.id,
|
|
541
|
+
error: e.message,
|
|
542
|
+
type: "invalid_tool_call",
|
|
543
|
+
});
|
|
544
|
+
}
|
|
527
545
|
}
|
|
546
|
+
fields.additional_kwargs = {
|
|
547
|
+
tool_calls: tools,
|
|
548
|
+
};
|
|
528
549
|
}
|
|
529
|
-
fields
|
|
530
|
-
|
|
550
|
+
return fields;
|
|
551
|
+
}
|
|
552
|
+
function responseToBaseMessage(response) {
|
|
553
|
+
const fields = responseToBaseMessageFields(response);
|
|
554
|
+
return new AIMessage(fields);
|
|
555
|
+
}
|
|
556
|
+
function safeResponseToBaseMessage(response, safetyHandler) {
|
|
557
|
+
return safeResponseTo(response, safetyHandler, responseToBaseMessage);
|
|
558
|
+
}
|
|
559
|
+
function responseToChatResult(response) {
|
|
560
|
+
const generations = responseToChatGenerations(response);
|
|
561
|
+
return {
|
|
562
|
+
generations,
|
|
563
|
+
llmOutput: responseToGenerationInfo(response),
|
|
531
564
|
};
|
|
532
565
|
}
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
const fields = responseToBaseMessageFields(response);
|
|
537
|
-
return new AIMessage(fields);
|
|
538
|
-
}
|
|
539
|
-
export function safeResponseToBaseMessage(response, safetyHandler) {
|
|
540
|
-
return safeResponseTo(response, safetyHandler, responseToBaseMessage);
|
|
541
|
-
}
|
|
542
|
-
export function responseToChatResult(response) {
|
|
543
|
-
const generations = responseToChatGenerations(response);
|
|
566
|
+
function safeResponseToChatResult(response, safetyHandler) {
|
|
567
|
+
return safeResponseTo(response, safetyHandler, responseToChatResult);
|
|
568
|
+
}
|
|
544
569
|
return {
|
|
545
|
-
|
|
546
|
-
|
|
570
|
+
messageContentToParts,
|
|
571
|
+
baseMessageToContent,
|
|
572
|
+
safeResponseToString,
|
|
573
|
+
safeResponseToChatGeneration,
|
|
574
|
+
chunkToString,
|
|
575
|
+
safeResponseToBaseMessage,
|
|
576
|
+
safeResponseToChatResult,
|
|
547
577
|
};
|
|
548
578
|
}
|
|
549
|
-
export function safeResponseToChatResult(response, safetyHandler) {
|
|
550
|
-
return safeResponseTo(response, safetyHandler, responseToChatResult);
|
|
551
|
-
}
|
|
552
579
|
export function validateGeminiParams(params) {
|
|
553
580
|
if (params.maxOutputTokens && params.maxOutputTokens < 0) {
|
|
554
581
|
throw new Error("`maxOutputTokens` must be a positive integer");
|