@langchain/google-common 0.2.17 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/LICENSE +6 -6
- package/dist/_virtual/rolldown_runtime.cjs +25 -0
- package/dist/auth.cjs +82 -116
- package/dist/auth.cjs.map +1 -0
- package/dist/auth.d.cts +46 -0
- package/dist/auth.d.cts.map +1 -0
- package/dist/auth.d.ts +41 -36
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +80 -110
- package/dist/auth.js.map +1 -0
- package/dist/chat_models.cjs +251 -466
- package/dist/chat_models.cjs.map +1 -0
- package/dist/chat_models.d.cts +98 -0
- package/dist/chat_models.d.cts.map +1 -0
- package/dist/chat_models.d.ts +87 -73
- package/dist/chat_models.d.ts.map +1 -0
- package/dist/chat_models.js +245 -457
- package/dist/chat_models.js.map +1 -0
- package/dist/connection.cjs +321 -466
- package/dist/connection.cjs.map +1 -0
- package/dist/connection.d.cts +109 -0
- package/dist/connection.d.cts.map +1 -0
- package/dist/connection.d.ts +98 -91
- package/dist/connection.d.ts.map +1 -0
- package/dist/connection.js +317 -459
- package/dist/connection.js.map +1 -0
- package/dist/embeddings.cjs +135 -186
- package/dist/embeddings.cjs.map +1 -0
- package/dist/embeddings.d.cts +44 -0
- package/dist/embeddings.d.cts.map +1 -0
- package/dist/embeddings.d.ts +38 -32
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +133 -181
- package/dist/embeddings.js.map +1 -0
- package/dist/experimental/media.cjs +380 -482
- package/dist/experimental/media.cjs.map +1 -0
- package/dist/experimental/media.d.cts +198 -0
- package/dist/experimental/media.d.cts.map +1 -0
- package/dist/experimental/media.d.ts +190 -202
- package/dist/experimental/media.d.ts.map +1 -0
- package/dist/experimental/media.js +369 -468
- package/dist/experimental/media.js.map +1 -0
- package/dist/experimental/utils/media_core.cjs +403 -517
- package/dist/experimental/utils/media_core.cjs.map +1 -0
- package/dist/experimental/utils/media_core.d.cts +215 -0
- package/dist/experimental/utils/media_core.d.cts.map +1 -0
- package/dist/experimental/utils/media_core.d.ts +171 -165
- package/dist/experimental/utils/media_core.d.ts.map +1 -0
- package/dist/experimental/utils/media_core.js +395 -506
- package/dist/experimental/utils/media_core.js.map +1 -0
- package/dist/index.cjs +58 -27
- package/dist/index.d.cts +13 -0
- package/dist/index.d.ts +13 -11
- package/dist/index.js +13 -11
- package/dist/llms.cjs +157 -244
- package/dist/llms.cjs.map +1 -0
- package/dist/llms.d.cts +72 -0
- package/dist/llms.d.cts.map +1 -0
- package/dist/llms.d.ts +64 -54
- package/dist/llms.d.ts.map +1 -0
- package/dist/llms.js +154 -238
- package/dist/llms.js.map +1 -0
- package/dist/output_parsers.cjs +148 -173
- package/dist/output_parsers.cjs.map +1 -0
- package/dist/output_parsers.d.cts +53 -0
- package/dist/output_parsers.d.cts.map +1 -0
- package/dist/output_parsers.d.ts +46 -42
- package/dist/output_parsers.d.ts.map +1 -0
- package/dist/output_parsers.js +146 -168
- package/dist/output_parsers.js.map +1 -0
- package/dist/types-anthropic.d.cts +229 -0
- package/dist/types-anthropic.d.cts.map +1 -0
- package/dist/types-anthropic.d.ts +221 -215
- package/dist/types-anthropic.d.ts.map +1 -0
- package/dist/types.cjs +51 -62
- package/dist/types.cjs.map +1 -0
- package/dist/types.d.cts +748 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.ts +669 -656
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +46 -45
- package/dist/types.js.map +1 -0
- package/dist/utils/anthropic.cjs +598 -821
- package/dist/utils/anthropic.cjs.map +1 -0
- package/dist/utils/anthropic.js +597 -818
- package/dist/utils/anthropic.js.map +1 -0
- package/dist/utils/common.cjs +130 -211
- package/dist/utils/common.cjs.map +1 -0
- package/dist/utils/common.d.cts +13 -0
- package/dist/utils/common.d.cts.map +1 -0
- package/dist/utils/common.d.ts +12 -7
- package/dist/utils/common.d.ts.map +1 -0
- package/dist/utils/common.js +128 -207
- package/dist/utils/common.js.map +1 -0
- package/dist/utils/failed_handler.cjs +28 -30
- package/dist/utils/failed_handler.cjs.map +1 -0
- package/dist/utils/failed_handler.d.cts +9 -0
- package/dist/utils/failed_handler.d.cts.map +1 -0
- package/dist/utils/failed_handler.d.ts +8 -2
- package/dist/utils/failed_handler.d.ts.map +1 -0
- package/dist/utils/failed_handler.js +28 -28
- package/dist/utils/failed_handler.js.map +1 -0
- package/dist/utils/gemini.cjs +1020 -1488
- package/dist/utils/gemini.cjs.map +1 -0
- package/dist/utils/gemini.d.cts +51 -0
- package/dist/utils/gemini.d.cts.map +1 -0
- package/dist/utils/gemini.d.ts +51 -48
- package/dist/utils/gemini.d.ts.map +1 -0
- package/dist/utils/gemini.js +1015 -1479
- package/dist/utils/gemini.js.map +1 -0
- package/dist/utils/index.cjs +38 -23
- package/dist/utils/index.d.cts +8 -0
- package/dist/utils/index.d.ts +8 -7
- package/dist/utils/index.js +8 -7
- package/dist/utils/palm.d.cts +11 -0
- package/dist/utils/palm.d.cts.map +1 -0
- package/dist/utils/palm.d.ts +9 -4
- package/dist/utils/palm.d.ts.map +1 -0
- package/dist/utils/safety.cjs +13 -22
- package/dist/utils/safety.cjs.map +1 -0
- package/dist/utils/safety.d.cts +12 -0
- package/dist/utils/safety.d.cts.map +1 -0
- package/dist/utils/safety.d.ts +10 -4
- package/dist/utils/safety.d.ts.map +1 -0
- package/dist/utils/safety.js +13 -19
- package/dist/utils/safety.js.map +1 -0
- package/dist/utils/stream.cjs +296 -475
- package/dist/utils/stream.cjs.map +1 -0
- package/dist/utils/stream.d.cts +165 -0
- package/dist/utils/stream.d.cts.map +1 -0
- package/dist/utils/stream.d.ts +156 -131
- package/dist/utils/stream.d.ts.map +1 -0
- package/dist/utils/stream.js +293 -469
- package/dist/utils/stream.js.map +1 -0
- package/dist/utils/zod_to_gemini_parameters.cjs +43 -81
- package/dist/utils/zod_to_gemini_parameters.cjs.map +1 -0
- package/dist/utils/zod_to_gemini_parameters.d.cts +22 -0
- package/dist/utils/zod_to_gemini_parameters.d.cts.map +1 -0
- package/dist/utils/zod_to_gemini_parameters.d.ts +21 -6
- package/dist/utils/zod_to_gemini_parameters.d.ts.map +1 -0
- package/dist/utils/zod_to_gemini_parameters.js +40 -76
- package/dist/utils/zod_to_gemini_parameters.js.map +1 -0
- package/package.json +69 -85
- package/dist/types-anthropic.cjs +0 -2
- package/dist/types-anthropic.js +0 -1
- package/dist/utils/anthropic.d.ts +0 -4
- package/dist/utils/palm.cjs +0 -2
- package/dist/utils/palm.js +0 -1
- package/experimental/media.cjs +0 -1
- package/experimental/media.d.cts +0 -1
- package/experimental/media.d.ts +0 -1
- package/experimental/media.js +0 -1
- package/experimental/utils/media_core.cjs +0 -1
- package/experimental/utils/media_core.d.cts +0 -1
- package/experimental/utils/media_core.d.ts +0 -1
- package/experimental/utils/media_core.js +0 -1
- package/index.cjs +0 -1
- package/index.d.cts +0 -1
- package/index.d.ts +0 -1
- package/index.js +0 -1
- package/types.cjs +0 -1
- package/types.d.cts +0 -1
- package/types.d.ts +0 -1
- package/types.js +0 -1
- package/utils.cjs +0 -1
- package/utils.d.cts +0 -1
- package/utils.d.ts +0 -1
- package/utils.js +0 -1
package/dist/utils/gemini.js
CHANGED
|
@@ -1,1489 +1,1025 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { AIMessage, AIMessageChunk, isAIMessage, parseBase64DataUrl, isDataContentBlock, convertToProviderContentBlock, } from "@langchain/core/messages";
|
|
3
|
-
import { ChatGenerationChunk, } from "@langchain/core/outputs";
|
|
4
|
-
import { isLangChainTool } from "@langchain/core/utils/function_calling";
|
|
5
|
-
import { concat } from "@langchain/core/utils/stream";
|
|
6
|
-
import { GeminiSearchToolAttributes, } from "../types.js";
|
|
1
|
+
import { GeminiSearchToolAttributes } from "../types.js";
|
|
7
2
|
import { GoogleAISafetyError } from "./safety.js";
|
|
8
3
|
import { schemaToGeminiParameters } from "./zod_to_gemini_parameters.js";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
setMessage(data) {
|
|
92
|
-
const ret = data;
|
|
93
|
-
if (this.forceNewMessage ||
|
|
94
|
-
!data?.candidates?.[0]?.content?.parts?.length) {
|
|
95
|
-
ret.candidates = data.candidates ?? [];
|
|
96
|
-
ret.candidates[0] = data.candidates[0] ?? {};
|
|
97
|
-
ret.candidates[0].content = data.candidates[0].content ?? {};
|
|
98
|
-
ret.candidates[0].content = {
|
|
99
|
-
role: "model",
|
|
100
|
-
parts: [{ text: this.msg }],
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
return ret;
|
|
104
|
-
}
|
|
105
|
-
handleData(response, data) {
|
|
106
|
-
try {
|
|
107
|
-
return super.handleData(response, data);
|
|
108
|
-
}
|
|
109
|
-
catch (xx) {
|
|
110
|
-
return this.setMessage(data);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
}
|
|
4
|
+
import { ChatGenerationChunk } from "@langchain/core/outputs";
|
|
5
|
+
import { AIMessage, AIMessageChunk, HumanMessage, SystemMessage, ToolMessage, convertToProviderContentBlock, isAIMessage, isDataContentBlock, parseBase64DataUrl } from "@langchain/core/messages";
|
|
6
|
+
import { concat } from "@langchain/core/utils/stream";
|
|
7
|
+
import { isLangChainTool } from "@langchain/core/utils/function_calling";
|
|
8
|
+
import { v4 } from "uuid";
|
|
9
|
+
|
|
10
|
+
//#region src/utils/gemini.ts
|
|
11
|
+
var DefaultGeminiSafetyHandler = class {
|
|
12
|
+
errorFinish = [
|
|
13
|
+
"SAFETY",
|
|
14
|
+
"RECITATION",
|
|
15
|
+
"OTHER"
|
|
16
|
+
];
|
|
17
|
+
constructor(settings) {
|
|
18
|
+
this.errorFinish = settings?.errorFinish ?? this.errorFinish;
|
|
19
|
+
}
|
|
20
|
+
handleDataPromptFeedback(response, data) {
|
|
21
|
+
const promptFeedback = data?.promptFeedback;
|
|
22
|
+
const blockReason = promptFeedback?.blockReason;
|
|
23
|
+
if (blockReason) throw new GoogleAISafetyError(response, `Prompt blocked: ${blockReason}`);
|
|
24
|
+
return data;
|
|
25
|
+
}
|
|
26
|
+
handleDataFinishReason(response, data) {
|
|
27
|
+
const firstCandidate = data?.candidates?.[0];
|
|
28
|
+
const finishReason = firstCandidate?.finishReason;
|
|
29
|
+
if (this.errorFinish.includes(finishReason)) throw new GoogleAISafetyError(response, `Finish reason: ${finishReason}`);
|
|
30
|
+
return data;
|
|
31
|
+
}
|
|
32
|
+
handleData(response, data) {
|
|
33
|
+
let ret = data;
|
|
34
|
+
ret = this.handleDataPromptFeedback(response, ret);
|
|
35
|
+
ret = this.handleDataFinishReason(response, ret);
|
|
36
|
+
return ret;
|
|
37
|
+
}
|
|
38
|
+
handle(response) {
|
|
39
|
+
let newdata;
|
|
40
|
+
if ("nextChunk" in response.data) newdata = response.data;
|
|
41
|
+
else if (Array.isArray(response.data)) try {
|
|
42
|
+
newdata = response.data.map((item) => this.handleData(response, item));
|
|
43
|
+
} catch (xx) {
|
|
44
|
+
if (xx instanceof GoogleAISafetyError) throw new GoogleAISafetyError(response, xx.message);
|
|
45
|
+
else throw xx;
|
|
46
|
+
}
|
|
47
|
+
else {
|
|
48
|
+
const data = response.data;
|
|
49
|
+
newdata = this.handleData(response, data);
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
...response,
|
|
53
|
+
data: newdata
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
var MessageGeminiSafetyHandler = class extends DefaultGeminiSafetyHandler {
|
|
58
|
+
msg = "";
|
|
59
|
+
forceNewMessage = false;
|
|
60
|
+
constructor(settings) {
|
|
61
|
+
super(settings);
|
|
62
|
+
this.msg = settings?.msg ?? this.msg;
|
|
63
|
+
this.forceNewMessage = settings?.forceNewMessage ?? this.forceNewMessage;
|
|
64
|
+
}
|
|
65
|
+
setMessage(data) {
|
|
66
|
+
const ret = data;
|
|
67
|
+
if (this.forceNewMessage || !data?.candidates?.[0]?.content?.parts?.length) {
|
|
68
|
+
ret.candidates = data.candidates ?? [];
|
|
69
|
+
ret.candidates[0] = data.candidates[0] ?? {};
|
|
70
|
+
ret.candidates[0].content = data.candidates[0].content ?? {};
|
|
71
|
+
ret.candidates[0].content = {
|
|
72
|
+
role: "model",
|
|
73
|
+
parts: [{ text: this.msg }]
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
return ret;
|
|
77
|
+
}
|
|
78
|
+
handleData(response, data) {
|
|
79
|
+
try {
|
|
80
|
+
return super.handleData(response, data);
|
|
81
|
+
} catch {
|
|
82
|
+
return this.setMessage(data);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
};
|
|
114
86
|
const extractMimeType = (str) => {
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
}
|
|
121
|
-
return null;
|
|
87
|
+
if (str.startsWith("data:")) return {
|
|
88
|
+
mimeType: str.split(":")[1].split(";")[0],
|
|
89
|
+
data: str.split(",")[1]
|
|
90
|
+
};
|
|
91
|
+
return null;
|
|
122
92
|
};
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
ret = {
|
|
156
|
-
voiceConfig: {
|
|
157
|
-
prebuiltVoiceConfig: {
|
|
158
|
-
voiceName: voice,
|
|
159
|
-
},
|
|
160
|
-
},
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
else {
|
|
164
|
-
// This is multi-speaker, so we have speaker/name pairs
|
|
165
|
-
// If we have just one (why?), turn it into an array for the moment
|
|
166
|
-
const voices = Array.isArray(voice)
|
|
167
|
-
? voice
|
|
168
|
-
: [voice];
|
|
169
|
-
// Go through all the speaker/name pairs and turn this into the voice config array
|
|
170
|
-
const speakerVoiceConfigs = voices.map((v) => ({
|
|
171
|
-
speaker: v.speaker,
|
|
172
|
-
voiceConfig: {
|
|
173
|
-
prebuiltVoiceConfig: {
|
|
174
|
-
voiceName: v.name,
|
|
175
|
-
},
|
|
176
|
-
},
|
|
177
|
-
}));
|
|
178
|
-
// Create the multi-speaker voice configuration
|
|
179
|
-
ret = {
|
|
180
|
-
multiSpeakerVoiceConfig: {
|
|
181
|
-
speakerVoiceConfigs,
|
|
182
|
-
},
|
|
183
|
-
};
|
|
184
|
-
}
|
|
185
|
-
if (languageCode) {
|
|
186
|
-
ret.languageCode = languageCode;
|
|
187
|
-
}
|
|
188
|
-
return ret;
|
|
93
|
+
/**
|
|
94
|
+
* Infers the MIME type from a URL based on its file extension.
|
|
95
|
+
* This is used as a fallback when the MIME type is not provided.
|
|
96
|
+
*
|
|
97
|
+
* @param url - The URL to infer the MIME type from
|
|
98
|
+
* @returns The inferred MIME type or undefined if it cannot be determined
|
|
99
|
+
*/
|
|
100
|
+
function inferMimeTypeFromUrl(url) {
|
|
101
|
+
const mimeTypeMap = {
|
|
102
|
+
jpg: "image/jpeg",
|
|
103
|
+
jpeg: "image/jpeg",
|
|
104
|
+
png: "image/png",
|
|
105
|
+
gif: "image/gif",
|
|
106
|
+
webp: "image/webp",
|
|
107
|
+
bmp: "image/bmp",
|
|
108
|
+
svg: "image/svg+xml",
|
|
109
|
+
ico: "image/x-icon",
|
|
110
|
+
tiff: "image/tiff",
|
|
111
|
+
tif: "image/tiff"
|
|
112
|
+
};
|
|
113
|
+
try {
|
|
114
|
+
const pathname = new URL(url).pathname;
|
|
115
|
+
const extension = pathname.split(".").pop()?.toLowerCase().split(/[?#]/)[0];
|
|
116
|
+
return extension ? mimeTypeMap[extension] : void 0;
|
|
117
|
+
} catch {
|
|
118
|
+
const match = url.match(/\.([a-zA-Z0-9]+)(?:[?#]|$)/);
|
|
119
|
+
if (match) {
|
|
120
|
+
const extension = match[1].toLowerCase();
|
|
121
|
+
return mimeTypeMap[extension];
|
|
122
|
+
}
|
|
123
|
+
return void 0;
|
|
124
|
+
}
|
|
189
125
|
}
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
const { file_data } = c.file;
|
|
224
|
-
return {
|
|
225
|
-
type: "file",
|
|
226
|
-
source_type: "base64",
|
|
227
|
-
data: file_data,
|
|
228
|
-
};
|
|
229
|
-
}
|
|
230
|
-
return c;
|
|
231
|
-
});
|
|
126
|
+
function normalizeSpeechConfig(config) {
|
|
127
|
+
function isSpeechConfig(config$1) {
|
|
128
|
+
return typeof config$1 === "object" && (Object.hasOwn(config$1, "voiceConfig") || Object.hasOwn(config$1, "multiSpeakerVoiceConfig"));
|
|
129
|
+
}
|
|
130
|
+
function hasLanguage(config$1) {
|
|
131
|
+
return typeof config$1 === "object" && Object.hasOwn(config$1, "languageCode");
|
|
132
|
+
}
|
|
133
|
+
function hasVoice(config$1) {
|
|
134
|
+
return Object.hasOwn(config$1, "voice");
|
|
135
|
+
}
|
|
136
|
+
if (typeof config === "undefined") return void 0;
|
|
137
|
+
if (isSpeechConfig(config)) return config;
|
|
138
|
+
let languageCode;
|
|
139
|
+
let voice;
|
|
140
|
+
if (hasLanguage(config)) {
|
|
141
|
+
languageCode = config.languageCode;
|
|
142
|
+
voice = hasVoice(config) ? config.voice : config.voices;
|
|
143
|
+
} else {
|
|
144
|
+
languageCode = void 0;
|
|
145
|
+
voice = config;
|
|
146
|
+
}
|
|
147
|
+
let ret;
|
|
148
|
+
if (typeof voice === "string") ret = { voiceConfig: { prebuiltVoiceConfig: { voiceName: voice } } };
|
|
149
|
+
else {
|
|
150
|
+
const voices = Array.isArray(voice) ? voice : [voice];
|
|
151
|
+
const speakerVoiceConfigs = voices.map((v) => ({
|
|
152
|
+
speaker: v.speaker,
|
|
153
|
+
voiceConfig: { prebuiltVoiceConfig: { voiceName: v.name } }
|
|
154
|
+
}));
|
|
155
|
+
ret = { multiSpeakerVoiceConfig: { speakerVoiceConfigs } };
|
|
156
|
+
}
|
|
157
|
+
if (languageCode) ret.languageCode = languageCode;
|
|
158
|
+
return ret;
|
|
232
159
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
function combineGenerations(generations, response) {
|
|
1078
|
-
const gen = splitGenerationTypes(generations, response);
|
|
1079
|
-
const combinedContent = combineContent(gen.content);
|
|
1080
|
-
const combinedText = combineText(gen.content);
|
|
1081
|
-
const combinedToolCalls = combineToolCalls(gen.content);
|
|
1082
|
-
const kwargs = combineAdditionalKwargs(gen.content);
|
|
1083
|
-
const lastContent = gen.content[gen.content.length - 1];
|
|
1084
|
-
// Add usage metadata
|
|
1085
|
-
const usage_metadata = responseToUsageMetadata(response);
|
|
1086
|
-
// Add thinking / reasoning
|
|
1087
|
-
// if (gen.reasoning && gen.reasoning.length > 0) {
|
|
1088
|
-
// kwargs.reasoning_content = combineContent(gen.reasoning, true);
|
|
1089
|
-
// }
|
|
1090
|
-
// Build the message and the generation chunk to return
|
|
1091
|
-
const message = new AIMessageChunk({
|
|
1092
|
-
content: combinedContent,
|
|
1093
|
-
additional_kwargs: kwargs,
|
|
1094
|
-
usage_metadata,
|
|
1095
|
-
tool_calls: combinedToolCalls.tool_calls,
|
|
1096
|
-
invalid_tool_calls: combinedToolCalls.invalid_tool_calls,
|
|
1097
|
-
});
|
|
1098
|
-
return [
|
|
1099
|
-
new ChatGenerationChunk({
|
|
1100
|
-
message,
|
|
1101
|
-
text: combinedText,
|
|
1102
|
-
generationInfo: lastContent.generationInfo,
|
|
1103
|
-
}),
|
|
1104
|
-
];
|
|
1105
|
-
}
|
|
1106
|
-
function splitGenerationTypes(generations, _response) {
|
|
1107
|
-
const content = [];
|
|
1108
|
-
const reasoning = [];
|
|
1109
|
-
generations.forEach((gen) => {
|
|
1110
|
-
if (gen?.generationInfo?.thought) {
|
|
1111
|
-
reasoning.push(gen);
|
|
1112
|
-
}
|
|
1113
|
-
else {
|
|
1114
|
-
content.push(gen);
|
|
1115
|
-
}
|
|
1116
|
-
});
|
|
1117
|
-
return {
|
|
1118
|
-
content,
|
|
1119
|
-
reasoning,
|
|
1120
|
-
};
|
|
1121
|
-
}
|
|
1122
|
-
/**
|
|
1123
|
-
* Although this returns an array, only the first (or maybe last)
|
|
1124
|
-
* element in the array is used. So we need to combine them into
|
|
1125
|
-
* just one element that contains everything we need.
|
|
1126
|
-
* @param response
|
|
1127
|
-
*/
|
|
1128
|
-
function responseToChatGenerations(response) {
|
|
1129
|
-
const generations = responseToGroundedChatGenerations(response);
|
|
1130
|
-
if (generations.length === 0) {
|
|
1131
|
-
return [];
|
|
1132
|
-
}
|
|
1133
|
-
const ret = combineGenerations(generations, response);
|
|
1134
|
-
// Add logprobs information to the message
|
|
1135
|
-
const candidate = response?.data
|
|
1136
|
-
?.candidates?.[0];
|
|
1137
|
-
const avgLogprobs = candidate?.avgLogprobs;
|
|
1138
|
-
const logprobs = candidateToLogprobs(candidate);
|
|
1139
|
-
if (logprobs) {
|
|
1140
|
-
ret[0].message.response_metadata = {
|
|
1141
|
-
...ret[0].message.response_metadata,
|
|
1142
|
-
logprobs,
|
|
1143
|
-
avgLogprobs,
|
|
1144
|
-
};
|
|
1145
|
-
}
|
|
1146
|
-
return ret;
|
|
1147
|
-
}
|
|
1148
|
-
function responseToBaseMessageFields(response) {
|
|
1149
|
-
const parts = responseToParts(response);
|
|
1150
|
-
return partsToBaseMessageChunkFields(parts);
|
|
1151
|
-
}
|
|
1152
|
-
function partsToSignatures(parts) {
|
|
1153
|
-
return parts.map((part) => part?.thoughtSignature ?? "");
|
|
1154
|
-
}
|
|
1155
|
-
function partsToBaseMessageChunkFields(parts) {
|
|
1156
|
-
const fields = {
|
|
1157
|
-
content: partsToMessageContent(parts),
|
|
1158
|
-
tool_call_chunks: [],
|
|
1159
|
-
tool_calls: [],
|
|
1160
|
-
invalid_tool_calls: [],
|
|
1161
|
-
};
|
|
1162
|
-
fields.additional_kwargs = {};
|
|
1163
|
-
const rawTools = partsToToolsRaw(parts);
|
|
1164
|
-
if (rawTools.length > 0) {
|
|
1165
|
-
const tools = toolsRawToTools(rawTools);
|
|
1166
|
-
for (const tool of tools) {
|
|
1167
|
-
fields.tool_call_chunks?.push({
|
|
1168
|
-
name: tool.function.name,
|
|
1169
|
-
args: tool.function.arguments,
|
|
1170
|
-
id: tool.id,
|
|
1171
|
-
type: "tool_call_chunk",
|
|
1172
|
-
});
|
|
1173
|
-
try {
|
|
1174
|
-
fields.tool_calls?.push({
|
|
1175
|
-
name: tool.function.name,
|
|
1176
|
-
args: JSON.parse(tool.function.arguments),
|
|
1177
|
-
id: tool.id,
|
|
1178
|
-
});
|
|
1179
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1180
|
-
}
|
|
1181
|
-
catch (e) {
|
|
1182
|
-
fields.invalid_tool_calls?.push({
|
|
1183
|
-
name: tool.function.name,
|
|
1184
|
-
args: tool.function.arguments,
|
|
1185
|
-
id: tool.id,
|
|
1186
|
-
error: e.message,
|
|
1187
|
-
type: "invalid_tool_call",
|
|
1188
|
-
});
|
|
1189
|
-
}
|
|
1190
|
-
}
|
|
1191
|
-
fields.additional_kwargs.tool_calls = tools;
|
|
1192
|
-
}
|
|
1193
|
-
fields.additional_kwargs.signatures = partsToSignatures(parts);
|
|
1194
|
-
return fields;
|
|
1195
|
-
}
|
|
1196
|
-
function responseToBaseMessage(response) {
|
|
1197
|
-
const fields = responseToBaseMessageFields(response);
|
|
1198
|
-
return new AIMessage(fields);
|
|
1199
|
-
}
|
|
1200
|
-
function safeResponseToBaseMessage(response) {
|
|
1201
|
-
return safeResponseTo(response, responseToBaseMessage);
|
|
1202
|
-
}
|
|
1203
|
-
function responseToChatResult(response) {
|
|
1204
|
-
const generations = responseToChatGenerations(response);
|
|
1205
|
-
return {
|
|
1206
|
-
generations,
|
|
1207
|
-
llmOutput: responseToGenerationInfo(response),
|
|
1208
|
-
};
|
|
1209
|
-
}
|
|
1210
|
-
function safeResponseToChatResult(response) {
|
|
1211
|
-
return safeResponseTo(response, responseToChatResult);
|
|
1212
|
-
}
|
|
1213
|
-
function inputType(input) {
|
|
1214
|
-
if (typeof input === "string") {
|
|
1215
|
-
return "MessageContent";
|
|
1216
|
-
}
|
|
1217
|
-
else {
|
|
1218
|
-
const firstItem = input[0];
|
|
1219
|
-
if (Object.hasOwn(firstItem, "content")) {
|
|
1220
|
-
return "BaseMessageArray";
|
|
1221
|
-
}
|
|
1222
|
-
else {
|
|
1223
|
-
return "MessageContent";
|
|
1224
|
-
}
|
|
1225
|
-
}
|
|
1226
|
-
}
|
|
1227
|
-
async function formatMessageContents(input, _parameters) {
|
|
1228
|
-
const parts = await messageContentToParts(input);
|
|
1229
|
-
const contents = [
|
|
1230
|
-
{
|
|
1231
|
-
role: "user", // Required by Vertex AI
|
|
1232
|
-
parts,
|
|
1233
|
-
},
|
|
1234
|
-
];
|
|
1235
|
-
return contents;
|
|
1236
|
-
}
|
|
1237
|
-
async function formatBaseMessageContents(input, _parameters) {
|
|
1238
|
-
const inputPromises = input.map((msg, i) => baseMessageToContent(msg, input[i - 1]));
|
|
1239
|
-
const inputs = await Promise.all(inputPromises);
|
|
1240
|
-
return inputs.reduce((acc, cur) => {
|
|
1241
|
-
// Filter out the system content
|
|
1242
|
-
if (cur.every((content) => content.role === "system")) {
|
|
1243
|
-
return acc;
|
|
1244
|
-
}
|
|
1245
|
-
// Combine adjacent function messages
|
|
1246
|
-
if (cur[0]?.role === "function" &&
|
|
1247
|
-
acc.length > 0 &&
|
|
1248
|
-
acc[acc.length - 1].role === "function") {
|
|
1249
|
-
acc[acc.length - 1].parts = [
|
|
1250
|
-
...acc[acc.length - 1].parts,
|
|
1251
|
-
...cur[0].parts,
|
|
1252
|
-
];
|
|
1253
|
-
}
|
|
1254
|
-
else {
|
|
1255
|
-
acc.push(...cur);
|
|
1256
|
-
}
|
|
1257
|
-
return acc;
|
|
1258
|
-
}, []);
|
|
1259
|
-
}
|
|
1260
|
-
async function formatContents(input, parameters) {
|
|
1261
|
-
const it = inputType(input);
|
|
1262
|
-
switch (it) {
|
|
1263
|
-
case "MessageContent":
|
|
1264
|
-
return formatMessageContents(input, parameters);
|
|
1265
|
-
case "BaseMessageArray":
|
|
1266
|
-
return formatBaseMessageContents(input, parameters);
|
|
1267
|
-
default:
|
|
1268
|
-
throw new Error(`Unknown input type "${it}": ${input}`);
|
|
1269
|
-
}
|
|
1270
|
-
}
|
|
1271
|
-
function formatGenerationConfig(parameters) {
|
|
1272
|
-
const ret = {
|
|
1273
|
-
temperature: parameters.temperature,
|
|
1274
|
-
topK: parameters.topK,
|
|
1275
|
-
topP: parameters.topP,
|
|
1276
|
-
seed: parameters.seed,
|
|
1277
|
-
presencePenalty: parameters.presencePenalty,
|
|
1278
|
-
frequencyPenalty: parameters.frequencyPenalty,
|
|
1279
|
-
maxOutputTokens: parameters.maxOutputTokens,
|
|
1280
|
-
stopSequences: parameters.stopSequences,
|
|
1281
|
-
responseMimeType: parameters.responseMimeType,
|
|
1282
|
-
responseModalities: parameters.responseModalities,
|
|
1283
|
-
speechConfig: normalizeSpeechConfig(parameters.speechConfig),
|
|
1284
|
-
};
|
|
1285
|
-
// Add the logprobs if explicitly set
|
|
1286
|
-
if (typeof parameters.logprobs !== "undefined") {
|
|
1287
|
-
ret.responseLogprobs = parameters.logprobs;
|
|
1288
|
-
if (parameters.logprobs &&
|
|
1289
|
-
typeof parameters.topLogprobs !== "undefined") {
|
|
1290
|
-
ret.logprobs = parameters.topLogprobs;
|
|
1291
|
-
}
|
|
1292
|
-
}
|
|
1293
|
-
// Add thinking configuration if explicitly set
|
|
1294
|
-
// Note that you cannot have thinkingBudget set to 0 and includeThoughts true
|
|
1295
|
-
if (typeof parameters.maxReasoningTokens !== "undefined") {
|
|
1296
|
-
const includeThoughts = parameters.maxReasoningTokens !== 0;
|
|
1297
|
-
ret.thinkingConfig = {
|
|
1298
|
-
thinkingBudget: parameters.maxReasoningTokens,
|
|
1299
|
-
includeThoughts,
|
|
1300
|
-
};
|
|
1301
|
-
}
|
|
1302
|
-
// Remove any undefined properties, so we don't send them
|
|
1303
|
-
let attribute;
|
|
1304
|
-
for (attribute in ret) {
|
|
1305
|
-
if (ret[attribute] === undefined) {
|
|
1306
|
-
delete ret[attribute];
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
return ret;
|
|
1310
|
-
}
|
|
1311
|
-
function formatSafetySettings(parameters) {
|
|
1312
|
-
return parameters.safetySettings ?? [];
|
|
1313
|
-
}
|
|
1314
|
-
async function formatBaseMessageSystemInstruction(input) {
|
|
1315
|
-
let ret = {};
|
|
1316
|
-
for (let index = 0; index < input.length; index += 1) {
|
|
1317
|
-
const message = input[index];
|
|
1318
|
-
if (message._getType() === "system") {
|
|
1319
|
-
// For system types, we only want it if it is the first message,
|
|
1320
|
-
// if it appears anywhere else, it should be an error.
|
|
1321
|
-
if (index === 0) {
|
|
1322
|
-
// eslint-disable-next-line prefer-destructuring
|
|
1323
|
-
ret = (await baseMessageToContent(message, undefined))[0];
|
|
1324
|
-
}
|
|
1325
|
-
else {
|
|
1326
|
-
throw new Error("System messages are only permitted as the first passed message.");
|
|
1327
|
-
}
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
return ret;
|
|
1331
|
-
}
|
|
1332
|
-
async function formatSystemInstruction(input) {
|
|
1333
|
-
if (!config?.useSystemInstruction) {
|
|
1334
|
-
return {};
|
|
1335
|
-
}
|
|
1336
|
-
const it = inputType(input);
|
|
1337
|
-
switch (it) {
|
|
1338
|
-
case "BaseMessageArray":
|
|
1339
|
-
return formatBaseMessageSystemInstruction(input);
|
|
1340
|
-
default:
|
|
1341
|
-
return {};
|
|
1342
|
-
}
|
|
1343
|
-
}
|
|
1344
|
-
function structuredToolToFunctionDeclaration(tool) {
|
|
1345
|
-
const jsonSchema = schemaToGeminiParameters(tool.schema);
|
|
1346
|
-
return {
|
|
1347
|
-
name: tool.name,
|
|
1348
|
-
description: tool.description ?? `A function available to call.`,
|
|
1349
|
-
parameters: jsonSchema,
|
|
1350
|
-
};
|
|
1351
|
-
}
|
|
1352
|
-
function searchToolName(tool) {
|
|
1353
|
-
for (const name of GeminiSearchToolAttributes) {
|
|
1354
|
-
if (name in tool) {
|
|
1355
|
-
return name;
|
|
1356
|
-
}
|
|
1357
|
-
}
|
|
1358
|
-
return undefined;
|
|
1359
|
-
}
|
|
1360
|
-
function cleanGeminiTool(tool) {
|
|
1361
|
-
const orig = searchToolName(tool);
|
|
1362
|
-
const adj = config?.googleSearchToolAdjustment;
|
|
1363
|
-
if (orig && adj && adj !== orig) {
|
|
1364
|
-
return {
|
|
1365
|
-
[adj]: {},
|
|
1366
|
-
};
|
|
1367
|
-
}
|
|
1368
|
-
else {
|
|
1369
|
-
return tool;
|
|
1370
|
-
}
|
|
1371
|
-
}
|
|
1372
|
-
function formatTools(parameters) {
|
|
1373
|
-
const tools = parameters?.tools;
|
|
1374
|
-
if (!tools || tools.length === 0) {
|
|
1375
|
-
return [];
|
|
1376
|
-
}
|
|
1377
|
-
// Group all LangChain tools into a single functionDeclarations array.
|
|
1378
|
-
// Gemini Tools may be normalized to different tool names
|
|
1379
|
-
const langChainTools = [];
|
|
1380
|
-
const otherTools = [];
|
|
1381
|
-
tools.forEach((tool) => {
|
|
1382
|
-
if (isLangChainTool(tool)) {
|
|
1383
|
-
langChainTools.push(tool);
|
|
1384
|
-
}
|
|
1385
|
-
else {
|
|
1386
|
-
otherTools.push(cleanGeminiTool(tool));
|
|
1387
|
-
}
|
|
1388
|
-
});
|
|
1389
|
-
const result = [...otherTools];
|
|
1390
|
-
if (langChainTools.length > 0) {
|
|
1391
|
-
result.push({
|
|
1392
|
-
functionDeclarations: langChainTools.map(structuredToolToFunctionDeclaration),
|
|
1393
|
-
});
|
|
1394
|
-
}
|
|
1395
|
-
return result;
|
|
1396
|
-
}
|
|
1397
|
-
function formatToolConfig(parameters) {
|
|
1398
|
-
if (!parameters.tool_choice || typeof parameters.tool_choice !== "string") {
|
|
1399
|
-
return undefined;
|
|
1400
|
-
}
|
|
1401
|
-
if (["auto", "any", "none"].includes(parameters.tool_choice)) {
|
|
1402
|
-
return {
|
|
1403
|
-
functionCallingConfig: {
|
|
1404
|
-
mode: parameters.tool_choice,
|
|
1405
|
-
allowedFunctionNames: parameters.allowed_function_names,
|
|
1406
|
-
},
|
|
1407
|
-
};
|
|
1408
|
-
}
|
|
1409
|
-
// force tool choice to be a single function name in case of structured output
|
|
1410
|
-
return {
|
|
1411
|
-
functionCallingConfig: {
|
|
1412
|
-
mode: "any",
|
|
1413
|
-
allowedFunctionNames: [parameters.tool_choice],
|
|
1414
|
-
},
|
|
1415
|
-
};
|
|
1416
|
-
}
|
|
1417
|
-
async function formatData(input, parameters) {
|
|
1418
|
-
const typedInput = input;
|
|
1419
|
-
const contents = await formatContents(typedInput, parameters);
|
|
1420
|
-
const generationConfig = formatGenerationConfig(parameters);
|
|
1421
|
-
const tools = formatTools(parameters);
|
|
1422
|
-
const toolConfig = formatToolConfig(parameters);
|
|
1423
|
-
const safetySettings = formatSafetySettings(parameters);
|
|
1424
|
-
const systemInstruction = await formatSystemInstruction(typedInput);
|
|
1425
|
-
const ret = {
|
|
1426
|
-
contents,
|
|
1427
|
-
generationConfig,
|
|
1428
|
-
};
|
|
1429
|
-
if (tools && tools.length) {
|
|
1430
|
-
ret.tools = tools;
|
|
1431
|
-
}
|
|
1432
|
-
if (toolConfig) {
|
|
1433
|
-
ret.toolConfig = toolConfig;
|
|
1434
|
-
}
|
|
1435
|
-
if (safetySettings && safetySettings.length) {
|
|
1436
|
-
ret.safetySettings = safetySettings;
|
|
1437
|
-
}
|
|
1438
|
-
if (systemInstruction?.role &&
|
|
1439
|
-
systemInstruction?.parts &&
|
|
1440
|
-
systemInstruction?.parts?.length) {
|
|
1441
|
-
ret.systemInstruction = systemInstruction;
|
|
1442
|
-
}
|
|
1443
|
-
if (parameters.cachedContent) {
|
|
1444
|
-
ret.cachedContent = parameters.cachedContent;
|
|
1445
|
-
}
|
|
1446
|
-
if (parameters.labels && Object.keys(parameters.labels).length > 0) {
|
|
1447
|
-
ret.labels = parameters.labels;
|
|
1448
|
-
}
|
|
1449
|
-
return ret;
|
|
1450
|
-
}
|
|
1451
|
-
return {
|
|
1452
|
-
messageContentToParts,
|
|
1453
|
-
baseMessageToContent,
|
|
1454
|
-
responseToString: safeResponseToString,
|
|
1455
|
-
responseToChatGeneration: safeResponseToChatGeneration,
|
|
1456
|
-
chunkToString,
|
|
1457
|
-
responseToBaseMessage: safeResponseToBaseMessage,
|
|
1458
|
-
responseToChatResult: safeResponseToChatResult,
|
|
1459
|
-
formatData,
|
|
1460
|
-
};
|
|
160
|
+
function getGeminiAPI(config) {
|
|
161
|
+
function messageContentText(content) {
|
|
162
|
+
if (content?.text && content?.text.length > 0) return { text: content.text };
|
|
163
|
+
else return null;
|
|
164
|
+
}
|
|
165
|
+
function messageContentImageUrlData(content) {
|
|
166
|
+
const url = typeof content.image_url === "string" ? content.image_url : content.image_url.url;
|
|
167
|
+
if (!url) throw new Error("Missing Image URL");
|
|
168
|
+
const mimeTypeAndData = extractMimeType(url);
|
|
169
|
+
if (mimeTypeAndData) return { inlineData: mimeTypeAndData };
|
|
170
|
+
else {
|
|
171
|
+
const mimeType = inferMimeTypeFromUrl(url) || "image/png";
|
|
172
|
+
return { fileData: {
|
|
173
|
+
mimeType,
|
|
174
|
+
fileUri: url
|
|
175
|
+
} };
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function messageContentImageUrl(content) {
|
|
179
|
+
const ret = messageContentImageUrlData(content);
|
|
180
|
+
supplementVideoMetadata(content, ret);
|
|
181
|
+
return ret;
|
|
182
|
+
}
|
|
183
|
+
async function blobToFileData(blob) {
|
|
184
|
+
return { fileData: {
|
|
185
|
+
fileUri: blob.path,
|
|
186
|
+
mimeType: blob.mimetype
|
|
187
|
+
} };
|
|
188
|
+
}
|
|
189
|
+
async function fileUriContentToBlob(uri) {
|
|
190
|
+
return config?.mediaManager?.getMediaBlob(uri);
|
|
191
|
+
}
|
|
192
|
+
async function messageContentMediaData(content) {
|
|
193
|
+
if ("mimeType" in content && "data" in content) return { inlineData: {
|
|
194
|
+
mimeType: content.mimeType,
|
|
195
|
+
data: content.data
|
|
196
|
+
} };
|
|
197
|
+
else if ("mimeType" in content && "fileUri" in content) return { fileData: {
|
|
198
|
+
mimeType: content.mimeType,
|
|
199
|
+
fileUri: content.fileUri
|
|
200
|
+
} };
|
|
201
|
+
else {
|
|
202
|
+
const uri = content.fileUri;
|
|
203
|
+
const blob = await fileUriContentToBlob(uri);
|
|
204
|
+
if (blob) return await blobToFileData(blob);
|
|
205
|
+
}
|
|
206
|
+
throw new Error(`Invalid media content: ${JSON.stringify(content, null, 1)}`);
|
|
207
|
+
}
|
|
208
|
+
function supplementVideoMetadata(content, ret) {
|
|
209
|
+
if ("videoMetadata" in content && typeof ret === "object") ret.videoMetadata = content.videoMetadata;
|
|
210
|
+
return ret;
|
|
211
|
+
}
|
|
212
|
+
async function messageContentMedia(content) {
|
|
213
|
+
const ret = await messageContentMediaData(content);
|
|
214
|
+
supplementVideoMetadata(content, ret);
|
|
215
|
+
return ret;
|
|
216
|
+
}
|
|
217
|
+
function messageContentReasoning(content) {
|
|
218
|
+
if (content?.reasoning && content?.reasoning.length > 0) return {
|
|
219
|
+
text: content.reasoning,
|
|
220
|
+
thought: true
|
|
221
|
+
};
|
|
222
|
+
else return null;
|
|
223
|
+
}
|
|
224
|
+
const standardContentBlockConverter = {
|
|
225
|
+
providerName: "Google Gemini",
|
|
226
|
+
fromStandardTextBlock(block) {
|
|
227
|
+
return { text: block.text };
|
|
228
|
+
},
|
|
229
|
+
fromStandardImageBlock(block) {
|
|
230
|
+
if (block.source_type === "url") {
|
|
231
|
+
const data = parseBase64DataUrl({ dataUrl: block.url });
|
|
232
|
+
if (data) return { inlineData: {
|
|
233
|
+
mimeType: data.mime_type,
|
|
234
|
+
data: data.data
|
|
235
|
+
} };
|
|
236
|
+
else {
|
|
237
|
+
let mimeType = block.mime_type;
|
|
238
|
+
if (!mimeType || mimeType === "") mimeType = inferMimeTypeFromUrl(block.url) || "image/png";
|
|
239
|
+
return { fileData: {
|
|
240
|
+
mimeType,
|
|
241
|
+
fileUri: block.url
|
|
242
|
+
} };
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (block.source_type === "base64") return { inlineData: {
|
|
246
|
+
mimeType: block.mime_type || "image/png",
|
|
247
|
+
data: block.data
|
|
248
|
+
} };
|
|
249
|
+
throw new Error(`Unsupported source type: ${block.source_type}`);
|
|
250
|
+
},
|
|
251
|
+
fromStandardAudioBlock(block) {
|
|
252
|
+
if (block.source_type === "url") {
|
|
253
|
+
const data = parseBase64DataUrl({ dataUrl: block.url });
|
|
254
|
+
if (data) return { inlineData: {
|
|
255
|
+
mimeType: data.mime_type,
|
|
256
|
+
data: data.data
|
|
257
|
+
} };
|
|
258
|
+
else return { fileData: {
|
|
259
|
+
mimeType: block.mime_type || "audio/mpeg",
|
|
260
|
+
fileUri: block.url
|
|
261
|
+
} };
|
|
262
|
+
}
|
|
263
|
+
if (block.source_type === "base64") return { inlineData: {
|
|
264
|
+
mimeType: block.mime_type || "audio/mpeg",
|
|
265
|
+
data: block.data
|
|
266
|
+
} };
|
|
267
|
+
throw new Error(`Unsupported source type: ${block.source_type}`);
|
|
268
|
+
},
|
|
269
|
+
fromStandardFileBlock(block) {
|
|
270
|
+
if (block.source_type === "text") return { text: block.text };
|
|
271
|
+
if (block.source_type === "url") {
|
|
272
|
+
const data = parseBase64DataUrl({ dataUrl: block.url });
|
|
273
|
+
if (data) return { inlineData: {
|
|
274
|
+
mimeType: data.mime_type,
|
|
275
|
+
data: data.data
|
|
276
|
+
} };
|
|
277
|
+
else return { fileData: {
|
|
278
|
+
mimeType: block.mime_type || "application/octet-stream",
|
|
279
|
+
fileUri: block.url
|
|
280
|
+
} };
|
|
281
|
+
}
|
|
282
|
+
if (block.source_type === "base64") return { inlineData: {
|
|
283
|
+
mimeType: block.mime_type || "application/octet-stream",
|
|
284
|
+
data: block.data
|
|
285
|
+
} };
|
|
286
|
+
throw new Error(`Unsupported source type: ${block.source_type}`);
|
|
287
|
+
}
|
|
288
|
+
};
|
|
289
|
+
async function messageContentComplexToPart(content) {
|
|
290
|
+
switch (content.type) {
|
|
291
|
+
case "text":
|
|
292
|
+
if ("text" in content) return messageContentText(content);
|
|
293
|
+
break;
|
|
294
|
+
case "image_url":
|
|
295
|
+
if ("image_url" in content) return messageContentImageUrl(content);
|
|
296
|
+
break;
|
|
297
|
+
case "media": return await messageContentMedia(content);
|
|
298
|
+
case "reasoning": return messageContentReasoning(content);
|
|
299
|
+
default: throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${JSON.stringify(content)}`);
|
|
300
|
+
}
|
|
301
|
+
throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
|
|
302
|
+
}
|
|
303
|
+
async function messageContentComplexToParts(content) {
|
|
304
|
+
const contents = content.map((m) => isDataContentBlock(m) ? convertToProviderContentBlock(m, standardContentBlockConverter) : messageContentComplexToPart(m));
|
|
305
|
+
return Promise.all(contents);
|
|
306
|
+
}
|
|
307
|
+
async function messageContentToParts(content) {
|
|
308
|
+
const messageContent = typeof content === "string" ? [{
|
|
309
|
+
type: "text",
|
|
310
|
+
text: content
|
|
311
|
+
}] : content;
|
|
312
|
+
const allParts = await messageContentComplexToParts(messageContent);
|
|
313
|
+
const parts = allParts.reduce((acc, val) => {
|
|
314
|
+
if (val) return [...acc, val];
|
|
315
|
+
else return acc;
|
|
316
|
+
}, []);
|
|
317
|
+
return parts;
|
|
318
|
+
}
|
|
319
|
+
function messageToolCallsToParts(toolCalls) {
|
|
320
|
+
if (!toolCalls || toolCalls.length === 0) return [];
|
|
321
|
+
return toolCalls.map((tool) => {
|
|
322
|
+
let args = {};
|
|
323
|
+
if (tool?.function?.arguments) {
|
|
324
|
+
const argStr = tool.function.arguments;
|
|
325
|
+
args = JSON.parse(argStr);
|
|
326
|
+
}
|
|
327
|
+
return { functionCall: {
|
|
328
|
+
name: tool.function.name,
|
|
329
|
+
args
|
|
330
|
+
} };
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
function messageKwargsToParts(kwargs) {
|
|
334
|
+
const ret = [];
|
|
335
|
+
if (kwargs?.tool_calls) ret.push(...messageToolCallsToParts(kwargs.tool_calls));
|
|
336
|
+
return ret;
|
|
337
|
+
}
|
|
338
|
+
async function roleMessageToContent(role, message) {
|
|
339
|
+
const contentParts = await messageContentToParts(message.content);
|
|
340
|
+
let toolParts;
|
|
341
|
+
if (isAIMessage(message) && !!message.tool_calls?.length) toolParts = message.tool_calls.map((toolCall) => ({ functionCall: {
|
|
342
|
+
name: toolCall.name,
|
|
343
|
+
args: toolCall.args
|
|
344
|
+
} }));
|
|
345
|
+
else toolParts = messageKwargsToParts(message.additional_kwargs);
|
|
346
|
+
const parts = [...contentParts, ...toolParts];
|
|
347
|
+
const signatures = message?.additional_kwargs?.signatures ?? [];
|
|
348
|
+
if (signatures.length === parts.length) for (let co = 0; co < signatures.length; co += 1) {
|
|
349
|
+
const signature = signatures[co];
|
|
350
|
+
if (signature && signature.length > 0) parts[co].thoughtSignature = signature;
|
|
351
|
+
}
|
|
352
|
+
return [{
|
|
353
|
+
role,
|
|
354
|
+
parts
|
|
355
|
+
}];
|
|
356
|
+
}
|
|
357
|
+
async function systemMessageToContent(message) {
|
|
358
|
+
return config?.useSystemInstruction ? roleMessageToContent("system", message) : [...await roleMessageToContent("user", message), ...await roleMessageToContent("model", new AIMessage("Ok"))];
|
|
359
|
+
}
|
|
360
|
+
function toolMessageToContent(message, prevMessage) {
|
|
361
|
+
const contentStr = typeof message.content === "string" ? message.content : message.content.reduce((acc, content) => {
|
|
362
|
+
if (content.type === "text") return acc + content.text;
|
|
363
|
+
else return acc;
|
|
364
|
+
}, "");
|
|
365
|
+
const responseName = (isAIMessage(prevMessage) && !!prevMessage.tool_calls?.length ? prevMessage.tool_calls[0].name : prevMessage.name) ?? message.tool_call_id;
|
|
366
|
+
try {
|
|
367
|
+
const content = JSON.parse(contentStr);
|
|
368
|
+
return [{
|
|
369
|
+
role: "function",
|
|
370
|
+
parts: [{ functionResponse: {
|
|
371
|
+
name: responseName,
|
|
372
|
+
response: { content }
|
|
373
|
+
} }]
|
|
374
|
+
}];
|
|
375
|
+
} catch (_) {
|
|
376
|
+
return [{
|
|
377
|
+
role: "function",
|
|
378
|
+
parts: [{ functionResponse: {
|
|
379
|
+
name: responseName,
|
|
380
|
+
response: { content: contentStr }
|
|
381
|
+
} }]
|
|
382
|
+
}];
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
async function baseMessageToContent(message, prevMessage) {
|
|
386
|
+
if (SystemMessage.isInstance(message)) return systemMessageToContent(message);
|
|
387
|
+
else if (HumanMessage.isInstance(message)) return roleMessageToContent("user", message);
|
|
388
|
+
else if (AIMessage.isInstance(message)) return roleMessageToContent("model", message);
|
|
389
|
+
else if (ToolMessage.isInstance(message)) {
|
|
390
|
+
if (!prevMessage) throw new Error("Tool messages cannot be the first message passed to the model.");
|
|
391
|
+
return toolMessageToContent(message, prevMessage);
|
|
392
|
+
} else {
|
|
393
|
+
console.log(`Unsupported message type: ${message.type}`);
|
|
394
|
+
return [];
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
function thoughtPartToMessageContent(part) {
|
|
398
|
+
return {
|
|
399
|
+
type: "reasoning",
|
|
400
|
+
reasoning: part.text
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
function textPartToMessageContent(part) {
|
|
404
|
+
return {
|
|
405
|
+
type: "text",
|
|
406
|
+
text: part.text
|
|
407
|
+
};
|
|
408
|
+
}
|
|
409
|
+
function inlineDataPartToMessageContentImage(part) {
|
|
410
|
+
return {
|
|
411
|
+
type: "image_url",
|
|
412
|
+
image_url: `data:${part.inlineData.mimeType};base64,${part.inlineData.data}`
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
function inlineDataPartToMessageContentMedia(part) {
|
|
416
|
+
return {
|
|
417
|
+
type: "media",
|
|
418
|
+
mimeType: part.inlineData.mimeType,
|
|
419
|
+
data: part.inlineData.data
|
|
420
|
+
};
|
|
421
|
+
}
|
|
422
|
+
function inlineDataPartToMessageContent(part) {
|
|
423
|
+
const mimeType = part?.inlineData?.mimeType ?? "";
|
|
424
|
+
if (mimeType.startsWith("image")) return inlineDataPartToMessageContentImage(part);
|
|
425
|
+
else return inlineDataPartToMessageContentMedia(part);
|
|
426
|
+
}
|
|
427
|
+
function fileDataPartToMessageContent(part) {
|
|
428
|
+
return {
|
|
429
|
+
type: "image_url",
|
|
430
|
+
image_url: part.fileData.fileUri
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
function partsToMessageContent(parts) {
|
|
434
|
+
return parts.map((part) => {
|
|
435
|
+
if (part === void 0 || part === null) return null;
|
|
436
|
+
else if (part.thought) return thoughtPartToMessageContent(part);
|
|
437
|
+
else if ("text" in part) return textPartToMessageContent(part);
|
|
438
|
+
else if ("inlineData" in part) return inlineDataPartToMessageContent(part);
|
|
439
|
+
else if ("fileData" in part) return fileDataPartToMessageContent(part);
|
|
440
|
+
else return null;
|
|
441
|
+
}).reduce((acc, content) => {
|
|
442
|
+
if (content) acc.push(content);
|
|
443
|
+
return acc;
|
|
444
|
+
}, []);
|
|
445
|
+
}
|
|
446
|
+
function toolRawToTool(raw) {
|
|
447
|
+
return {
|
|
448
|
+
id: raw.id,
|
|
449
|
+
type: raw.type,
|
|
450
|
+
function: {
|
|
451
|
+
name: raw.function.name,
|
|
452
|
+
arguments: JSON.stringify(raw.function.arguments)
|
|
453
|
+
}
|
|
454
|
+
};
|
|
455
|
+
}
|
|
456
|
+
function functionCallPartToToolRaw(part) {
|
|
457
|
+
return {
|
|
458
|
+
id: v4().replace(/-/g, ""),
|
|
459
|
+
type: "function",
|
|
460
|
+
function: {
|
|
461
|
+
name: part.functionCall.name,
|
|
462
|
+
arguments: part.functionCall.args ?? {}
|
|
463
|
+
}
|
|
464
|
+
};
|
|
465
|
+
}
|
|
466
|
+
function partsToToolsRaw(parts) {
|
|
467
|
+
return parts.map((part) => {
|
|
468
|
+
if (part === void 0 || part === null) return null;
|
|
469
|
+
else if ("functionCall" in part) return functionCallPartToToolRaw(part);
|
|
470
|
+
else return null;
|
|
471
|
+
}).reduce((acc, content) => {
|
|
472
|
+
if (content) acc.push(content);
|
|
473
|
+
return acc;
|
|
474
|
+
}, []);
|
|
475
|
+
}
|
|
476
|
+
function toolsRawToTools(raws) {
|
|
477
|
+
return raws.map((raw) => toolRawToTool(raw));
|
|
478
|
+
}
|
|
479
|
+
function responseToGenerateContentResponseData(response) {
|
|
480
|
+
if ("nextChunk" in response.data) throw new Error("Cannot convert Stream to GenerateContentResponseData");
|
|
481
|
+
else if (Array.isArray(response.data)) return response.data.reduce((acc, val) => {
|
|
482
|
+
const valParts = val?.candidates?.[0]?.content?.parts ?? [];
|
|
483
|
+
acc.candidates[0].content.parts.push(...valParts);
|
|
484
|
+
acc.promptFeedback = val.promptFeedback;
|
|
485
|
+
return acc;
|
|
486
|
+
});
|
|
487
|
+
else return response.data;
|
|
488
|
+
}
|
|
489
|
+
function responseToParts(response) {
|
|
490
|
+
const responseData = responseToGenerateContentResponseData(response);
|
|
491
|
+
const parts = responseData?.candidates?.[0]?.content?.parts ?? [];
|
|
492
|
+
return parts;
|
|
493
|
+
}
|
|
494
|
+
function partToText(part) {
|
|
495
|
+
return "text" in part ? part.text : "";
|
|
496
|
+
}
|
|
497
|
+
function responseToString(response) {
|
|
498
|
+
const parts = responseToParts(response);
|
|
499
|
+
const ret = parts.reduce((acc, part) => {
|
|
500
|
+
const val = partToText(part);
|
|
501
|
+
return acc + val;
|
|
502
|
+
}, "");
|
|
503
|
+
return ret;
|
|
504
|
+
}
|
|
505
|
+
function safeResponseTo(response, responseTo) {
|
|
506
|
+
const safetyHandler = config?.safetyHandler ?? new DefaultGeminiSafetyHandler();
|
|
507
|
+
try {
|
|
508
|
+
const safeResponse = safetyHandler.handle(response);
|
|
509
|
+
return responseTo(safeResponse);
|
|
510
|
+
} catch (xx) {
|
|
511
|
+
if (xx instanceof GoogleAISafetyError) {
|
|
512
|
+
const ret = responseTo(xx.response);
|
|
513
|
+
xx.reply = ret;
|
|
514
|
+
}
|
|
515
|
+
throw xx;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
function safeResponseToString(response) {
|
|
519
|
+
return safeResponseTo(response, responseToString);
|
|
520
|
+
}
|
|
521
|
+
function logprobResultToLogprob(result) {
|
|
522
|
+
const token = result?.token;
|
|
523
|
+
const logprob = result?.logProbability;
|
|
524
|
+
const encoder = new TextEncoder();
|
|
525
|
+
const bytes = Array.from(encoder.encode(token));
|
|
526
|
+
return {
|
|
527
|
+
token,
|
|
528
|
+
logprob,
|
|
529
|
+
bytes
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
function candidateToLogprobs(candidate) {
|
|
533
|
+
const logprobs = candidate?.logprobsResult;
|
|
534
|
+
const chosenTokens = logprobs?.chosenCandidates ?? [];
|
|
535
|
+
const topTokens = logprobs?.topCandidates ?? [];
|
|
536
|
+
const content = [];
|
|
537
|
+
for (let co = 0; co < chosenTokens.length; co += 1) {
|
|
538
|
+
const chosen = chosenTokens[co];
|
|
539
|
+
const top = topTokens[co]?.candidates ?? [];
|
|
540
|
+
const logprob = logprobResultToLogprob(chosen);
|
|
541
|
+
logprob.top_logprobs = top.map((l) => logprobResultToLogprob(l));
|
|
542
|
+
content.push(logprob);
|
|
543
|
+
}
|
|
544
|
+
return { content };
|
|
545
|
+
}
|
|
546
|
+
function candidateToUrlContextMetadata(candidate) {
|
|
547
|
+
const retrieval = candidate?.urlRetrievalMetadata?.urlRetrievalContexts ?? [];
|
|
548
|
+
const context = candidate?.urlContextMetadata?.urlMetadata ?? [];
|
|
549
|
+
const all = [...retrieval, ...context];
|
|
550
|
+
if (all.length === 0) return void 0;
|
|
551
|
+
else return { urlMetadata: all };
|
|
552
|
+
}
|
|
553
|
+
function addModalityCounts(modalityTokenCounts, details) {
|
|
554
|
+
modalityTokenCounts?.forEach((modalityTokenCount) => {
|
|
555
|
+
const { modality, tokenCount } = modalityTokenCount;
|
|
556
|
+
const modalityLc = modality.toLowerCase();
|
|
557
|
+
const currentCount = details[modalityLc] ?? 0;
|
|
558
|
+
details[modalityLc] = currentCount + tokenCount;
|
|
559
|
+
});
|
|
560
|
+
}
|
|
561
|
+
function responseToUsageMetadata(response) {
|
|
562
|
+
if ("usageMetadata" in response.data) {
|
|
563
|
+
const data = response?.data;
|
|
564
|
+
const usageMetadata = data?.usageMetadata;
|
|
565
|
+
const input_tokens = usageMetadata.promptTokenCount ?? 0;
|
|
566
|
+
const candidatesTokenCount = usageMetadata.candidatesTokenCount ?? 0;
|
|
567
|
+
const thoughtsTokenCount = usageMetadata.thoughtsTokenCount ?? 0;
|
|
568
|
+
const output_tokens = candidatesTokenCount + thoughtsTokenCount;
|
|
569
|
+
const total_tokens = usageMetadata.totalTokenCount ?? input_tokens + output_tokens;
|
|
570
|
+
const input_token_details = {};
|
|
571
|
+
addModalityCounts(usageMetadata.promptTokensDetails, input_token_details);
|
|
572
|
+
if (typeof usageMetadata?.cachedContentTokenCount === "number") input_token_details.cache_read = usageMetadata.cachedContentTokenCount;
|
|
573
|
+
const output_token_details = {};
|
|
574
|
+
addModalityCounts(usageMetadata?.candidatesTokensDetails, output_token_details);
|
|
575
|
+
if (typeof usageMetadata?.thoughtsTokenCount === "number") output_token_details.reasoning = usageMetadata.thoughtsTokenCount;
|
|
576
|
+
const ret = {
|
|
577
|
+
input_tokens,
|
|
578
|
+
output_tokens,
|
|
579
|
+
total_tokens,
|
|
580
|
+
input_token_details,
|
|
581
|
+
output_token_details
|
|
582
|
+
};
|
|
583
|
+
return ret;
|
|
584
|
+
}
|
|
585
|
+
return void 0;
|
|
586
|
+
}
|
|
587
|
+
function responseToGenerationInfo(response) {
|
|
588
|
+
const data = Array.isArray(response.data) && response.data[0] ? response.data[0] : response.data && response.data.candidates ? response.data : void 0;
|
|
589
|
+
if (!data) return {};
|
|
590
|
+
const finish_reason = data.candidates[0]?.finishReason;
|
|
591
|
+
const ret = {
|
|
592
|
+
safety_ratings: data.candidates[0]?.safetyRatings?.map((rating) => ({
|
|
593
|
+
category: rating.category,
|
|
594
|
+
probability: rating.probability,
|
|
595
|
+
probability_score: rating.probabilityScore,
|
|
596
|
+
severity: rating.severity,
|
|
597
|
+
severity_score: rating.severityScore
|
|
598
|
+
})),
|
|
599
|
+
citation_metadata: data.candidates[0]?.citationMetadata,
|
|
600
|
+
grounding_metadata: data.candidates[0]?.groundingMetadata,
|
|
601
|
+
finish_reason,
|
|
602
|
+
finish_message: data.candidates[0]?.finishMessage,
|
|
603
|
+
url_context_metadata: candidateToUrlContextMetadata(data.candidates[0]),
|
|
604
|
+
avgLogprobs: data.candidates[0]?.avgLogprobs,
|
|
605
|
+
logprobs: candidateToLogprobs(data.candidates[0])
|
|
606
|
+
};
|
|
607
|
+
if (typeof finish_reason === "string") ret.usage_metadata = responseToUsageMetadata(response);
|
|
608
|
+
return ret;
|
|
609
|
+
}
|
|
610
|
+
function responseToChatGeneration(response) {
|
|
611
|
+
return new ChatGenerationChunk({
|
|
612
|
+
text: responseToString(response),
|
|
613
|
+
message: partToMessageChunk(responseToParts(response)[0]),
|
|
614
|
+
generationInfo: responseToGenerationInfo(response)
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
function safeResponseToChatGeneration(response) {
|
|
618
|
+
return safeResponseTo(response, responseToChatGeneration);
|
|
619
|
+
}
|
|
620
|
+
function chunkToString(chunk) {
|
|
621
|
+
if (chunk === null) return "";
|
|
622
|
+
else if (typeof chunk.content === "string") return chunk.content;
|
|
623
|
+
else if (chunk.content.length === 0) return "";
|
|
624
|
+
else if (chunk.content[0].type === "text") return chunk.content[0].text;
|
|
625
|
+
else throw new Error(`Unexpected chunk: ${chunk}`);
|
|
626
|
+
}
|
|
627
|
+
function partToMessageChunk(part) {
|
|
628
|
+
const fields = partsToBaseMessageChunkFields([part]);
|
|
629
|
+
if (typeof fields.content === "string") return new AIMessageChunk(fields);
|
|
630
|
+
else if (fields.content?.every((item) => item.type === "text")) {
|
|
631
|
+
const newContent = fields.content.map((item) => "text" in item ? item.text : "").join("");
|
|
632
|
+
return new AIMessageChunk({
|
|
633
|
+
...fields,
|
|
634
|
+
content: newContent,
|
|
635
|
+
response_metadata: {
|
|
636
|
+
...fields.response_metadata,
|
|
637
|
+
model_provider: "google-vertexai"
|
|
638
|
+
}
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
return new AIMessageChunk(fields);
|
|
642
|
+
}
|
|
643
|
+
function partToChatGeneration(part) {
|
|
644
|
+
const message = partToMessageChunk(part);
|
|
645
|
+
const text = partToText(part);
|
|
646
|
+
const generationInfo = {};
|
|
647
|
+
return new ChatGenerationChunk({
|
|
648
|
+
text,
|
|
649
|
+
message,
|
|
650
|
+
generationInfo
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
function groundingSupportByPart(groundingSupports) {
|
|
654
|
+
const ret = [];
|
|
655
|
+
if (!groundingSupports || groundingSupports.length === 0) return [];
|
|
656
|
+
groundingSupports?.forEach((groundingSupport) => {
|
|
657
|
+
const segment = groundingSupport?.segment;
|
|
658
|
+
const partIndex = segment?.partIndex ?? 0;
|
|
659
|
+
if (ret[partIndex]) ret[partIndex].push(groundingSupport);
|
|
660
|
+
else ret[partIndex] = [groundingSupport];
|
|
661
|
+
});
|
|
662
|
+
return ret;
|
|
663
|
+
}
|
|
664
|
+
function responseToGroundedChatGenerations(response) {
|
|
665
|
+
const parts = responseToParts(response);
|
|
666
|
+
if (parts.length === 0) return [];
|
|
667
|
+
const candidate = (response?.data)?.candidates?.[0];
|
|
668
|
+
const groundingMetadata = candidate?.groundingMetadata;
|
|
669
|
+
const citationMetadata = candidate?.citationMetadata;
|
|
670
|
+
const groundingParts = groundingSupportByPart(groundingMetadata?.groundingSupports);
|
|
671
|
+
const ret = parts.map((part, index) => {
|
|
672
|
+
const gen = partToChatGeneration(part);
|
|
673
|
+
if (!gen.generationInfo) gen.generationInfo = {};
|
|
674
|
+
if (groundingMetadata) {
|
|
675
|
+
gen.generationInfo.groundingMetadata = groundingMetadata;
|
|
676
|
+
const groundingPart = groundingParts[index];
|
|
677
|
+
if (groundingPart) gen.generationInfo.groundingSupport = groundingPart;
|
|
678
|
+
}
|
|
679
|
+
if (citationMetadata) gen.generationInfo.citationMetadata = citationMetadata;
|
|
680
|
+
return gen;
|
|
681
|
+
});
|
|
682
|
+
return ret;
|
|
683
|
+
}
|
|
684
|
+
function combineContent(gen, forceComplex = false) {
|
|
685
|
+
const allString = gen.every((item) => typeof item.message.content === "string");
|
|
686
|
+
if (allString && !forceComplex) return gen.map((item) => item.message.content).join("");
|
|
687
|
+
else {
|
|
688
|
+
const ret = [];
|
|
689
|
+
gen.forEach((item) => {
|
|
690
|
+
if (typeof item.message.content === "string") ret.push({
|
|
691
|
+
type: "text",
|
|
692
|
+
text: item.message.content
|
|
693
|
+
});
|
|
694
|
+
else item.message.content.forEach((c) => {
|
|
695
|
+
ret.push(c);
|
|
696
|
+
});
|
|
697
|
+
});
|
|
698
|
+
return ret;
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
function combineText(gen) {
|
|
702
|
+
return gen.map((item) => item.text ?? "").join("");
|
|
703
|
+
}
|
|
704
|
+
function combineToolCalls(gen) {
|
|
705
|
+
let ret = new AIMessageChunk("");
|
|
706
|
+
gen.forEach((item) => {
|
|
707
|
+
const message = item?.message;
|
|
708
|
+
ret = concat(ret, message);
|
|
709
|
+
});
|
|
710
|
+
return ret;
|
|
711
|
+
}
|
|
712
|
+
function combineAdditionalKwargs(gen) {
|
|
713
|
+
const ret = {};
|
|
714
|
+
gen.forEach((item) => {
|
|
715
|
+
const message = item?.message;
|
|
716
|
+
const kwargs = message?.additional_kwargs ?? {};
|
|
717
|
+
const keys = Object.keys(kwargs);
|
|
718
|
+
keys.forEach((key) => {
|
|
719
|
+
const value = kwargs[key];
|
|
720
|
+
if (Object.hasOwn(ret, key) && Array.isArray(ret[key]) && Array.isArray(value)) ret[key].push(...value);
|
|
721
|
+
else ret[key] = value;
|
|
722
|
+
});
|
|
723
|
+
});
|
|
724
|
+
return ret;
|
|
725
|
+
}
|
|
726
|
+
function combineGenerations(generations, response) {
|
|
727
|
+
const gen = splitGenerationTypes(generations, response);
|
|
728
|
+
const combinedContent = combineContent(gen.content);
|
|
729
|
+
const combinedText = combineText(gen.content);
|
|
730
|
+
const combinedToolCalls = combineToolCalls(gen.content);
|
|
731
|
+
const kwargs = combineAdditionalKwargs(gen.content);
|
|
732
|
+
const lastContent = gen.content[gen.content.length - 1];
|
|
733
|
+
const usage_metadata = responseToUsageMetadata(response);
|
|
734
|
+
const message = new AIMessageChunk({
|
|
735
|
+
content: combinedContent,
|
|
736
|
+
additional_kwargs: kwargs,
|
|
737
|
+
response_metadata: { model_provider: "google-vertexai" },
|
|
738
|
+
usage_metadata,
|
|
739
|
+
tool_calls: combinedToolCalls.tool_calls,
|
|
740
|
+
invalid_tool_calls: combinedToolCalls.invalid_tool_calls
|
|
741
|
+
});
|
|
742
|
+
return [new ChatGenerationChunk({
|
|
743
|
+
message,
|
|
744
|
+
text: combinedText,
|
|
745
|
+
generationInfo: lastContent.generationInfo
|
|
746
|
+
})];
|
|
747
|
+
}
|
|
748
|
+
function splitGenerationTypes(generations, _response) {
|
|
749
|
+
const content = [];
|
|
750
|
+
const reasoning = [];
|
|
751
|
+
generations.forEach((gen) => {
|
|
752
|
+
if (gen?.generationInfo?.thought) reasoning.push(gen);
|
|
753
|
+
else content.push(gen);
|
|
754
|
+
});
|
|
755
|
+
return {
|
|
756
|
+
content,
|
|
757
|
+
reasoning
|
|
758
|
+
};
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* Although this returns an array, only the first (or maybe last)
|
|
762
|
+
* element in the array is used. So we need to combine them into
|
|
763
|
+
* just one element that contains everything we need.
|
|
764
|
+
* @param response
|
|
765
|
+
*/
|
|
766
|
+
function responseToChatGenerations(response) {
|
|
767
|
+
const generations = responseToGroundedChatGenerations(response);
|
|
768
|
+
if (generations.length === 0) return [];
|
|
769
|
+
const ret = combineGenerations(generations, response);
|
|
770
|
+
const candidate = (response?.data)?.candidates?.[0];
|
|
771
|
+
const avgLogprobs = candidate?.avgLogprobs;
|
|
772
|
+
const logprobs = candidateToLogprobs(candidate);
|
|
773
|
+
if (logprobs) ret[0].message.response_metadata = {
|
|
774
|
+
model_provider: "google-vertexai",
|
|
775
|
+
...ret[0].message.response_metadata,
|
|
776
|
+
logprobs,
|
|
777
|
+
avgLogprobs
|
|
778
|
+
};
|
|
779
|
+
return ret;
|
|
780
|
+
}
|
|
781
|
+
function responseToBaseMessageFields(response) {
|
|
782
|
+
const parts = responseToParts(response);
|
|
783
|
+
return partsToBaseMessageChunkFields(parts);
|
|
784
|
+
}
|
|
785
|
+
function partsToSignatures(parts) {
|
|
786
|
+
return parts.map((part) => part?.thoughtSignature ?? "");
|
|
787
|
+
}
|
|
788
|
+
function partsToBaseMessageChunkFields(parts) {
|
|
789
|
+
const fields = {
|
|
790
|
+
content: partsToMessageContent(parts),
|
|
791
|
+
tool_call_chunks: [],
|
|
792
|
+
tool_calls: [],
|
|
793
|
+
invalid_tool_calls: [],
|
|
794
|
+
response_metadata: { model_provider: "google-vertexai" }
|
|
795
|
+
};
|
|
796
|
+
fields.additional_kwargs = {};
|
|
797
|
+
const rawTools = partsToToolsRaw(parts);
|
|
798
|
+
if (rawTools.length > 0) {
|
|
799
|
+
const tools = toolsRawToTools(rawTools);
|
|
800
|
+
for (const tool of tools) {
|
|
801
|
+
fields.tool_call_chunks?.push({
|
|
802
|
+
name: tool.function.name,
|
|
803
|
+
args: tool.function.arguments,
|
|
804
|
+
id: tool.id,
|
|
805
|
+
type: "tool_call_chunk"
|
|
806
|
+
});
|
|
807
|
+
try {
|
|
808
|
+
fields.tool_calls?.push({
|
|
809
|
+
name: tool.function.name,
|
|
810
|
+
args: JSON.parse(tool.function.arguments),
|
|
811
|
+
id: tool.id
|
|
812
|
+
});
|
|
813
|
+
} catch (e) {
|
|
814
|
+
fields.invalid_tool_calls?.push({
|
|
815
|
+
name: tool.function.name,
|
|
816
|
+
args: tool.function.arguments,
|
|
817
|
+
id: tool.id,
|
|
818
|
+
error: e.message,
|
|
819
|
+
type: "invalid_tool_call"
|
|
820
|
+
});
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
fields.additional_kwargs.tool_calls = tools;
|
|
824
|
+
}
|
|
825
|
+
fields.additional_kwargs.signatures = partsToSignatures(parts);
|
|
826
|
+
return fields;
|
|
827
|
+
}
|
|
828
|
+
function responseToBaseMessage(response) {
|
|
829
|
+
const fields = responseToBaseMessageFields(response);
|
|
830
|
+
return new AIMessage(fields);
|
|
831
|
+
}
|
|
832
|
+
function safeResponseToBaseMessage(response) {
|
|
833
|
+
return safeResponseTo(response, responseToBaseMessage);
|
|
834
|
+
}
|
|
835
|
+
function responseToChatResult(response) {
|
|
836
|
+
const generations = responseToChatGenerations(response);
|
|
837
|
+
return {
|
|
838
|
+
generations,
|
|
839
|
+
llmOutput: responseToGenerationInfo(response)
|
|
840
|
+
};
|
|
841
|
+
}
|
|
842
|
+
function safeResponseToChatResult(response) {
|
|
843
|
+
return safeResponseTo(response, responseToChatResult);
|
|
844
|
+
}
|
|
845
|
+
function inputType(input) {
|
|
846
|
+
if (typeof input === "string") return "MessageContent";
|
|
847
|
+
else {
|
|
848
|
+
const firstItem = input[0];
|
|
849
|
+
if (Object.hasOwn(firstItem, "content")) return "BaseMessageArray";
|
|
850
|
+
else return "MessageContent";
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
async function formatMessageContents(input, _parameters) {
|
|
854
|
+
const parts = await messageContentToParts(input);
|
|
855
|
+
const contents = [{
|
|
856
|
+
role: "user",
|
|
857
|
+
parts
|
|
858
|
+
}];
|
|
859
|
+
return contents;
|
|
860
|
+
}
|
|
861
|
+
async function formatBaseMessageContents(input, _parameters) {
|
|
862
|
+
const inputPromises = input.map((msg, i) => baseMessageToContent(msg, input[i - 1]));
|
|
863
|
+
const inputs = await Promise.all(inputPromises);
|
|
864
|
+
return inputs.reduce((acc, cur) => {
|
|
865
|
+
if (cur.every((content) => content.role === "system")) return acc;
|
|
866
|
+
if (cur[0]?.role === "function" && acc.length > 0 && acc[acc.length - 1].role === "function") acc[acc.length - 1].parts = [...acc[acc.length - 1].parts, ...cur[0].parts];
|
|
867
|
+
else acc.push(...cur);
|
|
868
|
+
return acc;
|
|
869
|
+
}, []);
|
|
870
|
+
}
|
|
871
|
+
async function formatContents(input, parameters) {
|
|
872
|
+
const it = inputType(input);
|
|
873
|
+
switch (it) {
|
|
874
|
+
case "MessageContent": return formatMessageContents(input, parameters);
|
|
875
|
+
case "BaseMessageArray": return formatBaseMessageContents(input, parameters);
|
|
876
|
+
default: throw new Error(`Unknown input type "${it}": ${input}`);
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
function formatGenerationConfig(parameters) {
|
|
880
|
+
const ret = {
|
|
881
|
+
temperature: parameters.temperature,
|
|
882
|
+
topK: parameters.topK,
|
|
883
|
+
topP: parameters.topP,
|
|
884
|
+
seed: parameters.seed,
|
|
885
|
+
presencePenalty: parameters.presencePenalty,
|
|
886
|
+
frequencyPenalty: parameters.frequencyPenalty,
|
|
887
|
+
maxOutputTokens: parameters.maxOutputTokens,
|
|
888
|
+
stopSequences: parameters.stopSequences,
|
|
889
|
+
responseMimeType: parameters.responseMimeType,
|
|
890
|
+
responseModalities: parameters.responseModalities,
|
|
891
|
+
speechConfig: normalizeSpeechConfig(parameters.speechConfig)
|
|
892
|
+
};
|
|
893
|
+
if (typeof parameters.logprobs !== "undefined") {
|
|
894
|
+
ret.responseLogprobs = parameters.logprobs;
|
|
895
|
+
if (parameters.logprobs && typeof parameters.topLogprobs !== "undefined") ret.logprobs = parameters.topLogprobs;
|
|
896
|
+
}
|
|
897
|
+
if (typeof parameters.maxReasoningTokens !== "undefined") {
|
|
898
|
+
const includeThoughts = parameters.maxReasoningTokens !== 0;
|
|
899
|
+
ret.thinkingConfig = {
|
|
900
|
+
thinkingBudget: parameters.maxReasoningTokens,
|
|
901
|
+
includeThoughts
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
let attribute;
|
|
905
|
+
for (attribute in ret) if (ret[attribute] === void 0) delete ret[attribute];
|
|
906
|
+
return ret;
|
|
907
|
+
}
|
|
908
|
+
function formatSafetySettings(parameters) {
|
|
909
|
+
return parameters.safetySettings ?? [];
|
|
910
|
+
}
|
|
911
|
+
async function formatBaseMessageSystemInstruction(input) {
|
|
912
|
+
let ret = {};
|
|
913
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
914
|
+
const message = input[index];
|
|
915
|
+
if (message.getType() === "system") if (index === 0) ret = (await baseMessageToContent(message, void 0))[0];
|
|
916
|
+
else throw new Error("System messages are only permitted as the first passed message.");
|
|
917
|
+
}
|
|
918
|
+
return ret;
|
|
919
|
+
}
|
|
920
|
+
async function formatSystemInstruction(input) {
|
|
921
|
+
if (!config?.useSystemInstruction) return {};
|
|
922
|
+
const it = inputType(input);
|
|
923
|
+
switch (it) {
|
|
924
|
+
case "BaseMessageArray": return formatBaseMessageSystemInstruction(input);
|
|
925
|
+
default: return {};
|
|
926
|
+
}
|
|
927
|
+
}
|
|
928
|
+
function structuredToolToFunctionDeclaration(tool) {
|
|
929
|
+
const jsonSchema = schemaToGeminiParameters(tool.schema);
|
|
930
|
+
return {
|
|
931
|
+
name: tool.name,
|
|
932
|
+
description: tool.description ?? `A function available to call.`,
|
|
933
|
+
parameters: jsonSchema
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
function searchToolName(tool) {
|
|
937
|
+
for (const name of GeminiSearchToolAttributes) if (name in tool) return name;
|
|
938
|
+
return void 0;
|
|
939
|
+
}
|
|
940
|
+
function cleanGeminiTool(tool) {
|
|
941
|
+
const orig = searchToolName(tool);
|
|
942
|
+
const adj = config?.googleSearchToolAdjustment;
|
|
943
|
+
if (orig && adj && adj !== orig) return { [adj]: {} };
|
|
944
|
+
else return tool;
|
|
945
|
+
}
|
|
946
|
+
function formatTools(parameters) {
|
|
947
|
+
const tools = parameters?.tools;
|
|
948
|
+
if (!tools || tools.length === 0) return [];
|
|
949
|
+
const langChainTools = [];
|
|
950
|
+
const otherTools = [];
|
|
951
|
+
tools.forEach((tool) => {
|
|
952
|
+
if (isLangChainTool(tool)) langChainTools.push(tool);
|
|
953
|
+
else otherTools.push(cleanGeminiTool(tool));
|
|
954
|
+
});
|
|
955
|
+
const result = [...otherTools];
|
|
956
|
+
if (langChainTools.length > 0) result.push({ functionDeclarations: langChainTools.map(structuredToolToFunctionDeclaration) });
|
|
957
|
+
return result;
|
|
958
|
+
}
|
|
959
|
+
function formatToolConfig(parameters) {
|
|
960
|
+
if (!parameters.tool_choice || typeof parameters.tool_choice !== "string") return void 0;
|
|
961
|
+
if ([
|
|
962
|
+
"auto",
|
|
963
|
+
"any",
|
|
964
|
+
"none"
|
|
965
|
+
].includes(parameters.tool_choice)) return { functionCallingConfig: {
|
|
966
|
+
mode: parameters.tool_choice,
|
|
967
|
+
allowedFunctionNames: parameters.allowed_function_names
|
|
968
|
+
} };
|
|
969
|
+
return { functionCallingConfig: {
|
|
970
|
+
mode: "any",
|
|
971
|
+
allowedFunctionNames: [parameters.tool_choice]
|
|
972
|
+
} };
|
|
973
|
+
}
|
|
974
|
+
async function formatData(input, parameters) {
|
|
975
|
+
const typedInput = input;
|
|
976
|
+
const contents = await formatContents(typedInput, parameters);
|
|
977
|
+
const generationConfig = formatGenerationConfig(parameters);
|
|
978
|
+
const tools = formatTools(parameters);
|
|
979
|
+
const toolConfig = formatToolConfig(parameters);
|
|
980
|
+
const safetySettings = formatSafetySettings(parameters);
|
|
981
|
+
const systemInstruction = await formatSystemInstruction(typedInput);
|
|
982
|
+
const ret = {
|
|
983
|
+
contents,
|
|
984
|
+
generationConfig
|
|
985
|
+
};
|
|
986
|
+
if (tools && tools.length) ret.tools = tools;
|
|
987
|
+
if (toolConfig) ret.toolConfig = toolConfig;
|
|
988
|
+
if (safetySettings && safetySettings.length) ret.safetySettings = safetySettings;
|
|
989
|
+
if (systemInstruction?.role && systemInstruction?.parts && systemInstruction?.parts?.length) ret.systemInstruction = systemInstruction;
|
|
990
|
+
if (parameters.cachedContent) ret.cachedContent = parameters.cachedContent;
|
|
991
|
+
if (parameters.labels && Object.keys(parameters.labels).length > 0) ret.labels = parameters.labels;
|
|
992
|
+
return ret;
|
|
993
|
+
}
|
|
994
|
+
return {
|
|
995
|
+
messageContentToParts,
|
|
996
|
+
baseMessageToContent,
|
|
997
|
+
responseToString: safeResponseToString,
|
|
998
|
+
responseToChatGeneration: safeResponseToChatGeneration,
|
|
999
|
+
chunkToString,
|
|
1000
|
+
responseToBaseMessage: safeResponseToBaseMessage,
|
|
1001
|
+
responseToChatResult: safeResponseToChatResult,
|
|
1002
|
+
formatData
|
|
1003
|
+
};
|
|
1461
1004
|
}
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
}
|
|
1473
|
-
if (params.temperature &&
|
|
1474
|
-
(params.temperature < 0 || params.temperature > 2)) {
|
|
1475
|
-
throw new Error("`temperature` must be in the range of [0.0,2.0]");
|
|
1476
|
-
}
|
|
1477
|
-
if (params.topP && (params.topP < 0 || params.topP > 1)) {
|
|
1478
|
-
throw new Error("`topP` must be in the range of [0.0,1.0]");
|
|
1479
|
-
}
|
|
1480
|
-
if (params.topK && params.topK < 0) {
|
|
1481
|
-
throw new Error("`topK` must be a positive integer");
|
|
1482
|
-
}
|
|
1005
|
+
function validateGeminiParams(params) {
|
|
1006
|
+
if (params.maxOutputTokens && params.maxOutputTokens < 0) throw new Error("`maxOutputTokens` must be a positive integer");
|
|
1007
|
+
if (typeof params.maxReasoningTokens !== "undefined") {
|
|
1008
|
+
if (typeof params.maxOutputTokens !== "undefined") {
|
|
1009
|
+
if (params.maxReasoningTokens >= params.maxOutputTokens) throw new Error("`maxOutputTokens` must be greater than `maxReasoningTokens`");
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
if (params.temperature && (params.temperature < 0 || params.temperature > 2)) throw new Error("`temperature` must be in the range of [0.0,2.0]");
|
|
1013
|
+
if (params.topP && (params.topP < 0 || params.topP > 1)) throw new Error("`topP` must be in the range of [0.0,1.0]");
|
|
1014
|
+
if (params.topK && params.topK < 0) throw new Error("`topK` must be a positive integer");
|
|
1483
1015
|
}
|
|
1484
|
-
|
|
1485
|
-
|
|
1016
|
+
function isModelGemini(modelName) {
|
|
1017
|
+
return modelName.toLowerCase().startsWith("gemini");
|
|
1486
1018
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1019
|
+
function isModelGemma(modelName) {
|
|
1020
|
+
return modelName.toLowerCase().startsWith("gemma");
|
|
1489
1021
|
}
|
|
1022
|
+
|
|
1023
|
+
//#endregion
|
|
1024
|
+
export { DefaultGeminiSafetyHandler, MessageGeminiSafetyHandler, getGeminiAPI, isModelGemini, isModelGemma, normalizeSpeechConfig, validateGeminiParams };
|
|
1025
|
+
//# sourceMappingURL=gemini.js.map
|