@micdrop/gemini 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/index.d.mts +52 -2
- package/dist/index.d.ts +52 -2
- package/dist/index.js +176 -44
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +166 -35
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -3
package/dist/index.mjs
CHANGED
|
@@ -1,8 +1,61 @@
|
|
|
1
1
|
// src/GeminiAgent.ts
|
|
2
2
|
import { GoogleGenAI } from "@google/genai";
|
|
3
3
|
import { Agent } from "@micdrop/server";
|
|
4
|
+
|
|
5
|
+
// src/models.ts
|
|
6
|
+
import { cachedList, cachedModels } from "@micdrop/server";
|
|
7
|
+
function fetchAll(apiKey) {
|
|
8
|
+
return cachedList(["gemini-raw-models"], apiKey, async () => {
|
|
9
|
+
const models = [];
|
|
10
|
+
let pageToken;
|
|
11
|
+
do {
|
|
12
|
+
const params = new URLSearchParams({ pageSize: "1000", key: apiKey });
|
|
13
|
+
if (pageToken) params.set("pageToken", pageToken);
|
|
14
|
+
const response = await fetch(
|
|
15
|
+
`https://generativelanguage.googleapis.com/v1beta/models?${params}`
|
|
16
|
+
);
|
|
17
|
+
if (!response.ok) throw new Error(`Gemini answered ${response.status}`);
|
|
18
|
+
const page = await response.json();
|
|
19
|
+
models.push(...page.models);
|
|
20
|
+
pageToken = page.nextPageToken;
|
|
21
|
+
} while (pageToken);
|
|
22
|
+
return models;
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
function kindOf(model) {
|
|
26
|
+
const id = model.name.replace(/^models\//, "");
|
|
27
|
+
const methods = model.supportedGenerationMethods ?? [];
|
|
28
|
+
if (!id.startsWith("gemini-")) return void 0;
|
|
29
|
+
if (/-tts\b/.test(id)) return "tts";
|
|
30
|
+
if (methods.includes("bidiGenerateContent")) {
|
|
31
|
+
if (id.includes("transcribe")) return "stt";
|
|
32
|
+
if (/translate|robotics/.test(id)) return void 0;
|
|
33
|
+
return "live";
|
|
34
|
+
}
|
|
35
|
+
if (!methods.includes("generateContent")) return void 0;
|
|
36
|
+
if (/image|embedding|transcribe|robotics|computer-use|omni|nano-banana/.test(id)) {
|
|
37
|
+
return void 0;
|
|
38
|
+
}
|
|
39
|
+
return "agent";
|
|
40
|
+
}
|
|
41
|
+
function listGeminiModels(kind, options) {
|
|
42
|
+
return cachedModels(
|
|
43
|
+
["gemini", kind],
|
|
44
|
+
options.apiKey,
|
|
45
|
+
async () => (await fetchAll(options.apiKey)).filter((model) => kindOf(model) === kind).map((model) => ({
|
|
46
|
+
id: model.name.replace(/^models\//, ""),
|
|
47
|
+
name: model.displayName,
|
|
48
|
+
description: model.description,
|
|
49
|
+
reasoning: !!model.thinking
|
|
50
|
+
})),
|
|
51
|
+
options.refresh
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// src/GeminiAgent.ts
|
|
4
56
|
import z, { toJSONSchema } from "zod";
|
|
5
57
|
var DEFAULT_MODEL = "gemini-3.8-flash";
|
|
58
|
+
var API_VERSION = "v1beta";
|
|
6
59
|
var DEFAULT_MAX_STEPS = 5;
|
|
7
60
|
var DEFAULT_MAX_RETRIES = 3;
|
|
8
61
|
var DEFAULT_RETRY_DELAY = 1e3;
|
|
@@ -15,6 +68,15 @@ var GeminiAgent = class extends Agent {
|
|
|
15
68
|
this.signatures = /* @__PURE__ */ new Map();
|
|
16
69
|
this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
|
|
17
70
|
}
|
|
71
|
+
/**
|
|
72
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
73
|
+
*
|
|
74
|
+
* The list is fetched once and shared by every call of the process for a
|
|
75
|
+
* day. `refresh` fetches it again.
|
|
76
|
+
*/
|
|
77
|
+
static listModels(options) {
|
|
78
|
+
return listGeminiModels("agent", options);
|
|
79
|
+
}
|
|
18
80
|
generateAnswer(stream, signal) {
|
|
19
81
|
return this.generateStep(stream, signal, 0);
|
|
20
82
|
}
|
|
@@ -61,11 +123,11 @@ var GeminiAgent = class extends Agent {
|
|
|
61
123
|
}
|
|
62
124
|
/** Streams one request, writing the text of the answer as it comes */
|
|
63
125
|
async streamStep(stream, signal, progress) {
|
|
64
|
-
|
|
65
|
-
const extractOptions = this.getExtractOptions();
|
|
126
|
+
const write = this.answerWriter(stream);
|
|
66
127
|
const response = await this.genai.interactions.create(
|
|
67
128
|
{
|
|
68
129
|
model: this.options.model || DEFAULT_MODEL,
|
|
130
|
+
api_version: API_VERSION,
|
|
69
131
|
store: false,
|
|
70
132
|
stream: true,
|
|
71
133
|
system_instruction: this.buildSystemInstruction(),
|
|
@@ -84,6 +146,12 @@ var GeminiAgent = class extends Agent {
|
|
|
84
146
|
let skipAnswer = false;
|
|
85
147
|
let calledTool = false;
|
|
86
148
|
let signature = "";
|
|
149
|
+
const writeText = (text) => {
|
|
150
|
+
if (!text) return;
|
|
151
|
+
this.log(`Answer chunk: "${text}"`);
|
|
152
|
+
progress.answer += text;
|
|
153
|
+
write(text);
|
|
154
|
+
};
|
|
87
155
|
for await (const event of response) {
|
|
88
156
|
if (signal.aborted) break;
|
|
89
157
|
switch (event.event_type) {
|
|
@@ -96,6 +164,11 @@ var GeminiAgent = class extends Agent {
|
|
|
96
164
|
arguments: "",
|
|
97
165
|
signature: ""
|
|
98
166
|
};
|
|
167
|
+
if (event.step.type === "model_output") {
|
|
168
|
+
for (const content of event.step.content ?? []) {
|
|
169
|
+
if (content?.type === "text") writeText(content.text);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
99
172
|
break;
|
|
100
173
|
case "step.delta": {
|
|
101
174
|
const step = steps[event.index];
|
|
@@ -106,18 +179,7 @@ var GeminiAgent = class extends Agent {
|
|
|
106
179
|
} else if (delta.type === "arguments_delta") {
|
|
107
180
|
step.arguments += delta.arguments ?? "";
|
|
108
181
|
} else if (delta.type === "text" && step.type === "model_output") {
|
|
109
|
-
|
|
110
|
-
progress.answer += delta.text;
|
|
111
|
-
if (extractOptions) {
|
|
112
|
-
if (extracting) break;
|
|
113
|
-
const startTagIndex = delta.text.indexOf(extractOptions.startTag);
|
|
114
|
-
if (startTagIndex !== -1) {
|
|
115
|
-
extracting = true;
|
|
116
|
-
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
stream.write(delta.text);
|
|
182
|
+
writeText(delta.text);
|
|
121
183
|
}
|
|
122
184
|
break;
|
|
123
185
|
}
|
|
@@ -259,7 +321,7 @@ function score(instructions, levels) {
|
|
|
259
321
|
}
|
|
260
322
|
var DEFAULT_MODEL2 = "gemini-decision-1";
|
|
261
323
|
var DEFAULT_TIMEOUT = 3e3;
|
|
262
|
-
var
|
|
324
|
+
var API_VERSION2 = "v1beta";
|
|
263
325
|
var GeminiClassifier = class extends Classifier {
|
|
264
326
|
constructor(options) {
|
|
265
327
|
super();
|
|
@@ -275,7 +337,7 @@ var GeminiClassifier = class extends Classifier {
|
|
|
275
337
|
input: this.buildInput(input),
|
|
276
338
|
questions: asked,
|
|
277
339
|
// Only on v1beta, while the client defaults to v1
|
|
278
|
-
api_version:
|
|
340
|
+
api_version: API_VERSION2
|
|
279
341
|
},
|
|
280
342
|
{
|
|
281
343
|
// The SDK drops its timeout when a signal is given
|
|
@@ -306,7 +368,6 @@ var GeminiClassifier = class extends Classifier {
|
|
|
306
368
|
|
|
307
369
|
// src/GeminiLive.ts
|
|
308
370
|
import {
|
|
309
|
-
Pcm16Resampler,
|
|
310
371
|
Realtime
|
|
311
372
|
} from "@micdrop/server";
|
|
312
373
|
import { toJSONSchema as toJSONSchema2 } from "zod";
|
|
@@ -476,10 +537,6 @@ var GeminiLive = class extends Realtime {
|
|
|
476
537
|
super(options);
|
|
477
538
|
this.everConnected = false;
|
|
478
539
|
this.goingAway = false;
|
|
479
|
-
this.outputResampler = new Pcm16Resampler(
|
|
480
|
-
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
481
|
-
SAMPLE_RATE2
|
|
482
|
-
);
|
|
483
540
|
this.hasTranscribedTurn = false;
|
|
484
541
|
// Transcript of a dropped turn, which the conversation should not show
|
|
485
542
|
this.ignoreUserTranscript = false;
|
|
@@ -553,6 +610,15 @@ var GeminiLive = class extends Realtime {
|
|
|
553
610
|
this.socket.on("Failed", () => this.emit("Failed"));
|
|
554
611
|
this.socket.connect();
|
|
555
612
|
}
|
|
613
|
+
/**
|
|
614
|
+
* Lists the Live models, which hear and speak.
|
|
615
|
+
*
|
|
616
|
+
* The list is fetched once and shared by every call of the process for a
|
|
617
|
+
* day. `refresh` fetches it again.
|
|
618
|
+
*/
|
|
619
|
+
static listModels(options) {
|
|
620
|
+
return listGeminiModels("live", options);
|
|
621
|
+
}
|
|
556
622
|
/**
|
|
557
623
|
* Stops the answer. The tools it is waiting for still send their results,
|
|
558
624
|
* silently, since a blocking call left unanswered would stall the model.
|
|
@@ -595,8 +661,10 @@ var GeminiLive = class extends Realtime {
|
|
|
595
661
|
`Say exactly the following text, word for word, and nothing else: ${text}`
|
|
596
662
|
);
|
|
597
663
|
}
|
|
664
|
+
get sampleRate() {
|
|
665
|
+
return GEMINI_OUTPUT_SAMPLE_RATE;
|
|
666
|
+
}
|
|
598
667
|
cancelAnswer() {
|
|
599
|
-
this.outputResampler.reset();
|
|
600
668
|
}
|
|
601
669
|
get isIdle() {
|
|
602
670
|
return super.isIdle && this.toolRound.running === 0;
|
|
@@ -694,7 +762,7 @@ var GeminiLive = class extends Realtime {
|
|
|
694
762
|
const data = part.inlineData?.data;
|
|
695
763
|
if (!data) continue;
|
|
696
764
|
this.onModelOutput();
|
|
697
|
-
const output =
|
|
765
|
+
const output = Buffer.from(data, "base64");
|
|
698
766
|
if (output.length > 0) this.emitAudio(output);
|
|
699
767
|
}
|
|
700
768
|
if (content.outputTranscription?.text) {
|
|
@@ -703,14 +771,12 @@ var GeminiLive = class extends Realtime {
|
|
|
703
771
|
}
|
|
704
772
|
if (content.interrupted) {
|
|
705
773
|
this.log("Interrupted");
|
|
706
|
-
this.outputResampler.reset();
|
|
707
774
|
}
|
|
708
775
|
if (content.generationComplete) {
|
|
709
776
|
this.commitTurnTranscript();
|
|
710
777
|
}
|
|
711
778
|
if (content.turnComplete) {
|
|
712
779
|
this.commitTurnTranscript();
|
|
713
|
-
this.outputResampler.reset();
|
|
714
780
|
if (this.toolRound.running > 0 || this.toolRound.followUp) {
|
|
715
781
|
this.toolRound.answerEndDeferred = true;
|
|
716
782
|
} else {
|
|
@@ -864,6 +930,15 @@ var GeminiSTT = class extends STT {
|
|
|
864
930
|
this.socket.on("Failed", this.onFailed);
|
|
865
931
|
this.socket.connect();
|
|
866
932
|
}
|
|
933
|
+
/**
|
|
934
|
+
* Lists the models that transcribe a Live session.
|
|
935
|
+
*
|
|
936
|
+
* The list is fetched once and shared by every call of the process for a
|
|
937
|
+
* day. `refresh` fetches it again.
|
|
938
|
+
*/
|
|
939
|
+
static listModels(options) {
|
|
940
|
+
return listGeminiModels("stt", options);
|
|
941
|
+
}
|
|
867
942
|
transcribe(audioStream) {
|
|
868
943
|
let utterance;
|
|
869
944
|
audioStream.on("data", (chunk) => {
|
|
@@ -981,18 +1056,77 @@ var GeminiSTT = class extends STT {
|
|
|
981
1056
|
|
|
982
1057
|
// src/GeminiTTS.ts
|
|
983
1058
|
import { GoogleGenAI as GoogleGenAI3 } from "@google/genai";
|
|
984
|
-
import {
|
|
985
|
-
var DEFAULT_MODEL5 = "gemini-
|
|
1059
|
+
import { SentenceTTS } from "@micdrop/server";
|
|
1060
|
+
var DEFAULT_MODEL5 = "gemini-3.8-flash-tts";
|
|
1061
|
+
var API_VERSION3 = "v1beta";
|
|
986
1062
|
var DEFAULT_VOICE = "Kore";
|
|
987
1063
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
988
|
-
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
989
1064
|
var DEFAULT_MAX_RETRY = 2;
|
|
1065
|
+
var VOICES = [
|
|
1066
|
+
["Zephyr", "Bright", "female"],
|
|
1067
|
+
["Puck", "Upbeat", "male"],
|
|
1068
|
+
["Charon", "Informative", "male"],
|
|
1069
|
+
["Kore", "Firm", "female"],
|
|
1070
|
+
["Fenrir", "Excitable", "male"],
|
|
1071
|
+
["Leda", "Youthful", "female"],
|
|
1072
|
+
["Orus", "Firm", "male"],
|
|
1073
|
+
["Aoede", "Breezy", "female"],
|
|
1074
|
+
["Callirrhoe", "Easy-going", "female"],
|
|
1075
|
+
["Laomedeia", "Upbeat", "female"],
|
|
1076
|
+
["Autonoe", "Bright", "female"],
|
|
1077
|
+
["Enceladus", "Breathy", "male"],
|
|
1078
|
+
["Iapetus", "Clear", "male"],
|
|
1079
|
+
["Umbriel", "Easy-going", "male"],
|
|
1080
|
+
["Algieba", "Smooth", "male"],
|
|
1081
|
+
["Despina", "Smooth", "female"],
|
|
1082
|
+
["Erinome", "Clear", "female"],
|
|
1083
|
+
["Algenib", "Gravelly", "male"],
|
|
1084
|
+
["Rasalgethi", "Informative", "male"],
|
|
1085
|
+
["Achernar", "Soft", "female"],
|
|
1086
|
+
["Alnilam", "Firm", "male"],
|
|
1087
|
+
["Schedar", "Even", "male"],
|
|
1088
|
+
["Gacrux", "Mature", "female"],
|
|
1089
|
+
["Pulcherrima", "Forward", "female"],
|
|
1090
|
+
["Achird", "Friendly", "male"],
|
|
1091
|
+
["Zubenelgenubi", "Casual", "male"],
|
|
1092
|
+
["Vindemiatrix", "Gentle", "female"],
|
|
1093
|
+
["Sadachbia", "Lively", "male"],
|
|
1094
|
+
["Sadaltager", "Knowledgeable", "male"],
|
|
1095
|
+
["Sulafat", "Warm", "female"]
|
|
1096
|
+
];
|
|
990
1097
|
var GeminiTTS = class extends SentenceTTS {
|
|
991
1098
|
constructor(options) {
|
|
992
1099
|
super();
|
|
993
1100
|
this.options = options;
|
|
1101
|
+
// Read from the audio, which states its rate
|
|
1102
|
+
this.rate = GEMINI_SAMPLE_RATE;
|
|
994
1103
|
this.genai = "genai" in options ? options.genai : new GoogleGenAI3({ apiKey: options.apiKey });
|
|
995
1104
|
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Lists the models that speak.
|
|
1107
|
+
*
|
|
1108
|
+
* The list is fetched once and shared by every call of the process for a
|
|
1109
|
+
* day. `refresh` fetches it again.
|
|
1110
|
+
*/
|
|
1111
|
+
static listModels(options) {
|
|
1112
|
+
return listGeminiModels("tts", options);
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
1116
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
1117
|
+
* here.
|
|
1118
|
+
*/
|
|
1119
|
+
static async listVoices(options = {}) {
|
|
1120
|
+
return VOICES.map(([name, tone, gender]) => ({
|
|
1121
|
+
id: name,
|
|
1122
|
+
name,
|
|
1123
|
+
gender,
|
|
1124
|
+
description: tone
|
|
1125
|
+
}));
|
|
1126
|
+
}
|
|
1127
|
+
get sampleRate() {
|
|
1128
|
+
return this.rate;
|
|
1129
|
+
}
|
|
996
1130
|
async synthesize(text, signal) {
|
|
997
1131
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
998
1132
|
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
|
|
@@ -1020,11 +1154,11 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
1020
1154
|
response_format: { type: "audio" },
|
|
1021
1155
|
generation_config: {
|
|
1022
1156
|
speech_config: [{ voice: this.options.voice || DEFAULT_VOICE }]
|
|
1023
|
-
}
|
|
1157
|
+
},
|
|
1158
|
+
api_version: API_VERSION3
|
|
1024
1159
|
},
|
|
1025
1160
|
{ fetchOptions: { signal } }
|
|
1026
1161
|
);
|
|
1027
|
-
let resampler;
|
|
1028
1162
|
for await (const event of stream) {
|
|
1029
1163
|
if (signal.aborted) break;
|
|
1030
1164
|
if (event.event_type === "error") {
|
|
@@ -1038,11 +1172,8 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
1038
1172
|
}
|
|
1039
1173
|
const { data, sample_rate } = event.delta;
|
|
1040
1174
|
if (!data) continue;
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
OUTPUT_SAMPLE_RATE
|
|
1044
|
-
));
|
|
1045
|
-
const audio = resampler.process(Buffer.from(data, "base64"));
|
|
1175
|
+
this.rate = sample_rate || GEMINI_SAMPLE_RATE;
|
|
1176
|
+
const audio = Buffer.from(data, "base64");
|
|
1046
1177
|
if (audio.length > 0) progress.emitted = true;
|
|
1047
1178
|
if (!this.emitAudio(audio)) break;
|
|
1048
1179
|
}
|