@micdrop/gemini 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/index.d.mts +52 -2
- package/dist/index.d.ts +52 -2
- package/dist/index.js +176 -44
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +166 -35
- package/dist/index.mjs.map +1 -1
- package/package.json +4 -3
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# @micdrop/gemini
|
|
2
2
|
|
|
3
|
-
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/
|
|
3
|
+
[Micdrop website](https://micdrop.dev) | [Documentation](https://micdrop.dev/docs/providers)
|
|
4
4
|
|
|
5
5
|
Google Gemini implementation for [@micdrop/server](https://micdrop.dev/docs/server).
|
|
6
6
|
|
|
7
|
-
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/
|
|
7
|
+
This package provides an agent (`GeminiAgent`), a speech-to-text (`GeminiSTT`) and a text-to-speech (`GeminiTTS`) running on the Gemini API. `GeminiLive` is a [realtime model](https://micdrop.dev/docs/realtime) running on the Gemini Live API: it hears the user and answers with its own voice, in place of the three. `GeminiClassifier` answers typed questions about each turn of the user with the [Decisions API](https://aistudio.google.com/docs/decisions-api).
|
|
8
8
|
|
|
9
9
|
## Installation
|
|
10
10
|
|
|
@@ -34,7 +34,7 @@ new MicdropServer(socket, {
|
|
|
34
34
|
}),
|
|
35
35
|
tts: new GeminiTTS({
|
|
36
36
|
apiKey,
|
|
37
|
-
model: 'gemini-
|
|
37
|
+
model: 'gemini-3.8-flash-tts', // Default model
|
|
38
38
|
voice: 'Kore', // Default voice
|
|
39
39
|
}),
|
|
40
40
|
})
|
|
@@ -72,7 +72,7 @@ The Micdrop client detects when the user speaks, so the automatic activity detec
|
|
|
72
72
|
|
|
73
73
|
| Event | Payload | Description |
|
|
74
74
|
| ----------------------- | ------------------------- | ---------------------------------------------------------------------- |
|
|
75
|
-
| `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits,
|
|
75
|
+
| `Audio` | `Buffer` | A chunk of the voice of the model, PCM 16 bits, 24 kHz, mono. |
|
|
76
76
|
| `PartialMessage` | `string` | The transcript of the answer so far. |
|
|
77
77
|
| `Message` | `MicdropConversationItem` | A message, a tool call or a tool result was added to the conversation. |
|
|
78
78
|
| `ToolCall` | `MicdropToolCall` | A tool declared with `emitOutput` ran, with its parameters and output. |
|
|
@@ -114,7 +114,7 @@ new MicdropServer(socket, {
|
|
|
114
114
|
|
|
115
115
|
## Documentation
|
|
116
116
|
|
|
117
|
-
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/
|
|
117
|
+
Read full [documentation of the Gemini integration for Micdrop](https://micdrop.dev/docs/providers) on the [website](https://micdrop.dev).
|
|
118
118
|
|
|
119
119
|
## License
|
|
120
120
|
|
package/dist/index.d.mts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
2
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS, MicdropVoice } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -24,6 +24,16 @@ type GeminiOptions = {
|
|
|
24
24
|
genai: GoogleGenAI;
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
/**
|
|
28
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
29
|
+
*
|
|
30
|
+
* The list is fetched once and shared by every call of the process for a
|
|
31
|
+
* day. `refresh` fetches it again.
|
|
32
|
+
*/
|
|
33
|
+
static listModels(options: {
|
|
34
|
+
apiKey: string;
|
|
35
|
+
refresh?: boolean;
|
|
36
|
+
}): Promise<MicdropModel[]>;
|
|
27
37
|
private genai;
|
|
28
38
|
private signatures;
|
|
29
39
|
constructor(options: GeminiAgentOptions);
|
|
@@ -173,11 +183,20 @@ interface GeminiLiveOptions extends RealtimeOptions {
|
|
|
173
183
|
maxRetry?: number;
|
|
174
184
|
}
|
|
175
185
|
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
186
|
+
/**
|
|
187
|
+
* Lists the Live models, which hear and speak.
|
|
188
|
+
*
|
|
189
|
+
* The list is fetched once and shared by every call of the process for a
|
|
190
|
+
* day. `refresh` fetches it again.
|
|
191
|
+
*/
|
|
192
|
+
static listModels(options: {
|
|
193
|
+
apiKey: string;
|
|
194
|
+
refresh?: boolean;
|
|
195
|
+
}): Promise<MicdropModel[]>;
|
|
176
196
|
private socket;
|
|
177
197
|
private everConnected;
|
|
178
198
|
private resumptionHandle?;
|
|
179
199
|
private goingAway;
|
|
180
|
-
private outputResampler;
|
|
181
200
|
private hasTranscribedTurn;
|
|
182
201
|
private ignoreUserTranscript;
|
|
183
202
|
private cancelledToolCalls;
|
|
@@ -196,6 +215,7 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
196
215
|
protected clearTurn(): void;
|
|
197
216
|
protected generate(): void;
|
|
198
217
|
protected speakText(text: string): void;
|
|
218
|
+
get sampleRate(): number;
|
|
199
219
|
protected cancelAnswer(): void;
|
|
200
220
|
protected get isIdle(): boolean;
|
|
201
221
|
protected updateTools(): void;
|
|
@@ -251,6 +271,16 @@ interface GeminiSTTOptions {
|
|
|
251
271
|
}
|
|
252
272
|
declare class GeminiSTT extends STT {
|
|
253
273
|
private readonly options;
|
|
274
|
+
/**
|
|
275
|
+
* Lists the models that transcribe a Live session.
|
|
276
|
+
*
|
|
277
|
+
* The list is fetched once and shared by every call of the process for a
|
|
278
|
+
* day. `refresh` fetches it again.
|
|
279
|
+
*/
|
|
280
|
+
static listModels(options: {
|
|
281
|
+
apiKey: string;
|
|
282
|
+
refresh?: boolean;
|
|
283
|
+
}): Promise<MicdropModel[]>;
|
|
254
284
|
private socket;
|
|
255
285
|
private goingAway;
|
|
256
286
|
private utterances;
|
|
@@ -295,7 +325,27 @@ type GeminiTTSOptions = GeminiOptions & {
|
|
|
295
325
|
};
|
|
296
326
|
declare class GeminiTTS extends SentenceTTS {
|
|
297
327
|
private readonly options;
|
|
328
|
+
/**
|
|
329
|
+
* Lists the models that speak.
|
|
330
|
+
*
|
|
331
|
+
* The list is fetched once and shared by every call of the process for a
|
|
332
|
+
* day. `refresh` fetches it again.
|
|
333
|
+
*/
|
|
334
|
+
static listModels(options: {
|
|
335
|
+
apiKey: string;
|
|
336
|
+
refresh?: boolean;
|
|
337
|
+
}): Promise<MicdropModel[]>;
|
|
338
|
+
/**
|
|
339
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
340
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
341
|
+
* here.
|
|
342
|
+
*/
|
|
343
|
+
static listVoices(options?: {
|
|
344
|
+
language?: string;
|
|
345
|
+
}): Promise<MicdropVoice[]>;
|
|
298
346
|
private genai;
|
|
347
|
+
private rate;
|
|
348
|
+
get sampleRate(): number;
|
|
299
349
|
constructor(options: GeminiTTSOptions);
|
|
300
350
|
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
301
351
|
private synthesizeOnce;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { GoogleGenAI, Decisions } from '@google/genai';
|
|
2
|
-
import { Agent, AgentOptions, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS } from '@micdrop/server';
|
|
2
|
+
import { Agent, AgentOptions, MicdropModel, Classifier, ClassifierInput, Realtime, RealtimeOptions, STT, SentenceTTS, MicdropVoice } from '@micdrop/server';
|
|
3
3
|
import { PassThrough, Readable } from 'stream';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -24,6 +24,16 @@ type GeminiOptions = {
|
|
|
24
24
|
genai: GoogleGenAI;
|
|
25
25
|
};
|
|
26
26
|
declare class GeminiAgent extends Agent<GeminiAgentOptions> {
|
|
27
|
+
/**
|
|
28
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
29
|
+
*
|
|
30
|
+
* The list is fetched once and shared by every call of the process for a
|
|
31
|
+
* day. `refresh` fetches it again.
|
|
32
|
+
*/
|
|
33
|
+
static listModels(options: {
|
|
34
|
+
apiKey: string;
|
|
35
|
+
refresh?: boolean;
|
|
36
|
+
}): Promise<MicdropModel[]>;
|
|
27
37
|
private genai;
|
|
28
38
|
private signatures;
|
|
29
39
|
constructor(options: GeminiAgentOptions);
|
|
@@ -173,11 +183,20 @@ interface GeminiLiveOptions extends RealtimeOptions {
|
|
|
173
183
|
maxRetry?: number;
|
|
174
184
|
}
|
|
175
185
|
declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
186
|
+
/**
|
|
187
|
+
* Lists the Live models, which hear and speak.
|
|
188
|
+
*
|
|
189
|
+
* The list is fetched once and shared by every call of the process for a
|
|
190
|
+
* day. `refresh` fetches it again.
|
|
191
|
+
*/
|
|
192
|
+
static listModels(options: {
|
|
193
|
+
apiKey: string;
|
|
194
|
+
refresh?: boolean;
|
|
195
|
+
}): Promise<MicdropModel[]>;
|
|
176
196
|
private socket;
|
|
177
197
|
private everConnected;
|
|
178
198
|
private resumptionHandle?;
|
|
179
199
|
private goingAway;
|
|
180
|
-
private outputResampler;
|
|
181
200
|
private hasTranscribedTurn;
|
|
182
201
|
private ignoreUserTranscript;
|
|
183
202
|
private cancelledToolCalls;
|
|
@@ -196,6 +215,7 @@ declare class GeminiLive extends Realtime<GeminiLiveOptions> {
|
|
|
196
215
|
protected clearTurn(): void;
|
|
197
216
|
protected generate(): void;
|
|
198
217
|
protected speakText(text: string): void;
|
|
218
|
+
get sampleRate(): number;
|
|
199
219
|
protected cancelAnswer(): void;
|
|
200
220
|
protected get isIdle(): boolean;
|
|
201
221
|
protected updateTools(): void;
|
|
@@ -251,6 +271,16 @@ interface GeminiSTTOptions {
|
|
|
251
271
|
}
|
|
252
272
|
declare class GeminiSTT extends STT {
|
|
253
273
|
private readonly options;
|
|
274
|
+
/**
|
|
275
|
+
* Lists the models that transcribe a Live session.
|
|
276
|
+
*
|
|
277
|
+
* The list is fetched once and shared by every call of the process for a
|
|
278
|
+
* day. `refresh` fetches it again.
|
|
279
|
+
*/
|
|
280
|
+
static listModels(options: {
|
|
281
|
+
apiKey: string;
|
|
282
|
+
refresh?: boolean;
|
|
283
|
+
}): Promise<MicdropModel[]>;
|
|
254
284
|
private socket;
|
|
255
285
|
private goingAway;
|
|
256
286
|
private utterances;
|
|
@@ -295,7 +325,27 @@ type GeminiTTSOptions = GeminiOptions & {
|
|
|
295
325
|
};
|
|
296
326
|
declare class GeminiTTS extends SentenceTTS {
|
|
297
327
|
private readonly options;
|
|
328
|
+
/**
|
|
329
|
+
* Lists the models that speak.
|
|
330
|
+
*
|
|
331
|
+
* The list is fetched once and shared by every call of the process for a
|
|
332
|
+
* day. `refresh` fetches it again.
|
|
333
|
+
*/
|
|
334
|
+
static listModels(options: {
|
|
335
|
+
apiKey: string;
|
|
336
|
+
refresh?: boolean;
|
|
337
|
+
}): Promise<MicdropModel[]>;
|
|
338
|
+
/**
|
|
339
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
340
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
341
|
+
* here.
|
|
342
|
+
*/
|
|
343
|
+
static listVoices(options?: {
|
|
344
|
+
language?: string;
|
|
345
|
+
}): Promise<MicdropVoice[]>;
|
|
298
346
|
private genai;
|
|
347
|
+
private rate;
|
|
348
|
+
get sampleRate(): number;
|
|
299
349
|
constructor(options: GeminiTTSOptions);
|
|
300
350
|
protected synthesize(text: string, signal: AbortSignal): Promise<undefined>;
|
|
301
351
|
private synthesizeOnce;
|
package/dist/index.js
CHANGED
|
@@ -43,14 +43,67 @@ module.exports = __toCommonJS(index_exports);
|
|
|
43
43
|
|
|
44
44
|
// src/GeminiAgent.ts
|
|
45
45
|
var import_genai = require("@google/genai");
|
|
46
|
+
var import_server2 = require("@micdrop/server");
|
|
47
|
+
|
|
48
|
+
// src/models.ts
|
|
46
49
|
var import_server = require("@micdrop/server");
|
|
50
|
+
function fetchAll(apiKey) {
|
|
51
|
+
return (0, import_server.cachedList)(["gemini-raw-models"], apiKey, async () => {
|
|
52
|
+
const models = [];
|
|
53
|
+
let pageToken;
|
|
54
|
+
do {
|
|
55
|
+
const params = new URLSearchParams({ pageSize: "1000", key: apiKey });
|
|
56
|
+
if (pageToken) params.set("pageToken", pageToken);
|
|
57
|
+
const response = await fetch(
|
|
58
|
+
`https://generativelanguage.googleapis.com/v1beta/models?${params}`
|
|
59
|
+
);
|
|
60
|
+
if (!response.ok) throw new Error(`Gemini answered ${response.status}`);
|
|
61
|
+
const page = await response.json();
|
|
62
|
+
models.push(...page.models);
|
|
63
|
+
pageToken = page.nextPageToken;
|
|
64
|
+
} while (pageToken);
|
|
65
|
+
return models;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
function kindOf(model) {
|
|
69
|
+
const id = model.name.replace(/^models\//, "");
|
|
70
|
+
const methods = model.supportedGenerationMethods ?? [];
|
|
71
|
+
if (!id.startsWith("gemini-")) return void 0;
|
|
72
|
+
if (/-tts\b/.test(id)) return "tts";
|
|
73
|
+
if (methods.includes("bidiGenerateContent")) {
|
|
74
|
+
if (id.includes("transcribe")) return "stt";
|
|
75
|
+
if (/translate|robotics/.test(id)) return void 0;
|
|
76
|
+
return "live";
|
|
77
|
+
}
|
|
78
|
+
if (!methods.includes("generateContent")) return void 0;
|
|
79
|
+
if (/image|embedding|transcribe|robotics|computer-use|omni|nano-banana/.test(id)) {
|
|
80
|
+
return void 0;
|
|
81
|
+
}
|
|
82
|
+
return "agent";
|
|
83
|
+
}
|
|
84
|
+
function listGeminiModels(kind, options) {
|
|
85
|
+
return (0, import_server.cachedModels)(
|
|
86
|
+
["gemini", kind],
|
|
87
|
+
options.apiKey,
|
|
88
|
+
async () => (await fetchAll(options.apiKey)).filter((model) => kindOf(model) === kind).map((model) => ({
|
|
89
|
+
id: model.name.replace(/^models\//, ""),
|
|
90
|
+
name: model.displayName,
|
|
91
|
+
description: model.description,
|
|
92
|
+
reasoning: !!model.thinking
|
|
93
|
+
})),
|
|
94
|
+
options.refresh
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// src/GeminiAgent.ts
|
|
47
99
|
var import_zod = __toESM(require("zod"));
|
|
48
100
|
var DEFAULT_MODEL = "gemini-3.8-flash";
|
|
101
|
+
var API_VERSION = "v1beta";
|
|
49
102
|
var DEFAULT_MAX_STEPS = 5;
|
|
50
103
|
var DEFAULT_MAX_RETRIES = 3;
|
|
51
104
|
var DEFAULT_RETRY_DELAY = 1e3;
|
|
52
105
|
var FIRST_MESSAGE_PROMPT = "The call starts now.";
|
|
53
|
-
var GeminiAgent = class extends
|
|
106
|
+
var GeminiAgent = class extends import_server2.Agent {
|
|
54
107
|
constructor(options) {
|
|
55
108
|
super(options);
|
|
56
109
|
// Gemini asks for the signature of the thought behind a tool call when the
|
|
@@ -58,6 +111,15 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
58
111
|
this.signatures = /* @__PURE__ */ new Map();
|
|
59
112
|
this.genai = "genai" in options ? options.genai : new import_genai.GoogleGenAI({ apiKey: options.apiKey });
|
|
60
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* Lists the models that can answer a call. `reasoning` marks the ones that think, see `thinkingLevel`.
|
|
116
|
+
*
|
|
117
|
+
* The list is fetched once and shared by every call of the process for a
|
|
118
|
+
* day. `refresh` fetches it again.
|
|
119
|
+
*/
|
|
120
|
+
static listModels(options) {
|
|
121
|
+
return listGeminiModels("agent", options);
|
|
122
|
+
}
|
|
61
123
|
generateAnswer(stream, signal) {
|
|
62
124
|
return this.generateStep(stream, signal, 0);
|
|
63
125
|
}
|
|
@@ -104,11 +166,11 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
104
166
|
}
|
|
105
167
|
/** Streams one request, writing the text of the answer as it comes */
|
|
106
168
|
async streamStep(stream, signal, progress) {
|
|
107
|
-
|
|
108
|
-
const extractOptions = this.getExtractOptions();
|
|
169
|
+
const write = this.answerWriter(stream);
|
|
109
170
|
const response = await this.genai.interactions.create(
|
|
110
171
|
{
|
|
111
172
|
model: this.options.model || DEFAULT_MODEL,
|
|
173
|
+
api_version: API_VERSION,
|
|
112
174
|
store: false,
|
|
113
175
|
stream: true,
|
|
114
176
|
system_instruction: this.buildSystemInstruction(),
|
|
@@ -127,6 +189,12 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
127
189
|
let skipAnswer = false;
|
|
128
190
|
let calledTool = false;
|
|
129
191
|
let signature = "";
|
|
192
|
+
const writeText = (text) => {
|
|
193
|
+
if (!text) return;
|
|
194
|
+
this.log(`Answer chunk: "${text}"`);
|
|
195
|
+
progress.answer += text;
|
|
196
|
+
write(text);
|
|
197
|
+
};
|
|
130
198
|
for await (const event of response) {
|
|
131
199
|
if (signal.aborted) break;
|
|
132
200
|
switch (event.event_type) {
|
|
@@ -139,6 +207,11 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
139
207
|
arguments: "",
|
|
140
208
|
signature: ""
|
|
141
209
|
};
|
|
210
|
+
if (event.step.type === "model_output") {
|
|
211
|
+
for (const content of event.step.content ?? []) {
|
|
212
|
+
if (content?.type === "text") writeText(content.text);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
142
215
|
break;
|
|
143
216
|
case "step.delta": {
|
|
144
217
|
const step = steps[event.index];
|
|
@@ -149,18 +222,7 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
149
222
|
} else if (delta.type === "arguments_delta") {
|
|
150
223
|
step.arguments += delta.arguments ?? "";
|
|
151
224
|
} else if (delta.type === "text" && step.type === "model_output") {
|
|
152
|
-
|
|
153
|
-
progress.answer += delta.text;
|
|
154
|
-
if (extractOptions) {
|
|
155
|
-
if (extracting) break;
|
|
156
|
-
const startTagIndex = delta.text.indexOf(extractOptions.startTag);
|
|
157
|
-
if (startTagIndex !== -1) {
|
|
158
|
-
extracting = true;
|
|
159
|
-
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
160
|
-
break;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
stream.write(delta.text);
|
|
225
|
+
writeText(delta.text);
|
|
164
226
|
}
|
|
165
227
|
break;
|
|
166
228
|
}
|
|
@@ -283,7 +345,7 @@ function parseArguments(parameters) {
|
|
|
283
345
|
|
|
284
346
|
// src/GeminiClassifier.ts
|
|
285
347
|
var import_genai2 = require("@google/genai");
|
|
286
|
-
var
|
|
348
|
+
var import_server3 = require("@micdrop/server");
|
|
287
349
|
function noul(instructions, criteria) {
|
|
288
350
|
return {
|
|
289
351
|
type: "noul",
|
|
@@ -302,8 +364,8 @@ function score(instructions, levels) {
|
|
|
302
364
|
}
|
|
303
365
|
var DEFAULT_MODEL2 = "gemini-decision-1";
|
|
304
366
|
var DEFAULT_TIMEOUT = 3e3;
|
|
305
|
-
var
|
|
306
|
-
var GeminiClassifier = class extends
|
|
367
|
+
var API_VERSION2 = "v1beta";
|
|
368
|
+
var GeminiClassifier = class extends import_server3.Classifier {
|
|
307
369
|
constructor(options) {
|
|
308
370
|
super();
|
|
309
371
|
this.options = options;
|
|
@@ -318,7 +380,7 @@ var GeminiClassifier = class extends import_server2.Classifier {
|
|
|
318
380
|
input: this.buildInput(input),
|
|
319
381
|
questions: asked,
|
|
320
382
|
// Only on v1beta, while the client defaults to v1
|
|
321
|
-
api_version:
|
|
383
|
+
api_version: API_VERSION2
|
|
322
384
|
},
|
|
323
385
|
{
|
|
324
386
|
// The SDK drops its timeout when a signal is given
|
|
@@ -348,11 +410,11 @@ var GeminiClassifier = class extends import_server2.Classifier {
|
|
|
348
410
|
};
|
|
349
411
|
|
|
350
412
|
// src/GeminiLive.ts
|
|
351
|
-
var
|
|
413
|
+
var import_server5 = require("@micdrop/server");
|
|
352
414
|
var import_zod2 = require("zod");
|
|
353
415
|
|
|
354
416
|
// src/LiveSocket.ts
|
|
355
|
-
var
|
|
417
|
+
var import_server4 = require("@micdrop/server");
|
|
356
418
|
var import_eventemitter3 = require("eventemitter3");
|
|
357
419
|
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
358
420
|
var SAMPLE_RATE = 16e3;
|
|
@@ -399,7 +461,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
|
399
461
|
this.clearPending();
|
|
400
462
|
this.emit("Failed");
|
|
401
463
|
};
|
|
402
|
-
this.socket = new
|
|
464
|
+
this.socket = new import_server4.ReconnectingSocket({
|
|
403
465
|
url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
|
|
404
466
|
connectionTimeout: options.connectionTimeout,
|
|
405
467
|
retryDelay: options.retryDelay,
|
|
@@ -509,15 +571,11 @@ function newToolRound() {
|
|
|
509
571
|
return { running: 0, answerEndDeferred: false };
|
|
510
572
|
}
|
|
511
573
|
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
512
|
-
var GeminiLive = class extends
|
|
574
|
+
var GeminiLive = class extends import_server5.Realtime {
|
|
513
575
|
constructor(options) {
|
|
514
576
|
super(options);
|
|
515
577
|
this.everConnected = false;
|
|
516
578
|
this.goingAway = false;
|
|
517
|
-
this.outputResampler = new import_server4.Pcm16Resampler(
|
|
518
|
-
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
519
|
-
SAMPLE_RATE2
|
|
520
|
-
);
|
|
521
579
|
this.hasTranscribedTurn = false;
|
|
522
580
|
// Transcript of a dropped turn, which the conversation should not show
|
|
523
581
|
this.ignoreUserTranscript = false;
|
|
@@ -591,6 +649,15 @@ var GeminiLive = class extends import_server4.Realtime {
|
|
|
591
649
|
this.socket.on("Failed", () => this.emit("Failed"));
|
|
592
650
|
this.socket.connect();
|
|
593
651
|
}
|
|
652
|
+
/**
|
|
653
|
+
* Lists the Live models, which hear and speak.
|
|
654
|
+
*
|
|
655
|
+
* The list is fetched once and shared by every call of the process for a
|
|
656
|
+
* day. `refresh` fetches it again.
|
|
657
|
+
*/
|
|
658
|
+
static listModels(options) {
|
|
659
|
+
return listGeminiModels("live", options);
|
|
660
|
+
}
|
|
594
661
|
/**
|
|
595
662
|
* Stops the answer. The tools it is waiting for still send their results,
|
|
596
663
|
* silently, since a blocking call left unanswered would stall the model.
|
|
@@ -633,8 +700,10 @@ var GeminiLive = class extends import_server4.Realtime {
|
|
|
633
700
|
`Say exactly the following text, word for word, and nothing else: ${text}`
|
|
634
701
|
);
|
|
635
702
|
}
|
|
703
|
+
get sampleRate() {
|
|
704
|
+
return GEMINI_OUTPUT_SAMPLE_RATE;
|
|
705
|
+
}
|
|
636
706
|
cancelAnswer() {
|
|
637
|
-
this.outputResampler.reset();
|
|
638
707
|
}
|
|
639
708
|
get isIdle() {
|
|
640
709
|
return super.isIdle && this.toolRound.running === 0;
|
|
@@ -732,7 +801,7 @@ var GeminiLive = class extends import_server4.Realtime {
|
|
|
732
801
|
const data = part.inlineData?.data;
|
|
733
802
|
if (!data) continue;
|
|
734
803
|
this.onModelOutput();
|
|
735
|
-
const output =
|
|
804
|
+
const output = Buffer.from(data, "base64");
|
|
736
805
|
if (output.length > 0) this.emitAudio(output);
|
|
737
806
|
}
|
|
738
807
|
if (content.outputTranscription?.text) {
|
|
@@ -741,14 +810,12 @@ var GeminiLive = class extends import_server4.Realtime {
|
|
|
741
810
|
}
|
|
742
811
|
if (content.interrupted) {
|
|
743
812
|
this.log("Interrupted");
|
|
744
|
-
this.outputResampler.reset();
|
|
745
813
|
}
|
|
746
814
|
if (content.generationComplete) {
|
|
747
815
|
this.commitTurnTranscript();
|
|
748
816
|
}
|
|
749
817
|
if (content.turnComplete) {
|
|
750
818
|
this.commitTurnTranscript();
|
|
751
|
-
this.outputResampler.reset();
|
|
752
819
|
if (this.toolRound.running > 0 || this.toolRound.followUp) {
|
|
753
820
|
this.toolRound.answerEndDeferred = true;
|
|
754
821
|
} else {
|
|
@@ -838,11 +905,11 @@ var GeminiLive = class extends import_server4.Realtime {
|
|
|
838
905
|
};
|
|
839
906
|
|
|
840
907
|
// src/GeminiSTT.ts
|
|
841
|
-
var
|
|
908
|
+
var import_server6 = require("@micdrop/server");
|
|
842
909
|
var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
|
|
843
910
|
var SAMPLE_RATE3 = 16e3;
|
|
844
911
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
845
|
-
var GeminiSTT = class extends
|
|
912
|
+
var GeminiSTT = class extends import_server6.STT {
|
|
846
913
|
constructor(options) {
|
|
847
914
|
super();
|
|
848
915
|
this.options = options;
|
|
@@ -902,6 +969,15 @@ var GeminiSTT = class extends import_server5.STT {
|
|
|
902
969
|
this.socket.on("Failed", this.onFailed);
|
|
903
970
|
this.socket.connect();
|
|
904
971
|
}
|
|
972
|
+
/**
|
|
973
|
+
* Lists the models that transcribe a Live session.
|
|
974
|
+
*
|
|
975
|
+
* The list is fetched once and shared by every call of the process for a
|
|
976
|
+
* day. `refresh` fetches it again.
|
|
977
|
+
*/
|
|
978
|
+
static listModels(options) {
|
|
979
|
+
return listGeminiModels("stt", options);
|
|
980
|
+
}
|
|
905
981
|
transcribe(audioStream) {
|
|
906
982
|
let utterance;
|
|
907
983
|
audioStream.on("data", (chunk) => {
|
|
@@ -1019,18 +1095,77 @@ var GeminiSTT = class extends import_server5.STT {
|
|
|
1019
1095
|
|
|
1020
1096
|
// src/GeminiTTS.ts
|
|
1021
1097
|
var import_genai3 = require("@google/genai");
|
|
1022
|
-
var
|
|
1023
|
-
var DEFAULT_MODEL5 = "gemini-
|
|
1098
|
+
var import_server7 = require("@micdrop/server");
|
|
1099
|
+
var DEFAULT_MODEL5 = "gemini-3.8-flash-tts";
|
|
1100
|
+
var API_VERSION3 = "v1beta";
|
|
1024
1101
|
var DEFAULT_VOICE = "Kore";
|
|
1025
1102
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
1026
|
-
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
1027
1103
|
var DEFAULT_MAX_RETRY = 2;
|
|
1028
|
-
var
|
|
1104
|
+
var VOICES = [
|
|
1105
|
+
["Zephyr", "Bright", "female"],
|
|
1106
|
+
["Puck", "Upbeat", "male"],
|
|
1107
|
+
["Charon", "Informative", "male"],
|
|
1108
|
+
["Kore", "Firm", "female"],
|
|
1109
|
+
["Fenrir", "Excitable", "male"],
|
|
1110
|
+
["Leda", "Youthful", "female"],
|
|
1111
|
+
["Orus", "Firm", "male"],
|
|
1112
|
+
["Aoede", "Breezy", "female"],
|
|
1113
|
+
["Callirrhoe", "Easy-going", "female"],
|
|
1114
|
+
["Laomedeia", "Upbeat", "female"],
|
|
1115
|
+
["Autonoe", "Bright", "female"],
|
|
1116
|
+
["Enceladus", "Breathy", "male"],
|
|
1117
|
+
["Iapetus", "Clear", "male"],
|
|
1118
|
+
["Umbriel", "Easy-going", "male"],
|
|
1119
|
+
["Algieba", "Smooth", "male"],
|
|
1120
|
+
["Despina", "Smooth", "female"],
|
|
1121
|
+
["Erinome", "Clear", "female"],
|
|
1122
|
+
["Algenib", "Gravelly", "male"],
|
|
1123
|
+
["Rasalgethi", "Informative", "male"],
|
|
1124
|
+
["Achernar", "Soft", "female"],
|
|
1125
|
+
["Alnilam", "Firm", "male"],
|
|
1126
|
+
["Schedar", "Even", "male"],
|
|
1127
|
+
["Gacrux", "Mature", "female"],
|
|
1128
|
+
["Pulcherrima", "Forward", "female"],
|
|
1129
|
+
["Achird", "Friendly", "male"],
|
|
1130
|
+
["Zubenelgenubi", "Casual", "male"],
|
|
1131
|
+
["Vindemiatrix", "Gentle", "female"],
|
|
1132
|
+
["Sadachbia", "Lively", "male"],
|
|
1133
|
+
["Sadaltager", "Knowledgeable", "male"],
|
|
1134
|
+
["Sulafat", "Warm", "female"]
|
|
1135
|
+
];
|
|
1136
|
+
var GeminiTTS = class extends import_server7.SentenceTTS {
|
|
1029
1137
|
constructor(options) {
|
|
1030
1138
|
super();
|
|
1031
1139
|
this.options = options;
|
|
1140
|
+
// Read from the audio, which states its rate
|
|
1141
|
+
this.rate = GEMINI_SAMPLE_RATE;
|
|
1032
1142
|
this.genai = "genai" in options ? options.genai : new import_genai3.GoogleGenAI({ apiKey: options.apiKey });
|
|
1033
1143
|
}
|
|
1144
|
+
/**
|
|
1145
|
+
* Lists the models that speak.
|
|
1146
|
+
*
|
|
1147
|
+
* The list is fetched once and shared by every call of the process for a
|
|
1148
|
+
* day. `refresh` fetches it again.
|
|
1149
|
+
*/
|
|
1150
|
+
static listModels(options) {
|
|
1151
|
+
return listGeminiModels("tts", options);
|
|
1152
|
+
}
|
|
1153
|
+
/**
|
|
1154
|
+
* Lists the prebuilt voices, which speak every language the model does, so
|
|
1155
|
+
* `language` keeps them all. Gemini has no listing, so the list is written
|
|
1156
|
+
* here.
|
|
1157
|
+
*/
|
|
1158
|
+
static async listVoices(options = {}) {
|
|
1159
|
+
return VOICES.map(([name, tone, gender]) => ({
|
|
1160
|
+
id: name,
|
|
1161
|
+
name,
|
|
1162
|
+
gender,
|
|
1163
|
+
description: tone
|
|
1164
|
+
}));
|
|
1165
|
+
}
|
|
1166
|
+
get sampleRate() {
|
|
1167
|
+
return this.rate;
|
|
1168
|
+
}
|
|
1034
1169
|
async synthesize(text, signal) {
|
|
1035
1170
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
1036
1171
|
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
|
|
@@ -1058,11 +1193,11 @@ var GeminiTTS = class extends import_server6.SentenceTTS {
|
|
|
1058
1193
|
response_format: { type: "audio" },
|
|
1059
1194
|
generation_config: {
|
|
1060
1195
|
speech_config: [{ voice: this.options.voice || DEFAULT_VOICE }]
|
|
1061
|
-
}
|
|
1196
|
+
},
|
|
1197
|
+
api_version: API_VERSION3
|
|
1062
1198
|
},
|
|
1063
1199
|
{ fetchOptions: { signal } }
|
|
1064
1200
|
);
|
|
1065
|
-
let resampler;
|
|
1066
1201
|
for await (const event of stream) {
|
|
1067
1202
|
if (signal.aborted) break;
|
|
1068
1203
|
if (event.event_type === "error") {
|
|
@@ -1076,11 +1211,8 @@ var GeminiTTS = class extends import_server6.SentenceTTS {
|
|
|
1076
1211
|
}
|
|
1077
1212
|
const { data, sample_rate } = event.delta;
|
|
1078
1213
|
if (!data) continue;
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
OUTPUT_SAMPLE_RATE
|
|
1082
|
-
));
|
|
1083
|
-
const audio = resampler.process(Buffer.from(data, "base64"));
|
|
1214
|
+
this.rate = sample_rate || GEMINI_SAMPLE_RATE;
|
|
1215
|
+
const audio = Buffer.from(data, "base64");
|
|
1084
1216
|
if (audio.length > 0) progress.emitted = true;
|
|
1085
1217
|
if (!this.emitAudio(audio)) break;
|
|
1086
1218
|
}
|