@micdrop/gemini 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs ADDED
@@ -0,0 +1,922 @@
1
+ // src/GeminiAgent.ts
2
+ import { GoogleGenAI } from "@google/genai";
3
+ import { Agent } from "@micdrop/server";
4
+ import z, { toJSONSchema } from "zod";
5
+ var DEFAULT_MODEL = "gemini-3.8-flash";
6
+ var DEFAULT_MAX_STEPS = 5;
7
+ var DEFAULT_MAX_RETRIES = 3;
8
+ var DEFAULT_RETRY_DELAY = 1e3;
9
+ var FIRST_MESSAGE_PROMPT = "The call starts now.";
10
+ var GeminiAgent = class extends Agent {
11
+ constructor(options) {
12
+ super(options);
13
+ // Gemini asks for the signature of the thought behind a tool call when the
14
+ // call is sent back, and the conversation has no room for it
15
+ this.signatures = /* @__PURE__ */ new Map();
16
+ this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
17
+ }
18
+ async generateAnswer(stream, stepCount = 0, tryCount = 0) {
19
+ if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
20
+ console.error("[GeminiAgent] Max steps reached");
21
+ return;
22
+ }
23
+ const abortController = new AbortController();
24
+ this.abortController = abortController;
25
+ const signal = abortController.signal;
26
+ let extracting = false;
27
+ const extractOptions = this.getExtractOptions();
28
+ try {
29
+ const response = await this.genai.interactions.create(
30
+ {
31
+ model: this.options.model || DEFAULT_MODEL,
32
+ store: false,
33
+ stream: true,
34
+ system_instruction: this.buildSystemInstruction(),
35
+ input: this.buildInput(),
36
+ tools: this.buildTools(),
37
+ ...this.options.thinkingLevel || this.options.settings ? {
38
+ generation_config: {
39
+ ...this.options.settings,
40
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
41
+ }
42
+ } : {}
43
+ },
44
+ { fetchOptions: { signal } }
45
+ );
46
+ const steps = [];
47
+ let answer = "";
48
+ let skipAnswer = false;
49
+ let calledTool = false;
50
+ let signature = "";
51
+ for await (const event of response) {
52
+ if (abortController !== this.abortController) return;
53
+ switch (event.event_type) {
54
+ case "step.start":
55
+ steps[event.index] = {
56
+ type: event.step.type,
57
+ id: event.step.id,
58
+ name: event.step.name,
59
+ text: "",
60
+ arguments: "",
61
+ signature: ""
62
+ };
63
+ break;
64
+ case "step.delta": {
65
+ const step = steps[event.index];
66
+ if (!step) break;
67
+ const delta = event.delta;
68
+ if (delta.type === "thought_signature") {
69
+ step.signature += delta.signature ?? "";
70
+ } else if (delta.type === "arguments_delta") {
71
+ step.arguments += delta.arguments ?? "";
72
+ } else if (delta.type === "text" && step.type === "model_output") {
73
+ this.log(`Answer chunk: "${delta.text}"`);
74
+ answer += delta.text;
75
+ if (extractOptions) {
76
+ if (extracting) break;
77
+ const startTagIndex = delta.text.indexOf(
78
+ extractOptions.startTag
79
+ );
80
+ if (startTagIndex !== -1) {
81
+ extracting = true;
82
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
83
+ break;
84
+ }
85
+ }
86
+ stream.write(delta.text);
87
+ }
88
+ break;
89
+ }
90
+ case "step.stop": {
91
+ const step = steps[event.index];
92
+ if (step?.type === "thought") signature = step.signature;
93
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
94
+ calledTool = true;
95
+ if (signature) this.signatures.set(step.id, signature);
96
+ const result = await this.executeTool({
97
+ role: "tool_call",
98
+ toolCallId: step.id,
99
+ toolName: step.name,
100
+ parameters: step.arguments || "{}"
101
+ });
102
+ if (result.skipAnswer) skipAnswer = true;
103
+ break;
104
+ }
105
+ case "error":
106
+ throw new Error(event.error?.message || "Generation failed");
107
+ default:
108
+ break;
109
+ }
110
+ }
111
+ if (abortController !== this.abortController) return;
112
+ this.abortController = void 0;
113
+ if (answer) {
114
+ const { message, metadata } = this.extract(answer);
115
+ this.addAssistantMessage(message, metadata);
116
+ }
117
+ if (calledTool && !skipAnswer && !answer) {
118
+ await this.generateAnswer(stream, stepCount + 1);
119
+ }
120
+ } catch (error) {
121
+ if (signal.aborted) return;
122
+ console.error("[GeminiAgent] Error answering:", error);
123
+ if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
124
+ await new Promise(
125
+ (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
126
+ );
127
+ await this.generateAnswer(stream, stepCount, tryCount + 1);
128
+ } else {
129
+ this.log("Max retries reached, giving up");
130
+ this.emit("Failed");
131
+ }
132
+ }
133
+ }
134
+ cancel() {
135
+ if (!this.abortController) return;
136
+ this.log("Cancel");
137
+ this.abortController.abort();
138
+ this.abortController = void 0;
139
+ }
140
+ buildSystemInstruction() {
141
+ return this.conversation.flatMap(
142
+ (message) => message.role === "system" ? [message.content] : []
143
+ ).join("\n\n");
144
+ }
145
+ buildInput() {
146
+ const input = this.conversation.flatMap((message) => {
147
+ switch (message.role) {
148
+ case "user":
149
+ return [
150
+ {
151
+ type: "user_input",
152
+ content: [{ type: "text", text: message.content }]
153
+ }
154
+ ];
155
+ case "assistant":
156
+ return [
157
+ {
158
+ type: "model_output",
159
+ content: [{ type: "text", text: message.content }]
160
+ }
161
+ ];
162
+ case "tool_call": {
163
+ const signature = this.signatures.get(message.toolCallId);
164
+ return [
165
+ ...signature ? [{ type: "thought", signature }] : [],
166
+ {
167
+ type: "function_call",
168
+ id: message.toolCallId,
169
+ name: message.toolName,
170
+ arguments: parseArguments(message.parameters)
171
+ }
172
+ ];
173
+ }
174
+ case "tool_result":
175
+ return [
176
+ {
177
+ type: "function_result",
178
+ call_id: message.toolCallId,
179
+ name: message.toolName,
180
+ result: [{ type: "text", text: message.output }]
181
+ }
182
+ ];
183
+ default:
184
+ return [];
185
+ }
186
+ });
187
+ if (input.length === 0) {
188
+ input.push({
189
+ type: "user_input",
190
+ content: [{ type: "text", text: FIRST_MESSAGE_PROMPT }]
191
+ });
192
+ }
193
+ return input;
194
+ }
195
+ buildTools() {
196
+ if (this.tools.length === 0) return void 0;
197
+ return this.tools.map((tool) => {
198
+ const { $schema, ...parameters } = toJSONSchema(
199
+ tool.inputSchema || z.object()
200
+ );
201
+ return {
202
+ type: "function",
203
+ name: tool.name,
204
+ description: tool.description,
205
+ parameters
206
+ };
207
+ });
208
+ }
209
+ };
210
+ function parseArguments(parameters) {
211
+ try {
212
+ return JSON.parse(parameters);
213
+ } catch {
214
+ return {};
215
+ }
216
+ }
217
+
218
+ // src/GeminiLive.ts
219
+ import {
220
+ Pcm16Resampler,
221
+ Realtime
222
+ } from "@micdrop/server";
223
+ import { toJSONSchema as toJSONSchema2 } from "zod";
224
+
225
+ // src/LiveSocket.ts
226
+ import { EventEmitter } from "eventemitter3";
227
+ import WebSocket from "ws";
228
+ var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
229
+ var SAMPLE_RATE = 16e3;
230
+ var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
231
+ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
232
+ var DEFAULT_RETRY_DELAY2 = 1e3;
233
+ var DEFAULT_MAX_RETRY = 3;
234
+ var LiveSocket = class extends EventEmitter {
235
+ constructor(options, buildSetup, log) {
236
+ super();
237
+ this.options = options;
238
+ this.buildSetup = buildSetup;
239
+ this.log = log;
240
+ this.connected = false;
241
+ this.destroyed = false;
242
+ // Messages waiting for the session, or for the end of a turn to be due
243
+ this.pending = [];
244
+ // Audio of the turn as it went out, see flush()
245
+ this.turnAudioStartedAt = 0;
246
+ this.turnAudioMs = 0;
247
+ this.retryCount = 0;
248
+ }
249
+ /** The session is set up and takes messages right away */
250
+ get isConnected() {
251
+ return this.connected;
252
+ }
253
+ connect() {
254
+ this.initWS().catch((error) => {
255
+ this.log("Connection error:", error);
256
+ this.reconnect();
257
+ });
258
+ }
259
+ /** Sends a message once the session is ready, after the ones before it */
260
+ send(message) {
261
+ this.pending.push(message);
262
+ this.flush();
263
+ }
264
+ /** Sends a message ahead of the waiting ones, from a Ready listener */
265
+ sendFirst(message) {
266
+ this.socket?.send(JSON.stringify(message));
267
+ }
268
+ /** Drops the messages that have not gone out */
269
+ clearPending() {
270
+ this.pending = [];
271
+ this.clearPendingTimeout();
272
+ }
273
+ /** Moves to a new connection, keeping the messages that have not gone out */
274
+ renew() {
275
+ this.log("Renewing the connection");
276
+ this.closeSocket();
277
+ this.connect();
278
+ }
279
+ destroy() {
280
+ this.destroyed = true;
281
+ if (this.reconnectTimeout) {
282
+ clearTimeout(this.reconnectTimeout);
283
+ this.reconnectTimeout = void 0;
284
+ }
285
+ this.closeSocket();
286
+ this.pending = [];
287
+ this.removeAllListeners();
288
+ }
289
+ /**
290
+ * Sends the messages in order, as soon as they can go.
291
+ *
292
+ * Gemini refuses the end of a turn that comes before the audio of the turn
293
+ * had time to play, and closes the session. Audio sent faster than it is
294
+ * spoken, when the user talked while the connection opened, would end the
295
+ * session, so the end of such a turn waits for it.
296
+ */
297
+ flush() {
298
+ while (this.pending.length > 0 && !this.pendingTimeout) {
299
+ const socket = this.socket;
300
+ if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
301
+ const message = this.pending[0];
302
+ const input = message.realtimeInput;
303
+ if (input?.activityEnd) {
304
+ const due = this.turnAudioStartedAt + this.turnAudioMs - Date.now();
305
+ if (due > 0) {
306
+ this.pendingTimeout = setTimeout(() => {
307
+ this.pendingTimeout = void 0;
308
+ this.flush();
309
+ }, due);
310
+ return;
311
+ }
312
+ } else if (input?.activityStart) {
313
+ this.turnAudioStartedAt = 0;
314
+ this.turnAudioMs = 0;
315
+ } else if (input?.audio) {
316
+ if (this.turnAudioMs === 0) this.turnAudioStartedAt = Date.now();
317
+ this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
318
+ }
319
+ this.pending.shift();
320
+ socket.send(JSON.stringify(message));
321
+ }
322
+ }
323
+ clearPendingTimeout() {
324
+ if (!this.pendingTimeout) return;
325
+ clearTimeout(this.pendingTimeout);
326
+ this.pendingTimeout = void 0;
327
+ }
328
+ initWS() {
329
+ return new Promise((resolve, reject) => {
330
+ const socket = new WebSocket(
331
+ `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
332
+ );
333
+ this.socket = socket;
334
+ const timeout = setTimeout(() => {
335
+ this.log("Connection timeout");
336
+ this.closeSocket();
337
+ reject(new Error("WebSocket connection timeout"));
338
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
339
+ socket.on("open", () => {
340
+ this.log("Connection opened");
341
+ socket.send(JSON.stringify(this.buildSetup()));
342
+ });
343
+ socket.on("error", (error) => {
344
+ clearTimeout(timeout);
345
+ this.log("WebSocket error:", error);
346
+ reject(new Error("WebSocket connection error"));
347
+ });
348
+ socket.on("close", (code, reason) => {
349
+ clearTimeout(timeout);
350
+ const wasConnected = this.connected;
351
+ this.closeSocket();
352
+ if (this.destroyed) return;
353
+ this.log("Connection closed", { code, reason: reason.toString() });
354
+ if (wasConnected) {
355
+ this.reconnect();
356
+ } else {
357
+ reject(new Error(`Connection closed: ${reason.toString()}`));
358
+ }
359
+ });
360
+ socket.on("message", (data) => {
361
+ const message = JSON.parse(data.toString());
362
+ if (message.setupComplete) {
363
+ clearTimeout(timeout);
364
+ this.log("Session ready");
365
+ this.connected = true;
366
+ this.retryCount = 0;
367
+ this.emit("Ready");
368
+ this.flush();
369
+ resolve();
370
+ return;
371
+ }
372
+ this.emit("Message", message);
373
+ });
374
+ });
375
+ }
376
+ closeSocket() {
377
+ const socket = this.socket;
378
+ this.socket = void 0;
379
+ this.connected = false;
380
+ this.clearPendingTimeout();
381
+ if (!socket) return;
382
+ socket.removeAllListeners();
383
+ socket.on("error", () => {
384
+ });
385
+ if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
386
+ socket.close(1e3);
387
+ }
388
+ }
389
+ reconnect() {
390
+ if (this.destroyed) return;
391
+ this.retryCount++;
392
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
393
+ this.log("Max retries reached, giving up");
394
+ this.emit("Failed");
395
+ return;
396
+ }
397
+ this.emit("Reconnecting", this.retryCount);
398
+ this.log("Reconnecting...");
399
+ this.reconnectTimeout = setTimeout(() => {
400
+ this.reconnectTimeout = void 0;
401
+ this.connect();
402
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
403
+ }
404
+ };
405
+
406
+ // src/GeminiLive.ts
407
+ var DEFAULT_MODEL2 = "gemini-3.8-live";
408
+ var SAMPLE_RATE2 = 16e3;
409
+ var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
410
+ var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
411
+ var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
412
+ var GeminiLive = class extends Realtime {
413
+ constructor(options) {
414
+ super(options);
415
+ this.everConnected = false;
416
+ // Whether the setup being sent resumes the session
417
+ this.resuming = false;
418
+ this.goingAway = false;
419
+ this.outputResampler = new Pcm16Resampler(
420
+ GEMINI_OUTPUT_SAMPLE_RATE,
421
+ SAMPLE_RATE2
422
+ );
423
+ this.hasTranscribedTurn = false;
424
+ // Transcript of a dropped turn, which the conversation should not show
425
+ this.ignoreUserTranscript = false;
426
+ this.cancelledToolCalls = /* @__PURE__ */ new Set();
427
+ // Gemini ends its turn when it calls a tool, and speaks again once it has
428
+ // the result. The answer stays open in between, so the client keeps waiting
429
+ // for it rather than listening again.
430
+ this.runningTools = 0;
431
+ this.answerEndDeferred = false;
432
+ this.onReady = () => {
433
+ if (this.everConnected && !this.resuming) {
434
+ const turns = this.conversation.flatMap(
435
+ (message) => message.role === "user" || message.role === "assistant" ? [
436
+ {
437
+ role: message.role === "assistant" ? "model" : "user",
438
+ parts: [{ text: message.content }]
439
+ }
440
+ ] : []
441
+ );
442
+ if (turns.length > 0) {
443
+ this.socket.sendFirst({ clientContent: { turns, turnComplete: false } });
444
+ }
445
+ }
446
+ this.everConnected = true;
447
+ this.goingAway = false;
448
+ };
449
+ this.onReconnecting = (attempt) => {
450
+ if (attempt > 1) this.resumptionHandle = void 0;
451
+ if (!this.resumptionHandle) this.socket.clearPending();
452
+ this.endAnswer();
453
+ };
454
+ this.onMessage = (message) => {
455
+ if (message.serverContent) {
456
+ this.onServerContent(message.serverContent);
457
+ }
458
+ if (message.toolCall) {
459
+ for (const call of message.toolCall.functionCalls ?? []) {
460
+ this.onToolCall(call);
461
+ }
462
+ }
463
+ if (message.toolCallCancellation) {
464
+ for (const id of message.toolCallCancellation.ids ?? []) {
465
+ this.cancelledToolCalls.add(id);
466
+ }
467
+ }
468
+ if (message.sessionResumptionUpdate) {
469
+ const { newHandle, resumable } = message.sessionResumptionUpdate;
470
+ if (resumable && newHandle) this.resumptionHandle = newHandle;
471
+ }
472
+ if (message.goAway) {
473
+ this.log("Connection ending in", message.goAway.timeLeft);
474
+ this.goingAway = true;
475
+ this.renewIfGoingAway();
476
+ }
477
+ };
478
+ this.socket = new LiveSocket(
479
+ options,
480
+ () => this.buildSetup(),
481
+ (...message) => this.log(...message)
482
+ );
483
+ this.socket.on("Ready", this.onReady);
484
+ this.socket.on("Message", this.onMessage);
485
+ this.socket.on("Reconnecting", this.onReconnecting);
486
+ this.socket.on("Failed", () => this.emit("Failed"));
487
+ this.socket.connect();
488
+ }
489
+ destroy() {
490
+ super.destroy();
491
+ this.clearToolFollowUp();
492
+ this.socket.destroy();
493
+ }
494
+ openTurn() {
495
+ this.ignoreUserTranscript = false;
496
+ this.send({ realtimeInput: { activityStart: {} } });
497
+ }
498
+ appendAudio(chunk) {
499
+ this.send({
500
+ realtimeInput: {
501
+ audio: {
502
+ mimeType: `audio/pcm;rate=${SAMPLE_RATE2}`,
503
+ data: chunk.toString("base64")
504
+ }
505
+ }
506
+ });
507
+ }
508
+ closeTurn() {
509
+ this.hasTranscribedTurn = false;
510
+ this.send({ realtimeInput: { activityEnd: {} } });
511
+ }
512
+ clearTurn() {
513
+ this.ignoreUserTranscript = true;
514
+ this.send({ realtimeInput: { activityEnd: {} } });
515
+ }
516
+ generate() {
517
+ this.sendUserText(FIRST_MESSAGE_PROMPT2);
518
+ }
519
+ speakText(text) {
520
+ this.sendUserText(
521
+ `Say exactly the following text, word for word, and nothing else: ${text}`
522
+ );
523
+ }
524
+ cancelAnswer() {
525
+ this.outputResampler.reset();
526
+ this.clearToolFollowUp();
527
+ this.answerEndDeferred = false;
528
+ }
529
+ get isIdle() {
530
+ return super.isIdle && this.runningTools === 0;
531
+ }
532
+ updateTools() {
533
+ if (this.everConnected) {
534
+ this.log("Tools changed after the session started, Gemini keeps the old");
535
+ }
536
+ }
537
+ injectMessage(role, text) {
538
+ if (!this.socket.isConnected && this.everConnected && !this.resumptionHandle)
539
+ return;
540
+ this.send(this.contentMessage(role, text, false));
541
+ }
542
+ sendUserText(text) {
543
+ this.send(this.contentMessage("user", text, true));
544
+ }
545
+ contentMessage(role, text, turnComplete) {
546
+ return {
547
+ clientContent: {
548
+ turns: [
549
+ {
550
+ role: role === "assistant" ? "model" : "user",
551
+ parts: [{ text }]
552
+ }
553
+ ],
554
+ turnComplete
555
+ }
556
+ };
557
+ }
558
+ send(message) {
559
+ this.socket.send(message);
560
+ }
561
+ buildSetup() {
562
+ this.resuming = !!this.resumptionHandle;
563
+ const functionDeclarations = this.tools.map((tool) => ({
564
+ name: tool.name,
565
+ description: tool.description,
566
+ ...tool.inputSchema ? { parametersJsonSchema: toJSONSchema2(tool.inputSchema) } : {}
567
+ }));
568
+ return {
569
+ setup: {
570
+ model: `models/${this.options.model || DEFAULT_MODEL2}`,
571
+ generationConfig: {
572
+ responseModalities: ["AUDIO"],
573
+ ...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
574
+ ...this.options.voice ? {
575
+ speechConfig: {
576
+ voiceConfig: {
577
+ prebuiltVoiceConfig: { voiceName: this.options.voice }
578
+ }
579
+ }
580
+ } : {}
581
+ },
582
+ systemInstruction: { parts: [{ text: this.options.systemPrompt }] },
583
+ ...functionDeclarations.length > 0 ? { tools: [{ functionDeclarations }] } : {},
584
+ // The Micdrop client detects when the user speaks
585
+ realtimeInputConfig: {
586
+ automaticActivityDetection: { disabled: true }
587
+ },
588
+ inputAudioTranscription: {},
589
+ outputAudioTranscription: {},
590
+ sessionResumption: this.resumptionHandle ? { handle: this.resumptionHandle } : {},
591
+ // Lifts the length limit of a session
592
+ contextWindowCompression: { slidingWindow: {} }
593
+ }
594
+ };
595
+ }
596
+ /** Moves to a new connection before Gemini closes this one */
597
+ renewIfGoingAway() {
598
+ if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
599
+ this.goingAway = false;
600
+ this.socket.renew();
601
+ }
602
+ onServerContent(content) {
603
+ if (content.inputTranscription?.text && !this.ignoreUserTranscript) {
604
+ this.addUserTranscript(content.inputTranscription.text);
605
+ }
606
+ for (const part of content.modelTurn?.parts ?? []) {
607
+ const data = part.inlineData?.data;
608
+ if (!data) continue;
609
+ this.onModelOutput();
610
+ const output = this.outputResampler.process(Buffer.from(data, "base64"));
611
+ if (output.length > 0) this.emitAudio(output);
612
+ }
613
+ if (content.outputTranscription?.text) {
614
+ this.onModelOutput();
615
+ this.addAnswerTranscript(content.outputTranscription.text);
616
+ }
617
+ if (content.interrupted) {
618
+ this.log("Interrupted");
619
+ this.outputResampler.reset();
620
+ }
621
+ if (content.generationComplete) {
622
+ this.commitTurnTranscript();
623
+ }
624
+ if (content.turnComplete) {
625
+ this.commitTurnTranscript();
626
+ this.outputResampler.reset();
627
+ if (this.runningTools > 0 || this.toolFollowUp) {
628
+ this.answerEndDeferred = true;
629
+ } else {
630
+ this.endAnswer();
631
+ this.renewIfGoingAway();
632
+ }
633
+ }
634
+ }
635
+ onModelOutput() {
636
+ this.commitTurnTranscript();
637
+ if (this.toolFollowUp) {
638
+ this.clearToolFollowUp();
639
+ this.answerEndDeferred = false;
640
+ }
641
+ }
642
+ /** Ends the answer the model stopped giving to wait for a tool */
643
+ endDeferredAnswer() {
644
+ if (!this.answerEndDeferred) return;
645
+ if (this.runningTools > 0 || this.toolFollowUp) return;
646
+ this.answerEndDeferred = false;
647
+ this.endAnswer();
648
+ this.renewIfGoingAway();
649
+ }
650
+ clearToolFollowUp() {
651
+ if (!this.toolFollowUp) return;
652
+ clearTimeout(this.toolFollowUp);
653
+ this.toolFollowUp = void 0;
654
+ }
655
+ /**
656
+ * The transcript of the turn comes before the answer, so the first piece of
657
+ * the answer closes it.
658
+ */
659
+ commitTurnTranscript() {
660
+ if (this.hasTranscribedTurn) return;
661
+ this.hasTranscribedTurn = true;
662
+ this.commitUserTranscript();
663
+ }
664
+ async onToolCall(call) {
665
+ this.commitTurnTranscript();
666
+ const toolCall = {
667
+ role: "tool_call",
668
+ toolCallId: call.id,
669
+ toolName: call.name,
670
+ parameters: JSON.stringify(call.args ?? {})
671
+ };
672
+ this.runningTools++;
673
+ let result;
674
+ try {
675
+ result = await this.runTool(toolCall);
676
+ } finally {
677
+ this.runningTools--;
678
+ }
679
+ if (this.cancelledToolCalls.delete(call.id)) {
680
+ this.endDeferredAnswer();
681
+ return;
682
+ }
683
+ this.send({
684
+ toolResponse: {
685
+ functionResponses: [
686
+ {
687
+ id: call.id,
688
+ name: call.name,
689
+ response: {
690
+ output: result.output ?? null,
691
+ // A tool without an answer lets the model know silently
692
+ scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
693
+ }
694
+ }
695
+ ]
696
+ }
697
+ });
698
+ if (!result.skipAnswer) {
699
+ this.clearToolFollowUp();
700
+ this.toolFollowUp = setTimeout(() => {
701
+ this.toolFollowUp = void 0;
702
+ this.log("Nothing said about the tool result");
703
+ this.endDeferredAnswer();
704
+ }, TOOL_FOLLOW_UP_TIMEOUT);
705
+ } else {
706
+ this.endDeferredAnswer();
707
+ }
708
+ }
709
+ };
710
+
711
+ // src/GeminiSTT.ts
712
+ import { STT } from "@micdrop/server";
713
+ var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
714
+ var SAMPLE_RATE3 = 16e3;
715
+ var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
716
+ var GeminiSTT = class extends STT {
717
+ constructor(options) {
718
+ super();
719
+ this.options = options;
720
+ this.goingAway = false;
721
+ // Audio of the utterance, sent again if the connection drops before its
722
+ // transcript comes back
723
+ this.chunks = [];
724
+ this.utteranceOpen = false;
725
+ this.utteranceEnded = false;
726
+ this.transcript = "";
727
+ this.onMessage = (message) => {
728
+ const content = message.serverContent;
729
+ if (content?.inputTranscription?.text && this.utteranceOpen) {
730
+ this.transcript += content.inputTranscription.text;
731
+ }
732
+ if (content?.generationComplete && this.utteranceEnded) {
733
+ this.finishUtterance();
734
+ }
735
+ if (message.goAway) {
736
+ this.log("Connection ending in", message.goAway.timeLeft);
737
+ if (this.utteranceOpen) {
738
+ this.goingAway = true;
739
+ } else {
740
+ this.socket.renew();
741
+ }
742
+ }
743
+ };
744
+ this.onReconnecting = () => {
745
+ this.socket.clearPending();
746
+ if (!this.utteranceOpen) return;
747
+ this.transcript = "";
748
+ this.socket.send({ realtimeInput: { activityStart: {} } });
749
+ this.chunks.forEach((chunk) => this.sendAudio(chunk));
750
+ if (this.utteranceEnded) {
751
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
752
+ }
753
+ };
754
+ this.socket = new LiveSocket(
755
+ options,
756
+ () => this.buildSetup(),
757
+ (...message) => this.log(...message)
758
+ );
759
+ this.socket.on("Message", this.onMessage);
760
+ this.socket.on("Reconnecting", this.onReconnecting);
761
+ this.socket.on("Failed", () => this.emit("Failed", this.chunks));
762
+ this.socket.connect();
763
+ }
764
+ transcribe(audioStream) {
765
+ audioStream.on("data", (chunk) => {
766
+ if (!this.utteranceOpen) this.openUtterance();
767
+ this.chunks.push(chunk);
768
+ this.sendAudio(chunk);
769
+ });
770
+ audioStream.on("end", () => {
771
+ if (!this.utteranceOpen || this.utteranceEnded) return;
772
+ this.utteranceEnded = true;
773
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
774
+ this.clearTranscriptionTimeout();
775
+ this.transcriptionTimeout = setTimeout(() => {
776
+ this.transcriptionTimeout = void 0;
777
+ this.log("Transcription timeout");
778
+ this.finishUtterance();
779
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
780
+ });
781
+ }
782
+ destroy() {
783
+ super.destroy();
784
+ this.clearTranscriptionTimeout();
785
+ this.socket.destroy();
786
+ }
787
+ buildSetup() {
788
+ return {
789
+ setup: {
790
+ model: `models/${this.options.model || DEFAULT_MODEL3}`,
791
+ generationConfig: { responseModalities: ["TEXT"] },
792
+ inputAudioTranscription: {
793
+ languageCodes: this.options.language ? [this.options.language] : [],
794
+ ...this.options.vocabulary?.length ? { customVocabulary: this.options.vocabulary } : {},
795
+ ...this.options.mode ? { mode: this.options.mode } : {}
796
+ },
797
+ // The Micdrop client detects when the user speaks
798
+ realtimeInputConfig: {
799
+ automaticActivityDetection: { disabled: true }
800
+ }
801
+ }
802
+ };
803
+ }
804
+ openUtterance() {
805
+ if (this.utteranceOpen) this.finishUtterance();
806
+ this.utteranceOpen = true;
807
+ this.utteranceEnded = false;
808
+ this.chunks = [];
809
+ this.transcript = "";
810
+ this.socket.send({ realtimeInput: { activityStart: {} } });
811
+ }
812
+ sendAudio(chunk) {
813
+ this.socket.send({
814
+ realtimeInput: {
815
+ audio: {
816
+ mimeType: `audio/pcm;rate=${SAMPLE_RATE3}`,
817
+ data: chunk.toString("base64")
818
+ }
819
+ }
820
+ });
821
+ }
822
+ /** Emits what was transcribed of the utterance, possibly nothing */
823
+ finishUtterance() {
824
+ this.clearTranscriptionTimeout();
825
+ const transcript = this.transcript.trim();
826
+ this.utteranceOpen = false;
827
+ this.utteranceEnded = false;
828
+ this.chunks = [];
829
+ this.transcript = "";
830
+ this.log(`Transcript: "${transcript}"`);
831
+ this.emit("Transcript", transcript);
832
+ if (this.goingAway) {
833
+ this.goingAway = false;
834
+ this.socket.renew();
835
+ }
836
+ }
837
+ clearTranscriptionTimeout() {
838
+ if (!this.transcriptionTimeout) return;
839
+ clearTimeout(this.transcriptionTimeout);
840
+ this.transcriptionTimeout = void 0;
841
+ }
842
+ };
843
+
844
+ // src/GeminiTTS.ts
845
+ import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
846
+ import { Pcm16Resampler as Pcm16Resampler2, SentenceTTS } from "@micdrop/server";
847
+ var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
848
+ var DEFAULT_VOICE = "Kore";
849
+ var GEMINI_SAMPLE_RATE = 24e3;
850
+ var OUTPUT_SAMPLE_RATE = 16e3;
851
+ var DEFAULT_MAX_RETRY2 = 2;
852
+ var GeminiTTS = class extends SentenceTTS {
853
+ constructor(options) {
854
+ super();
855
+ this.options = options;
856
+ this.genai = "genai" in options ? options.genai : new GoogleGenAI2({ apiKey: options.apiKey });
857
+ }
858
+ async synthesize(text, signal) {
859
+ const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
860
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
861
+ for (let attempt = 0; ; attempt++) {
862
+ const progress = { emitted: false };
863
+ try {
864
+ await this.synthesizeOnce(input, signal, progress);
865
+ return void 0;
866
+ } catch (error) {
867
+ if (signal.aborted || progress.emitted || attempt >= maxRetry) {
868
+ throw error;
869
+ }
870
+ if (!isBlocked(error)) throw error;
871
+ this.log(`Sentence blocked, trying again: "${text}"`);
872
+ }
873
+ }
874
+ }
875
+ async synthesizeOnce(input, signal, progress) {
876
+ const stream = await this.genai.interactions.create(
877
+ {
878
+ model: this.options.model || DEFAULT_MODEL4,
879
+ store: false,
880
+ stream: true,
881
+ input,
882
+ response_format: { type: "audio" },
883
+ generation_config: {
884
+ speech_config: [{ voice: this.options.voice || DEFAULT_VOICE }]
885
+ }
886
+ },
887
+ { fetchOptions: { signal } }
888
+ );
889
+ let resampler;
890
+ for await (const event of stream) {
891
+ if (signal.aborted) break;
892
+ if (event.event_type === "error") {
893
+ throw Object.assign(
894
+ new Error(event.error?.message || "Speech generation failed"),
895
+ { code: event.error?.code }
896
+ );
897
+ }
898
+ if (event.event_type !== "step.delta" || event.delta.type !== "audio") {
899
+ continue;
900
+ }
901
+ const { data, sample_rate } = event.delta;
902
+ if (!data) continue;
903
+ resampler ?? (resampler = new Pcm16Resampler2(
904
+ sample_rate || GEMINI_SAMPLE_RATE,
905
+ OUTPUT_SAMPLE_RATE
906
+ ));
907
+ const audio = resampler.process(Buffer.from(data, "base64"));
908
+ if (audio.length > 0) progress.emitted = true;
909
+ if (!this.emitAudio(audio)) break;
910
+ }
911
+ }
912
+ };
913
+ function isBlocked(error) {
914
+ return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
915
+ }
916
+ export {
917
+ GeminiAgent,
918
+ GeminiLive,
919
+ GeminiSTT,
920
+ GeminiTTS
921
+ };
922
+ //# sourceMappingURL=index.mjs.map