@micdrop/gemini 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,959 @@
1
+ "use strict";
2
+ var __create = Object.create;
3
+ var __defProp = Object.defineProperty;
4
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
5
+ var __getOwnPropNames = Object.getOwnPropertyNames;
6
+ var __getProtoOf = Object.getPrototypeOf;
7
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
8
+ var __export = (target, all) => {
9
+ for (var name in all)
10
+ __defProp(target, name, { get: all[name], enumerable: true });
11
+ };
12
+ var __copyProps = (to, from, except, desc) => {
13
+ if (from && typeof from === "object" || typeof from === "function") {
14
+ for (let key of __getOwnPropNames(from))
15
+ if (!__hasOwnProp.call(to, key) && key !== except)
16
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
17
+ }
18
+ return to;
19
+ };
20
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
21
+ // If the importer is in node compatibility mode or this is not an ESM
22
+ // file that has been converted to a CommonJS file using a Babel-
23
+ // compatible transform (i.e. "__esModule" has not been set), then set
24
+ // "default" to the CommonJS "module.exports" for node compatibility.
25
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
26
+ mod
27
+ ));
28
+ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
29
+
30
+ // src/index.ts
31
+ var index_exports = {};
32
+ __export(index_exports, {
33
+ GeminiAgent: () => GeminiAgent,
34
+ GeminiLive: () => GeminiLive,
35
+ GeminiSTT: () => GeminiSTT,
36
+ GeminiTTS: () => GeminiTTS
37
+ });
38
+ module.exports = __toCommonJS(index_exports);
39
+
40
+ // src/GeminiAgent.ts
41
+ var import_genai = require("@google/genai");
42
+ var import_server = require("@micdrop/server");
43
+ var import_zod = __toESM(require("zod"));
44
+ var DEFAULT_MODEL = "gemini-3.8-flash";
45
+ var DEFAULT_MAX_STEPS = 5;
46
+ var DEFAULT_MAX_RETRIES = 3;
47
+ var DEFAULT_RETRY_DELAY = 1e3;
48
+ var FIRST_MESSAGE_PROMPT = "The call starts now.";
49
+ var GeminiAgent = class extends import_server.Agent {
50
+ constructor(options) {
51
+ super(options);
52
+ // Gemini asks for the signature of the thought behind a tool call when the
53
+ // call is sent back, and the conversation has no room for it
54
+ this.signatures = /* @__PURE__ */ new Map();
55
+ this.genai = "genai" in options ? options.genai : new import_genai.GoogleGenAI({ apiKey: options.apiKey });
56
+ }
57
+ async generateAnswer(stream, stepCount = 0, tryCount = 0) {
58
+ if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
59
+ console.error("[GeminiAgent] Max steps reached");
60
+ return;
61
+ }
62
+ const abortController = new AbortController();
63
+ this.abortController = abortController;
64
+ const signal = abortController.signal;
65
+ let extracting = false;
66
+ const extractOptions = this.getExtractOptions();
67
+ try {
68
+ const response = await this.genai.interactions.create(
69
+ {
70
+ model: this.options.model || DEFAULT_MODEL,
71
+ store: false,
72
+ stream: true,
73
+ system_instruction: this.buildSystemInstruction(),
74
+ input: this.buildInput(),
75
+ tools: this.buildTools(),
76
+ ...this.options.thinkingLevel || this.options.settings ? {
77
+ generation_config: {
78
+ ...this.options.settings,
79
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
80
+ }
81
+ } : {}
82
+ },
83
+ { fetchOptions: { signal } }
84
+ );
85
+ const steps = [];
86
+ let answer = "";
87
+ let skipAnswer = false;
88
+ let calledTool = false;
89
+ let signature = "";
90
+ for await (const event of response) {
91
+ if (abortController !== this.abortController) return;
92
+ switch (event.event_type) {
93
+ case "step.start":
94
+ steps[event.index] = {
95
+ type: event.step.type,
96
+ id: event.step.id,
97
+ name: event.step.name,
98
+ text: "",
99
+ arguments: "",
100
+ signature: ""
101
+ };
102
+ break;
103
+ case "step.delta": {
104
+ const step = steps[event.index];
105
+ if (!step) break;
106
+ const delta = event.delta;
107
+ if (delta.type === "thought_signature") {
108
+ step.signature += delta.signature ?? "";
109
+ } else if (delta.type === "arguments_delta") {
110
+ step.arguments += delta.arguments ?? "";
111
+ } else if (delta.type === "text" && step.type === "model_output") {
112
+ this.log(`Answer chunk: "${delta.text}"`);
113
+ answer += delta.text;
114
+ if (extractOptions) {
115
+ if (extracting) break;
116
+ const startTagIndex = delta.text.indexOf(
117
+ extractOptions.startTag
118
+ );
119
+ if (startTagIndex !== -1) {
120
+ extracting = true;
121
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
122
+ break;
123
+ }
124
+ }
125
+ stream.write(delta.text);
126
+ }
127
+ break;
128
+ }
129
+ case "step.stop": {
130
+ const step = steps[event.index];
131
+ if (step?.type === "thought") signature = step.signature;
132
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
133
+ calledTool = true;
134
+ if (signature) this.signatures.set(step.id, signature);
135
+ const result = await this.executeTool({
136
+ role: "tool_call",
137
+ toolCallId: step.id,
138
+ toolName: step.name,
139
+ parameters: step.arguments || "{}"
140
+ });
141
+ if (result.skipAnswer) skipAnswer = true;
142
+ break;
143
+ }
144
+ case "error":
145
+ throw new Error(event.error?.message || "Generation failed");
146
+ default:
147
+ break;
148
+ }
149
+ }
150
+ if (abortController !== this.abortController) return;
151
+ this.abortController = void 0;
152
+ if (answer) {
153
+ const { message, metadata } = this.extract(answer);
154
+ this.addAssistantMessage(message, metadata);
155
+ }
156
+ if (calledTool && !skipAnswer && !answer) {
157
+ await this.generateAnswer(stream, stepCount + 1);
158
+ }
159
+ } catch (error) {
160
+ if (signal.aborted) return;
161
+ console.error("[GeminiAgent] Error answering:", error);
162
+ if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
163
+ await new Promise(
164
+ (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
165
+ );
166
+ await this.generateAnswer(stream, stepCount, tryCount + 1);
167
+ } else {
168
+ this.log("Max retries reached, giving up");
169
+ this.emit("Failed");
170
+ }
171
+ }
172
+ }
173
+ cancel() {
174
+ if (!this.abortController) return;
175
+ this.log("Cancel");
176
+ this.abortController.abort();
177
+ this.abortController = void 0;
178
+ }
179
+ buildSystemInstruction() {
180
+ return this.conversation.flatMap(
181
+ (message) => message.role === "system" ? [message.content] : []
182
+ ).join("\n\n");
183
+ }
184
+ buildInput() {
185
+ const input = this.conversation.flatMap((message) => {
186
+ switch (message.role) {
187
+ case "user":
188
+ return [
189
+ {
190
+ type: "user_input",
191
+ content: [{ type: "text", text: message.content }]
192
+ }
193
+ ];
194
+ case "assistant":
195
+ return [
196
+ {
197
+ type: "model_output",
198
+ content: [{ type: "text", text: message.content }]
199
+ }
200
+ ];
201
+ case "tool_call": {
202
+ const signature = this.signatures.get(message.toolCallId);
203
+ return [
204
+ ...signature ? [{ type: "thought", signature }] : [],
205
+ {
206
+ type: "function_call",
207
+ id: message.toolCallId,
208
+ name: message.toolName,
209
+ arguments: parseArguments(message.parameters)
210
+ }
211
+ ];
212
+ }
213
+ case "tool_result":
214
+ return [
215
+ {
216
+ type: "function_result",
217
+ call_id: message.toolCallId,
218
+ name: message.toolName,
219
+ result: [{ type: "text", text: message.output }]
220
+ }
221
+ ];
222
+ default:
223
+ return [];
224
+ }
225
+ });
226
+ if (input.length === 0) {
227
+ input.push({
228
+ type: "user_input",
229
+ content: [{ type: "text", text: FIRST_MESSAGE_PROMPT }]
230
+ });
231
+ }
232
+ return input;
233
+ }
234
+ buildTools() {
235
+ if (this.tools.length === 0) return void 0;
236
+ return this.tools.map((tool) => {
237
+ const { $schema, ...parameters } = (0, import_zod.toJSONSchema)(
238
+ tool.inputSchema || import_zod.default.object()
239
+ );
240
+ return {
241
+ type: "function",
242
+ name: tool.name,
243
+ description: tool.description,
244
+ parameters
245
+ };
246
+ });
247
+ }
248
+ };
249
+ function parseArguments(parameters) {
250
+ try {
251
+ return JSON.parse(parameters);
252
+ } catch {
253
+ return {};
254
+ }
255
+ }
256
+
257
+ // src/GeminiLive.ts
258
+ var import_server2 = require("@micdrop/server");
259
+ var import_zod2 = require("zod");
260
+
261
+ // src/LiveSocket.ts
262
+ var import_eventemitter3 = require("eventemitter3");
263
+ var import_ws = __toESM(require("ws"));
264
+ var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
265
+ var SAMPLE_RATE = 16e3;
266
+ var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
267
+ var DEFAULT_CONNECTION_TIMEOUT = 5e3;
268
+ var DEFAULT_RETRY_DELAY2 = 1e3;
269
+ var DEFAULT_MAX_RETRY = 3;
270
+ var LiveSocket = class extends import_eventemitter3.EventEmitter {
271
+ constructor(options, buildSetup, log) {
272
+ super();
273
+ this.options = options;
274
+ this.buildSetup = buildSetup;
275
+ this.log = log;
276
+ this.connected = false;
277
+ this.destroyed = false;
278
+ // Messages waiting for the session, or for the end of a turn to be due
279
+ this.pending = [];
280
+ // Audio of the turn as it went out, see flush()
281
+ this.turnAudioStartedAt = 0;
282
+ this.turnAudioMs = 0;
283
+ this.retryCount = 0;
284
+ }
285
+ /** The session is set up and takes messages right away */
286
+ get isConnected() {
287
+ return this.connected;
288
+ }
289
+ connect() {
290
+ this.initWS().catch((error) => {
291
+ this.log("Connection error:", error);
292
+ this.reconnect();
293
+ });
294
+ }
295
+ /** Sends a message once the session is ready, after the ones before it */
296
+ send(message) {
297
+ this.pending.push(message);
298
+ this.flush();
299
+ }
300
+ /** Sends a message ahead of the waiting ones, from a Ready listener */
301
+ sendFirst(message) {
302
+ this.socket?.send(JSON.stringify(message));
303
+ }
304
+ /** Drops the messages that have not gone out */
305
+ clearPending() {
306
+ this.pending = [];
307
+ this.clearPendingTimeout();
308
+ }
309
+ /** Moves to a new connection, keeping the messages that have not gone out */
310
+ renew() {
311
+ this.log("Renewing the connection");
312
+ this.closeSocket();
313
+ this.connect();
314
+ }
315
+ destroy() {
316
+ this.destroyed = true;
317
+ if (this.reconnectTimeout) {
318
+ clearTimeout(this.reconnectTimeout);
319
+ this.reconnectTimeout = void 0;
320
+ }
321
+ this.closeSocket();
322
+ this.pending = [];
323
+ this.removeAllListeners();
324
+ }
325
+ /**
326
+ * Sends the messages in order, as soon as they can go.
327
+ *
328
+ * Gemini refuses the end of a turn that comes before the audio of the turn
329
+ * had time to play, and closes the session. Audio sent faster than it is
330
+ * spoken, when the user talked while the connection opened, would end the
331
+ * session, so the end of such a turn waits for it.
332
+ */
333
+ flush() {
334
+ while (this.pending.length > 0 && !this.pendingTimeout) {
335
+ const socket = this.socket;
336
+ if (!this.connected || socket?.readyState !== import_ws.default.OPEN) return;
337
+ const message = this.pending[0];
338
+ const input = message.realtimeInput;
339
+ if (input?.activityEnd) {
340
+ const due = this.turnAudioStartedAt + this.turnAudioMs - Date.now();
341
+ if (due > 0) {
342
+ this.pendingTimeout = setTimeout(() => {
343
+ this.pendingTimeout = void 0;
344
+ this.flush();
345
+ }, due);
346
+ return;
347
+ }
348
+ } else if (input?.activityStart) {
349
+ this.turnAudioStartedAt = 0;
350
+ this.turnAudioMs = 0;
351
+ } else if (input?.audio) {
352
+ if (this.turnAudioMs === 0) this.turnAudioStartedAt = Date.now();
353
+ this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
354
+ }
355
+ this.pending.shift();
356
+ socket.send(JSON.stringify(message));
357
+ }
358
+ }
359
+ clearPendingTimeout() {
360
+ if (!this.pendingTimeout) return;
361
+ clearTimeout(this.pendingTimeout);
362
+ this.pendingTimeout = void 0;
363
+ }
364
+ initWS() {
365
+ return new Promise((resolve, reject) => {
366
+ const socket = new import_ws.default(
367
+ `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
368
+ );
369
+ this.socket = socket;
370
+ const timeout = setTimeout(() => {
371
+ this.log("Connection timeout");
372
+ this.closeSocket();
373
+ reject(new Error("WebSocket connection timeout"));
374
+ }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
375
+ socket.on("open", () => {
376
+ this.log("Connection opened");
377
+ socket.send(JSON.stringify(this.buildSetup()));
378
+ });
379
+ socket.on("error", (error) => {
380
+ clearTimeout(timeout);
381
+ this.log("WebSocket error:", error);
382
+ reject(new Error("WebSocket connection error"));
383
+ });
384
+ socket.on("close", (code, reason) => {
385
+ clearTimeout(timeout);
386
+ const wasConnected = this.connected;
387
+ this.closeSocket();
388
+ if (this.destroyed) return;
389
+ this.log("Connection closed", { code, reason: reason.toString() });
390
+ if (wasConnected) {
391
+ this.reconnect();
392
+ } else {
393
+ reject(new Error(`Connection closed: ${reason.toString()}`));
394
+ }
395
+ });
396
+ socket.on("message", (data) => {
397
+ const message = JSON.parse(data.toString());
398
+ if (message.setupComplete) {
399
+ clearTimeout(timeout);
400
+ this.log("Session ready");
401
+ this.connected = true;
402
+ this.retryCount = 0;
403
+ this.emit("Ready");
404
+ this.flush();
405
+ resolve();
406
+ return;
407
+ }
408
+ this.emit("Message", message);
409
+ });
410
+ });
411
+ }
412
+ closeSocket() {
413
+ const socket = this.socket;
414
+ this.socket = void 0;
415
+ this.connected = false;
416
+ this.clearPendingTimeout();
417
+ if (!socket) return;
418
+ socket.removeAllListeners();
419
+ socket.on("error", () => {
420
+ });
421
+ if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
422
+ socket.close(1e3);
423
+ }
424
+ }
425
+ reconnect() {
426
+ if (this.destroyed) return;
427
+ this.retryCount++;
428
+ if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
429
+ this.log("Max retries reached, giving up");
430
+ this.emit("Failed");
431
+ return;
432
+ }
433
+ this.emit("Reconnecting", this.retryCount);
434
+ this.log("Reconnecting...");
435
+ this.reconnectTimeout = setTimeout(() => {
436
+ this.reconnectTimeout = void 0;
437
+ this.connect();
438
+ }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
439
+ }
440
+ };
441
+
442
+ // src/GeminiLive.ts
443
+ var DEFAULT_MODEL2 = "gemini-3.8-live";
444
+ var SAMPLE_RATE2 = 16e3;
445
+ var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
446
+ var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
447
+ var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
448
+ var GeminiLive = class extends import_server2.Realtime {
449
+ constructor(options) {
450
+ super(options);
451
+ this.everConnected = false;
452
+ // Whether the setup being sent resumes the session
453
+ this.resuming = false;
454
+ this.goingAway = false;
455
+ this.outputResampler = new import_server2.Pcm16Resampler(
456
+ GEMINI_OUTPUT_SAMPLE_RATE,
457
+ SAMPLE_RATE2
458
+ );
459
+ this.hasTranscribedTurn = false;
460
+ // Transcript of a dropped turn, which the conversation should not show
461
+ this.ignoreUserTranscript = false;
462
+ this.cancelledToolCalls = /* @__PURE__ */ new Set();
463
+ // Gemini ends its turn when it calls a tool, and speaks again once it has
464
+ // the result. The answer stays open in between, so the client keeps waiting
465
+ // for it rather than listening again.
466
+ this.runningTools = 0;
467
+ this.answerEndDeferred = false;
468
+ this.onReady = () => {
469
+ if (this.everConnected && !this.resuming) {
470
+ const turns = this.conversation.flatMap(
471
+ (message) => message.role === "user" || message.role === "assistant" ? [
472
+ {
473
+ role: message.role === "assistant" ? "model" : "user",
474
+ parts: [{ text: message.content }]
475
+ }
476
+ ] : []
477
+ );
478
+ if (turns.length > 0) {
479
+ this.socket.sendFirst({ clientContent: { turns, turnComplete: false } });
480
+ }
481
+ }
482
+ this.everConnected = true;
483
+ this.goingAway = false;
484
+ };
485
+ this.onReconnecting = (attempt) => {
486
+ if (attempt > 1) this.resumptionHandle = void 0;
487
+ if (!this.resumptionHandle) this.socket.clearPending();
488
+ this.endAnswer();
489
+ };
490
+ this.onMessage = (message) => {
491
+ if (message.serverContent) {
492
+ this.onServerContent(message.serverContent);
493
+ }
494
+ if (message.toolCall) {
495
+ for (const call of message.toolCall.functionCalls ?? []) {
496
+ this.onToolCall(call);
497
+ }
498
+ }
499
+ if (message.toolCallCancellation) {
500
+ for (const id of message.toolCallCancellation.ids ?? []) {
501
+ this.cancelledToolCalls.add(id);
502
+ }
503
+ }
504
+ if (message.sessionResumptionUpdate) {
505
+ const { newHandle, resumable } = message.sessionResumptionUpdate;
506
+ if (resumable && newHandle) this.resumptionHandle = newHandle;
507
+ }
508
+ if (message.goAway) {
509
+ this.log("Connection ending in", message.goAway.timeLeft);
510
+ this.goingAway = true;
511
+ this.renewIfGoingAway();
512
+ }
513
+ };
514
+ this.socket = new LiveSocket(
515
+ options,
516
+ () => this.buildSetup(),
517
+ (...message) => this.log(...message)
518
+ );
519
+ this.socket.on("Ready", this.onReady);
520
+ this.socket.on("Message", this.onMessage);
521
+ this.socket.on("Reconnecting", this.onReconnecting);
522
+ this.socket.on("Failed", () => this.emit("Failed"));
523
+ this.socket.connect();
524
+ }
525
+ destroy() {
526
+ super.destroy();
527
+ this.clearToolFollowUp();
528
+ this.socket.destroy();
529
+ }
530
+ openTurn() {
531
+ this.ignoreUserTranscript = false;
532
+ this.send({ realtimeInput: { activityStart: {} } });
533
+ }
534
+ appendAudio(chunk) {
535
+ this.send({
536
+ realtimeInput: {
537
+ audio: {
538
+ mimeType: `audio/pcm;rate=${SAMPLE_RATE2}`,
539
+ data: chunk.toString("base64")
540
+ }
541
+ }
542
+ });
543
+ }
544
+ closeTurn() {
545
+ this.hasTranscribedTurn = false;
546
+ this.send({ realtimeInput: { activityEnd: {} } });
547
+ }
548
+ clearTurn() {
549
+ this.ignoreUserTranscript = true;
550
+ this.send({ realtimeInput: { activityEnd: {} } });
551
+ }
552
+ generate() {
553
+ this.sendUserText(FIRST_MESSAGE_PROMPT2);
554
+ }
555
+ speakText(text) {
556
+ this.sendUserText(
557
+ `Say exactly the following text, word for word, and nothing else: ${text}`
558
+ );
559
+ }
560
+ cancelAnswer() {
561
+ this.outputResampler.reset();
562
+ this.clearToolFollowUp();
563
+ this.answerEndDeferred = false;
564
+ }
565
+ get isIdle() {
566
+ return super.isIdle && this.runningTools === 0;
567
+ }
568
+ updateTools() {
569
+ if (this.everConnected) {
570
+ this.log("Tools changed after the session started, Gemini keeps the old");
571
+ }
572
+ }
573
+ injectMessage(role, text) {
574
+ if (!this.socket.isConnected && this.everConnected && !this.resumptionHandle)
575
+ return;
576
+ this.send(this.contentMessage(role, text, false));
577
+ }
578
+ sendUserText(text) {
579
+ this.send(this.contentMessage("user", text, true));
580
+ }
581
+ contentMessage(role, text, turnComplete) {
582
+ return {
583
+ clientContent: {
584
+ turns: [
585
+ {
586
+ role: role === "assistant" ? "model" : "user",
587
+ parts: [{ text }]
588
+ }
589
+ ],
590
+ turnComplete
591
+ }
592
+ };
593
+ }
594
+ send(message) {
595
+ this.socket.send(message);
596
+ }
597
+ buildSetup() {
598
+ this.resuming = !!this.resumptionHandle;
599
+ const functionDeclarations = this.tools.map((tool) => ({
600
+ name: tool.name,
601
+ description: tool.description,
602
+ ...tool.inputSchema ? { parametersJsonSchema: (0, import_zod2.toJSONSchema)(tool.inputSchema) } : {}
603
+ }));
604
+ return {
605
+ setup: {
606
+ model: `models/${this.options.model || DEFAULT_MODEL2}`,
607
+ generationConfig: {
608
+ responseModalities: ["AUDIO"],
609
+ ...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
610
+ ...this.options.voice ? {
611
+ speechConfig: {
612
+ voiceConfig: {
613
+ prebuiltVoiceConfig: { voiceName: this.options.voice }
614
+ }
615
+ }
616
+ } : {}
617
+ },
618
+ systemInstruction: { parts: [{ text: this.options.systemPrompt }] },
619
+ ...functionDeclarations.length > 0 ? { tools: [{ functionDeclarations }] } : {},
620
+ // The Micdrop client detects when the user speaks
621
+ realtimeInputConfig: {
622
+ automaticActivityDetection: { disabled: true }
623
+ },
624
+ inputAudioTranscription: {},
625
+ outputAudioTranscription: {},
626
+ sessionResumption: this.resumptionHandle ? { handle: this.resumptionHandle } : {},
627
+ // Lifts the length limit of a session
628
+ contextWindowCompression: { slidingWindow: {} }
629
+ }
630
+ };
631
+ }
632
+ /** Moves to a new connection before Gemini closes this one */
633
+ renewIfGoingAway() {
634
+ if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
635
+ this.goingAway = false;
636
+ this.socket.renew();
637
+ }
638
+ onServerContent(content) {
639
+ if (content.inputTranscription?.text && !this.ignoreUserTranscript) {
640
+ this.addUserTranscript(content.inputTranscription.text);
641
+ }
642
+ for (const part of content.modelTurn?.parts ?? []) {
643
+ const data = part.inlineData?.data;
644
+ if (!data) continue;
645
+ this.onModelOutput();
646
+ const output = this.outputResampler.process(Buffer.from(data, "base64"));
647
+ if (output.length > 0) this.emitAudio(output);
648
+ }
649
+ if (content.outputTranscription?.text) {
650
+ this.onModelOutput();
651
+ this.addAnswerTranscript(content.outputTranscription.text);
652
+ }
653
+ if (content.interrupted) {
654
+ this.log("Interrupted");
655
+ this.outputResampler.reset();
656
+ }
657
+ if (content.generationComplete) {
658
+ this.commitTurnTranscript();
659
+ }
660
+ if (content.turnComplete) {
661
+ this.commitTurnTranscript();
662
+ this.outputResampler.reset();
663
+ if (this.runningTools > 0 || this.toolFollowUp) {
664
+ this.answerEndDeferred = true;
665
+ } else {
666
+ this.endAnswer();
667
+ this.renewIfGoingAway();
668
+ }
669
+ }
670
+ }
671
+ onModelOutput() {
672
+ this.commitTurnTranscript();
673
+ if (this.toolFollowUp) {
674
+ this.clearToolFollowUp();
675
+ this.answerEndDeferred = false;
676
+ }
677
+ }
678
+ /** Ends the answer the model stopped giving to wait for a tool */
679
+ endDeferredAnswer() {
680
+ if (!this.answerEndDeferred) return;
681
+ if (this.runningTools > 0 || this.toolFollowUp) return;
682
+ this.answerEndDeferred = false;
683
+ this.endAnswer();
684
+ this.renewIfGoingAway();
685
+ }
686
+ clearToolFollowUp() {
687
+ if (!this.toolFollowUp) return;
688
+ clearTimeout(this.toolFollowUp);
689
+ this.toolFollowUp = void 0;
690
+ }
691
+ /**
692
+ * The transcript of the turn comes before the answer, so the first piece of
693
+ * the answer closes it.
694
+ */
695
+ commitTurnTranscript() {
696
+ if (this.hasTranscribedTurn) return;
697
+ this.hasTranscribedTurn = true;
698
+ this.commitUserTranscript();
699
+ }
700
+ async onToolCall(call) {
701
+ this.commitTurnTranscript();
702
+ const toolCall = {
703
+ role: "tool_call",
704
+ toolCallId: call.id,
705
+ toolName: call.name,
706
+ parameters: JSON.stringify(call.args ?? {})
707
+ };
708
+ this.runningTools++;
709
+ let result;
710
+ try {
711
+ result = await this.runTool(toolCall);
712
+ } finally {
713
+ this.runningTools--;
714
+ }
715
+ if (this.cancelledToolCalls.delete(call.id)) {
716
+ this.endDeferredAnswer();
717
+ return;
718
+ }
719
+ this.send({
720
+ toolResponse: {
721
+ functionResponses: [
722
+ {
723
+ id: call.id,
724
+ name: call.name,
725
+ response: {
726
+ output: result.output ?? null,
727
+ // A tool without an answer lets the model know silently
728
+ scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
729
+ }
730
+ }
731
+ ]
732
+ }
733
+ });
734
+ if (!result.skipAnswer) {
735
+ this.clearToolFollowUp();
736
+ this.toolFollowUp = setTimeout(() => {
737
+ this.toolFollowUp = void 0;
738
+ this.log("Nothing said about the tool result");
739
+ this.endDeferredAnswer();
740
+ }, TOOL_FOLLOW_UP_TIMEOUT);
741
+ } else {
742
+ this.endDeferredAnswer();
743
+ }
744
+ }
745
+ };
746
+
747
+ // src/GeminiSTT.ts
748
+ var import_server3 = require("@micdrop/server");
749
+ var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
750
+ var SAMPLE_RATE3 = 16e3;
751
+ var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
752
+ var GeminiSTT = class extends import_server3.STT {
753
+ constructor(options) {
754
+ super();
755
+ this.options = options;
756
+ this.goingAway = false;
757
+ // Audio of the utterance, sent again if the connection drops before its
758
+ // transcript comes back
759
+ this.chunks = [];
760
+ this.utteranceOpen = false;
761
+ this.utteranceEnded = false;
762
+ this.transcript = "";
763
+ this.onMessage = (message) => {
764
+ const content = message.serverContent;
765
+ if (content?.inputTranscription?.text && this.utteranceOpen) {
766
+ this.transcript += content.inputTranscription.text;
767
+ }
768
+ if (content?.generationComplete && this.utteranceEnded) {
769
+ this.finishUtterance();
770
+ }
771
+ if (message.goAway) {
772
+ this.log("Connection ending in", message.goAway.timeLeft);
773
+ if (this.utteranceOpen) {
774
+ this.goingAway = true;
775
+ } else {
776
+ this.socket.renew();
777
+ }
778
+ }
779
+ };
780
+ this.onReconnecting = () => {
781
+ this.socket.clearPending();
782
+ if (!this.utteranceOpen) return;
783
+ this.transcript = "";
784
+ this.socket.send({ realtimeInput: { activityStart: {} } });
785
+ this.chunks.forEach((chunk) => this.sendAudio(chunk));
786
+ if (this.utteranceEnded) {
787
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
788
+ }
789
+ };
790
+ this.socket = new LiveSocket(
791
+ options,
792
+ () => this.buildSetup(),
793
+ (...message) => this.log(...message)
794
+ );
795
+ this.socket.on("Message", this.onMessage);
796
+ this.socket.on("Reconnecting", this.onReconnecting);
797
+ this.socket.on("Failed", () => this.emit("Failed", this.chunks));
798
+ this.socket.connect();
799
+ }
800
+ transcribe(audioStream) {
801
+ audioStream.on("data", (chunk) => {
802
+ if (!this.utteranceOpen) this.openUtterance();
803
+ this.chunks.push(chunk);
804
+ this.sendAudio(chunk);
805
+ });
806
+ audioStream.on("end", () => {
807
+ if (!this.utteranceOpen || this.utteranceEnded) return;
808
+ this.utteranceEnded = true;
809
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
810
+ this.clearTranscriptionTimeout();
811
+ this.transcriptionTimeout = setTimeout(() => {
812
+ this.transcriptionTimeout = void 0;
813
+ this.log("Transcription timeout");
814
+ this.finishUtterance();
815
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
816
+ });
817
+ }
818
+ destroy() {
819
+ super.destroy();
820
+ this.clearTranscriptionTimeout();
821
+ this.socket.destroy();
822
+ }
823
+ buildSetup() {
824
+ return {
825
+ setup: {
826
+ model: `models/${this.options.model || DEFAULT_MODEL3}`,
827
+ generationConfig: { responseModalities: ["TEXT"] },
828
+ inputAudioTranscription: {
829
+ languageCodes: this.options.language ? [this.options.language] : [],
830
+ ...this.options.vocabulary?.length ? { customVocabulary: this.options.vocabulary } : {},
831
+ ...this.options.mode ? { mode: this.options.mode } : {}
832
+ },
833
+ // The Micdrop client detects when the user speaks
834
+ realtimeInputConfig: {
835
+ automaticActivityDetection: { disabled: true }
836
+ }
837
+ }
838
+ };
839
+ }
840
+ openUtterance() {
841
+ if (this.utteranceOpen) this.finishUtterance();
842
+ this.utteranceOpen = true;
843
+ this.utteranceEnded = false;
844
+ this.chunks = [];
845
+ this.transcript = "";
846
+ this.socket.send({ realtimeInput: { activityStart: {} } });
847
+ }
848
+ sendAudio(chunk) {
849
+ this.socket.send({
850
+ realtimeInput: {
851
+ audio: {
852
+ mimeType: `audio/pcm;rate=${SAMPLE_RATE3}`,
853
+ data: chunk.toString("base64")
854
+ }
855
+ }
856
+ });
857
+ }
858
+ /** Emits what was transcribed of the utterance, possibly nothing */
859
+ finishUtterance() {
860
+ this.clearTranscriptionTimeout();
861
+ const transcript = this.transcript.trim();
862
+ this.utteranceOpen = false;
863
+ this.utteranceEnded = false;
864
+ this.chunks = [];
865
+ this.transcript = "";
866
+ this.log(`Transcript: "${transcript}"`);
867
+ this.emit("Transcript", transcript);
868
+ if (this.goingAway) {
869
+ this.goingAway = false;
870
+ this.socket.renew();
871
+ }
872
+ }
873
+ clearTranscriptionTimeout() {
874
+ if (!this.transcriptionTimeout) return;
875
+ clearTimeout(this.transcriptionTimeout);
876
+ this.transcriptionTimeout = void 0;
877
+ }
878
+ };
879
+
880
+ // src/GeminiTTS.ts
881
+ var import_genai2 = require("@google/genai");
882
+ var import_server4 = require("@micdrop/server");
883
+ var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
884
+ var DEFAULT_VOICE = "Kore";
885
+ var GEMINI_SAMPLE_RATE = 24e3;
886
+ var OUTPUT_SAMPLE_RATE = 16e3;
887
+ var DEFAULT_MAX_RETRY2 = 2;
888
+ var GeminiTTS = class extends import_server4.SentenceTTS {
889
+ constructor(options) {
890
+ super();
891
+ this.options = options;
892
+ this.genai = "genai" in options ? options.genai : new import_genai2.GoogleGenAI({ apiKey: options.apiKey });
893
+ }
894
+ async synthesize(text, signal) {
895
+ const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
896
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
897
+ for (let attempt = 0; ; attempt++) {
898
+ const progress = { emitted: false };
899
+ try {
900
+ await this.synthesizeOnce(input, signal, progress);
901
+ return void 0;
902
+ } catch (error) {
903
+ if (signal.aborted || progress.emitted || attempt >= maxRetry) {
904
+ throw error;
905
+ }
906
+ if (!isBlocked(error)) throw error;
907
+ this.log(`Sentence blocked, trying again: "${text}"`);
908
+ }
909
+ }
910
+ }
911
+ async synthesizeOnce(input, signal, progress) {
912
+ const stream = await this.genai.interactions.create(
913
+ {
914
+ model: this.options.model || DEFAULT_MODEL4,
915
+ store: false,
916
+ stream: true,
917
+ input,
918
+ response_format: { type: "audio" },
919
+ generation_config: {
920
+ speech_config: [{ voice: this.options.voice || DEFAULT_VOICE }]
921
+ }
922
+ },
923
+ { fetchOptions: { signal } }
924
+ );
925
+ let resampler;
926
+ for await (const event of stream) {
927
+ if (signal.aborted) break;
928
+ if (event.event_type === "error") {
929
+ throw Object.assign(
930
+ new Error(event.error?.message || "Speech generation failed"),
931
+ { code: event.error?.code }
932
+ );
933
+ }
934
+ if (event.event_type !== "step.delta" || event.delta.type !== "audio") {
935
+ continue;
936
+ }
937
+ const { data, sample_rate } = event.delta;
938
+ if (!data) continue;
939
+ resampler ?? (resampler = new import_server4.Pcm16Resampler(
940
+ sample_rate || GEMINI_SAMPLE_RATE,
941
+ OUTPUT_SAMPLE_RATE
942
+ ));
943
+ const audio = resampler.process(Buffer.from(data, "base64"));
944
+ if (audio.length > 0) progress.emitted = true;
945
+ if (!this.emitAudio(audio)) break;
946
+ }
947
+ }
948
+ };
949
+ function isBlocked(error) {
950
+ return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
951
+ }
952
+ // Annotate the CommonJS export names for ESM import in node:
953
+ 0 && (module.exports = {
954
+ GeminiAgent,
955
+ GeminiLive,
956
+ GeminiSTT,
957
+ GeminiTTS
958
+ });
959
+ //# sourceMappingURL=index.js.map