@micdrop/gemini 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -54,127 +54,139 @@ var GeminiAgent = class extends import_server.Agent {
54
54
  this.signatures = /* @__PURE__ */ new Map();
55
55
  this.genai = "genai" in options ? options.genai : new import_genai.GoogleGenAI({ apiKey: options.apiKey });
56
56
  }
57
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
57
+ generateAnswer(stream, signal) {
58
+ return this.generateStep(stream, signal, 0);
59
+ }
60
+ /**
61
+ * Generates one step of the answer, retried on error.
62
+ *
63
+ * A retry starts the step over, so it only happens while nothing of the
64
+ * step reached the stream: what was said cannot be taken back, and saying
65
+ * it twice is worse than stopping there.
66
+ */
67
+ async generateStep(stream, signal, stepCount) {
58
68
  if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
59
69
  console.error("[GeminiAgent] Max steps reached");
60
70
  return;
61
71
  }
62
- const abortController = new AbortController();
63
- this.abortController = abortController;
64
- const signal = abortController.signal;
72
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
73
+ for (let tryCount = 0; ; tryCount++) {
74
+ const progress = { answer: "" };
75
+ try {
76
+ const step = await this.streamStep(stream, signal, progress);
77
+ if (signal.aborted) return;
78
+ this.recordAnswer(progress.answer);
79
+ if (step.calledTool && !step.skipAnswer && !progress.answer) {
80
+ await this.generateStep(stream, signal, stepCount + 1);
81
+ }
82
+ return;
83
+ } catch (error) {
84
+ if (signal.aborted) return;
85
+ console.error("[GeminiAgent] Error answering:", error);
86
+ if (progress.answer) {
87
+ this.log("Error after part of the answer was said, keeping it");
88
+ this.recordAnswer(progress.answer);
89
+ return;
90
+ }
91
+ if (tryCount >= maxRetry) {
92
+ this.log("Max retries reached, giving up");
93
+ this.emit("Failed");
94
+ return;
95
+ }
96
+ await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
97
+ if (signal.aborted) return;
98
+ }
99
+ }
100
+ }
101
+ /** Streams one request, writing the text of the answer as it comes */
102
+ async streamStep(stream, signal, progress) {
65
103
  let extracting = false;
66
104
  const extractOptions = this.getExtractOptions();
67
- try {
68
- const response = await this.genai.interactions.create(
69
- {
70
- model: this.options.model || DEFAULT_MODEL,
71
- store: false,
72
- stream: true,
73
- system_instruction: this.buildSystemInstruction(),
74
- input: this.buildInput(),
75
- tools: this.buildTools(),
76
- ...this.options.thinkingLevel || this.options.settings ? {
77
- generation_config: {
78
- ...this.options.settings,
79
- ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
80
- }
81
- } : {}
82
- },
83
- { fetchOptions: { signal } }
84
- );
85
- const steps = [];
86
- let answer = "";
87
- let skipAnswer = false;
88
- let calledTool = false;
89
- let signature = "";
90
- for await (const event of response) {
91
- if (abortController !== this.abortController) return;
92
- switch (event.event_type) {
93
- case "step.start":
94
- steps[event.index] = {
95
- type: event.step.type,
96
- id: event.step.id,
97
- name: event.step.name,
98
- text: "",
99
- arguments: "",
100
- signature: ""
101
- };
102
- break;
103
- case "step.delta": {
104
- const step = steps[event.index];
105
- if (!step) break;
106
- const delta = event.delta;
107
- if (delta.type === "thought_signature") {
108
- step.signature += delta.signature ?? "";
109
- } else if (delta.type === "arguments_delta") {
110
- step.arguments += delta.arguments ?? "";
111
- } else if (delta.type === "text" && step.type === "model_output") {
112
- this.log(`Answer chunk: "${delta.text}"`);
113
- answer += delta.text;
114
- if (extractOptions) {
115
- if (extracting) break;
116
- const startTagIndex = delta.text.indexOf(
117
- extractOptions.startTag
118
- );
119
- if (startTagIndex !== -1) {
120
- extracting = true;
121
- stream.write(delta.text.slice(0, startTagIndex).trimEnd());
122
- break;
123
- }
105
+ const response = await this.genai.interactions.create(
106
+ {
107
+ model: this.options.model || DEFAULT_MODEL,
108
+ store: false,
109
+ stream: true,
110
+ system_instruction: this.buildSystemInstruction(),
111
+ input: this.buildInput(),
112
+ tools: this.buildTools(),
113
+ ...this.options.thinkingLevel || this.options.settings ? {
114
+ generation_config: {
115
+ ...this.options.settings,
116
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
117
+ }
118
+ } : {}
119
+ },
120
+ { signal }
121
+ );
122
+ const steps = [];
123
+ let skipAnswer = false;
124
+ let calledTool = false;
125
+ let signature = "";
126
+ for await (const event of response) {
127
+ if (signal.aborted) break;
128
+ switch (event.event_type) {
129
+ case "step.start":
130
+ steps[event.index] = {
131
+ type: event.step.type,
132
+ id: event.step.id,
133
+ name: event.step.name,
134
+ text: "",
135
+ arguments: "",
136
+ signature: ""
137
+ };
138
+ break;
139
+ case "step.delta": {
140
+ const step = steps[event.index];
141
+ if (!step) break;
142
+ const delta = event.delta;
143
+ if (delta.type === "thought_signature") {
144
+ step.signature += delta.signature ?? "";
145
+ } else if (delta.type === "arguments_delta") {
146
+ step.arguments += delta.arguments ?? "";
147
+ } else if (delta.type === "text" && step.type === "model_output") {
148
+ this.log(`Answer chunk: "${delta.text}"`);
149
+ progress.answer += delta.text;
150
+ if (extractOptions) {
151
+ if (extracting) break;
152
+ const startTagIndex = delta.text.indexOf(extractOptions.startTag);
153
+ if (startTagIndex !== -1) {
154
+ extracting = true;
155
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
156
+ break;
124
157
  }
125
- stream.write(delta.text);
126
158
  }
127
- break;
159
+ stream.write(delta.text);
128
160
  }
129
- case "step.stop": {
130
- const step = steps[event.index];
131
- if (step?.type === "thought") signature = step.signature;
132
- if (step?.type !== "function_call" || !step.id || !step.name) break;
133
- calledTool = true;
134
- if (signature) this.signatures.set(step.id, signature);
135
- const result = await this.executeTool({
136
- role: "tool_call",
137
- toolCallId: step.id,
138
- toolName: step.name,
139
- parameters: step.arguments || "{}"
140
- });
141
- if (result.skipAnswer) skipAnswer = true;
142
- break;
143
- }
144
- case "error":
145
- throw new Error(event.error?.message || "Generation failed");
146
- default:
147
- break;
161
+ break;
148
162
  }
149
- }
150
- if (abortController !== this.abortController) return;
151
- this.abortController = void 0;
152
- if (answer) {
153
- const { message, metadata } = this.extract(answer);
154
- this.addAssistantMessage(message, metadata);
155
- }
156
- if (calledTool && !skipAnswer && !answer) {
157
- await this.generateAnswer(stream, stepCount + 1);
158
- }
159
- } catch (error) {
160
- if (signal.aborted) return;
161
- console.error("[GeminiAgent] Error answering:", error);
162
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
163
- await new Promise(
164
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
165
- );
166
- await this.generateAnswer(stream, stepCount, tryCount + 1);
167
- } else {
168
- this.log("Max retries reached, giving up");
169
- this.emit("Failed");
163
+ case "step.stop": {
164
+ const step = steps[event.index];
165
+ if (step?.type === "thought") signature = step.signature;
166
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
167
+ calledTool = true;
168
+ if (signature) this.signatures.set(step.id, signature);
169
+ const result = await this.executeTool({
170
+ role: "tool_call",
171
+ toolCallId: step.id,
172
+ toolName: step.name,
173
+ parameters: step.arguments || "{}"
174
+ });
175
+ if (result.skipAnswer) skipAnswer = true;
176
+ break;
177
+ }
178
+ case "error":
179
+ throw new Error(event.error?.message || "Generation failed");
180
+ default:
181
+ break;
170
182
  }
171
183
  }
184
+ return { calledTool, skipAnswer };
172
185
  }
173
- cancel() {
174
- if (!this.abortController) return;
175
- this.log("Cancel");
176
- this.abortController.abort();
177
- this.abortController = void 0;
186
+ recordAnswer(answer) {
187
+ if (!answer) return;
188
+ const { message, metadata } = this.extract(answer);
189
+ this.addAssistantMessage(message, metadata);
178
190
  }
179
191
  buildSystemInstruction() {
180
192
  return this.conversation.flatMap(
@@ -246,6 +258,17 @@ var GeminiAgent = class extends import_server.Agent {
246
258
  });
247
259
  }
248
260
  };
261
+ function sleep(ms, signal) {
262
+ return new Promise((resolve) => {
263
+ const done = () => {
264
+ clearTimeout(timer);
265
+ signal.removeEventListener("abort", done);
266
+ resolve();
267
+ };
268
+ const timer = setTimeout(done, ms);
269
+ signal.addEventListener("abort", done);
270
+ });
271
+ }
249
272
  function parseArguments(parameters) {
250
273
  try {
251
274
  return JSON.parse(parameters);
@@ -255,71 +278,109 @@ function parseArguments(parameters) {
255
278
  }
256
279
 
257
280
  // src/GeminiLive.ts
258
- var import_server2 = require("@micdrop/server");
281
+ var import_server3 = require("@micdrop/server");
259
282
  var import_zod2 = require("zod");
260
283
 
261
284
  // src/LiveSocket.ts
285
+ var import_server2 = require("@micdrop/server");
262
286
  var import_eventemitter3 = require("eventemitter3");
263
- var import_ws = __toESM(require("ws"));
264
287
  var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
265
288
  var SAMPLE_RATE = 16e3;
266
289
  var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
267
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
268
- var DEFAULT_RETRY_DELAY2 = 1e3;
269
- var DEFAULT_MAX_RETRY = 3;
270
290
  var LiveSocket = class extends import_eventemitter3.EventEmitter {
271
291
  constructor(options, buildSetup, log) {
272
292
  super();
273
- this.options = options;
274
293
  this.buildSetup = buildSetup;
275
294
  this.log = log;
276
- this.connected = false;
277
- this.destroyed = false;
278
295
  // Messages waiting for the session, or for the end of a turn to be due
279
296
  this.pending = [];
280
297
  // Audio of the turn as it went out, see flush()
281
298
  this.turnAudioStartedAt = 0;
282
299
  this.turnAudioMs = 0;
283
- this.retryCount = 0;
300
+ // Attempts since the last session was ready, for the owner to tell a drop
301
+ // from a resumption that keeps failing
302
+ this.reconnectAttempts = 0;
303
+ /** Sends the setup, and holds the connection back until Gemini accepts it */
304
+ this.handshake = {
305
+ start: (socket) => {
306
+ const setup = this.buildSetup();
307
+ this.setup = setup;
308
+ socket.send(JSON.stringify(setup));
309
+ },
310
+ read: (data) => !!parseMessage(data.toString())?.setupComplete
311
+ };
312
+ this.onOpen = () => {
313
+ this.log("Session ready");
314
+ this.reconnectAttempts = 0;
315
+ this.emit("Ready", this.setup);
316
+ this.flush();
317
+ };
318
+ this.onMessage = (data) => {
319
+ const message = parseMessage(data.toString());
320
+ if (message) this.emit("Message", message);
321
+ };
322
+ this.onStatusChange = (status) => {
323
+ if (status !== "open") this.clearPendingTimeout();
324
+ if (status === "reconnecting") {
325
+ this.emit("Reconnecting", ++this.reconnectAttempts);
326
+ }
327
+ };
328
+ this.onFailed = () => {
329
+ this.clearPending();
330
+ this.emit("Failed");
331
+ };
332
+ this.socket = new import_server2.ReconnectingSocket({
333
+ url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
334
+ connectionTimeout: options.connectionTimeout,
335
+ retryDelay: options.retryDelay,
336
+ maxRetry: options.maxRetry,
337
+ handshake: this.handshake,
338
+ log
339
+ });
340
+ this.socket.on("Open", this.onOpen);
341
+ this.socket.on("Message", this.onMessage);
342
+ this.socket.on("StatusChange", this.onStatusChange);
343
+ this.socket.on("Failed", this.onFailed);
284
344
  }
285
345
  /** The session is set up and takes messages right away */
286
346
  get isConnected() {
287
- return this.connected;
347
+ return this.socket.isOpen;
288
348
  }
289
349
  connect() {
290
- this.initWS().catch((error) => {
291
- this.log("Connection error:", error);
292
- this.reconnect();
293
- });
350
+ this.socket.connect();
294
351
  }
295
- /** Sends a message once the session is ready, after the ones before it */
352
+ /**
353
+ * Sends a message once the session is ready, after the ones before it.
354
+ * Dropped once the connection failed for good, as nothing would send it.
355
+ */
296
356
  send(message) {
357
+ const status = this.socket.status;
358
+ if (status === "failed" || status === "destroyed") return;
297
359
  this.pending.push(message);
298
360
  this.flush();
299
361
  }
300
362
  /** Sends a message ahead of the waiting ones, from a Ready listener */
301
363
  sendFirst(message) {
302
- this.socket?.send(JSON.stringify(message));
364
+ this.socket.send(JSON.stringify(message));
303
365
  }
304
366
  /** Drops the messages that have not gone out */
305
367
  clearPending() {
306
368
  this.pending = [];
307
369
  this.clearPendingTimeout();
308
370
  }
309
- /** Moves to a new connection, keeping the messages that have not gone out */
371
+ /**
372
+ * Moves to a new connection, keeping the messages that have not gone out.
373
+ * A reconnection waiting for its delay is replaced by this one.
374
+ */
310
375
  renew() {
311
376
  this.log("Renewing the connection");
312
- this.closeSocket();
313
- this.connect();
377
+ this.clearPendingTimeout();
378
+ this.reconnectAttempts = 0;
379
+ this.socket.restart();
314
380
  }
315
381
  destroy() {
316
- this.destroyed = true;
317
- if (this.reconnectTimeout) {
318
- clearTimeout(this.reconnectTimeout);
319
- this.reconnectTimeout = void 0;
320
- }
321
- this.closeSocket();
322
- this.pending = [];
382
+ this.clearPending();
383
+ this.socket.destroy();
323
384
  this.removeAllListeners();
324
385
  }
325
386
  /**
@@ -332,8 +393,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
332
393
  */
333
394
  flush() {
334
395
  while (this.pending.length > 0 && !this.pendingTimeout) {
335
- const socket = this.socket;
336
- if (!this.connected || socket?.readyState !== import_ws.default.OPEN) return;
396
+ if (!this.socket.isOpen) return;
337
397
  const message = this.pending[0];
338
398
  const input = message.realtimeInput;
339
399
  if (input?.activityEnd) {
@@ -353,7 +413,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
353
413
  this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
354
414
  }
355
415
  this.pending.shift();
356
- socket.send(JSON.stringify(message));
416
+ this.socket.send(JSON.stringify(message));
357
417
  }
358
418
  }
359
419
  clearPendingTimeout() {
@@ -361,112 +421,46 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
361
421
  clearTimeout(this.pendingTimeout);
362
422
  this.pendingTimeout = void 0;
363
423
  }
364
- initWS() {
365
- return new Promise((resolve, reject) => {
366
- const socket = new import_ws.default(
367
- `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
368
- );
369
- this.socket = socket;
370
- const timeout = setTimeout(() => {
371
- this.log("Connection timeout");
372
- this.closeSocket();
373
- reject(new Error("WebSocket connection timeout"));
374
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
375
- socket.on("open", () => {
376
- this.log("Connection opened");
377
- socket.send(JSON.stringify(this.buildSetup()));
378
- });
379
- socket.on("error", (error) => {
380
- clearTimeout(timeout);
381
- this.log("WebSocket error:", error);
382
- reject(new Error("WebSocket connection error"));
383
- });
384
- socket.on("close", (code, reason) => {
385
- clearTimeout(timeout);
386
- const wasConnected = this.connected;
387
- this.closeSocket();
388
- if (this.destroyed) return;
389
- this.log("Connection closed", { code, reason: reason.toString() });
390
- if (wasConnected) {
391
- this.reconnect();
392
- } else {
393
- reject(new Error(`Connection closed: ${reason.toString()}`));
394
- }
395
- });
396
- socket.on("message", (data) => {
397
- const message = JSON.parse(data.toString());
398
- if (message.setupComplete) {
399
- clearTimeout(timeout);
400
- this.log("Session ready");
401
- this.connected = true;
402
- this.retryCount = 0;
403
- this.emit("Ready");
404
- this.flush();
405
- resolve();
406
- return;
407
- }
408
- this.emit("Message", message);
409
- });
410
- });
411
- }
412
- closeSocket() {
413
- const socket = this.socket;
414
- this.socket = void 0;
415
- this.connected = false;
416
- this.clearPendingTimeout();
417
- if (!socket) return;
418
- socket.removeAllListeners();
419
- socket.on("error", () => {
420
- });
421
- if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
422
- socket.close(1e3);
423
- }
424
- }
425
- reconnect() {
426
- if (this.destroyed) return;
427
- this.retryCount++;
428
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
429
- this.log("Max retries reached, giving up");
430
- this.emit("Failed");
431
- return;
432
- }
433
- this.emit("Reconnecting", this.retryCount);
434
- this.log("Reconnecting...");
435
- this.reconnectTimeout = setTimeout(() => {
436
- this.reconnectTimeout = void 0;
437
- this.connect();
438
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
439
- }
440
424
  };
425
+ function parseMessage(text) {
426
+ try {
427
+ return JSON.parse(text);
428
+ } catch {
429
+ return void 0;
430
+ }
431
+ }
441
432
 
442
433
  // src/GeminiLive.ts
443
434
  var DEFAULT_MODEL2 = "gemini-3.8-live";
444
435
  var SAMPLE_RATE2 = 16e3;
445
436
  var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
446
437
  var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
438
+ function newToolRound() {
439
+ return { running: 0, answerEndDeferred: false };
440
+ }
447
441
  var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
448
- var GeminiLive = class extends import_server2.Realtime {
442
+ var GeminiLive = class extends import_server3.Realtime {
449
443
  constructor(options) {
450
444
  super(options);
451
445
  this.everConnected = false;
452
- // Whether the setup being sent resumes the session
453
- this.resuming = false;
454
446
  this.goingAway = false;
455
- this.outputResampler = new import_server2.Pcm16Resampler(
447
+ this.outputResampler = new import_server3.Pcm16Resampler(
456
448
  GEMINI_OUTPUT_SAMPLE_RATE,
457
449
  SAMPLE_RATE2
458
450
  );
459
451
  this.hasTranscribedTurn = false;
460
452
  // Transcript of a dropped turn, which the conversation should not show
461
453
  this.ignoreUserTranscript = false;
454
+ // Calls of the session Gemini told not to answer
462
455
  this.cancelledToolCalls = /* @__PURE__ */ new Set();
463
- // Gemini ends its turn when it calls a tool, and speaks again once it has
464
- // the result. The answer stays open in between, so the client keeps waiting
465
- // for it rather than listening again.
466
- this.runningTools = 0;
467
- this.answerEndDeferred = false;
468
- this.onReady = () => {
469
- if (this.everConnected && !this.resuming) {
456
+ // Bumped by every session that does not resume the previous one, whose tool
457
+ // calls it cannot answer
458
+ this.session = 0;
459
+ this.toolRound = newToolRound();
460
+ this.onReady = (setup) => {
461
+ const resumed = !!setup?.setup?.sessionResumption?.handle;
462
+ if (this.everConnected && !resumed) {
463
+ this.startFreshSession();
470
464
  const turns = this.conversation.flatMap(
471
465
  (message) => message.role === "user" || message.role === "assistant" ? [
472
466
  {
@@ -484,7 +478,12 @@ var GeminiLive = class extends import_server2.Realtime {
484
478
  };
485
479
  this.onReconnecting = (attempt) => {
486
480
  if (attempt > 1) this.resumptionHandle = void 0;
487
- if (!this.resumptionHandle) this.socket.clearPending();
481
+ if (!this.resumptionHandle) {
482
+ this.socket.clearPending();
483
+ this.startFreshSession();
484
+ }
485
+ this.toolRound.answerEndDeferred = false;
486
+ this.commitTurnTranscript();
488
487
  this.endAnswer();
489
488
  };
490
489
  this.onMessage = (message) => {
@@ -522,9 +521,16 @@ var GeminiLive = class extends import_server2.Realtime {
522
521
  this.socket.on("Failed", () => this.emit("Failed"));
523
522
  this.socket.connect();
524
523
  }
524
+ /**
525
+ * Stops the answer. The tools it is waiting for still send their results,
526
+ * silently, since a blocking call left unanswered would stall the model.
527
+ */
528
+ cancel() {
529
+ super.cancel();
530
+ this.resetToolRound();
531
+ }
525
532
  destroy() {
526
533
  super.destroy();
527
- this.clearToolFollowUp();
528
534
  this.socket.destroy();
529
535
  }
530
536
  openTurn() {
@@ -559,11 +565,9 @@ var GeminiLive = class extends import_server2.Realtime {
559
565
  }
560
566
  cancelAnswer() {
561
567
  this.outputResampler.reset();
562
- this.clearToolFollowUp();
563
- this.answerEndDeferred = false;
564
568
  }
565
569
  get isIdle() {
566
- return super.isIdle && this.runningTools === 0;
570
+ return super.isIdle && this.toolRound.running === 0;
567
571
  }
568
572
  updateTools() {
569
573
  if (this.everConnected) {
@@ -595,7 +599,6 @@ var GeminiLive = class extends import_server2.Realtime {
595
599
  this.socket.send(message);
596
600
  }
597
601
  buildSetup() {
598
- this.resuming = !!this.resumptionHandle;
599
602
  const functionDeclarations = this.tools.map((tool) => ({
600
603
  name: tool.name,
601
604
  description: tool.description,
@@ -629,6 +632,22 @@ var GeminiLive = class extends import_server2.Realtime {
629
632
  }
630
633
  };
631
634
  }
635
+ /** Forgets what belonged to the previous session */
636
+ startFreshSession() {
637
+ this.session++;
638
+ this.resetToolRound();
639
+ this.cancelledToolCalls.clear();
640
+ this.ignoreUserTranscript = false;
641
+ }
642
+ /**
643
+ * Starts a new round of tools. The tools still running belong to the old
644
+ * one: their results go out silently, and nothing waits for the model to
645
+ * talk about them.
646
+ */
647
+ resetToolRound() {
648
+ if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
649
+ this.toolRound = newToolRound();
650
+ }
632
651
  /** Moves to a new connection before Gemini closes this one */
633
652
  renewIfGoingAway() {
634
653
  if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
@@ -660,8 +679,8 @@ var GeminiLive = class extends import_server2.Realtime {
660
679
  if (content.turnComplete) {
661
680
  this.commitTurnTranscript();
662
681
  this.outputResampler.reset();
663
- if (this.runningTools > 0 || this.toolFollowUp) {
664
- this.answerEndDeferred = true;
682
+ if (this.toolRound.running > 0 || this.toolRound.followUp) {
683
+ this.toolRound.answerEndDeferred = true;
665
684
  } else {
666
685
  this.endAnswer();
667
686
  this.renewIfGoingAway();
@@ -670,24 +689,22 @@ var GeminiLive = class extends import_server2.Realtime {
670
689
  }
671
690
  onModelOutput() {
672
691
  this.commitTurnTranscript();
673
- if (this.toolFollowUp) {
674
- this.clearToolFollowUp();
675
- this.answerEndDeferred = false;
692
+ const round = this.toolRound;
693
+ if (round.followUp) {
694
+ clearTimeout(round.followUp);
695
+ round.followUp = void 0;
696
+ round.answerEndDeferred = false;
676
697
  }
677
698
  }
678
699
  /** Ends the answer the model stopped giving to wait for a tool */
679
700
  endDeferredAnswer() {
680
- if (!this.answerEndDeferred) return;
681
- if (this.runningTools > 0 || this.toolFollowUp) return;
682
- this.answerEndDeferred = false;
701
+ const round = this.toolRound;
702
+ if (!round.answerEndDeferred) return;
703
+ if (round.running > 0 || round.followUp) return;
704
+ round.answerEndDeferred = false;
683
705
  this.endAnswer();
684
706
  this.renewIfGoingAway();
685
707
  }
686
- clearToolFollowUp() {
687
- if (!this.toolFollowUp) return;
688
- clearTimeout(this.toolFollowUp);
689
- this.toolFollowUp = void 0;
690
- }
691
708
  /**
692
709
  * The transcript of the turn comes before the answer, so the first piece of
693
710
  * the answer closes it.
@@ -705,17 +722,22 @@ var GeminiLive = class extends import_server2.Realtime {
705
722
  toolName: call.name,
706
723
  parameters: JSON.stringify(call.args ?? {})
707
724
  };
708
- this.runningTools++;
725
+ const round = this.toolRound;
726
+ const session = this.session;
727
+ round.running++;
709
728
  let result;
710
729
  try {
711
730
  result = await this.runTool(toolCall);
712
731
  } finally {
713
- this.runningTools--;
732
+ round.running--;
714
733
  }
734
+ if (session !== this.session) return;
715
735
  if (this.cancelledToolCalls.delete(call.id)) {
716
736
  this.endDeferredAnswer();
717
737
  return;
718
738
  }
739
+ const interrupted = round !== this.toolRound;
740
+ const silent = result.skipAnswer || interrupted;
719
741
  this.send({
720
742
  toolResponse: {
721
743
  functionResponses: [
@@ -725,16 +747,17 @@ var GeminiLive = class extends import_server2.Realtime {
725
747
  response: {
726
748
  output: result.output ?? null,
727
749
  // A tool without an answer lets the model know silently
728
- scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
750
+ scheduling: silent ? "SILENT" : "WHEN_IDLE"
729
751
  }
730
752
  }
731
753
  ]
732
754
  }
733
755
  });
756
+ if (interrupted) return;
734
757
  if (!result.skipAnswer) {
735
- this.clearToolFollowUp();
736
- this.toolFollowUp = setTimeout(() => {
737
- this.toolFollowUp = void 0;
758
+ if (round.followUp) clearTimeout(round.followUp);
759
+ round.followUp = setTimeout(() => {
760
+ round.followUp = void 0;
738
761
  this.log("Nothing said about the tool result");
739
762
  this.endDeferredAnswer();
740
763
  }, TOOL_FOLLOW_UP_TIMEOUT);
@@ -745,79 +768,85 @@ var GeminiLive = class extends import_server2.Realtime {
745
768
  };
746
769
 
747
770
  // src/GeminiSTT.ts
748
- var import_server3 = require("@micdrop/server");
771
+ var import_server4 = require("@micdrop/server");
749
772
  var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
750
773
  var SAMPLE_RATE3 = 16e3;
751
774
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
752
- var GeminiSTT = class extends import_server3.STT {
775
+ var GeminiSTT = class extends import_server4.STT {
753
776
  constructor(options) {
754
777
  super();
755
778
  this.options = options;
779
+ // Gemini closes the session soon, and it is renewed once nothing waits
756
780
  this.goingAway = false;
757
- // Audio of the utterance, sent again if the connection drops before its
758
- // transcript comes back
759
- this.chunks = [];
760
- this.utteranceOpen = false;
761
- this.utteranceEnded = false;
762
- this.transcript = "";
781
+ // Utterances waiting for their transcript, in the order they were spoken.
782
+ // Gemini names no utterance in what it sends, and goes on to the next one
783
+ // before it is done with the previous. So it hears one at a time: the first
784
+ // one, and the audio of the next waits here until its transcript is in.
785
+ this.utterances = [];
786
+ this.onReady = () => {
787
+ this.goingAway = false;
788
+ };
763
789
  this.onMessage = (message) => {
764
790
  const content = message.serverContent;
765
- if (content?.inputTranscription?.text && this.utteranceOpen) {
766
- this.transcript += content.inputTranscription.text;
791
+ const utterance = this.utterances[0];
792
+ if (content?.inputTranscription?.text && utterance) {
793
+ utterance.transcript += content.inputTranscription.text;
767
794
  }
768
- if (content?.generationComplete && this.utteranceEnded) {
769
- this.finishUtterance();
795
+ if (content?.generationComplete && utterance) {
796
+ if (utterance.state === "ended") this.give(utterance);
797
+ if (utterance.state === "given") this.remove(utterance);
770
798
  }
771
799
  if (message.goAway) {
772
800
  this.log("Connection ending in", message.goAway.timeLeft);
773
- if (this.utteranceOpen) {
774
- this.goingAway = true;
775
- } else {
776
- this.socket.renew();
777
- }
801
+ this.goingAway = true;
802
+ this.renewIfGoingAway();
778
803
  }
779
804
  };
780
805
  this.onReconnecting = () => {
806
+ this.goingAway = false;
781
807
  this.socket.clearPending();
782
- if (!this.utteranceOpen) return;
783
- this.transcript = "";
784
- this.socket.send({ realtimeInput: { activityStart: {} } });
785
- this.chunks.forEach((chunk) => this.sendAudio(chunk));
786
- if (this.utteranceEnded) {
787
- this.socket.send({ realtimeInput: { activityEnd: {} } });
808
+ for (const utterance of this.utterances) {
809
+ if (utterance.state === "given") this.remove(utterance);
810
+ }
811
+ for (const utterance of this.utterances) {
812
+ if (utterance.timeout) clearTimeout(utterance.timeout);
813
+ utterance.timeout = void 0;
814
+ utterance.sent = false;
815
+ utterance.transcript = "";
788
816
  }
817
+ this.sendFirst();
818
+ };
819
+ this.onFailed = () => {
820
+ const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
821
+ this.dropUtterances();
822
+ this.emit("Failed", chunks);
789
823
  };
790
824
  this.socket = new LiveSocket(
791
825
  options,
792
826
  () => this.buildSetup(),
793
827
  (...message) => this.log(...message)
794
828
  );
829
+ this.socket.on("Ready", this.onReady);
795
830
  this.socket.on("Message", this.onMessage);
796
831
  this.socket.on("Reconnecting", this.onReconnecting);
797
- this.socket.on("Failed", () => this.emit("Failed", this.chunks));
832
+ this.socket.on("Failed", this.onFailed);
798
833
  this.socket.connect();
799
834
  }
800
835
  transcribe(audioStream) {
836
+ let utterance;
801
837
  audioStream.on("data", (chunk) => {
802
- if (!this.utteranceOpen) this.openUtterance();
803
- this.chunks.push(chunk);
804
- this.sendAudio(chunk);
838
+ if (!utterance) utterance = this.openUtterance();
839
+ if (utterance.state !== "open") return;
840
+ utterance.chunks.push(chunk);
841
+ if (utterance.sent) this.sendAudio(chunk);
805
842
  });
806
843
  audioStream.on("end", () => {
807
- if (!this.utteranceOpen || this.utteranceEnded) return;
808
- this.utteranceEnded = true;
809
- this.socket.send({ realtimeInput: { activityEnd: {} } });
810
- this.clearTranscriptionTimeout();
811
- this.transcriptionTimeout = setTimeout(() => {
812
- this.transcriptionTimeout = void 0;
813
- this.log("Transcription timeout");
814
- this.finishUtterance();
815
- }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
844
+ if (utterance?.state === "open") this.endUtterance(utterance);
816
845
  });
817
846
  }
818
847
  destroy() {
819
848
  super.destroy();
820
- this.clearTranscriptionTimeout();
849
+ this.dropUtterances();
821
850
  this.socket.destroy();
822
851
  }
823
852
  buildSetup() {
@@ -838,12 +867,38 @@ var GeminiSTT = class extends import_server3.STT {
838
867
  };
839
868
  }
840
869
  openUtterance() {
841
- if (this.utteranceOpen) this.finishUtterance();
842
- this.utteranceOpen = true;
843
- this.utteranceEnded = false;
844
- this.chunks = [];
845
- this.transcript = "";
870
+ const last = this.utterances[this.utterances.length - 1];
871
+ if (last?.state === "open") this.endUtterance(last);
872
+ const utterance = {
873
+ state: "open",
874
+ sent: false,
875
+ chunks: [],
876
+ transcript: ""
877
+ };
878
+ this.utterances.push(utterance);
879
+ this.sendFirst();
880
+ return utterance;
881
+ }
882
+ endUtterance(utterance) {
883
+ utterance.state = "ended";
884
+ if (utterance.sent) this.sendEnd(utterance);
885
+ }
886
+ /** Sends the first utterance to Gemini, with the audio it held meanwhile */
887
+ sendFirst() {
888
+ const utterance = this.utterances[0];
889
+ if (!utterance || utterance.sent) return;
890
+ utterance.sent = true;
846
891
  this.socket.send({ realtimeInput: { activityStart: {} } });
892
+ utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
893
+ if (utterance.state === "ended") this.sendEnd(utterance);
894
+ }
895
+ sendEnd(utterance) {
896
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
897
+ utterance.timeout = setTimeout(() => {
898
+ utterance.timeout = void 0;
899
+ this.log("Transcription timeout");
900
+ this.give(utterance);
901
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
847
902
  }
848
903
  sendAudio(chunk) {
849
904
  this.socket.send({
@@ -855,37 +910,52 @@ var GeminiSTT = class extends import_server3.STT {
855
910
  }
856
911
  });
857
912
  }
858
- /** Emits what was transcribed of the utterance, possibly nothing */
859
- finishUtterance() {
860
- this.clearTranscriptionTimeout();
861
- const transcript = this.transcript.trim();
862
- this.utteranceOpen = false;
863
- this.utteranceEnded = false;
864
- this.chunks = [];
865
- this.transcript = "";
913
+ /**
914
+ * Emits what was transcribed of an utterance, possibly nothing, and keeps
915
+ * it in line for what Gemini still sends about it
916
+ */
917
+ give(utterance) {
918
+ if (utterance.timeout) clearTimeout(utterance.timeout);
919
+ utterance.state = "given";
920
+ const transcript = utterance.transcript.trim();
866
921
  this.log(`Transcript: "${transcript}"`);
867
922
  this.emit("Transcript", transcript);
868
- if (this.goingAway) {
869
- this.goingAway = false;
870
- this.socket.renew();
871
- }
923
+ utterance.timeout = setTimeout(
924
+ () => this.remove(utterance),
925
+ this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
926
+ );
872
927
  }
873
- clearTranscriptionTimeout() {
874
- if (!this.transcriptionTimeout) return;
875
- clearTimeout(this.transcriptionTimeout);
876
- this.transcriptionTimeout = void 0;
928
+ remove(utterance) {
929
+ if (utterance.timeout) clearTimeout(utterance.timeout);
930
+ utterance.timeout = void 0;
931
+ this.utterances = this.utterances.filter((other) => other !== utterance);
932
+ this.renewIfGoingAway();
933
+ this.sendFirst();
934
+ }
935
+ /** Moves to a new connection once no utterance waits on this one */
936
+ renewIfGoingAway() {
937
+ if (!this.goingAway || this.utterances.length > 0) return;
938
+ this.goingAway = false;
939
+ this.socket.renew();
940
+ }
941
+ dropUtterances() {
942
+ for (const utterance of this.utterances) {
943
+ if (utterance.timeout) clearTimeout(utterance.timeout);
944
+ if (utterance.state === "open") utterance.state = "ended";
945
+ }
946
+ this.utterances = [];
877
947
  }
878
948
  };
879
949
 
880
950
  // src/GeminiTTS.ts
881
951
  var import_genai2 = require("@google/genai");
882
- var import_server4 = require("@micdrop/server");
952
+ var import_server5 = require("@micdrop/server");
883
953
  var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
884
954
  var DEFAULT_VOICE = "Kore";
885
955
  var GEMINI_SAMPLE_RATE = 24e3;
886
956
  var OUTPUT_SAMPLE_RATE = 16e3;
887
- var DEFAULT_MAX_RETRY2 = 2;
888
- var GeminiTTS = class extends import_server4.SentenceTTS {
957
+ var DEFAULT_MAX_RETRY = 2;
958
+ var GeminiTTS = class extends import_server5.SentenceTTS {
889
959
  constructor(options) {
890
960
  super();
891
961
  this.options = options;
@@ -893,7 +963,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
893
963
  }
894
964
  async synthesize(text, signal) {
895
965
  const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
896
- const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
966
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
897
967
  for (let attempt = 0; ; attempt++) {
898
968
  const progress = { emitted: false };
899
969
  try {
@@ -936,7 +1006,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
936
1006
  }
937
1007
  const { data, sample_rate } = event.delta;
938
1008
  if (!data) continue;
939
- resampler ?? (resampler = new import_server4.Pcm16Resampler(
1009
+ resampler ?? (resampler = new import_server5.Pcm16Resampler(
940
1010
  sample_rate || GEMINI_SAMPLE_RATE,
941
1011
  OUTPUT_SAMPLE_RATE
942
1012
  ));
@@ -947,7 +1017,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
947
1017
  }
948
1018
  };
949
1019
  function isBlocked(error) {
950
- return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
1020
+ return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
951
1021
  }
952
1022
  // Annotate the CommonJS export names for ESM import in node:
953
1023
  0 && (module.exports = {