@micdrop/gemini 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15,127 +15,139 @@ var GeminiAgent = class extends Agent {
15
15
  this.signatures = /* @__PURE__ */ new Map();
16
16
  this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
17
17
  }
18
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
18
+ generateAnswer(stream, signal) {
19
+ return this.generateStep(stream, signal, 0);
20
+ }
21
+ /**
22
+ * Generates one step of the answer, retried on error.
23
+ *
24
+ * A retry starts the step over, so it only happens while nothing of the
25
+ * step reached the stream: what was said cannot be taken back, and saying
26
+ * it twice is worse than stopping there.
27
+ */
28
+ async generateStep(stream, signal, stepCount) {
19
29
  if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
20
30
  console.error("[GeminiAgent] Max steps reached");
21
31
  return;
22
32
  }
23
- const abortController = new AbortController();
24
- this.abortController = abortController;
25
- const signal = abortController.signal;
33
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
34
+ for (let tryCount = 0; ; tryCount++) {
35
+ const progress = { answer: "" };
36
+ try {
37
+ const step = await this.streamStep(stream, signal, progress);
38
+ if (signal.aborted) return;
39
+ this.recordAnswer(progress.answer);
40
+ if (step.calledTool && !step.skipAnswer && !progress.answer) {
41
+ await this.generateStep(stream, signal, stepCount + 1);
42
+ }
43
+ return;
44
+ } catch (error) {
45
+ if (signal.aborted) return;
46
+ console.error("[GeminiAgent] Error answering:", error);
47
+ if (progress.answer) {
48
+ this.log("Error after part of the answer was said, keeping it");
49
+ this.recordAnswer(progress.answer);
50
+ return;
51
+ }
52
+ if (tryCount >= maxRetry) {
53
+ this.log("Max retries reached, giving up");
54
+ this.emit("Failed");
55
+ return;
56
+ }
57
+ await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
58
+ if (signal.aborted) return;
59
+ }
60
+ }
61
+ }
62
+ /** Streams one request, writing the text of the answer as it comes */
63
+ async streamStep(stream, signal, progress) {
26
64
  let extracting = false;
27
65
  const extractOptions = this.getExtractOptions();
28
- try {
29
- const response = await this.genai.interactions.create(
30
- {
31
- model: this.options.model || DEFAULT_MODEL,
32
- store: false,
33
- stream: true,
34
- system_instruction: this.buildSystemInstruction(),
35
- input: this.buildInput(),
36
- tools: this.buildTools(),
37
- ...this.options.thinkingLevel || this.options.settings ? {
38
- generation_config: {
39
- ...this.options.settings,
40
- ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
41
- }
42
- } : {}
43
- },
44
- { fetchOptions: { signal } }
45
- );
46
- const steps = [];
47
- let answer = "";
48
- let skipAnswer = false;
49
- let calledTool = false;
50
- let signature = "";
51
- for await (const event of response) {
52
- if (abortController !== this.abortController) return;
53
- switch (event.event_type) {
54
- case "step.start":
55
- steps[event.index] = {
56
- type: event.step.type,
57
- id: event.step.id,
58
- name: event.step.name,
59
- text: "",
60
- arguments: "",
61
- signature: ""
62
- };
63
- break;
64
- case "step.delta": {
65
- const step = steps[event.index];
66
- if (!step) break;
67
- const delta = event.delta;
68
- if (delta.type === "thought_signature") {
69
- step.signature += delta.signature ?? "";
70
- } else if (delta.type === "arguments_delta") {
71
- step.arguments += delta.arguments ?? "";
72
- } else if (delta.type === "text" && step.type === "model_output") {
73
- this.log(`Answer chunk: "${delta.text}"`);
74
- answer += delta.text;
75
- if (extractOptions) {
76
- if (extracting) break;
77
- const startTagIndex = delta.text.indexOf(
78
- extractOptions.startTag
79
- );
80
- if (startTagIndex !== -1) {
81
- extracting = true;
82
- stream.write(delta.text.slice(0, startTagIndex).trimEnd());
83
- break;
84
- }
66
+ const response = await this.genai.interactions.create(
67
+ {
68
+ model: this.options.model || DEFAULT_MODEL,
69
+ store: false,
70
+ stream: true,
71
+ system_instruction: this.buildSystemInstruction(),
72
+ input: this.buildInput(),
73
+ tools: this.buildTools(),
74
+ ...this.options.thinkingLevel || this.options.settings ? {
75
+ generation_config: {
76
+ ...this.options.settings,
77
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
78
+ }
79
+ } : {}
80
+ },
81
+ { signal }
82
+ );
83
+ const steps = [];
84
+ let skipAnswer = false;
85
+ let calledTool = false;
86
+ let signature = "";
87
+ for await (const event of response) {
88
+ if (signal.aborted) break;
89
+ switch (event.event_type) {
90
+ case "step.start":
91
+ steps[event.index] = {
92
+ type: event.step.type,
93
+ id: event.step.id,
94
+ name: event.step.name,
95
+ text: "",
96
+ arguments: "",
97
+ signature: ""
98
+ };
99
+ break;
100
+ case "step.delta": {
101
+ const step = steps[event.index];
102
+ if (!step) break;
103
+ const delta = event.delta;
104
+ if (delta.type === "thought_signature") {
105
+ step.signature += delta.signature ?? "";
106
+ } else if (delta.type === "arguments_delta") {
107
+ step.arguments += delta.arguments ?? "";
108
+ } else if (delta.type === "text" && step.type === "model_output") {
109
+ this.log(`Answer chunk: "${delta.text}"`);
110
+ progress.answer += delta.text;
111
+ if (extractOptions) {
112
+ if (extracting) break;
113
+ const startTagIndex = delta.text.indexOf(extractOptions.startTag);
114
+ if (startTagIndex !== -1) {
115
+ extracting = true;
116
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
117
+ break;
85
118
  }
86
- stream.write(delta.text);
87
119
  }
88
- break;
89
- }
90
- case "step.stop": {
91
- const step = steps[event.index];
92
- if (step?.type === "thought") signature = step.signature;
93
- if (step?.type !== "function_call" || !step.id || !step.name) break;
94
- calledTool = true;
95
- if (signature) this.signatures.set(step.id, signature);
96
- const result = await this.executeTool({
97
- role: "tool_call",
98
- toolCallId: step.id,
99
- toolName: step.name,
100
- parameters: step.arguments || "{}"
101
- });
102
- if (result.skipAnswer) skipAnswer = true;
103
- break;
120
+ stream.write(delta.text);
104
121
  }
105
- case "error":
106
- throw new Error(event.error?.message || "Generation failed");
107
- default:
108
- break;
122
+ break;
109
123
  }
110
- }
111
- if (abortController !== this.abortController) return;
112
- this.abortController = void 0;
113
- if (answer) {
114
- const { message, metadata } = this.extract(answer);
115
- this.addAssistantMessage(message, metadata);
116
- }
117
- if (calledTool && !skipAnswer && !answer) {
118
- await this.generateAnswer(stream, stepCount + 1);
119
- }
120
- } catch (error) {
121
- if (signal.aborted) return;
122
- console.error("[GeminiAgent] Error answering:", error);
123
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
124
- await new Promise(
125
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
126
- );
127
- await this.generateAnswer(stream, stepCount, tryCount + 1);
128
- } else {
129
- this.log("Max retries reached, giving up");
130
- this.emit("Failed");
124
+ case "step.stop": {
125
+ const step = steps[event.index];
126
+ if (step?.type === "thought") signature = step.signature;
127
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
128
+ calledTool = true;
129
+ if (signature) this.signatures.set(step.id, signature);
130
+ const result = await this.executeTool({
131
+ role: "tool_call",
132
+ toolCallId: step.id,
133
+ toolName: step.name,
134
+ parameters: step.arguments || "{}"
135
+ });
136
+ if (result.skipAnswer) skipAnswer = true;
137
+ break;
138
+ }
139
+ case "error":
140
+ throw new Error(event.error?.message || "Generation failed");
141
+ default:
142
+ break;
131
143
  }
132
144
  }
145
+ return { calledTool, skipAnswer };
133
146
  }
134
- cancel() {
135
- if (!this.abortController) return;
136
- this.log("Cancel");
137
- this.abortController.abort();
138
- this.abortController = void 0;
147
+ recordAnswer(answer) {
148
+ if (!answer) return;
149
+ const { message, metadata } = this.extract(answer);
150
+ this.addAssistantMessage(message, metadata);
139
151
  }
140
152
  buildSystemInstruction() {
141
153
  return this.conversation.flatMap(
@@ -207,6 +219,17 @@ var GeminiAgent = class extends Agent {
207
219
  });
208
220
  }
209
221
  };
222
+ function sleep(ms, signal) {
223
+ return new Promise((resolve) => {
224
+ const done = () => {
225
+ clearTimeout(timer);
226
+ signal.removeEventListener("abort", done);
227
+ resolve();
228
+ };
229
+ const timer = setTimeout(done, ms);
230
+ signal.addEventListener("abort", done);
231
+ });
232
+ }
210
233
  function parseArguments(parameters) {
211
234
  try {
212
235
  return JSON.parse(parameters);
@@ -223,67 +246,107 @@ import {
223
246
  import { toJSONSchema as toJSONSchema2 } from "zod";
224
247
 
225
248
  // src/LiveSocket.ts
249
+ import {
250
+ ReconnectingSocket
251
+ } from "@micdrop/server";
226
252
  import { EventEmitter } from "eventemitter3";
227
- import WebSocket from "ws";
228
253
  var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
229
254
  var SAMPLE_RATE = 16e3;
230
255
  var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
231
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
232
- var DEFAULT_RETRY_DELAY2 = 1e3;
233
- var DEFAULT_MAX_RETRY = 3;
234
256
  var LiveSocket = class extends EventEmitter {
235
257
  constructor(options, buildSetup, log) {
236
258
  super();
237
- this.options = options;
238
259
  this.buildSetup = buildSetup;
239
260
  this.log = log;
240
- this.connected = false;
241
- this.destroyed = false;
242
261
  // Messages waiting for the session, or for the end of a turn to be due
243
262
  this.pending = [];
244
263
  // Audio of the turn as it went out, see flush()
245
264
  this.turnAudioStartedAt = 0;
246
265
  this.turnAudioMs = 0;
247
- this.retryCount = 0;
266
+ // Attempts since the last session was ready, for the owner to tell a drop
267
+ // from a resumption that keeps failing
268
+ this.reconnectAttempts = 0;
269
+ /** Sends the setup, and holds the connection back until Gemini accepts it */
270
+ this.handshake = {
271
+ start: (socket) => {
272
+ const setup = this.buildSetup();
273
+ this.setup = setup;
274
+ socket.send(JSON.stringify(setup));
275
+ },
276
+ read: (data) => !!parseMessage(data.toString())?.setupComplete
277
+ };
278
+ this.onOpen = () => {
279
+ this.log("Session ready");
280
+ this.reconnectAttempts = 0;
281
+ this.emit("Ready", this.setup);
282
+ this.flush();
283
+ };
284
+ this.onMessage = (data) => {
285
+ const message = parseMessage(data.toString());
286
+ if (message) this.emit("Message", message);
287
+ };
288
+ this.onStatusChange = (status) => {
289
+ if (status !== "open") this.clearPendingTimeout();
290
+ if (status === "reconnecting") {
291
+ this.emit("Reconnecting", ++this.reconnectAttempts);
292
+ }
293
+ };
294
+ this.onFailed = () => {
295
+ this.clearPending();
296
+ this.emit("Failed");
297
+ };
298
+ this.socket = new ReconnectingSocket({
299
+ url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
300
+ connectionTimeout: options.connectionTimeout,
301
+ retryDelay: options.retryDelay,
302
+ maxRetry: options.maxRetry,
303
+ handshake: this.handshake,
304
+ log
305
+ });
306
+ this.socket.on("Open", this.onOpen);
307
+ this.socket.on("Message", this.onMessage);
308
+ this.socket.on("StatusChange", this.onStatusChange);
309
+ this.socket.on("Failed", this.onFailed);
248
310
  }
249
311
  /** The session is set up and takes messages right away */
250
312
  get isConnected() {
251
- return this.connected;
313
+ return this.socket.isOpen;
252
314
  }
253
315
  connect() {
254
- this.initWS().catch((error) => {
255
- this.log("Connection error:", error);
256
- this.reconnect();
257
- });
316
+ this.socket.connect();
258
317
  }
259
- /** Sends a message once the session is ready, after the ones before it */
318
+ /**
319
+ * Sends a message once the session is ready, after the ones before it.
320
+ * Dropped once the connection failed for good, as nothing would send it.
321
+ */
260
322
  send(message) {
323
+ const status = this.socket.status;
324
+ if (status === "failed" || status === "destroyed") return;
261
325
  this.pending.push(message);
262
326
  this.flush();
263
327
  }
264
328
  /** Sends a message ahead of the waiting ones, from a Ready listener */
265
329
  sendFirst(message) {
266
- this.socket?.send(JSON.stringify(message));
330
+ this.socket.send(JSON.stringify(message));
267
331
  }
268
332
  /** Drops the messages that have not gone out */
269
333
  clearPending() {
270
334
  this.pending = [];
271
335
  this.clearPendingTimeout();
272
336
  }
273
- /** Moves to a new connection, keeping the messages that have not gone out */
337
+ /**
338
+ * Moves to a new connection, keeping the messages that have not gone out.
339
+ * A reconnection waiting for its delay is replaced by this one.
340
+ */
274
341
  renew() {
275
342
  this.log("Renewing the connection");
276
- this.closeSocket();
277
- this.connect();
343
+ this.clearPendingTimeout();
344
+ this.reconnectAttempts = 0;
345
+ this.socket.restart();
278
346
  }
279
347
  destroy() {
280
- this.destroyed = true;
281
- if (this.reconnectTimeout) {
282
- clearTimeout(this.reconnectTimeout);
283
- this.reconnectTimeout = void 0;
284
- }
285
- this.closeSocket();
286
- this.pending = [];
348
+ this.clearPending();
349
+ this.socket.destroy();
287
350
  this.removeAllListeners();
288
351
  }
289
352
  /**
@@ -296,8 +359,7 @@ var LiveSocket = class extends EventEmitter {
296
359
  */
297
360
  flush() {
298
361
  while (this.pending.length > 0 && !this.pendingTimeout) {
299
- const socket = this.socket;
300
- if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
362
+ if (!this.socket.isOpen) return;
301
363
  const message = this.pending[0];
302
364
  const input = message.realtimeInput;
303
365
  if (input?.activityEnd) {
@@ -317,7 +379,7 @@ var LiveSocket = class extends EventEmitter {
317
379
  this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
318
380
  }
319
381
  this.pending.shift();
320
- socket.send(JSON.stringify(message));
382
+ this.socket.send(JSON.stringify(message));
321
383
  }
322
384
  }
323
385
  clearPendingTimeout() {
@@ -325,96 +387,28 @@ var LiveSocket = class extends EventEmitter {
325
387
  clearTimeout(this.pendingTimeout);
326
388
  this.pendingTimeout = void 0;
327
389
  }
328
- initWS() {
329
- return new Promise((resolve, reject) => {
330
- const socket = new WebSocket(
331
- `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
332
- );
333
- this.socket = socket;
334
- const timeout = setTimeout(() => {
335
- this.log("Connection timeout");
336
- this.closeSocket();
337
- reject(new Error("WebSocket connection timeout"));
338
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
339
- socket.on("open", () => {
340
- this.log("Connection opened");
341
- socket.send(JSON.stringify(this.buildSetup()));
342
- });
343
- socket.on("error", (error) => {
344
- clearTimeout(timeout);
345
- this.log("WebSocket error:", error);
346
- reject(new Error("WebSocket connection error"));
347
- });
348
- socket.on("close", (code, reason) => {
349
- clearTimeout(timeout);
350
- const wasConnected = this.connected;
351
- this.closeSocket();
352
- if (this.destroyed) return;
353
- this.log("Connection closed", { code, reason: reason.toString() });
354
- if (wasConnected) {
355
- this.reconnect();
356
- } else {
357
- reject(new Error(`Connection closed: ${reason.toString()}`));
358
- }
359
- });
360
- socket.on("message", (data) => {
361
- const message = JSON.parse(data.toString());
362
- if (message.setupComplete) {
363
- clearTimeout(timeout);
364
- this.log("Session ready");
365
- this.connected = true;
366
- this.retryCount = 0;
367
- this.emit("Ready");
368
- this.flush();
369
- resolve();
370
- return;
371
- }
372
- this.emit("Message", message);
373
- });
374
- });
375
- }
376
- closeSocket() {
377
- const socket = this.socket;
378
- this.socket = void 0;
379
- this.connected = false;
380
- this.clearPendingTimeout();
381
- if (!socket) return;
382
- socket.removeAllListeners();
383
- socket.on("error", () => {
384
- });
385
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
386
- socket.close(1e3);
387
- }
388
- }
389
- reconnect() {
390
- if (this.destroyed) return;
391
- this.retryCount++;
392
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
393
- this.log("Max retries reached, giving up");
394
- this.emit("Failed");
395
- return;
396
- }
397
- this.emit("Reconnecting", this.retryCount);
398
- this.log("Reconnecting...");
399
- this.reconnectTimeout = setTimeout(() => {
400
- this.reconnectTimeout = void 0;
401
- this.connect();
402
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
403
- }
404
390
  };
391
+ function parseMessage(text) {
392
+ try {
393
+ return JSON.parse(text);
394
+ } catch {
395
+ return void 0;
396
+ }
397
+ }
405
398
 
406
399
  // src/GeminiLive.ts
407
400
  var DEFAULT_MODEL2 = "gemini-3.8-live";
408
401
  var SAMPLE_RATE2 = 16e3;
409
402
  var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
410
403
  var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
404
+ function newToolRound() {
405
+ return { running: 0, answerEndDeferred: false };
406
+ }
411
407
  var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
412
408
  var GeminiLive = class extends Realtime {
413
409
  constructor(options) {
414
410
  super(options);
415
411
  this.everConnected = false;
416
- // Whether the setup being sent resumes the session
417
- this.resuming = false;
418
412
  this.goingAway = false;
419
413
  this.outputResampler = new Pcm16Resampler(
420
414
  GEMINI_OUTPUT_SAMPLE_RATE,
@@ -423,14 +417,16 @@ var GeminiLive = class extends Realtime {
423
417
  this.hasTranscribedTurn = false;
424
418
  // Transcript of a dropped turn, which the conversation should not show
425
419
  this.ignoreUserTranscript = false;
420
+ // Calls of the session Gemini told not to answer
426
421
  this.cancelledToolCalls = /* @__PURE__ */ new Set();
427
- // Gemini ends its turn when it calls a tool, and speaks again once it has
428
- // the result. The answer stays open in between, so the client keeps waiting
429
- // for it rather than listening again.
430
- this.runningTools = 0;
431
- this.answerEndDeferred = false;
432
- this.onReady = () => {
433
- if (this.everConnected && !this.resuming) {
422
+ // Bumped by every session that does not resume the previous one, whose tool
423
+ // calls it cannot answer
424
+ this.session = 0;
425
+ this.toolRound = newToolRound();
426
+ this.onReady = (setup) => {
427
+ const resumed = !!setup?.setup?.sessionResumption?.handle;
428
+ if (this.everConnected && !resumed) {
429
+ this.startFreshSession();
434
430
  const turns = this.conversation.flatMap(
435
431
  (message) => message.role === "user" || message.role === "assistant" ? [
436
432
  {
@@ -448,7 +444,12 @@ var GeminiLive = class extends Realtime {
448
444
  };
449
445
  this.onReconnecting = (attempt) => {
450
446
  if (attempt > 1) this.resumptionHandle = void 0;
451
- if (!this.resumptionHandle) this.socket.clearPending();
447
+ if (!this.resumptionHandle) {
448
+ this.socket.clearPending();
449
+ this.startFreshSession();
450
+ }
451
+ this.toolRound.answerEndDeferred = false;
452
+ this.commitTurnTranscript();
452
453
  this.endAnswer();
453
454
  };
454
455
  this.onMessage = (message) => {
@@ -486,9 +487,16 @@ var GeminiLive = class extends Realtime {
486
487
  this.socket.on("Failed", () => this.emit("Failed"));
487
488
  this.socket.connect();
488
489
  }
490
+ /**
491
+ * Stops the answer. The tools it is waiting for still send their results,
492
+ * silently, since a blocking call left unanswered would stall the model.
493
+ */
494
+ cancel() {
495
+ super.cancel();
496
+ this.resetToolRound();
497
+ }
489
498
  destroy() {
490
499
  super.destroy();
491
- this.clearToolFollowUp();
492
500
  this.socket.destroy();
493
501
  }
494
502
  openTurn() {
@@ -523,11 +531,9 @@ var GeminiLive = class extends Realtime {
523
531
  }
524
532
  cancelAnswer() {
525
533
  this.outputResampler.reset();
526
- this.clearToolFollowUp();
527
- this.answerEndDeferred = false;
528
534
  }
529
535
  get isIdle() {
530
- return super.isIdle && this.runningTools === 0;
536
+ return super.isIdle && this.toolRound.running === 0;
531
537
  }
532
538
  updateTools() {
533
539
  if (this.everConnected) {
@@ -559,7 +565,6 @@ var GeminiLive = class extends Realtime {
559
565
  this.socket.send(message);
560
566
  }
561
567
  buildSetup() {
562
- this.resuming = !!this.resumptionHandle;
563
568
  const functionDeclarations = this.tools.map((tool) => ({
564
569
  name: tool.name,
565
570
  description: tool.description,
@@ -593,6 +598,22 @@ var GeminiLive = class extends Realtime {
593
598
  }
594
599
  };
595
600
  }
601
+ /** Forgets what belonged to the previous session */
602
+ startFreshSession() {
603
+ this.session++;
604
+ this.resetToolRound();
605
+ this.cancelledToolCalls.clear();
606
+ this.ignoreUserTranscript = false;
607
+ }
608
+ /**
609
+ * Starts a new round of tools. The tools still running belong to the old
610
+ * one: their results go out silently, and nothing waits for the model to
611
+ * talk about them.
612
+ */
613
+ resetToolRound() {
614
+ if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
615
+ this.toolRound = newToolRound();
616
+ }
596
617
  /** Moves to a new connection before Gemini closes this one */
597
618
  renewIfGoingAway() {
598
619
  if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
@@ -624,8 +645,8 @@ var GeminiLive = class extends Realtime {
624
645
  if (content.turnComplete) {
625
646
  this.commitTurnTranscript();
626
647
  this.outputResampler.reset();
627
- if (this.runningTools > 0 || this.toolFollowUp) {
628
- this.answerEndDeferred = true;
648
+ if (this.toolRound.running > 0 || this.toolRound.followUp) {
649
+ this.toolRound.answerEndDeferred = true;
629
650
  } else {
630
651
  this.endAnswer();
631
652
  this.renewIfGoingAway();
@@ -634,24 +655,22 @@ var GeminiLive = class extends Realtime {
634
655
  }
635
656
  onModelOutput() {
636
657
  this.commitTurnTranscript();
637
- if (this.toolFollowUp) {
638
- this.clearToolFollowUp();
639
- this.answerEndDeferred = false;
658
+ const round = this.toolRound;
659
+ if (round.followUp) {
660
+ clearTimeout(round.followUp);
661
+ round.followUp = void 0;
662
+ round.answerEndDeferred = false;
640
663
  }
641
664
  }
642
665
  /** Ends the answer the model stopped giving to wait for a tool */
643
666
  endDeferredAnswer() {
644
- if (!this.answerEndDeferred) return;
645
- if (this.runningTools > 0 || this.toolFollowUp) return;
646
- this.answerEndDeferred = false;
667
+ const round = this.toolRound;
668
+ if (!round.answerEndDeferred) return;
669
+ if (round.running > 0 || round.followUp) return;
670
+ round.answerEndDeferred = false;
647
671
  this.endAnswer();
648
672
  this.renewIfGoingAway();
649
673
  }
650
- clearToolFollowUp() {
651
- if (!this.toolFollowUp) return;
652
- clearTimeout(this.toolFollowUp);
653
- this.toolFollowUp = void 0;
654
- }
655
674
  /**
656
675
  * The transcript of the turn comes before the answer, so the first piece of
657
676
  * the answer closes it.
@@ -669,17 +688,22 @@ var GeminiLive = class extends Realtime {
669
688
  toolName: call.name,
670
689
  parameters: JSON.stringify(call.args ?? {})
671
690
  };
672
- this.runningTools++;
691
+ const round = this.toolRound;
692
+ const session = this.session;
693
+ round.running++;
673
694
  let result;
674
695
  try {
675
696
  result = await this.runTool(toolCall);
676
697
  } finally {
677
- this.runningTools--;
698
+ round.running--;
678
699
  }
700
+ if (session !== this.session) return;
679
701
  if (this.cancelledToolCalls.delete(call.id)) {
680
702
  this.endDeferredAnswer();
681
703
  return;
682
704
  }
705
+ const interrupted = round !== this.toolRound;
706
+ const silent = result.skipAnswer || interrupted;
683
707
  this.send({
684
708
  toolResponse: {
685
709
  functionResponses: [
@@ -689,16 +713,17 @@ var GeminiLive = class extends Realtime {
689
713
  response: {
690
714
  output: result.output ?? null,
691
715
  // A tool without an answer lets the model know silently
692
- scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
716
+ scheduling: silent ? "SILENT" : "WHEN_IDLE"
693
717
  }
694
718
  }
695
719
  ]
696
720
  }
697
721
  });
722
+ if (interrupted) return;
698
723
  if (!result.skipAnswer) {
699
- this.clearToolFollowUp();
700
- this.toolFollowUp = setTimeout(() => {
701
- this.toolFollowUp = void 0;
724
+ if (round.followUp) clearTimeout(round.followUp);
725
+ round.followUp = setTimeout(() => {
726
+ round.followUp = void 0;
702
727
  this.log("Nothing said about the tool result");
703
728
  this.endDeferredAnswer();
704
729
  }, TOOL_FOLLOW_UP_TIMEOUT);
@@ -717,71 +742,77 @@ var GeminiSTT = class extends STT {
717
742
  constructor(options) {
718
743
  super();
719
744
  this.options = options;
745
+ // Gemini closes the session soon, and it is renewed once nothing waits
720
746
  this.goingAway = false;
721
- // Audio of the utterance, sent again if the connection drops before its
722
- // transcript comes back
723
- this.chunks = [];
724
- this.utteranceOpen = false;
725
- this.utteranceEnded = false;
726
- this.transcript = "";
747
+ // Utterances waiting for their transcript, in the order they were spoken.
748
+ // Gemini names no utterance in what it sends, and goes on to the next one
749
+ // before it is done with the previous. So it hears one at a time: the first
750
+ // one, and the audio of the next waits here until its transcript is in.
751
+ this.utterances = [];
752
+ this.onReady = () => {
753
+ this.goingAway = false;
754
+ };
727
755
  this.onMessage = (message) => {
728
756
  const content = message.serverContent;
729
- if (content?.inputTranscription?.text && this.utteranceOpen) {
730
- this.transcript += content.inputTranscription.text;
757
+ const utterance = this.utterances[0];
758
+ if (content?.inputTranscription?.text && utterance) {
759
+ utterance.transcript += content.inputTranscription.text;
731
760
  }
732
- if (content?.generationComplete && this.utteranceEnded) {
733
- this.finishUtterance();
761
+ if (content?.generationComplete && utterance) {
762
+ if (utterance.state === "ended") this.give(utterance);
763
+ if (utterance.state === "given") this.remove(utterance);
734
764
  }
735
765
  if (message.goAway) {
736
766
  this.log("Connection ending in", message.goAway.timeLeft);
737
- if (this.utteranceOpen) {
738
- this.goingAway = true;
739
- } else {
740
- this.socket.renew();
741
- }
767
+ this.goingAway = true;
768
+ this.renewIfGoingAway();
742
769
  }
743
770
  };
744
771
  this.onReconnecting = () => {
772
+ this.goingAway = false;
745
773
  this.socket.clearPending();
746
- if (!this.utteranceOpen) return;
747
- this.transcript = "";
748
- this.socket.send({ realtimeInput: { activityStart: {} } });
749
- this.chunks.forEach((chunk) => this.sendAudio(chunk));
750
- if (this.utteranceEnded) {
751
- this.socket.send({ realtimeInput: { activityEnd: {} } });
774
+ for (const utterance of this.utterances) {
775
+ if (utterance.state === "given") this.remove(utterance);
776
+ }
777
+ for (const utterance of this.utterances) {
778
+ if (utterance.timeout) clearTimeout(utterance.timeout);
779
+ utterance.timeout = void 0;
780
+ utterance.sent = false;
781
+ utterance.transcript = "";
752
782
  }
783
+ this.sendFirst();
784
+ };
785
+ this.onFailed = () => {
786
+ const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
787
+ this.dropUtterances();
788
+ this.emit("Failed", chunks);
753
789
  };
754
790
  this.socket = new LiveSocket(
755
791
  options,
756
792
  () => this.buildSetup(),
757
793
  (...message) => this.log(...message)
758
794
  );
795
+ this.socket.on("Ready", this.onReady);
759
796
  this.socket.on("Message", this.onMessage);
760
797
  this.socket.on("Reconnecting", this.onReconnecting);
761
- this.socket.on("Failed", () => this.emit("Failed", this.chunks));
798
+ this.socket.on("Failed", this.onFailed);
762
799
  this.socket.connect();
763
800
  }
764
801
  transcribe(audioStream) {
802
+ let utterance;
765
803
  audioStream.on("data", (chunk) => {
766
- if (!this.utteranceOpen) this.openUtterance();
767
- this.chunks.push(chunk);
768
- this.sendAudio(chunk);
804
+ if (!utterance) utterance = this.openUtterance();
805
+ if (utterance.state !== "open") return;
806
+ utterance.chunks.push(chunk);
807
+ if (utterance.sent) this.sendAudio(chunk);
769
808
  });
770
809
  audioStream.on("end", () => {
771
- if (!this.utteranceOpen || this.utteranceEnded) return;
772
- this.utteranceEnded = true;
773
- this.socket.send({ realtimeInput: { activityEnd: {} } });
774
- this.clearTranscriptionTimeout();
775
- this.transcriptionTimeout = setTimeout(() => {
776
- this.transcriptionTimeout = void 0;
777
- this.log("Transcription timeout");
778
- this.finishUtterance();
779
- }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
810
+ if (utterance?.state === "open") this.endUtterance(utterance);
780
811
  });
781
812
  }
782
813
  destroy() {
783
814
  super.destroy();
784
- this.clearTranscriptionTimeout();
815
+ this.dropUtterances();
785
816
  this.socket.destroy();
786
817
  }
787
818
  buildSetup() {
@@ -802,12 +833,38 @@ var GeminiSTT = class extends STT {
802
833
  };
803
834
  }
804
835
  openUtterance() {
805
- if (this.utteranceOpen) this.finishUtterance();
806
- this.utteranceOpen = true;
807
- this.utteranceEnded = false;
808
- this.chunks = [];
809
- this.transcript = "";
836
+ const last = this.utterances[this.utterances.length - 1];
837
+ if (last?.state === "open") this.endUtterance(last);
838
+ const utterance = {
839
+ state: "open",
840
+ sent: false,
841
+ chunks: [],
842
+ transcript: ""
843
+ };
844
+ this.utterances.push(utterance);
845
+ this.sendFirst();
846
+ return utterance;
847
+ }
848
+ endUtterance(utterance) {
849
+ utterance.state = "ended";
850
+ if (utterance.sent) this.sendEnd(utterance);
851
+ }
852
+ /** Sends the first utterance to Gemini, with the audio it held meanwhile */
853
+ sendFirst() {
854
+ const utterance = this.utterances[0];
855
+ if (!utterance || utterance.sent) return;
856
+ utterance.sent = true;
810
857
  this.socket.send({ realtimeInput: { activityStart: {} } });
858
+ utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
859
+ if (utterance.state === "ended") this.sendEnd(utterance);
860
+ }
861
+ sendEnd(utterance) {
862
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
863
+ utterance.timeout = setTimeout(() => {
864
+ utterance.timeout = void 0;
865
+ this.log("Transcription timeout");
866
+ this.give(utterance);
867
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
811
868
  }
812
869
  sendAudio(chunk) {
813
870
  this.socket.send({
@@ -819,25 +876,40 @@ var GeminiSTT = class extends STT {
819
876
  }
820
877
  });
821
878
  }
822
- /** Emits what was transcribed of the utterance, possibly nothing */
823
- finishUtterance() {
824
- this.clearTranscriptionTimeout();
825
- const transcript = this.transcript.trim();
826
- this.utteranceOpen = false;
827
- this.utteranceEnded = false;
828
- this.chunks = [];
829
- this.transcript = "";
879
+ /**
880
+ * Emits what was transcribed of an utterance, possibly nothing, and keeps
881
+ * it in line for what Gemini still sends about it
882
+ */
883
+ give(utterance) {
884
+ if (utterance.timeout) clearTimeout(utterance.timeout);
885
+ utterance.state = "given";
886
+ const transcript = utterance.transcript.trim();
830
887
  this.log(`Transcript: "${transcript}"`);
831
888
  this.emit("Transcript", transcript);
832
- if (this.goingAway) {
833
- this.goingAway = false;
834
- this.socket.renew();
835
- }
889
+ utterance.timeout = setTimeout(
890
+ () => this.remove(utterance),
891
+ this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
892
+ );
836
893
  }
837
- clearTranscriptionTimeout() {
838
- if (!this.transcriptionTimeout) return;
839
- clearTimeout(this.transcriptionTimeout);
840
- this.transcriptionTimeout = void 0;
894
+ remove(utterance) {
895
+ if (utterance.timeout) clearTimeout(utterance.timeout);
896
+ utterance.timeout = void 0;
897
+ this.utterances = this.utterances.filter((other) => other !== utterance);
898
+ this.renewIfGoingAway();
899
+ this.sendFirst();
900
+ }
901
+ /** Moves to a new connection once no utterance waits on this one */
902
+ renewIfGoingAway() {
903
+ if (!this.goingAway || this.utterances.length > 0) return;
904
+ this.goingAway = false;
905
+ this.socket.renew();
906
+ }
907
+ dropUtterances() {
908
+ for (const utterance of this.utterances) {
909
+ if (utterance.timeout) clearTimeout(utterance.timeout);
910
+ if (utterance.state === "open") utterance.state = "ended";
911
+ }
912
+ this.utterances = [];
841
913
  }
842
914
  };
843
915
 
@@ -848,7 +920,7 @@ var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
848
920
  var DEFAULT_VOICE = "Kore";
849
921
  var GEMINI_SAMPLE_RATE = 24e3;
850
922
  var OUTPUT_SAMPLE_RATE = 16e3;
851
- var DEFAULT_MAX_RETRY2 = 2;
923
+ var DEFAULT_MAX_RETRY = 2;
852
924
  var GeminiTTS = class extends SentenceTTS {
853
925
  constructor(options) {
854
926
  super();
@@ -857,7 +929,7 @@ var GeminiTTS = class extends SentenceTTS {
857
929
  }
858
930
  async synthesize(text, signal) {
859
931
  const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
860
- const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
932
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
861
933
  for (let attempt = 0; ; attempt++) {
862
934
  const progress = { emitted: false };
863
935
  try {
@@ -911,7 +983,7 @@ var GeminiTTS = class extends SentenceTTS {
911
983
  }
912
984
  };
913
985
  function isBlocked(error) {
914
- return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
986
+ return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
915
987
  }
916
988
  export {
917
989
  GeminiAgent,