@micdrop/gemini 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.mjs CHANGED
@@ -15,127 +15,139 @@ var GeminiAgent = class extends Agent {
15
15
  this.signatures = /* @__PURE__ */ new Map();
16
16
  this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
17
17
  }
18
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
18
+ generateAnswer(stream, signal) {
19
+ return this.generateStep(stream, signal, 0);
20
+ }
21
+ /**
22
+ * Generates one step of the answer, retried on error.
23
+ *
24
+ * A retry starts the step over, so it only happens while nothing of the
25
+ * step reached the stream: what was said cannot be taken back, and saying
26
+ * it twice is worse than stopping there.
27
+ */
28
+ async generateStep(stream, signal, stepCount) {
19
29
  if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
20
30
  console.error("[GeminiAgent] Max steps reached");
21
31
  return;
22
32
  }
23
- const abortController = new AbortController();
24
- this.abortController = abortController;
25
- const signal = abortController.signal;
33
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
34
+ for (let tryCount = 0; ; tryCount++) {
35
+ const progress = { answer: "" };
36
+ try {
37
+ const step = await this.streamStep(stream, signal, progress);
38
+ if (signal.aborted) return;
39
+ this.recordAnswer(progress.answer);
40
+ if (step.calledTool && !step.skipAnswer && !progress.answer) {
41
+ await this.generateStep(stream, signal, stepCount + 1);
42
+ }
43
+ return;
44
+ } catch (error) {
45
+ if (signal.aborted) return;
46
+ console.error("[GeminiAgent] Error answering:", error);
47
+ if (progress.answer) {
48
+ this.log("Error after part of the answer was said, keeping it");
49
+ this.recordAnswer(progress.answer);
50
+ return;
51
+ }
52
+ if (tryCount >= maxRetry) {
53
+ this.log("Max retries reached, giving up");
54
+ this.emit("Failed");
55
+ return;
56
+ }
57
+ await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
58
+ if (signal.aborted) return;
59
+ }
60
+ }
61
+ }
62
+ /** Streams one request, writing the text of the answer as it comes */
63
+ async streamStep(stream, signal, progress) {
26
64
  let extracting = false;
27
65
  const extractOptions = this.getExtractOptions();
28
- try {
29
- const response = await this.genai.interactions.create(
30
- {
31
- model: this.options.model || DEFAULT_MODEL,
32
- store: false,
33
- stream: true,
34
- system_instruction: this.buildSystemInstruction(),
35
- input: this.buildInput(),
36
- tools: this.buildTools(),
37
- ...this.options.thinkingLevel || this.options.settings ? {
38
- generation_config: {
39
- ...this.options.settings,
40
- ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
41
- }
42
- } : {}
43
- },
44
- { fetchOptions: { signal } }
45
- );
46
- const steps = [];
47
- let answer = "";
48
- let skipAnswer = false;
49
- let calledTool = false;
50
- let signature = "";
51
- for await (const event of response) {
52
- if (abortController !== this.abortController) return;
53
- switch (event.event_type) {
54
- case "step.start":
55
- steps[event.index] = {
56
- type: event.step.type,
57
- id: event.step.id,
58
- name: event.step.name,
59
- text: "",
60
- arguments: "",
61
- signature: ""
62
- };
63
- break;
64
- case "step.delta": {
65
- const step = steps[event.index];
66
- if (!step) break;
67
- const delta = event.delta;
68
- if (delta.type === "thought_signature") {
69
- step.signature += delta.signature ?? "";
70
- } else if (delta.type === "arguments_delta") {
71
- step.arguments += delta.arguments ?? "";
72
- } else if (delta.type === "text" && step.type === "model_output") {
73
- this.log(`Answer chunk: "${delta.text}"`);
74
- answer += delta.text;
75
- if (extractOptions) {
76
- if (extracting) break;
77
- const startTagIndex = delta.text.indexOf(
78
- extractOptions.startTag
79
- );
80
- if (startTagIndex !== -1) {
81
- extracting = true;
82
- stream.write(delta.text.slice(0, startTagIndex).trimEnd());
83
- break;
84
- }
66
+ const response = await this.genai.interactions.create(
67
+ {
68
+ model: this.options.model || DEFAULT_MODEL,
69
+ store: false,
70
+ stream: true,
71
+ system_instruction: this.buildSystemInstruction(),
72
+ input: this.buildInput(),
73
+ tools: this.buildTools(),
74
+ ...this.options.thinkingLevel || this.options.settings ? {
75
+ generation_config: {
76
+ ...this.options.settings,
77
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
78
+ }
79
+ } : {}
80
+ },
81
+ { signal }
82
+ );
83
+ const steps = [];
84
+ let skipAnswer = false;
85
+ let calledTool = false;
86
+ let signature = "";
87
+ for await (const event of response) {
88
+ if (signal.aborted) break;
89
+ switch (event.event_type) {
90
+ case "step.start":
91
+ steps[event.index] = {
92
+ type: event.step.type,
93
+ id: event.step.id,
94
+ name: event.step.name,
95
+ text: "",
96
+ arguments: "",
97
+ signature: ""
98
+ };
99
+ break;
100
+ case "step.delta": {
101
+ const step = steps[event.index];
102
+ if (!step) break;
103
+ const delta = event.delta;
104
+ if (delta.type === "thought_signature") {
105
+ step.signature += delta.signature ?? "";
106
+ } else if (delta.type === "arguments_delta") {
107
+ step.arguments += delta.arguments ?? "";
108
+ } else if (delta.type === "text" && step.type === "model_output") {
109
+ this.log(`Answer chunk: "${delta.text}"`);
110
+ progress.answer += delta.text;
111
+ if (extractOptions) {
112
+ if (extracting) break;
113
+ const startTagIndex = delta.text.indexOf(extractOptions.startTag);
114
+ if (startTagIndex !== -1) {
115
+ extracting = true;
116
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
117
+ break;
85
118
  }
86
- stream.write(delta.text);
87
119
  }
88
- break;
89
- }
90
- case "step.stop": {
91
- const step = steps[event.index];
92
- if (step?.type === "thought") signature = step.signature;
93
- if (step?.type !== "function_call" || !step.id || !step.name) break;
94
- calledTool = true;
95
- if (signature) this.signatures.set(step.id, signature);
96
- const result = await this.executeTool({
97
- role: "tool_call",
98
- toolCallId: step.id,
99
- toolName: step.name,
100
- parameters: step.arguments || "{}"
101
- });
102
- if (result.skipAnswer) skipAnswer = true;
103
- break;
120
+ stream.write(delta.text);
104
121
  }
105
- case "error":
106
- throw new Error(event.error?.message || "Generation failed");
107
- default:
108
- break;
122
+ break;
109
123
  }
110
- }
111
- if (abortController !== this.abortController) return;
112
- this.abortController = void 0;
113
- if (answer) {
114
- const { message, metadata } = this.extract(answer);
115
- this.addAssistantMessage(message, metadata);
116
- }
117
- if (calledTool && !skipAnswer && !answer) {
118
- await this.generateAnswer(stream, stepCount + 1);
119
- }
120
- } catch (error) {
121
- if (signal.aborted) return;
122
- console.error("[GeminiAgent] Error answering:", error);
123
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
124
- await new Promise(
125
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
126
- );
127
- await this.generateAnswer(stream, stepCount, tryCount + 1);
128
- } else {
129
- this.log("Max retries reached, giving up");
130
- this.emit("Failed");
124
+ case "step.stop": {
125
+ const step = steps[event.index];
126
+ if (step?.type === "thought") signature = step.signature;
127
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
128
+ calledTool = true;
129
+ if (signature) this.signatures.set(step.id, signature);
130
+ const result = await this.executeTool({
131
+ role: "tool_call",
132
+ toolCallId: step.id,
133
+ toolName: step.name,
134
+ parameters: step.arguments || "{}"
135
+ });
136
+ if (result.skipAnswer) skipAnswer = true;
137
+ break;
138
+ }
139
+ case "error":
140
+ throw new Error(event.error?.message || "Generation failed");
141
+ default:
142
+ break;
131
143
  }
132
144
  }
145
+ return { calledTool, skipAnswer };
133
146
  }
134
- cancel() {
135
- if (!this.abortController) return;
136
- this.log("Cancel");
137
- this.abortController.abort();
138
- this.abortController = void 0;
147
+ recordAnswer(answer) {
148
+ if (!answer) return;
149
+ const { message, metadata } = this.extract(answer);
150
+ this.addAssistantMessage(message, metadata);
139
151
  }
140
152
  buildSystemInstruction() {
141
153
  return this.conversation.flatMap(
@@ -207,6 +219,17 @@ var GeminiAgent = class extends Agent {
207
219
  });
208
220
  }
209
221
  };
222
+ function sleep(ms, signal) {
223
+ return new Promise((resolve) => {
224
+ const done = () => {
225
+ clearTimeout(timer);
226
+ signal.removeEventListener("abort", done);
227
+ resolve();
228
+ };
229
+ const timer = setTimeout(done, ms);
230
+ signal.addEventListener("abort", done);
231
+ });
232
+ }
210
233
  function parseArguments(parameters) {
211
234
  try {
212
235
  return JSON.parse(parameters);
@@ -215,6 +238,72 @@ function parseArguments(parameters) {
215
238
  }
216
239
  }
217
240
 
241
+ // src/GeminiClassifier.ts
242
+ import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
243
+ import { Classifier } from "@micdrop/server";
244
+ function noul(instructions, criteria) {
245
+ return {
246
+ type: "noul",
247
+ instructions,
248
+ criteria: criteria && {
249
+ true_description: criteria.true,
250
+ false_description: criteria.false
251
+ }
252
+ };
253
+ }
254
+ function choice(instructions, criteria) {
255
+ return { type: "choice", instructions, criteria };
256
+ }
257
+ function score(instructions, levels) {
258
+ return { type: "score", instructions, criteria: levels };
259
+ }
260
+ var DEFAULT_MODEL2 = "gemini-decision-1";
261
+ var DEFAULT_TIMEOUT = 3e3;
262
+ var API_VERSION = "v1beta";
263
+ var GeminiClassifier = class extends Classifier {
264
+ constructor(options) {
265
+ super();
266
+ this.options = options;
267
+ this.genai = "genai" in options ? options.genai : new GoogleGenAI2({ apiKey: options.apiKey });
268
+ }
269
+ async evaluate(input, signal) {
270
+ const { questions, model, timeout } = this.options;
271
+ const asked = typeof questions === "function" ? questions(input) : questions;
272
+ const decision = await this.genai.decisions.create(
273
+ {
274
+ model: model || DEFAULT_MODEL2,
275
+ input: this.buildInput(input),
276
+ questions: asked,
277
+ // Only on v1beta, while the client defaults to v1
278
+ api_version: API_VERSION
279
+ },
280
+ {
281
+ // The SDK drops its timeout when a signal is given
282
+ signal: AbortSignal.any([
283
+ signal,
284
+ AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT)
285
+ ]),
286
+ retries: { strategy: "none" }
287
+ }
288
+ );
289
+ const blocked = decision.prompt_feedback?.block_reason;
290
+ if (blocked) {
291
+ const message = decision.prompt_feedback?.block_reason_message;
292
+ throw new Error(
293
+ `Input blocked by Gemini (${blocked})${message ? `: ${message}` : ""}`
294
+ );
295
+ }
296
+ return {
297
+ answers: decision.answers ?? {},
298
+ usage: decision.usage
299
+ };
300
+ }
301
+ buildInput(input) {
302
+ if (this.options.input) return this.options.input(input);
303
+ return typeof input === "string" ? input : JSON.stringify(input);
304
+ }
305
+ };
306
+
218
307
  // src/GeminiLive.ts
219
308
  import {
220
309
  Pcm16Resampler,
@@ -223,67 +312,107 @@ import {
223
312
  import { toJSONSchema as toJSONSchema2 } from "zod";
224
313
 
225
314
  // src/LiveSocket.ts
315
+ import {
316
+ ReconnectingSocket
317
+ } from "@micdrop/server";
226
318
  import { EventEmitter } from "eventemitter3";
227
- import WebSocket from "ws";
228
319
  var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
229
320
  var SAMPLE_RATE = 16e3;
230
321
  var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
231
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
232
- var DEFAULT_RETRY_DELAY2 = 1e3;
233
- var DEFAULT_MAX_RETRY = 3;
234
322
  var LiveSocket = class extends EventEmitter {
235
323
  constructor(options, buildSetup, log) {
236
324
  super();
237
- this.options = options;
238
325
  this.buildSetup = buildSetup;
239
326
  this.log = log;
240
- this.connected = false;
241
- this.destroyed = false;
242
327
  // Messages waiting for the session, or for the end of a turn to be due
243
328
  this.pending = [];
244
329
  // Audio of the turn as it went out, see flush()
245
330
  this.turnAudioStartedAt = 0;
246
331
  this.turnAudioMs = 0;
247
- this.retryCount = 0;
332
+ // Attempts since the last session was ready, for the owner to tell a drop
333
+ // from a resumption that keeps failing
334
+ this.reconnectAttempts = 0;
335
+ /** Sends the setup, and holds the connection back until Gemini accepts it */
336
+ this.handshake = {
337
+ start: (socket) => {
338
+ const setup = this.buildSetup();
339
+ this.setup = setup;
340
+ socket.send(JSON.stringify(setup));
341
+ },
342
+ read: (data) => !!parseMessage(data.toString())?.setupComplete
343
+ };
344
+ this.onOpen = () => {
345
+ this.log("Session ready");
346
+ this.reconnectAttempts = 0;
347
+ this.emit("Ready", this.setup);
348
+ this.flush();
349
+ };
350
+ this.onMessage = (data) => {
351
+ const message = parseMessage(data.toString());
352
+ if (message) this.emit("Message", message);
353
+ };
354
+ this.onStatusChange = (status) => {
355
+ if (status !== "open") this.clearPendingTimeout();
356
+ if (status === "reconnecting") {
357
+ this.emit("Reconnecting", ++this.reconnectAttempts);
358
+ }
359
+ };
360
+ this.onFailed = () => {
361
+ this.clearPending();
362
+ this.emit("Failed");
363
+ };
364
+ this.socket = new ReconnectingSocket({
365
+ url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
366
+ connectionTimeout: options.connectionTimeout,
367
+ retryDelay: options.retryDelay,
368
+ maxRetry: options.maxRetry,
369
+ handshake: this.handshake,
370
+ log
371
+ });
372
+ this.socket.on("Open", this.onOpen);
373
+ this.socket.on("Message", this.onMessage);
374
+ this.socket.on("StatusChange", this.onStatusChange);
375
+ this.socket.on("Failed", this.onFailed);
248
376
  }
249
377
  /** The session is set up and takes messages right away */
250
378
  get isConnected() {
251
- return this.connected;
379
+ return this.socket.isOpen;
252
380
  }
253
381
  connect() {
254
- this.initWS().catch((error) => {
255
- this.log("Connection error:", error);
256
- this.reconnect();
257
- });
382
+ this.socket.connect();
258
383
  }
259
- /** Sends a message once the session is ready, after the ones before it */
384
+ /**
385
+ * Sends a message once the session is ready, after the ones before it.
386
+ * Dropped once the connection failed for good, as nothing would send it.
387
+ */
260
388
  send(message) {
389
+ const status = this.socket.status;
390
+ if (status === "failed" || status === "destroyed") return;
261
391
  this.pending.push(message);
262
392
  this.flush();
263
393
  }
264
394
  /** Sends a message ahead of the waiting ones, from a Ready listener */
265
395
  sendFirst(message) {
266
- this.socket?.send(JSON.stringify(message));
396
+ this.socket.send(JSON.stringify(message));
267
397
  }
268
398
  /** Drops the messages that have not gone out */
269
399
  clearPending() {
270
400
  this.pending = [];
271
401
  this.clearPendingTimeout();
272
402
  }
273
- /** Moves to a new connection, keeping the messages that have not gone out */
403
+ /**
404
+ * Moves to a new connection, keeping the messages that have not gone out.
405
+ * A reconnection waiting for its delay is replaced by this one.
406
+ */
274
407
  renew() {
275
408
  this.log("Renewing the connection");
276
- this.closeSocket();
277
- this.connect();
409
+ this.clearPendingTimeout();
410
+ this.reconnectAttempts = 0;
411
+ this.socket.restart();
278
412
  }
279
413
  destroy() {
280
- this.destroyed = true;
281
- if (this.reconnectTimeout) {
282
- clearTimeout(this.reconnectTimeout);
283
- this.reconnectTimeout = void 0;
284
- }
285
- this.closeSocket();
286
- this.pending = [];
414
+ this.clearPending();
415
+ this.socket.destroy();
287
416
  this.removeAllListeners();
288
417
  }
289
418
  /**
@@ -296,8 +425,7 @@ var LiveSocket = class extends EventEmitter {
296
425
  */
297
426
  flush() {
298
427
  while (this.pending.length > 0 && !this.pendingTimeout) {
299
- const socket = this.socket;
300
- if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
428
+ if (!this.socket.isOpen) return;
301
429
  const message = this.pending[0];
302
430
  const input = message.realtimeInput;
303
431
  if (input?.activityEnd) {
@@ -317,7 +445,7 @@ var LiveSocket = class extends EventEmitter {
317
445
  this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
318
446
  }
319
447
  this.pending.shift();
320
- socket.send(JSON.stringify(message));
448
+ this.socket.send(JSON.stringify(message));
321
449
  }
322
450
  }
323
451
  clearPendingTimeout() {
@@ -325,96 +453,28 @@ var LiveSocket = class extends EventEmitter {
325
453
  clearTimeout(this.pendingTimeout);
326
454
  this.pendingTimeout = void 0;
327
455
  }
328
- initWS() {
329
- return new Promise((resolve, reject) => {
330
- const socket = new WebSocket(
331
- `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
332
- );
333
- this.socket = socket;
334
- const timeout = setTimeout(() => {
335
- this.log("Connection timeout");
336
- this.closeSocket();
337
- reject(new Error("WebSocket connection timeout"));
338
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
339
- socket.on("open", () => {
340
- this.log("Connection opened");
341
- socket.send(JSON.stringify(this.buildSetup()));
342
- });
343
- socket.on("error", (error) => {
344
- clearTimeout(timeout);
345
- this.log("WebSocket error:", error);
346
- reject(new Error("WebSocket connection error"));
347
- });
348
- socket.on("close", (code, reason) => {
349
- clearTimeout(timeout);
350
- const wasConnected = this.connected;
351
- this.closeSocket();
352
- if (this.destroyed) return;
353
- this.log("Connection closed", { code, reason: reason.toString() });
354
- if (wasConnected) {
355
- this.reconnect();
356
- } else {
357
- reject(new Error(`Connection closed: ${reason.toString()}`));
358
- }
359
- });
360
- socket.on("message", (data) => {
361
- const message = JSON.parse(data.toString());
362
- if (message.setupComplete) {
363
- clearTimeout(timeout);
364
- this.log("Session ready");
365
- this.connected = true;
366
- this.retryCount = 0;
367
- this.emit("Ready");
368
- this.flush();
369
- resolve();
370
- return;
371
- }
372
- this.emit("Message", message);
373
- });
374
- });
375
- }
376
- closeSocket() {
377
- const socket = this.socket;
378
- this.socket = void 0;
379
- this.connected = false;
380
- this.clearPendingTimeout();
381
- if (!socket) return;
382
- socket.removeAllListeners();
383
- socket.on("error", () => {
384
- });
385
- if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
386
- socket.close(1e3);
387
- }
388
- }
389
- reconnect() {
390
- if (this.destroyed) return;
391
- this.retryCount++;
392
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
393
- this.log("Max retries reached, giving up");
394
- this.emit("Failed");
395
- return;
396
- }
397
- this.emit("Reconnecting", this.retryCount);
398
- this.log("Reconnecting...");
399
- this.reconnectTimeout = setTimeout(() => {
400
- this.reconnectTimeout = void 0;
401
- this.connect();
402
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
403
- }
404
456
  };
457
+ function parseMessage(text) {
458
+ try {
459
+ return JSON.parse(text);
460
+ } catch {
461
+ return void 0;
462
+ }
463
+ }
405
464
 
406
465
  // src/GeminiLive.ts
407
- var DEFAULT_MODEL2 = "gemini-3.8-live";
466
+ var DEFAULT_MODEL3 = "gemini-3.8-live";
408
467
  var SAMPLE_RATE2 = 16e3;
409
468
  var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
410
469
  var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
470
+ function newToolRound() {
471
+ return { running: 0, answerEndDeferred: false };
472
+ }
411
473
  var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
412
474
  var GeminiLive = class extends Realtime {
413
475
  constructor(options) {
414
476
  super(options);
415
477
  this.everConnected = false;
416
- // Whether the setup being sent resumes the session
417
- this.resuming = false;
418
478
  this.goingAway = false;
419
479
  this.outputResampler = new Pcm16Resampler(
420
480
  GEMINI_OUTPUT_SAMPLE_RATE,
@@ -423,14 +483,16 @@ var GeminiLive = class extends Realtime {
423
483
  this.hasTranscribedTurn = false;
424
484
  // Transcript of a dropped turn, which the conversation should not show
425
485
  this.ignoreUserTranscript = false;
486
+ // Calls of the session Gemini told not to answer
426
487
  this.cancelledToolCalls = /* @__PURE__ */ new Set();
427
- // Gemini ends its turn when it calls a tool, and speaks again once it has
428
- // the result. The answer stays open in between, so the client keeps waiting
429
- // for it rather than listening again.
430
- this.runningTools = 0;
431
- this.answerEndDeferred = false;
432
- this.onReady = () => {
433
- if (this.everConnected && !this.resuming) {
488
+ // Bumped by every session that does not resume the previous one, whose tool
489
+ // calls it cannot answer
490
+ this.session = 0;
491
+ this.toolRound = newToolRound();
492
+ this.onReady = (setup) => {
493
+ const resumed = !!setup?.setup?.sessionResumption?.handle;
494
+ if (this.everConnected && !resumed) {
495
+ this.startFreshSession();
434
496
  const turns = this.conversation.flatMap(
435
497
  (message) => message.role === "user" || message.role === "assistant" ? [
436
498
  {
@@ -448,7 +510,12 @@ var GeminiLive = class extends Realtime {
448
510
  };
449
511
  this.onReconnecting = (attempt) => {
450
512
  if (attempt > 1) this.resumptionHandle = void 0;
451
- if (!this.resumptionHandle) this.socket.clearPending();
513
+ if (!this.resumptionHandle) {
514
+ this.socket.clearPending();
515
+ this.startFreshSession();
516
+ }
517
+ this.toolRound.answerEndDeferred = false;
518
+ this.commitTurnTranscript();
452
519
  this.endAnswer();
453
520
  };
454
521
  this.onMessage = (message) => {
@@ -486,9 +553,16 @@ var GeminiLive = class extends Realtime {
486
553
  this.socket.on("Failed", () => this.emit("Failed"));
487
554
  this.socket.connect();
488
555
  }
556
+ /**
557
+ * Stops the answer. The tools it is waiting for still send their results,
558
+ * silently, since a blocking call left unanswered would stall the model.
559
+ */
560
+ cancel() {
561
+ super.cancel();
562
+ this.resetToolRound();
563
+ }
489
564
  destroy() {
490
565
  super.destroy();
491
- this.clearToolFollowUp();
492
566
  this.socket.destroy();
493
567
  }
494
568
  openTurn() {
@@ -523,11 +597,9 @@ var GeminiLive = class extends Realtime {
523
597
  }
524
598
  cancelAnswer() {
525
599
  this.outputResampler.reset();
526
- this.clearToolFollowUp();
527
- this.answerEndDeferred = false;
528
600
  }
529
601
  get isIdle() {
530
- return super.isIdle && this.runningTools === 0;
602
+ return super.isIdle && this.toolRound.running === 0;
531
603
  }
532
604
  updateTools() {
533
605
  if (this.everConnected) {
@@ -559,7 +631,6 @@ var GeminiLive = class extends Realtime {
559
631
  this.socket.send(message);
560
632
  }
561
633
  buildSetup() {
562
- this.resuming = !!this.resumptionHandle;
563
634
  const functionDeclarations = this.tools.map((tool) => ({
564
635
  name: tool.name,
565
636
  description: tool.description,
@@ -567,7 +638,7 @@ var GeminiLive = class extends Realtime {
567
638
  }));
568
639
  return {
569
640
  setup: {
570
- model: `models/${this.options.model || DEFAULT_MODEL2}`,
641
+ model: `models/${this.options.model || DEFAULT_MODEL3}`,
571
642
  generationConfig: {
572
643
  responseModalities: ["AUDIO"],
573
644
  ...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
@@ -593,6 +664,22 @@ var GeminiLive = class extends Realtime {
593
664
  }
594
665
  };
595
666
  }
667
+ /** Forgets what belonged to the previous session */
668
+ startFreshSession() {
669
+ this.session++;
670
+ this.resetToolRound();
671
+ this.cancelledToolCalls.clear();
672
+ this.ignoreUserTranscript = false;
673
+ }
674
+ /**
675
+ * Starts a new round of tools. The tools still running belong to the old
676
+ * one: their results go out silently, and nothing waits for the model to
677
+ * talk about them.
678
+ */
679
+ resetToolRound() {
680
+ if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
681
+ this.toolRound = newToolRound();
682
+ }
596
683
  /** Moves to a new connection before Gemini closes this one */
597
684
  renewIfGoingAway() {
598
685
  if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
@@ -624,8 +711,8 @@ var GeminiLive = class extends Realtime {
624
711
  if (content.turnComplete) {
625
712
  this.commitTurnTranscript();
626
713
  this.outputResampler.reset();
627
- if (this.runningTools > 0 || this.toolFollowUp) {
628
- this.answerEndDeferred = true;
714
+ if (this.toolRound.running > 0 || this.toolRound.followUp) {
715
+ this.toolRound.answerEndDeferred = true;
629
716
  } else {
630
717
  this.endAnswer();
631
718
  this.renewIfGoingAway();
@@ -634,24 +721,22 @@ var GeminiLive = class extends Realtime {
634
721
  }
635
722
  onModelOutput() {
636
723
  this.commitTurnTranscript();
637
- if (this.toolFollowUp) {
638
- this.clearToolFollowUp();
639
- this.answerEndDeferred = false;
724
+ const round = this.toolRound;
725
+ if (round.followUp) {
726
+ clearTimeout(round.followUp);
727
+ round.followUp = void 0;
728
+ round.answerEndDeferred = false;
640
729
  }
641
730
  }
642
731
  /** Ends the answer the model stopped giving to wait for a tool */
643
732
  endDeferredAnswer() {
644
- if (!this.answerEndDeferred) return;
645
- if (this.runningTools > 0 || this.toolFollowUp) return;
646
- this.answerEndDeferred = false;
733
+ const round = this.toolRound;
734
+ if (!round.answerEndDeferred) return;
735
+ if (round.running > 0 || round.followUp) return;
736
+ round.answerEndDeferred = false;
647
737
  this.endAnswer();
648
738
  this.renewIfGoingAway();
649
739
  }
650
- clearToolFollowUp() {
651
- if (!this.toolFollowUp) return;
652
- clearTimeout(this.toolFollowUp);
653
- this.toolFollowUp = void 0;
654
- }
655
740
  /**
656
741
  * The transcript of the turn comes before the answer, so the first piece of
657
742
  * the answer closes it.
@@ -669,17 +754,22 @@ var GeminiLive = class extends Realtime {
669
754
  toolName: call.name,
670
755
  parameters: JSON.stringify(call.args ?? {})
671
756
  };
672
- this.runningTools++;
757
+ const round = this.toolRound;
758
+ const session = this.session;
759
+ round.running++;
673
760
  let result;
674
761
  try {
675
762
  result = await this.runTool(toolCall);
676
763
  } finally {
677
- this.runningTools--;
764
+ round.running--;
678
765
  }
766
+ if (session !== this.session) return;
679
767
  if (this.cancelledToolCalls.delete(call.id)) {
680
768
  this.endDeferredAnswer();
681
769
  return;
682
770
  }
771
+ const interrupted = round !== this.toolRound;
772
+ const silent = result.skipAnswer || interrupted;
683
773
  this.send({
684
774
  toolResponse: {
685
775
  functionResponses: [
@@ -689,16 +779,17 @@ var GeminiLive = class extends Realtime {
689
779
  response: {
690
780
  output: result.output ?? null,
691
781
  // A tool without an answer lets the model know silently
692
- scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
782
+ scheduling: silent ? "SILENT" : "WHEN_IDLE"
693
783
  }
694
784
  }
695
785
  ]
696
786
  }
697
787
  });
788
+ if (interrupted) return;
698
789
  if (!result.skipAnswer) {
699
- this.clearToolFollowUp();
700
- this.toolFollowUp = setTimeout(() => {
701
- this.toolFollowUp = void 0;
790
+ if (round.followUp) clearTimeout(round.followUp);
791
+ round.followUp = setTimeout(() => {
792
+ round.followUp = void 0;
702
793
  this.log("Nothing said about the tool result");
703
794
  this.endDeferredAnswer();
704
795
  }, TOOL_FOLLOW_UP_TIMEOUT);
@@ -710,84 +801,90 @@ var GeminiLive = class extends Realtime {
710
801
 
711
802
  // src/GeminiSTT.ts
712
803
  import { STT } from "@micdrop/server";
713
- var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
804
+ var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
714
805
  var SAMPLE_RATE3 = 16e3;
715
806
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
716
807
  var GeminiSTT = class extends STT {
717
808
  constructor(options) {
718
809
  super();
719
810
  this.options = options;
811
+ // Gemini closes the session soon, and it is renewed once nothing waits
720
812
  this.goingAway = false;
721
- // Audio of the utterance, sent again if the connection drops before its
722
- // transcript comes back
723
- this.chunks = [];
724
- this.utteranceOpen = false;
725
- this.utteranceEnded = false;
726
- this.transcript = "";
813
+ // Utterances waiting for their transcript, in the order they were spoken.
814
+ // Gemini names no utterance in what it sends, and goes on to the next one
815
+ // before it is done with the previous. So it hears one at a time: the first
816
+ // one, and the audio of the next waits here until its transcript is in.
817
+ this.utterances = [];
818
+ this.onReady = () => {
819
+ this.goingAway = false;
820
+ };
727
821
  this.onMessage = (message) => {
728
822
  const content = message.serverContent;
729
- if (content?.inputTranscription?.text && this.utteranceOpen) {
730
- this.transcript += content.inputTranscription.text;
823
+ const utterance = this.utterances[0];
824
+ if (content?.inputTranscription?.text && utterance) {
825
+ utterance.transcript += content.inputTranscription.text;
731
826
  }
732
- if (content?.generationComplete && this.utteranceEnded) {
733
- this.finishUtterance();
827
+ if (content?.generationComplete && utterance) {
828
+ if (utterance.state === "ended") this.give(utterance);
829
+ if (utterance.state === "given") this.remove(utterance);
734
830
  }
735
831
  if (message.goAway) {
736
832
  this.log("Connection ending in", message.goAway.timeLeft);
737
- if (this.utteranceOpen) {
738
- this.goingAway = true;
739
- } else {
740
- this.socket.renew();
741
- }
833
+ this.goingAway = true;
834
+ this.renewIfGoingAway();
742
835
  }
743
836
  };
744
837
  this.onReconnecting = () => {
838
+ this.goingAway = false;
745
839
  this.socket.clearPending();
746
- if (!this.utteranceOpen) return;
747
- this.transcript = "";
748
- this.socket.send({ realtimeInput: { activityStart: {} } });
749
- this.chunks.forEach((chunk) => this.sendAudio(chunk));
750
- if (this.utteranceEnded) {
751
- this.socket.send({ realtimeInput: { activityEnd: {} } });
840
+ for (const utterance of this.utterances) {
841
+ if (utterance.state === "given") this.remove(utterance);
752
842
  }
843
+ for (const utterance of this.utterances) {
844
+ if (utterance.timeout) clearTimeout(utterance.timeout);
845
+ utterance.timeout = void 0;
846
+ utterance.sent = false;
847
+ utterance.transcript = "";
848
+ }
849
+ this.sendFirst();
850
+ };
851
+ this.onFailed = () => {
852
+ const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
853
+ this.dropUtterances();
854
+ this.emit("Failed", chunks);
753
855
  };
754
856
  this.socket = new LiveSocket(
755
857
  options,
756
858
  () => this.buildSetup(),
757
859
  (...message) => this.log(...message)
758
860
  );
861
+ this.socket.on("Ready", this.onReady);
759
862
  this.socket.on("Message", this.onMessage);
760
863
  this.socket.on("Reconnecting", this.onReconnecting);
761
- this.socket.on("Failed", () => this.emit("Failed", this.chunks));
864
+ this.socket.on("Failed", this.onFailed);
762
865
  this.socket.connect();
763
866
  }
764
867
  transcribe(audioStream) {
868
+ let utterance;
765
869
  audioStream.on("data", (chunk) => {
766
- if (!this.utteranceOpen) this.openUtterance();
767
- this.chunks.push(chunk);
768
- this.sendAudio(chunk);
870
+ if (!utterance) utterance = this.openUtterance();
871
+ if (utterance.state !== "open") return;
872
+ utterance.chunks.push(chunk);
873
+ if (utterance.sent) this.sendAudio(chunk);
769
874
  });
770
875
  audioStream.on("end", () => {
771
- if (!this.utteranceOpen || this.utteranceEnded) return;
772
- this.utteranceEnded = true;
773
- this.socket.send({ realtimeInput: { activityEnd: {} } });
774
- this.clearTranscriptionTimeout();
775
- this.transcriptionTimeout = setTimeout(() => {
776
- this.transcriptionTimeout = void 0;
777
- this.log("Transcription timeout");
778
- this.finishUtterance();
779
- }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
876
+ if (utterance?.state === "open") this.endUtterance(utterance);
780
877
  });
781
878
  }
782
879
  destroy() {
783
880
  super.destroy();
784
- this.clearTranscriptionTimeout();
881
+ this.dropUtterances();
785
882
  this.socket.destroy();
786
883
  }
787
884
  buildSetup() {
788
885
  return {
789
886
  setup: {
790
- model: `models/${this.options.model || DEFAULT_MODEL3}`,
887
+ model: `models/${this.options.model || DEFAULT_MODEL4}`,
791
888
  generationConfig: { responseModalities: ["TEXT"] },
792
889
  inputAudioTranscription: {
793
890
  languageCodes: this.options.language ? [this.options.language] : [],
@@ -802,12 +899,38 @@ var GeminiSTT = class extends STT {
802
899
  };
803
900
  }
804
901
  openUtterance() {
805
- if (this.utteranceOpen) this.finishUtterance();
806
- this.utteranceOpen = true;
807
- this.utteranceEnded = false;
808
- this.chunks = [];
809
- this.transcript = "";
902
+ const last = this.utterances[this.utterances.length - 1];
903
+ if (last?.state === "open") this.endUtterance(last);
904
+ const utterance = {
905
+ state: "open",
906
+ sent: false,
907
+ chunks: [],
908
+ transcript: ""
909
+ };
910
+ this.utterances.push(utterance);
911
+ this.sendFirst();
912
+ return utterance;
913
+ }
914
+ endUtterance(utterance) {
915
+ utterance.state = "ended";
916
+ if (utterance.sent) this.sendEnd(utterance);
917
+ }
918
+ /** Sends the first utterance to Gemini, with the audio it held meanwhile */
919
+ sendFirst() {
920
+ const utterance = this.utterances[0];
921
+ if (!utterance || utterance.sent) return;
922
+ utterance.sent = true;
810
923
  this.socket.send({ realtimeInput: { activityStart: {} } });
924
+ utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
925
+ if (utterance.state === "ended") this.sendEnd(utterance);
926
+ }
927
+ sendEnd(utterance) {
928
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
929
+ utterance.timeout = setTimeout(() => {
930
+ utterance.timeout = void 0;
931
+ this.log("Transcription timeout");
932
+ this.give(utterance);
933
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
811
934
  }
812
935
  sendAudio(chunk) {
813
936
  this.socket.send({
@@ -819,45 +942,60 @@ var GeminiSTT = class extends STT {
819
942
  }
820
943
  });
821
944
  }
822
- /** Emits what was transcribed of the utterance, possibly nothing */
823
- finishUtterance() {
824
- this.clearTranscriptionTimeout();
825
- const transcript = this.transcript.trim();
826
- this.utteranceOpen = false;
827
- this.utteranceEnded = false;
828
- this.chunks = [];
829
- this.transcript = "";
945
+ /**
946
+ * Emits what was transcribed of an utterance, possibly nothing, and keeps
947
+ * it in line for what Gemini still sends about it
948
+ */
949
+ give(utterance) {
950
+ if (utterance.timeout) clearTimeout(utterance.timeout);
951
+ utterance.state = "given";
952
+ const transcript = utterance.transcript.trim();
830
953
  this.log(`Transcript: "${transcript}"`);
831
954
  this.emit("Transcript", transcript);
832
- if (this.goingAway) {
833
- this.goingAway = false;
834
- this.socket.renew();
835
- }
955
+ utterance.timeout = setTimeout(
956
+ () => this.remove(utterance),
957
+ this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
958
+ );
836
959
  }
837
- clearTranscriptionTimeout() {
838
- if (!this.transcriptionTimeout) return;
839
- clearTimeout(this.transcriptionTimeout);
840
- this.transcriptionTimeout = void 0;
960
+ remove(utterance) {
961
+ if (utterance.timeout) clearTimeout(utterance.timeout);
962
+ utterance.timeout = void 0;
963
+ this.utterances = this.utterances.filter((other) => other !== utterance);
964
+ this.renewIfGoingAway();
965
+ this.sendFirst();
966
+ }
967
+ /** Moves to a new connection once no utterance waits on this one */
968
+ renewIfGoingAway() {
969
+ if (!this.goingAway || this.utterances.length > 0) return;
970
+ this.goingAway = false;
971
+ this.socket.renew();
972
+ }
973
+ dropUtterances() {
974
+ for (const utterance of this.utterances) {
975
+ if (utterance.timeout) clearTimeout(utterance.timeout);
976
+ if (utterance.state === "open") utterance.state = "ended";
977
+ }
978
+ this.utterances = [];
841
979
  }
842
980
  };
843
981
 
844
982
  // src/GeminiTTS.ts
845
- import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
983
+ import { GoogleGenAI as GoogleGenAI3 } from "@google/genai";
846
984
  import { Pcm16Resampler as Pcm16Resampler2, SentenceTTS } from "@micdrop/server";
847
- var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
985
+ var DEFAULT_MODEL5 = "gemini-2.5-flash-preview-tts";
848
986
  var DEFAULT_VOICE = "Kore";
849
987
  var GEMINI_SAMPLE_RATE = 24e3;
850
988
  var OUTPUT_SAMPLE_RATE = 16e3;
851
- var DEFAULT_MAX_RETRY2 = 2;
989
+ var DEFAULT_MAX_RETRY = 2;
852
990
  var GeminiTTS = class extends SentenceTTS {
853
991
  constructor(options) {
854
992
  super();
855
993
  this.options = options;
856
- this.genai = "genai" in options ? options.genai : new GoogleGenAI2({ apiKey: options.apiKey });
994
+ this.genai = "genai" in options ? options.genai : new GoogleGenAI3({ apiKey: options.apiKey });
857
995
  }
858
996
  async synthesize(text, signal) {
859
997
  const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
860
- const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
998
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
861
999
  for (let attempt = 0; ; attempt++) {
862
1000
  const progress = { emitted: false };
863
1001
  try {
@@ -875,7 +1013,7 @@ var GeminiTTS = class extends SentenceTTS {
875
1013
  async synthesizeOnce(input, signal, progress) {
876
1014
  const stream = await this.genai.interactions.create(
877
1015
  {
878
- model: this.options.model || DEFAULT_MODEL4,
1016
+ model: this.options.model || DEFAULT_MODEL5,
879
1017
  store: false,
880
1018
  stream: true,
881
1019
  input,
@@ -911,12 +1049,16 @@ var GeminiTTS = class extends SentenceTTS {
911
1049
  }
912
1050
  };
913
1051
  function isBlocked(error) {
914
- return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
1052
+ return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
915
1053
  }
916
1054
  export {
917
1055
  GeminiAgent,
1056
+ GeminiClassifier,
918
1057
  GeminiLive,
919
1058
  GeminiSTT,
920
- GeminiTTS
1059
+ GeminiTTS,
1060
+ choice,
1061
+ noul,
1062
+ score
921
1063
  };
922
1064
  //# sourceMappingURL=index.mjs.map