@micdrop/gemini 1.0.1 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -31,9 +31,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
31
31
  var index_exports = {};
32
32
  __export(index_exports, {
33
33
  GeminiAgent: () => GeminiAgent,
34
+ GeminiClassifier: () => GeminiClassifier,
34
35
  GeminiLive: () => GeminiLive,
35
36
  GeminiSTT: () => GeminiSTT,
36
- GeminiTTS: () => GeminiTTS
37
+ GeminiTTS: () => GeminiTTS,
38
+ choice: () => choice,
39
+ noul: () => noul,
40
+ score: () => score
37
41
  });
38
42
  module.exports = __toCommonJS(index_exports);
39
43
 
@@ -54,127 +58,139 @@ var GeminiAgent = class extends import_server.Agent {
54
58
  this.signatures = /* @__PURE__ */ new Map();
55
59
  this.genai = "genai" in options ? options.genai : new import_genai.GoogleGenAI({ apiKey: options.apiKey });
56
60
  }
57
- async generateAnswer(stream, stepCount = 0, tryCount = 0) {
61
+ generateAnswer(stream, signal) {
62
+ return this.generateStep(stream, signal, 0);
63
+ }
64
+ /**
65
+ * Generates one step of the answer, retried on error.
66
+ *
67
+ * A retry starts the step over, so it only happens while nothing of the
68
+ * step reached the stream: what was said cannot be taken back, and saying
69
+ * it twice is worse than stopping there.
70
+ */
71
+ async generateStep(stream, signal, stepCount) {
58
72
  if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
59
73
  console.error("[GeminiAgent] Max steps reached");
60
74
  return;
61
75
  }
62
- const abortController = new AbortController();
63
- this.abortController = abortController;
64
- const signal = abortController.signal;
76
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
77
+ for (let tryCount = 0; ; tryCount++) {
78
+ const progress = { answer: "" };
79
+ try {
80
+ const step = await this.streamStep(stream, signal, progress);
81
+ if (signal.aborted) return;
82
+ this.recordAnswer(progress.answer);
83
+ if (step.calledTool && !step.skipAnswer && !progress.answer) {
84
+ await this.generateStep(stream, signal, stepCount + 1);
85
+ }
86
+ return;
87
+ } catch (error) {
88
+ if (signal.aborted) return;
89
+ console.error("[GeminiAgent] Error answering:", error);
90
+ if (progress.answer) {
91
+ this.log("Error after part of the answer was said, keeping it");
92
+ this.recordAnswer(progress.answer);
93
+ return;
94
+ }
95
+ if (tryCount >= maxRetry) {
96
+ this.log("Max retries reached, giving up");
97
+ this.emit("Failed");
98
+ return;
99
+ }
100
+ await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
101
+ if (signal.aborted) return;
102
+ }
103
+ }
104
+ }
105
+ /** Streams one request, writing the text of the answer as it comes */
106
+ async streamStep(stream, signal, progress) {
65
107
  let extracting = false;
66
108
  const extractOptions = this.getExtractOptions();
67
- try {
68
- const response = await this.genai.interactions.create(
69
- {
70
- model: this.options.model || DEFAULT_MODEL,
71
- store: false,
72
- stream: true,
73
- system_instruction: this.buildSystemInstruction(),
74
- input: this.buildInput(),
75
- tools: this.buildTools(),
76
- ...this.options.thinkingLevel || this.options.settings ? {
77
- generation_config: {
78
- ...this.options.settings,
79
- ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
80
- }
81
- } : {}
82
- },
83
- { fetchOptions: { signal } }
84
- );
85
- const steps = [];
86
- let answer = "";
87
- let skipAnswer = false;
88
- let calledTool = false;
89
- let signature = "";
90
- for await (const event of response) {
91
- if (abortController !== this.abortController) return;
92
- switch (event.event_type) {
93
- case "step.start":
94
- steps[event.index] = {
95
- type: event.step.type,
96
- id: event.step.id,
97
- name: event.step.name,
98
- text: "",
99
- arguments: "",
100
- signature: ""
101
- };
102
- break;
103
- case "step.delta": {
104
- const step = steps[event.index];
105
- if (!step) break;
106
- const delta = event.delta;
107
- if (delta.type === "thought_signature") {
108
- step.signature += delta.signature ?? "";
109
- } else if (delta.type === "arguments_delta") {
110
- step.arguments += delta.arguments ?? "";
111
- } else if (delta.type === "text" && step.type === "model_output") {
112
- this.log(`Answer chunk: "${delta.text}"`);
113
- answer += delta.text;
114
- if (extractOptions) {
115
- if (extracting) break;
116
- const startTagIndex = delta.text.indexOf(
117
- extractOptions.startTag
118
- );
119
- if (startTagIndex !== -1) {
120
- extracting = true;
121
- stream.write(delta.text.slice(0, startTagIndex).trimEnd());
122
- break;
123
- }
109
+ const response = await this.genai.interactions.create(
110
+ {
111
+ model: this.options.model || DEFAULT_MODEL,
112
+ store: false,
113
+ stream: true,
114
+ system_instruction: this.buildSystemInstruction(),
115
+ input: this.buildInput(),
116
+ tools: this.buildTools(),
117
+ ...this.options.thinkingLevel || this.options.settings ? {
118
+ generation_config: {
119
+ ...this.options.settings,
120
+ ...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
121
+ }
122
+ } : {}
123
+ },
124
+ { signal }
125
+ );
126
+ const steps = [];
127
+ let skipAnswer = false;
128
+ let calledTool = false;
129
+ let signature = "";
130
+ for await (const event of response) {
131
+ if (signal.aborted) break;
132
+ switch (event.event_type) {
133
+ case "step.start":
134
+ steps[event.index] = {
135
+ type: event.step.type,
136
+ id: event.step.id,
137
+ name: event.step.name,
138
+ text: "",
139
+ arguments: "",
140
+ signature: ""
141
+ };
142
+ break;
143
+ case "step.delta": {
144
+ const step = steps[event.index];
145
+ if (!step) break;
146
+ const delta = event.delta;
147
+ if (delta.type === "thought_signature") {
148
+ step.signature += delta.signature ?? "";
149
+ } else if (delta.type === "arguments_delta") {
150
+ step.arguments += delta.arguments ?? "";
151
+ } else if (delta.type === "text" && step.type === "model_output") {
152
+ this.log(`Answer chunk: "${delta.text}"`);
153
+ progress.answer += delta.text;
154
+ if (extractOptions) {
155
+ if (extracting) break;
156
+ const startTagIndex = delta.text.indexOf(extractOptions.startTag);
157
+ if (startTagIndex !== -1) {
158
+ extracting = true;
159
+ stream.write(delta.text.slice(0, startTagIndex).trimEnd());
160
+ break;
124
161
  }
125
- stream.write(delta.text);
126
162
  }
127
- break;
128
- }
129
- case "step.stop": {
130
- const step = steps[event.index];
131
- if (step?.type === "thought") signature = step.signature;
132
- if (step?.type !== "function_call" || !step.id || !step.name) break;
133
- calledTool = true;
134
- if (signature) this.signatures.set(step.id, signature);
135
- const result = await this.executeTool({
136
- role: "tool_call",
137
- toolCallId: step.id,
138
- toolName: step.name,
139
- parameters: step.arguments || "{}"
140
- });
141
- if (result.skipAnswer) skipAnswer = true;
142
- break;
163
+ stream.write(delta.text);
143
164
  }
144
- case "error":
145
- throw new Error(event.error?.message || "Generation failed");
146
- default:
147
- break;
165
+ break;
148
166
  }
149
- }
150
- if (abortController !== this.abortController) return;
151
- this.abortController = void 0;
152
- if (answer) {
153
- const { message, metadata } = this.extract(answer);
154
- this.addAssistantMessage(message, metadata);
155
- }
156
- if (calledTool && !skipAnswer && !answer) {
157
- await this.generateAnswer(stream, stepCount + 1);
158
- }
159
- } catch (error) {
160
- if (signal.aborted) return;
161
- console.error("[GeminiAgent] Error answering:", error);
162
- if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
163
- await new Promise(
164
- (resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
165
- );
166
- await this.generateAnswer(stream, stepCount, tryCount + 1);
167
- } else {
168
- this.log("Max retries reached, giving up");
169
- this.emit("Failed");
167
+ case "step.stop": {
168
+ const step = steps[event.index];
169
+ if (step?.type === "thought") signature = step.signature;
170
+ if (step?.type !== "function_call" || !step.id || !step.name) break;
171
+ calledTool = true;
172
+ if (signature) this.signatures.set(step.id, signature);
173
+ const result = await this.executeTool({
174
+ role: "tool_call",
175
+ toolCallId: step.id,
176
+ toolName: step.name,
177
+ parameters: step.arguments || "{}"
178
+ });
179
+ if (result.skipAnswer) skipAnswer = true;
180
+ break;
181
+ }
182
+ case "error":
183
+ throw new Error(event.error?.message || "Generation failed");
184
+ default:
185
+ break;
170
186
  }
171
187
  }
188
+ return { calledTool, skipAnswer };
172
189
  }
173
- cancel() {
174
- if (!this.abortController) return;
175
- this.log("Cancel");
176
- this.abortController.abort();
177
- this.abortController = void 0;
190
+ recordAnswer(answer) {
191
+ if (!answer) return;
192
+ const { message, metadata } = this.extract(answer);
193
+ this.addAssistantMessage(message, metadata);
178
194
  }
179
195
  buildSystemInstruction() {
180
196
  return this.conversation.flatMap(
@@ -246,6 +262,17 @@ var GeminiAgent = class extends import_server.Agent {
246
262
  });
247
263
  }
248
264
  };
265
+ function sleep(ms, signal) {
266
+ return new Promise((resolve) => {
267
+ const done = () => {
268
+ clearTimeout(timer);
269
+ signal.removeEventListener("abort", done);
270
+ resolve();
271
+ };
272
+ const timer = setTimeout(done, ms);
273
+ signal.addEventListener("abort", done);
274
+ });
275
+ }
249
276
  function parseArguments(parameters) {
250
277
  try {
251
278
  return JSON.parse(parameters);
@@ -254,72 +281,176 @@ function parseArguments(parameters) {
254
281
  }
255
282
  }
256
283
 
257
- // src/GeminiLive.ts
284
+ // src/GeminiClassifier.ts
285
+ var import_genai2 = require("@google/genai");
258
286
  var import_server2 = require("@micdrop/server");
287
+ function noul(instructions, criteria) {
288
+ return {
289
+ type: "noul",
290
+ instructions,
291
+ criteria: criteria && {
292
+ true_description: criteria.true,
293
+ false_description: criteria.false
294
+ }
295
+ };
296
+ }
297
+ function choice(instructions, criteria) {
298
+ return { type: "choice", instructions, criteria };
299
+ }
300
+ function score(instructions, levels) {
301
+ return { type: "score", instructions, criteria: levels };
302
+ }
303
+ var DEFAULT_MODEL2 = "gemini-decision-1";
304
+ var DEFAULT_TIMEOUT = 3e3;
305
+ var API_VERSION = "v1beta";
306
+ var GeminiClassifier = class extends import_server2.Classifier {
307
+ constructor(options) {
308
+ super();
309
+ this.options = options;
310
+ this.genai = "genai" in options ? options.genai : new import_genai2.GoogleGenAI({ apiKey: options.apiKey });
311
+ }
312
+ async evaluate(input, signal) {
313
+ const { questions, model, timeout } = this.options;
314
+ const asked = typeof questions === "function" ? questions(input) : questions;
315
+ const decision = await this.genai.decisions.create(
316
+ {
317
+ model: model || DEFAULT_MODEL2,
318
+ input: this.buildInput(input),
319
+ questions: asked,
320
+ // Only on v1beta, while the client defaults to v1
321
+ api_version: API_VERSION
322
+ },
323
+ {
324
+ // The SDK drops its timeout when a signal is given
325
+ signal: AbortSignal.any([
326
+ signal,
327
+ AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT)
328
+ ]),
329
+ retries: { strategy: "none" }
330
+ }
331
+ );
332
+ const blocked = decision.prompt_feedback?.block_reason;
333
+ if (blocked) {
334
+ const message = decision.prompt_feedback?.block_reason_message;
335
+ throw new Error(
336
+ `Input blocked by Gemini (${blocked})${message ? `: ${message}` : ""}`
337
+ );
338
+ }
339
+ return {
340
+ answers: decision.answers ?? {},
341
+ usage: decision.usage
342
+ };
343
+ }
344
+ buildInput(input) {
345
+ if (this.options.input) return this.options.input(input);
346
+ return typeof input === "string" ? input : JSON.stringify(input);
347
+ }
348
+ };
349
+
350
+ // src/GeminiLive.ts
351
+ var import_server4 = require("@micdrop/server");
259
352
  var import_zod2 = require("zod");
260
353
 
261
354
  // src/LiveSocket.ts
355
+ var import_server3 = require("@micdrop/server");
262
356
  var import_eventemitter3 = require("eventemitter3");
263
- var import_ws = __toESM(require("ws"));
264
357
  var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
265
358
  var SAMPLE_RATE = 16e3;
266
359
  var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
267
- var DEFAULT_CONNECTION_TIMEOUT = 5e3;
268
- var DEFAULT_RETRY_DELAY2 = 1e3;
269
- var DEFAULT_MAX_RETRY = 3;
270
360
  var LiveSocket = class extends import_eventemitter3.EventEmitter {
271
361
  constructor(options, buildSetup, log) {
272
362
  super();
273
- this.options = options;
274
363
  this.buildSetup = buildSetup;
275
364
  this.log = log;
276
- this.connected = false;
277
- this.destroyed = false;
278
365
  // Messages waiting for the session, or for the end of a turn to be due
279
366
  this.pending = [];
280
367
  // Audio of the turn as it went out, see flush()
281
368
  this.turnAudioStartedAt = 0;
282
369
  this.turnAudioMs = 0;
283
- this.retryCount = 0;
370
+ // Attempts since the last session was ready, for the owner to tell a drop
371
+ // from a resumption that keeps failing
372
+ this.reconnectAttempts = 0;
373
+ /** Sends the setup, and holds the connection back until Gemini accepts it */
374
+ this.handshake = {
375
+ start: (socket) => {
376
+ const setup = this.buildSetup();
377
+ this.setup = setup;
378
+ socket.send(JSON.stringify(setup));
379
+ },
380
+ read: (data) => !!parseMessage(data.toString())?.setupComplete
381
+ };
382
+ this.onOpen = () => {
383
+ this.log("Session ready");
384
+ this.reconnectAttempts = 0;
385
+ this.emit("Ready", this.setup);
386
+ this.flush();
387
+ };
388
+ this.onMessage = (data) => {
389
+ const message = parseMessage(data.toString());
390
+ if (message) this.emit("Message", message);
391
+ };
392
+ this.onStatusChange = (status) => {
393
+ if (status !== "open") this.clearPendingTimeout();
394
+ if (status === "reconnecting") {
395
+ this.emit("Reconnecting", ++this.reconnectAttempts);
396
+ }
397
+ };
398
+ this.onFailed = () => {
399
+ this.clearPending();
400
+ this.emit("Failed");
401
+ };
402
+ this.socket = new import_server3.ReconnectingSocket({
403
+ url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
404
+ connectionTimeout: options.connectionTimeout,
405
+ retryDelay: options.retryDelay,
406
+ maxRetry: options.maxRetry,
407
+ handshake: this.handshake,
408
+ log
409
+ });
410
+ this.socket.on("Open", this.onOpen);
411
+ this.socket.on("Message", this.onMessage);
412
+ this.socket.on("StatusChange", this.onStatusChange);
413
+ this.socket.on("Failed", this.onFailed);
284
414
  }
285
415
  /** The session is set up and takes messages right away */
286
416
  get isConnected() {
287
- return this.connected;
417
+ return this.socket.isOpen;
288
418
  }
289
419
  connect() {
290
- this.initWS().catch((error) => {
291
- this.log("Connection error:", error);
292
- this.reconnect();
293
- });
420
+ this.socket.connect();
294
421
  }
295
- /** Sends a message once the session is ready, after the ones before it */
422
+ /**
423
+ * Sends a message once the session is ready, after the ones before it.
424
+ * Dropped once the connection failed for good, as nothing would send it.
425
+ */
296
426
  send(message) {
427
+ const status = this.socket.status;
428
+ if (status === "failed" || status === "destroyed") return;
297
429
  this.pending.push(message);
298
430
  this.flush();
299
431
  }
300
432
  /** Sends a message ahead of the waiting ones, from a Ready listener */
301
433
  sendFirst(message) {
302
- this.socket?.send(JSON.stringify(message));
434
+ this.socket.send(JSON.stringify(message));
303
435
  }
304
436
  /** Drops the messages that have not gone out */
305
437
  clearPending() {
306
438
  this.pending = [];
307
439
  this.clearPendingTimeout();
308
440
  }
309
- /** Moves to a new connection, keeping the messages that have not gone out */
441
+ /**
442
+ * Moves to a new connection, keeping the messages that have not gone out.
443
+ * A reconnection waiting for its delay is replaced by this one.
444
+ */
310
445
  renew() {
311
446
  this.log("Renewing the connection");
312
- this.closeSocket();
313
- this.connect();
447
+ this.clearPendingTimeout();
448
+ this.reconnectAttempts = 0;
449
+ this.socket.restart();
314
450
  }
315
451
  destroy() {
316
- this.destroyed = true;
317
- if (this.reconnectTimeout) {
318
- clearTimeout(this.reconnectTimeout);
319
- this.reconnectTimeout = void 0;
320
- }
321
- this.closeSocket();
322
- this.pending = [];
452
+ this.clearPending();
453
+ this.socket.destroy();
323
454
  this.removeAllListeners();
324
455
  }
325
456
  /**
@@ -332,8 +463,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
332
463
  */
333
464
  flush() {
334
465
  while (this.pending.length > 0 && !this.pendingTimeout) {
335
- const socket = this.socket;
336
- if (!this.connected || socket?.readyState !== import_ws.default.OPEN) return;
466
+ if (!this.socket.isOpen) return;
337
467
  const message = this.pending[0];
338
468
  const input = message.realtimeInput;
339
469
  if (input?.activityEnd) {
@@ -353,7 +483,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
353
483
  this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
354
484
  }
355
485
  this.pending.shift();
356
- socket.send(JSON.stringify(message));
486
+ this.socket.send(JSON.stringify(message));
357
487
  }
358
488
  }
359
489
  clearPendingTimeout() {
@@ -361,112 +491,46 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
361
491
  clearTimeout(this.pendingTimeout);
362
492
  this.pendingTimeout = void 0;
363
493
  }
364
- initWS() {
365
- return new Promise((resolve, reject) => {
366
- const socket = new import_ws.default(
367
- `${URL}?key=${encodeURIComponent(this.options.apiKey)}`
368
- );
369
- this.socket = socket;
370
- const timeout = setTimeout(() => {
371
- this.log("Connection timeout");
372
- this.closeSocket();
373
- reject(new Error("WebSocket connection timeout"));
374
- }, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
375
- socket.on("open", () => {
376
- this.log("Connection opened");
377
- socket.send(JSON.stringify(this.buildSetup()));
378
- });
379
- socket.on("error", (error) => {
380
- clearTimeout(timeout);
381
- this.log("WebSocket error:", error);
382
- reject(new Error("WebSocket connection error"));
383
- });
384
- socket.on("close", (code, reason) => {
385
- clearTimeout(timeout);
386
- const wasConnected = this.connected;
387
- this.closeSocket();
388
- if (this.destroyed) return;
389
- this.log("Connection closed", { code, reason: reason.toString() });
390
- if (wasConnected) {
391
- this.reconnect();
392
- } else {
393
- reject(new Error(`Connection closed: ${reason.toString()}`));
394
- }
395
- });
396
- socket.on("message", (data) => {
397
- const message = JSON.parse(data.toString());
398
- if (message.setupComplete) {
399
- clearTimeout(timeout);
400
- this.log("Session ready");
401
- this.connected = true;
402
- this.retryCount = 0;
403
- this.emit("Ready");
404
- this.flush();
405
- resolve();
406
- return;
407
- }
408
- this.emit("Message", message);
409
- });
410
- });
411
- }
412
- closeSocket() {
413
- const socket = this.socket;
414
- this.socket = void 0;
415
- this.connected = false;
416
- this.clearPendingTimeout();
417
- if (!socket) return;
418
- socket.removeAllListeners();
419
- socket.on("error", () => {
420
- });
421
- if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
422
- socket.close(1e3);
423
- }
424
- }
425
- reconnect() {
426
- if (this.destroyed) return;
427
- this.retryCount++;
428
- if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
429
- this.log("Max retries reached, giving up");
430
- this.emit("Failed");
431
- return;
432
- }
433
- this.emit("Reconnecting", this.retryCount);
434
- this.log("Reconnecting...");
435
- this.reconnectTimeout = setTimeout(() => {
436
- this.reconnectTimeout = void 0;
437
- this.connect();
438
- }, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
439
- }
440
494
  };
495
+ function parseMessage(text) {
496
+ try {
497
+ return JSON.parse(text);
498
+ } catch {
499
+ return void 0;
500
+ }
501
+ }
441
502
 
442
503
  // src/GeminiLive.ts
443
- var DEFAULT_MODEL2 = "gemini-3.8-live";
504
+ var DEFAULT_MODEL3 = "gemini-3.8-live";
444
505
  var SAMPLE_RATE2 = 16e3;
445
506
  var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
446
507
  var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
508
+ function newToolRound() {
509
+ return { running: 0, answerEndDeferred: false };
510
+ }
447
511
  var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
448
- var GeminiLive = class extends import_server2.Realtime {
512
+ var GeminiLive = class extends import_server4.Realtime {
449
513
  constructor(options) {
450
514
  super(options);
451
515
  this.everConnected = false;
452
- // Whether the setup being sent resumes the session
453
- this.resuming = false;
454
516
  this.goingAway = false;
455
- this.outputResampler = new import_server2.Pcm16Resampler(
517
+ this.outputResampler = new import_server4.Pcm16Resampler(
456
518
  GEMINI_OUTPUT_SAMPLE_RATE,
457
519
  SAMPLE_RATE2
458
520
  );
459
521
  this.hasTranscribedTurn = false;
460
522
  // Transcript of a dropped turn, which the conversation should not show
461
523
  this.ignoreUserTranscript = false;
524
+ // Calls of the session Gemini told not to answer
462
525
  this.cancelledToolCalls = /* @__PURE__ */ new Set();
463
- // Gemini ends its turn when it calls a tool, and speaks again once it has
464
- // the result. The answer stays open in between, so the client keeps waiting
465
- // for it rather than listening again.
466
- this.runningTools = 0;
467
- this.answerEndDeferred = false;
468
- this.onReady = () => {
469
- if (this.everConnected && !this.resuming) {
526
+ // Bumped by every session that does not resume the previous one, whose tool
527
+ // calls it cannot answer
528
+ this.session = 0;
529
+ this.toolRound = newToolRound();
530
+ this.onReady = (setup) => {
531
+ const resumed = !!setup?.setup?.sessionResumption?.handle;
532
+ if (this.everConnected && !resumed) {
533
+ this.startFreshSession();
470
534
  const turns = this.conversation.flatMap(
471
535
  (message) => message.role === "user" || message.role === "assistant" ? [
472
536
  {
@@ -484,7 +548,12 @@ var GeminiLive = class extends import_server2.Realtime {
484
548
  };
485
549
  this.onReconnecting = (attempt) => {
486
550
  if (attempt > 1) this.resumptionHandle = void 0;
487
- if (!this.resumptionHandle) this.socket.clearPending();
551
+ if (!this.resumptionHandle) {
552
+ this.socket.clearPending();
553
+ this.startFreshSession();
554
+ }
555
+ this.toolRound.answerEndDeferred = false;
556
+ this.commitTurnTranscript();
488
557
  this.endAnswer();
489
558
  };
490
559
  this.onMessage = (message) => {
@@ -522,9 +591,16 @@ var GeminiLive = class extends import_server2.Realtime {
522
591
  this.socket.on("Failed", () => this.emit("Failed"));
523
592
  this.socket.connect();
524
593
  }
594
+ /**
595
+ * Stops the answer. The tools it is waiting for still send their results,
596
+ * silently, since a blocking call left unanswered would stall the model.
597
+ */
598
+ cancel() {
599
+ super.cancel();
600
+ this.resetToolRound();
601
+ }
525
602
  destroy() {
526
603
  super.destroy();
527
- this.clearToolFollowUp();
528
604
  this.socket.destroy();
529
605
  }
530
606
  openTurn() {
@@ -559,11 +635,9 @@ var GeminiLive = class extends import_server2.Realtime {
559
635
  }
560
636
  cancelAnswer() {
561
637
  this.outputResampler.reset();
562
- this.clearToolFollowUp();
563
- this.answerEndDeferred = false;
564
638
  }
565
639
  get isIdle() {
566
- return super.isIdle && this.runningTools === 0;
640
+ return super.isIdle && this.toolRound.running === 0;
567
641
  }
568
642
  updateTools() {
569
643
  if (this.everConnected) {
@@ -595,7 +669,6 @@ var GeminiLive = class extends import_server2.Realtime {
595
669
  this.socket.send(message);
596
670
  }
597
671
  buildSetup() {
598
- this.resuming = !!this.resumptionHandle;
599
672
  const functionDeclarations = this.tools.map((tool) => ({
600
673
  name: tool.name,
601
674
  description: tool.description,
@@ -603,7 +676,7 @@ var GeminiLive = class extends import_server2.Realtime {
603
676
  }));
604
677
  return {
605
678
  setup: {
606
- model: `models/${this.options.model || DEFAULT_MODEL2}`,
679
+ model: `models/${this.options.model || DEFAULT_MODEL3}`,
607
680
  generationConfig: {
608
681
  responseModalities: ["AUDIO"],
609
682
  ...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
@@ -629,6 +702,22 @@ var GeminiLive = class extends import_server2.Realtime {
629
702
  }
630
703
  };
631
704
  }
705
+ /** Forgets what belonged to the previous session */
706
+ startFreshSession() {
707
+ this.session++;
708
+ this.resetToolRound();
709
+ this.cancelledToolCalls.clear();
710
+ this.ignoreUserTranscript = false;
711
+ }
712
+ /**
713
+ * Starts a new round of tools. The tools still running belong to the old
714
+ * one: their results go out silently, and nothing waits for the model to
715
+ * talk about them.
716
+ */
717
+ resetToolRound() {
718
+ if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
719
+ this.toolRound = newToolRound();
720
+ }
632
721
  /** Moves to a new connection before Gemini closes this one */
633
722
  renewIfGoingAway() {
634
723
  if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
@@ -660,8 +749,8 @@ var GeminiLive = class extends import_server2.Realtime {
660
749
  if (content.turnComplete) {
661
750
  this.commitTurnTranscript();
662
751
  this.outputResampler.reset();
663
- if (this.runningTools > 0 || this.toolFollowUp) {
664
- this.answerEndDeferred = true;
752
+ if (this.toolRound.running > 0 || this.toolRound.followUp) {
753
+ this.toolRound.answerEndDeferred = true;
665
754
  } else {
666
755
  this.endAnswer();
667
756
  this.renewIfGoingAway();
@@ -670,24 +759,22 @@ var GeminiLive = class extends import_server2.Realtime {
670
759
  }
671
760
  onModelOutput() {
672
761
  this.commitTurnTranscript();
673
- if (this.toolFollowUp) {
674
- this.clearToolFollowUp();
675
- this.answerEndDeferred = false;
762
+ const round = this.toolRound;
763
+ if (round.followUp) {
764
+ clearTimeout(round.followUp);
765
+ round.followUp = void 0;
766
+ round.answerEndDeferred = false;
676
767
  }
677
768
  }
678
769
  /** Ends the answer the model stopped giving to wait for a tool */
679
770
  endDeferredAnswer() {
680
- if (!this.answerEndDeferred) return;
681
- if (this.runningTools > 0 || this.toolFollowUp) return;
682
- this.answerEndDeferred = false;
771
+ const round = this.toolRound;
772
+ if (!round.answerEndDeferred) return;
773
+ if (round.running > 0 || round.followUp) return;
774
+ round.answerEndDeferred = false;
683
775
  this.endAnswer();
684
776
  this.renewIfGoingAway();
685
777
  }
686
- clearToolFollowUp() {
687
- if (!this.toolFollowUp) return;
688
- clearTimeout(this.toolFollowUp);
689
- this.toolFollowUp = void 0;
690
- }
691
778
  /**
692
779
  * The transcript of the turn comes before the answer, so the first piece of
693
780
  * the answer closes it.
@@ -705,17 +792,22 @@ var GeminiLive = class extends import_server2.Realtime {
705
792
  toolName: call.name,
706
793
  parameters: JSON.stringify(call.args ?? {})
707
794
  };
708
- this.runningTools++;
795
+ const round = this.toolRound;
796
+ const session = this.session;
797
+ round.running++;
709
798
  let result;
710
799
  try {
711
800
  result = await this.runTool(toolCall);
712
801
  } finally {
713
- this.runningTools--;
802
+ round.running--;
714
803
  }
804
+ if (session !== this.session) return;
715
805
  if (this.cancelledToolCalls.delete(call.id)) {
716
806
  this.endDeferredAnswer();
717
807
  return;
718
808
  }
809
+ const interrupted = round !== this.toolRound;
810
+ const silent = result.skipAnswer || interrupted;
719
811
  this.send({
720
812
  toolResponse: {
721
813
  functionResponses: [
@@ -725,16 +817,17 @@ var GeminiLive = class extends import_server2.Realtime {
725
817
  response: {
726
818
  output: result.output ?? null,
727
819
  // A tool without an answer lets the model know silently
728
- scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
820
+ scheduling: silent ? "SILENT" : "WHEN_IDLE"
729
821
  }
730
822
  }
731
823
  ]
732
824
  }
733
825
  });
826
+ if (interrupted) return;
734
827
  if (!result.skipAnswer) {
735
- this.clearToolFollowUp();
736
- this.toolFollowUp = setTimeout(() => {
737
- this.toolFollowUp = void 0;
828
+ if (round.followUp) clearTimeout(round.followUp);
829
+ round.followUp = setTimeout(() => {
830
+ round.followUp = void 0;
738
831
  this.log("Nothing said about the tool result");
739
832
  this.endDeferredAnswer();
740
833
  }, TOOL_FOLLOW_UP_TIMEOUT);
@@ -745,85 +838,91 @@ var GeminiLive = class extends import_server2.Realtime {
745
838
  };
746
839
 
747
840
  // src/GeminiSTT.ts
748
- var import_server3 = require("@micdrop/server");
749
- var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
841
+ var import_server5 = require("@micdrop/server");
842
+ var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
750
843
  var SAMPLE_RATE3 = 16e3;
751
844
  var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
752
- var GeminiSTT = class extends import_server3.STT {
845
+ var GeminiSTT = class extends import_server5.STT {
753
846
  constructor(options) {
754
847
  super();
755
848
  this.options = options;
849
+ // Gemini closes the session soon, and it is renewed once nothing waits
756
850
  this.goingAway = false;
757
- // Audio of the utterance, sent again if the connection drops before its
758
- // transcript comes back
759
- this.chunks = [];
760
- this.utteranceOpen = false;
761
- this.utteranceEnded = false;
762
- this.transcript = "";
851
+ // Utterances waiting for their transcript, in the order they were spoken.
852
+ // Gemini names no utterance in what it sends, and goes on to the next one
853
+ // before it is done with the previous. So it hears one at a time: the first
854
+ // one, and the audio of the next waits here until its transcript is in.
855
+ this.utterances = [];
856
+ this.onReady = () => {
857
+ this.goingAway = false;
858
+ };
763
859
  this.onMessage = (message) => {
764
860
  const content = message.serverContent;
765
- if (content?.inputTranscription?.text && this.utteranceOpen) {
766
- this.transcript += content.inputTranscription.text;
861
+ const utterance = this.utterances[0];
862
+ if (content?.inputTranscription?.text && utterance) {
863
+ utterance.transcript += content.inputTranscription.text;
767
864
  }
768
- if (content?.generationComplete && this.utteranceEnded) {
769
- this.finishUtterance();
865
+ if (content?.generationComplete && utterance) {
866
+ if (utterance.state === "ended") this.give(utterance);
867
+ if (utterance.state === "given") this.remove(utterance);
770
868
  }
771
869
  if (message.goAway) {
772
870
  this.log("Connection ending in", message.goAway.timeLeft);
773
- if (this.utteranceOpen) {
774
- this.goingAway = true;
775
- } else {
776
- this.socket.renew();
777
- }
871
+ this.goingAway = true;
872
+ this.renewIfGoingAway();
778
873
  }
779
874
  };
780
875
  this.onReconnecting = () => {
876
+ this.goingAway = false;
781
877
  this.socket.clearPending();
782
- if (!this.utteranceOpen) return;
783
- this.transcript = "";
784
- this.socket.send({ realtimeInput: { activityStart: {} } });
785
- this.chunks.forEach((chunk) => this.sendAudio(chunk));
786
- if (this.utteranceEnded) {
787
- this.socket.send({ realtimeInput: { activityEnd: {} } });
878
+ for (const utterance of this.utterances) {
879
+ if (utterance.state === "given") this.remove(utterance);
788
880
  }
881
+ for (const utterance of this.utterances) {
882
+ if (utterance.timeout) clearTimeout(utterance.timeout);
883
+ utterance.timeout = void 0;
884
+ utterance.sent = false;
885
+ utterance.transcript = "";
886
+ }
887
+ this.sendFirst();
888
+ };
889
+ this.onFailed = () => {
890
+ const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
891
+ this.dropUtterances();
892
+ this.emit("Failed", chunks);
789
893
  };
790
894
  this.socket = new LiveSocket(
791
895
  options,
792
896
  () => this.buildSetup(),
793
897
  (...message) => this.log(...message)
794
898
  );
899
+ this.socket.on("Ready", this.onReady);
795
900
  this.socket.on("Message", this.onMessage);
796
901
  this.socket.on("Reconnecting", this.onReconnecting);
797
- this.socket.on("Failed", () => this.emit("Failed", this.chunks));
902
+ this.socket.on("Failed", this.onFailed);
798
903
  this.socket.connect();
799
904
  }
800
905
  transcribe(audioStream) {
906
+ let utterance;
801
907
  audioStream.on("data", (chunk) => {
802
- if (!this.utteranceOpen) this.openUtterance();
803
- this.chunks.push(chunk);
804
- this.sendAudio(chunk);
908
+ if (!utterance) utterance = this.openUtterance();
909
+ if (utterance.state !== "open") return;
910
+ utterance.chunks.push(chunk);
911
+ if (utterance.sent) this.sendAudio(chunk);
805
912
  });
806
913
  audioStream.on("end", () => {
807
- if (!this.utteranceOpen || this.utteranceEnded) return;
808
- this.utteranceEnded = true;
809
- this.socket.send({ realtimeInput: { activityEnd: {} } });
810
- this.clearTranscriptionTimeout();
811
- this.transcriptionTimeout = setTimeout(() => {
812
- this.transcriptionTimeout = void 0;
813
- this.log("Transcription timeout");
814
- this.finishUtterance();
815
- }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
914
+ if (utterance?.state === "open") this.endUtterance(utterance);
816
915
  });
817
916
  }
818
917
  destroy() {
819
918
  super.destroy();
820
- this.clearTranscriptionTimeout();
919
+ this.dropUtterances();
821
920
  this.socket.destroy();
822
921
  }
823
922
  buildSetup() {
824
923
  return {
825
924
  setup: {
826
- model: `models/${this.options.model || DEFAULT_MODEL3}`,
925
+ model: `models/${this.options.model || DEFAULT_MODEL4}`,
827
926
  generationConfig: { responseModalities: ["TEXT"] },
828
927
  inputAudioTranscription: {
829
928
  languageCodes: this.options.language ? [this.options.language] : [],
@@ -838,12 +937,38 @@ var GeminiSTT = class extends import_server3.STT {
838
937
  };
839
938
  }
840
939
  openUtterance() {
841
- if (this.utteranceOpen) this.finishUtterance();
842
- this.utteranceOpen = true;
843
- this.utteranceEnded = false;
844
- this.chunks = [];
845
- this.transcript = "";
940
+ const last = this.utterances[this.utterances.length - 1];
941
+ if (last?.state === "open") this.endUtterance(last);
942
+ const utterance = {
943
+ state: "open",
944
+ sent: false,
945
+ chunks: [],
946
+ transcript: ""
947
+ };
948
+ this.utterances.push(utterance);
949
+ this.sendFirst();
950
+ return utterance;
951
+ }
952
+ endUtterance(utterance) {
953
+ utterance.state = "ended";
954
+ if (utterance.sent) this.sendEnd(utterance);
955
+ }
956
+ /** Sends the first utterance to Gemini, with the audio it held meanwhile */
957
+ sendFirst() {
958
+ const utterance = this.utterances[0];
959
+ if (!utterance || utterance.sent) return;
960
+ utterance.sent = true;
846
961
  this.socket.send({ realtimeInput: { activityStart: {} } });
962
+ utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
963
+ if (utterance.state === "ended") this.sendEnd(utterance);
964
+ }
965
+ sendEnd(utterance) {
966
+ this.socket.send({ realtimeInput: { activityEnd: {} } });
967
+ utterance.timeout = setTimeout(() => {
968
+ utterance.timeout = void 0;
969
+ this.log("Transcription timeout");
970
+ this.give(utterance);
971
+ }, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
847
972
  }
848
973
  sendAudio(chunk) {
849
974
  this.socket.send({
@@ -855,45 +980,60 @@ var GeminiSTT = class extends import_server3.STT {
855
980
  }
856
981
  });
857
982
  }
858
- /** Emits what was transcribed of the utterance, possibly nothing */
859
- finishUtterance() {
860
- this.clearTranscriptionTimeout();
861
- const transcript = this.transcript.trim();
862
- this.utteranceOpen = false;
863
- this.utteranceEnded = false;
864
- this.chunks = [];
865
- this.transcript = "";
983
+ /**
984
+ * Emits what was transcribed of an utterance, possibly nothing, and keeps
985
+ * it in line for what Gemini still sends about it
986
+ */
987
+ give(utterance) {
988
+ if (utterance.timeout) clearTimeout(utterance.timeout);
989
+ utterance.state = "given";
990
+ const transcript = utterance.transcript.trim();
866
991
  this.log(`Transcript: "${transcript}"`);
867
992
  this.emit("Transcript", transcript);
868
- if (this.goingAway) {
869
- this.goingAway = false;
870
- this.socket.renew();
871
- }
993
+ utterance.timeout = setTimeout(
994
+ () => this.remove(utterance),
995
+ this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
996
+ );
872
997
  }
873
- clearTranscriptionTimeout() {
874
- if (!this.transcriptionTimeout) return;
875
- clearTimeout(this.transcriptionTimeout);
876
- this.transcriptionTimeout = void 0;
998
+ remove(utterance) {
999
+ if (utterance.timeout) clearTimeout(utterance.timeout);
1000
+ utterance.timeout = void 0;
1001
+ this.utterances = this.utterances.filter((other) => other !== utterance);
1002
+ this.renewIfGoingAway();
1003
+ this.sendFirst();
1004
+ }
1005
+ /** Moves to a new connection once no utterance waits on this one */
1006
+ renewIfGoingAway() {
1007
+ if (!this.goingAway || this.utterances.length > 0) return;
1008
+ this.goingAway = false;
1009
+ this.socket.renew();
1010
+ }
1011
+ dropUtterances() {
1012
+ for (const utterance of this.utterances) {
1013
+ if (utterance.timeout) clearTimeout(utterance.timeout);
1014
+ if (utterance.state === "open") utterance.state = "ended";
1015
+ }
1016
+ this.utterances = [];
877
1017
  }
878
1018
  };
879
1019
 
880
1020
  // src/GeminiTTS.ts
881
- var import_genai2 = require("@google/genai");
882
- var import_server4 = require("@micdrop/server");
883
- var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
1021
+ var import_genai3 = require("@google/genai");
1022
+ var import_server6 = require("@micdrop/server");
1023
+ var DEFAULT_MODEL5 = "gemini-2.5-flash-preview-tts";
884
1024
  var DEFAULT_VOICE = "Kore";
885
1025
  var GEMINI_SAMPLE_RATE = 24e3;
886
1026
  var OUTPUT_SAMPLE_RATE = 16e3;
887
- var DEFAULT_MAX_RETRY2 = 2;
888
- var GeminiTTS = class extends import_server4.SentenceTTS {
1027
+ var DEFAULT_MAX_RETRY = 2;
1028
+ var GeminiTTS = class extends import_server6.SentenceTTS {
889
1029
  constructor(options) {
890
1030
  super();
891
1031
  this.options = options;
892
- this.genai = "genai" in options ? options.genai : new import_genai2.GoogleGenAI({ apiKey: options.apiKey });
1032
+ this.genai = "genai" in options ? options.genai : new import_genai3.GoogleGenAI({ apiKey: options.apiKey });
893
1033
  }
894
1034
  async synthesize(text, signal) {
895
1035
  const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
896
- const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
1036
+ const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
897
1037
  for (let attempt = 0; ; attempt++) {
898
1038
  const progress = { emitted: false };
899
1039
  try {
@@ -911,7 +1051,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
911
1051
  async synthesizeOnce(input, signal, progress) {
912
1052
  const stream = await this.genai.interactions.create(
913
1053
  {
914
- model: this.options.model || DEFAULT_MODEL4,
1054
+ model: this.options.model || DEFAULT_MODEL5,
915
1055
  store: false,
916
1056
  stream: true,
917
1057
  input,
@@ -936,7 +1076,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
936
1076
  }
937
1077
  const { data, sample_rate } = event.delta;
938
1078
  if (!data) continue;
939
- resampler ?? (resampler = new import_server4.Pcm16Resampler(
1079
+ resampler ?? (resampler = new import_server6.Pcm16Resampler(
940
1080
  sample_rate || GEMINI_SAMPLE_RATE,
941
1081
  OUTPUT_SAMPLE_RATE
942
1082
  ));
@@ -947,13 +1087,17 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
947
1087
  }
948
1088
  };
949
1089
  function isBlocked(error) {
950
- return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
1090
+ return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
951
1091
  }
952
1092
  // Annotate the CommonJS export names for ESM import in node:
953
1093
  0 && (module.exports = {
954
1094
  GeminiAgent,
1095
+ GeminiClassifier,
955
1096
  GeminiLive,
956
1097
  GeminiSTT,
957
- GeminiTTS
1098
+ GeminiTTS,
1099
+ choice,
1100
+ noul,
1101
+ score
958
1102
  });
959
1103
  //# sourceMappingURL=index.js.map