@micdrop/gemini 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -2
- package/dist/index.d.mts +156 -19
- package/dist/index.d.ts +156 -19
- package/dist/index.js +467 -323
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +456 -314
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -31,9 +31,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var index_exports = {};
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
GeminiAgent: () => GeminiAgent,
|
|
34
|
+
GeminiClassifier: () => GeminiClassifier,
|
|
34
35
|
GeminiLive: () => GeminiLive,
|
|
35
36
|
GeminiSTT: () => GeminiSTT,
|
|
36
|
-
GeminiTTS: () => GeminiTTS
|
|
37
|
+
GeminiTTS: () => GeminiTTS,
|
|
38
|
+
choice: () => choice,
|
|
39
|
+
noul: () => noul,
|
|
40
|
+
score: () => score
|
|
37
41
|
});
|
|
38
42
|
module.exports = __toCommonJS(index_exports);
|
|
39
43
|
|
|
@@ -54,127 +58,139 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
54
58
|
this.signatures = /* @__PURE__ */ new Map();
|
|
55
59
|
this.genai = "genai" in options ? options.genai : new import_genai.GoogleGenAI({ apiKey: options.apiKey });
|
|
56
60
|
}
|
|
57
|
-
|
|
61
|
+
generateAnswer(stream, signal) {
|
|
62
|
+
return this.generateStep(stream, signal, 0);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Generates one step of the answer, retried on error.
|
|
66
|
+
*
|
|
67
|
+
* A retry starts the step over, so it only happens while nothing of the
|
|
68
|
+
* step reached the stream: what was said cannot be taken back, and saying
|
|
69
|
+
* it twice is worse than stopping there.
|
|
70
|
+
*/
|
|
71
|
+
async generateStep(stream, signal, stepCount) {
|
|
58
72
|
if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
|
|
59
73
|
console.error("[GeminiAgent] Max steps reached");
|
|
60
74
|
return;
|
|
61
75
|
}
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
76
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
|
|
77
|
+
for (let tryCount = 0; ; tryCount++) {
|
|
78
|
+
const progress = { answer: "" };
|
|
79
|
+
try {
|
|
80
|
+
const step = await this.streamStep(stream, signal, progress);
|
|
81
|
+
if (signal.aborted) return;
|
|
82
|
+
this.recordAnswer(progress.answer);
|
|
83
|
+
if (step.calledTool && !step.skipAnswer && !progress.answer) {
|
|
84
|
+
await this.generateStep(stream, signal, stepCount + 1);
|
|
85
|
+
}
|
|
86
|
+
return;
|
|
87
|
+
} catch (error) {
|
|
88
|
+
if (signal.aborted) return;
|
|
89
|
+
console.error("[GeminiAgent] Error answering:", error);
|
|
90
|
+
if (progress.answer) {
|
|
91
|
+
this.log("Error after part of the answer was said, keeping it");
|
|
92
|
+
this.recordAnswer(progress.answer);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (tryCount >= maxRetry) {
|
|
96
|
+
this.log("Max retries reached, giving up");
|
|
97
|
+
this.emit("Failed");
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
|
|
101
|
+
if (signal.aborted) return;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
/** Streams one request, writing the text of the answer as it comes */
|
|
106
|
+
async streamStep(stream, signal, progress) {
|
|
65
107
|
let extracting = false;
|
|
66
108
|
const extractOptions = this.getExtractOptions();
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
if (startTagIndex !== -1) {
|
|
120
|
-
extracting = true;
|
|
121
|
-
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
122
|
-
break;
|
|
123
|
-
}
|
|
109
|
+
const response = await this.genai.interactions.create(
|
|
110
|
+
{
|
|
111
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
112
|
+
store: false,
|
|
113
|
+
stream: true,
|
|
114
|
+
system_instruction: this.buildSystemInstruction(),
|
|
115
|
+
input: this.buildInput(),
|
|
116
|
+
tools: this.buildTools(),
|
|
117
|
+
...this.options.thinkingLevel || this.options.settings ? {
|
|
118
|
+
generation_config: {
|
|
119
|
+
...this.options.settings,
|
|
120
|
+
...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
|
|
121
|
+
}
|
|
122
|
+
} : {}
|
|
123
|
+
},
|
|
124
|
+
{ signal }
|
|
125
|
+
);
|
|
126
|
+
const steps = [];
|
|
127
|
+
let skipAnswer = false;
|
|
128
|
+
let calledTool = false;
|
|
129
|
+
let signature = "";
|
|
130
|
+
for await (const event of response) {
|
|
131
|
+
if (signal.aborted) break;
|
|
132
|
+
switch (event.event_type) {
|
|
133
|
+
case "step.start":
|
|
134
|
+
steps[event.index] = {
|
|
135
|
+
type: event.step.type,
|
|
136
|
+
id: event.step.id,
|
|
137
|
+
name: event.step.name,
|
|
138
|
+
text: "",
|
|
139
|
+
arguments: "",
|
|
140
|
+
signature: ""
|
|
141
|
+
};
|
|
142
|
+
break;
|
|
143
|
+
case "step.delta": {
|
|
144
|
+
const step = steps[event.index];
|
|
145
|
+
if (!step) break;
|
|
146
|
+
const delta = event.delta;
|
|
147
|
+
if (delta.type === "thought_signature") {
|
|
148
|
+
step.signature += delta.signature ?? "";
|
|
149
|
+
} else if (delta.type === "arguments_delta") {
|
|
150
|
+
step.arguments += delta.arguments ?? "";
|
|
151
|
+
} else if (delta.type === "text" && step.type === "model_output") {
|
|
152
|
+
this.log(`Answer chunk: "${delta.text}"`);
|
|
153
|
+
progress.answer += delta.text;
|
|
154
|
+
if (extractOptions) {
|
|
155
|
+
if (extracting) break;
|
|
156
|
+
const startTagIndex = delta.text.indexOf(extractOptions.startTag);
|
|
157
|
+
if (startTagIndex !== -1) {
|
|
158
|
+
extracting = true;
|
|
159
|
+
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
160
|
+
break;
|
|
124
161
|
}
|
|
125
|
-
stream.write(delta.text);
|
|
126
162
|
}
|
|
127
|
-
|
|
128
|
-
}
|
|
129
|
-
case "step.stop": {
|
|
130
|
-
const step = steps[event.index];
|
|
131
|
-
if (step?.type === "thought") signature = step.signature;
|
|
132
|
-
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
133
|
-
calledTool = true;
|
|
134
|
-
if (signature) this.signatures.set(step.id, signature);
|
|
135
|
-
const result = await this.executeTool({
|
|
136
|
-
role: "tool_call",
|
|
137
|
-
toolCallId: step.id,
|
|
138
|
-
toolName: step.name,
|
|
139
|
-
parameters: step.arguments || "{}"
|
|
140
|
-
});
|
|
141
|
-
if (result.skipAnswer) skipAnswer = true;
|
|
142
|
-
break;
|
|
163
|
+
stream.write(delta.text);
|
|
143
164
|
}
|
|
144
|
-
|
|
145
|
-
throw new Error(event.error?.message || "Generation failed");
|
|
146
|
-
default:
|
|
147
|
-
break;
|
|
165
|
+
break;
|
|
148
166
|
}
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
this.log("Max retries reached, giving up");
|
|
169
|
-
this.emit("Failed");
|
|
167
|
+
case "step.stop": {
|
|
168
|
+
const step = steps[event.index];
|
|
169
|
+
if (step?.type === "thought") signature = step.signature;
|
|
170
|
+
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
171
|
+
calledTool = true;
|
|
172
|
+
if (signature) this.signatures.set(step.id, signature);
|
|
173
|
+
const result = await this.executeTool({
|
|
174
|
+
role: "tool_call",
|
|
175
|
+
toolCallId: step.id,
|
|
176
|
+
toolName: step.name,
|
|
177
|
+
parameters: step.arguments || "{}"
|
|
178
|
+
});
|
|
179
|
+
if (result.skipAnswer) skipAnswer = true;
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
182
|
+
case "error":
|
|
183
|
+
throw new Error(event.error?.message || "Generation failed");
|
|
184
|
+
default:
|
|
185
|
+
break;
|
|
170
186
|
}
|
|
171
187
|
}
|
|
188
|
+
return { calledTool, skipAnswer };
|
|
172
189
|
}
|
|
173
|
-
|
|
174
|
-
if (!
|
|
175
|
-
this.
|
|
176
|
-
this.
|
|
177
|
-
this.abortController = void 0;
|
|
190
|
+
recordAnswer(answer) {
|
|
191
|
+
if (!answer) return;
|
|
192
|
+
const { message, metadata } = this.extract(answer);
|
|
193
|
+
this.addAssistantMessage(message, metadata);
|
|
178
194
|
}
|
|
179
195
|
buildSystemInstruction() {
|
|
180
196
|
return this.conversation.flatMap(
|
|
@@ -246,6 +262,17 @@ var GeminiAgent = class extends import_server.Agent {
|
|
|
246
262
|
});
|
|
247
263
|
}
|
|
248
264
|
};
|
|
265
|
+
function sleep(ms, signal) {
|
|
266
|
+
return new Promise((resolve) => {
|
|
267
|
+
const done = () => {
|
|
268
|
+
clearTimeout(timer);
|
|
269
|
+
signal.removeEventListener("abort", done);
|
|
270
|
+
resolve();
|
|
271
|
+
};
|
|
272
|
+
const timer = setTimeout(done, ms);
|
|
273
|
+
signal.addEventListener("abort", done);
|
|
274
|
+
});
|
|
275
|
+
}
|
|
249
276
|
function parseArguments(parameters) {
|
|
250
277
|
try {
|
|
251
278
|
return JSON.parse(parameters);
|
|
@@ -254,72 +281,176 @@ function parseArguments(parameters) {
|
|
|
254
281
|
}
|
|
255
282
|
}
|
|
256
283
|
|
|
257
|
-
// src/
|
|
284
|
+
// src/GeminiClassifier.ts
|
|
285
|
+
var import_genai2 = require("@google/genai");
|
|
258
286
|
var import_server2 = require("@micdrop/server");
|
|
287
|
+
function noul(instructions, criteria) {
|
|
288
|
+
return {
|
|
289
|
+
type: "noul",
|
|
290
|
+
instructions,
|
|
291
|
+
criteria: criteria && {
|
|
292
|
+
true_description: criteria.true,
|
|
293
|
+
false_description: criteria.false
|
|
294
|
+
}
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
function choice(instructions, criteria) {
|
|
298
|
+
return { type: "choice", instructions, criteria };
|
|
299
|
+
}
|
|
300
|
+
function score(instructions, levels) {
|
|
301
|
+
return { type: "score", instructions, criteria: levels };
|
|
302
|
+
}
|
|
303
|
+
var DEFAULT_MODEL2 = "gemini-decision-1";
|
|
304
|
+
var DEFAULT_TIMEOUT = 3e3;
|
|
305
|
+
var API_VERSION = "v1beta";
|
|
306
|
+
var GeminiClassifier = class extends import_server2.Classifier {
|
|
307
|
+
constructor(options) {
|
|
308
|
+
super();
|
|
309
|
+
this.options = options;
|
|
310
|
+
this.genai = "genai" in options ? options.genai : new import_genai2.GoogleGenAI({ apiKey: options.apiKey });
|
|
311
|
+
}
|
|
312
|
+
async evaluate(input, signal) {
|
|
313
|
+
const { questions, model, timeout } = this.options;
|
|
314
|
+
const asked = typeof questions === "function" ? questions(input) : questions;
|
|
315
|
+
const decision = await this.genai.decisions.create(
|
|
316
|
+
{
|
|
317
|
+
model: model || DEFAULT_MODEL2,
|
|
318
|
+
input: this.buildInput(input),
|
|
319
|
+
questions: asked,
|
|
320
|
+
// Only on v1beta, while the client defaults to v1
|
|
321
|
+
api_version: API_VERSION
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
// The SDK drops its timeout when a signal is given
|
|
325
|
+
signal: AbortSignal.any([
|
|
326
|
+
signal,
|
|
327
|
+
AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT)
|
|
328
|
+
]),
|
|
329
|
+
retries: { strategy: "none" }
|
|
330
|
+
}
|
|
331
|
+
);
|
|
332
|
+
const blocked = decision.prompt_feedback?.block_reason;
|
|
333
|
+
if (blocked) {
|
|
334
|
+
const message = decision.prompt_feedback?.block_reason_message;
|
|
335
|
+
throw new Error(
|
|
336
|
+
`Input blocked by Gemini (${blocked})${message ? `: ${message}` : ""}`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
answers: decision.answers ?? {},
|
|
341
|
+
usage: decision.usage
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
buildInput(input) {
|
|
345
|
+
if (this.options.input) return this.options.input(input);
|
|
346
|
+
return typeof input === "string" ? input : JSON.stringify(input);
|
|
347
|
+
}
|
|
348
|
+
};
|
|
349
|
+
|
|
350
|
+
// src/GeminiLive.ts
|
|
351
|
+
var import_server4 = require("@micdrop/server");
|
|
259
352
|
var import_zod2 = require("zod");
|
|
260
353
|
|
|
261
354
|
// src/LiveSocket.ts
|
|
355
|
+
var import_server3 = require("@micdrop/server");
|
|
262
356
|
var import_eventemitter3 = require("eventemitter3");
|
|
263
|
-
var import_ws = __toESM(require("ws"));
|
|
264
357
|
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
265
358
|
var SAMPLE_RATE = 16e3;
|
|
266
359
|
var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
|
|
267
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
268
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
269
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
270
360
|
var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
271
361
|
constructor(options, buildSetup, log) {
|
|
272
362
|
super();
|
|
273
|
-
this.options = options;
|
|
274
363
|
this.buildSetup = buildSetup;
|
|
275
364
|
this.log = log;
|
|
276
|
-
this.connected = false;
|
|
277
|
-
this.destroyed = false;
|
|
278
365
|
// Messages waiting for the session, or for the end of a turn to be due
|
|
279
366
|
this.pending = [];
|
|
280
367
|
// Audio of the turn as it went out, see flush()
|
|
281
368
|
this.turnAudioStartedAt = 0;
|
|
282
369
|
this.turnAudioMs = 0;
|
|
283
|
-
|
|
370
|
+
// Attempts since the last session was ready, for the owner to tell a drop
|
|
371
|
+
// from a resumption that keeps failing
|
|
372
|
+
this.reconnectAttempts = 0;
|
|
373
|
+
/** Sends the setup, and holds the connection back until Gemini accepts it */
|
|
374
|
+
this.handshake = {
|
|
375
|
+
start: (socket) => {
|
|
376
|
+
const setup = this.buildSetup();
|
|
377
|
+
this.setup = setup;
|
|
378
|
+
socket.send(JSON.stringify(setup));
|
|
379
|
+
},
|
|
380
|
+
read: (data) => !!parseMessage(data.toString())?.setupComplete
|
|
381
|
+
};
|
|
382
|
+
this.onOpen = () => {
|
|
383
|
+
this.log("Session ready");
|
|
384
|
+
this.reconnectAttempts = 0;
|
|
385
|
+
this.emit("Ready", this.setup);
|
|
386
|
+
this.flush();
|
|
387
|
+
};
|
|
388
|
+
this.onMessage = (data) => {
|
|
389
|
+
const message = parseMessage(data.toString());
|
|
390
|
+
if (message) this.emit("Message", message);
|
|
391
|
+
};
|
|
392
|
+
this.onStatusChange = (status) => {
|
|
393
|
+
if (status !== "open") this.clearPendingTimeout();
|
|
394
|
+
if (status === "reconnecting") {
|
|
395
|
+
this.emit("Reconnecting", ++this.reconnectAttempts);
|
|
396
|
+
}
|
|
397
|
+
};
|
|
398
|
+
this.onFailed = () => {
|
|
399
|
+
this.clearPending();
|
|
400
|
+
this.emit("Failed");
|
|
401
|
+
};
|
|
402
|
+
this.socket = new import_server3.ReconnectingSocket({
|
|
403
|
+
url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
|
|
404
|
+
connectionTimeout: options.connectionTimeout,
|
|
405
|
+
retryDelay: options.retryDelay,
|
|
406
|
+
maxRetry: options.maxRetry,
|
|
407
|
+
handshake: this.handshake,
|
|
408
|
+
log
|
|
409
|
+
});
|
|
410
|
+
this.socket.on("Open", this.onOpen);
|
|
411
|
+
this.socket.on("Message", this.onMessage);
|
|
412
|
+
this.socket.on("StatusChange", this.onStatusChange);
|
|
413
|
+
this.socket.on("Failed", this.onFailed);
|
|
284
414
|
}
|
|
285
415
|
/** The session is set up and takes messages right away */
|
|
286
416
|
get isConnected() {
|
|
287
|
-
return this.
|
|
417
|
+
return this.socket.isOpen;
|
|
288
418
|
}
|
|
289
419
|
connect() {
|
|
290
|
-
this.
|
|
291
|
-
this.log("Connection error:", error);
|
|
292
|
-
this.reconnect();
|
|
293
|
-
});
|
|
420
|
+
this.socket.connect();
|
|
294
421
|
}
|
|
295
|
-
/**
|
|
422
|
+
/**
|
|
423
|
+
* Sends a message once the session is ready, after the ones before it.
|
|
424
|
+
* Dropped once the connection failed for good, as nothing would send it.
|
|
425
|
+
*/
|
|
296
426
|
send(message) {
|
|
427
|
+
const status = this.socket.status;
|
|
428
|
+
if (status === "failed" || status === "destroyed") return;
|
|
297
429
|
this.pending.push(message);
|
|
298
430
|
this.flush();
|
|
299
431
|
}
|
|
300
432
|
/** Sends a message ahead of the waiting ones, from a Ready listener */
|
|
301
433
|
sendFirst(message) {
|
|
302
|
-
this.socket
|
|
434
|
+
this.socket.send(JSON.stringify(message));
|
|
303
435
|
}
|
|
304
436
|
/** Drops the messages that have not gone out */
|
|
305
437
|
clearPending() {
|
|
306
438
|
this.pending = [];
|
|
307
439
|
this.clearPendingTimeout();
|
|
308
440
|
}
|
|
309
|
-
/**
|
|
441
|
+
/**
|
|
442
|
+
* Moves to a new connection, keeping the messages that have not gone out.
|
|
443
|
+
* A reconnection waiting for its delay is replaced by this one.
|
|
444
|
+
*/
|
|
310
445
|
renew() {
|
|
311
446
|
this.log("Renewing the connection");
|
|
312
|
-
this.
|
|
313
|
-
this.
|
|
447
|
+
this.clearPendingTimeout();
|
|
448
|
+
this.reconnectAttempts = 0;
|
|
449
|
+
this.socket.restart();
|
|
314
450
|
}
|
|
315
451
|
destroy() {
|
|
316
|
-
this.
|
|
317
|
-
|
|
318
|
-
clearTimeout(this.reconnectTimeout);
|
|
319
|
-
this.reconnectTimeout = void 0;
|
|
320
|
-
}
|
|
321
|
-
this.closeSocket();
|
|
322
|
-
this.pending = [];
|
|
452
|
+
this.clearPending();
|
|
453
|
+
this.socket.destroy();
|
|
323
454
|
this.removeAllListeners();
|
|
324
455
|
}
|
|
325
456
|
/**
|
|
@@ -332,8 +463,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
|
332
463
|
*/
|
|
333
464
|
flush() {
|
|
334
465
|
while (this.pending.length > 0 && !this.pendingTimeout) {
|
|
335
|
-
|
|
336
|
-
if (!this.connected || socket?.readyState !== import_ws.default.OPEN) return;
|
|
466
|
+
if (!this.socket.isOpen) return;
|
|
337
467
|
const message = this.pending[0];
|
|
338
468
|
const input = message.realtimeInput;
|
|
339
469
|
if (input?.activityEnd) {
|
|
@@ -353,7 +483,7 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
|
353
483
|
this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
|
|
354
484
|
}
|
|
355
485
|
this.pending.shift();
|
|
356
|
-
socket.send(JSON.stringify(message));
|
|
486
|
+
this.socket.send(JSON.stringify(message));
|
|
357
487
|
}
|
|
358
488
|
}
|
|
359
489
|
clearPendingTimeout() {
|
|
@@ -361,112 +491,46 @@ var LiveSocket = class extends import_eventemitter3.EventEmitter {
|
|
|
361
491
|
clearTimeout(this.pendingTimeout);
|
|
362
492
|
this.pendingTimeout = void 0;
|
|
363
493
|
}
|
|
364
|
-
initWS() {
|
|
365
|
-
return new Promise((resolve, reject) => {
|
|
366
|
-
const socket = new import_ws.default(
|
|
367
|
-
`${URL}?key=${encodeURIComponent(this.options.apiKey)}`
|
|
368
|
-
);
|
|
369
|
-
this.socket = socket;
|
|
370
|
-
const timeout = setTimeout(() => {
|
|
371
|
-
this.log("Connection timeout");
|
|
372
|
-
this.closeSocket();
|
|
373
|
-
reject(new Error("WebSocket connection timeout"));
|
|
374
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
375
|
-
socket.on("open", () => {
|
|
376
|
-
this.log("Connection opened");
|
|
377
|
-
socket.send(JSON.stringify(this.buildSetup()));
|
|
378
|
-
});
|
|
379
|
-
socket.on("error", (error) => {
|
|
380
|
-
clearTimeout(timeout);
|
|
381
|
-
this.log("WebSocket error:", error);
|
|
382
|
-
reject(new Error("WebSocket connection error"));
|
|
383
|
-
});
|
|
384
|
-
socket.on("close", (code, reason) => {
|
|
385
|
-
clearTimeout(timeout);
|
|
386
|
-
const wasConnected = this.connected;
|
|
387
|
-
this.closeSocket();
|
|
388
|
-
if (this.destroyed) return;
|
|
389
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
390
|
-
if (wasConnected) {
|
|
391
|
-
this.reconnect();
|
|
392
|
-
} else {
|
|
393
|
-
reject(new Error(`Connection closed: ${reason.toString()}`));
|
|
394
|
-
}
|
|
395
|
-
});
|
|
396
|
-
socket.on("message", (data) => {
|
|
397
|
-
const message = JSON.parse(data.toString());
|
|
398
|
-
if (message.setupComplete) {
|
|
399
|
-
clearTimeout(timeout);
|
|
400
|
-
this.log("Session ready");
|
|
401
|
-
this.connected = true;
|
|
402
|
-
this.retryCount = 0;
|
|
403
|
-
this.emit("Ready");
|
|
404
|
-
this.flush();
|
|
405
|
-
resolve();
|
|
406
|
-
return;
|
|
407
|
-
}
|
|
408
|
-
this.emit("Message", message);
|
|
409
|
-
});
|
|
410
|
-
});
|
|
411
|
-
}
|
|
412
|
-
closeSocket() {
|
|
413
|
-
const socket = this.socket;
|
|
414
|
-
this.socket = void 0;
|
|
415
|
-
this.connected = false;
|
|
416
|
-
this.clearPendingTimeout();
|
|
417
|
-
if (!socket) return;
|
|
418
|
-
socket.removeAllListeners();
|
|
419
|
-
socket.on("error", () => {
|
|
420
|
-
});
|
|
421
|
-
if (socket.readyState === import_ws.default.OPEN || socket.readyState === import_ws.default.CONNECTING) {
|
|
422
|
-
socket.close(1e3);
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
reconnect() {
|
|
426
|
-
if (this.destroyed) return;
|
|
427
|
-
this.retryCount++;
|
|
428
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
429
|
-
this.log("Max retries reached, giving up");
|
|
430
|
-
this.emit("Failed");
|
|
431
|
-
return;
|
|
432
|
-
}
|
|
433
|
-
this.emit("Reconnecting", this.retryCount);
|
|
434
|
-
this.log("Reconnecting...");
|
|
435
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
436
|
-
this.reconnectTimeout = void 0;
|
|
437
|
-
this.connect();
|
|
438
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
439
|
-
}
|
|
440
494
|
};
|
|
495
|
+
function parseMessage(text) {
|
|
496
|
+
try {
|
|
497
|
+
return JSON.parse(text);
|
|
498
|
+
} catch {
|
|
499
|
+
return void 0;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
441
502
|
|
|
442
503
|
// src/GeminiLive.ts
|
|
443
|
-
var
|
|
504
|
+
var DEFAULT_MODEL3 = "gemini-3.8-live";
|
|
444
505
|
var SAMPLE_RATE2 = 16e3;
|
|
445
506
|
var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
|
|
446
507
|
var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
|
|
508
|
+
function newToolRound() {
|
|
509
|
+
return { running: 0, answerEndDeferred: false };
|
|
510
|
+
}
|
|
447
511
|
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
448
|
-
var GeminiLive = class extends
|
|
512
|
+
var GeminiLive = class extends import_server4.Realtime {
|
|
449
513
|
constructor(options) {
|
|
450
514
|
super(options);
|
|
451
515
|
this.everConnected = false;
|
|
452
|
-
// Whether the setup being sent resumes the session
|
|
453
|
-
this.resuming = false;
|
|
454
516
|
this.goingAway = false;
|
|
455
|
-
this.outputResampler = new
|
|
517
|
+
this.outputResampler = new import_server4.Pcm16Resampler(
|
|
456
518
|
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
457
519
|
SAMPLE_RATE2
|
|
458
520
|
);
|
|
459
521
|
this.hasTranscribedTurn = false;
|
|
460
522
|
// Transcript of a dropped turn, which the conversation should not show
|
|
461
523
|
this.ignoreUserTranscript = false;
|
|
524
|
+
// Calls of the session Gemini told not to answer
|
|
462
525
|
this.cancelledToolCalls = /* @__PURE__ */ new Set();
|
|
463
|
-
//
|
|
464
|
-
//
|
|
465
|
-
|
|
466
|
-
this.
|
|
467
|
-
this.
|
|
468
|
-
|
|
469
|
-
if (this.everConnected && !
|
|
526
|
+
// Bumped by every session that does not resume the previous one, whose tool
|
|
527
|
+
// calls it cannot answer
|
|
528
|
+
this.session = 0;
|
|
529
|
+
this.toolRound = newToolRound();
|
|
530
|
+
this.onReady = (setup) => {
|
|
531
|
+
const resumed = !!setup?.setup?.sessionResumption?.handle;
|
|
532
|
+
if (this.everConnected && !resumed) {
|
|
533
|
+
this.startFreshSession();
|
|
470
534
|
const turns = this.conversation.flatMap(
|
|
471
535
|
(message) => message.role === "user" || message.role === "assistant" ? [
|
|
472
536
|
{
|
|
@@ -484,7 +548,12 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
484
548
|
};
|
|
485
549
|
this.onReconnecting = (attempt) => {
|
|
486
550
|
if (attempt > 1) this.resumptionHandle = void 0;
|
|
487
|
-
if (!this.resumptionHandle)
|
|
551
|
+
if (!this.resumptionHandle) {
|
|
552
|
+
this.socket.clearPending();
|
|
553
|
+
this.startFreshSession();
|
|
554
|
+
}
|
|
555
|
+
this.toolRound.answerEndDeferred = false;
|
|
556
|
+
this.commitTurnTranscript();
|
|
488
557
|
this.endAnswer();
|
|
489
558
|
};
|
|
490
559
|
this.onMessage = (message) => {
|
|
@@ -522,9 +591,16 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
522
591
|
this.socket.on("Failed", () => this.emit("Failed"));
|
|
523
592
|
this.socket.connect();
|
|
524
593
|
}
|
|
594
|
+
/**
|
|
595
|
+
* Stops the answer. The tools it is waiting for still send their results,
|
|
596
|
+
* silently, since a blocking call left unanswered would stall the model.
|
|
597
|
+
*/
|
|
598
|
+
cancel() {
|
|
599
|
+
super.cancel();
|
|
600
|
+
this.resetToolRound();
|
|
601
|
+
}
|
|
525
602
|
destroy() {
|
|
526
603
|
super.destroy();
|
|
527
|
-
this.clearToolFollowUp();
|
|
528
604
|
this.socket.destroy();
|
|
529
605
|
}
|
|
530
606
|
openTurn() {
|
|
@@ -559,11 +635,9 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
559
635
|
}
|
|
560
636
|
cancelAnswer() {
|
|
561
637
|
this.outputResampler.reset();
|
|
562
|
-
this.clearToolFollowUp();
|
|
563
|
-
this.answerEndDeferred = false;
|
|
564
638
|
}
|
|
565
639
|
get isIdle() {
|
|
566
|
-
return super.isIdle && this.
|
|
640
|
+
return super.isIdle && this.toolRound.running === 0;
|
|
567
641
|
}
|
|
568
642
|
updateTools() {
|
|
569
643
|
if (this.everConnected) {
|
|
@@ -595,7 +669,6 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
595
669
|
this.socket.send(message);
|
|
596
670
|
}
|
|
597
671
|
buildSetup() {
|
|
598
|
-
this.resuming = !!this.resumptionHandle;
|
|
599
672
|
const functionDeclarations = this.tools.map((tool) => ({
|
|
600
673
|
name: tool.name,
|
|
601
674
|
description: tool.description,
|
|
@@ -603,7 +676,7 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
603
676
|
}));
|
|
604
677
|
return {
|
|
605
678
|
setup: {
|
|
606
|
-
model: `models/${this.options.model ||
|
|
679
|
+
model: `models/${this.options.model || DEFAULT_MODEL3}`,
|
|
607
680
|
generationConfig: {
|
|
608
681
|
responseModalities: ["AUDIO"],
|
|
609
682
|
...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
|
|
@@ -629,6 +702,22 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
629
702
|
}
|
|
630
703
|
};
|
|
631
704
|
}
|
|
705
|
+
/** Forgets what belonged to the previous session */
|
|
706
|
+
startFreshSession() {
|
|
707
|
+
this.session++;
|
|
708
|
+
this.resetToolRound();
|
|
709
|
+
this.cancelledToolCalls.clear();
|
|
710
|
+
this.ignoreUserTranscript = false;
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Starts a new round of tools. The tools still running belong to the old
|
|
714
|
+
* one: their results go out silently, and nothing waits for the model to
|
|
715
|
+
* talk about them.
|
|
716
|
+
*/
|
|
717
|
+
resetToolRound() {
|
|
718
|
+
if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
|
|
719
|
+
this.toolRound = newToolRound();
|
|
720
|
+
}
|
|
632
721
|
/** Moves to a new connection before Gemini closes this one */
|
|
633
722
|
renewIfGoingAway() {
|
|
634
723
|
if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
|
|
@@ -660,8 +749,8 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
660
749
|
if (content.turnComplete) {
|
|
661
750
|
this.commitTurnTranscript();
|
|
662
751
|
this.outputResampler.reset();
|
|
663
|
-
if (this.
|
|
664
|
-
this.answerEndDeferred = true;
|
|
752
|
+
if (this.toolRound.running > 0 || this.toolRound.followUp) {
|
|
753
|
+
this.toolRound.answerEndDeferred = true;
|
|
665
754
|
} else {
|
|
666
755
|
this.endAnswer();
|
|
667
756
|
this.renewIfGoingAway();
|
|
@@ -670,24 +759,22 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
670
759
|
}
|
|
671
760
|
onModelOutput() {
|
|
672
761
|
this.commitTurnTranscript();
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
762
|
+
const round = this.toolRound;
|
|
763
|
+
if (round.followUp) {
|
|
764
|
+
clearTimeout(round.followUp);
|
|
765
|
+
round.followUp = void 0;
|
|
766
|
+
round.answerEndDeferred = false;
|
|
676
767
|
}
|
|
677
768
|
}
|
|
678
769
|
/** Ends the answer the model stopped giving to wait for a tool */
|
|
679
770
|
endDeferredAnswer() {
|
|
680
|
-
|
|
681
|
-
if (
|
|
682
|
-
|
|
771
|
+
const round = this.toolRound;
|
|
772
|
+
if (!round.answerEndDeferred) return;
|
|
773
|
+
if (round.running > 0 || round.followUp) return;
|
|
774
|
+
round.answerEndDeferred = false;
|
|
683
775
|
this.endAnswer();
|
|
684
776
|
this.renewIfGoingAway();
|
|
685
777
|
}
|
|
686
|
-
clearToolFollowUp() {
|
|
687
|
-
if (!this.toolFollowUp) return;
|
|
688
|
-
clearTimeout(this.toolFollowUp);
|
|
689
|
-
this.toolFollowUp = void 0;
|
|
690
|
-
}
|
|
691
778
|
/**
|
|
692
779
|
* The transcript of the turn comes before the answer, so the first piece of
|
|
693
780
|
* the answer closes it.
|
|
@@ -705,17 +792,22 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
705
792
|
toolName: call.name,
|
|
706
793
|
parameters: JSON.stringify(call.args ?? {})
|
|
707
794
|
};
|
|
708
|
-
this.
|
|
795
|
+
const round = this.toolRound;
|
|
796
|
+
const session = this.session;
|
|
797
|
+
round.running++;
|
|
709
798
|
let result;
|
|
710
799
|
try {
|
|
711
800
|
result = await this.runTool(toolCall);
|
|
712
801
|
} finally {
|
|
713
|
-
|
|
802
|
+
round.running--;
|
|
714
803
|
}
|
|
804
|
+
if (session !== this.session) return;
|
|
715
805
|
if (this.cancelledToolCalls.delete(call.id)) {
|
|
716
806
|
this.endDeferredAnswer();
|
|
717
807
|
return;
|
|
718
808
|
}
|
|
809
|
+
const interrupted = round !== this.toolRound;
|
|
810
|
+
const silent = result.skipAnswer || interrupted;
|
|
719
811
|
this.send({
|
|
720
812
|
toolResponse: {
|
|
721
813
|
functionResponses: [
|
|
@@ -725,16 +817,17 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
725
817
|
response: {
|
|
726
818
|
output: result.output ?? null,
|
|
727
819
|
// A tool without an answer lets the model know silently
|
|
728
|
-
scheduling:
|
|
820
|
+
scheduling: silent ? "SILENT" : "WHEN_IDLE"
|
|
729
821
|
}
|
|
730
822
|
}
|
|
731
823
|
]
|
|
732
824
|
}
|
|
733
825
|
});
|
|
826
|
+
if (interrupted) return;
|
|
734
827
|
if (!result.skipAnswer) {
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
828
|
+
if (round.followUp) clearTimeout(round.followUp);
|
|
829
|
+
round.followUp = setTimeout(() => {
|
|
830
|
+
round.followUp = void 0;
|
|
738
831
|
this.log("Nothing said about the tool result");
|
|
739
832
|
this.endDeferredAnswer();
|
|
740
833
|
}, TOOL_FOLLOW_UP_TIMEOUT);
|
|
@@ -745,85 +838,91 @@ var GeminiLive = class extends import_server2.Realtime {
|
|
|
745
838
|
};
|
|
746
839
|
|
|
747
840
|
// src/GeminiSTT.ts
|
|
748
|
-
var
|
|
749
|
-
var
|
|
841
|
+
var import_server5 = require("@micdrop/server");
|
|
842
|
+
var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
|
|
750
843
|
var SAMPLE_RATE3 = 16e3;
|
|
751
844
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
752
|
-
var GeminiSTT = class extends
|
|
845
|
+
var GeminiSTT = class extends import_server5.STT {
|
|
753
846
|
constructor(options) {
|
|
754
847
|
super();
|
|
755
848
|
this.options = options;
|
|
849
|
+
// Gemini closes the session soon, and it is renewed once nothing waits
|
|
756
850
|
this.goingAway = false;
|
|
757
|
-
//
|
|
758
|
-
//
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
this.
|
|
762
|
-
this.
|
|
851
|
+
// Utterances waiting for their transcript, in the order they were spoken.
|
|
852
|
+
// Gemini names no utterance in what it sends, and goes on to the next one
|
|
853
|
+
// before it is done with the previous. So it hears one at a time: the first
|
|
854
|
+
// one, and the audio of the next waits here until its transcript is in.
|
|
855
|
+
this.utterances = [];
|
|
856
|
+
this.onReady = () => {
|
|
857
|
+
this.goingAway = false;
|
|
858
|
+
};
|
|
763
859
|
this.onMessage = (message) => {
|
|
764
860
|
const content = message.serverContent;
|
|
765
|
-
|
|
766
|
-
|
|
861
|
+
const utterance = this.utterances[0];
|
|
862
|
+
if (content?.inputTranscription?.text && utterance) {
|
|
863
|
+
utterance.transcript += content.inputTranscription.text;
|
|
767
864
|
}
|
|
768
|
-
if (content?.generationComplete &&
|
|
769
|
-
this.
|
|
865
|
+
if (content?.generationComplete && utterance) {
|
|
866
|
+
if (utterance.state === "ended") this.give(utterance);
|
|
867
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
770
868
|
}
|
|
771
869
|
if (message.goAway) {
|
|
772
870
|
this.log("Connection ending in", message.goAway.timeLeft);
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
} else {
|
|
776
|
-
this.socket.renew();
|
|
777
|
-
}
|
|
871
|
+
this.goingAway = true;
|
|
872
|
+
this.renewIfGoingAway();
|
|
778
873
|
}
|
|
779
874
|
};
|
|
780
875
|
this.onReconnecting = () => {
|
|
876
|
+
this.goingAway = false;
|
|
781
877
|
this.socket.clearPending();
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
785
|
-
this.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
786
|
-
if (this.utteranceEnded) {
|
|
787
|
-
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
878
|
+
for (const utterance of this.utterances) {
|
|
879
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
788
880
|
}
|
|
881
|
+
for (const utterance of this.utterances) {
|
|
882
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
883
|
+
utterance.timeout = void 0;
|
|
884
|
+
utterance.sent = false;
|
|
885
|
+
utterance.transcript = "";
|
|
886
|
+
}
|
|
887
|
+
this.sendFirst();
|
|
888
|
+
};
|
|
889
|
+
this.onFailed = () => {
|
|
890
|
+
const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
|
|
891
|
+
this.dropUtterances();
|
|
892
|
+
this.emit("Failed", chunks);
|
|
789
893
|
};
|
|
790
894
|
this.socket = new LiveSocket(
|
|
791
895
|
options,
|
|
792
896
|
() => this.buildSetup(),
|
|
793
897
|
(...message) => this.log(...message)
|
|
794
898
|
);
|
|
899
|
+
this.socket.on("Ready", this.onReady);
|
|
795
900
|
this.socket.on("Message", this.onMessage);
|
|
796
901
|
this.socket.on("Reconnecting", this.onReconnecting);
|
|
797
|
-
this.socket.on("Failed",
|
|
902
|
+
this.socket.on("Failed", this.onFailed);
|
|
798
903
|
this.socket.connect();
|
|
799
904
|
}
|
|
800
905
|
transcribe(audioStream) {
|
|
906
|
+
let utterance;
|
|
801
907
|
audioStream.on("data", (chunk) => {
|
|
802
|
-
if (!
|
|
803
|
-
|
|
804
|
-
|
|
908
|
+
if (!utterance) utterance = this.openUtterance();
|
|
909
|
+
if (utterance.state !== "open") return;
|
|
910
|
+
utterance.chunks.push(chunk);
|
|
911
|
+
if (utterance.sent) this.sendAudio(chunk);
|
|
805
912
|
});
|
|
806
913
|
audioStream.on("end", () => {
|
|
807
|
-
if (
|
|
808
|
-
this.utteranceEnded = true;
|
|
809
|
-
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
810
|
-
this.clearTranscriptionTimeout();
|
|
811
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
812
|
-
this.transcriptionTimeout = void 0;
|
|
813
|
-
this.log("Transcription timeout");
|
|
814
|
-
this.finishUtterance();
|
|
815
|
-
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
914
|
+
if (utterance?.state === "open") this.endUtterance(utterance);
|
|
816
915
|
});
|
|
817
916
|
}
|
|
818
917
|
destroy() {
|
|
819
918
|
super.destroy();
|
|
820
|
-
this.
|
|
919
|
+
this.dropUtterances();
|
|
821
920
|
this.socket.destroy();
|
|
822
921
|
}
|
|
823
922
|
buildSetup() {
|
|
824
923
|
return {
|
|
825
924
|
setup: {
|
|
826
|
-
model: `models/${this.options.model ||
|
|
925
|
+
model: `models/${this.options.model || DEFAULT_MODEL4}`,
|
|
827
926
|
generationConfig: { responseModalities: ["TEXT"] },
|
|
828
927
|
inputAudioTranscription: {
|
|
829
928
|
languageCodes: this.options.language ? [this.options.language] : [],
|
|
@@ -838,12 +937,38 @@ var GeminiSTT = class extends import_server3.STT {
|
|
|
838
937
|
};
|
|
839
938
|
}
|
|
840
939
|
openUtterance() {
|
|
841
|
-
|
|
842
|
-
this.
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
940
|
+
const last = this.utterances[this.utterances.length - 1];
|
|
941
|
+
if (last?.state === "open") this.endUtterance(last);
|
|
942
|
+
const utterance = {
|
|
943
|
+
state: "open",
|
|
944
|
+
sent: false,
|
|
945
|
+
chunks: [],
|
|
946
|
+
transcript: ""
|
|
947
|
+
};
|
|
948
|
+
this.utterances.push(utterance);
|
|
949
|
+
this.sendFirst();
|
|
950
|
+
return utterance;
|
|
951
|
+
}
|
|
952
|
+
endUtterance(utterance) {
|
|
953
|
+
utterance.state = "ended";
|
|
954
|
+
if (utterance.sent) this.sendEnd(utterance);
|
|
955
|
+
}
|
|
956
|
+
/** Sends the first utterance to Gemini, with the audio it held meanwhile */
|
|
957
|
+
sendFirst() {
|
|
958
|
+
const utterance = this.utterances[0];
|
|
959
|
+
if (!utterance || utterance.sent) return;
|
|
960
|
+
utterance.sent = true;
|
|
846
961
|
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
962
|
+
utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
963
|
+
if (utterance.state === "ended") this.sendEnd(utterance);
|
|
964
|
+
}
|
|
965
|
+
sendEnd(utterance) {
|
|
966
|
+
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
967
|
+
utterance.timeout = setTimeout(() => {
|
|
968
|
+
utterance.timeout = void 0;
|
|
969
|
+
this.log("Transcription timeout");
|
|
970
|
+
this.give(utterance);
|
|
971
|
+
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
847
972
|
}
|
|
848
973
|
sendAudio(chunk) {
|
|
849
974
|
this.socket.send({
|
|
@@ -855,45 +980,60 @@ var GeminiSTT = class extends import_server3.STT {
|
|
|
855
980
|
}
|
|
856
981
|
});
|
|
857
982
|
}
|
|
858
|
-
/**
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
983
|
+
/**
|
|
984
|
+
* Emits what was transcribed of an utterance, possibly nothing, and keeps
|
|
985
|
+
* it in line for what Gemini still sends about it
|
|
986
|
+
*/
|
|
987
|
+
give(utterance) {
|
|
988
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
989
|
+
utterance.state = "given";
|
|
990
|
+
const transcript = utterance.transcript.trim();
|
|
866
991
|
this.log(`Transcript: "${transcript}"`);
|
|
867
992
|
this.emit("Transcript", transcript);
|
|
868
|
-
|
|
869
|
-
this.
|
|
870
|
-
this.
|
|
871
|
-
|
|
993
|
+
utterance.timeout = setTimeout(
|
|
994
|
+
() => this.remove(utterance),
|
|
995
|
+
this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
|
|
996
|
+
);
|
|
872
997
|
}
|
|
873
|
-
|
|
874
|
-
if (
|
|
875
|
-
|
|
876
|
-
this.
|
|
998
|
+
remove(utterance) {
|
|
999
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
1000
|
+
utterance.timeout = void 0;
|
|
1001
|
+
this.utterances = this.utterances.filter((other) => other !== utterance);
|
|
1002
|
+
this.renewIfGoingAway();
|
|
1003
|
+
this.sendFirst();
|
|
1004
|
+
}
|
|
1005
|
+
/** Moves to a new connection once no utterance waits on this one */
|
|
1006
|
+
renewIfGoingAway() {
|
|
1007
|
+
if (!this.goingAway || this.utterances.length > 0) return;
|
|
1008
|
+
this.goingAway = false;
|
|
1009
|
+
this.socket.renew();
|
|
1010
|
+
}
|
|
1011
|
+
dropUtterances() {
|
|
1012
|
+
for (const utterance of this.utterances) {
|
|
1013
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
1014
|
+
if (utterance.state === "open") utterance.state = "ended";
|
|
1015
|
+
}
|
|
1016
|
+
this.utterances = [];
|
|
877
1017
|
}
|
|
878
1018
|
};
|
|
879
1019
|
|
|
880
1020
|
// src/GeminiTTS.ts
|
|
881
|
-
var
|
|
882
|
-
var
|
|
883
|
-
var
|
|
1021
|
+
var import_genai3 = require("@google/genai");
|
|
1022
|
+
var import_server6 = require("@micdrop/server");
|
|
1023
|
+
var DEFAULT_MODEL5 = "gemini-2.5-flash-preview-tts";
|
|
884
1024
|
var DEFAULT_VOICE = "Kore";
|
|
885
1025
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
886
1026
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
887
|
-
var
|
|
888
|
-
var GeminiTTS = class extends
|
|
1027
|
+
var DEFAULT_MAX_RETRY = 2;
|
|
1028
|
+
var GeminiTTS = class extends import_server6.SentenceTTS {
|
|
889
1029
|
constructor(options) {
|
|
890
1030
|
super();
|
|
891
1031
|
this.options = options;
|
|
892
|
-
this.genai = "genai" in options ? options.genai : new
|
|
1032
|
+
this.genai = "genai" in options ? options.genai : new import_genai3.GoogleGenAI({ apiKey: options.apiKey });
|
|
893
1033
|
}
|
|
894
1034
|
async synthesize(text, signal) {
|
|
895
1035
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
896
|
-
const maxRetry = this.options.maxRetry ??
|
|
1036
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
|
|
897
1037
|
for (let attempt = 0; ; attempt++) {
|
|
898
1038
|
const progress = { emitted: false };
|
|
899
1039
|
try {
|
|
@@ -911,7 +1051,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
|
|
|
911
1051
|
async synthesizeOnce(input, signal, progress) {
|
|
912
1052
|
const stream = await this.genai.interactions.create(
|
|
913
1053
|
{
|
|
914
|
-
model: this.options.model ||
|
|
1054
|
+
model: this.options.model || DEFAULT_MODEL5,
|
|
915
1055
|
store: false,
|
|
916
1056
|
stream: true,
|
|
917
1057
|
input,
|
|
@@ -936,7 +1076,7 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
|
|
|
936
1076
|
}
|
|
937
1077
|
const { data, sample_rate } = event.delta;
|
|
938
1078
|
if (!data) continue;
|
|
939
|
-
resampler ?? (resampler = new
|
|
1079
|
+
resampler ?? (resampler = new import_server6.Pcm16Resampler(
|
|
940
1080
|
sample_rate || GEMINI_SAMPLE_RATE,
|
|
941
1081
|
OUTPUT_SAMPLE_RATE
|
|
942
1082
|
));
|
|
@@ -947,13 +1087,17 @@ var GeminiTTS = class extends import_server4.SentenceTTS {
|
|
|
947
1087
|
}
|
|
948
1088
|
};
|
|
949
1089
|
function isBlocked(error) {
|
|
950
|
-
return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
1090
|
+
return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
951
1091
|
}
|
|
952
1092
|
// Annotate the CommonJS export names for ESM import in node:
|
|
953
1093
|
0 && (module.exports = {
|
|
954
1094
|
GeminiAgent,
|
|
1095
|
+
GeminiClassifier,
|
|
955
1096
|
GeminiLive,
|
|
956
1097
|
GeminiSTT,
|
|
957
|
-
GeminiTTS
|
|
1098
|
+
GeminiTTS,
|
|
1099
|
+
choice,
|
|
1100
|
+
noul,
|
|
1101
|
+
score
|
|
958
1102
|
});
|
|
959
1103
|
//# sourceMappingURL=index.js.map
|