@micdrop/gemini 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -2
- package/dist/index.d.mts +156 -19
- package/dist/index.d.ts +156 -19
- package/dist/index.js +467 -323
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +456 -314
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -4
package/dist/index.mjs
CHANGED
|
@@ -15,127 +15,139 @@ var GeminiAgent = class extends Agent {
|
|
|
15
15
|
this.signatures = /* @__PURE__ */ new Map();
|
|
16
16
|
this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
|
|
17
17
|
}
|
|
18
|
-
|
|
18
|
+
generateAnswer(stream, signal) {
|
|
19
|
+
return this.generateStep(stream, signal, 0);
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Generates one step of the answer, retried on error.
|
|
23
|
+
*
|
|
24
|
+
* A retry starts the step over, so it only happens while nothing of the
|
|
25
|
+
* step reached the stream: what was said cannot be taken back, and saying
|
|
26
|
+
* it twice is worse than stopping there.
|
|
27
|
+
*/
|
|
28
|
+
async generateStep(stream, signal, stepCount) {
|
|
19
29
|
if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
|
|
20
30
|
console.error("[GeminiAgent] Max steps reached");
|
|
21
31
|
return;
|
|
22
32
|
}
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
|
|
34
|
+
for (let tryCount = 0; ; tryCount++) {
|
|
35
|
+
const progress = { answer: "" };
|
|
36
|
+
try {
|
|
37
|
+
const step = await this.streamStep(stream, signal, progress);
|
|
38
|
+
if (signal.aborted) return;
|
|
39
|
+
this.recordAnswer(progress.answer);
|
|
40
|
+
if (step.calledTool && !step.skipAnswer && !progress.answer) {
|
|
41
|
+
await this.generateStep(stream, signal, stepCount + 1);
|
|
42
|
+
}
|
|
43
|
+
return;
|
|
44
|
+
} catch (error) {
|
|
45
|
+
if (signal.aborted) return;
|
|
46
|
+
console.error("[GeminiAgent] Error answering:", error);
|
|
47
|
+
if (progress.answer) {
|
|
48
|
+
this.log("Error after part of the answer was said, keeping it");
|
|
49
|
+
this.recordAnswer(progress.answer);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (tryCount >= maxRetry) {
|
|
53
|
+
this.log("Max retries reached, giving up");
|
|
54
|
+
this.emit("Failed");
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
|
|
58
|
+
if (signal.aborted) return;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Streams one request, writing the text of the answer as it comes */
|
|
63
|
+
async streamStep(stream, signal, progress) {
|
|
26
64
|
let extracting = false;
|
|
27
65
|
const extractOptions = this.getExtractOptions();
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
if (startTagIndex !== -1) {
|
|
81
|
-
extracting = true;
|
|
82
|
-
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
83
|
-
break;
|
|
84
|
-
}
|
|
66
|
+
const response = await this.genai.interactions.create(
|
|
67
|
+
{
|
|
68
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
69
|
+
store: false,
|
|
70
|
+
stream: true,
|
|
71
|
+
system_instruction: this.buildSystemInstruction(),
|
|
72
|
+
input: this.buildInput(),
|
|
73
|
+
tools: this.buildTools(),
|
|
74
|
+
...this.options.thinkingLevel || this.options.settings ? {
|
|
75
|
+
generation_config: {
|
|
76
|
+
...this.options.settings,
|
|
77
|
+
...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
|
|
78
|
+
}
|
|
79
|
+
} : {}
|
|
80
|
+
},
|
|
81
|
+
{ signal }
|
|
82
|
+
);
|
|
83
|
+
const steps = [];
|
|
84
|
+
let skipAnswer = false;
|
|
85
|
+
let calledTool = false;
|
|
86
|
+
let signature = "";
|
|
87
|
+
for await (const event of response) {
|
|
88
|
+
if (signal.aborted) break;
|
|
89
|
+
switch (event.event_type) {
|
|
90
|
+
case "step.start":
|
|
91
|
+
steps[event.index] = {
|
|
92
|
+
type: event.step.type,
|
|
93
|
+
id: event.step.id,
|
|
94
|
+
name: event.step.name,
|
|
95
|
+
text: "",
|
|
96
|
+
arguments: "",
|
|
97
|
+
signature: ""
|
|
98
|
+
};
|
|
99
|
+
break;
|
|
100
|
+
case "step.delta": {
|
|
101
|
+
const step = steps[event.index];
|
|
102
|
+
if (!step) break;
|
|
103
|
+
const delta = event.delta;
|
|
104
|
+
if (delta.type === "thought_signature") {
|
|
105
|
+
step.signature += delta.signature ?? "";
|
|
106
|
+
} else if (delta.type === "arguments_delta") {
|
|
107
|
+
step.arguments += delta.arguments ?? "";
|
|
108
|
+
} else if (delta.type === "text" && step.type === "model_output") {
|
|
109
|
+
this.log(`Answer chunk: "${delta.text}"`);
|
|
110
|
+
progress.answer += delta.text;
|
|
111
|
+
if (extractOptions) {
|
|
112
|
+
if (extracting) break;
|
|
113
|
+
const startTagIndex = delta.text.indexOf(extractOptions.startTag);
|
|
114
|
+
if (startTagIndex !== -1) {
|
|
115
|
+
extracting = true;
|
|
116
|
+
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
117
|
+
break;
|
|
85
118
|
}
|
|
86
|
-
stream.write(delta.text);
|
|
87
119
|
}
|
|
88
|
-
|
|
89
|
-
}
|
|
90
|
-
case "step.stop": {
|
|
91
|
-
const step = steps[event.index];
|
|
92
|
-
if (step?.type === "thought") signature = step.signature;
|
|
93
|
-
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
94
|
-
calledTool = true;
|
|
95
|
-
if (signature) this.signatures.set(step.id, signature);
|
|
96
|
-
const result = await this.executeTool({
|
|
97
|
-
role: "tool_call",
|
|
98
|
-
toolCallId: step.id,
|
|
99
|
-
toolName: step.name,
|
|
100
|
-
parameters: step.arguments || "{}"
|
|
101
|
-
});
|
|
102
|
-
if (result.skipAnswer) skipAnswer = true;
|
|
103
|
-
break;
|
|
120
|
+
stream.write(delta.text);
|
|
104
121
|
}
|
|
105
|
-
|
|
106
|
-
throw new Error(event.error?.message || "Generation failed");
|
|
107
|
-
default:
|
|
108
|
-
break;
|
|
122
|
+
break;
|
|
109
123
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
this.log("Max retries reached, giving up");
|
|
130
|
-
this.emit("Failed");
|
|
124
|
+
case "step.stop": {
|
|
125
|
+
const step = steps[event.index];
|
|
126
|
+
if (step?.type === "thought") signature = step.signature;
|
|
127
|
+
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
128
|
+
calledTool = true;
|
|
129
|
+
if (signature) this.signatures.set(step.id, signature);
|
|
130
|
+
const result = await this.executeTool({
|
|
131
|
+
role: "tool_call",
|
|
132
|
+
toolCallId: step.id,
|
|
133
|
+
toolName: step.name,
|
|
134
|
+
parameters: step.arguments || "{}"
|
|
135
|
+
});
|
|
136
|
+
if (result.skipAnswer) skipAnswer = true;
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
case "error":
|
|
140
|
+
throw new Error(event.error?.message || "Generation failed");
|
|
141
|
+
default:
|
|
142
|
+
break;
|
|
131
143
|
}
|
|
132
144
|
}
|
|
145
|
+
return { calledTool, skipAnswer };
|
|
133
146
|
}
|
|
134
|
-
|
|
135
|
-
if (!
|
|
136
|
-
this.
|
|
137
|
-
this.
|
|
138
|
-
this.abortController = void 0;
|
|
147
|
+
recordAnswer(answer) {
|
|
148
|
+
if (!answer) return;
|
|
149
|
+
const { message, metadata } = this.extract(answer);
|
|
150
|
+
this.addAssistantMessage(message, metadata);
|
|
139
151
|
}
|
|
140
152
|
buildSystemInstruction() {
|
|
141
153
|
return this.conversation.flatMap(
|
|
@@ -207,6 +219,17 @@ var GeminiAgent = class extends Agent {
|
|
|
207
219
|
});
|
|
208
220
|
}
|
|
209
221
|
};
|
|
222
|
+
function sleep(ms, signal) {
|
|
223
|
+
return new Promise((resolve) => {
|
|
224
|
+
const done = () => {
|
|
225
|
+
clearTimeout(timer);
|
|
226
|
+
signal.removeEventListener("abort", done);
|
|
227
|
+
resolve();
|
|
228
|
+
};
|
|
229
|
+
const timer = setTimeout(done, ms);
|
|
230
|
+
signal.addEventListener("abort", done);
|
|
231
|
+
});
|
|
232
|
+
}
|
|
210
233
|
function parseArguments(parameters) {
|
|
211
234
|
try {
|
|
212
235
|
return JSON.parse(parameters);
|
|
@@ -215,6 +238,72 @@ function parseArguments(parameters) {
|
|
|
215
238
|
}
|
|
216
239
|
}
|
|
217
240
|
|
|
241
|
+
// src/GeminiClassifier.ts
|
|
242
|
+
import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
|
|
243
|
+
import { Classifier } from "@micdrop/server";
|
|
244
|
+
function noul(instructions, criteria) {
|
|
245
|
+
return {
|
|
246
|
+
type: "noul",
|
|
247
|
+
instructions,
|
|
248
|
+
criteria: criteria && {
|
|
249
|
+
true_description: criteria.true,
|
|
250
|
+
false_description: criteria.false
|
|
251
|
+
}
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
function choice(instructions, criteria) {
|
|
255
|
+
return { type: "choice", instructions, criteria };
|
|
256
|
+
}
|
|
257
|
+
function score(instructions, levels) {
|
|
258
|
+
return { type: "score", instructions, criteria: levels };
|
|
259
|
+
}
|
|
260
|
+
var DEFAULT_MODEL2 = "gemini-decision-1";
|
|
261
|
+
var DEFAULT_TIMEOUT = 3e3;
|
|
262
|
+
var API_VERSION = "v1beta";
|
|
263
|
+
var GeminiClassifier = class extends Classifier {
|
|
264
|
+
constructor(options) {
|
|
265
|
+
super();
|
|
266
|
+
this.options = options;
|
|
267
|
+
this.genai = "genai" in options ? options.genai : new GoogleGenAI2({ apiKey: options.apiKey });
|
|
268
|
+
}
|
|
269
|
+
async evaluate(input, signal) {
|
|
270
|
+
const { questions, model, timeout } = this.options;
|
|
271
|
+
const asked = typeof questions === "function" ? questions(input) : questions;
|
|
272
|
+
const decision = await this.genai.decisions.create(
|
|
273
|
+
{
|
|
274
|
+
model: model || DEFAULT_MODEL2,
|
|
275
|
+
input: this.buildInput(input),
|
|
276
|
+
questions: asked,
|
|
277
|
+
// Only on v1beta, while the client defaults to v1
|
|
278
|
+
api_version: API_VERSION
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
// The SDK drops its timeout when a signal is given
|
|
282
|
+
signal: AbortSignal.any([
|
|
283
|
+
signal,
|
|
284
|
+
AbortSignal.timeout(timeout ?? DEFAULT_TIMEOUT)
|
|
285
|
+
]),
|
|
286
|
+
retries: { strategy: "none" }
|
|
287
|
+
}
|
|
288
|
+
);
|
|
289
|
+
const blocked = decision.prompt_feedback?.block_reason;
|
|
290
|
+
if (blocked) {
|
|
291
|
+
const message = decision.prompt_feedback?.block_reason_message;
|
|
292
|
+
throw new Error(
|
|
293
|
+
`Input blocked by Gemini (${blocked})${message ? `: ${message}` : ""}`
|
|
294
|
+
);
|
|
295
|
+
}
|
|
296
|
+
return {
|
|
297
|
+
answers: decision.answers ?? {},
|
|
298
|
+
usage: decision.usage
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
buildInput(input) {
|
|
302
|
+
if (this.options.input) return this.options.input(input);
|
|
303
|
+
return typeof input === "string" ? input : JSON.stringify(input);
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
|
|
218
307
|
// src/GeminiLive.ts
|
|
219
308
|
import {
|
|
220
309
|
Pcm16Resampler,
|
|
@@ -223,67 +312,107 @@ import {
|
|
|
223
312
|
import { toJSONSchema as toJSONSchema2 } from "zod";
|
|
224
313
|
|
|
225
314
|
// src/LiveSocket.ts
|
|
315
|
+
import {
|
|
316
|
+
ReconnectingSocket
|
|
317
|
+
} from "@micdrop/server";
|
|
226
318
|
import { EventEmitter } from "eventemitter3";
|
|
227
|
-
import WebSocket from "ws";
|
|
228
319
|
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
229
320
|
var SAMPLE_RATE = 16e3;
|
|
230
321
|
var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
|
|
231
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
232
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
233
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
234
322
|
var LiveSocket = class extends EventEmitter {
|
|
235
323
|
constructor(options, buildSetup, log) {
|
|
236
324
|
super();
|
|
237
|
-
this.options = options;
|
|
238
325
|
this.buildSetup = buildSetup;
|
|
239
326
|
this.log = log;
|
|
240
|
-
this.connected = false;
|
|
241
|
-
this.destroyed = false;
|
|
242
327
|
// Messages waiting for the session, or for the end of a turn to be due
|
|
243
328
|
this.pending = [];
|
|
244
329
|
// Audio of the turn as it went out, see flush()
|
|
245
330
|
this.turnAudioStartedAt = 0;
|
|
246
331
|
this.turnAudioMs = 0;
|
|
247
|
-
|
|
332
|
+
// Attempts since the last session was ready, for the owner to tell a drop
|
|
333
|
+
// from a resumption that keeps failing
|
|
334
|
+
this.reconnectAttempts = 0;
|
|
335
|
+
/** Sends the setup, and holds the connection back until Gemini accepts it */
|
|
336
|
+
this.handshake = {
|
|
337
|
+
start: (socket) => {
|
|
338
|
+
const setup = this.buildSetup();
|
|
339
|
+
this.setup = setup;
|
|
340
|
+
socket.send(JSON.stringify(setup));
|
|
341
|
+
},
|
|
342
|
+
read: (data) => !!parseMessage(data.toString())?.setupComplete
|
|
343
|
+
};
|
|
344
|
+
this.onOpen = () => {
|
|
345
|
+
this.log("Session ready");
|
|
346
|
+
this.reconnectAttempts = 0;
|
|
347
|
+
this.emit("Ready", this.setup);
|
|
348
|
+
this.flush();
|
|
349
|
+
};
|
|
350
|
+
this.onMessage = (data) => {
|
|
351
|
+
const message = parseMessage(data.toString());
|
|
352
|
+
if (message) this.emit("Message", message);
|
|
353
|
+
};
|
|
354
|
+
this.onStatusChange = (status) => {
|
|
355
|
+
if (status !== "open") this.clearPendingTimeout();
|
|
356
|
+
if (status === "reconnecting") {
|
|
357
|
+
this.emit("Reconnecting", ++this.reconnectAttempts);
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
this.onFailed = () => {
|
|
361
|
+
this.clearPending();
|
|
362
|
+
this.emit("Failed");
|
|
363
|
+
};
|
|
364
|
+
this.socket = new ReconnectingSocket({
|
|
365
|
+
url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
|
|
366
|
+
connectionTimeout: options.connectionTimeout,
|
|
367
|
+
retryDelay: options.retryDelay,
|
|
368
|
+
maxRetry: options.maxRetry,
|
|
369
|
+
handshake: this.handshake,
|
|
370
|
+
log
|
|
371
|
+
});
|
|
372
|
+
this.socket.on("Open", this.onOpen);
|
|
373
|
+
this.socket.on("Message", this.onMessage);
|
|
374
|
+
this.socket.on("StatusChange", this.onStatusChange);
|
|
375
|
+
this.socket.on("Failed", this.onFailed);
|
|
248
376
|
}
|
|
249
377
|
/** The session is set up and takes messages right away */
|
|
250
378
|
get isConnected() {
|
|
251
|
-
return this.
|
|
379
|
+
return this.socket.isOpen;
|
|
252
380
|
}
|
|
253
381
|
connect() {
|
|
254
|
-
this.
|
|
255
|
-
this.log("Connection error:", error);
|
|
256
|
-
this.reconnect();
|
|
257
|
-
});
|
|
382
|
+
this.socket.connect();
|
|
258
383
|
}
|
|
259
|
-
/**
|
|
384
|
+
/**
|
|
385
|
+
* Sends a message once the session is ready, after the ones before it.
|
|
386
|
+
* Dropped once the connection failed for good, as nothing would send it.
|
|
387
|
+
*/
|
|
260
388
|
send(message) {
|
|
389
|
+
const status = this.socket.status;
|
|
390
|
+
if (status === "failed" || status === "destroyed") return;
|
|
261
391
|
this.pending.push(message);
|
|
262
392
|
this.flush();
|
|
263
393
|
}
|
|
264
394
|
/** Sends a message ahead of the waiting ones, from a Ready listener */
|
|
265
395
|
sendFirst(message) {
|
|
266
|
-
this.socket
|
|
396
|
+
this.socket.send(JSON.stringify(message));
|
|
267
397
|
}
|
|
268
398
|
/** Drops the messages that have not gone out */
|
|
269
399
|
clearPending() {
|
|
270
400
|
this.pending = [];
|
|
271
401
|
this.clearPendingTimeout();
|
|
272
402
|
}
|
|
273
|
-
/**
|
|
403
|
+
/**
|
|
404
|
+
* Moves to a new connection, keeping the messages that have not gone out.
|
|
405
|
+
* A reconnection waiting for its delay is replaced by this one.
|
|
406
|
+
*/
|
|
274
407
|
renew() {
|
|
275
408
|
this.log("Renewing the connection");
|
|
276
|
-
this.
|
|
277
|
-
this.
|
|
409
|
+
this.clearPendingTimeout();
|
|
410
|
+
this.reconnectAttempts = 0;
|
|
411
|
+
this.socket.restart();
|
|
278
412
|
}
|
|
279
413
|
destroy() {
|
|
280
|
-
this.
|
|
281
|
-
|
|
282
|
-
clearTimeout(this.reconnectTimeout);
|
|
283
|
-
this.reconnectTimeout = void 0;
|
|
284
|
-
}
|
|
285
|
-
this.closeSocket();
|
|
286
|
-
this.pending = [];
|
|
414
|
+
this.clearPending();
|
|
415
|
+
this.socket.destroy();
|
|
287
416
|
this.removeAllListeners();
|
|
288
417
|
}
|
|
289
418
|
/**
|
|
@@ -296,8 +425,7 @@ var LiveSocket = class extends EventEmitter {
|
|
|
296
425
|
*/
|
|
297
426
|
flush() {
|
|
298
427
|
while (this.pending.length > 0 && !this.pendingTimeout) {
|
|
299
|
-
|
|
300
|
-
if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
|
|
428
|
+
if (!this.socket.isOpen) return;
|
|
301
429
|
const message = this.pending[0];
|
|
302
430
|
const input = message.realtimeInput;
|
|
303
431
|
if (input?.activityEnd) {
|
|
@@ -317,7 +445,7 @@ var LiveSocket = class extends EventEmitter {
|
|
|
317
445
|
this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
|
|
318
446
|
}
|
|
319
447
|
this.pending.shift();
|
|
320
|
-
socket.send(JSON.stringify(message));
|
|
448
|
+
this.socket.send(JSON.stringify(message));
|
|
321
449
|
}
|
|
322
450
|
}
|
|
323
451
|
clearPendingTimeout() {
|
|
@@ -325,96 +453,28 @@ var LiveSocket = class extends EventEmitter {
|
|
|
325
453
|
clearTimeout(this.pendingTimeout);
|
|
326
454
|
this.pendingTimeout = void 0;
|
|
327
455
|
}
|
|
328
|
-
initWS() {
|
|
329
|
-
return new Promise((resolve, reject) => {
|
|
330
|
-
const socket = new WebSocket(
|
|
331
|
-
`${URL}?key=${encodeURIComponent(this.options.apiKey)}`
|
|
332
|
-
);
|
|
333
|
-
this.socket = socket;
|
|
334
|
-
const timeout = setTimeout(() => {
|
|
335
|
-
this.log("Connection timeout");
|
|
336
|
-
this.closeSocket();
|
|
337
|
-
reject(new Error("WebSocket connection timeout"));
|
|
338
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
339
|
-
socket.on("open", () => {
|
|
340
|
-
this.log("Connection opened");
|
|
341
|
-
socket.send(JSON.stringify(this.buildSetup()));
|
|
342
|
-
});
|
|
343
|
-
socket.on("error", (error) => {
|
|
344
|
-
clearTimeout(timeout);
|
|
345
|
-
this.log("WebSocket error:", error);
|
|
346
|
-
reject(new Error("WebSocket connection error"));
|
|
347
|
-
});
|
|
348
|
-
socket.on("close", (code, reason) => {
|
|
349
|
-
clearTimeout(timeout);
|
|
350
|
-
const wasConnected = this.connected;
|
|
351
|
-
this.closeSocket();
|
|
352
|
-
if (this.destroyed) return;
|
|
353
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
354
|
-
if (wasConnected) {
|
|
355
|
-
this.reconnect();
|
|
356
|
-
} else {
|
|
357
|
-
reject(new Error(`Connection closed: ${reason.toString()}`));
|
|
358
|
-
}
|
|
359
|
-
});
|
|
360
|
-
socket.on("message", (data) => {
|
|
361
|
-
const message = JSON.parse(data.toString());
|
|
362
|
-
if (message.setupComplete) {
|
|
363
|
-
clearTimeout(timeout);
|
|
364
|
-
this.log("Session ready");
|
|
365
|
-
this.connected = true;
|
|
366
|
-
this.retryCount = 0;
|
|
367
|
-
this.emit("Ready");
|
|
368
|
-
this.flush();
|
|
369
|
-
resolve();
|
|
370
|
-
return;
|
|
371
|
-
}
|
|
372
|
-
this.emit("Message", message);
|
|
373
|
-
});
|
|
374
|
-
});
|
|
375
|
-
}
|
|
376
|
-
closeSocket() {
|
|
377
|
-
const socket = this.socket;
|
|
378
|
-
this.socket = void 0;
|
|
379
|
-
this.connected = false;
|
|
380
|
-
this.clearPendingTimeout();
|
|
381
|
-
if (!socket) return;
|
|
382
|
-
socket.removeAllListeners();
|
|
383
|
-
socket.on("error", () => {
|
|
384
|
-
});
|
|
385
|
-
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
386
|
-
socket.close(1e3);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
reconnect() {
|
|
390
|
-
if (this.destroyed) return;
|
|
391
|
-
this.retryCount++;
|
|
392
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
393
|
-
this.log("Max retries reached, giving up");
|
|
394
|
-
this.emit("Failed");
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
this.emit("Reconnecting", this.retryCount);
|
|
398
|
-
this.log("Reconnecting...");
|
|
399
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
400
|
-
this.reconnectTimeout = void 0;
|
|
401
|
-
this.connect();
|
|
402
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
403
|
-
}
|
|
404
456
|
};
|
|
457
|
+
function parseMessage(text) {
|
|
458
|
+
try {
|
|
459
|
+
return JSON.parse(text);
|
|
460
|
+
} catch {
|
|
461
|
+
return void 0;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
405
464
|
|
|
406
465
|
// src/GeminiLive.ts
|
|
407
|
-
var
|
|
466
|
+
var DEFAULT_MODEL3 = "gemini-3.8-live";
|
|
408
467
|
var SAMPLE_RATE2 = 16e3;
|
|
409
468
|
var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
|
|
410
469
|
var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
|
|
470
|
+
function newToolRound() {
|
|
471
|
+
return { running: 0, answerEndDeferred: false };
|
|
472
|
+
}
|
|
411
473
|
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
412
474
|
var GeminiLive = class extends Realtime {
|
|
413
475
|
constructor(options) {
|
|
414
476
|
super(options);
|
|
415
477
|
this.everConnected = false;
|
|
416
|
-
// Whether the setup being sent resumes the session
|
|
417
|
-
this.resuming = false;
|
|
418
478
|
this.goingAway = false;
|
|
419
479
|
this.outputResampler = new Pcm16Resampler(
|
|
420
480
|
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
@@ -423,14 +483,16 @@ var GeminiLive = class extends Realtime {
|
|
|
423
483
|
this.hasTranscribedTurn = false;
|
|
424
484
|
// Transcript of a dropped turn, which the conversation should not show
|
|
425
485
|
this.ignoreUserTranscript = false;
|
|
486
|
+
// Calls of the session Gemini told not to answer
|
|
426
487
|
this.cancelledToolCalls = /* @__PURE__ */ new Set();
|
|
427
|
-
//
|
|
428
|
-
//
|
|
429
|
-
|
|
430
|
-
this.
|
|
431
|
-
this.
|
|
432
|
-
|
|
433
|
-
if (this.everConnected && !
|
|
488
|
+
// Bumped by every session that does not resume the previous one, whose tool
|
|
489
|
+
// calls it cannot answer
|
|
490
|
+
this.session = 0;
|
|
491
|
+
this.toolRound = newToolRound();
|
|
492
|
+
this.onReady = (setup) => {
|
|
493
|
+
const resumed = !!setup?.setup?.sessionResumption?.handle;
|
|
494
|
+
if (this.everConnected && !resumed) {
|
|
495
|
+
this.startFreshSession();
|
|
434
496
|
const turns = this.conversation.flatMap(
|
|
435
497
|
(message) => message.role === "user" || message.role === "assistant" ? [
|
|
436
498
|
{
|
|
@@ -448,7 +510,12 @@ var GeminiLive = class extends Realtime {
|
|
|
448
510
|
};
|
|
449
511
|
this.onReconnecting = (attempt) => {
|
|
450
512
|
if (attempt > 1) this.resumptionHandle = void 0;
|
|
451
|
-
if (!this.resumptionHandle)
|
|
513
|
+
if (!this.resumptionHandle) {
|
|
514
|
+
this.socket.clearPending();
|
|
515
|
+
this.startFreshSession();
|
|
516
|
+
}
|
|
517
|
+
this.toolRound.answerEndDeferred = false;
|
|
518
|
+
this.commitTurnTranscript();
|
|
452
519
|
this.endAnswer();
|
|
453
520
|
};
|
|
454
521
|
this.onMessage = (message) => {
|
|
@@ -486,9 +553,16 @@ var GeminiLive = class extends Realtime {
|
|
|
486
553
|
this.socket.on("Failed", () => this.emit("Failed"));
|
|
487
554
|
this.socket.connect();
|
|
488
555
|
}
|
|
556
|
+
/**
|
|
557
|
+
* Stops the answer. The tools it is waiting for still send their results,
|
|
558
|
+
* silently, since a blocking call left unanswered would stall the model.
|
|
559
|
+
*/
|
|
560
|
+
cancel() {
|
|
561
|
+
super.cancel();
|
|
562
|
+
this.resetToolRound();
|
|
563
|
+
}
|
|
489
564
|
destroy() {
|
|
490
565
|
super.destroy();
|
|
491
|
-
this.clearToolFollowUp();
|
|
492
566
|
this.socket.destroy();
|
|
493
567
|
}
|
|
494
568
|
openTurn() {
|
|
@@ -523,11 +597,9 @@ var GeminiLive = class extends Realtime {
|
|
|
523
597
|
}
|
|
524
598
|
cancelAnswer() {
|
|
525
599
|
this.outputResampler.reset();
|
|
526
|
-
this.clearToolFollowUp();
|
|
527
|
-
this.answerEndDeferred = false;
|
|
528
600
|
}
|
|
529
601
|
get isIdle() {
|
|
530
|
-
return super.isIdle && this.
|
|
602
|
+
return super.isIdle && this.toolRound.running === 0;
|
|
531
603
|
}
|
|
532
604
|
updateTools() {
|
|
533
605
|
if (this.everConnected) {
|
|
@@ -559,7 +631,6 @@ var GeminiLive = class extends Realtime {
|
|
|
559
631
|
this.socket.send(message);
|
|
560
632
|
}
|
|
561
633
|
buildSetup() {
|
|
562
|
-
this.resuming = !!this.resumptionHandle;
|
|
563
634
|
const functionDeclarations = this.tools.map((tool) => ({
|
|
564
635
|
name: tool.name,
|
|
565
636
|
description: tool.description,
|
|
@@ -567,7 +638,7 @@ var GeminiLive = class extends Realtime {
|
|
|
567
638
|
}));
|
|
568
639
|
return {
|
|
569
640
|
setup: {
|
|
570
|
-
model: `models/${this.options.model ||
|
|
641
|
+
model: `models/${this.options.model || DEFAULT_MODEL3}`,
|
|
571
642
|
generationConfig: {
|
|
572
643
|
responseModalities: ["AUDIO"],
|
|
573
644
|
...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
|
|
@@ -593,6 +664,22 @@ var GeminiLive = class extends Realtime {
|
|
|
593
664
|
}
|
|
594
665
|
};
|
|
595
666
|
}
|
|
667
|
+
/** Forgets what belonged to the previous session */
|
|
668
|
+
startFreshSession() {
|
|
669
|
+
this.session++;
|
|
670
|
+
this.resetToolRound();
|
|
671
|
+
this.cancelledToolCalls.clear();
|
|
672
|
+
this.ignoreUserTranscript = false;
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Starts a new round of tools. The tools still running belong to the old
|
|
676
|
+
* one: their results go out silently, and nothing waits for the model to
|
|
677
|
+
* talk about them.
|
|
678
|
+
*/
|
|
679
|
+
resetToolRound() {
|
|
680
|
+
if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
|
|
681
|
+
this.toolRound = newToolRound();
|
|
682
|
+
}
|
|
596
683
|
/** Moves to a new connection before Gemini closes this one */
|
|
597
684
|
renewIfGoingAway() {
|
|
598
685
|
if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
|
|
@@ -624,8 +711,8 @@ var GeminiLive = class extends Realtime {
|
|
|
624
711
|
if (content.turnComplete) {
|
|
625
712
|
this.commitTurnTranscript();
|
|
626
713
|
this.outputResampler.reset();
|
|
627
|
-
if (this.
|
|
628
|
-
this.answerEndDeferred = true;
|
|
714
|
+
if (this.toolRound.running > 0 || this.toolRound.followUp) {
|
|
715
|
+
this.toolRound.answerEndDeferred = true;
|
|
629
716
|
} else {
|
|
630
717
|
this.endAnswer();
|
|
631
718
|
this.renewIfGoingAway();
|
|
@@ -634,24 +721,22 @@ var GeminiLive = class extends Realtime {
|
|
|
634
721
|
}
|
|
635
722
|
onModelOutput() {
|
|
636
723
|
this.commitTurnTranscript();
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
724
|
+
const round = this.toolRound;
|
|
725
|
+
if (round.followUp) {
|
|
726
|
+
clearTimeout(round.followUp);
|
|
727
|
+
round.followUp = void 0;
|
|
728
|
+
round.answerEndDeferred = false;
|
|
640
729
|
}
|
|
641
730
|
}
|
|
642
731
|
/** Ends the answer the model stopped giving to wait for a tool */
|
|
643
732
|
endDeferredAnswer() {
|
|
644
|
-
|
|
645
|
-
if (
|
|
646
|
-
|
|
733
|
+
const round = this.toolRound;
|
|
734
|
+
if (!round.answerEndDeferred) return;
|
|
735
|
+
if (round.running > 0 || round.followUp) return;
|
|
736
|
+
round.answerEndDeferred = false;
|
|
647
737
|
this.endAnswer();
|
|
648
738
|
this.renewIfGoingAway();
|
|
649
739
|
}
|
|
650
|
-
clearToolFollowUp() {
|
|
651
|
-
if (!this.toolFollowUp) return;
|
|
652
|
-
clearTimeout(this.toolFollowUp);
|
|
653
|
-
this.toolFollowUp = void 0;
|
|
654
|
-
}
|
|
655
740
|
/**
|
|
656
741
|
* The transcript of the turn comes before the answer, so the first piece of
|
|
657
742
|
* the answer closes it.
|
|
@@ -669,17 +754,22 @@ var GeminiLive = class extends Realtime {
|
|
|
669
754
|
toolName: call.name,
|
|
670
755
|
parameters: JSON.stringify(call.args ?? {})
|
|
671
756
|
};
|
|
672
|
-
this.
|
|
757
|
+
const round = this.toolRound;
|
|
758
|
+
const session = this.session;
|
|
759
|
+
round.running++;
|
|
673
760
|
let result;
|
|
674
761
|
try {
|
|
675
762
|
result = await this.runTool(toolCall);
|
|
676
763
|
} finally {
|
|
677
|
-
|
|
764
|
+
round.running--;
|
|
678
765
|
}
|
|
766
|
+
if (session !== this.session) return;
|
|
679
767
|
if (this.cancelledToolCalls.delete(call.id)) {
|
|
680
768
|
this.endDeferredAnswer();
|
|
681
769
|
return;
|
|
682
770
|
}
|
|
771
|
+
const interrupted = round !== this.toolRound;
|
|
772
|
+
const silent = result.skipAnswer || interrupted;
|
|
683
773
|
this.send({
|
|
684
774
|
toolResponse: {
|
|
685
775
|
functionResponses: [
|
|
@@ -689,16 +779,17 @@ var GeminiLive = class extends Realtime {
|
|
|
689
779
|
response: {
|
|
690
780
|
output: result.output ?? null,
|
|
691
781
|
// A tool without an answer lets the model know silently
|
|
692
|
-
scheduling:
|
|
782
|
+
scheduling: silent ? "SILENT" : "WHEN_IDLE"
|
|
693
783
|
}
|
|
694
784
|
}
|
|
695
785
|
]
|
|
696
786
|
}
|
|
697
787
|
});
|
|
788
|
+
if (interrupted) return;
|
|
698
789
|
if (!result.skipAnswer) {
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
790
|
+
if (round.followUp) clearTimeout(round.followUp);
|
|
791
|
+
round.followUp = setTimeout(() => {
|
|
792
|
+
round.followUp = void 0;
|
|
702
793
|
this.log("Nothing said about the tool result");
|
|
703
794
|
this.endDeferredAnswer();
|
|
704
795
|
}, TOOL_FOLLOW_UP_TIMEOUT);
|
|
@@ -710,84 +801,90 @@ var GeminiLive = class extends Realtime {
|
|
|
710
801
|
|
|
711
802
|
// src/GeminiSTT.ts
|
|
712
803
|
import { STT } from "@micdrop/server";
|
|
713
|
-
var
|
|
804
|
+
var DEFAULT_MODEL4 = "gemini-3.5-transcribe-live";
|
|
714
805
|
var SAMPLE_RATE3 = 16e3;
|
|
715
806
|
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
716
807
|
var GeminiSTT = class extends STT {
|
|
717
808
|
constructor(options) {
|
|
718
809
|
super();
|
|
719
810
|
this.options = options;
|
|
811
|
+
// Gemini closes the session soon, and it is renewed once nothing waits
|
|
720
812
|
this.goingAway = false;
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
this.
|
|
726
|
-
this.
|
|
813
|
+
// Utterances waiting for their transcript, in the order they were spoken.
|
|
814
|
+
// Gemini names no utterance in what it sends, and goes on to the next one
|
|
815
|
+
// before it is done with the previous. So it hears one at a time: the first
|
|
816
|
+
// one, and the audio of the next waits here until its transcript is in.
|
|
817
|
+
this.utterances = [];
|
|
818
|
+
this.onReady = () => {
|
|
819
|
+
this.goingAway = false;
|
|
820
|
+
};
|
|
727
821
|
this.onMessage = (message) => {
|
|
728
822
|
const content = message.serverContent;
|
|
729
|
-
|
|
730
|
-
|
|
823
|
+
const utterance = this.utterances[0];
|
|
824
|
+
if (content?.inputTranscription?.text && utterance) {
|
|
825
|
+
utterance.transcript += content.inputTranscription.text;
|
|
731
826
|
}
|
|
732
|
-
if (content?.generationComplete &&
|
|
733
|
-
this.
|
|
827
|
+
if (content?.generationComplete && utterance) {
|
|
828
|
+
if (utterance.state === "ended") this.give(utterance);
|
|
829
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
734
830
|
}
|
|
735
831
|
if (message.goAway) {
|
|
736
832
|
this.log("Connection ending in", message.goAway.timeLeft);
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
} else {
|
|
740
|
-
this.socket.renew();
|
|
741
|
-
}
|
|
833
|
+
this.goingAway = true;
|
|
834
|
+
this.renewIfGoingAway();
|
|
742
835
|
}
|
|
743
836
|
};
|
|
744
837
|
this.onReconnecting = () => {
|
|
838
|
+
this.goingAway = false;
|
|
745
839
|
this.socket.clearPending();
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
749
|
-
this.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
750
|
-
if (this.utteranceEnded) {
|
|
751
|
-
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
840
|
+
for (const utterance of this.utterances) {
|
|
841
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
752
842
|
}
|
|
843
|
+
for (const utterance of this.utterances) {
|
|
844
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
845
|
+
utterance.timeout = void 0;
|
|
846
|
+
utterance.sent = false;
|
|
847
|
+
utterance.transcript = "";
|
|
848
|
+
}
|
|
849
|
+
this.sendFirst();
|
|
850
|
+
};
|
|
851
|
+
this.onFailed = () => {
|
|
852
|
+
const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
|
|
853
|
+
this.dropUtterances();
|
|
854
|
+
this.emit("Failed", chunks);
|
|
753
855
|
};
|
|
754
856
|
this.socket = new LiveSocket(
|
|
755
857
|
options,
|
|
756
858
|
() => this.buildSetup(),
|
|
757
859
|
(...message) => this.log(...message)
|
|
758
860
|
);
|
|
861
|
+
this.socket.on("Ready", this.onReady);
|
|
759
862
|
this.socket.on("Message", this.onMessage);
|
|
760
863
|
this.socket.on("Reconnecting", this.onReconnecting);
|
|
761
|
-
this.socket.on("Failed",
|
|
864
|
+
this.socket.on("Failed", this.onFailed);
|
|
762
865
|
this.socket.connect();
|
|
763
866
|
}
|
|
764
867
|
transcribe(audioStream) {
|
|
868
|
+
let utterance;
|
|
765
869
|
audioStream.on("data", (chunk) => {
|
|
766
|
-
if (!
|
|
767
|
-
|
|
768
|
-
|
|
870
|
+
if (!utterance) utterance = this.openUtterance();
|
|
871
|
+
if (utterance.state !== "open") return;
|
|
872
|
+
utterance.chunks.push(chunk);
|
|
873
|
+
if (utterance.sent) this.sendAudio(chunk);
|
|
769
874
|
});
|
|
770
875
|
audioStream.on("end", () => {
|
|
771
|
-
if (
|
|
772
|
-
this.utteranceEnded = true;
|
|
773
|
-
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
774
|
-
this.clearTranscriptionTimeout();
|
|
775
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
776
|
-
this.transcriptionTimeout = void 0;
|
|
777
|
-
this.log("Transcription timeout");
|
|
778
|
-
this.finishUtterance();
|
|
779
|
-
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
876
|
+
if (utterance?.state === "open") this.endUtterance(utterance);
|
|
780
877
|
});
|
|
781
878
|
}
|
|
782
879
|
destroy() {
|
|
783
880
|
super.destroy();
|
|
784
|
-
this.
|
|
881
|
+
this.dropUtterances();
|
|
785
882
|
this.socket.destroy();
|
|
786
883
|
}
|
|
787
884
|
buildSetup() {
|
|
788
885
|
return {
|
|
789
886
|
setup: {
|
|
790
|
-
model: `models/${this.options.model ||
|
|
887
|
+
model: `models/${this.options.model || DEFAULT_MODEL4}`,
|
|
791
888
|
generationConfig: { responseModalities: ["TEXT"] },
|
|
792
889
|
inputAudioTranscription: {
|
|
793
890
|
languageCodes: this.options.language ? [this.options.language] : [],
|
|
@@ -802,12 +899,38 @@ var GeminiSTT = class extends STT {
|
|
|
802
899
|
};
|
|
803
900
|
}
|
|
804
901
|
openUtterance() {
|
|
805
|
-
|
|
806
|
-
this.
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
902
|
+
const last = this.utterances[this.utterances.length - 1];
|
|
903
|
+
if (last?.state === "open") this.endUtterance(last);
|
|
904
|
+
const utterance = {
|
|
905
|
+
state: "open",
|
|
906
|
+
sent: false,
|
|
907
|
+
chunks: [],
|
|
908
|
+
transcript: ""
|
|
909
|
+
};
|
|
910
|
+
this.utterances.push(utterance);
|
|
911
|
+
this.sendFirst();
|
|
912
|
+
return utterance;
|
|
913
|
+
}
|
|
914
|
+
endUtterance(utterance) {
|
|
915
|
+
utterance.state = "ended";
|
|
916
|
+
if (utterance.sent) this.sendEnd(utterance);
|
|
917
|
+
}
|
|
918
|
+
/** Sends the first utterance to Gemini, with the audio it held meanwhile */
|
|
919
|
+
sendFirst() {
|
|
920
|
+
const utterance = this.utterances[0];
|
|
921
|
+
if (!utterance || utterance.sent) return;
|
|
922
|
+
utterance.sent = true;
|
|
810
923
|
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
924
|
+
utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
925
|
+
if (utterance.state === "ended") this.sendEnd(utterance);
|
|
926
|
+
}
|
|
927
|
+
sendEnd(utterance) {
|
|
928
|
+
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
929
|
+
utterance.timeout = setTimeout(() => {
|
|
930
|
+
utterance.timeout = void 0;
|
|
931
|
+
this.log("Transcription timeout");
|
|
932
|
+
this.give(utterance);
|
|
933
|
+
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
811
934
|
}
|
|
812
935
|
sendAudio(chunk) {
|
|
813
936
|
this.socket.send({
|
|
@@ -819,45 +942,60 @@ var GeminiSTT = class extends STT {
|
|
|
819
942
|
}
|
|
820
943
|
});
|
|
821
944
|
}
|
|
822
|
-
/**
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
945
|
+
/**
|
|
946
|
+
* Emits what was transcribed of an utterance, possibly nothing, and keeps
|
|
947
|
+
* it in line for what Gemini still sends about it
|
|
948
|
+
*/
|
|
949
|
+
give(utterance) {
|
|
950
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
951
|
+
utterance.state = "given";
|
|
952
|
+
const transcript = utterance.transcript.trim();
|
|
830
953
|
this.log(`Transcript: "${transcript}"`);
|
|
831
954
|
this.emit("Transcript", transcript);
|
|
832
|
-
|
|
833
|
-
this.
|
|
834
|
-
this.
|
|
835
|
-
|
|
955
|
+
utterance.timeout = setTimeout(
|
|
956
|
+
() => this.remove(utterance),
|
|
957
|
+
this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
|
|
958
|
+
);
|
|
836
959
|
}
|
|
837
|
-
|
|
838
|
-
if (
|
|
839
|
-
|
|
840
|
-
this.
|
|
960
|
+
remove(utterance) {
|
|
961
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
962
|
+
utterance.timeout = void 0;
|
|
963
|
+
this.utterances = this.utterances.filter((other) => other !== utterance);
|
|
964
|
+
this.renewIfGoingAway();
|
|
965
|
+
this.sendFirst();
|
|
966
|
+
}
|
|
967
|
+
/** Moves to a new connection once no utterance waits on this one */
|
|
968
|
+
renewIfGoingAway() {
|
|
969
|
+
if (!this.goingAway || this.utterances.length > 0) return;
|
|
970
|
+
this.goingAway = false;
|
|
971
|
+
this.socket.renew();
|
|
972
|
+
}
|
|
973
|
+
dropUtterances() {
|
|
974
|
+
for (const utterance of this.utterances) {
|
|
975
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
976
|
+
if (utterance.state === "open") utterance.state = "ended";
|
|
977
|
+
}
|
|
978
|
+
this.utterances = [];
|
|
841
979
|
}
|
|
842
980
|
};
|
|
843
981
|
|
|
844
982
|
// src/GeminiTTS.ts
|
|
845
|
-
import { GoogleGenAI as
|
|
983
|
+
import { GoogleGenAI as GoogleGenAI3 } from "@google/genai";
|
|
846
984
|
import { Pcm16Resampler as Pcm16Resampler2, SentenceTTS } from "@micdrop/server";
|
|
847
|
-
var
|
|
985
|
+
var DEFAULT_MODEL5 = "gemini-2.5-flash-preview-tts";
|
|
848
986
|
var DEFAULT_VOICE = "Kore";
|
|
849
987
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
850
988
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
851
|
-
var
|
|
989
|
+
var DEFAULT_MAX_RETRY = 2;
|
|
852
990
|
var GeminiTTS = class extends SentenceTTS {
|
|
853
991
|
constructor(options) {
|
|
854
992
|
super();
|
|
855
993
|
this.options = options;
|
|
856
|
-
this.genai = "genai" in options ? options.genai : new
|
|
994
|
+
this.genai = "genai" in options ? options.genai : new GoogleGenAI3({ apiKey: options.apiKey });
|
|
857
995
|
}
|
|
858
996
|
async synthesize(text, signal) {
|
|
859
997
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
860
|
-
const maxRetry = this.options.maxRetry ??
|
|
998
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
|
|
861
999
|
for (let attempt = 0; ; attempt++) {
|
|
862
1000
|
const progress = { emitted: false };
|
|
863
1001
|
try {
|
|
@@ -875,7 +1013,7 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
875
1013
|
async synthesizeOnce(input, signal, progress) {
|
|
876
1014
|
const stream = await this.genai.interactions.create(
|
|
877
1015
|
{
|
|
878
|
-
model: this.options.model ||
|
|
1016
|
+
model: this.options.model || DEFAULT_MODEL5,
|
|
879
1017
|
store: false,
|
|
880
1018
|
stream: true,
|
|
881
1019
|
input,
|
|
@@ -911,12 +1049,16 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
911
1049
|
}
|
|
912
1050
|
};
|
|
913
1051
|
function isBlocked(error) {
|
|
914
|
-
return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
1052
|
+
return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
915
1053
|
}
|
|
916
1054
|
export {
|
|
917
1055
|
GeminiAgent,
|
|
1056
|
+
GeminiClassifier,
|
|
918
1057
|
GeminiLive,
|
|
919
1058
|
GeminiSTT,
|
|
920
|
-
GeminiTTS
|
|
1059
|
+
GeminiTTS,
|
|
1060
|
+
choice,
|
|
1061
|
+
noul,
|
|
1062
|
+
score
|
|
921
1063
|
};
|
|
922
1064
|
//# sourceMappingURL=index.mjs.map
|