@micdrop/gemini 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -0
- package/dist/index.d.mts +167 -0
- package/dist/index.d.ts +167 -0
- package/dist/index.js +959 -0
- package/dist/index.js.map +1 -0
- package/dist/index.mjs +922 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +56 -0
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,922 @@
|
|
|
1
|
+
// src/GeminiAgent.ts
|
|
2
|
+
import { GoogleGenAI } from "@google/genai";
|
|
3
|
+
import { Agent } from "@micdrop/server";
|
|
4
|
+
import z, { toJSONSchema } from "zod";
|
|
5
|
+
var DEFAULT_MODEL = "gemini-3.8-flash";
|
|
6
|
+
var DEFAULT_MAX_STEPS = 5;
|
|
7
|
+
var DEFAULT_MAX_RETRIES = 3;
|
|
8
|
+
var DEFAULT_RETRY_DELAY = 1e3;
|
|
9
|
+
var FIRST_MESSAGE_PROMPT = "The call starts now.";
|
|
10
|
+
var GeminiAgent = class extends Agent {
|
|
11
|
+
constructor(options) {
|
|
12
|
+
super(options);
|
|
13
|
+
// Gemini asks for the signature of the thought behind a tool call when the
|
|
14
|
+
// call is sent back, and the conversation has no room for it
|
|
15
|
+
this.signatures = /* @__PURE__ */ new Map();
|
|
16
|
+
this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
|
|
17
|
+
}
|
|
18
|
+
async generateAnswer(stream, stepCount = 0, tryCount = 0) {
|
|
19
|
+
if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
|
|
20
|
+
console.error("[GeminiAgent] Max steps reached");
|
|
21
|
+
return;
|
|
22
|
+
}
|
|
23
|
+
const abortController = new AbortController();
|
|
24
|
+
this.abortController = abortController;
|
|
25
|
+
const signal = abortController.signal;
|
|
26
|
+
let extracting = false;
|
|
27
|
+
const extractOptions = this.getExtractOptions();
|
|
28
|
+
try {
|
|
29
|
+
const response = await this.genai.interactions.create(
|
|
30
|
+
{
|
|
31
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
32
|
+
store: false,
|
|
33
|
+
stream: true,
|
|
34
|
+
system_instruction: this.buildSystemInstruction(),
|
|
35
|
+
input: this.buildInput(),
|
|
36
|
+
tools: this.buildTools(),
|
|
37
|
+
...this.options.thinkingLevel || this.options.settings ? {
|
|
38
|
+
generation_config: {
|
|
39
|
+
...this.options.settings,
|
|
40
|
+
...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
|
|
41
|
+
}
|
|
42
|
+
} : {}
|
|
43
|
+
},
|
|
44
|
+
{ fetchOptions: { signal } }
|
|
45
|
+
);
|
|
46
|
+
const steps = [];
|
|
47
|
+
let answer = "";
|
|
48
|
+
let skipAnswer = false;
|
|
49
|
+
let calledTool = false;
|
|
50
|
+
let signature = "";
|
|
51
|
+
for await (const event of response) {
|
|
52
|
+
if (abortController !== this.abortController) return;
|
|
53
|
+
switch (event.event_type) {
|
|
54
|
+
case "step.start":
|
|
55
|
+
steps[event.index] = {
|
|
56
|
+
type: event.step.type,
|
|
57
|
+
id: event.step.id,
|
|
58
|
+
name: event.step.name,
|
|
59
|
+
text: "",
|
|
60
|
+
arguments: "",
|
|
61
|
+
signature: ""
|
|
62
|
+
};
|
|
63
|
+
break;
|
|
64
|
+
case "step.delta": {
|
|
65
|
+
const step = steps[event.index];
|
|
66
|
+
if (!step) break;
|
|
67
|
+
const delta = event.delta;
|
|
68
|
+
if (delta.type === "thought_signature") {
|
|
69
|
+
step.signature += delta.signature ?? "";
|
|
70
|
+
} else if (delta.type === "arguments_delta") {
|
|
71
|
+
step.arguments += delta.arguments ?? "";
|
|
72
|
+
} else if (delta.type === "text" && step.type === "model_output") {
|
|
73
|
+
this.log(`Answer chunk: "${delta.text}"`);
|
|
74
|
+
answer += delta.text;
|
|
75
|
+
if (extractOptions) {
|
|
76
|
+
if (extracting) break;
|
|
77
|
+
const startTagIndex = delta.text.indexOf(
|
|
78
|
+
extractOptions.startTag
|
|
79
|
+
);
|
|
80
|
+
if (startTagIndex !== -1) {
|
|
81
|
+
extracting = true;
|
|
82
|
+
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
stream.write(delta.text);
|
|
87
|
+
}
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
case "step.stop": {
|
|
91
|
+
const step = steps[event.index];
|
|
92
|
+
if (step?.type === "thought") signature = step.signature;
|
|
93
|
+
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
94
|
+
calledTool = true;
|
|
95
|
+
if (signature) this.signatures.set(step.id, signature);
|
|
96
|
+
const result = await this.executeTool({
|
|
97
|
+
role: "tool_call",
|
|
98
|
+
toolCallId: step.id,
|
|
99
|
+
toolName: step.name,
|
|
100
|
+
parameters: step.arguments || "{}"
|
|
101
|
+
});
|
|
102
|
+
if (result.skipAnswer) skipAnswer = true;
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
case "error":
|
|
106
|
+
throw new Error(event.error?.message || "Generation failed");
|
|
107
|
+
default:
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
if (abortController !== this.abortController) return;
|
|
112
|
+
this.abortController = void 0;
|
|
113
|
+
if (answer) {
|
|
114
|
+
const { message, metadata } = this.extract(answer);
|
|
115
|
+
this.addAssistantMessage(message, metadata);
|
|
116
|
+
}
|
|
117
|
+
if (calledTool && !skipAnswer && !answer) {
|
|
118
|
+
await this.generateAnswer(stream, stepCount + 1);
|
|
119
|
+
}
|
|
120
|
+
} catch (error) {
|
|
121
|
+
if (signal.aborted) return;
|
|
122
|
+
console.error("[GeminiAgent] Error answering:", error);
|
|
123
|
+
if (tryCount < (this.options.maxRetry ?? DEFAULT_MAX_RETRIES)) {
|
|
124
|
+
await new Promise(
|
|
125
|
+
(resolve) => setTimeout(resolve, this.options.retryDelay ?? DEFAULT_RETRY_DELAY)
|
|
126
|
+
);
|
|
127
|
+
await this.generateAnswer(stream, stepCount, tryCount + 1);
|
|
128
|
+
} else {
|
|
129
|
+
this.log("Max retries reached, giving up");
|
|
130
|
+
this.emit("Failed");
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
cancel() {
|
|
135
|
+
if (!this.abortController) return;
|
|
136
|
+
this.log("Cancel");
|
|
137
|
+
this.abortController.abort();
|
|
138
|
+
this.abortController = void 0;
|
|
139
|
+
}
|
|
140
|
+
buildSystemInstruction() {
|
|
141
|
+
return this.conversation.flatMap(
|
|
142
|
+
(message) => message.role === "system" ? [message.content] : []
|
|
143
|
+
).join("\n\n");
|
|
144
|
+
}
|
|
145
|
+
buildInput() {
|
|
146
|
+
const input = this.conversation.flatMap((message) => {
|
|
147
|
+
switch (message.role) {
|
|
148
|
+
case "user":
|
|
149
|
+
return [
|
|
150
|
+
{
|
|
151
|
+
type: "user_input",
|
|
152
|
+
content: [{ type: "text", text: message.content }]
|
|
153
|
+
}
|
|
154
|
+
];
|
|
155
|
+
case "assistant":
|
|
156
|
+
return [
|
|
157
|
+
{
|
|
158
|
+
type: "model_output",
|
|
159
|
+
content: [{ type: "text", text: message.content }]
|
|
160
|
+
}
|
|
161
|
+
];
|
|
162
|
+
case "tool_call": {
|
|
163
|
+
const signature = this.signatures.get(message.toolCallId);
|
|
164
|
+
return [
|
|
165
|
+
...signature ? [{ type: "thought", signature }] : [],
|
|
166
|
+
{
|
|
167
|
+
type: "function_call",
|
|
168
|
+
id: message.toolCallId,
|
|
169
|
+
name: message.toolName,
|
|
170
|
+
arguments: parseArguments(message.parameters)
|
|
171
|
+
}
|
|
172
|
+
];
|
|
173
|
+
}
|
|
174
|
+
case "tool_result":
|
|
175
|
+
return [
|
|
176
|
+
{
|
|
177
|
+
type: "function_result",
|
|
178
|
+
call_id: message.toolCallId,
|
|
179
|
+
name: message.toolName,
|
|
180
|
+
result: [{ type: "text", text: message.output }]
|
|
181
|
+
}
|
|
182
|
+
];
|
|
183
|
+
default:
|
|
184
|
+
return [];
|
|
185
|
+
}
|
|
186
|
+
});
|
|
187
|
+
if (input.length === 0) {
|
|
188
|
+
input.push({
|
|
189
|
+
type: "user_input",
|
|
190
|
+
content: [{ type: "text", text: FIRST_MESSAGE_PROMPT }]
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
return input;
|
|
194
|
+
}
|
|
195
|
+
buildTools() {
|
|
196
|
+
if (this.tools.length === 0) return void 0;
|
|
197
|
+
return this.tools.map((tool) => {
|
|
198
|
+
const { $schema, ...parameters } = toJSONSchema(
|
|
199
|
+
tool.inputSchema || z.object()
|
|
200
|
+
);
|
|
201
|
+
return {
|
|
202
|
+
type: "function",
|
|
203
|
+
name: tool.name,
|
|
204
|
+
description: tool.description,
|
|
205
|
+
parameters
|
|
206
|
+
};
|
|
207
|
+
});
|
|
208
|
+
}
|
|
209
|
+
};
|
|
210
|
+
function parseArguments(parameters) {
|
|
211
|
+
try {
|
|
212
|
+
return JSON.parse(parameters);
|
|
213
|
+
} catch {
|
|
214
|
+
return {};
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// src/GeminiLive.ts
|
|
219
|
+
import {
|
|
220
|
+
Pcm16Resampler,
|
|
221
|
+
Realtime
|
|
222
|
+
} from "@micdrop/server";
|
|
223
|
+
import { toJSONSchema as toJSONSchema2 } from "zod";
|
|
224
|
+
|
|
225
|
+
// src/LiveSocket.ts
|
|
226
|
+
import { EventEmitter } from "eventemitter3";
|
|
227
|
+
import WebSocket from "ws";
|
|
228
|
+
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
229
|
+
var SAMPLE_RATE = 16e3;
|
|
230
|
+
var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
|
|
231
|
+
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
232
|
+
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
233
|
+
var DEFAULT_MAX_RETRY = 3;
|
|
234
|
+
var LiveSocket = class extends EventEmitter {
|
|
235
|
+
constructor(options, buildSetup, log) {
|
|
236
|
+
super();
|
|
237
|
+
this.options = options;
|
|
238
|
+
this.buildSetup = buildSetup;
|
|
239
|
+
this.log = log;
|
|
240
|
+
this.connected = false;
|
|
241
|
+
this.destroyed = false;
|
|
242
|
+
// Messages waiting for the session, or for the end of a turn to be due
|
|
243
|
+
this.pending = [];
|
|
244
|
+
// Audio of the turn as it went out, see flush()
|
|
245
|
+
this.turnAudioStartedAt = 0;
|
|
246
|
+
this.turnAudioMs = 0;
|
|
247
|
+
this.retryCount = 0;
|
|
248
|
+
}
|
|
249
|
+
/** The session is set up and takes messages right away */
|
|
250
|
+
get isConnected() {
|
|
251
|
+
return this.connected;
|
|
252
|
+
}
|
|
253
|
+
connect() {
|
|
254
|
+
this.initWS().catch((error) => {
|
|
255
|
+
this.log("Connection error:", error);
|
|
256
|
+
this.reconnect();
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
/** Sends a message once the session is ready, after the ones before it */
|
|
260
|
+
send(message) {
|
|
261
|
+
this.pending.push(message);
|
|
262
|
+
this.flush();
|
|
263
|
+
}
|
|
264
|
+
/** Sends a message ahead of the waiting ones, from a Ready listener */
|
|
265
|
+
sendFirst(message) {
|
|
266
|
+
this.socket?.send(JSON.stringify(message));
|
|
267
|
+
}
|
|
268
|
+
/** Drops the messages that have not gone out */
|
|
269
|
+
clearPending() {
|
|
270
|
+
this.pending = [];
|
|
271
|
+
this.clearPendingTimeout();
|
|
272
|
+
}
|
|
273
|
+
/** Moves to a new connection, keeping the messages that have not gone out */
|
|
274
|
+
renew() {
|
|
275
|
+
this.log("Renewing the connection");
|
|
276
|
+
this.closeSocket();
|
|
277
|
+
this.connect();
|
|
278
|
+
}
|
|
279
|
+
destroy() {
|
|
280
|
+
this.destroyed = true;
|
|
281
|
+
if (this.reconnectTimeout) {
|
|
282
|
+
clearTimeout(this.reconnectTimeout);
|
|
283
|
+
this.reconnectTimeout = void 0;
|
|
284
|
+
}
|
|
285
|
+
this.closeSocket();
|
|
286
|
+
this.pending = [];
|
|
287
|
+
this.removeAllListeners();
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Sends the messages in order, as soon as they can go.
|
|
291
|
+
*
|
|
292
|
+
* Gemini refuses the end of a turn that comes before the audio of the turn
|
|
293
|
+
* had time to play, and closes the session. Audio sent faster than it is
|
|
294
|
+
* spoken, when the user talked while the connection opened, would end the
|
|
295
|
+
* session, so the end of such a turn waits for it.
|
|
296
|
+
*/
|
|
297
|
+
flush() {
|
|
298
|
+
while (this.pending.length > 0 && !this.pendingTimeout) {
|
|
299
|
+
const socket = this.socket;
|
|
300
|
+
if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
|
|
301
|
+
const message = this.pending[0];
|
|
302
|
+
const input = message.realtimeInput;
|
|
303
|
+
if (input?.activityEnd) {
|
|
304
|
+
const due = this.turnAudioStartedAt + this.turnAudioMs - Date.now();
|
|
305
|
+
if (due > 0) {
|
|
306
|
+
this.pendingTimeout = setTimeout(() => {
|
|
307
|
+
this.pendingTimeout = void 0;
|
|
308
|
+
this.flush();
|
|
309
|
+
}, due);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
} else if (input?.activityStart) {
|
|
313
|
+
this.turnAudioStartedAt = 0;
|
|
314
|
+
this.turnAudioMs = 0;
|
|
315
|
+
} else if (input?.audio) {
|
|
316
|
+
if (this.turnAudioMs === 0) this.turnAudioStartedAt = Date.now();
|
|
317
|
+
this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
|
|
318
|
+
}
|
|
319
|
+
this.pending.shift();
|
|
320
|
+
socket.send(JSON.stringify(message));
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
clearPendingTimeout() {
|
|
324
|
+
if (!this.pendingTimeout) return;
|
|
325
|
+
clearTimeout(this.pendingTimeout);
|
|
326
|
+
this.pendingTimeout = void 0;
|
|
327
|
+
}
|
|
328
|
+
initWS() {
|
|
329
|
+
return new Promise((resolve, reject) => {
|
|
330
|
+
const socket = new WebSocket(
|
|
331
|
+
`${URL}?key=${encodeURIComponent(this.options.apiKey)}`
|
|
332
|
+
);
|
|
333
|
+
this.socket = socket;
|
|
334
|
+
const timeout = setTimeout(() => {
|
|
335
|
+
this.log("Connection timeout");
|
|
336
|
+
this.closeSocket();
|
|
337
|
+
reject(new Error("WebSocket connection timeout"));
|
|
338
|
+
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
339
|
+
socket.on("open", () => {
|
|
340
|
+
this.log("Connection opened");
|
|
341
|
+
socket.send(JSON.stringify(this.buildSetup()));
|
|
342
|
+
});
|
|
343
|
+
socket.on("error", (error) => {
|
|
344
|
+
clearTimeout(timeout);
|
|
345
|
+
this.log("WebSocket error:", error);
|
|
346
|
+
reject(new Error("WebSocket connection error"));
|
|
347
|
+
});
|
|
348
|
+
socket.on("close", (code, reason) => {
|
|
349
|
+
clearTimeout(timeout);
|
|
350
|
+
const wasConnected = this.connected;
|
|
351
|
+
this.closeSocket();
|
|
352
|
+
if (this.destroyed) return;
|
|
353
|
+
this.log("Connection closed", { code, reason: reason.toString() });
|
|
354
|
+
if (wasConnected) {
|
|
355
|
+
this.reconnect();
|
|
356
|
+
} else {
|
|
357
|
+
reject(new Error(`Connection closed: ${reason.toString()}`));
|
|
358
|
+
}
|
|
359
|
+
});
|
|
360
|
+
socket.on("message", (data) => {
|
|
361
|
+
const message = JSON.parse(data.toString());
|
|
362
|
+
if (message.setupComplete) {
|
|
363
|
+
clearTimeout(timeout);
|
|
364
|
+
this.log("Session ready");
|
|
365
|
+
this.connected = true;
|
|
366
|
+
this.retryCount = 0;
|
|
367
|
+
this.emit("Ready");
|
|
368
|
+
this.flush();
|
|
369
|
+
resolve();
|
|
370
|
+
return;
|
|
371
|
+
}
|
|
372
|
+
this.emit("Message", message);
|
|
373
|
+
});
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
closeSocket() {
|
|
377
|
+
const socket = this.socket;
|
|
378
|
+
this.socket = void 0;
|
|
379
|
+
this.connected = false;
|
|
380
|
+
this.clearPendingTimeout();
|
|
381
|
+
if (!socket) return;
|
|
382
|
+
socket.removeAllListeners();
|
|
383
|
+
socket.on("error", () => {
|
|
384
|
+
});
|
|
385
|
+
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
386
|
+
socket.close(1e3);
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
reconnect() {
|
|
390
|
+
if (this.destroyed) return;
|
|
391
|
+
this.retryCount++;
|
|
392
|
+
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
393
|
+
this.log("Max retries reached, giving up");
|
|
394
|
+
this.emit("Failed");
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
this.emit("Reconnecting", this.retryCount);
|
|
398
|
+
this.log("Reconnecting...");
|
|
399
|
+
this.reconnectTimeout = setTimeout(() => {
|
|
400
|
+
this.reconnectTimeout = void 0;
|
|
401
|
+
this.connect();
|
|
402
|
+
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
403
|
+
}
|
|
404
|
+
};
|
|
405
|
+
|
|
406
|
+
// src/GeminiLive.ts
|
|
407
|
+
var DEFAULT_MODEL2 = "gemini-3.8-live";
|
|
408
|
+
var SAMPLE_RATE2 = 16e3;
|
|
409
|
+
var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
|
|
410
|
+
var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
|
|
411
|
+
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
412
|
+
var GeminiLive = class extends Realtime {
|
|
413
|
+
constructor(options) {
|
|
414
|
+
super(options);
|
|
415
|
+
this.everConnected = false;
|
|
416
|
+
// Whether the setup being sent resumes the session
|
|
417
|
+
this.resuming = false;
|
|
418
|
+
this.goingAway = false;
|
|
419
|
+
this.outputResampler = new Pcm16Resampler(
|
|
420
|
+
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
421
|
+
SAMPLE_RATE2
|
|
422
|
+
);
|
|
423
|
+
this.hasTranscribedTurn = false;
|
|
424
|
+
// Transcript of a dropped turn, which the conversation should not show
|
|
425
|
+
this.ignoreUserTranscript = false;
|
|
426
|
+
this.cancelledToolCalls = /* @__PURE__ */ new Set();
|
|
427
|
+
// Gemini ends its turn when it calls a tool, and speaks again once it has
|
|
428
|
+
// the result. The answer stays open in between, so the client keeps waiting
|
|
429
|
+
// for it rather than listening again.
|
|
430
|
+
this.runningTools = 0;
|
|
431
|
+
this.answerEndDeferred = false;
|
|
432
|
+
this.onReady = () => {
|
|
433
|
+
if (this.everConnected && !this.resuming) {
|
|
434
|
+
const turns = this.conversation.flatMap(
|
|
435
|
+
(message) => message.role === "user" || message.role === "assistant" ? [
|
|
436
|
+
{
|
|
437
|
+
role: message.role === "assistant" ? "model" : "user",
|
|
438
|
+
parts: [{ text: message.content }]
|
|
439
|
+
}
|
|
440
|
+
] : []
|
|
441
|
+
);
|
|
442
|
+
if (turns.length > 0) {
|
|
443
|
+
this.socket.sendFirst({ clientContent: { turns, turnComplete: false } });
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
this.everConnected = true;
|
|
447
|
+
this.goingAway = false;
|
|
448
|
+
};
|
|
449
|
+
this.onReconnecting = (attempt) => {
|
|
450
|
+
if (attempt > 1) this.resumptionHandle = void 0;
|
|
451
|
+
if (!this.resumptionHandle) this.socket.clearPending();
|
|
452
|
+
this.endAnswer();
|
|
453
|
+
};
|
|
454
|
+
this.onMessage = (message) => {
|
|
455
|
+
if (message.serverContent) {
|
|
456
|
+
this.onServerContent(message.serverContent);
|
|
457
|
+
}
|
|
458
|
+
if (message.toolCall) {
|
|
459
|
+
for (const call of message.toolCall.functionCalls ?? []) {
|
|
460
|
+
this.onToolCall(call);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
if (message.toolCallCancellation) {
|
|
464
|
+
for (const id of message.toolCallCancellation.ids ?? []) {
|
|
465
|
+
this.cancelledToolCalls.add(id);
|
|
466
|
+
}
|
|
467
|
+
}
|
|
468
|
+
if (message.sessionResumptionUpdate) {
|
|
469
|
+
const { newHandle, resumable } = message.sessionResumptionUpdate;
|
|
470
|
+
if (resumable && newHandle) this.resumptionHandle = newHandle;
|
|
471
|
+
}
|
|
472
|
+
if (message.goAway) {
|
|
473
|
+
this.log("Connection ending in", message.goAway.timeLeft);
|
|
474
|
+
this.goingAway = true;
|
|
475
|
+
this.renewIfGoingAway();
|
|
476
|
+
}
|
|
477
|
+
};
|
|
478
|
+
this.socket = new LiveSocket(
|
|
479
|
+
options,
|
|
480
|
+
() => this.buildSetup(),
|
|
481
|
+
(...message) => this.log(...message)
|
|
482
|
+
);
|
|
483
|
+
this.socket.on("Ready", this.onReady);
|
|
484
|
+
this.socket.on("Message", this.onMessage);
|
|
485
|
+
this.socket.on("Reconnecting", this.onReconnecting);
|
|
486
|
+
this.socket.on("Failed", () => this.emit("Failed"));
|
|
487
|
+
this.socket.connect();
|
|
488
|
+
}
|
|
489
|
+
destroy() {
|
|
490
|
+
super.destroy();
|
|
491
|
+
this.clearToolFollowUp();
|
|
492
|
+
this.socket.destroy();
|
|
493
|
+
}
|
|
494
|
+
openTurn() {
|
|
495
|
+
this.ignoreUserTranscript = false;
|
|
496
|
+
this.send({ realtimeInput: { activityStart: {} } });
|
|
497
|
+
}
|
|
498
|
+
appendAudio(chunk) {
|
|
499
|
+
this.send({
|
|
500
|
+
realtimeInput: {
|
|
501
|
+
audio: {
|
|
502
|
+
mimeType: `audio/pcm;rate=${SAMPLE_RATE2}`,
|
|
503
|
+
data: chunk.toString("base64")
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
closeTurn() {
|
|
509
|
+
this.hasTranscribedTurn = false;
|
|
510
|
+
this.send({ realtimeInput: { activityEnd: {} } });
|
|
511
|
+
}
|
|
512
|
+
clearTurn() {
|
|
513
|
+
this.ignoreUserTranscript = true;
|
|
514
|
+
this.send({ realtimeInput: { activityEnd: {} } });
|
|
515
|
+
}
|
|
516
|
+
generate() {
|
|
517
|
+
this.sendUserText(FIRST_MESSAGE_PROMPT2);
|
|
518
|
+
}
|
|
519
|
+
speakText(text) {
|
|
520
|
+
this.sendUserText(
|
|
521
|
+
`Say exactly the following text, word for word, and nothing else: ${text}`
|
|
522
|
+
);
|
|
523
|
+
}
|
|
524
|
+
cancelAnswer() {
|
|
525
|
+
this.outputResampler.reset();
|
|
526
|
+
this.clearToolFollowUp();
|
|
527
|
+
this.answerEndDeferred = false;
|
|
528
|
+
}
|
|
529
|
+
get isIdle() {
|
|
530
|
+
return super.isIdle && this.runningTools === 0;
|
|
531
|
+
}
|
|
532
|
+
updateTools() {
|
|
533
|
+
if (this.everConnected) {
|
|
534
|
+
this.log("Tools changed after the session started, Gemini keeps the old");
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
injectMessage(role, text) {
|
|
538
|
+
if (!this.socket.isConnected && this.everConnected && !this.resumptionHandle)
|
|
539
|
+
return;
|
|
540
|
+
this.send(this.contentMessage(role, text, false));
|
|
541
|
+
}
|
|
542
|
+
sendUserText(text) {
|
|
543
|
+
this.send(this.contentMessage("user", text, true));
|
|
544
|
+
}
|
|
545
|
+
contentMessage(role, text, turnComplete) {
|
|
546
|
+
return {
|
|
547
|
+
clientContent: {
|
|
548
|
+
turns: [
|
|
549
|
+
{
|
|
550
|
+
role: role === "assistant" ? "model" : "user",
|
|
551
|
+
parts: [{ text }]
|
|
552
|
+
}
|
|
553
|
+
],
|
|
554
|
+
turnComplete
|
|
555
|
+
}
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
send(message) {
|
|
559
|
+
this.socket.send(message);
|
|
560
|
+
}
|
|
561
|
+
buildSetup() {
|
|
562
|
+
this.resuming = !!this.resumptionHandle;
|
|
563
|
+
const functionDeclarations = this.tools.map((tool) => ({
|
|
564
|
+
name: tool.name,
|
|
565
|
+
description: tool.description,
|
|
566
|
+
...tool.inputSchema ? { parametersJsonSchema: toJSONSchema2(tool.inputSchema) } : {}
|
|
567
|
+
}));
|
|
568
|
+
return {
|
|
569
|
+
setup: {
|
|
570
|
+
model: `models/${this.options.model || DEFAULT_MODEL2}`,
|
|
571
|
+
generationConfig: {
|
|
572
|
+
responseModalities: ["AUDIO"],
|
|
573
|
+
...this.options.thinkingLevel ? { thinkingConfig: { thinkingLevel: this.options.thinkingLevel } } : {},
|
|
574
|
+
...this.options.voice ? {
|
|
575
|
+
speechConfig: {
|
|
576
|
+
voiceConfig: {
|
|
577
|
+
prebuiltVoiceConfig: { voiceName: this.options.voice }
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
} : {}
|
|
581
|
+
},
|
|
582
|
+
systemInstruction: { parts: [{ text: this.options.systemPrompt }] },
|
|
583
|
+
...functionDeclarations.length > 0 ? { tools: [{ functionDeclarations }] } : {},
|
|
584
|
+
// The Micdrop client detects when the user speaks
|
|
585
|
+
realtimeInputConfig: {
|
|
586
|
+
automaticActivityDetection: { disabled: true }
|
|
587
|
+
},
|
|
588
|
+
inputAudioTranscription: {},
|
|
589
|
+
outputAudioTranscription: {},
|
|
590
|
+
sessionResumption: this.resumptionHandle ? { handle: this.resumptionHandle } : {},
|
|
591
|
+
// Lifts the length limit of a session
|
|
592
|
+
contextWindowCompression: { slidingWindow: {} }
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
}
|
|
596
|
+
/** Moves to a new connection before Gemini closes this one */
|
|
597
|
+
renewIfGoingAway() {
|
|
598
|
+
if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
|
|
599
|
+
this.goingAway = false;
|
|
600
|
+
this.socket.renew();
|
|
601
|
+
}
|
|
602
|
+
onServerContent(content) {
|
|
603
|
+
if (content.inputTranscription?.text && !this.ignoreUserTranscript) {
|
|
604
|
+
this.addUserTranscript(content.inputTranscription.text);
|
|
605
|
+
}
|
|
606
|
+
for (const part of content.modelTurn?.parts ?? []) {
|
|
607
|
+
const data = part.inlineData?.data;
|
|
608
|
+
if (!data) continue;
|
|
609
|
+
this.onModelOutput();
|
|
610
|
+
const output = this.outputResampler.process(Buffer.from(data, "base64"));
|
|
611
|
+
if (output.length > 0) this.emitAudio(output);
|
|
612
|
+
}
|
|
613
|
+
if (content.outputTranscription?.text) {
|
|
614
|
+
this.onModelOutput();
|
|
615
|
+
this.addAnswerTranscript(content.outputTranscription.text);
|
|
616
|
+
}
|
|
617
|
+
if (content.interrupted) {
|
|
618
|
+
this.log("Interrupted");
|
|
619
|
+
this.outputResampler.reset();
|
|
620
|
+
}
|
|
621
|
+
if (content.generationComplete) {
|
|
622
|
+
this.commitTurnTranscript();
|
|
623
|
+
}
|
|
624
|
+
if (content.turnComplete) {
|
|
625
|
+
this.commitTurnTranscript();
|
|
626
|
+
this.outputResampler.reset();
|
|
627
|
+
if (this.runningTools > 0 || this.toolFollowUp) {
|
|
628
|
+
this.answerEndDeferred = true;
|
|
629
|
+
} else {
|
|
630
|
+
this.endAnswer();
|
|
631
|
+
this.renewIfGoingAway();
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
onModelOutput() {
|
|
636
|
+
this.commitTurnTranscript();
|
|
637
|
+
if (this.toolFollowUp) {
|
|
638
|
+
this.clearToolFollowUp();
|
|
639
|
+
this.answerEndDeferred = false;
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
/** Ends the answer the model stopped giving to wait for a tool */
|
|
643
|
+
endDeferredAnswer() {
|
|
644
|
+
if (!this.answerEndDeferred) return;
|
|
645
|
+
if (this.runningTools > 0 || this.toolFollowUp) return;
|
|
646
|
+
this.answerEndDeferred = false;
|
|
647
|
+
this.endAnswer();
|
|
648
|
+
this.renewIfGoingAway();
|
|
649
|
+
}
|
|
650
|
+
clearToolFollowUp() {
|
|
651
|
+
if (!this.toolFollowUp) return;
|
|
652
|
+
clearTimeout(this.toolFollowUp);
|
|
653
|
+
this.toolFollowUp = void 0;
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* The transcript of the turn comes before the answer, so the first piece of
|
|
657
|
+
* the answer closes it.
|
|
658
|
+
*/
|
|
659
|
+
commitTurnTranscript() {
|
|
660
|
+
if (this.hasTranscribedTurn) return;
|
|
661
|
+
this.hasTranscribedTurn = true;
|
|
662
|
+
this.commitUserTranscript();
|
|
663
|
+
}
|
|
664
|
+
async onToolCall(call) {
|
|
665
|
+
this.commitTurnTranscript();
|
|
666
|
+
const toolCall = {
|
|
667
|
+
role: "tool_call",
|
|
668
|
+
toolCallId: call.id,
|
|
669
|
+
toolName: call.name,
|
|
670
|
+
parameters: JSON.stringify(call.args ?? {})
|
|
671
|
+
};
|
|
672
|
+
this.runningTools++;
|
|
673
|
+
let result;
|
|
674
|
+
try {
|
|
675
|
+
result = await this.runTool(toolCall);
|
|
676
|
+
} finally {
|
|
677
|
+
this.runningTools--;
|
|
678
|
+
}
|
|
679
|
+
if (this.cancelledToolCalls.delete(call.id)) {
|
|
680
|
+
this.endDeferredAnswer();
|
|
681
|
+
return;
|
|
682
|
+
}
|
|
683
|
+
this.send({
|
|
684
|
+
toolResponse: {
|
|
685
|
+
functionResponses: [
|
|
686
|
+
{
|
|
687
|
+
id: call.id,
|
|
688
|
+
name: call.name,
|
|
689
|
+
response: {
|
|
690
|
+
output: result.output ?? null,
|
|
691
|
+
// A tool without an answer lets the model know silently
|
|
692
|
+
scheduling: result.skipAnswer ? "SILENT" : "WHEN_IDLE"
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
]
|
|
696
|
+
}
|
|
697
|
+
});
|
|
698
|
+
if (!result.skipAnswer) {
|
|
699
|
+
this.clearToolFollowUp();
|
|
700
|
+
this.toolFollowUp = setTimeout(() => {
|
|
701
|
+
this.toolFollowUp = void 0;
|
|
702
|
+
this.log("Nothing said about the tool result");
|
|
703
|
+
this.endDeferredAnswer();
|
|
704
|
+
}, TOOL_FOLLOW_UP_TIMEOUT);
|
|
705
|
+
} else {
|
|
706
|
+
this.endDeferredAnswer();
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
// src/GeminiSTT.ts
|
|
712
|
+
import { STT } from "@micdrop/server";
|
|
713
|
+
var DEFAULT_MODEL3 = "gemini-3.5-transcribe-live";
|
|
714
|
+
var SAMPLE_RATE3 = 16e3;
|
|
715
|
+
var DEFAULT_TRANSCRIPTION_TIMEOUT = 4e3;
|
|
716
|
+
var GeminiSTT = class extends STT {
|
|
717
|
+
constructor(options) {
|
|
718
|
+
super();
|
|
719
|
+
this.options = options;
|
|
720
|
+
this.goingAway = false;
|
|
721
|
+
// Audio of the utterance, sent again if the connection drops before its
|
|
722
|
+
// transcript comes back
|
|
723
|
+
this.chunks = [];
|
|
724
|
+
this.utteranceOpen = false;
|
|
725
|
+
this.utteranceEnded = false;
|
|
726
|
+
this.transcript = "";
|
|
727
|
+
this.onMessage = (message) => {
|
|
728
|
+
const content = message.serverContent;
|
|
729
|
+
if (content?.inputTranscription?.text && this.utteranceOpen) {
|
|
730
|
+
this.transcript += content.inputTranscription.text;
|
|
731
|
+
}
|
|
732
|
+
if (content?.generationComplete && this.utteranceEnded) {
|
|
733
|
+
this.finishUtterance();
|
|
734
|
+
}
|
|
735
|
+
if (message.goAway) {
|
|
736
|
+
this.log("Connection ending in", message.goAway.timeLeft);
|
|
737
|
+
if (this.utteranceOpen) {
|
|
738
|
+
this.goingAway = true;
|
|
739
|
+
} else {
|
|
740
|
+
this.socket.renew();
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
};
|
|
744
|
+
this.onReconnecting = () => {
|
|
745
|
+
this.socket.clearPending();
|
|
746
|
+
if (!this.utteranceOpen) return;
|
|
747
|
+
this.transcript = "";
|
|
748
|
+
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
749
|
+
this.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
750
|
+
if (this.utteranceEnded) {
|
|
751
|
+
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
752
|
+
}
|
|
753
|
+
};
|
|
754
|
+
this.socket = new LiveSocket(
|
|
755
|
+
options,
|
|
756
|
+
() => this.buildSetup(),
|
|
757
|
+
(...message) => this.log(...message)
|
|
758
|
+
);
|
|
759
|
+
this.socket.on("Message", this.onMessage);
|
|
760
|
+
this.socket.on("Reconnecting", this.onReconnecting);
|
|
761
|
+
this.socket.on("Failed", () => this.emit("Failed", this.chunks));
|
|
762
|
+
this.socket.connect();
|
|
763
|
+
}
|
|
764
|
+
transcribe(audioStream) {
|
|
765
|
+
audioStream.on("data", (chunk) => {
|
|
766
|
+
if (!this.utteranceOpen) this.openUtterance();
|
|
767
|
+
this.chunks.push(chunk);
|
|
768
|
+
this.sendAudio(chunk);
|
|
769
|
+
});
|
|
770
|
+
audioStream.on("end", () => {
|
|
771
|
+
if (!this.utteranceOpen || this.utteranceEnded) return;
|
|
772
|
+
this.utteranceEnded = true;
|
|
773
|
+
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
774
|
+
this.clearTranscriptionTimeout();
|
|
775
|
+
this.transcriptionTimeout = setTimeout(() => {
|
|
776
|
+
this.transcriptionTimeout = void 0;
|
|
777
|
+
this.log("Transcription timeout");
|
|
778
|
+
this.finishUtterance();
|
|
779
|
+
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
780
|
+
});
|
|
781
|
+
}
|
|
782
|
+
destroy() {
|
|
783
|
+
super.destroy();
|
|
784
|
+
this.clearTranscriptionTimeout();
|
|
785
|
+
this.socket.destroy();
|
|
786
|
+
}
|
|
787
|
+
buildSetup() {
|
|
788
|
+
return {
|
|
789
|
+
setup: {
|
|
790
|
+
model: `models/${this.options.model || DEFAULT_MODEL3}`,
|
|
791
|
+
generationConfig: { responseModalities: ["TEXT"] },
|
|
792
|
+
inputAudioTranscription: {
|
|
793
|
+
languageCodes: this.options.language ? [this.options.language] : [],
|
|
794
|
+
...this.options.vocabulary?.length ? { customVocabulary: this.options.vocabulary } : {},
|
|
795
|
+
...this.options.mode ? { mode: this.options.mode } : {}
|
|
796
|
+
},
|
|
797
|
+
// The Micdrop client detects when the user speaks
|
|
798
|
+
realtimeInputConfig: {
|
|
799
|
+
automaticActivityDetection: { disabled: true }
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
};
|
|
803
|
+
}
|
|
804
|
+
openUtterance() {
|
|
805
|
+
if (this.utteranceOpen) this.finishUtterance();
|
|
806
|
+
this.utteranceOpen = true;
|
|
807
|
+
this.utteranceEnded = false;
|
|
808
|
+
this.chunks = [];
|
|
809
|
+
this.transcript = "";
|
|
810
|
+
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
811
|
+
}
|
|
812
|
+
sendAudio(chunk) {
|
|
813
|
+
this.socket.send({
|
|
814
|
+
realtimeInput: {
|
|
815
|
+
audio: {
|
|
816
|
+
mimeType: `audio/pcm;rate=${SAMPLE_RATE3}`,
|
|
817
|
+
data: chunk.toString("base64")
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
});
|
|
821
|
+
}
|
|
822
|
+
/** Emits what was transcribed of the utterance, possibly nothing */
|
|
823
|
+
finishUtterance() {
|
|
824
|
+
this.clearTranscriptionTimeout();
|
|
825
|
+
const transcript = this.transcript.trim();
|
|
826
|
+
this.utteranceOpen = false;
|
|
827
|
+
this.utteranceEnded = false;
|
|
828
|
+
this.chunks = [];
|
|
829
|
+
this.transcript = "";
|
|
830
|
+
this.log(`Transcript: "${transcript}"`);
|
|
831
|
+
this.emit("Transcript", transcript);
|
|
832
|
+
if (this.goingAway) {
|
|
833
|
+
this.goingAway = false;
|
|
834
|
+
this.socket.renew();
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
clearTranscriptionTimeout() {
|
|
838
|
+
if (!this.transcriptionTimeout) return;
|
|
839
|
+
clearTimeout(this.transcriptionTimeout);
|
|
840
|
+
this.transcriptionTimeout = void 0;
|
|
841
|
+
}
|
|
842
|
+
};
|
|
843
|
+
|
|
844
|
+
// src/GeminiTTS.ts
|
|
845
|
+
import { GoogleGenAI as GoogleGenAI2 } from "@google/genai";
|
|
846
|
+
import { Pcm16Resampler as Pcm16Resampler2, SentenceTTS } from "@micdrop/server";
|
|
847
|
+
var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
|
|
848
|
+
var DEFAULT_VOICE = "Kore";
|
|
849
|
+
var GEMINI_SAMPLE_RATE = 24e3;
|
|
850
|
+
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
851
|
+
var DEFAULT_MAX_RETRY2 = 2;
|
|
852
|
+
var GeminiTTS = class extends SentenceTTS {
|
|
853
|
+
constructor(options) {
|
|
854
|
+
super();
|
|
855
|
+
this.options = options;
|
|
856
|
+
this.genai = "genai" in options ? options.genai : new GoogleGenAI2({ apiKey: options.apiKey });
|
|
857
|
+
}
|
|
858
|
+
async synthesize(text, signal) {
|
|
859
|
+
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
860
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY2;
|
|
861
|
+
for (let attempt = 0; ; attempt++) {
|
|
862
|
+
const progress = { emitted: false };
|
|
863
|
+
try {
|
|
864
|
+
await this.synthesizeOnce(input, signal, progress);
|
|
865
|
+
return void 0;
|
|
866
|
+
} catch (error) {
|
|
867
|
+
if (signal.aborted || progress.emitted || attempt >= maxRetry) {
|
|
868
|
+
throw error;
|
|
869
|
+
}
|
|
870
|
+
if (!isBlocked(error)) throw error;
|
|
871
|
+
this.log(`Sentence blocked, trying again: "${text}"`);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
async synthesizeOnce(input, signal, progress) {
|
|
876
|
+
const stream = await this.genai.interactions.create(
|
|
877
|
+
{
|
|
878
|
+
model: this.options.model || DEFAULT_MODEL4,
|
|
879
|
+
store: false,
|
|
880
|
+
stream: true,
|
|
881
|
+
input,
|
|
882
|
+
response_format: { type: "audio" },
|
|
883
|
+
generation_config: {
|
|
884
|
+
speech_config: [{ voice: this.options.voice || DEFAULT_VOICE }]
|
|
885
|
+
}
|
|
886
|
+
},
|
|
887
|
+
{ fetchOptions: { signal } }
|
|
888
|
+
);
|
|
889
|
+
let resampler;
|
|
890
|
+
for await (const event of stream) {
|
|
891
|
+
if (signal.aborted) break;
|
|
892
|
+
if (event.event_type === "error") {
|
|
893
|
+
throw Object.assign(
|
|
894
|
+
new Error(event.error?.message || "Speech generation failed"),
|
|
895
|
+
{ code: event.error?.code }
|
|
896
|
+
);
|
|
897
|
+
}
|
|
898
|
+
if (event.event_type !== "step.delta" || event.delta.type !== "audio") {
|
|
899
|
+
continue;
|
|
900
|
+
}
|
|
901
|
+
const { data, sample_rate } = event.delta;
|
|
902
|
+
if (!data) continue;
|
|
903
|
+
resampler ?? (resampler = new Pcm16Resampler2(
|
|
904
|
+
sample_rate || GEMINI_SAMPLE_RATE,
|
|
905
|
+
OUTPUT_SAMPLE_RATE
|
|
906
|
+
));
|
|
907
|
+
const audio = resampler.process(Buffer.from(data, "base64"));
|
|
908
|
+
if (audio.length > 0) progress.emitted = true;
|
|
909
|
+
if (!this.emitAudio(audio)) break;
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
};
|
|
913
|
+
function isBlocked(error) {
|
|
914
|
+
return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
915
|
+
}
|
|
916
|
+
export {
|
|
917
|
+
GeminiAgent,
|
|
918
|
+
GeminiLive,
|
|
919
|
+
GeminiSTT,
|
|
920
|
+
GeminiTTS
|
|
921
|
+
};
|
|
922
|
+
//# sourceMappingURL=index.mjs.map
|