@micdrop/gemini 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/index.d.mts +43 -16
- package/dist/index.d.ts +43 -16
- package/dist/index.js +383 -313
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +377 -305
- package/dist/index.mjs.map +1 -1
- package/package.json +3 -3
package/dist/index.mjs
CHANGED
|
@@ -15,127 +15,139 @@ var GeminiAgent = class extends Agent {
|
|
|
15
15
|
this.signatures = /* @__PURE__ */ new Map();
|
|
16
16
|
this.genai = "genai" in options ? options.genai : new GoogleGenAI({ apiKey: options.apiKey });
|
|
17
17
|
}
|
|
18
|
-
|
|
18
|
+
generateAnswer(stream, signal) {
|
|
19
|
+
return this.generateStep(stream, signal, 0);
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Generates one step of the answer, retried on error.
|
|
23
|
+
*
|
|
24
|
+
* A retry starts the step over, so it only happens while nothing of the
|
|
25
|
+
* step reached the stream: what was said cannot be taken back, and saying
|
|
26
|
+
* it twice is worse than stopping there.
|
|
27
|
+
*/
|
|
28
|
+
async generateStep(stream, signal, stepCount) {
|
|
19
29
|
if (stepCount >= (this.options.maxSteps || DEFAULT_MAX_STEPS)) {
|
|
20
30
|
console.error("[GeminiAgent] Max steps reached");
|
|
21
31
|
return;
|
|
22
32
|
}
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
33
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRIES;
|
|
34
|
+
for (let tryCount = 0; ; tryCount++) {
|
|
35
|
+
const progress = { answer: "" };
|
|
36
|
+
try {
|
|
37
|
+
const step = await this.streamStep(stream, signal, progress);
|
|
38
|
+
if (signal.aborted) return;
|
|
39
|
+
this.recordAnswer(progress.answer);
|
|
40
|
+
if (step.calledTool && !step.skipAnswer && !progress.answer) {
|
|
41
|
+
await this.generateStep(stream, signal, stepCount + 1);
|
|
42
|
+
}
|
|
43
|
+
return;
|
|
44
|
+
} catch (error) {
|
|
45
|
+
if (signal.aborted) return;
|
|
46
|
+
console.error("[GeminiAgent] Error answering:", error);
|
|
47
|
+
if (progress.answer) {
|
|
48
|
+
this.log("Error after part of the answer was said, keeping it");
|
|
49
|
+
this.recordAnswer(progress.answer);
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (tryCount >= maxRetry) {
|
|
53
|
+
this.log("Max retries reached, giving up");
|
|
54
|
+
this.emit("Failed");
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
await sleep(this.options.retryDelay ?? DEFAULT_RETRY_DELAY, signal);
|
|
58
|
+
if (signal.aborted) return;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/** Streams one request, writing the text of the answer as it comes */
|
|
63
|
+
async streamStep(stream, signal, progress) {
|
|
26
64
|
let extracting = false;
|
|
27
65
|
const extractOptions = this.getExtractOptions();
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
if (startTagIndex !== -1) {
|
|
81
|
-
extracting = true;
|
|
82
|
-
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
83
|
-
break;
|
|
84
|
-
}
|
|
66
|
+
const response = await this.genai.interactions.create(
|
|
67
|
+
{
|
|
68
|
+
model: this.options.model || DEFAULT_MODEL,
|
|
69
|
+
store: false,
|
|
70
|
+
stream: true,
|
|
71
|
+
system_instruction: this.buildSystemInstruction(),
|
|
72
|
+
input: this.buildInput(),
|
|
73
|
+
tools: this.buildTools(),
|
|
74
|
+
...this.options.thinkingLevel || this.options.settings ? {
|
|
75
|
+
generation_config: {
|
|
76
|
+
...this.options.settings,
|
|
77
|
+
...this.options.thinkingLevel ? { thinking_level: this.options.thinkingLevel } : {}
|
|
78
|
+
}
|
|
79
|
+
} : {}
|
|
80
|
+
},
|
|
81
|
+
{ signal }
|
|
82
|
+
);
|
|
83
|
+
const steps = [];
|
|
84
|
+
let skipAnswer = false;
|
|
85
|
+
let calledTool = false;
|
|
86
|
+
let signature = "";
|
|
87
|
+
for await (const event of response) {
|
|
88
|
+
if (signal.aborted) break;
|
|
89
|
+
switch (event.event_type) {
|
|
90
|
+
case "step.start":
|
|
91
|
+
steps[event.index] = {
|
|
92
|
+
type: event.step.type,
|
|
93
|
+
id: event.step.id,
|
|
94
|
+
name: event.step.name,
|
|
95
|
+
text: "",
|
|
96
|
+
arguments: "",
|
|
97
|
+
signature: ""
|
|
98
|
+
};
|
|
99
|
+
break;
|
|
100
|
+
case "step.delta": {
|
|
101
|
+
const step = steps[event.index];
|
|
102
|
+
if (!step) break;
|
|
103
|
+
const delta = event.delta;
|
|
104
|
+
if (delta.type === "thought_signature") {
|
|
105
|
+
step.signature += delta.signature ?? "";
|
|
106
|
+
} else if (delta.type === "arguments_delta") {
|
|
107
|
+
step.arguments += delta.arguments ?? "";
|
|
108
|
+
} else if (delta.type === "text" && step.type === "model_output") {
|
|
109
|
+
this.log(`Answer chunk: "${delta.text}"`);
|
|
110
|
+
progress.answer += delta.text;
|
|
111
|
+
if (extractOptions) {
|
|
112
|
+
if (extracting) break;
|
|
113
|
+
const startTagIndex = delta.text.indexOf(extractOptions.startTag);
|
|
114
|
+
if (startTagIndex !== -1) {
|
|
115
|
+
extracting = true;
|
|
116
|
+
stream.write(delta.text.slice(0, startTagIndex).trimEnd());
|
|
117
|
+
break;
|
|
85
118
|
}
|
|
86
|
-
stream.write(delta.text);
|
|
87
119
|
}
|
|
88
|
-
|
|
89
|
-
}
|
|
90
|
-
case "step.stop": {
|
|
91
|
-
const step = steps[event.index];
|
|
92
|
-
if (step?.type === "thought") signature = step.signature;
|
|
93
|
-
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
94
|
-
calledTool = true;
|
|
95
|
-
if (signature) this.signatures.set(step.id, signature);
|
|
96
|
-
const result = await this.executeTool({
|
|
97
|
-
role: "tool_call",
|
|
98
|
-
toolCallId: step.id,
|
|
99
|
-
toolName: step.name,
|
|
100
|
-
parameters: step.arguments || "{}"
|
|
101
|
-
});
|
|
102
|
-
if (result.skipAnswer) skipAnswer = true;
|
|
103
|
-
break;
|
|
120
|
+
stream.write(delta.text);
|
|
104
121
|
}
|
|
105
|
-
|
|
106
|
-
throw new Error(event.error?.message || "Generation failed");
|
|
107
|
-
default:
|
|
108
|
-
break;
|
|
122
|
+
break;
|
|
109
123
|
}
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
this.log("Max retries reached, giving up");
|
|
130
|
-
this.emit("Failed");
|
|
124
|
+
case "step.stop": {
|
|
125
|
+
const step = steps[event.index];
|
|
126
|
+
if (step?.type === "thought") signature = step.signature;
|
|
127
|
+
if (step?.type !== "function_call" || !step.id || !step.name) break;
|
|
128
|
+
calledTool = true;
|
|
129
|
+
if (signature) this.signatures.set(step.id, signature);
|
|
130
|
+
const result = await this.executeTool({
|
|
131
|
+
role: "tool_call",
|
|
132
|
+
toolCallId: step.id,
|
|
133
|
+
toolName: step.name,
|
|
134
|
+
parameters: step.arguments || "{}"
|
|
135
|
+
});
|
|
136
|
+
if (result.skipAnswer) skipAnswer = true;
|
|
137
|
+
break;
|
|
138
|
+
}
|
|
139
|
+
case "error":
|
|
140
|
+
throw new Error(event.error?.message || "Generation failed");
|
|
141
|
+
default:
|
|
142
|
+
break;
|
|
131
143
|
}
|
|
132
144
|
}
|
|
145
|
+
return { calledTool, skipAnswer };
|
|
133
146
|
}
|
|
134
|
-
|
|
135
|
-
if (!
|
|
136
|
-
this.
|
|
137
|
-
this.
|
|
138
|
-
this.abortController = void 0;
|
|
147
|
+
recordAnswer(answer) {
|
|
148
|
+
if (!answer) return;
|
|
149
|
+
const { message, metadata } = this.extract(answer);
|
|
150
|
+
this.addAssistantMessage(message, metadata);
|
|
139
151
|
}
|
|
140
152
|
buildSystemInstruction() {
|
|
141
153
|
return this.conversation.flatMap(
|
|
@@ -207,6 +219,17 @@ var GeminiAgent = class extends Agent {
|
|
|
207
219
|
});
|
|
208
220
|
}
|
|
209
221
|
};
|
|
222
|
+
function sleep(ms, signal) {
|
|
223
|
+
return new Promise((resolve) => {
|
|
224
|
+
const done = () => {
|
|
225
|
+
clearTimeout(timer);
|
|
226
|
+
signal.removeEventListener("abort", done);
|
|
227
|
+
resolve();
|
|
228
|
+
};
|
|
229
|
+
const timer = setTimeout(done, ms);
|
|
230
|
+
signal.addEventListener("abort", done);
|
|
231
|
+
});
|
|
232
|
+
}
|
|
210
233
|
function parseArguments(parameters) {
|
|
211
234
|
try {
|
|
212
235
|
return JSON.parse(parameters);
|
|
@@ -223,67 +246,107 @@ import {
|
|
|
223
246
|
import { toJSONSchema as toJSONSchema2 } from "zod";
|
|
224
247
|
|
|
225
248
|
// src/LiveSocket.ts
|
|
249
|
+
import {
|
|
250
|
+
ReconnectingSocket
|
|
251
|
+
} from "@micdrop/server";
|
|
226
252
|
import { EventEmitter } from "eventemitter3";
|
|
227
|
-
import WebSocket from "ws";
|
|
228
253
|
var URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent";
|
|
229
254
|
var SAMPLE_RATE = 16e3;
|
|
230
255
|
var BYTES_PER_MS = SAMPLE_RATE * 2 / 1e3;
|
|
231
|
-
var DEFAULT_CONNECTION_TIMEOUT = 5e3;
|
|
232
|
-
var DEFAULT_RETRY_DELAY2 = 1e3;
|
|
233
|
-
var DEFAULT_MAX_RETRY = 3;
|
|
234
256
|
var LiveSocket = class extends EventEmitter {
|
|
235
257
|
constructor(options, buildSetup, log) {
|
|
236
258
|
super();
|
|
237
|
-
this.options = options;
|
|
238
259
|
this.buildSetup = buildSetup;
|
|
239
260
|
this.log = log;
|
|
240
|
-
this.connected = false;
|
|
241
|
-
this.destroyed = false;
|
|
242
261
|
// Messages waiting for the session, or for the end of a turn to be due
|
|
243
262
|
this.pending = [];
|
|
244
263
|
// Audio of the turn as it went out, see flush()
|
|
245
264
|
this.turnAudioStartedAt = 0;
|
|
246
265
|
this.turnAudioMs = 0;
|
|
247
|
-
|
|
266
|
+
// Attempts since the last session was ready, for the owner to tell a drop
|
|
267
|
+
// from a resumption that keeps failing
|
|
268
|
+
this.reconnectAttempts = 0;
|
|
269
|
+
/** Sends the setup, and holds the connection back until Gemini accepts it */
|
|
270
|
+
this.handshake = {
|
|
271
|
+
start: (socket) => {
|
|
272
|
+
const setup = this.buildSetup();
|
|
273
|
+
this.setup = setup;
|
|
274
|
+
socket.send(JSON.stringify(setup));
|
|
275
|
+
},
|
|
276
|
+
read: (data) => !!parseMessage(data.toString())?.setupComplete
|
|
277
|
+
};
|
|
278
|
+
this.onOpen = () => {
|
|
279
|
+
this.log("Session ready");
|
|
280
|
+
this.reconnectAttempts = 0;
|
|
281
|
+
this.emit("Ready", this.setup);
|
|
282
|
+
this.flush();
|
|
283
|
+
};
|
|
284
|
+
this.onMessage = (data) => {
|
|
285
|
+
const message = parseMessage(data.toString());
|
|
286
|
+
if (message) this.emit("Message", message);
|
|
287
|
+
};
|
|
288
|
+
this.onStatusChange = (status) => {
|
|
289
|
+
if (status !== "open") this.clearPendingTimeout();
|
|
290
|
+
if (status === "reconnecting") {
|
|
291
|
+
this.emit("Reconnecting", ++this.reconnectAttempts);
|
|
292
|
+
}
|
|
293
|
+
};
|
|
294
|
+
this.onFailed = () => {
|
|
295
|
+
this.clearPending();
|
|
296
|
+
this.emit("Failed");
|
|
297
|
+
};
|
|
298
|
+
this.socket = new ReconnectingSocket({
|
|
299
|
+
url: `${URL}?key=${encodeURIComponent(options.apiKey)}`,
|
|
300
|
+
connectionTimeout: options.connectionTimeout,
|
|
301
|
+
retryDelay: options.retryDelay,
|
|
302
|
+
maxRetry: options.maxRetry,
|
|
303
|
+
handshake: this.handshake,
|
|
304
|
+
log
|
|
305
|
+
});
|
|
306
|
+
this.socket.on("Open", this.onOpen);
|
|
307
|
+
this.socket.on("Message", this.onMessage);
|
|
308
|
+
this.socket.on("StatusChange", this.onStatusChange);
|
|
309
|
+
this.socket.on("Failed", this.onFailed);
|
|
248
310
|
}
|
|
249
311
|
/** The session is set up and takes messages right away */
|
|
250
312
|
get isConnected() {
|
|
251
|
-
return this.
|
|
313
|
+
return this.socket.isOpen;
|
|
252
314
|
}
|
|
253
315
|
connect() {
|
|
254
|
-
this.
|
|
255
|
-
this.log("Connection error:", error);
|
|
256
|
-
this.reconnect();
|
|
257
|
-
});
|
|
316
|
+
this.socket.connect();
|
|
258
317
|
}
|
|
259
|
-
/**
|
|
318
|
+
/**
|
|
319
|
+
* Sends a message once the session is ready, after the ones before it.
|
|
320
|
+
* Dropped once the connection failed for good, as nothing would send it.
|
|
321
|
+
*/
|
|
260
322
|
send(message) {
|
|
323
|
+
const status = this.socket.status;
|
|
324
|
+
if (status === "failed" || status === "destroyed") return;
|
|
261
325
|
this.pending.push(message);
|
|
262
326
|
this.flush();
|
|
263
327
|
}
|
|
264
328
|
/** Sends a message ahead of the waiting ones, from a Ready listener */
|
|
265
329
|
sendFirst(message) {
|
|
266
|
-
this.socket
|
|
330
|
+
this.socket.send(JSON.stringify(message));
|
|
267
331
|
}
|
|
268
332
|
/** Drops the messages that have not gone out */
|
|
269
333
|
clearPending() {
|
|
270
334
|
this.pending = [];
|
|
271
335
|
this.clearPendingTimeout();
|
|
272
336
|
}
|
|
273
|
-
/**
|
|
337
|
+
/**
|
|
338
|
+
* Moves to a new connection, keeping the messages that have not gone out.
|
|
339
|
+
* A reconnection waiting for its delay is replaced by this one.
|
|
340
|
+
*/
|
|
274
341
|
renew() {
|
|
275
342
|
this.log("Renewing the connection");
|
|
276
|
-
this.
|
|
277
|
-
this.
|
|
343
|
+
this.clearPendingTimeout();
|
|
344
|
+
this.reconnectAttempts = 0;
|
|
345
|
+
this.socket.restart();
|
|
278
346
|
}
|
|
279
347
|
destroy() {
|
|
280
|
-
this.
|
|
281
|
-
|
|
282
|
-
clearTimeout(this.reconnectTimeout);
|
|
283
|
-
this.reconnectTimeout = void 0;
|
|
284
|
-
}
|
|
285
|
-
this.closeSocket();
|
|
286
|
-
this.pending = [];
|
|
348
|
+
this.clearPending();
|
|
349
|
+
this.socket.destroy();
|
|
287
350
|
this.removeAllListeners();
|
|
288
351
|
}
|
|
289
352
|
/**
|
|
@@ -296,8 +359,7 @@ var LiveSocket = class extends EventEmitter {
|
|
|
296
359
|
*/
|
|
297
360
|
flush() {
|
|
298
361
|
while (this.pending.length > 0 && !this.pendingTimeout) {
|
|
299
|
-
|
|
300
|
-
if (!this.connected || socket?.readyState !== WebSocket.OPEN) return;
|
|
362
|
+
if (!this.socket.isOpen) return;
|
|
301
363
|
const message = this.pending[0];
|
|
302
364
|
const input = message.realtimeInput;
|
|
303
365
|
if (input?.activityEnd) {
|
|
@@ -317,7 +379,7 @@ var LiveSocket = class extends EventEmitter {
|
|
|
317
379
|
this.turnAudioMs += Buffer.byteLength(input.audio.data, "base64") / BYTES_PER_MS;
|
|
318
380
|
}
|
|
319
381
|
this.pending.shift();
|
|
320
|
-
socket.send(JSON.stringify(message));
|
|
382
|
+
this.socket.send(JSON.stringify(message));
|
|
321
383
|
}
|
|
322
384
|
}
|
|
323
385
|
clearPendingTimeout() {
|
|
@@ -325,96 +387,28 @@ var LiveSocket = class extends EventEmitter {
|
|
|
325
387
|
clearTimeout(this.pendingTimeout);
|
|
326
388
|
this.pendingTimeout = void 0;
|
|
327
389
|
}
|
|
328
|
-
initWS() {
|
|
329
|
-
return new Promise((resolve, reject) => {
|
|
330
|
-
const socket = new WebSocket(
|
|
331
|
-
`${URL}?key=${encodeURIComponent(this.options.apiKey)}`
|
|
332
|
-
);
|
|
333
|
-
this.socket = socket;
|
|
334
|
-
const timeout = setTimeout(() => {
|
|
335
|
-
this.log("Connection timeout");
|
|
336
|
-
this.closeSocket();
|
|
337
|
-
reject(new Error("WebSocket connection timeout"));
|
|
338
|
-
}, this.options.connectionTimeout ?? DEFAULT_CONNECTION_TIMEOUT);
|
|
339
|
-
socket.on("open", () => {
|
|
340
|
-
this.log("Connection opened");
|
|
341
|
-
socket.send(JSON.stringify(this.buildSetup()));
|
|
342
|
-
});
|
|
343
|
-
socket.on("error", (error) => {
|
|
344
|
-
clearTimeout(timeout);
|
|
345
|
-
this.log("WebSocket error:", error);
|
|
346
|
-
reject(new Error("WebSocket connection error"));
|
|
347
|
-
});
|
|
348
|
-
socket.on("close", (code, reason) => {
|
|
349
|
-
clearTimeout(timeout);
|
|
350
|
-
const wasConnected = this.connected;
|
|
351
|
-
this.closeSocket();
|
|
352
|
-
if (this.destroyed) return;
|
|
353
|
-
this.log("Connection closed", { code, reason: reason.toString() });
|
|
354
|
-
if (wasConnected) {
|
|
355
|
-
this.reconnect();
|
|
356
|
-
} else {
|
|
357
|
-
reject(new Error(`Connection closed: ${reason.toString()}`));
|
|
358
|
-
}
|
|
359
|
-
});
|
|
360
|
-
socket.on("message", (data) => {
|
|
361
|
-
const message = JSON.parse(data.toString());
|
|
362
|
-
if (message.setupComplete) {
|
|
363
|
-
clearTimeout(timeout);
|
|
364
|
-
this.log("Session ready");
|
|
365
|
-
this.connected = true;
|
|
366
|
-
this.retryCount = 0;
|
|
367
|
-
this.emit("Ready");
|
|
368
|
-
this.flush();
|
|
369
|
-
resolve();
|
|
370
|
-
return;
|
|
371
|
-
}
|
|
372
|
-
this.emit("Message", message);
|
|
373
|
-
});
|
|
374
|
-
});
|
|
375
|
-
}
|
|
376
|
-
closeSocket() {
|
|
377
|
-
const socket = this.socket;
|
|
378
|
-
this.socket = void 0;
|
|
379
|
-
this.connected = false;
|
|
380
|
-
this.clearPendingTimeout();
|
|
381
|
-
if (!socket) return;
|
|
382
|
-
socket.removeAllListeners();
|
|
383
|
-
socket.on("error", () => {
|
|
384
|
-
});
|
|
385
|
-
if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
|
|
386
|
-
socket.close(1e3);
|
|
387
|
-
}
|
|
388
|
-
}
|
|
389
|
-
reconnect() {
|
|
390
|
-
if (this.destroyed) return;
|
|
391
|
-
this.retryCount++;
|
|
392
|
-
if (this.retryCount > (this.options.maxRetry ?? DEFAULT_MAX_RETRY)) {
|
|
393
|
-
this.log("Max retries reached, giving up");
|
|
394
|
-
this.emit("Failed");
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
this.emit("Reconnecting", this.retryCount);
|
|
398
|
-
this.log("Reconnecting...");
|
|
399
|
-
this.reconnectTimeout = setTimeout(() => {
|
|
400
|
-
this.reconnectTimeout = void 0;
|
|
401
|
-
this.connect();
|
|
402
|
-
}, this.options.retryDelay ?? DEFAULT_RETRY_DELAY2);
|
|
403
|
-
}
|
|
404
390
|
};
|
|
391
|
+
function parseMessage(text) {
|
|
392
|
+
try {
|
|
393
|
+
return JSON.parse(text);
|
|
394
|
+
} catch {
|
|
395
|
+
return void 0;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
405
398
|
|
|
406
399
|
// src/GeminiLive.ts
|
|
407
400
|
var DEFAULT_MODEL2 = "gemini-3.8-live";
|
|
408
401
|
var SAMPLE_RATE2 = 16e3;
|
|
409
402
|
var GEMINI_OUTPUT_SAMPLE_RATE = 24e3;
|
|
410
403
|
var TOOL_FOLLOW_UP_TIMEOUT = 1e4;
|
|
404
|
+
function newToolRound() {
|
|
405
|
+
return { running: 0, answerEndDeferred: false };
|
|
406
|
+
}
|
|
411
407
|
var FIRST_MESSAGE_PROMPT2 = "The call starts now. Speak first, as your instructions say.";
|
|
412
408
|
var GeminiLive = class extends Realtime {
|
|
413
409
|
constructor(options) {
|
|
414
410
|
super(options);
|
|
415
411
|
this.everConnected = false;
|
|
416
|
-
// Whether the setup being sent resumes the session
|
|
417
|
-
this.resuming = false;
|
|
418
412
|
this.goingAway = false;
|
|
419
413
|
this.outputResampler = new Pcm16Resampler(
|
|
420
414
|
GEMINI_OUTPUT_SAMPLE_RATE,
|
|
@@ -423,14 +417,16 @@ var GeminiLive = class extends Realtime {
|
|
|
423
417
|
this.hasTranscribedTurn = false;
|
|
424
418
|
// Transcript of a dropped turn, which the conversation should not show
|
|
425
419
|
this.ignoreUserTranscript = false;
|
|
420
|
+
// Calls of the session Gemini told not to answer
|
|
426
421
|
this.cancelledToolCalls = /* @__PURE__ */ new Set();
|
|
427
|
-
//
|
|
428
|
-
//
|
|
429
|
-
|
|
430
|
-
this.
|
|
431
|
-
this.
|
|
432
|
-
|
|
433
|
-
if (this.everConnected && !
|
|
422
|
+
// Bumped by every session that does not resume the previous one, whose tool
|
|
423
|
+
// calls it cannot answer
|
|
424
|
+
this.session = 0;
|
|
425
|
+
this.toolRound = newToolRound();
|
|
426
|
+
this.onReady = (setup) => {
|
|
427
|
+
const resumed = !!setup?.setup?.sessionResumption?.handle;
|
|
428
|
+
if (this.everConnected && !resumed) {
|
|
429
|
+
this.startFreshSession();
|
|
434
430
|
const turns = this.conversation.flatMap(
|
|
435
431
|
(message) => message.role === "user" || message.role === "assistant" ? [
|
|
436
432
|
{
|
|
@@ -448,7 +444,12 @@ var GeminiLive = class extends Realtime {
|
|
|
448
444
|
};
|
|
449
445
|
this.onReconnecting = (attempt) => {
|
|
450
446
|
if (attempt > 1) this.resumptionHandle = void 0;
|
|
451
|
-
if (!this.resumptionHandle)
|
|
447
|
+
if (!this.resumptionHandle) {
|
|
448
|
+
this.socket.clearPending();
|
|
449
|
+
this.startFreshSession();
|
|
450
|
+
}
|
|
451
|
+
this.toolRound.answerEndDeferred = false;
|
|
452
|
+
this.commitTurnTranscript();
|
|
452
453
|
this.endAnswer();
|
|
453
454
|
};
|
|
454
455
|
this.onMessage = (message) => {
|
|
@@ -486,9 +487,16 @@ var GeminiLive = class extends Realtime {
|
|
|
486
487
|
this.socket.on("Failed", () => this.emit("Failed"));
|
|
487
488
|
this.socket.connect();
|
|
488
489
|
}
|
|
490
|
+
/**
|
|
491
|
+
* Stops the answer. The tools it is waiting for still send their results,
|
|
492
|
+
* silently, since a blocking call left unanswered would stall the model.
|
|
493
|
+
*/
|
|
494
|
+
cancel() {
|
|
495
|
+
super.cancel();
|
|
496
|
+
this.resetToolRound();
|
|
497
|
+
}
|
|
489
498
|
destroy() {
|
|
490
499
|
super.destroy();
|
|
491
|
-
this.clearToolFollowUp();
|
|
492
500
|
this.socket.destroy();
|
|
493
501
|
}
|
|
494
502
|
openTurn() {
|
|
@@ -523,11 +531,9 @@ var GeminiLive = class extends Realtime {
|
|
|
523
531
|
}
|
|
524
532
|
cancelAnswer() {
|
|
525
533
|
this.outputResampler.reset();
|
|
526
|
-
this.clearToolFollowUp();
|
|
527
|
-
this.answerEndDeferred = false;
|
|
528
534
|
}
|
|
529
535
|
get isIdle() {
|
|
530
|
-
return super.isIdle && this.
|
|
536
|
+
return super.isIdle && this.toolRound.running === 0;
|
|
531
537
|
}
|
|
532
538
|
updateTools() {
|
|
533
539
|
if (this.everConnected) {
|
|
@@ -559,7 +565,6 @@ var GeminiLive = class extends Realtime {
|
|
|
559
565
|
this.socket.send(message);
|
|
560
566
|
}
|
|
561
567
|
buildSetup() {
|
|
562
|
-
this.resuming = !!this.resumptionHandle;
|
|
563
568
|
const functionDeclarations = this.tools.map((tool) => ({
|
|
564
569
|
name: tool.name,
|
|
565
570
|
description: tool.description,
|
|
@@ -593,6 +598,22 @@ var GeminiLive = class extends Realtime {
|
|
|
593
598
|
}
|
|
594
599
|
};
|
|
595
600
|
}
|
|
601
|
+
/** Forgets what belonged to the previous session */
|
|
602
|
+
startFreshSession() {
|
|
603
|
+
this.session++;
|
|
604
|
+
this.resetToolRound();
|
|
605
|
+
this.cancelledToolCalls.clear();
|
|
606
|
+
this.ignoreUserTranscript = false;
|
|
607
|
+
}
|
|
608
|
+
/**
|
|
609
|
+
* Starts a new round of tools. The tools still running belong to the old
|
|
610
|
+
* one: their results go out silently, and nothing waits for the model to
|
|
611
|
+
* talk about them.
|
|
612
|
+
*/
|
|
613
|
+
resetToolRound() {
|
|
614
|
+
if (this.toolRound.followUp) clearTimeout(this.toolRound.followUp);
|
|
615
|
+
this.toolRound = newToolRound();
|
|
616
|
+
}
|
|
596
617
|
/** Moves to a new connection before Gemini closes this one */
|
|
597
618
|
renewIfGoingAway() {
|
|
598
619
|
if (!this.goingAway || !this.isIdle || !this.resumptionHandle) return;
|
|
@@ -624,8 +645,8 @@ var GeminiLive = class extends Realtime {
|
|
|
624
645
|
if (content.turnComplete) {
|
|
625
646
|
this.commitTurnTranscript();
|
|
626
647
|
this.outputResampler.reset();
|
|
627
|
-
if (this.
|
|
628
|
-
this.answerEndDeferred = true;
|
|
648
|
+
if (this.toolRound.running > 0 || this.toolRound.followUp) {
|
|
649
|
+
this.toolRound.answerEndDeferred = true;
|
|
629
650
|
} else {
|
|
630
651
|
this.endAnswer();
|
|
631
652
|
this.renewIfGoingAway();
|
|
@@ -634,24 +655,22 @@ var GeminiLive = class extends Realtime {
|
|
|
634
655
|
}
|
|
635
656
|
onModelOutput() {
|
|
636
657
|
this.commitTurnTranscript();
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
658
|
+
const round = this.toolRound;
|
|
659
|
+
if (round.followUp) {
|
|
660
|
+
clearTimeout(round.followUp);
|
|
661
|
+
round.followUp = void 0;
|
|
662
|
+
round.answerEndDeferred = false;
|
|
640
663
|
}
|
|
641
664
|
}
|
|
642
665
|
/** Ends the answer the model stopped giving to wait for a tool */
|
|
643
666
|
endDeferredAnswer() {
|
|
644
|
-
|
|
645
|
-
if (
|
|
646
|
-
|
|
667
|
+
const round = this.toolRound;
|
|
668
|
+
if (!round.answerEndDeferred) return;
|
|
669
|
+
if (round.running > 0 || round.followUp) return;
|
|
670
|
+
round.answerEndDeferred = false;
|
|
647
671
|
this.endAnswer();
|
|
648
672
|
this.renewIfGoingAway();
|
|
649
673
|
}
|
|
650
|
-
clearToolFollowUp() {
|
|
651
|
-
if (!this.toolFollowUp) return;
|
|
652
|
-
clearTimeout(this.toolFollowUp);
|
|
653
|
-
this.toolFollowUp = void 0;
|
|
654
|
-
}
|
|
655
674
|
/**
|
|
656
675
|
* The transcript of the turn comes before the answer, so the first piece of
|
|
657
676
|
* the answer closes it.
|
|
@@ -669,17 +688,22 @@ var GeminiLive = class extends Realtime {
|
|
|
669
688
|
toolName: call.name,
|
|
670
689
|
parameters: JSON.stringify(call.args ?? {})
|
|
671
690
|
};
|
|
672
|
-
this.
|
|
691
|
+
const round = this.toolRound;
|
|
692
|
+
const session = this.session;
|
|
693
|
+
round.running++;
|
|
673
694
|
let result;
|
|
674
695
|
try {
|
|
675
696
|
result = await this.runTool(toolCall);
|
|
676
697
|
} finally {
|
|
677
|
-
|
|
698
|
+
round.running--;
|
|
678
699
|
}
|
|
700
|
+
if (session !== this.session) return;
|
|
679
701
|
if (this.cancelledToolCalls.delete(call.id)) {
|
|
680
702
|
this.endDeferredAnswer();
|
|
681
703
|
return;
|
|
682
704
|
}
|
|
705
|
+
const interrupted = round !== this.toolRound;
|
|
706
|
+
const silent = result.skipAnswer || interrupted;
|
|
683
707
|
this.send({
|
|
684
708
|
toolResponse: {
|
|
685
709
|
functionResponses: [
|
|
@@ -689,16 +713,17 @@ var GeminiLive = class extends Realtime {
|
|
|
689
713
|
response: {
|
|
690
714
|
output: result.output ?? null,
|
|
691
715
|
// A tool without an answer lets the model know silently
|
|
692
|
-
scheduling:
|
|
716
|
+
scheduling: silent ? "SILENT" : "WHEN_IDLE"
|
|
693
717
|
}
|
|
694
718
|
}
|
|
695
719
|
]
|
|
696
720
|
}
|
|
697
721
|
});
|
|
722
|
+
if (interrupted) return;
|
|
698
723
|
if (!result.skipAnswer) {
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
724
|
+
if (round.followUp) clearTimeout(round.followUp);
|
|
725
|
+
round.followUp = setTimeout(() => {
|
|
726
|
+
round.followUp = void 0;
|
|
702
727
|
this.log("Nothing said about the tool result");
|
|
703
728
|
this.endDeferredAnswer();
|
|
704
729
|
}, TOOL_FOLLOW_UP_TIMEOUT);
|
|
@@ -717,71 +742,77 @@ var GeminiSTT = class extends STT {
|
|
|
717
742
|
constructor(options) {
|
|
718
743
|
super();
|
|
719
744
|
this.options = options;
|
|
745
|
+
// Gemini closes the session soon, and it is renewed once nothing waits
|
|
720
746
|
this.goingAway = false;
|
|
721
|
-
//
|
|
722
|
-
//
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
this.
|
|
726
|
-
this.
|
|
747
|
+
// Utterances waiting for their transcript, in the order they were spoken.
|
|
748
|
+
// Gemini names no utterance in what it sends, and goes on to the next one
|
|
749
|
+
// before it is done with the previous. So it hears one at a time: the first
|
|
750
|
+
// one, and the audio of the next waits here until its transcript is in.
|
|
751
|
+
this.utterances = [];
|
|
752
|
+
this.onReady = () => {
|
|
753
|
+
this.goingAway = false;
|
|
754
|
+
};
|
|
727
755
|
this.onMessage = (message) => {
|
|
728
756
|
const content = message.serverContent;
|
|
729
|
-
|
|
730
|
-
|
|
757
|
+
const utterance = this.utterances[0];
|
|
758
|
+
if (content?.inputTranscription?.text && utterance) {
|
|
759
|
+
utterance.transcript += content.inputTranscription.text;
|
|
731
760
|
}
|
|
732
|
-
if (content?.generationComplete &&
|
|
733
|
-
this.
|
|
761
|
+
if (content?.generationComplete && utterance) {
|
|
762
|
+
if (utterance.state === "ended") this.give(utterance);
|
|
763
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
734
764
|
}
|
|
735
765
|
if (message.goAway) {
|
|
736
766
|
this.log("Connection ending in", message.goAway.timeLeft);
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
} else {
|
|
740
|
-
this.socket.renew();
|
|
741
|
-
}
|
|
767
|
+
this.goingAway = true;
|
|
768
|
+
this.renewIfGoingAway();
|
|
742
769
|
}
|
|
743
770
|
};
|
|
744
771
|
this.onReconnecting = () => {
|
|
772
|
+
this.goingAway = false;
|
|
745
773
|
this.socket.clearPending();
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
774
|
+
for (const utterance of this.utterances) {
|
|
775
|
+
if (utterance.state === "given") this.remove(utterance);
|
|
776
|
+
}
|
|
777
|
+
for (const utterance of this.utterances) {
|
|
778
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
779
|
+
utterance.timeout = void 0;
|
|
780
|
+
utterance.sent = false;
|
|
781
|
+
utterance.transcript = "";
|
|
752
782
|
}
|
|
783
|
+
this.sendFirst();
|
|
784
|
+
};
|
|
785
|
+
this.onFailed = () => {
|
|
786
|
+
const chunks = this.utterances.filter((utterance) => utterance.state !== "given").flatMap((utterance) => utterance.chunks);
|
|
787
|
+
this.dropUtterances();
|
|
788
|
+
this.emit("Failed", chunks);
|
|
753
789
|
};
|
|
754
790
|
this.socket = new LiveSocket(
|
|
755
791
|
options,
|
|
756
792
|
() => this.buildSetup(),
|
|
757
793
|
(...message) => this.log(...message)
|
|
758
794
|
);
|
|
795
|
+
this.socket.on("Ready", this.onReady);
|
|
759
796
|
this.socket.on("Message", this.onMessage);
|
|
760
797
|
this.socket.on("Reconnecting", this.onReconnecting);
|
|
761
|
-
this.socket.on("Failed",
|
|
798
|
+
this.socket.on("Failed", this.onFailed);
|
|
762
799
|
this.socket.connect();
|
|
763
800
|
}
|
|
764
801
|
transcribe(audioStream) {
|
|
802
|
+
let utterance;
|
|
765
803
|
audioStream.on("data", (chunk) => {
|
|
766
|
-
if (!
|
|
767
|
-
|
|
768
|
-
|
|
804
|
+
if (!utterance) utterance = this.openUtterance();
|
|
805
|
+
if (utterance.state !== "open") return;
|
|
806
|
+
utterance.chunks.push(chunk);
|
|
807
|
+
if (utterance.sent) this.sendAudio(chunk);
|
|
769
808
|
});
|
|
770
809
|
audioStream.on("end", () => {
|
|
771
|
-
if (
|
|
772
|
-
this.utteranceEnded = true;
|
|
773
|
-
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
774
|
-
this.clearTranscriptionTimeout();
|
|
775
|
-
this.transcriptionTimeout = setTimeout(() => {
|
|
776
|
-
this.transcriptionTimeout = void 0;
|
|
777
|
-
this.log("Transcription timeout");
|
|
778
|
-
this.finishUtterance();
|
|
779
|
-
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
810
|
+
if (utterance?.state === "open") this.endUtterance(utterance);
|
|
780
811
|
});
|
|
781
812
|
}
|
|
782
813
|
destroy() {
|
|
783
814
|
super.destroy();
|
|
784
|
-
this.
|
|
815
|
+
this.dropUtterances();
|
|
785
816
|
this.socket.destroy();
|
|
786
817
|
}
|
|
787
818
|
buildSetup() {
|
|
@@ -802,12 +833,38 @@ var GeminiSTT = class extends STT {
|
|
|
802
833
|
};
|
|
803
834
|
}
|
|
804
835
|
openUtterance() {
|
|
805
|
-
|
|
806
|
-
this.
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
836
|
+
const last = this.utterances[this.utterances.length - 1];
|
|
837
|
+
if (last?.state === "open") this.endUtterance(last);
|
|
838
|
+
const utterance = {
|
|
839
|
+
state: "open",
|
|
840
|
+
sent: false,
|
|
841
|
+
chunks: [],
|
|
842
|
+
transcript: ""
|
|
843
|
+
};
|
|
844
|
+
this.utterances.push(utterance);
|
|
845
|
+
this.sendFirst();
|
|
846
|
+
return utterance;
|
|
847
|
+
}
|
|
848
|
+
endUtterance(utterance) {
|
|
849
|
+
utterance.state = "ended";
|
|
850
|
+
if (utterance.sent) this.sendEnd(utterance);
|
|
851
|
+
}
|
|
852
|
+
/** Sends the first utterance to Gemini, with the audio it held meanwhile */
|
|
853
|
+
sendFirst() {
|
|
854
|
+
const utterance = this.utterances[0];
|
|
855
|
+
if (!utterance || utterance.sent) return;
|
|
856
|
+
utterance.sent = true;
|
|
810
857
|
this.socket.send({ realtimeInput: { activityStart: {} } });
|
|
858
|
+
utterance.chunks.forEach((chunk) => this.sendAudio(chunk));
|
|
859
|
+
if (utterance.state === "ended") this.sendEnd(utterance);
|
|
860
|
+
}
|
|
861
|
+
sendEnd(utterance) {
|
|
862
|
+
this.socket.send({ realtimeInput: { activityEnd: {} } });
|
|
863
|
+
utterance.timeout = setTimeout(() => {
|
|
864
|
+
utterance.timeout = void 0;
|
|
865
|
+
this.log("Transcription timeout");
|
|
866
|
+
this.give(utterance);
|
|
867
|
+
}, this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT);
|
|
811
868
|
}
|
|
812
869
|
sendAudio(chunk) {
|
|
813
870
|
this.socket.send({
|
|
@@ -819,25 +876,40 @@ var GeminiSTT = class extends STT {
|
|
|
819
876
|
}
|
|
820
877
|
});
|
|
821
878
|
}
|
|
822
|
-
/**
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
879
|
+
/**
|
|
880
|
+
* Emits what was transcribed of an utterance, possibly nothing, and keeps
|
|
881
|
+
* it in line for what Gemini still sends about it
|
|
882
|
+
*/
|
|
883
|
+
give(utterance) {
|
|
884
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
885
|
+
utterance.state = "given";
|
|
886
|
+
const transcript = utterance.transcript.trim();
|
|
830
887
|
this.log(`Transcript: "${transcript}"`);
|
|
831
888
|
this.emit("Transcript", transcript);
|
|
832
|
-
|
|
833
|
-
this.
|
|
834
|
-
this.
|
|
835
|
-
|
|
889
|
+
utterance.timeout = setTimeout(
|
|
890
|
+
() => this.remove(utterance),
|
|
891
|
+
this.options.transcriptionTimeout ?? DEFAULT_TRANSCRIPTION_TIMEOUT
|
|
892
|
+
);
|
|
836
893
|
}
|
|
837
|
-
|
|
838
|
-
if (
|
|
839
|
-
|
|
840
|
-
this.
|
|
894
|
+
remove(utterance) {
|
|
895
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
896
|
+
utterance.timeout = void 0;
|
|
897
|
+
this.utterances = this.utterances.filter((other) => other !== utterance);
|
|
898
|
+
this.renewIfGoingAway();
|
|
899
|
+
this.sendFirst();
|
|
900
|
+
}
|
|
901
|
+
/** Moves to a new connection once no utterance waits on this one */
|
|
902
|
+
renewIfGoingAway() {
|
|
903
|
+
if (!this.goingAway || this.utterances.length > 0) return;
|
|
904
|
+
this.goingAway = false;
|
|
905
|
+
this.socket.renew();
|
|
906
|
+
}
|
|
907
|
+
dropUtterances() {
|
|
908
|
+
for (const utterance of this.utterances) {
|
|
909
|
+
if (utterance.timeout) clearTimeout(utterance.timeout);
|
|
910
|
+
if (utterance.state === "open") utterance.state = "ended";
|
|
911
|
+
}
|
|
912
|
+
this.utterances = [];
|
|
841
913
|
}
|
|
842
914
|
};
|
|
843
915
|
|
|
@@ -848,7 +920,7 @@ var DEFAULT_MODEL4 = "gemini-2.5-flash-preview-tts";
|
|
|
848
920
|
var DEFAULT_VOICE = "Kore";
|
|
849
921
|
var GEMINI_SAMPLE_RATE = 24e3;
|
|
850
922
|
var OUTPUT_SAMPLE_RATE = 16e3;
|
|
851
|
-
var
|
|
923
|
+
var DEFAULT_MAX_RETRY = 2;
|
|
852
924
|
var GeminiTTS = class extends SentenceTTS {
|
|
853
925
|
constructor(options) {
|
|
854
926
|
super();
|
|
@@ -857,7 +929,7 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
857
929
|
}
|
|
858
930
|
async synthesize(text, signal) {
|
|
859
931
|
const input = this.options.instructions ? `${this.options.instructions}: ${text}` : text;
|
|
860
|
-
const maxRetry = this.options.maxRetry ??
|
|
932
|
+
const maxRetry = this.options.maxRetry ?? DEFAULT_MAX_RETRY;
|
|
861
933
|
for (let attempt = 0; ; attempt++) {
|
|
862
934
|
const progress = { emitted: false };
|
|
863
935
|
try {
|
|
@@ -911,7 +983,7 @@ var GeminiTTS = class extends SentenceTTS {
|
|
|
911
983
|
}
|
|
912
984
|
};
|
|
913
985
|
function isBlocked(error) {
|
|
914
|
-
return error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
986
|
+
return error?.status === 400 || error?.code === "content_blocked" || error?.error?.error?.code === "content_blocked" || String(error?.body ?? "").includes("content_blocked");
|
|
915
987
|
}
|
|
916
988
|
export {
|
|
917
989
|
GeminiAgent,
|