@charivo/realtime 0.18.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -26
- package/dist/chunk-3YPRYU2M.mjs +1251 -0
- package/dist/chunk-3YPRYU2M.mjs.map +1 -0
- package/dist/chunk-5OO737VD.mjs +33 -0
- package/dist/chunk-5OO737VD.mjs.map +1 -0
- package/dist/{chunk-U7ER2VBF.mjs → chunk-EIYYLUWF.mjs} +10 -17
- package/dist/chunk-EIYYLUWF.mjs.map +1 -0
- package/dist/{chunk-VFIMTFOP.mjs → chunk-JEJGTULX.mjs} +24 -31
- package/dist/chunk-JEJGTULX.mjs.map +1 -0
- package/dist/{chunk-EXLERHST.mjs → chunk-VTHBTUH3.mjs} +7 -10
- package/dist/chunk-VTHBTUH3.mjs.map +1 -0
- package/dist/gemini/index.d.mts +22 -0
- package/dist/gemini/index.d.ts +22 -0
- package/dist/gemini/index.js +1282 -0
- package/dist/gemini/index.js.map +1 -0
- package/dist/gemini/index.mjs +4 -0
- package/dist/gemini/index.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs.map +1 -1
- package/dist/openai/index.js +22 -7
- package/dist/openai/index.js.map +1 -1
- package/dist/openai/index.mjs +3 -2
- package/dist/openai-agents/index.js +25 -14
- package/dist/openai-agents/index.js.map +1 -1
- package/dist/openai-agents/index.mjs +3 -2
- package/dist/remote/index.d.mts +1 -0
- package/dist/remote/index.d.ts +1 -0
- package/dist/remote/index.js +1296 -37
- package/dist/remote/index.js.map +1 -1
- package/dist/remote/index.mjs +13 -4
- package/dist/remote/index.mjs.map +1 -1
- package/package.json +7 -2
- package/dist/chunk-EXLERHST.mjs.map +0 -1
- package/dist/chunk-U7ER2VBF.mjs.map +0 -1
- package/dist/chunk-VFIMTFOP.mjs.map +0 -1
|
@@ -0,0 +1,1251 @@
|
|
|
1
|
+
import { acquireMicrophoneStream, DEFAULT_REQUEST_TIMEOUT_MS, isRealtimeSessionBootstrap, isRecord } from './chunk-5OO737VD.mjs';
|
|
2
|
+
import { createLipSyncAnalyzer, GEMINI_LIVE_ADAPTER, CharivoStateError, fetchWithTimeout, CharivoProviderError } from '@charivo/core';
|
|
3
|
+
|
|
4
|
+
// src/gemini/defaults.ts
|
|
5
|
+
var DEFAULT_GEMINI_LIVE_MODEL = "gemini-3.1-flash-live-preview";
|
|
6
|
+
var DEFAULT_GEMINI_LIVE_VOICE = "Kore";
|
|
7
|
+
var INPUT_SAMPLE_RATE = 16e3;
|
|
8
|
+
var OUTPUT_SAMPLE_RATE = 24e3;
|
|
9
|
+
var CAPTURE_FRAME_SAMPLES = 320;
|
|
10
|
+
var CONVERGENCE_GATE_MS = 700;
|
|
11
|
+
|
|
12
|
+
// src/gemini/dev-bootstrap.ts
|
|
13
|
+
var AUTH_TOKENS_URL = "https://generativelanguage.googleapis.com/v1beta/auth_tokens";
|
|
14
|
+
var GEMINI_LIVE_WEBSOCKET_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained";
|
|
15
|
+
var ALLOWED_VOICES = /* @__PURE__ */ new Set([
|
|
16
|
+
"Zephyr",
|
|
17
|
+
"Puck",
|
|
18
|
+
"Charon",
|
|
19
|
+
"Kore",
|
|
20
|
+
"Fenrir",
|
|
21
|
+
"Leda",
|
|
22
|
+
"Orus",
|
|
23
|
+
"Aoede",
|
|
24
|
+
"Callirrhoe",
|
|
25
|
+
"Autonoe",
|
|
26
|
+
"Enceladus",
|
|
27
|
+
"Iapetus",
|
|
28
|
+
"Umbriel",
|
|
29
|
+
"Algieba",
|
|
30
|
+
"Despina",
|
|
31
|
+
"Erinome",
|
|
32
|
+
"Algenib",
|
|
33
|
+
"Rasalgethi",
|
|
34
|
+
"Laomedeia",
|
|
35
|
+
"Achernar",
|
|
36
|
+
"Alnilam",
|
|
37
|
+
"Schedar",
|
|
38
|
+
"Gacrux",
|
|
39
|
+
"Pulcherrima",
|
|
40
|
+
"Achird",
|
|
41
|
+
"Zubenelgenubi",
|
|
42
|
+
"Vindemiatrix",
|
|
43
|
+
"Sadachbia",
|
|
44
|
+
"Sadaltager",
|
|
45
|
+
"Sulafat"
|
|
46
|
+
]);
|
|
47
|
+
function toMintBody(session) {
|
|
48
|
+
if (session.toolChoice === "none" || session.toolChoice === "required") {
|
|
49
|
+
throw new CharivoStateError(
|
|
50
|
+
`Gemini Live does not support toolChoice "${session.toolChoice}"`
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
if (session.inputAudioTranscription?.model !== void 0) {
|
|
54
|
+
throw new CharivoStateError(
|
|
55
|
+
`Gemini Live does not support inputAudioTranscription.model "${session.inputAudioTranscription.model}"`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
const voice = session.voice && ALLOWED_VOICES.has(session.voice) ? session.voice : DEFAULT_GEMINI_LIVE_VOICE;
|
|
59
|
+
const generationConfig = {
|
|
60
|
+
responseModalities: ["AUDIO"],
|
|
61
|
+
speechConfig: {
|
|
62
|
+
voiceConfig: {
|
|
63
|
+
prebuiltVoiceConfig: { voiceName: voice }
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
if (session.maxTokens !== void 0) {
|
|
68
|
+
generationConfig.maxOutputTokens = session.maxTokens;
|
|
69
|
+
}
|
|
70
|
+
const setup = {
|
|
71
|
+
model: `models/${session.model ?? DEFAULT_GEMINI_LIVE_MODEL}`,
|
|
72
|
+
generationConfig,
|
|
73
|
+
// The only assistant-text source on native-audio models.
|
|
74
|
+
outputAudioTranscription: {}
|
|
75
|
+
};
|
|
76
|
+
if (session.instructions) {
|
|
77
|
+
setup.systemInstruction = { parts: [{ text: session.instructions }] };
|
|
78
|
+
}
|
|
79
|
+
if (session.tools?.length) {
|
|
80
|
+
setup.tools = [
|
|
81
|
+
{
|
|
82
|
+
functionDeclarations: session.tools.map((tool) => ({
|
|
83
|
+
name: tool.name,
|
|
84
|
+
description: tool.description,
|
|
85
|
+
parameters: tool.parameters
|
|
86
|
+
}))
|
|
87
|
+
}
|
|
88
|
+
];
|
|
89
|
+
}
|
|
90
|
+
if (session.inputAudioTranscription?.enabled === true) {
|
|
91
|
+
setup.inputAudioTranscription = {};
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
uses: 1,
|
|
95
|
+
bidiGenerateContentSetup: setup
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function extractTokenName(payload) {
|
|
99
|
+
if (!isRecord(payload)) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
return typeof payload.name === "string" && payload.name.length > 0 ? payload.name : null;
|
|
103
|
+
}
|
|
104
|
+
function createGeminiLiveDevBootstrap(apiKey) {
|
|
105
|
+
return async (request) => {
|
|
106
|
+
const response = await fetchWithTimeout(
|
|
107
|
+
AUTH_TOKENS_URL,
|
|
108
|
+
{
|
|
109
|
+
method: "POST",
|
|
110
|
+
headers: {
|
|
111
|
+
// Never in the URL: proxies and request logs capture query strings.
|
|
112
|
+
"x-goog-api-key": apiKey,
|
|
113
|
+
"Content-Type": "application/json"
|
|
114
|
+
},
|
|
115
|
+
body: JSON.stringify(toMintBody(request.session))
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
timeoutMessage: `Gemini Live token request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,
|
|
119
|
+
failureMessage: "Realtime request failed"
|
|
120
|
+
}
|
|
121
|
+
);
|
|
122
|
+
if (!response.ok) {
|
|
123
|
+
const errorText = await response.text();
|
|
124
|
+
throw new CharivoProviderError(
|
|
125
|
+
`Failed to mint Gemini Live ephemeral token: ${errorText}`
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
let body;
|
|
129
|
+
try {
|
|
130
|
+
body = await response.json();
|
|
131
|
+
} catch {
|
|
132
|
+
throw new CharivoProviderError(
|
|
133
|
+
"Invalid ephemeral token response: response body could not be parsed as JSON"
|
|
134
|
+
);
|
|
135
|
+
}
|
|
136
|
+
const token = extractTokenName(body);
|
|
137
|
+
if (!token) {
|
|
138
|
+
throw new CharivoProviderError(
|
|
139
|
+
"Invalid ephemeral token response: name missing or empty"
|
|
140
|
+
);
|
|
141
|
+
}
|
|
142
|
+
return {
|
|
143
|
+
adapter: GEMINI_LIVE_ADAPTER,
|
|
144
|
+
transport: "websocket",
|
|
145
|
+
url: GEMINI_LIVE_WEBSOCKET_URL,
|
|
146
|
+
token
|
|
147
|
+
};
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// src/gemini/bootstrap.ts
|
|
152
|
+
async function getGeminiLiveBootstrap(options, request) {
|
|
153
|
+
if (options.sessionBootstrap) {
|
|
154
|
+
return options.sessionBootstrap(request);
|
|
155
|
+
}
|
|
156
|
+
const apiEndpoint = options.apiEndpoint;
|
|
157
|
+
if (!apiEndpoint && options.apiKey) {
|
|
158
|
+
return createGeminiLiveDevBootstrap(options.apiKey)(request);
|
|
159
|
+
}
|
|
160
|
+
if (!apiEndpoint) {
|
|
161
|
+
throw new CharivoStateError(
|
|
162
|
+
"Gemini Live client requires sessionBootstrap, apiEndpoint, or apiKey"
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
const response = await fetchWithTimeout(
|
|
166
|
+
apiEndpoint,
|
|
167
|
+
{
|
|
168
|
+
method: "POST",
|
|
169
|
+
headers: {
|
|
170
|
+
"Content-Type": "application/json"
|
|
171
|
+
},
|
|
172
|
+
body: JSON.stringify(request)
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
timeoutMessage: `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,
|
|
176
|
+
failureMessage: "Realtime request failed"
|
|
177
|
+
}
|
|
178
|
+
);
|
|
179
|
+
if (!response.ok) {
|
|
180
|
+
const errorText = await response.text();
|
|
181
|
+
throw new CharivoProviderError(
|
|
182
|
+
`Failed to create Realtime session: ${errorText}`
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
const bootstrap = await response.json();
|
|
186
|
+
if (!isRealtimeSessionBootstrap(bootstrap)) {
|
|
187
|
+
throw new CharivoProviderError(
|
|
188
|
+
"Invalid realtime session bootstrap response"
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
return bootstrap;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// src/gemini/capture.ts
|
|
195
|
+
var CAPTURE_PROCESSOR_NAME = "charivo-gemini-capture";
|
|
196
|
+
var CAPTURE_WORKLET_SOURCE = `
|
|
197
|
+
class CaptureProcessor extends AudioWorkletProcessor {
|
|
198
|
+
constructor(options) {
|
|
199
|
+
super();
|
|
200
|
+
this.ratio = sampleRate / options.processorOptions.targetRate;
|
|
201
|
+
this.frameSamples = options.processorOptions.frameSamples;
|
|
202
|
+
this.position = 0;
|
|
203
|
+
this.sum = 0;
|
|
204
|
+
this.count = 0;
|
|
205
|
+
this.pending = [];
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
process(inputs) {
|
|
209
|
+
const channel = inputs[0] && inputs[0][0];
|
|
210
|
+
if (!channel) {
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Box-average decimation. Not a proper anti-alias filter, but far better
|
|
215
|
+
// than picking every Nth sample, which folds speech harmonics back down.
|
|
216
|
+
// The position accumulator carries the fractional remainder across groups,
|
|
217
|
+
// so a device rate that is not a whole multiple of the target \u2014 44.1 kHz,
|
|
218
|
+
// say \u2014 alternates group sizes rather than drifting out of step.
|
|
219
|
+
for (let i = 0; i < channel.length; i += 1) {
|
|
220
|
+
this.sum += channel[i];
|
|
221
|
+
this.count += 1;
|
|
222
|
+
this.position += 1;
|
|
223
|
+
if (this.position >= this.ratio) {
|
|
224
|
+
this.position -= this.ratio;
|
|
225
|
+
this.pending.push(this.count > 0 ? this.sum / this.count : 0);
|
|
226
|
+
this.sum = 0;
|
|
227
|
+
this.count = 0;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
while (this.pending.length >= this.frameSamples) {
|
|
232
|
+
const chunk = this.pending.splice(0, this.frameSamples);
|
|
233
|
+
const frame = new Int16Array(chunk.length);
|
|
234
|
+
for (let i = 0; i < chunk.length; i += 1) {
|
|
235
|
+
const clamped = Math.max(-1, Math.min(1, chunk[i]));
|
|
236
|
+
frame[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
|
|
237
|
+
}
|
|
238
|
+
this.port.postMessage(frame.buffer, [frame.buffer]);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
registerProcessor("${CAPTURE_PROCESSOR_NAME}", CaptureProcessor);
|
|
246
|
+
`;
|
|
247
|
+
async function createCapturePipeline({
|
|
248
|
+
stream,
|
|
249
|
+
onFrame
|
|
250
|
+
}) {
|
|
251
|
+
const context = new AudioContext();
|
|
252
|
+
const workletUrl = URL.createObjectURL(
|
|
253
|
+
new Blob([CAPTURE_WORKLET_SOURCE], { type: "application/javascript" })
|
|
254
|
+
);
|
|
255
|
+
try {
|
|
256
|
+
await context.audioWorklet.addModule(workletUrl);
|
|
257
|
+
const source = context.createMediaStreamSource(stream);
|
|
258
|
+
const capture = new AudioWorkletNode(context, CAPTURE_PROCESSOR_NAME, {
|
|
259
|
+
processorOptions: {
|
|
260
|
+
targetRate: INPUT_SAMPLE_RATE,
|
|
261
|
+
frameSamples: CAPTURE_FRAME_SAMPLES
|
|
262
|
+
}
|
|
263
|
+
});
|
|
264
|
+
source.connect(capture);
|
|
265
|
+
capture.port.onmessage = (event) => {
|
|
266
|
+
onFrame(new Uint8Array(event.data));
|
|
267
|
+
};
|
|
268
|
+
capture.onprocessorerror = () => {
|
|
269
|
+
console.error("The Gemini capture worklet stopped processing");
|
|
270
|
+
};
|
|
271
|
+
return {
|
|
272
|
+
stop() {
|
|
273
|
+
capture.port.onmessage = null;
|
|
274
|
+
capture.port.close();
|
|
275
|
+
source.disconnect();
|
|
276
|
+
capture.disconnect();
|
|
277
|
+
void closeContext(context);
|
|
278
|
+
}
|
|
279
|
+
};
|
|
280
|
+
} catch (error) {
|
|
281
|
+
void closeContext(context);
|
|
282
|
+
throw error;
|
|
283
|
+
} finally {
|
|
284
|
+
URL.revokeObjectURL(workletUrl);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
function closeContext(context) {
|
|
288
|
+
return context.close().catch(
|
|
289
|
+
(error) => console.error("Failed to close the capture audio context:", error)
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
// src/gemini/playback.ts
|
|
294
|
+
function createPlaybackGraph(context) {
|
|
295
|
+
const output = context.createGain();
|
|
296
|
+
output.connect(context.destination);
|
|
297
|
+
const tap = context.createMediaStreamDestination();
|
|
298
|
+
output.connect(tap);
|
|
299
|
+
return { output, lipSyncStream: tap.stream };
|
|
300
|
+
}
|
|
301
|
+
var PlaybackScheduler = class {
|
|
302
|
+
constructor(context, destination, callbacks) {
|
|
303
|
+
this.context = context;
|
|
304
|
+
this.destination = destination;
|
|
305
|
+
this.callbacks = callbacks;
|
|
306
|
+
}
|
|
307
|
+
nextStartTime = 0;
|
|
308
|
+
/**
|
|
309
|
+
* Bumped by `flush()`. Every source captures the value it was scheduled
|
|
310
|
+
* under, so a discarded turn's late `onended` cannot report against the turn
|
|
311
|
+
* that replaced it.
|
|
312
|
+
*/
|
|
313
|
+
generation = 0;
|
|
314
|
+
active = /* @__PURE__ */ new Set();
|
|
315
|
+
/**
|
|
316
|
+
* Whether the scheduler has nothing left to play — nothing sounding, and
|
|
317
|
+
* nothing queued behind it. Derived from the live source set on
|
|
318
|
+
* every call rather than remembered: `enqueue()` clears idleness by adding to
|
|
319
|
+
* that set, so a caller cannot be holding a "drained" answer that the next
|
|
320
|
+
* chunk already invalidated.
|
|
321
|
+
*/
|
|
322
|
+
isIdle() {
|
|
323
|
+
return this.active.size === 0;
|
|
324
|
+
}
|
|
325
|
+
/** Schedule one `audio/pcm;rate=24000` chunk after everything already queued. */
|
|
326
|
+
enqueue(pcm) {
|
|
327
|
+
const sampleCount = Math.floor(pcm.byteLength / 2);
|
|
328
|
+
if (sampleCount === 0) {
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
const generation = this.generation;
|
|
332
|
+
const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
|
|
333
|
+
const buffer = this.context.createBuffer(
|
|
334
|
+
1,
|
|
335
|
+
sampleCount,
|
|
336
|
+
OUTPUT_SAMPLE_RATE
|
|
337
|
+
);
|
|
338
|
+
const channel = buffer.getChannelData(0);
|
|
339
|
+
for (let index = 0; index < sampleCount; index += 1) {
|
|
340
|
+
channel[index] = view.getInt16(index * 2, true) / 32768;
|
|
341
|
+
}
|
|
342
|
+
const source = this.context.createBufferSource();
|
|
343
|
+
source.buffer = buffer;
|
|
344
|
+
source.connect(this.destination);
|
|
345
|
+
source.onended = () => {
|
|
346
|
+
this.active.delete(source);
|
|
347
|
+
if (generation !== this.generation || this.active.size > 0) {
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
this.callbacks.onPlayingChange(false);
|
|
351
|
+
this.callbacks.onDrain();
|
|
352
|
+
};
|
|
353
|
+
const wasIdle = this.isIdle();
|
|
354
|
+
const startAt = Math.max(this.context.currentTime, this.nextStartTime);
|
|
355
|
+
this.active.add(source);
|
|
356
|
+
source.start(startAt);
|
|
357
|
+
this.nextStartTime = startAt + buffer.duration;
|
|
358
|
+
if (wasIdle) {
|
|
359
|
+
this.callbacks.onPlayingChange(true);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
/** Discard everything scheduled, audible or merely queued. */
|
|
363
|
+
flush() {
|
|
364
|
+
this.generation += 1;
|
|
365
|
+
const wasPlaying = !this.isIdle();
|
|
366
|
+
for (const source of this.active) {
|
|
367
|
+
source.onended = null;
|
|
368
|
+
try {
|
|
369
|
+
source.stop();
|
|
370
|
+
} catch {
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
this.active.clear();
|
|
374
|
+
this.nextStartTime = 0;
|
|
375
|
+
if (wasPlaying) {
|
|
376
|
+
this.callbacks.onPlayingChange(false);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
};
|
|
380
|
+
|
|
381
|
+
// src/gemini/client.ts
|
|
382
|
+
var GeminiLiveClient = class {
|
|
383
|
+
constructor(options = {}) {
|
|
384
|
+
this.options = options;
|
|
385
|
+
}
|
|
386
|
+
activeSocket = null;
|
|
387
|
+
/** Bumped by every transient reset; see `GeminiSocketBinding`. */
|
|
388
|
+
socketEpoch = 0;
|
|
389
|
+
pendingConnect = null;
|
|
390
|
+
lipSyncAnalyzer = createLipSyncAnalyzer({
|
|
391
|
+
onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
|
|
392
|
+
onError: (error) => console.error("Failed to setup audio analysis:", error)
|
|
393
|
+
});
|
|
394
|
+
/**
|
|
395
|
+
* Created once, inside a user gesture, and reused for the client's whole
|
|
396
|
+
* life. Only the terminal `cleanup()` closes it.
|
|
397
|
+
*/
|
|
398
|
+
playbackContext = null;
|
|
399
|
+
playbackScheduler = null;
|
|
400
|
+
/**
|
|
401
|
+
* Per-session, unlike the playback context above: both the stream and the
|
|
402
|
+
* pipeline reading it are acquired by every `connect()` and dropped by the
|
|
403
|
+
* transient reset.
|
|
404
|
+
*/
|
|
405
|
+
captureStream = null;
|
|
406
|
+
capturePipeline = null;
|
|
407
|
+
/**
|
|
408
|
+
* Whether the server has acknowledged setup. Microphone frames wait on it —
|
|
409
|
+
* see `sendCaptureFrame`.
|
|
410
|
+
*/
|
|
411
|
+
isSetupComplete = false;
|
|
412
|
+
/**
|
|
413
|
+
* Convergence gate. Safari's echo canceller leaks the character's own voice
|
|
414
|
+
* for roughly the first half-second after it starts speaking, and converges
|
|
415
|
+
* cumulatively across turns (measured,
|
|
416
|
+
* `tests/gemini-live-smoke/README.md`), so while armed the client drops the
|
|
417
|
+
* microphone frames landing within `CONVERGENCE_GATE_MS` of playback becoming
|
|
418
|
+
* audible.
|
|
419
|
+
*
|
|
420
|
+
* Dropping them does not slow that adaptation down: the canceller runs inside
|
|
421
|
+
* the `getUserMedia` pipeline and adapts to whatever plays out, forwarded or
|
|
422
|
+
* not. The gate only keeps the not-yet-converged residue from reaching the
|
|
423
|
+
* model, at a bounded cost — a barge-in inside the gate window at the start
|
|
424
|
+
* of a reply is lost, against the character killing its own turn twice.
|
|
425
|
+
*
|
|
426
|
+
* Disarmed by accumulated audible exposure, in `endAudioOutput`.
|
|
427
|
+
*/
|
|
428
|
+
gateArmed = true;
|
|
429
|
+
/**
|
|
430
|
+
* `performance.now()` at the moment the current turn's playback became
|
|
431
|
+
* audible, and the only thing the gate window is measured from: a fact this
|
|
432
|
+
* client owns, never a guess about what the microphone contains. Re-anchored
|
|
433
|
+
* at every turn's first audible playback by `beginAudiblePlayback`; read by
|
|
434
|
+
* `sendCaptureFrame` and cleared by the transient reset.
|
|
435
|
+
*/
|
|
436
|
+
playbackAudibleSince = null;
|
|
437
|
+
/**
|
|
438
|
+
* How much audible playback the echo canceller has had to adapt to, summed
|
|
439
|
+
* over the scheduler's playing → not-playing intervals for the whole session
|
|
440
|
+
* and the only quantity the gate disarms on. `audibleSpanStartedAt` is the
|
|
441
|
+
* open interval, or null while nothing is sounding.
|
|
442
|
+
*/
|
|
443
|
+
audibleExposureMs = 0;
|
|
444
|
+
audibleSpanStartedAt = null;
|
|
445
|
+
/**
|
|
446
|
+
* Assistant-side turn state. Output transcription is native audio's only
|
|
447
|
+
* source of assistant text, and both fields are cleared at `turnComplete`.
|
|
448
|
+
*/
|
|
449
|
+
hasStartedAssistantResponse = false;
|
|
450
|
+
assistantText = "";
|
|
451
|
+
/**
|
|
452
|
+
* Audio bookkeeping, which deliberately outlives `turnComplete`: in the
|
|
453
|
+
* measured normal sequence the final drain lands ~3 ms *after* it, so both
|
|
454
|
+
* are cleared where `audio.output.ended` actually fires.
|
|
455
|
+
*
|
|
456
|
+
* `hasStartedAudioOutput` is what keeps the pair balanced — every ending path
|
|
457
|
+
* requires it and clears it, so a raw transport listener sees exactly one
|
|
458
|
+
* start/end per turn. `turnCompleteSeen` means an ending is still owed, and
|
|
459
|
+
* is the second half of `ended` = currently idle AND `turnComplete` seen. The
|
|
460
|
+
* transient reset drops both, along with the turn they described.
|
|
461
|
+
*/
|
|
462
|
+
hasStartedAudioOutput = false;
|
|
463
|
+
turnCompleteSeen = false;
|
|
464
|
+
/**
|
|
465
|
+
* Whether server VAD killed this turn. Set by `handleInterrupted`, then read
|
|
466
|
+
* and cleared by the `endAudioOutput` it goes on to call: an interrupted turn
|
|
467
|
+
* banks its audible exposure like any other but never disarms the gate.
|
|
468
|
+
*/
|
|
469
|
+
turnInterrupted = false;
|
|
470
|
+
/**
|
|
471
|
+
* Whether the server has a model turn open — content has arrived and its
|
|
472
|
+
* `turnComplete` has not. It is the only proof `interrupt()` accepts that
|
|
473
|
+
* there is a turn to condemn, and neither existing flag answers that
|
|
474
|
+
* question: `hasStartedAssistantResponse` follows output transcription, which
|
|
475
|
+
* is a server-side session setting this client does not control, and the
|
|
476
|
+
* audio flags deliberately outlive `turnComplete`.
|
|
477
|
+
*/
|
|
478
|
+
modelTurnInFlight = false;
|
|
479
|
+
/**
|
|
480
|
+
* Whether a local `interrupt()` killed the turn in flight. Nothing on the
|
|
481
|
+
* wire stops the server generating it, so while this is set the rest of that
|
|
482
|
+
* turn is dropped at the guard in `applyServerMessage`, which is where what
|
|
483
|
+
* "the rest" covers is settled, and its completion is never reported:
|
|
484
|
+
* `RealtimeTransportEvent` forbids reporting an interrupted turn once a
|
|
485
|
+
* replacement may have started, because `RealtimeManager` would credit that
|
|
486
|
+
* completion to the replacement and release its send lock.
|
|
487
|
+
* `handleTurnComplete` is the single exit.
|
|
488
|
+
*
|
|
489
|
+
* A boolean where `openai/client.ts` learned to keep a count. That lesson
|
|
490
|
+
* does not transfer: it can issue `response.create` itself and stack a
|
|
491
|
+
* replacement behind a cancel, whereas turns here are the server's to open
|
|
492
|
+
* and it opens them one at a time, so there is never more than one to
|
|
493
|
+
* condemn.
|
|
494
|
+
*/
|
|
495
|
+
turnCondemned = false;
|
|
496
|
+
/**
|
|
497
|
+
* Live `functionCall.id` -> tool name, which `sendToolResult` needs because a
|
|
498
|
+
* Gemini `functionResponse` carries the name alongside the id. An entry lives
|
|
499
|
+
* from the `toolCall` frame until that call is answered or cancelled, and the
|
|
500
|
+
* map as a whole belongs to the session: the transient reset drops it, so a
|
|
501
|
+
* handler finishing after a reconnect throws instead of answering into a
|
|
502
|
+
* session that never asked.
|
|
503
|
+
*/
|
|
504
|
+
pendingToolCalls = /* @__PURE__ */ new Map();
|
|
505
|
+
/**
|
|
506
|
+
* Newest `usageMetadata` of the turn in progress, reported as `usage` when it
|
|
507
|
+
* completes and dropped there. Nothing measured guarantees one frame per
|
|
508
|
+
* turn, so carrying it forward would report the previous turn's numbers as
|
|
509
|
+
* this one's — and since audio prompt tokens grow turn over turn, it would
|
|
510
|
+
* under-report rather than fail visibly.
|
|
511
|
+
*/
|
|
512
|
+
latestUsage;
|
|
513
|
+
currentSessionConfig;
|
|
514
|
+
connectionLossNotified = false;
|
|
515
|
+
isExplicitDisconnect = false;
|
|
516
|
+
isRecovering = false;
|
|
517
|
+
/**
|
|
518
|
+
* Set only for the span of the terminal `cleanup()`. It is what stops a
|
|
519
|
+
* teardown step from reporting the session it is dismantling as lost — most
|
|
520
|
+
* visibly on a failed initial connect, where the manager would hear
|
|
521
|
+
* `connection.lost` for a session that never started.
|
|
522
|
+
*/
|
|
523
|
+
isCleaningUp = false;
|
|
524
|
+
eventCallbacks = /* @__PURE__ */ new Set();
|
|
525
|
+
async connect(config) {
|
|
526
|
+
try {
|
|
527
|
+
if (this.isExplicitDisconnect && this.isRecovering) {
|
|
528
|
+
throw new Error("Realtime session disconnect requested");
|
|
529
|
+
}
|
|
530
|
+
this.log("Starting Gemini Live websocket connection");
|
|
531
|
+
this.isExplicitDisconnect = false;
|
|
532
|
+
this.currentSessionConfig = config;
|
|
533
|
+
const bootstrap = await this.getSessionBootstrap({
|
|
534
|
+
transport: "websocket",
|
|
535
|
+
session: config ?? {}
|
|
536
|
+
});
|
|
537
|
+
const { url, token } = resolveWebSocketBootstrap(bootstrap);
|
|
538
|
+
if (this.isExplicitDisconnect) {
|
|
539
|
+
throw new Error("Realtime session disconnect requested");
|
|
540
|
+
}
|
|
541
|
+
await this.ensurePlaybackReady();
|
|
542
|
+
try {
|
|
543
|
+
this.captureStream = await acquireMicrophoneStream();
|
|
544
|
+
} catch (error) {
|
|
545
|
+
throw new Error("Microphone access required for Realtime API", {
|
|
546
|
+
cause: error
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
if (this.isExplicitDisconnect) {
|
|
550
|
+
throw new Error("Realtime session disconnect requested");
|
|
551
|
+
}
|
|
552
|
+
const capturePipeline = await createCapturePipeline({
|
|
553
|
+
stream: this.captureStream,
|
|
554
|
+
onFrame: (frame) => this.sendCaptureFrame(frame)
|
|
555
|
+
});
|
|
556
|
+
if (this.isExplicitDisconnect) {
|
|
557
|
+
capturePipeline.stop();
|
|
558
|
+
throw new Error("Realtime session disconnect requested");
|
|
559
|
+
}
|
|
560
|
+
this.capturePipeline = capturePipeline;
|
|
561
|
+
await this.openSocket(url, token, config?.model);
|
|
562
|
+
this.connectionLossNotified = false;
|
|
563
|
+
this.emitEvent({ type: "session.started" });
|
|
564
|
+
} catch (error) {
|
|
565
|
+
if (this.isRecovering) {
|
|
566
|
+
this.resetTransportState();
|
|
567
|
+
} else {
|
|
568
|
+
this.cleanup();
|
|
569
|
+
}
|
|
570
|
+
throw error;
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
async disconnect() {
|
|
574
|
+
this.log("Disconnecting Gemini Live websocket");
|
|
575
|
+
this.isExplicitDisconnect = true;
|
|
576
|
+
this.isRecovering = false;
|
|
577
|
+
this.cleanup();
|
|
578
|
+
}
|
|
579
|
+
async updateSession(_config) {
|
|
580
|
+
throw new Error(
|
|
581
|
+
"updateSession() is not supported on an open Gemini Live session. Reconnect with the new configuration instead."
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
async recover(config) {
|
|
585
|
+
this.isRecovering = true;
|
|
586
|
+
this.connectionLossNotified = true;
|
|
587
|
+
this.currentSessionConfig = config ?? this.currentSessionConfig;
|
|
588
|
+
try {
|
|
589
|
+
this.resetTransportState();
|
|
590
|
+
await this.connect(this.currentSessionConfig);
|
|
591
|
+
} finally {
|
|
592
|
+
this.isRecovering = false;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
async sendText(text) {
|
|
596
|
+
if (this.isRecovering) {
|
|
597
|
+
throw new Error("Realtime transport reconnecting");
|
|
598
|
+
}
|
|
599
|
+
const socket = this.requireReadySocket();
|
|
600
|
+
socket.send(
|
|
601
|
+
JSON.stringify({
|
|
602
|
+
clientContent: {
|
|
603
|
+
turns: [{ role: "user", parts: [{ text }] }],
|
|
604
|
+
turnComplete: true
|
|
605
|
+
}
|
|
606
|
+
})
|
|
607
|
+
);
|
|
608
|
+
}
|
|
609
|
+
/**
|
|
610
|
+
* Stop the character mid-sentence.
|
|
611
|
+
*
|
|
612
|
+
* Entirely local, and it has to be: the Live API documents no client message
|
|
613
|
+
* that cancels an in-flight generation. So the turn is condemned instead,
|
|
614
|
+
* which rests on the killed turn still running to its own `turnComplete` —
|
|
615
|
+
* the single exit from condemnation. That was measured rather than assumed
|
|
616
|
+
* (`tests/gemini-live-smoke/README.md`, Q2): interrupting mid-count emitted
|
|
617
|
+
* no completion for the killed turn, and the very next turn completed
|
|
618
|
+
* normally, which it could not have done had the condemnation never lifted.
|
|
619
|
+
* What has been scheduled is discarded, and its remaining speech and
|
|
620
|
+
* text are dropped — though not its tool calls, which `applyServerMessage`
|
|
621
|
+
* still dispatches on purpose. Nothing goes on the wire, which is why this
|
|
622
|
+
* needs neither an open socket nor a recovery guard.
|
|
623
|
+
*/
|
|
624
|
+
async interrupt() {
|
|
625
|
+
if (this.modelTurnInFlight) {
|
|
626
|
+
this.turnCondemned = true;
|
|
627
|
+
this.log("Gemini Live turn condemned by a local interrupt");
|
|
628
|
+
}
|
|
629
|
+
this.playbackScheduler?.flush();
|
|
630
|
+
this.endAudioOutput();
|
|
631
|
+
}
|
|
632
|
+
async sendToolResult(callId, output) {
|
|
633
|
+
if (this.isRecovering) {
|
|
634
|
+
throw new Error("Realtime transport reconnecting");
|
|
635
|
+
}
|
|
636
|
+
const name = this.pendingToolCalls.get(callId);
|
|
637
|
+
if (!name) {
|
|
638
|
+
throw new Error(
|
|
639
|
+
`Unknown Gemini Live tool call "${callId}": it was already answered, cancelled by the server, or belongs to a session that has ended`
|
|
640
|
+
);
|
|
641
|
+
}
|
|
642
|
+
const socket = this.requireReadySocket();
|
|
643
|
+
socket.send(
|
|
644
|
+
JSON.stringify({
|
|
645
|
+
toolResponse: {
|
|
646
|
+
functionResponses: [{ id: callId, name, response: output }]
|
|
647
|
+
}
|
|
648
|
+
})
|
|
649
|
+
);
|
|
650
|
+
this.pendingToolCalls.delete(callId);
|
|
651
|
+
}
|
|
652
|
+
async sendAudio(_audio) {
|
|
653
|
+
console.warn(
|
|
654
|
+
"sendAudio is not needed with the Gemini Live transport - the microphone is captured internally"
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
onEvent(callback) {
|
|
658
|
+
this.eventCallbacks.add(callback);
|
|
659
|
+
}
|
|
660
|
+
async prepareAudio() {
|
|
661
|
+
await this.ensurePlaybackReady();
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* Build the playback context, graph, scheduler, and lip-sync analyzer once,
|
|
665
|
+
* and afterwards only nudge that same playback context back towards running.
|
|
666
|
+
* It does not promise a running context on return; see the resume below.
|
|
667
|
+
*
|
|
668
|
+
* Both `prepareAudio()` and `connect()` come here, and the order they arrive
|
|
669
|
+
* in is the point: whichever runs first builds everything, the other reuses
|
|
670
|
+
* it. That is what keeps the `AudioContext` the one created inside a user
|
|
671
|
+
* gesture instead of a replacement built during an automatic reconnect,
|
|
672
|
+
* which the browser would refuse to start.
|
|
673
|
+
*
|
|
674
|
+
* The playback context is the only one this covers, and that leaves a gap
|
|
675
|
+
* worth knowing about: `lipSyncAnalyzer.prepare()` creates a second
|
|
676
|
+
* `AudioContext` of its own, and nothing in `@charivo/core` ever resumes it.
|
|
677
|
+
* A suspended `AnalyserNode` returns all-zero frequency data, so in exactly
|
|
678
|
+
* the case the nudge below exists for — a caller that skipped
|
|
679
|
+
* `prepareAudio()` — lip sync reads flat and no error surfaces anywhere.
|
|
680
|
+
* Pre-existing and identical in `openai/client.ts`; closing it means changing
|
|
681
|
+
* the analyzer contract, not this file.
|
|
682
|
+
*/
|
|
683
|
+
async ensurePlaybackReady() {
|
|
684
|
+
const existing = this.playbackContext;
|
|
685
|
+
if (existing) {
|
|
686
|
+
if (existing.state !== "running") {
|
|
687
|
+
void existing.resume().catch(
|
|
688
|
+
(error) => console.error(
|
|
689
|
+
"Failed to resume the playback audio context:",
|
|
690
|
+
error
|
|
691
|
+
)
|
|
692
|
+
);
|
|
693
|
+
}
|
|
694
|
+
return;
|
|
695
|
+
}
|
|
696
|
+
const context = new AudioContext({ sampleRate: OUTPUT_SAMPLE_RATE });
|
|
697
|
+
const graph = createPlaybackGraph(context);
|
|
698
|
+
this.playbackContext = context;
|
|
699
|
+
this.playbackScheduler = new PlaybackScheduler(context, graph.output, {
|
|
700
|
+
onDrain: () => {
|
|
701
|
+
if (this.turnCompleteSeen) {
|
|
702
|
+
this.endAudioOutput();
|
|
703
|
+
}
|
|
704
|
+
},
|
|
705
|
+
onPlayingChange: (playing) => {
|
|
706
|
+
if (playing) {
|
|
707
|
+
this.beginAudiblePlayback();
|
|
708
|
+
} else {
|
|
709
|
+
this.bankAudibleExposure();
|
|
710
|
+
}
|
|
711
|
+
}
|
|
712
|
+
});
|
|
713
|
+
await this.lipSyncAnalyzer.prepare();
|
|
714
|
+
this.lipSyncAnalyzer.attachMediaStream(graph.lipSyncStream);
|
|
715
|
+
this.lipSyncAnalyzer.pause();
|
|
716
|
+
}
|
|
717
|
+
openSocket(url, token, model) {
|
|
718
|
+
return new Promise((resolve, reject) => {
|
|
719
|
+
let socket;
|
|
720
|
+
try {
|
|
721
|
+
socket = new WebSocket(
|
|
722
|
+
`${url}?access_token=${encodeURIComponent(token)}`
|
|
723
|
+
);
|
|
724
|
+
} catch {
|
|
725
|
+
reject(new Error("Failed to open the Gemini Live websocket"));
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
const binding = {
|
|
729
|
+
socket,
|
|
730
|
+
epoch: this.socketEpoch,
|
|
731
|
+
pump: Promise.resolve()
|
|
732
|
+
};
|
|
733
|
+
this.activeSocket = binding;
|
|
734
|
+
this.pendingConnect = {
|
|
735
|
+
resolve,
|
|
736
|
+
reject,
|
|
737
|
+
timeoutId: setTimeout(() => {
|
|
738
|
+
this.rejectPendingConnect(
|
|
739
|
+
new Error(
|
|
740
|
+
`Gemini Live session setup timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
|
|
741
|
+
)
|
|
742
|
+
);
|
|
743
|
+
}, DEFAULT_REQUEST_TIMEOUT_MS)
|
|
744
|
+
};
|
|
745
|
+
socket.onopen = () => {
|
|
746
|
+
this.log("Gemini Live websocket open \u2014 sending setup");
|
|
747
|
+
socket.send(
|
|
748
|
+
JSON.stringify({
|
|
749
|
+
setup: { model: `models/${model ?? DEFAULT_GEMINI_LIVE_MODEL}` }
|
|
750
|
+
})
|
|
751
|
+
);
|
|
752
|
+
};
|
|
753
|
+
socket.onmessage = (event) => {
|
|
754
|
+
this.enqueueServerMessage(binding, event.data);
|
|
755
|
+
};
|
|
756
|
+
socket.onerror = () => {
|
|
757
|
+
this.failTransport(binding, new Error("Gemini Live websocket error"));
|
|
758
|
+
};
|
|
759
|
+
socket.onclose = (event) => {
|
|
760
|
+
this.log(
|
|
761
|
+
`Gemini Live websocket closed code=${event.code} reason="${event.reason}" clean=${event.wasClean}`
|
|
762
|
+
);
|
|
763
|
+
this.failTransport(
|
|
764
|
+
binding,
|
|
765
|
+
new Error(
|
|
766
|
+
`Gemini Live websocket closed (code ${event.code}${event.reason ? `: ${event.reason}` : ""})`
|
|
767
|
+
)
|
|
768
|
+
);
|
|
769
|
+
};
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
enqueueServerMessage(binding, data) {
|
|
773
|
+
if (binding.epoch !== this.socketEpoch) {
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
binding.pump = binding.pump.then(async () => {
|
|
777
|
+
const raw = typeof data === "string" ? data : await data.text();
|
|
778
|
+
if (binding.epoch !== this.socketEpoch) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
this.applyServerMessage(JSON.parse(raw));
|
|
782
|
+
}).catch((error) => {
|
|
783
|
+
this.failTransport(
|
|
784
|
+
binding,
|
|
785
|
+
new Error("Failed to handle a Gemini Live message", { cause: error })
|
|
786
|
+
);
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
applyServerMessage(message) {
|
|
790
|
+
if (message.setupComplete) {
|
|
791
|
+
this.log("Gemini Live setup complete");
|
|
792
|
+
this.isSetupComplete = true;
|
|
793
|
+
this.resolvePendingConnect();
|
|
794
|
+
return;
|
|
795
|
+
}
|
|
796
|
+
if (message.usageMetadata) {
|
|
797
|
+
this.latestUsage = message.usageMetadata;
|
|
798
|
+
}
|
|
799
|
+
if (message.toolCall) {
|
|
800
|
+
this.handleToolCall(message.toolCall.functionCalls ?? []);
|
|
801
|
+
}
|
|
802
|
+
for (const id of message.toolCallCancellation?.ids ?? []) {
|
|
803
|
+
this.pendingToolCalls.delete(id);
|
|
804
|
+
}
|
|
805
|
+
const content = message.serverContent;
|
|
806
|
+
if (!content) {
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
if (content.inputTranscription?.text) {
|
|
810
|
+
this.emitEvent({
|
|
811
|
+
type: "user.transcript",
|
|
812
|
+
text: content.inputTranscription.text
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
if (this.turnCondemned) {
|
|
816
|
+
if (content.turnComplete) {
|
|
817
|
+
this.handleTurnComplete();
|
|
818
|
+
}
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
if (content.outputTranscription?.text || content.modelTurn) {
|
|
822
|
+
this.modelTurnInFlight = true;
|
|
823
|
+
}
|
|
824
|
+
if (content.outputTranscription?.text) {
|
|
825
|
+
this.appendAssistantText(content.outputTranscription.text);
|
|
826
|
+
}
|
|
827
|
+
for (const part of content.modelTurn?.parts ?? []) {
|
|
828
|
+
if (part.inlineData?.data) {
|
|
829
|
+
this.playbackScheduler?.enqueue(base64ToBytes(part.inlineData.data));
|
|
830
|
+
}
|
|
831
|
+
}
|
|
832
|
+
if (content.interrupted) {
|
|
833
|
+
this.handleInterrupted();
|
|
834
|
+
}
|
|
835
|
+
if (content.turnComplete) {
|
|
836
|
+
this.handleTurnComplete();
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
/**
|
|
840
|
+
* Server VAD cut the turn off. The accumulated text deliberately survives:
|
|
841
|
+
* the turn still reports `assistant.response.completed` at its own
|
|
842
|
+
* `turnComplete` with whatever it managed to say, which is the shape the
|
|
843
|
+
* OpenAI client already has for a VAD-cancelled response reporting its
|
|
844
|
+
* `response.done`.
|
|
845
|
+
*/
|
|
846
|
+
handleInterrupted() {
|
|
847
|
+
const offset = this.hasStartedAudioOutput && this.playbackAudibleSince !== null ? `${Math.round(performance.now() - this.playbackAudibleSince)}ms after playback started` : "with nothing playing";
|
|
848
|
+
this.log(`Gemini Live turn interrupted ${offset}`);
|
|
849
|
+
this.turnInterrupted = true;
|
|
850
|
+
this.modelTurnInFlight = false;
|
|
851
|
+
this.playbackScheduler?.flush();
|
|
852
|
+
this.endAudioOutput();
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Settle the turn that `turnComplete` just closed.
|
|
856
|
+
*
|
|
857
|
+
* Two callers, two worlds. `applyServerMessage`'s condemned branch brings a
|
|
858
|
+
* turn a local `interrupt()` killed: it reports nothing and its audio ended
|
|
859
|
+
* back at the interrupt. The normal path brings a turn that ran: it reports
|
|
860
|
+
* its completion, and may still owe an ending. Everything between is per-turn
|
|
861
|
+
* state both worlds drop.
|
|
862
|
+
*/
|
|
863
|
+
handleTurnComplete() {
|
|
864
|
+
const wasCondemned = this.turnCondemned;
|
|
865
|
+
this.turnCondemned = false;
|
|
866
|
+
if (!wasCondemned) {
|
|
867
|
+
this.emitEvent({
|
|
868
|
+
type: "assistant.response.completed",
|
|
869
|
+
text: this.assistantText,
|
|
870
|
+
usage: this.latestUsage
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
this.modelTurnInFlight = false;
|
|
874
|
+
this.assistantText = "";
|
|
875
|
+
this.hasStartedAssistantResponse = false;
|
|
876
|
+
this.latestUsage = void 0;
|
|
877
|
+
this.turnCompleteSeen = this.hasStartedAudioOutput;
|
|
878
|
+
if (!this.turnCompleteSeen) {
|
|
879
|
+
return;
|
|
880
|
+
}
|
|
881
|
+
if (this.playbackScheduler?.isIdle()) {
|
|
882
|
+
this.endAudioOutput();
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
/**
|
|
886
|
+
* Fan a `toolCall` frame out, one `tool.call` per usable entry, recording the
|
|
887
|
+
* id -> name pairing each answer will need.
|
|
888
|
+
*
|
|
889
|
+
* This frame's shape was written from the API reference and confirmed live
|
|
890
|
+
* only afterwards (`tests/gemini-live-smoke/README.md`, 2026-09-01); the
|
|
891
|
+
* leniency stays. An entry missing either field is *reported* rather than
|
|
892
|
+
* trusted or dropped in silence: without both there is no answer this client
|
|
893
|
+
* can send,
|
|
894
|
+
* and a session that runs on while the model waits forever for a response
|
|
895
|
+
* fails invisibly. It travels as a transport `error`, which `RealtimeManager`
|
|
896
|
+
* handles without ending the session — releasing the send lock, so the user
|
|
897
|
+
* is not stuck. Killing the socket over it would instead livelock: recovery
|
|
898
|
+
* succeeds because the socket was never the problem, the model re-issues the
|
|
899
|
+
* same call on the fresh session, and every lap burns another `uses: 1`
|
|
900
|
+
* token. Strictness belongs on shapes that have been measured.
|
|
901
|
+
*
|
|
902
|
+
* The whole frame is decided before any of it is emitted, so a malformed
|
|
903
|
+
* entry cannot destroy a well-formed sibling whose handler is already
|
|
904
|
+
* running and whose id would be cleared out from under it. The report names
|
|
905
|
+
* the keys that did arrive — enough to read the real shape off one failure —
|
|
906
|
+
* and never quotes argument values.
|
|
907
|
+
*
|
|
908
|
+
* This covers the one absence worth surviving: an entry that arrived and
|
|
909
|
+
* cannot be answered. A payload that is not this shape at all — a null entry,
|
|
910
|
+
* a `functionCalls` that is not an array — still throws into the message
|
|
911
|
+
* pump's fatal catch, and nothing here parses defensively for it.
|
|
912
|
+
*/
|
|
913
|
+
handleToolCall(calls) {
|
|
914
|
+
const usable = [];
|
|
915
|
+
const unusable = [];
|
|
916
|
+
for (const call of calls) {
|
|
917
|
+
if (call.id && call.name) {
|
|
918
|
+
usable.push({ id: call.id, name: call.name, args: call.args ?? {} });
|
|
919
|
+
} else {
|
|
920
|
+
unusable.push(`{${Object.keys(call).join(", ")}}`);
|
|
921
|
+
}
|
|
922
|
+
}
|
|
923
|
+
for (const call of usable) {
|
|
924
|
+
this.pendingToolCalls.set(call.id, call.name);
|
|
925
|
+
this.emitEvent({
|
|
926
|
+
type: "tool.call",
|
|
927
|
+
name: call.name,
|
|
928
|
+
args: call.args,
|
|
929
|
+
callId: call.id
|
|
930
|
+
});
|
|
931
|
+
}
|
|
932
|
+
if (unusable.length > 0) {
|
|
933
|
+
this.emitEvent({
|
|
934
|
+
type: "error",
|
|
935
|
+
error: new Error(
|
|
936
|
+
`Gemini Live sent tool calls this client cannot answer: an id or name is missing (keys received: ${unusable.join(" ")})`
|
|
937
|
+
)
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
}
|
|
941
|
+
/**
|
|
942
|
+
* Output transcription is the only assistant text a native-audio session
|
|
943
|
+
* produces, and the server keeps it on, so the response's whole lifecycle
|
|
944
|
+
* hangs off these fragments.
|
|
945
|
+
*/
|
|
946
|
+
appendAssistantText(text) {
|
|
947
|
+
if (!this.hasStartedAssistantResponse) {
|
|
948
|
+
this.hasStartedAssistantResponse = true;
|
|
949
|
+
this.emitEvent({ type: "assistant.response.started" });
|
|
950
|
+
}
|
|
951
|
+
this.assistantText += text;
|
|
952
|
+
this.emitEvent({ type: "assistant.text.delta", text });
|
|
953
|
+
}
|
|
954
|
+
/**
|
|
955
|
+
* The scheduler started sounding.
|
|
956
|
+
*
|
|
957
|
+
* Only the first such transition of a turn opens the turn's audio. This fires
|
|
958
|
+
* again after every mid-turn underrun, and re-anchoring the gate on those
|
|
959
|
+
* would let a stalling connection hold it open for a whole reply — silently,
|
|
960
|
+
* which is the failure class the measured record warns about. Anchoring once
|
|
961
|
+
* per session is equally wrong: the measured Safari run was interrupted 0.5 s
|
|
962
|
+
* into its *second* turn. So the anchor is per turn, and it is left standing
|
|
963
|
+
* when playback stops, because the window has to expire on its own clock
|
|
964
|
+
* rather than on a silence the acoustic path has not caught up with yet.
|
|
965
|
+
*
|
|
966
|
+
* The unconditional first line is the other half of the pairing: it opens the
|
|
967
|
+
* audible span that `bankAudibleExposure` closes, once per sounding stretch
|
|
968
|
+
* rather than once per turn.
|
|
969
|
+
*/
|
|
970
|
+
beginAudiblePlayback() {
|
|
971
|
+
const now = performance.now();
|
|
972
|
+
this.audibleSpanStartedAt = now;
|
|
973
|
+
if (this.hasStartedAudioOutput) {
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
this.hasStartedAudioOutput = true;
|
|
977
|
+
this.playbackAudibleSince = now;
|
|
978
|
+
this.lipSyncAnalyzer.resume();
|
|
979
|
+
this.emitEvent({ type: "audio.output.started" });
|
|
980
|
+
}
|
|
981
|
+
/**
|
|
982
|
+
* Close the open audible span into the exposure total.
|
|
983
|
+
*
|
|
984
|
+
* Only these playing → not-playing intervals count towards the gate. A
|
|
985
|
+
* first-audible → `audio.output.ended` span would count silence the canceller
|
|
986
|
+
* never got to adapt to: `turnComplete` is paced to the audio's duration from
|
|
987
|
+
* the *first* chunk, so a burst-delivered turn sits quiet for seconds waiting
|
|
988
|
+
* on it — a measured 12 s span for 10.5 s of audio, and 6.5 s for 5.1 s.
|
|
989
|
+
*/
|
|
990
|
+
bankAudibleExposure() {
|
|
991
|
+
if (this.audibleSpanStartedAt === null) {
|
|
992
|
+
return;
|
|
993
|
+
}
|
|
994
|
+
this.audibleExposureMs += performance.now() - this.audibleSpanStartedAt;
|
|
995
|
+
this.audibleSpanStartedAt = null;
|
|
996
|
+
}
|
|
997
|
+
/**
|
|
998
|
+
* The single place `audio.output.ended` is emitted. Callers decide *that* the
|
|
999
|
+
* audio ended — always from the scheduler being idle and `turnComplete` seen,
|
|
1000
|
+
* never from a timer or an RMS reading — and this keeps the start/end pair
|
|
1001
|
+
* balanced and settles the gate.
|
|
1002
|
+
*
|
|
1003
|
+
* Both interrupt paths flush the scheduler immediately before calling in.
|
|
1004
|
+
* That flush is what closes the open audible span through
|
|
1005
|
+
* `onPlayingChange(false)`, so the gate below settles against a total that
|
|
1006
|
+
* includes it.
|
|
1007
|
+
*/
|
|
1008
|
+
endAudioOutput() {
|
|
1009
|
+
const wasInterrupted = this.turnInterrupted;
|
|
1010
|
+
this.turnInterrupted = false;
|
|
1011
|
+
if (!this.hasStartedAudioOutput) {
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
this.hasStartedAudioOutput = false;
|
|
1015
|
+
this.turnCompleteSeen = false;
|
|
1016
|
+
this.lipSyncAnalyzer.pause();
|
|
1017
|
+
this.emitEvent({ type: "audio.output.ended" });
|
|
1018
|
+
if (this.gateArmed && !wasInterrupted && this.audibleExposureMs >= CONVERGENCE_GATE_MS) {
|
|
1019
|
+
this.gateArmed = false;
|
|
1020
|
+
this.log(
|
|
1021
|
+
`Convergence gate disarmed after ${Math.round(this.audibleExposureMs)}ms of audible playback`
|
|
1022
|
+
);
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* The socket a frame can go out on right now, or null. `setupComplete` is
|
|
1027
|
+
* part of the answer: until the server acknowledges setup there is no session
|
|
1028
|
+
* to receive anything. Callers differ only in what they make of a null.
|
|
1029
|
+
*/
|
|
1030
|
+
getReadySocket() {
|
|
1031
|
+
const socket = this.activeSocket?.socket;
|
|
1032
|
+
if (!socket || !this.isSetupComplete || socket.readyState !== WebSocket.OPEN) {
|
|
1033
|
+
return null;
|
|
1034
|
+
}
|
|
1035
|
+
return socket;
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* `getReadySocket()` for the caller-driven frames, which refuse rather than
|
|
1039
|
+
* drop: `sendText()` and `sendToolResult()` have someone to tell.
|
|
1040
|
+
*
|
|
1041
|
+
* The recovery guard stays at those call sites instead of folding in here as
|
|
1042
|
+
* the OpenAI client's does, so a `sendToolResult()` during a reconnect
|
|
1043
|
+
* reports the reconnect rather than the unknown call id it would otherwise
|
|
1044
|
+
* hit — the map is already empty by then.
|
|
1045
|
+
*/
|
|
1046
|
+
requireReadySocket() {
|
|
1047
|
+
const socket = this.getReadySocket();
|
|
1048
|
+
if (!socket) {
|
|
1049
|
+
throw new Error("Gemini Live session is not ready to send");
|
|
1050
|
+
}
|
|
1051
|
+
return socket;
|
|
1052
|
+
}
|
|
1053
|
+
/**
|
|
1054
|
+
* The only path microphone audio takes to the wire.
|
|
1055
|
+
*
|
|
1056
|
+
* Frames are dropped rather than queued whenever `getReadySocket()` says they
|
|
1057
|
+
* cannot go out now: a frame held across a reconnect is audio the user spoke
|
|
1058
|
+
* into a session that has ended.
|
|
1059
|
+
*/
|
|
1060
|
+
sendCaptureFrame(frame) {
|
|
1061
|
+
const socket = this.getReadySocket();
|
|
1062
|
+
if (!socket) {
|
|
1063
|
+
return;
|
|
1064
|
+
}
|
|
1065
|
+
if (this.isWithinConvergenceGate()) {
|
|
1066
|
+
return;
|
|
1067
|
+
}
|
|
1068
|
+
socket.send(
|
|
1069
|
+
JSON.stringify({
|
|
1070
|
+
realtimeInput: {
|
|
1071
|
+
audio: {
|
|
1072
|
+
data: bytesToBase64(frame),
|
|
1073
|
+
mimeType: `audio/pcm;rate=${INPUT_SAMPLE_RATE}`
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
})
|
|
1077
|
+
);
|
|
1078
|
+
}
|
|
1079
|
+
/**
|
|
1080
|
+
* Whether this moment falls inside the convergence gate: still armed, and
|
|
1081
|
+
* within `CONVERGENCE_GATE_MS` of the current turn's playback becoming
|
|
1082
|
+
* audible.
|
|
1083
|
+
*/
|
|
1084
|
+
isWithinConvergenceGate() {
|
|
1085
|
+
return this.gateArmed && this.playbackAudibleSince !== null && performance.now() - this.playbackAudibleSince < CONVERGENCE_GATE_MS;
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* The single settlement point for every way a socket can fail.
|
|
1089
|
+
*
|
|
1090
|
+
* Before `setupComplete` the pending `connect()` owns the failure and settles
|
|
1091
|
+
* immediately — during the initial connect nothing is listening for
|
|
1092
|
+
* `connection.lost`, so waiting out the timeout would hang the caller, and
|
|
1093
|
+
* `connect()`'s catch runs the teardown at the depth the situation calls for.
|
|
1094
|
+
* After setup, the transient reset runs first so `recover()` starts from a
|
|
1095
|
+
* clean session, and the loss is reported once.
|
|
1096
|
+
*/
|
|
1097
|
+
failTransport(binding, error) {
|
|
1098
|
+
if (binding.epoch !== this.socketEpoch) {
|
|
1099
|
+
return;
|
|
1100
|
+
}
|
|
1101
|
+
if (this.pendingConnect) {
|
|
1102
|
+
this.rejectPendingConnect(error);
|
|
1103
|
+
return;
|
|
1104
|
+
}
|
|
1105
|
+
this.resetTransportState();
|
|
1106
|
+
this.emitConnectionLost(error);
|
|
1107
|
+
}
|
|
1108
|
+
/**
|
|
1109
|
+
* Transient teardown: everything belonging to one websocket session, and
|
|
1110
|
+
* nothing belonging to the user gesture. The prepared playback context and
|
|
1111
|
+
* the lip-sync analyzer deliberately survive, so an automatic reconnect does
|
|
1112
|
+
* not have to create and resume an `AudioContext` outside a user gesture.
|
|
1113
|
+
*/
|
|
1114
|
+
resetTransportState() {
|
|
1115
|
+
this.socketEpoch += 1;
|
|
1116
|
+
this.closeActiveSocket();
|
|
1117
|
+
this.isSetupComplete = false;
|
|
1118
|
+
this.rejectPendingConnect(
|
|
1119
|
+
new Error("Gemini Live session ended before setup completed")
|
|
1120
|
+
);
|
|
1121
|
+
this.lipSyncAnalyzer.pause();
|
|
1122
|
+
this.playbackScheduler?.flush();
|
|
1123
|
+
this.capturePipeline?.stop();
|
|
1124
|
+
this.capturePipeline = null;
|
|
1125
|
+
this.captureStream?.getTracks().forEach((track) => track.stop());
|
|
1126
|
+
this.captureStream = null;
|
|
1127
|
+
this.gateArmed = true;
|
|
1128
|
+
this.playbackAudibleSince = null;
|
|
1129
|
+
this.audibleExposureMs = 0;
|
|
1130
|
+
this.audibleSpanStartedAt = null;
|
|
1131
|
+
this.hasStartedAudioOutput = false;
|
|
1132
|
+
this.turnCompleteSeen = false;
|
|
1133
|
+
this.turnInterrupted = false;
|
|
1134
|
+
this.modelTurnInFlight = false;
|
|
1135
|
+
this.turnCondemned = false;
|
|
1136
|
+
this.hasStartedAssistantResponse = false;
|
|
1137
|
+
this.assistantText = "";
|
|
1138
|
+
this.latestUsage = void 0;
|
|
1139
|
+
this.pendingToolCalls.clear();
|
|
1140
|
+
}
|
|
1141
|
+
/**
|
|
1142
|
+
* Terminal teardown: the transient reset plus everything the user gesture
|
|
1143
|
+
* warmed up. Only an explicit `disconnect()` and a failed initial `connect()`
|
|
1144
|
+
* come here; every recovery path takes the transient reset instead.
|
|
1145
|
+
*/
|
|
1146
|
+
cleanup() {
|
|
1147
|
+
this.isCleaningUp = true;
|
|
1148
|
+
this.resetTransportState();
|
|
1149
|
+
this.lipSyncAnalyzer.cleanup().catch(
|
|
1150
|
+
(error) => console.error("Failed to clean up lip-sync analyzer:", error)
|
|
1151
|
+
);
|
|
1152
|
+
const playbackContext = this.playbackContext;
|
|
1153
|
+
this.playbackContext = null;
|
|
1154
|
+
this.playbackScheduler = null;
|
|
1155
|
+
playbackContext?.close().catch(
|
|
1156
|
+
(error) => console.error("Failed to close the playback audio context:", error)
|
|
1157
|
+
);
|
|
1158
|
+
this.connectionLossNotified = false;
|
|
1159
|
+
this.isCleaningUp = false;
|
|
1160
|
+
}
|
|
1161
|
+
closeActiveSocket() {
|
|
1162
|
+
const binding = this.activeSocket;
|
|
1163
|
+
this.activeSocket = null;
|
|
1164
|
+
if (!binding) {
|
|
1165
|
+
return;
|
|
1166
|
+
}
|
|
1167
|
+
binding.socket.onopen = null;
|
|
1168
|
+
binding.socket.onmessage = null;
|
|
1169
|
+
binding.socket.onerror = null;
|
|
1170
|
+
binding.socket.onclose = null;
|
|
1171
|
+
binding.socket.close();
|
|
1172
|
+
}
|
|
1173
|
+
emitConnectionLost(error) {
|
|
1174
|
+
if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp) {
|
|
1175
|
+
return;
|
|
1176
|
+
}
|
|
1177
|
+
this.connectionLossNotified = true;
|
|
1178
|
+
this.emitEvent({
|
|
1179
|
+
type: "connection.lost",
|
|
1180
|
+
cause: "connection-failed",
|
|
1181
|
+
error
|
|
1182
|
+
});
|
|
1183
|
+
}
|
|
1184
|
+
resolvePendingConnect() {
|
|
1185
|
+
const pending = this.pendingConnect;
|
|
1186
|
+
if (!pending) {
|
|
1187
|
+
return;
|
|
1188
|
+
}
|
|
1189
|
+
this.pendingConnect = null;
|
|
1190
|
+
clearTimeout(pending.timeoutId);
|
|
1191
|
+
pending.resolve();
|
|
1192
|
+
}
|
|
1193
|
+
rejectPendingConnect(error) {
|
|
1194
|
+
const pending = this.pendingConnect;
|
|
1195
|
+
if (!pending) {
|
|
1196
|
+
return;
|
|
1197
|
+
}
|
|
1198
|
+
this.pendingConnect = null;
|
|
1199
|
+
clearTimeout(pending.timeoutId);
|
|
1200
|
+
pending.reject(error);
|
|
1201
|
+
}
|
|
1202
|
+
async getSessionBootstrap(request) {
|
|
1203
|
+
return getGeminiLiveBootstrap(this.options, request);
|
|
1204
|
+
}
|
|
1205
|
+
emitEvent(event) {
|
|
1206
|
+
for (const callback of this.eventCallbacks) {
|
|
1207
|
+
callback(event);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
log(...args) {
|
|
1211
|
+
if (this.options.debug) {
|
|
1212
|
+
console.log(...args);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
};
|
|
1216
|
+
function bytesToBase64(bytes) {
|
|
1217
|
+
let binary = "";
|
|
1218
|
+
const stride = 32768;
|
|
1219
|
+
for (let offset = 0; offset < bytes.length; offset += stride) {
|
|
1220
|
+
binary += String.fromCharCode(...bytes.subarray(offset, offset + stride));
|
|
1221
|
+
}
|
|
1222
|
+
return btoa(binary);
|
|
1223
|
+
}
|
|
1224
|
+
function base64ToBytes(base64) {
|
|
1225
|
+
const binary = atob(base64);
|
|
1226
|
+
const bytes = new Uint8Array(binary.length);
|
|
1227
|
+
for (let index = 0; index < binary.length; index += 1) {
|
|
1228
|
+
bytes[index] = binary.charCodeAt(index);
|
|
1229
|
+
}
|
|
1230
|
+
return bytes;
|
|
1231
|
+
}
|
|
1232
|
+
function resolveWebSocketBootstrap(bootstrap) {
|
|
1233
|
+
if (bootstrap.adapter !== GEMINI_LIVE_ADAPTER || bootstrap.transport !== "websocket") {
|
|
1234
|
+
throw new Error(
|
|
1235
|
+
`Gemini Live client only supports ${GEMINI_LIVE_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
|
|
1236
|
+
);
|
|
1237
|
+
}
|
|
1238
|
+
if (!bootstrap.url || !bootstrap.token) {
|
|
1239
|
+
throw new Error(
|
|
1240
|
+
"Gemini Live bootstrap is missing its websocket url or token"
|
|
1241
|
+
);
|
|
1242
|
+
}
|
|
1243
|
+
return { url: bootstrap.url, token: bootstrap.token };
|
|
1244
|
+
}
|
|
1245
|
+
function createGeminiLiveClient(options) {
|
|
1246
|
+
return new GeminiLiveClient(options);
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
export { createGeminiLiveClient };
|
|
1250
|
+
//# sourceMappingURL=chunk-3YPRYU2M.mjs.map
|
|
1251
|
+
//# sourceMappingURL=chunk-3YPRYU2M.mjs.map
|