@charivo/realtime 0.18.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +83 -26
  2. package/dist/chunk-3YPRYU2M.mjs +1251 -0
  3. package/dist/chunk-3YPRYU2M.mjs.map +1 -0
  4. package/dist/chunk-5OO737VD.mjs +33 -0
  5. package/dist/chunk-5OO737VD.mjs.map +1 -0
  6. package/dist/{chunk-U7ER2VBF.mjs → chunk-EIYYLUWF.mjs} +10 -17
  7. package/dist/chunk-EIYYLUWF.mjs.map +1 -0
  8. package/dist/{chunk-VFIMTFOP.mjs → chunk-JEJGTULX.mjs} +24 -31
  9. package/dist/chunk-JEJGTULX.mjs.map +1 -0
  10. package/dist/{chunk-EXLERHST.mjs → chunk-VTHBTUH3.mjs} +7 -10
  11. package/dist/chunk-VTHBTUH3.mjs.map +1 -0
  12. package/dist/gemini/index.d.mts +22 -0
  13. package/dist/gemini/index.d.ts +22 -0
  14. package/dist/gemini/index.js +1282 -0
  15. package/dist/gemini/index.js.map +1 -0
  16. package/dist/gemini/index.mjs +4 -0
  17. package/dist/gemini/index.mjs.map +1 -0
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js.map +1 -1
  21. package/dist/index.mjs.map +1 -1
  22. package/dist/openai/index.js +22 -7
  23. package/dist/openai/index.js.map +1 -1
  24. package/dist/openai/index.mjs +3 -2
  25. package/dist/openai-agents/index.js +25 -14
  26. package/dist/openai-agents/index.js.map +1 -1
  27. package/dist/openai-agents/index.mjs +3 -2
  28. package/dist/remote/index.d.mts +1 -0
  29. package/dist/remote/index.d.ts +1 -0
  30. package/dist/remote/index.js +1296 -37
  31. package/dist/remote/index.js.map +1 -1
  32. package/dist/remote/index.mjs +13 -4
  33. package/dist/remote/index.mjs.map +1 -1
  34. package/package.json +7 -2
  35. package/dist/chunk-EXLERHST.mjs.map +0 -1
  36. package/dist/chunk-U7ER2VBF.mjs.map +0 -1
  37. package/dist/chunk-VFIMTFOP.mjs.map +0 -1
@@ -0,0 +1,1282 @@
1
+ 'use strict';
2
+
3
+ var core = require('@charivo/core');
4
+
5
+ // src/gemini/client.ts
6
+
7
+ // src/internal/microphone.ts
8
+ var DEFAULT_MICROPHONE_CONSTRAINTS = {
9
+ echoCancellation: true,
10
+ noiseSuppression: true,
11
+ autoGainControl: true
12
+ };
13
+ async function acquireMicrophoneStream() {
14
+ return navigator.mediaDevices.getUserMedia({
15
+ audio: DEFAULT_MICROPHONE_CONSTRAINTS
16
+ });
17
+ }
18
+ var DEFAULT_REQUEST_TIMEOUT_MS = core.DEFAULT_FETCH_TIMEOUT_MS;
19
+ function isRecord(value) {
20
+ return typeof value === "object" && value !== null;
21
+ }
22
+ function isRealtimeSessionBootstrap(value) {
23
+ if (!isRecord(value) || typeof value.adapter !== "string" || typeof value.transport !== "string") {
24
+ return false;
25
+ }
26
+ if (value.transport === "webrtc") {
27
+ return typeof value.answerSdp === "string" || typeof value.clientSecret === "string";
28
+ }
29
+ if (value.transport === "websocket") {
30
+ return typeof value.url === "string" && typeof value.token === "string";
31
+ }
32
+ return false;
33
+ }
34
+
35
+ // src/gemini/defaults.ts
36
+ var DEFAULT_GEMINI_LIVE_MODEL = "gemini-3.1-flash-live-preview";
37
+ var DEFAULT_GEMINI_LIVE_VOICE = "Kore";
38
+ var INPUT_SAMPLE_RATE = 16e3;
39
+ var OUTPUT_SAMPLE_RATE = 24e3;
40
+ var CAPTURE_FRAME_SAMPLES = 320;
41
+ var CONVERGENCE_GATE_MS = 700;
42
+
43
+ // src/gemini/dev-bootstrap.ts
44
+ var AUTH_TOKENS_URL = "https://generativelanguage.googleapis.com/v1beta/auth_tokens";
45
+ var GEMINI_LIVE_WEBSOCKET_URL = "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContentConstrained";
46
+ var ALLOWED_VOICES = /* @__PURE__ */ new Set([
47
+ "Zephyr",
48
+ "Puck",
49
+ "Charon",
50
+ "Kore",
51
+ "Fenrir",
52
+ "Leda",
53
+ "Orus",
54
+ "Aoede",
55
+ "Callirrhoe",
56
+ "Autonoe",
57
+ "Enceladus",
58
+ "Iapetus",
59
+ "Umbriel",
60
+ "Algieba",
61
+ "Despina",
62
+ "Erinome",
63
+ "Algenib",
64
+ "Rasalgethi",
65
+ "Laomedeia",
66
+ "Achernar",
67
+ "Alnilam",
68
+ "Schedar",
69
+ "Gacrux",
70
+ "Pulcherrima",
71
+ "Achird",
72
+ "Zubenelgenubi",
73
+ "Vindemiatrix",
74
+ "Sadachbia",
75
+ "Sadaltager",
76
+ "Sulafat"
77
+ ]);
78
+ function toMintBody(session) {
79
+ if (session.toolChoice === "none" || session.toolChoice === "required") {
80
+ throw new core.CharivoStateError(
81
+ `Gemini Live does not support toolChoice "${session.toolChoice}"`
82
+ );
83
+ }
84
+ if (session.inputAudioTranscription?.model !== void 0) {
85
+ throw new core.CharivoStateError(
86
+ `Gemini Live does not support inputAudioTranscription.model "${session.inputAudioTranscription.model}"`
87
+ );
88
+ }
89
+ const voice = session.voice && ALLOWED_VOICES.has(session.voice) ? session.voice : DEFAULT_GEMINI_LIVE_VOICE;
90
+ const generationConfig = {
91
+ responseModalities: ["AUDIO"],
92
+ speechConfig: {
93
+ voiceConfig: {
94
+ prebuiltVoiceConfig: { voiceName: voice }
95
+ }
96
+ }
97
+ };
98
+ if (session.maxTokens !== void 0) {
99
+ generationConfig.maxOutputTokens = session.maxTokens;
100
+ }
101
+ const setup = {
102
+ model: `models/${session.model ?? DEFAULT_GEMINI_LIVE_MODEL}`,
103
+ generationConfig,
104
+ // The only assistant-text source on native-audio models.
105
+ outputAudioTranscription: {}
106
+ };
107
+ if (session.instructions) {
108
+ setup.systemInstruction = { parts: [{ text: session.instructions }] };
109
+ }
110
+ if (session.tools?.length) {
111
+ setup.tools = [
112
+ {
113
+ functionDeclarations: session.tools.map((tool) => ({
114
+ name: tool.name,
115
+ description: tool.description,
116
+ parameters: tool.parameters
117
+ }))
118
+ }
119
+ ];
120
+ }
121
+ if (session.inputAudioTranscription?.enabled === true) {
122
+ setup.inputAudioTranscription = {};
123
+ }
124
+ return {
125
+ uses: 1,
126
+ bidiGenerateContentSetup: setup
127
+ };
128
+ }
129
+ function extractTokenName(payload) {
130
+ if (!isRecord(payload)) {
131
+ return null;
132
+ }
133
+ return typeof payload.name === "string" && payload.name.length > 0 ? payload.name : null;
134
+ }
135
+ function createGeminiLiveDevBootstrap(apiKey) {
136
+ return async (request) => {
137
+ const response = await core.fetchWithTimeout(
138
+ AUTH_TOKENS_URL,
139
+ {
140
+ method: "POST",
141
+ headers: {
142
+ // Never in the URL: proxies and request logs capture query strings.
143
+ "x-goog-api-key": apiKey,
144
+ "Content-Type": "application/json"
145
+ },
146
+ body: JSON.stringify(toMintBody(request.session))
147
+ },
148
+ {
149
+ timeoutMessage: `Gemini Live token request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,
150
+ failureMessage: "Realtime request failed"
151
+ }
152
+ );
153
+ if (!response.ok) {
154
+ const errorText = await response.text();
155
+ throw new core.CharivoProviderError(
156
+ `Failed to mint Gemini Live ephemeral token: ${errorText}`
157
+ );
158
+ }
159
+ let body;
160
+ try {
161
+ body = await response.json();
162
+ } catch {
163
+ throw new core.CharivoProviderError(
164
+ "Invalid ephemeral token response: response body could not be parsed as JSON"
165
+ );
166
+ }
167
+ const token = extractTokenName(body);
168
+ if (!token) {
169
+ throw new core.CharivoProviderError(
170
+ "Invalid ephemeral token response: name missing or empty"
171
+ );
172
+ }
173
+ return {
174
+ adapter: core.GEMINI_LIVE_ADAPTER,
175
+ transport: "websocket",
176
+ url: GEMINI_LIVE_WEBSOCKET_URL,
177
+ token
178
+ };
179
+ };
180
+ }
181
+
182
+ // src/gemini/bootstrap.ts
183
+ async function getGeminiLiveBootstrap(options, request) {
184
+ if (options.sessionBootstrap) {
185
+ return options.sessionBootstrap(request);
186
+ }
187
+ const apiEndpoint = options.apiEndpoint;
188
+ if (!apiEndpoint && options.apiKey) {
189
+ return createGeminiLiveDevBootstrap(options.apiKey)(request);
190
+ }
191
+ if (!apiEndpoint) {
192
+ throw new core.CharivoStateError(
193
+ "Gemini Live client requires sessionBootstrap, apiEndpoint, or apiKey"
194
+ );
195
+ }
196
+ const response = await core.fetchWithTimeout(
197
+ apiEndpoint,
198
+ {
199
+ method: "POST",
200
+ headers: {
201
+ "Content-Type": "application/json"
202
+ },
203
+ body: JSON.stringify(request)
204
+ },
205
+ {
206
+ timeoutMessage: `Realtime session request timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`,
207
+ failureMessage: "Realtime request failed"
208
+ }
209
+ );
210
+ if (!response.ok) {
211
+ const errorText = await response.text();
212
+ throw new core.CharivoProviderError(
213
+ `Failed to create Realtime session: ${errorText}`
214
+ );
215
+ }
216
+ const bootstrap = await response.json();
217
+ if (!isRealtimeSessionBootstrap(bootstrap)) {
218
+ throw new core.CharivoProviderError(
219
+ "Invalid realtime session bootstrap response"
220
+ );
221
+ }
222
+ return bootstrap;
223
+ }
224
+
225
+ // src/gemini/capture.ts
226
+ var CAPTURE_PROCESSOR_NAME = "charivo-gemini-capture";
227
+ var CAPTURE_WORKLET_SOURCE = `
228
+ class CaptureProcessor extends AudioWorkletProcessor {
229
+ constructor(options) {
230
+ super();
231
+ this.ratio = sampleRate / options.processorOptions.targetRate;
232
+ this.frameSamples = options.processorOptions.frameSamples;
233
+ this.position = 0;
234
+ this.sum = 0;
235
+ this.count = 0;
236
+ this.pending = [];
237
+ }
238
+
239
+ process(inputs) {
240
+ const channel = inputs[0] && inputs[0][0];
241
+ if (!channel) {
242
+ return true;
243
+ }
244
+
245
+ // Box-average decimation. Not a proper anti-alias filter, but far better
246
+ // than picking every Nth sample, which folds speech harmonics back down.
247
+ // The position accumulator carries the fractional remainder across groups,
248
+ // so a device rate that is not a whole multiple of the target \u2014 44.1 kHz,
249
+ // say \u2014 alternates group sizes rather than drifting out of step.
250
+ for (let i = 0; i < channel.length; i += 1) {
251
+ this.sum += channel[i];
252
+ this.count += 1;
253
+ this.position += 1;
254
+ if (this.position >= this.ratio) {
255
+ this.position -= this.ratio;
256
+ this.pending.push(this.count > 0 ? this.sum / this.count : 0);
257
+ this.sum = 0;
258
+ this.count = 0;
259
+ }
260
+ }
261
+
262
+ while (this.pending.length >= this.frameSamples) {
263
+ const chunk = this.pending.splice(0, this.frameSamples);
264
+ const frame = new Int16Array(chunk.length);
265
+ for (let i = 0; i < chunk.length; i += 1) {
266
+ const clamped = Math.max(-1, Math.min(1, chunk[i]));
267
+ frame[i] = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
268
+ }
269
+ this.port.postMessage(frame.buffer, [frame.buffer]);
270
+ }
271
+
272
+ return true;
273
+ }
274
+ }
275
+
276
+ registerProcessor("${CAPTURE_PROCESSOR_NAME}", CaptureProcessor);
277
+ `;
278
+ async function createCapturePipeline({
279
+ stream,
280
+ onFrame
281
+ }) {
282
+ const context = new AudioContext();
283
+ const workletUrl = URL.createObjectURL(
284
+ new Blob([CAPTURE_WORKLET_SOURCE], { type: "application/javascript" })
285
+ );
286
+ try {
287
+ await context.audioWorklet.addModule(workletUrl);
288
+ const source = context.createMediaStreamSource(stream);
289
+ const capture = new AudioWorkletNode(context, CAPTURE_PROCESSOR_NAME, {
290
+ processorOptions: {
291
+ targetRate: INPUT_SAMPLE_RATE,
292
+ frameSamples: CAPTURE_FRAME_SAMPLES
293
+ }
294
+ });
295
+ source.connect(capture);
296
+ capture.port.onmessage = (event) => {
297
+ onFrame(new Uint8Array(event.data));
298
+ };
299
+ capture.onprocessorerror = () => {
300
+ console.error("The Gemini capture worklet stopped processing");
301
+ };
302
+ return {
303
+ stop() {
304
+ capture.port.onmessage = null;
305
+ capture.port.close();
306
+ source.disconnect();
307
+ capture.disconnect();
308
+ void closeContext(context);
309
+ }
310
+ };
311
+ } catch (error) {
312
+ void closeContext(context);
313
+ throw error;
314
+ } finally {
315
+ URL.revokeObjectURL(workletUrl);
316
+ }
317
+ }
318
+ function closeContext(context) {
319
+ return context.close().catch(
320
+ (error) => console.error("Failed to close the capture audio context:", error)
321
+ );
322
+ }
323
+
324
+ // src/gemini/playback.ts
325
+ function createPlaybackGraph(context) {
326
+ const output = context.createGain();
327
+ output.connect(context.destination);
328
+ const tap = context.createMediaStreamDestination();
329
+ output.connect(tap);
330
+ return { output, lipSyncStream: tap.stream };
331
+ }
332
+ var PlaybackScheduler = class {
333
+ constructor(context, destination, callbacks) {
334
+ this.context = context;
335
+ this.destination = destination;
336
+ this.callbacks = callbacks;
337
+ }
338
+ nextStartTime = 0;
339
+ /**
340
+ * Bumped by `flush()`. Every source captures the value it was scheduled
341
+ * under, so a discarded turn's late `onended` cannot report against the turn
342
+ * that replaced it.
343
+ */
344
+ generation = 0;
345
+ active = /* @__PURE__ */ new Set();
346
+ /**
347
+ * Whether the scheduler has nothing left to play — nothing sounding, and
348
+ * nothing queued behind it. Derived from the live source set on
349
+ * every call rather than remembered: `enqueue()` clears idleness by adding to
350
+ * that set, so a caller cannot be holding a "drained" answer that the next
351
+ * chunk already invalidated.
352
+ */
353
+ isIdle() {
354
+ return this.active.size === 0;
355
+ }
356
+ /** Schedule one `audio/pcm;rate=24000` chunk after everything already queued. */
357
+ enqueue(pcm) {
358
+ const sampleCount = Math.floor(pcm.byteLength / 2);
359
+ if (sampleCount === 0) {
360
+ return;
361
+ }
362
+ const generation = this.generation;
363
+ const view = new DataView(pcm.buffer, pcm.byteOffset, pcm.byteLength);
364
+ const buffer = this.context.createBuffer(
365
+ 1,
366
+ sampleCount,
367
+ OUTPUT_SAMPLE_RATE
368
+ );
369
+ const channel = buffer.getChannelData(0);
370
+ for (let index = 0; index < sampleCount; index += 1) {
371
+ channel[index] = view.getInt16(index * 2, true) / 32768;
372
+ }
373
+ const source = this.context.createBufferSource();
374
+ source.buffer = buffer;
375
+ source.connect(this.destination);
376
+ source.onended = () => {
377
+ this.active.delete(source);
378
+ if (generation !== this.generation || this.active.size > 0) {
379
+ return;
380
+ }
381
+ this.callbacks.onPlayingChange(false);
382
+ this.callbacks.onDrain();
383
+ };
384
+ const wasIdle = this.isIdle();
385
+ const startAt = Math.max(this.context.currentTime, this.nextStartTime);
386
+ this.active.add(source);
387
+ source.start(startAt);
388
+ this.nextStartTime = startAt + buffer.duration;
389
+ if (wasIdle) {
390
+ this.callbacks.onPlayingChange(true);
391
+ }
392
+ }
393
+ /** Discard everything scheduled, audible or merely queued. */
394
+ flush() {
395
+ this.generation += 1;
396
+ const wasPlaying = !this.isIdle();
397
+ for (const source of this.active) {
398
+ source.onended = null;
399
+ try {
400
+ source.stop();
401
+ } catch {
402
+ }
403
+ }
404
+ this.active.clear();
405
+ this.nextStartTime = 0;
406
+ if (wasPlaying) {
407
+ this.callbacks.onPlayingChange(false);
408
+ }
409
+ }
410
+ };
411
+
412
+ // src/gemini/client.ts
413
+ var GeminiLiveClient = class {
414
+ constructor(options = {}) {
415
+ this.options = options;
416
+ }
417
+ activeSocket = null;
418
+ /** Bumped by every transient reset; see `GeminiSocketBinding`. */
419
+ socketEpoch = 0;
420
+ pendingConnect = null;
421
+ lipSyncAnalyzer = core.createLipSyncAnalyzer({
422
+ onRms: (rms) => this.emitEvent({ type: "audio.lipsync", rms }),
423
+ onError: (error) => console.error("Failed to setup audio analysis:", error)
424
+ });
425
+ /**
426
+ * Created once, inside a user gesture, and reused for the client's whole
427
+ * life. Only the terminal `cleanup()` closes it.
428
+ */
429
+ playbackContext = null;
430
+ playbackScheduler = null;
431
+ /**
432
+ * Per-session, unlike the playback context above: both the stream and the
433
+ * pipeline reading it are acquired by every `connect()` and dropped by the
434
+ * transient reset.
435
+ */
436
+ captureStream = null;
437
+ capturePipeline = null;
438
+ /**
439
+ * Whether the server has acknowledged setup. Microphone frames wait on it —
440
+ * see `sendCaptureFrame`.
441
+ */
442
+ isSetupComplete = false;
443
+ /**
444
+ * Convergence gate. Safari's echo canceller leaks the character's own voice
445
+ * for roughly the first half-second after it starts speaking, and converges
446
+ * cumulatively across turns (measured,
447
+ * `tests/gemini-live-smoke/README.md`), so while armed the client drops the
448
+ * microphone frames landing within `CONVERGENCE_GATE_MS` of playback becoming
449
+ * audible.
450
+ *
451
+ * Dropping them does not slow that adaptation down: the canceller runs inside
452
+ * the `getUserMedia` pipeline and adapts to whatever plays out, forwarded or
453
+ * not. The gate only keeps the not-yet-converged residue from reaching the
454
+ * model, at a bounded cost — a barge-in inside the gate window at the start
455
+ * of a reply is lost, against the character killing its own turn twice.
456
+ *
457
+ * Disarmed by accumulated audible exposure, in `endAudioOutput`.
458
+ */
459
+ gateArmed = true;
460
+ /**
461
+ * `performance.now()` at the moment the current turn's playback became
462
+ * audible, and the only thing the gate window is measured from: a fact this
463
+ * client owns, never a guess about what the microphone contains. Re-anchored
464
+ * at every turn's first audible playback by `beginAudiblePlayback`; read by
465
+ * `sendCaptureFrame` and cleared by the transient reset.
466
+ */
467
+ playbackAudibleSince = null;
468
+ /**
469
+ * How much audible playback the echo canceller has had to adapt to, summed
470
+ * over the scheduler's playing → not-playing intervals for the whole session
471
+ * and the only quantity the gate disarms on. `audibleSpanStartedAt` is the
472
+ * open interval, or null while nothing is sounding.
473
+ */
474
+ audibleExposureMs = 0;
475
+ audibleSpanStartedAt = null;
476
+ /**
477
+ * Assistant-side turn state. Output transcription is native audio's only
478
+ * source of assistant text, and both fields are cleared at `turnComplete`.
479
+ */
480
+ hasStartedAssistantResponse = false;
481
+ assistantText = "";
482
+ /**
483
+ * Audio bookkeeping, which deliberately outlives `turnComplete`: in the
484
+ * measured normal sequence the final drain lands ~3 ms *after* it, so both
485
+ * are cleared where `audio.output.ended` actually fires.
486
+ *
487
+ * `hasStartedAudioOutput` is what keeps the pair balanced — every ending path
488
+ * requires it and clears it, so a raw transport listener sees exactly one
489
+ * start/end per turn. `turnCompleteSeen` means an ending is still owed, and
490
+ * is the second half of `ended` = currently idle AND `turnComplete` seen. The
491
+ * transient reset drops both, along with the turn they described.
492
+ */
493
+ hasStartedAudioOutput = false;
494
+ turnCompleteSeen = false;
495
+ /**
496
+ * Whether server VAD killed this turn. Set by `handleInterrupted`, then read
497
+ * and cleared by the `endAudioOutput` it goes on to call: an interrupted turn
498
+ * banks its audible exposure like any other but never disarms the gate.
499
+ */
500
+ turnInterrupted = false;
501
+ /**
502
+ * Whether the server has a model turn open — content has arrived and its
503
+ * `turnComplete` has not. It is the only proof `interrupt()` accepts that
504
+ * there is a turn to condemn, and neither existing flag answers that
505
+ * question: `hasStartedAssistantResponse` follows output transcription, which
506
+ * is a server-side session setting this client does not control, and the
507
+ * audio flags deliberately outlive `turnComplete`.
508
+ */
509
+ modelTurnInFlight = false;
510
+ /**
511
+ * Whether a local `interrupt()` killed the turn in flight. Nothing on the
512
+ * wire stops the server generating it, so while this is set the rest of that
513
+ * turn is dropped at the guard in `applyServerMessage`, which is where what
514
+ * "the rest" covers is settled, and its completion is never reported:
515
+ * `RealtimeTransportEvent` forbids reporting an interrupted turn once a
516
+ * replacement may have started, because `RealtimeManager` would credit that
517
+ * completion to the replacement and release its send lock.
518
+ * `handleTurnComplete` is the single exit.
519
+ *
520
+ * A boolean where `openai/client.ts` learned to keep a count. That lesson
521
+ * does not transfer: it can issue `response.create` itself and stack a
522
+ * replacement behind a cancel, whereas turns here are the server's to open
523
+ * and it opens them one at a time, so there is never more than one to
524
+ * condemn.
525
+ */
526
+ turnCondemned = false;
527
+ /**
528
+ * Live `functionCall.id` -> tool name, which `sendToolResult` needs because a
529
+ * Gemini `functionResponse` carries the name alongside the id. An entry lives
530
+ * from the `toolCall` frame until that call is answered or cancelled, and the
531
+ * map as a whole belongs to the session: the transient reset drops it, so a
532
+ * handler finishing after a reconnect throws instead of answering into a
533
+ * session that never asked.
534
+ */
535
+ pendingToolCalls = /* @__PURE__ */ new Map();
536
+ /**
537
+ * Newest `usageMetadata` of the turn in progress, reported as `usage` when it
538
+ * completes and dropped there. Nothing measured guarantees one frame per
539
+ * turn, so carrying it forward would report the previous turn's numbers as
540
+ * this one's — and since audio prompt tokens grow turn over turn, it would
541
+ * under-report rather than fail visibly.
542
+ */
543
+ latestUsage;
544
+ currentSessionConfig;
545
+ connectionLossNotified = false;
546
+ isExplicitDisconnect = false;
547
+ isRecovering = false;
548
+ /**
549
+ * Set only for the span of the terminal `cleanup()`. It is what stops a
550
+ * teardown step from reporting the session it is dismantling as lost — most
551
+ * visibly on a failed initial connect, where the manager would hear
552
+ * `connection.lost` for a session that never started.
553
+ */
554
+ isCleaningUp = false;
555
+ eventCallbacks = /* @__PURE__ */ new Set();
556
+ async connect(config) {
557
+ try {
558
+ if (this.isExplicitDisconnect && this.isRecovering) {
559
+ throw new Error("Realtime session disconnect requested");
560
+ }
561
+ this.log("Starting Gemini Live websocket connection");
562
+ this.isExplicitDisconnect = false;
563
+ this.currentSessionConfig = config;
564
+ const bootstrap = await this.getSessionBootstrap({
565
+ transport: "websocket",
566
+ session: config ?? {}
567
+ });
568
+ const { url, token } = resolveWebSocketBootstrap(bootstrap);
569
+ if (this.isExplicitDisconnect) {
570
+ throw new Error("Realtime session disconnect requested");
571
+ }
572
+ await this.ensurePlaybackReady();
573
+ try {
574
+ this.captureStream = await acquireMicrophoneStream();
575
+ } catch (error) {
576
+ throw new Error("Microphone access required for Realtime API", {
577
+ cause: error
578
+ });
579
+ }
580
+ if (this.isExplicitDisconnect) {
581
+ throw new Error("Realtime session disconnect requested");
582
+ }
583
+ const capturePipeline = await createCapturePipeline({
584
+ stream: this.captureStream,
585
+ onFrame: (frame) => this.sendCaptureFrame(frame)
586
+ });
587
+ if (this.isExplicitDisconnect) {
588
+ capturePipeline.stop();
589
+ throw new Error("Realtime session disconnect requested");
590
+ }
591
+ this.capturePipeline = capturePipeline;
592
+ await this.openSocket(url, token, config?.model);
593
+ this.connectionLossNotified = false;
594
+ this.emitEvent({ type: "session.started" });
595
+ } catch (error) {
596
+ if (this.isRecovering) {
597
+ this.resetTransportState();
598
+ } else {
599
+ this.cleanup();
600
+ }
601
+ throw error;
602
+ }
603
+ }
604
+ async disconnect() {
605
+ this.log("Disconnecting Gemini Live websocket");
606
+ this.isExplicitDisconnect = true;
607
+ this.isRecovering = false;
608
+ this.cleanup();
609
+ }
610
+ async updateSession(_config) {
611
+ throw new Error(
612
+ "updateSession() is not supported on an open Gemini Live session. Reconnect with the new configuration instead."
613
+ );
614
+ }
615
+ async recover(config) {
616
+ this.isRecovering = true;
617
+ this.connectionLossNotified = true;
618
+ this.currentSessionConfig = config ?? this.currentSessionConfig;
619
+ try {
620
+ this.resetTransportState();
621
+ await this.connect(this.currentSessionConfig);
622
+ } finally {
623
+ this.isRecovering = false;
624
+ }
625
+ }
626
+ async sendText(text) {
627
+ if (this.isRecovering) {
628
+ throw new Error("Realtime transport reconnecting");
629
+ }
630
+ const socket = this.requireReadySocket();
631
+ socket.send(
632
+ JSON.stringify({
633
+ clientContent: {
634
+ turns: [{ role: "user", parts: [{ text }] }],
635
+ turnComplete: true
636
+ }
637
+ })
638
+ );
639
+ }
640
+ /**
641
+ * Stop the character mid-sentence.
642
+ *
643
+ * Entirely local, and it has to be: the Live API documents no client message
644
+ * that cancels an in-flight generation. So the turn is condemned instead,
645
+ * which rests on the killed turn still running to its own `turnComplete` —
646
+ * the single exit from condemnation. That was measured rather than assumed
647
+ * (`tests/gemini-live-smoke/README.md`, Q2): interrupting mid-count emitted
648
+ * no completion for the killed turn, and the very next turn completed
649
+ * normally, which it could not have done had the condemnation never lifted.
650
+ * What has been scheduled is discarded, and its remaining speech and
651
+ * text are dropped — though not its tool calls, which `applyServerMessage`
652
+ * still dispatches on purpose. Nothing goes on the wire, which is why this
653
+ * needs neither an open socket nor a recovery guard.
654
+ */
655
+ async interrupt() {
656
+ if (this.modelTurnInFlight) {
657
+ this.turnCondemned = true;
658
+ this.log("Gemini Live turn condemned by a local interrupt");
659
+ }
660
+ this.playbackScheduler?.flush();
661
+ this.endAudioOutput();
662
+ }
663
+ async sendToolResult(callId, output) {
664
+ if (this.isRecovering) {
665
+ throw new Error("Realtime transport reconnecting");
666
+ }
667
+ const name = this.pendingToolCalls.get(callId);
668
+ if (!name) {
669
+ throw new Error(
670
+ `Unknown Gemini Live tool call "${callId}": it was already answered, cancelled by the server, or belongs to a session that has ended`
671
+ );
672
+ }
673
+ const socket = this.requireReadySocket();
674
+ socket.send(
675
+ JSON.stringify({
676
+ toolResponse: {
677
+ functionResponses: [{ id: callId, name, response: output }]
678
+ }
679
+ })
680
+ );
681
+ this.pendingToolCalls.delete(callId);
682
+ }
683
+ async sendAudio(_audio) {
684
+ console.warn(
685
+ "sendAudio is not needed with the Gemini Live transport - the microphone is captured internally"
686
+ );
687
+ }
688
+ onEvent(callback) {
689
+ this.eventCallbacks.add(callback);
690
+ }
691
+ async prepareAudio() {
692
+ await this.ensurePlaybackReady();
693
+ }
694
+ /**
695
+ * Build the playback context, graph, scheduler, and lip-sync analyzer once,
696
+ * and afterwards only nudge that same playback context back towards running.
697
+ * It does not promise a running context on return; see the resume below.
698
+ *
699
+ * Both `prepareAudio()` and `connect()` come here, and the order they arrive
700
+ * in is the point: whichever runs first builds everything, the other reuses
701
+ * it. That is what keeps the `AudioContext` the one created inside a user
702
+ * gesture instead of a replacement built during an automatic reconnect,
703
+ * which the browser would refuse to start.
704
+ *
705
+ * The playback context is the only one this covers, and that leaves a gap
706
+ * worth knowing about: `lipSyncAnalyzer.prepare()` creates a second
707
+ * `AudioContext` of its own, and nothing in `@charivo/core` ever resumes it.
708
+ * A suspended `AnalyserNode` returns all-zero frequency data, so in exactly
709
+ * the case the nudge below exists for — a caller that skipped
710
+ * `prepareAudio()` — lip sync reads flat and no error surfaces anywhere.
711
+ * Pre-existing and identical in `openai/client.ts`; closing it means changing
712
+ * the analyzer contract, not this file.
713
+ */
714
+ async ensurePlaybackReady() {
715
+ const existing = this.playbackContext;
716
+ if (existing) {
717
+ if (existing.state !== "running") {
718
+ void existing.resume().catch(
719
+ (error) => console.error(
720
+ "Failed to resume the playback audio context:",
721
+ error
722
+ )
723
+ );
724
+ }
725
+ return;
726
+ }
727
+ const context = new AudioContext({ sampleRate: OUTPUT_SAMPLE_RATE });
728
+ const graph = createPlaybackGraph(context);
729
+ this.playbackContext = context;
730
+ this.playbackScheduler = new PlaybackScheduler(context, graph.output, {
731
+ onDrain: () => {
732
+ if (this.turnCompleteSeen) {
733
+ this.endAudioOutput();
734
+ }
735
+ },
736
+ onPlayingChange: (playing) => {
737
+ if (playing) {
738
+ this.beginAudiblePlayback();
739
+ } else {
740
+ this.bankAudibleExposure();
741
+ }
742
+ }
743
+ });
744
+ await this.lipSyncAnalyzer.prepare();
745
+ this.lipSyncAnalyzer.attachMediaStream(graph.lipSyncStream);
746
+ this.lipSyncAnalyzer.pause();
747
+ }
748
+ openSocket(url, token, model) {
749
+ return new Promise((resolve, reject) => {
750
+ let socket;
751
+ try {
752
+ socket = new WebSocket(
753
+ `${url}?access_token=${encodeURIComponent(token)}`
754
+ );
755
+ } catch {
756
+ reject(new Error("Failed to open the Gemini Live websocket"));
757
+ return;
758
+ }
759
+ const binding = {
760
+ socket,
761
+ epoch: this.socketEpoch,
762
+ pump: Promise.resolve()
763
+ };
764
+ this.activeSocket = binding;
765
+ this.pendingConnect = {
766
+ resolve,
767
+ reject,
768
+ timeoutId: setTimeout(() => {
769
+ this.rejectPendingConnect(
770
+ new Error(
771
+ `Gemini Live session setup timed out after ${DEFAULT_REQUEST_TIMEOUT_MS}ms`
772
+ )
773
+ );
774
+ }, DEFAULT_REQUEST_TIMEOUT_MS)
775
+ };
776
+ socket.onopen = () => {
777
+ this.log("Gemini Live websocket open \u2014 sending setup");
778
+ socket.send(
779
+ JSON.stringify({
780
+ setup: { model: `models/${model ?? DEFAULT_GEMINI_LIVE_MODEL}` }
781
+ })
782
+ );
783
+ };
784
+ socket.onmessage = (event) => {
785
+ this.enqueueServerMessage(binding, event.data);
786
+ };
787
+ socket.onerror = () => {
788
+ this.failTransport(binding, new Error("Gemini Live websocket error"));
789
+ };
790
+ socket.onclose = (event) => {
791
+ this.log(
792
+ `Gemini Live websocket closed code=${event.code} reason="${event.reason}" clean=${event.wasClean}`
793
+ );
794
+ this.failTransport(
795
+ binding,
796
+ new Error(
797
+ `Gemini Live websocket closed (code ${event.code}${event.reason ? `: ${event.reason}` : ""})`
798
+ )
799
+ );
800
+ };
801
+ });
802
+ }
803
+ enqueueServerMessage(binding, data) {
804
+ if (binding.epoch !== this.socketEpoch) {
805
+ return;
806
+ }
807
+ binding.pump = binding.pump.then(async () => {
808
+ const raw = typeof data === "string" ? data : await data.text();
809
+ if (binding.epoch !== this.socketEpoch) {
810
+ return;
811
+ }
812
+ this.applyServerMessage(JSON.parse(raw));
813
+ }).catch((error) => {
814
+ this.failTransport(
815
+ binding,
816
+ new Error("Failed to handle a Gemini Live message", { cause: error })
817
+ );
818
+ });
819
+ }
820
+ applyServerMessage(message) {
821
+ if (message.setupComplete) {
822
+ this.log("Gemini Live setup complete");
823
+ this.isSetupComplete = true;
824
+ this.resolvePendingConnect();
825
+ return;
826
+ }
827
+ if (message.usageMetadata) {
828
+ this.latestUsage = message.usageMetadata;
829
+ }
830
+ if (message.toolCall) {
831
+ this.handleToolCall(message.toolCall.functionCalls ?? []);
832
+ }
833
+ for (const id of message.toolCallCancellation?.ids ?? []) {
834
+ this.pendingToolCalls.delete(id);
835
+ }
836
+ const content = message.serverContent;
837
+ if (!content) {
838
+ return;
839
+ }
840
+ if (content.inputTranscription?.text) {
841
+ this.emitEvent({
842
+ type: "user.transcript",
843
+ text: content.inputTranscription.text
844
+ });
845
+ }
846
+ if (this.turnCondemned) {
847
+ if (content.turnComplete) {
848
+ this.handleTurnComplete();
849
+ }
850
+ return;
851
+ }
852
+ if (content.outputTranscription?.text || content.modelTurn) {
853
+ this.modelTurnInFlight = true;
854
+ }
855
+ if (content.outputTranscription?.text) {
856
+ this.appendAssistantText(content.outputTranscription.text);
857
+ }
858
+ for (const part of content.modelTurn?.parts ?? []) {
859
+ if (part.inlineData?.data) {
860
+ this.playbackScheduler?.enqueue(base64ToBytes(part.inlineData.data));
861
+ }
862
+ }
863
+ if (content.interrupted) {
864
+ this.handleInterrupted();
865
+ }
866
+ if (content.turnComplete) {
867
+ this.handleTurnComplete();
868
+ }
869
+ }
870
+ /**
871
+ * Server VAD cut the turn off. The accumulated text deliberately survives:
872
+ * the turn still reports `assistant.response.completed` at its own
873
+ * `turnComplete` with whatever it managed to say, which is the shape the
874
+ * OpenAI client already has for a VAD-cancelled response reporting its
875
+ * `response.done`.
876
+ */
877
+ handleInterrupted() {
878
+ const offset = this.hasStartedAudioOutput && this.playbackAudibleSince !== null ? `${Math.round(performance.now() - this.playbackAudibleSince)}ms after playback started` : "with nothing playing";
879
+ this.log(`Gemini Live turn interrupted ${offset}`);
880
+ this.turnInterrupted = true;
881
+ this.modelTurnInFlight = false;
882
+ this.playbackScheduler?.flush();
883
+ this.endAudioOutput();
884
+ }
885
+ /**
886
+ * Settle the turn that `turnComplete` just closed.
887
+ *
888
+ * Two callers, two worlds. `applyServerMessage`'s condemned branch brings a
889
+ * turn a local `interrupt()` killed: it reports nothing and its audio ended
890
+ * back at the interrupt. The normal path brings a turn that ran: it reports
891
+ * its completion, and may still owe an ending. Everything between is per-turn
892
+ * state both worlds drop.
893
+ */
894
+ handleTurnComplete() {
895
+ const wasCondemned = this.turnCondemned;
896
+ this.turnCondemned = false;
897
+ if (!wasCondemned) {
898
+ this.emitEvent({
899
+ type: "assistant.response.completed",
900
+ text: this.assistantText,
901
+ usage: this.latestUsage
902
+ });
903
+ }
904
+ this.modelTurnInFlight = false;
905
+ this.assistantText = "";
906
+ this.hasStartedAssistantResponse = false;
907
+ this.latestUsage = void 0;
908
+ this.turnCompleteSeen = this.hasStartedAudioOutput;
909
+ if (!this.turnCompleteSeen) {
910
+ return;
911
+ }
912
+ if (this.playbackScheduler?.isIdle()) {
913
+ this.endAudioOutput();
914
+ }
915
+ }
916
+ /**
917
+ * Fan a `toolCall` frame out, one `tool.call` per usable entry, recording the
918
+ * id -> name pairing each answer will need.
919
+ *
920
+ * This frame's shape was written from the API reference and confirmed live
921
+ * only afterwards (`tests/gemini-live-smoke/README.md`, 2026-09-01); the
922
+ * leniency stays. An entry missing either field is *reported* rather than
923
+ * trusted or dropped in silence: without both there is no answer this client
924
+ * can send,
925
+ * and a session that runs on while the model waits forever for a response
926
+ * fails invisibly. It travels as a transport `error`, which `RealtimeManager`
927
+ * handles without ending the session — releasing the send lock, so the user
928
+ * is not stuck. Killing the socket over it would instead livelock: recovery
929
+ * succeeds because the socket was never the problem, the model re-issues the
930
+ * same call on the fresh session, and every lap burns another `uses: 1`
931
+ * token. Strictness belongs on shapes that have been measured.
932
+ *
933
+ * The whole frame is decided before any of it is emitted, so a malformed
934
+ * entry cannot destroy a well-formed sibling whose handler is already
935
+ * running and whose id would be cleared out from under it. The report names
936
+ * the keys that did arrive — enough to read the real shape off one failure —
937
+ * and never quotes argument values.
938
+ *
939
+ * This covers the one absence worth surviving: an entry that arrived and
940
+ * cannot be answered. A payload that is not this shape at all — a null entry,
941
+ * a `functionCalls` that is not an array — still throws into the message
942
+ * pump's fatal catch, and nothing here parses defensively for it.
943
+ */
944
+ handleToolCall(calls) {
945
+ const usable = [];
946
+ const unusable = [];
947
+ for (const call of calls) {
948
+ if (call.id && call.name) {
949
+ usable.push({ id: call.id, name: call.name, args: call.args ?? {} });
950
+ } else {
951
+ unusable.push(`{${Object.keys(call).join(", ")}}`);
952
+ }
953
+ }
954
+ for (const call of usable) {
955
+ this.pendingToolCalls.set(call.id, call.name);
956
+ this.emitEvent({
957
+ type: "tool.call",
958
+ name: call.name,
959
+ args: call.args,
960
+ callId: call.id
961
+ });
962
+ }
963
+ if (unusable.length > 0) {
964
+ this.emitEvent({
965
+ type: "error",
966
+ error: new Error(
967
+ `Gemini Live sent tool calls this client cannot answer: an id or name is missing (keys received: ${unusable.join(" ")})`
968
+ )
969
+ });
970
+ }
971
+ }
972
+ /**
973
+ * Output transcription is the only assistant text a native-audio session
974
+ * produces, and the server keeps it on, so the response's whole lifecycle
975
+ * hangs off these fragments.
976
+ */
977
+ appendAssistantText(text) {
978
+ if (!this.hasStartedAssistantResponse) {
979
+ this.hasStartedAssistantResponse = true;
980
+ this.emitEvent({ type: "assistant.response.started" });
981
+ }
982
+ this.assistantText += text;
983
+ this.emitEvent({ type: "assistant.text.delta", text });
984
+ }
985
+ /**
986
+ * The scheduler started sounding.
987
+ *
988
+ * Only the first such transition of a turn opens the turn's audio. This fires
989
+ * again after every mid-turn underrun, and re-anchoring the gate on those
990
+ * would let a stalling connection hold it open for a whole reply — silently,
991
+ * which is the failure class the measured record warns about. Anchoring once
992
+ * per session is equally wrong: the measured Safari run was interrupted 0.5 s
993
+ * into its *second* turn. So the anchor is per turn, and it is left standing
994
+ * when playback stops, because the window has to expire on its own clock
995
+ * rather than on a silence the acoustic path has not caught up with yet.
996
+ *
997
+ * The unconditional first line is the other half of the pairing: it opens the
998
+ * audible span that `bankAudibleExposure` closes, once per sounding stretch
999
+ * rather than once per turn.
1000
+ */
1001
+ beginAudiblePlayback() {
1002
+ const now = performance.now();
1003
+ this.audibleSpanStartedAt = now;
1004
+ if (this.hasStartedAudioOutput) {
1005
+ return;
1006
+ }
1007
+ this.hasStartedAudioOutput = true;
1008
+ this.playbackAudibleSince = now;
1009
+ this.lipSyncAnalyzer.resume();
1010
+ this.emitEvent({ type: "audio.output.started" });
1011
+ }
1012
+ /**
1013
+ * Close the open audible span into the exposure total.
1014
+ *
1015
+ * Only these playing → not-playing intervals count towards the gate. A
1016
+ * first-audible → `audio.output.ended` span would count silence the canceller
1017
+ * never got to adapt to: `turnComplete` is paced to the audio's duration from
1018
+ * the *first* chunk, so a burst-delivered turn sits quiet for seconds waiting
1019
+ * on it — a measured 12 s span for 10.5 s of audio, and 6.5 s for 5.1 s.
1020
+ */
1021
+ bankAudibleExposure() {
1022
+ if (this.audibleSpanStartedAt === null) {
1023
+ return;
1024
+ }
1025
+ this.audibleExposureMs += performance.now() - this.audibleSpanStartedAt;
1026
+ this.audibleSpanStartedAt = null;
1027
+ }
1028
+ /**
1029
+ * The single place `audio.output.ended` is emitted. Callers decide *that* the
1030
+ * audio ended — always from the scheduler being idle and `turnComplete` seen,
1031
+ * never from a timer or an RMS reading — and this keeps the start/end pair
1032
+ * balanced and settles the gate.
1033
+ *
1034
+ * Both interrupt paths flush the scheduler immediately before calling in.
1035
+ * That flush is what closes the open audible span through
1036
+ * `onPlayingChange(false)`, so the gate below settles against a total that
1037
+ * includes it.
1038
+ */
1039
+ endAudioOutput() {
1040
+ const wasInterrupted = this.turnInterrupted;
1041
+ this.turnInterrupted = false;
1042
+ if (!this.hasStartedAudioOutput) {
1043
+ return;
1044
+ }
1045
+ this.hasStartedAudioOutput = false;
1046
+ this.turnCompleteSeen = false;
1047
+ this.lipSyncAnalyzer.pause();
1048
+ this.emitEvent({ type: "audio.output.ended" });
1049
+ if (this.gateArmed && !wasInterrupted && this.audibleExposureMs >= CONVERGENCE_GATE_MS) {
1050
+ this.gateArmed = false;
1051
+ this.log(
1052
+ `Convergence gate disarmed after ${Math.round(this.audibleExposureMs)}ms of audible playback`
1053
+ );
1054
+ }
1055
+ }
1056
+ /**
1057
+ * The socket a frame can go out on right now, or null. `setupComplete` is
1058
+ * part of the answer: until the server acknowledges setup there is no session
1059
+ * to receive anything. Callers differ only in what they make of a null.
1060
+ */
1061
+ getReadySocket() {
1062
+ const socket = this.activeSocket?.socket;
1063
+ if (!socket || !this.isSetupComplete || socket.readyState !== WebSocket.OPEN) {
1064
+ return null;
1065
+ }
1066
+ return socket;
1067
+ }
1068
+ /**
1069
+ * `getReadySocket()` for the caller-driven frames, which refuse rather than
1070
+ * drop: `sendText()` and `sendToolResult()` have someone to tell.
1071
+ *
1072
+ * The recovery guard stays at those call sites instead of folding in here as
1073
+ * the OpenAI client's does, so a `sendToolResult()` during a reconnect
1074
+ * reports the reconnect rather than the unknown call id it would otherwise
1075
+ * hit — the map is already empty by then.
1076
+ */
1077
+ requireReadySocket() {
1078
+ const socket = this.getReadySocket();
1079
+ if (!socket) {
1080
+ throw new Error("Gemini Live session is not ready to send");
1081
+ }
1082
+ return socket;
1083
+ }
1084
+ /**
1085
+ * The only path microphone audio takes to the wire.
1086
+ *
1087
+ * Frames are dropped rather than queued whenever `getReadySocket()` says they
1088
+ * cannot go out now: a frame held across a reconnect is audio the user spoke
1089
+ * into a session that has ended.
1090
+ */
1091
+ sendCaptureFrame(frame) {
1092
+ const socket = this.getReadySocket();
1093
+ if (!socket) {
1094
+ return;
1095
+ }
1096
+ if (this.isWithinConvergenceGate()) {
1097
+ return;
1098
+ }
1099
+ socket.send(
1100
+ JSON.stringify({
1101
+ realtimeInput: {
1102
+ audio: {
1103
+ data: bytesToBase64(frame),
1104
+ mimeType: `audio/pcm;rate=${INPUT_SAMPLE_RATE}`
1105
+ }
1106
+ }
1107
+ })
1108
+ );
1109
+ }
1110
+ /**
1111
+ * Whether this moment falls inside the convergence gate: still armed, and
1112
+ * within `CONVERGENCE_GATE_MS` of the current turn's playback becoming
1113
+ * audible.
1114
+ */
1115
+ isWithinConvergenceGate() {
1116
+ return this.gateArmed && this.playbackAudibleSince !== null && performance.now() - this.playbackAudibleSince < CONVERGENCE_GATE_MS;
1117
+ }
1118
+ /**
1119
+ * The single settlement point for every way a socket can fail.
1120
+ *
1121
+ * Before `setupComplete` the pending `connect()` owns the failure and settles
1122
+ * immediately — during the initial connect nothing is listening for
1123
+ * `connection.lost`, so waiting out the timeout would hang the caller, and
1124
+ * `connect()`'s catch runs the teardown at the depth the situation calls for.
1125
+ * After setup, the transient reset runs first so `recover()` starts from a
1126
+ * clean session, and the loss is reported once.
1127
+ */
1128
+ failTransport(binding, error) {
1129
+ if (binding.epoch !== this.socketEpoch) {
1130
+ return;
1131
+ }
1132
+ if (this.pendingConnect) {
1133
+ this.rejectPendingConnect(error);
1134
+ return;
1135
+ }
1136
+ this.resetTransportState();
1137
+ this.emitConnectionLost(error);
1138
+ }
1139
+ /**
1140
+ * Transient teardown: everything belonging to one websocket session, and
1141
+ * nothing belonging to the user gesture. The prepared playback context and
1142
+ * the lip-sync analyzer deliberately survive, so an automatic reconnect does
1143
+ * not have to create and resume an `AudioContext` outside a user gesture.
1144
+ */
1145
+ resetTransportState() {
1146
+ this.socketEpoch += 1;
1147
+ this.closeActiveSocket();
1148
+ this.isSetupComplete = false;
1149
+ this.rejectPendingConnect(
1150
+ new Error("Gemini Live session ended before setup completed")
1151
+ );
1152
+ this.lipSyncAnalyzer.pause();
1153
+ this.playbackScheduler?.flush();
1154
+ this.capturePipeline?.stop();
1155
+ this.capturePipeline = null;
1156
+ this.captureStream?.getTracks().forEach((track) => track.stop());
1157
+ this.captureStream = null;
1158
+ this.gateArmed = true;
1159
+ this.playbackAudibleSince = null;
1160
+ this.audibleExposureMs = 0;
1161
+ this.audibleSpanStartedAt = null;
1162
+ this.hasStartedAudioOutput = false;
1163
+ this.turnCompleteSeen = false;
1164
+ this.turnInterrupted = false;
1165
+ this.modelTurnInFlight = false;
1166
+ this.turnCondemned = false;
1167
+ this.hasStartedAssistantResponse = false;
1168
+ this.assistantText = "";
1169
+ this.latestUsage = void 0;
1170
+ this.pendingToolCalls.clear();
1171
+ }
1172
+ /**
1173
+ * Terminal teardown: the transient reset plus everything the user gesture
1174
+ * warmed up. Only an explicit `disconnect()` and a failed initial `connect()`
1175
+ * come here; every recovery path takes the transient reset instead.
1176
+ */
1177
+ cleanup() {
1178
+ this.isCleaningUp = true;
1179
+ this.resetTransportState();
1180
+ this.lipSyncAnalyzer.cleanup().catch(
1181
+ (error) => console.error("Failed to clean up lip-sync analyzer:", error)
1182
+ );
1183
+ const playbackContext = this.playbackContext;
1184
+ this.playbackContext = null;
1185
+ this.playbackScheduler = null;
1186
+ playbackContext?.close().catch(
1187
+ (error) => console.error("Failed to close the playback audio context:", error)
1188
+ );
1189
+ this.connectionLossNotified = false;
1190
+ this.isCleaningUp = false;
1191
+ }
1192
+ closeActiveSocket() {
1193
+ const binding = this.activeSocket;
1194
+ this.activeSocket = null;
1195
+ if (!binding) {
1196
+ return;
1197
+ }
1198
+ binding.socket.onopen = null;
1199
+ binding.socket.onmessage = null;
1200
+ binding.socket.onerror = null;
1201
+ binding.socket.onclose = null;
1202
+ binding.socket.close();
1203
+ }
1204
+ emitConnectionLost(error) {
1205
+ if (this.connectionLossNotified || this.isExplicitDisconnect || this.isCleaningUp) {
1206
+ return;
1207
+ }
1208
+ this.connectionLossNotified = true;
1209
+ this.emitEvent({
1210
+ type: "connection.lost",
1211
+ cause: "connection-failed",
1212
+ error
1213
+ });
1214
+ }
1215
+ resolvePendingConnect() {
1216
+ const pending = this.pendingConnect;
1217
+ if (!pending) {
1218
+ return;
1219
+ }
1220
+ this.pendingConnect = null;
1221
+ clearTimeout(pending.timeoutId);
1222
+ pending.resolve();
1223
+ }
1224
+ rejectPendingConnect(error) {
1225
+ const pending = this.pendingConnect;
1226
+ if (!pending) {
1227
+ return;
1228
+ }
1229
+ this.pendingConnect = null;
1230
+ clearTimeout(pending.timeoutId);
1231
+ pending.reject(error);
1232
+ }
1233
+ async getSessionBootstrap(request) {
1234
+ return getGeminiLiveBootstrap(this.options, request);
1235
+ }
1236
+ emitEvent(event) {
1237
+ for (const callback of this.eventCallbacks) {
1238
+ callback(event);
1239
+ }
1240
+ }
1241
+ log(...args) {
1242
+ if (this.options.debug) {
1243
+ console.log(...args);
1244
+ }
1245
+ }
1246
+ };
1247
+ function bytesToBase64(bytes) {
1248
+ let binary = "";
1249
+ const stride = 32768;
1250
+ for (let offset = 0; offset < bytes.length; offset += stride) {
1251
+ binary += String.fromCharCode(...bytes.subarray(offset, offset + stride));
1252
+ }
1253
+ return btoa(binary);
1254
+ }
1255
+ function base64ToBytes(base64) {
1256
+ const binary = atob(base64);
1257
+ const bytes = new Uint8Array(binary.length);
1258
+ for (let index = 0; index < binary.length; index += 1) {
1259
+ bytes[index] = binary.charCodeAt(index);
1260
+ }
1261
+ return bytes;
1262
+ }
1263
+ function resolveWebSocketBootstrap(bootstrap) {
1264
+ if (bootstrap.adapter !== core.GEMINI_LIVE_ADAPTER || bootstrap.transport !== "websocket") {
1265
+ throw new Error(
1266
+ `Gemini Live client only supports ${core.GEMINI_LIVE_ADAPTER} bootstrap, received ${bootstrap.adapter}/${bootstrap.transport}`
1267
+ );
1268
+ }
1269
+ if (!bootstrap.url || !bootstrap.token) {
1270
+ throw new Error(
1271
+ "Gemini Live bootstrap is missing its websocket url or token"
1272
+ );
1273
+ }
1274
+ return { url: bootstrap.url, token: bootstrap.token };
1275
+ }
1276
+ function createGeminiLiveClient(options) {
1277
+ return new GeminiLiveClient(options);
1278
+ }
1279
+
1280
+ exports.createGeminiLiveClient = createGeminiLiveClient;
1281
+ //# sourceMappingURL=index.js.map
1282
+ //# sourceMappingURL=index.js.map