@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/drivers/geminiRealtimeClient.js

@@ -6,14 +6,18 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 };
 var GeminiRealtimeClient_1;
 import { RegisterClass } from '@memberjunction/global';
-import { RealtimeDiagLog, RealtimeToolBatchBarrier, ExtractToolSchedulingHint, } from '@memberjunction/ai';
-import { GoogleGenAI, FunctionResponseScheduling, } from '@google/genai';
-import { BaseRealtimeClient, REQUESTED_TRACKS_SESSION_KEY } from '../generic/baseRealtimeClient.js';
+import { RealtimeDiagLog, IsPcmAudioMimeType, ParseDurationToMs, RealtimeConnectionSetup, RealtimeSessionResumption, RealtimeToolBatchBarrier, ExtractToolSchedulingHint, REALTIME_RELAY_TIME_LIMIT_CLOSE_CODE, } from '@memberjunction/ai';
+import { GoogleGenAI, FunctionResponseScheduling, Modality, } from '@google/genai';
+import { BaseRealtimeClient, REALTIME_CLIENT_TIME_LIMIT_ERROR_CODE, REQUESTED_TRACKS_SESSION_KEY, } from '../generic/baseRealtimeClient.js';
 import { Base64ToArrayBuffer } from '../audio/pcmUtils.js';
 import { RealtimePcmPlayback } from '../audio/pcmPlayback.js';
 import { RealtimeAudioMeter } from '../audio/audioMeter.js';
 import { CreatePcmMicCapture } from '../audio/micCapture.js';
-import { CreateStreamFrameCapture } from '../media/frameCapture.js';
+import { FrameSampler } from '../media/frameSampler.js';
+import { DEFAULT_INBOUND_VIDEO_RATE, MinVideoFrameSpacingMs } from '../media/videoPacing.js';
+import { VideoSourceArbiter } from '../media/videoSourceArbiter.js';
+import { GEMINI_AVATAR_MP4_TYPE, VideoPlayout } from '../media/videoPlayout.js';
+import { GeminiAvatarOutput } from './geminiAvatarOutput.js';
 // ── Audio constants (Gemini Live wire formats) ─────────────────────────────────
 /** Gemini Live expects client audio as 16-bit signed PCM, 16 kHz, mono. */
 const GEMINI_INPUT_SAMPLE_RATE = 16000;
@@ -21,6 +25,17 @@ const GEMINI_INPUT_SAMPLE_RATE = 16000;
 const GEMINI_INPUT_AUDIO_MIME_TYPE = 'audio/pcm;rate=16000';
 /** Gemini Live emits model audio as 16-bit signed PCM, 24 kHz, mono. */
 const GEMINI_OUTPUT_SAMPLE_RATE = 24000;
+/** How much microphone audio is held while the session moves to a new connection; older audio is dropped first. */
+const HELD_MIC_AUDIO_SECONDS = 2;
+/** {@link HELD_MIC_AUDIO_SECONDS} of 16 kHz, 16-bit mono PCM, as the base64 characters the chunks travel in. */
+const MAX_HELD_MIC_AUDIO_BASE64_CHARS = Math.ceil((GEMINI_INPUT_SAMPLE_RATE * 2 * HELD_MIC_AUDIO_SECONDS * 4) / 3);
+/** The detail field each Gemini `usageMetadata` modality's token count goes into; other modalities are not kept. */
+const GEMINI_MODALITY_TOKEN_FIELDS = {
+    TEXT: 'TextTokens',
+    AUDIO: 'AudioTokens',
+    IMAGE: 'ImageTokens',
+    VIDEO: 'VideoTokens',
+};
 // ── Legacy video-capability fallback ───────────────────────────────────────────
 //
 // Video capability and its rate ceiling are per-model data, minted from the provider's profile
@@ -33,6 +48,8 @@ const GEMINI_OUTPUT_SAMPLE_RATE = 24000;
 const LEGACY_VIDEO_MODEL_PREFIX = 'gemini-3.8-live';
 /** The frame-rate ceiling this client hardcoded before `MaxInboundVideoRate` was minted. */
 const LEGACY_VIDEO_MODEL_RATE = 1;
+/** The arbiter source id of the camera stream passed to `Connect`. */
+const CONNECT_CAMERA_SOURCE_ID = 'connect-camera';
 // ── Production playback engine ─────────────────────────────────────────────────
 /**
  * Web Audio playout scheduler for Gemini's 24 kHz PCM16 model audio.
@@ -62,21 +79,36 @@ export class GeminiPcmPlayback extends RealtimePcmPlayback {
  *   ({@link createMicCapture} seam) and streamed with `sendRealtimeInput`.
  * - **Audio out**: model PCM16 @ 24 kHz chunks scheduled through {@link GeminiPcmPlayback}
  *   ({@link createPlayback} seam); {@link IsAudioPlaying} is computed from the playout clock.
+ * - **Avatar out**: when the minted config grants an avatar and the host shows it (the outbound
+ *   video track is live), fragmented MP4 parts play through a {@link VideoPlayout}
+ *   ({@link CreateVideoPlayout} seam) handed to the host at connect, whose audio routes into the
+ *   PCM playback's graph; PCM follows the avatar's voice rule (see `GeminiAvatarOutput`). A host
+ *   that shows no avatar gets an audio-only session.
  * - **Event translation** (Gemini → contract): `inputTranscription` → User deltas/finals,
  *   `outputTranscription` → Assistant deltas/finals (accumulated like the OpenAI driver),
  *   `toolCall.functionCalls` → {@link OnToolCall} (callID→name cached for
  *   {@link SendToolResult}), `interrupted` → playback flush + `'listening'`,
  *   `turnComplete` → busy cleared + queued sends flushed, `usageMetadata` → {@link OnUsage}
- *   (per-turn prompt/response token deltas — see {@link handleUsageMetadata}).
+ *   (per-turn prompt/response token deltas with their modality split — see {@link handleUsageMetadata}).
+ *   An avatar's generated video seconds go to {@link OnUsage} in updates of their own
+ *   (`OutputTokenDetails.VideoSeconds`) at each turn boundary.
  * - **Busy mapping**: Gemini has no `response.created` frame, so `IsBusy` is set EAGERLY when
  *   this client triggers a response (text / narration / tool result) and on the first model
  *   output of a turn (audio part or output-transcription delta); cleared on `turnComplete`,
  *   and on a `toolCall` frame (the model has yielded the floor pending the tool result — so a
- *   slow `turnComplete` can never deadlock the queued result).
+ *   slow `turnComplete` can never deadlock the queued result). In an avatar session, video sets
+ *   it only while the model's answer is under way: Vertex AI streams the avatar's video between
+ *   answers too, and that idle video plays without marking the agent speaking or busy.
  * - **Collision safety**: ANY `sendClientContent` interrupts in-flight Gemini generation (per
  *   the Live API contract), so text / narration / context-note / tool-result sends issued
  *   while a turn is in flight are queued and flushed in order on `turnComplete` (the flush
  *   stops at the first send that starts a new response).
+ * - **Moving to a new connection**: Google ends a connection after about 10 minutes, announcing
+ *   it with `goAway`, and the session resumes on a new one with Google's handle
+ *   ({@link RealtimeSessionResumption}). A planned move waits until no turn is in progress
+ *   ({@link IsTurnOpen}). A connection is put to use only once Google confirms its setup
+ *   (`setupComplete`): input that reaches it earlier is dropped. During a move, sends queue and
+ *   the latest 2 s of mic audio are held; both go out on the new connection once it is ready.
  * - **Open-turn commit**: context notes ride as `turnComplete: false` client content, which
  *   tells the Live API MORE INPUT IS COMING — the server holds ALL generation (including the
  *   normally-automatic continuation after a tool response) until a `turnComplete: true`
@@ -105,13 +137,49 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         this.micStream = null;
         this.cameraStream = null;
         this.micCapture = null;
-        this.cameraCapture = null;
+        this.cameraSampler = null;
         this.playback = null;
+        /**
+         * The avatar half of the session: created at connect when the server granted an avatar and the host shows it (the
+         * outbound video track is live). `null` otherwise, and the session then plays model output as it always has.
+         */
+        this.avatarOutput = null;
         this.firstVideoSendTimestamp = 0;
         this.lastVideoSendTimestamp = 0;
         this.videoFramesSent = 0;
-        this.resumptionHandle = null;
-        this.lastConnectArgs = null;
+        /**
+         * Moves the session to a new connection with Google's resumption handle: when Google
+         * announces the connection is ending (`goAway`, with the time left; on Vertex AI it came about
+         * 9 minutes into a connection, with 30 s left) and after an unexpected drop. A planned move
+         * waits until no turn is in progress ({@link IsTurnOpen}). Created per {@link Connect}.
+         */
+        this.resumption = null;
+        /**
+         * Whether input can go out: the connection in use has confirmed its setup (`setupComplete`) and
+         * no move to a new connection is in progress. Google drops input that reaches a connection
+         * before its setup completes, and a connection being replaced resumes from a handle issued
+         * before the move, so what it receives never reaches the new one. While `false`, sends wait in
+         * {@link queuedSends} and microphone audio in {@link heldMicAudio}.
+         */
+        this.connectionReady = false;
+        /** Microphone audio captured during a move, sent on the new connection once it is ready. */
+        this.heldMicAudio = [];
+        /** Base64 characters in {@link heldMicAudio}, kept under {@link MAX_HELD_MIC_AUDIO_BASE64_CHARS}. */
+        this.heldMicAudioChars = 0;
+        /** See {@link IsTurnOpen}. */
+        this.turnOpen = false;
+        /** A turn's `turnComplete` came while a tool call was pending: the turn ends once the last pending call's result goes out. */
+        this.turnEndAwaitsToolResults = false;
+        /** Model, config and token of the current connection; a resume reuses them with the new handle. */
+        this.connectTarget = null;
+        /** Last connection number handed out by {@link openConnection}. */
+        this.issuedConnections = 0;
+        /**
+         * Number of the connection in use. Callbacks from any other connection (one that was
+         * replaced, closed, or is still opening) are ignored, so a replaced socket's close can't end
+         * the session that replaced it. `0` while no connection is in use.
+         */
+        this.currentConnection = 0;
         // ── Model capability & profile state ───────────────────────────────────────
         this.idleSignal = 'turnComplete';
         this.supportsScheduling = true;
@@ -126,6 +194,8 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         this.pendingUserText = '';
         /** Accumulates in-flight thought text deltas until finalized on turn completion. */
         this.pendingThoughtText = '';
+        /** MIME types of model output this session dropped, so each is reported once. */
+        this.droppedOutputTypes = new Set();
         /** True while a model turn is in flight; gates (queues) client-triggered sends. */
         this.responseActive = false;
         /** The kind of the turn currently in flight; stamped at send time, reset on turnComplete. */
@@ -159,7 +229,19 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
     static { this.ASSISTANT_SAFETY_BACKSTOP_MS = 15000; }
     /** Returns the latest session resumption handle reported by the server, if any. */
     get ResumptionHandle() {
-        return this.resumptionHandle;
+        return this.resumption?.Handle ?? null;
+    }
+    /**
+     * Whether a turn is in progress. A turn opens at its first sign: the user's first transcribed words, a turn this
+     * client sends (typed text, a spoken update), or the model's output (a transcription delta, a thought, a tool call,
+     * an audio part, which in an avatar session must play as the voice, or, on an `interactionStatus` model, the
+     * `IN_PROGRESS` status; the video an avatar streams between turns never opens one). It closes at the turn's
+     * `turnComplete` (the true idle signal on an `interactionStatus` model) once no tool call is pending, or when the last
+     * pending call's result goes out after that; and at a resume. The session moves to a new connection only between
+     * turns.
+     */
+    get IsTurnOpen() {
+        return this.turnOpen;
     }
     /** Returns the count of video frames successfully sent over the established video track. */
     get VideoFramesSent() {
@@ -184,20 +266,60 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         return !this.supportsBlocking;
     }
     // ── BaseRealtimeClient: connection lifecycle ───────────────────────────────
-    /**
-     * Opens the client-direct Gemini Live session: creates the playout engine, connects with
-     * the ephemeral token + the server-built `SessionConfig` (`{ model, config }` — the same
-     * values the server LOCKED into the token, so tampering is ignored by the API), then wires
-     * the mic-capture worklet. Reports `'listening'` once audio is flowing.
-     */
     /**
      * Opens the client-direct Gemini Live session: creates the playout engine, connects with
      * the ephemeral token + the server-built `SessionConfig` (`{ model, config }` — the same
      * values the server LOCKED into the token, so tampering is ignored by the API), negotiates
      * tracks, then wires the mic-capture worklet and optional video capture.
-     * Reports `'listening'` once audio is flowing.
+     * Reports `'connected'` once Google confirms the session's setup (`setupComplete`): input
+     * sent before that is dropped, so nothing goes out on the connection, and the microphone
+     * does not start, until then. Reports `'listening'` once audio is flowing.
+     *
+     * When the server granted an avatar (the minted `avatar` block) and the host shows it (the
+     * outbound video track is live), the avatar's video goes to the host once the connection is
+     * open. When the host doesn't show it, the session connects audio only.
+     *
+     * @throws When the connection closes or fails before Google confirms its setup, or no
+     *   confirmation comes in time (see `RealtimeConnectionSetup`).
      */
     async Connect(config, micStream, cameraStream) {
+        this.AssertTransportSupported(config);
+        this.resetForConnect(micStream, cameraStream);
+        this.setState('connecting');
+        const session = this.parseSessionConfig(config);
+        this.idleSignal = session.idleSignal;
+        this.supportsScheduling = session.supportsScheduling;
+        this.supportsBlocking = session.supportsBlocking;
+        this.negotiateSessionTracks(session);
+        const playback = this.createPlayback();
+        this.playback = playback;
+        // The agent voice plays through Web Audio only; publish it so a host recorder can mix
+        // it in (issue #5153). Null for playbacks with no output stream (fakes, no WebAudio).
+        this.publishRemoteMediaStream(playback.GetOutputStream?.() ?? null);
+        const liveConfig = this.prepareAvatar(session, playback);
+        this.resumption?.Dispose();
+        this.resumption = this.createResumption();
+        const target = {
+            Model: session.model,
+            Config: liveConfig,
+            EphemeralToken: config.EphemeralToken,
+            Transport: config.Transport,
+            RelayUrl: config.RelayUrl,
+        };
+        this.connectTarget = target;
+        if (!(await this.openFirstConnection(target))) {
+            return;
+        }
+        this.setState('connected');
+        // Handed over now rather than at the first part, so the avatar's tile is up before the agent speaks.
+        if (this.avatarOutput) {
+            this.emitRemoteVideo(this.avatarOutput.Source);
+        }
+        await this.startLocalCapture(micStream);
+        this.setState('listening');
+    }
+    /** Takes the streams for a new connect and clears what an earlier one left behind. */
+    resetForConnect(micStream, cameraStream) {
         this.micStream = micStream;
         this.cameraStream = cameraStream ?? null;
         this.clearSafetyBackstop();
@@ -205,18 +327,53 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         this.firstVideoSendTimestamp = 0;
         this.lastVideoSendTimestamp = 0;
         this.videoFramesSent = 0;
-        this.setState('connecting');
-        const { model, liveConfig, idleSignal, supportsScheduling, supportsBlocking, supportsInboundVideo, maxInboundVideoRate, requestedTracks } = this.parseSessionConfig(config);
-        this.idleSignal = idleSignal;
-        this.supportsScheduling = supportsScheduling;
-        this.supportsBlocking = supportsBlocking;
-        // Negotiate tracks. Video capability and its frame-rate ceiling are PER-MODEL DATA, minted
-        // from the provider's profile table (GeminiLiveModelProfile.SupportsInboundVideo /
-        // .MaxInboundVideoRate) and carried in the session config. Deriving either from the model
-        // id would put a second answer to the same question in a second place: the two agreed only
-        // because the model names happened to line up, and the next model to break that pattern
-        // would diverge silently.
-        const isVideoModel = supportsInboundVideo ?? model.toLowerCase().startsWith(LEGACY_VIDEO_MODEL_PREFIX);
+        this.resetConnectionState();
+        this.disposeAvatarOutput();
+    }
+    /** No connection ready, no audio held, no turn open: the state of a session before its first connection. */
+    resetConnectionState() {
+        this.connectionReady = false;
+        this.heldMicAudio = [];
+        this.heldMicAudioChars = 0;
+        this.turnOpen = false;
+        this.turnEndAwaitsToolResults = false;
+    }
+    /**
+     * Opens the session's first connection and puts it to use once Google confirms its setup.
+     *
+     * @returns `false` when the consumer disconnected while the setup was in flight; the connection is then closed, and a
+     *   setup failure after the disconnect is not reported.
+     */
+    async openFirstConnection(target) {
+        const opened = await this.openConnection(target);
+        try {
+            await GeminiRealtimeClient_1.awaitSetup(opened);
+        }
+        catch (error) {
+            if (this.connectTarget !== target) {
+                return false;
+            }
+            throw error;
+        }
+        if (this.connectTarget !== target) {
+            GeminiRealtimeClient_1.closeQuietly(opened.Session);
+            return false;
+        }
+        this.useConnection(opened);
+        this.connectionReady = true;
+        return true;
+    }
+    /**
+     * Negotiates tracks. Video capability and its frame-rate ceiling are PER-MODEL DATA, minted
+     * from the provider's profile table (GeminiLiveModelProfile.SupportsInboundVideo /
+     * .MaxInboundVideoRate) and carried in the session config. Deriving either from the model
+     * id would put a second answer to the same question in a second place: the two agreed only
+     * because the model names happened to line up, and the next model to break that pattern
+     * would diverge silently. A granted avatar adds an outbound video track when this browser
+     * can play it.
+     */
+    negotiateSessionTracks(session) {
+        const isVideoModel = session.supportsInboundVideo ?? session.model.toLowerCase().startsWith(LEGACY_VIDEO_MODEL_PREFIX);
         const supportedTracks = [
             { Modality: 'audio', Direction: 'inbound' },
             { Modality: 'audio', Direction: 'outbound' },
@@ -228,38 +385,110 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
                 Encoding: 'image/jpeg',
                 // The model's own ceiling. ResolveRequestedTracks takes the more restrictive of
                 // this and what the session requested, so this is what bounds the live track.
-                Rate: maxInboundVideoRate ?? LEGACY_VIDEO_MODEL_RATE,
+                Rate: session.maxInboundVideoRate ?? LEGACY_VIDEO_MODEL_RATE,
                 UsageBasis: ['tokens', 'frames'],
                 RequiresConsent: true,
             });
         }
-        this.negotiateTracks(requestedTracks, supportedTracks);
-        this.playback = this.createPlayback();
-        const connectArgs = {
-            Model: model,
-            Config: liveConfig,
-            EphemeralToken: config.EphemeralToken,
-            OnMessage: (message) => this.handleServerMessage(message),
-            OnError: (event) => this.handleTransportError(event),
-            OnClose: (event) => this.handleTransportClose(event),
-        };
-        this.lastConnectArgs = connectArgs;
-        this.session = await this.connectLiveSession(connectArgs);
-        this.setState('connected');
+        const avatarTrack = session.avatar ? GeminiRealtimeClient_1.avatarTrackFor(session.avatar) : null;
+        if (avatarTrack) {
+            supportedTracks.push(avatarTrack);
+        }
+        // How many inbound video streams the model takes, minted from the profile like the rate. A mint
+        // that predates the field falls back to one stream for a video model (every model that had video
+        // took exactly one) and none otherwise.
+        this.negotiateTracks(session.requestedTracks, supportedTracks, isVideoModel ? (session.maxInboundVideoStreams ?? 1) : 0);
+    }
+    /** The outbound video track a granted avatar offers, or `null` when this browser can't play its type through MSE. */
+    static avatarTrackFor(grant) {
+        const encoding = grant.Encoding ?? GEMINI_AVATAR_MP4_TYPE;
+        if (!VideoPlayout.IsSupported(encoding)) {
+            return null;
+        }
+        return { Modality: 'video', Direction: 'outbound', Encoding: encoding, UsageBasis: ['seconds'] };
+    }
+    /**
+     * Sets up the avatar the server granted, when the host shows it (the outbound video track is
+     * live). Otherwise the session runs audio only: the config sent at connect asks for audio and
+     * no avatar, and one line says why. Returns the config to connect with.
+     */
+    prepareAvatar(session, playback) {
+        const grant = session.avatar;
+        if (!grant) {
+            return session.liveConfig;
+        }
+        if (!this.IsTrackEstablished('video', 'outbound')) {
+            console.warn(this.avatarDowngradeMessage(session.liveConfig, grant));
+            return GeminiRealtimeClient_1.withoutAvatar(session.liveConfig);
+        }
+        const playout = this.CreateVideoPlayout({
+            MimeType: grant.Encoding ?? GEMINI_AVATAR_MP4_TYPE,
+            CarriesVoice: grant.AudioMuxed,
+            // The avatar's voice plays through the PCM engine's graph, so the meter and the recording carry it.
+            OnElementAttached: (element) => playback.ConnectMediaElement?.(element),
+        });
+        this.avatarOutput = new GeminiAvatarOutput(playout, playback, grant);
+        return session.liveConfig;
+    }
+    /**
+     * The one line for a granted avatar this session won't show, in the server driver's format: `browser` when the host
+     * asked for the agent's video and this browser can't play it, `host` when the host asked for none.
+     */
+    avatarDowngradeMessage(config, grant) {
+        const requested = this.AllTracks.some((t) => t.Descriptor.Direction === 'outbound' && String(t.Descriptor.Modality).trim().toLowerCase() === 'video');
+        const why = requested ? `this browser cannot play ${grant.Encoding ?? GEMINI_AVATAR_MP4_TYPE}` : 'the host shows no agent video';
+        const reason = requested ? 'browser' : 'host';
+        return `[GeminiRealtimeClient] Avatar "${config.avatarConfig?.avatarName ?? ''}" not used: ${why}. The call is audio only. Reason: ${reason}.`;
+    }
+    /** The connect config of a session whose avatar won't show: audio out, and no avatar. */
+    static withoutAvatar(config) {
+        const { avatarConfig: _avatar, ...rest } = config;
+        return { ...rest, responseModalities: [Modality.AUDIO] };
+    }
+    /** Starts the microphone capture, the camera passed to `Connect` when inbound video is live, and both audio meters. */
+    async startLocalCapture(micStream) {
         this.micCapture = await this.createMicCapture(micStream, (base64Pcm16) => this.sendMicChunk(base64Pcm16));
-        // Start camera capture if inbound video is established and cameraStream provided
         if (this.cameraStream && this.IsTrackEstablished('video', 'inbound')) {
-            this.cameraCapture = CreateStreamFrameCapture(this.cameraStream, {
-                Rate: 1,
-                OnFrame: (frame) => this.SendVideoFrame(frame.data, frame.mimeType),
-            });
+            this.cameraSampler = this.startConnectCamera(this.cameraStream);
         }
         // Audio-activity capability (base obligation #9): agent side taps the playout
         // engine's master gain; user side meters the mic stream. Null-safe — test fakes /
         // no-WebAudio environments simply leave the session un-metered.
         this.attachOutputAudioMeter(this.playback?.CreateMeter?.() ?? null);
         this.attachInputAudioMeter(RealtimeAudioMeter.ForMicStream(micStream));
-        this.setState('listening');
+    }
+    /**
+     * Stops the avatar's video and releases its element; the session then has no avatar. The video it generated and
+     * did not yet report is emitted first.
+     */
+    disposeAvatarOutput() {
+        this.emitAvatarVideoSeconds();
+        this.avatarOutput?.Dispose();
+        this.avatarOutput = null;
+    }
+    /**
+     * Samples the camera passed to {@link Connect} at the rate the video track negotiated and feeds it to
+     * the session's {@link VideoSourceArbiter} as a `'camera'` source. The arbiter stays the only writer of
+     * inbound video, and it tells the model when it switches to the camera.
+     */
+    startConnectCamera(stream) {
+        const arbiter = VideoSourceArbiter.ForSink(this);
+        arbiter.RegisterSource({ SourceID: CONNECT_CAMERA_SOURCE_ID, Label: 'Camera', Kind: 'camera' });
+        const sampler = new FrameSampler(stream, {
+            Rate: this.InboundVideoRate ?? DEFAULT_INBOUND_VIDEO_RATE,
+            OnFrame: (frame) => arbiter.PushFrame(CONNECT_CAMERA_SOURCE_ID, frame.Data, frame.MimeType),
+        });
+        sampler.Start();
+        return sampler;
+    }
+    /** Stops sampling the `Connect` camera and removes it from the arbiter. Does not stop its tracks. */
+    stopConnectCamera() {
+        if (!this.cameraSampler) {
+            return;
+        }
+        this.cameraSampler.Stop();
+        this.cameraSampler = null;
+        VideoSourceArbiter.ForSink(this).UnregisterSource(CONNECT_CAMERA_SOURCE_ID);
     }
     /**
      * Tears down the session, mic capture, mic tracks, camera capture, and playout engine,
@@ -272,15 +501,21 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         this.toolBatchBarrier.Clear();
         this.micStream?.getTracks().forEach((track) => track.stop());
         this.micStream = null;
-        this.cameraCapture?.Stop();
-        this.cameraCapture = null;
+        this.stopConnectCamera();
         this.cameraStream?.getTracks().forEach((track) => track.stop());
         this.cameraStream = null;
         this.micCapture?.Stop();
         this.micCapture = null;
+        this.disposeAvatarOutput();
         this.playback?.Close();
         this.playback = null;
-        this.resumptionHandle = null;
+        // Stop resuming before the socket closes, and drop events still in flight from it.
+        this.resumption?.Dispose();
+        this.resumption = null;
+        this.connectTarget = null;
+        this.currentConnection = 0;
+        this.resetConnectionState();
+        this.clearRemoteMediaStream();
         this.firstVideoSendTimestamp = 0;
         this.lastVideoSendTimestamp = 0;
         this.videoFramesSent = 0;
@@ -323,25 +558,32 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
     /**
      * Streams one base64 image frame over the established inbound video track.
      *
-     * Enforces a 750ms minimum inter-frame spacing to serve as a backstop with deliberate jitter
-     * headroom for 1 fps (1000ms) pacers (such as `ChannelInboundVideoBridge`'s `setInterval`,
-     * `frameCapture`, and channel-level gates like `OnScreencastFrame`'s 1000ms pacer). Upstream
-     * cadence generators and channel gates are the primary enforcers of the nominal 1 fps ceiling,
-     * while this 750ms gate absorbs event loop and async dispatch jitter without dropping intended
-     * 1Hz frames, while preventing any unpaced callers from bursting above 1.33 fps.
+     * Enforces a minimum inter-frame spacing as a backstop with deliberate jitter headroom for the
+     * upstream pacers (the source arbiter, `ChannelInboundVideoBridge`, `frameCapture`, channel-level
+     * gates such as `OnScreencastFrame`). The spacing is derived from the rate the track NEGOTIATED
+     * ({@link MinVideoFrameSpacingMs} of {@link BaseRealtimeClient.InboundVideoRate}), so it follows the
+     * model's profile rather than a constant: at the 1 fps every Live model accepts today it is the
+     * 750 ms gate this driver has always applied, and a faster model needs no change here. Upstream
+     * generators are the primary enforcers of the nominal rate; this gate absorbs event-loop and async
+     * dispatch jitter without dropping intended frames, while preventing an unpaced caller from bursting.
+     *
+     * Gemini accepts ONE inbound video stream, so `sourceId` is ignored: choosing which source feeds that
+     * stream is the arbiter's job, and by the time a frame arrives here the choice is made.
      *
      * If inbound video is not established, returns `false` without error or frame sends (fallback).
+     * A frame that comes while no connection is ready (before Google confirms the setup, or while
+     * the session moves to a new connection) is dropped too: the source sends the next one at the track's rate.
      *
-     * @returns `true` if the frame was dispatched to the session; `false` if dropped (throttled
-     *   or track unestablished).
+     * @returns `true` if the frame was dispatched to the session; `false` if dropped (throttled,
+     *   track unestablished, or no connection ready).
      */
-    SendVideoFrame(base64Image, mimeType = 'image/jpeg') {
-        if (!this.IsTrackEstablished('video', 'inbound')) {
+    SendVideoFrame(base64Image, mimeType = 'image/jpeg', _sourceId) {
+        if (!this.IsTrackEstablished('video', 'inbound') || !this.connectionReady) {
             return false;
         }
         const now = Date.now();
-        if (this.lastVideoSendTimestamp > 0 && now - this.lastVideoSendTimestamp < 750) {
-            return false; // Throttled: 750ms jitter headroom backstop for upstream 1 fps pacers (Reviewer Items 25, 30, 33)
+        if (this.lastVideoSendTimestamp > 0 && now - this.lastVideoSendTimestamp < MinVideoFrameSpacingMs(this.InboundVideoRate)) {
+            return false; // Throttled: jitter-headroom backstop for upstream pacers (Reviewer Items 25, 30, 33)
         }
         this.lastVideoSendTimestamp = now;
         if (this.firstVideoSendTimestamp === 0) {
@@ -364,7 +606,7 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      * by a cancel). Server-side, the next client content sent naturally interrupts any
      * residual generation per the Live API contract. The interrupted turn's accumulated
      * transcript is kept — the provider's trailing frames finalize what WAS spoken. No-op
-     * when nothing is active.
+     * when nothing is active. An avatar's video stops with the voice.
      */
     CancelActiveResponse() {
         if (!this.session) {
@@ -375,6 +617,7 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         }
         this.clearSafetyBackstop();
         this.playback?.Flush();
+        this.avatarOutput?.Cancel();
         this.responseActive = false;
         this.interactionInProgress = false;
         this.activeResponseKind = 'normal';
@@ -435,12 +678,9 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             return;
         }
         const name = this.pendingToolCallNames.get(callID) ?? '';
-        if (this.isNonBlocking) {
-            this.sendToolResponseTurn(callID, name, outputJson);
-        }
-        else {
-            this.enqueueOrRun(() => this.sendToolResponseTurn(callID, name, outputJson));
-        }
+        // Non-blocking results go out while the model talks; blocking ones wait for the turn in flight. Both wait while
+        // the session moves to a new connection.
+        this.enqueueOrRun(() => this.sendToolResponseTurn(callID, name, outputJson), !this.isNonBlocking);
     }
     /**
      * Mutes / unmutes by toggling the mic tracks' `enabled` flag: the capture pipeline stays
@@ -454,6 +694,15 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             track.enabled = !muted;
         }
     }
+    /** Rebinds the PCM capture and the input meter to the stream's current track (obligation #10); the socket stays open. */
+    async ReplaceMicrophone(micStream) {
+        if (!this.micCapture) {
+            return;
+        }
+        this.micCapture.Rebind(micStream);
+        this.micStream = micStream;
+        this.attachInputAudioMeter(RealtimeAudioMeter.ForMicStream(micStream));
+    }
     /** @inheritdoc */
     get IsBusy() {
         if (this.isNonBlocking) {
@@ -468,28 +717,46 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      *
      * Computed directly from the playout engine's playhead clock — this client OWNS the output
      * buffer (no WebRTC playback events exist on Gemini), so "audibly playing" is precisely
-     * "scheduled audio extends beyond the audio context's current time".
+     * "scheduled audio extends beyond the audio context's current time". In an avatar session
+     * the avatar's video counts too, while some of an answer's media is still ahead of its
+     * playhead: its MP4 usually carries the voice. The idle video that follows an answer does
+     * not count.
      */
     get IsAudioPlaying() {
-        return this.playback?.IsPlaying ?? false;
+        return (this.playback?.IsPlaying ?? false) || (this.avatarOutput?.IsPlaying ?? false);
     }
     // ── Overridable creation seams (tests inject fakes — no network / audio) ──
     /**
      * Creation seam for the Gemini Live session. Production constructs a `GoogleGenAI` client
      * with the **ephemeral token as the API key** on the `v1alpha` API version (the only
      * version that accepts `auth_tokens`), then opens `live.connect` with the server-built
-     * model + config. Unit tests override this to return an in-memory fake.
+     * model + config. Rejects if the socket errors or closes before Gemini acknowledges the setup.
+     * Unit tests override this to return an in-memory fake.
      */
     async connectLiveSession(args) {
         const ai = new GoogleGenAI({ apiKey: args.EphemeralToken, httpOptions: { apiVersion: 'v1alpha' } });
-        return ai.live.connect({
-            model: args.Model,
-            config: args.Config,
-            callbacks: {
+        return this.connectUntilSetup(args, (callbacks) => ai.live.connect({ model: args.Model, config: args.Config, callbacks }));
+    }
+    /**
+     * Opens a Live connection with `connect`, which passes the given callbacks to `live.connect`, and rejects if the
+     * socket errors or closes before Gemini acknowledges the setup. `live.connect` (@google/genai 2.11+) resolves only on
+     * the server's setupComplete frame and never rejects when the socket errors or closes first (a rejected token, model
+     * or setup field), so without this the connect waits forever. A promise settles once, so callbacks after a successful
+     * connect only reach the handlers. Every {@link connectLiveSession} override should open through this.
+     */
+    connectUntilSetup(args, connect) {
+        return new Promise((resolve, reject) => {
+            connect({
                 onmessage: args.OnMessage,
-                onerror: args.OnError,
-                onclose: args.OnClose,
-            },
+                onerror: (event) => {
+                    reject(new Error(`Gemini Live connection failed: ${event?.message ?? 'websocket error'}`));
+                    args.OnError(event);
+                },
+                onclose: (event) => {
+                    reject(new Error(`Gemini Live closed during setup (code ${event?.code ?? 'unknown'}${event?.reason ? `: ${event.reason}` : ''})`));
+                    args.OnClose(event);
+                },
+            }).then(resolve, reject);
         });
     }
     /**
@@ -506,6 +773,17 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
     createPlayback() {
         return new GeminiPcmPlayback();
     }
+    /**
+     * Creation seam for the avatar's video player, called at connect when the server granted an
+     * avatar and the host shows it. Production returns {@link VideoPlayout} (Media Source
+     * Extensions); unit tests return a fake.
+     *
+     * @param options The type the grant names, whether the video carries the voice, and the hook
+     *   that routes the element's audio into the PCM playback's Web Audio graph.
+     */
+    CreateVideoPlayout(options) {
+        return new VideoPlayout(options);
+    }
     // ── Connection internals ───────────────────────────────────────────────────
     /**
      * Extracts the model + Live connect config from the server-minted `SessionConfig`
@@ -527,36 +805,69 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         const supportsInboundVideo = typeof sessionConfig['supportsInboundVideo'] === 'boolean' ? sessionConfig['supportsInboundVideo'] : undefined;
         const rawMaxVideoRate = sessionConfig['maxInboundVideoRate'];
         const maxInboundVideoRate = typeof rawMaxVideoRate === 'number' && rawMaxVideoRate > 0 ? rawMaxVideoRate : undefined;
-        const rawRequestedTracks = sessionConfig[REQUESTED_TRACKS_SESSION_KEY];
-        let requestedTracks = undefined;
-        if (Array.isArray(rawRequestedTracks)) {
-            const list = [];
-            for (const item of rawRequestedTracks) {
-                if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
-                    const modality = typeof item['Modality'] === 'string' ? item['Modality'] : undefined;
-                    const direction = item['Direction'];
-                    if (modality && (direction === 'inbound' || direction === 'outbound')) {
-                        list.push({
-                            Modality: modality,
-                            Direction: direction,
-                            Encoding: typeof item['Encoding'] === 'string' ? item['Encoding'] : undefined,
-                            Rate: typeof item['Rate'] === 'number' ? item['Rate'] : undefined,
-                            RequiresConsent: typeof item['RequiresConsent'] === 'boolean' ? item['RequiresConsent'] : undefined,
-                        });
-                    }
+        const rawMaxStreams = sessionConfig['maxInboundVideoStreams'];
+        const maxInboundVideoStreams = typeof rawMaxStreams === 'number' && Number.isInteger(rawMaxStreams) && rawMaxStreams >= 0 ? rawMaxStreams : undefined;
+        return {
+            model, liveConfig, idleSignal, supportsScheduling, supportsBlocking, supportsInboundVideo, maxInboundVideoRate,
+            maxInboundVideoStreams,
+            requestedTracks: GeminiRealtimeClient_1.readRequestedTracks(sessionConfig[REQUESTED_TRACKS_SESSION_KEY]),
+            avatar: GeminiRealtimeClient_1.readAvatarGrant(sessionConfig['avatar']),
+        };
+    }
+    /** The tracks the runtime asks the session for (`requestedTracks`); `undefined` when it sent no list. */
+    static readRequestedTracks(raw) {
+        if (!Array.isArray(raw)) {
+            return undefined;
+        }
+        const list = [];
+        for (const item of raw) {
+            if (item !== null && typeof item === 'object' && !Array.isArray(item)) {
+                const modality = typeof item['Modality'] === 'string' ? item['Modality'] : undefined;
+                const direction = item['Direction'];
+                if (modality && (direction === 'inbound' || direction === 'outbound')) {
+                    list.push({
+                        Modality: modality,
+                        Direction: direction,
+                        Encoding: typeof item['Encoding'] === 'string' ? item['Encoding'] : undefined,
+                        Rate: typeof item['Rate'] === 'number' ? item['Rate'] : undefined,
+                        RequiresConsent: typeof item['RequiresConsent'] === 'boolean' ? item['RequiresConsent'] : undefined,
+                        SourceID: typeof item['SourceID'] === 'string' ? item['SourceID'] : undefined,
+                        Label: typeof item['Label'] === 'string' ? item['Label'] : undefined,
+                    });
                 }
             }
-            requestedTracks = list;
         }
-        return { model, liveConfig, idleSignal, supportsScheduling, supportsBlocking, supportsInboundVideo, maxInboundVideoRate, requestedTracks };
+        return list;
+    }
+    /**
+     * The avatar the server granted: the minted `avatar` block (`{ output: true, encoding, audioMuxed }`), which the
+     * server writes only when the session renders one. `audioMuxed` defaults to true, as the server's profile does.
+     */
+    static readAvatarGrant(raw) {
+        const block = GeminiRealtimeClient_1.readObject(raw);
+        if (block?.['output'] !== true) {
+            return null;
+        }
+        return { Encoding: GeminiRealtimeClient_1.readString(block['encoding']) ?? null, AudioMuxed: block['audioMuxed'] !== false };
     }
-    /** Streams one base64 PCM16 mic chunk to the model (no-op once the session is gone, closed, or in error). */
+    /**
+     * Streams one base64 PCM16 mic chunk to the model (no-op once the session is gone, closed, or in error). During a
+     * move to a new connection the chunk is held instead, and goes out on the new connection once it is ready.
+     */
     sendMicChunk(base64Pcm16) {
         if (!this.session || this.currentState === 'closed' || this.currentState === 'error') {
             return;
         }
+        if (!this.connectionReady) {
+            this.holdMicChunk(base64Pcm16);
+            return;
+        }
+        this.sendAudio(this.session, base64Pcm16);
+    }
+    /** Sends one base64 PCM16 mic chunk on `session`; a failed send is logged, not thrown. */
+    sendAudio(session, base64Pcm16) {
         try {
-            this.session.sendRealtimeInput({
+            session.sendRealtimeInput({
                 audio: { data: base64Pcm16, mimeType: GEMINI_INPUT_AUDIO_MIME_TYPE },
             });
         }
@@ -564,21 +875,69 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             RealtimeDiagLog(`[GeminiRealtimeClient] sendMicChunk failed: ${err instanceof Error ? err.message : String(err)}`);
         }
     }
-    /** Surfaces a fatal websocket error and marks the session unusable. */
+    /** Holds a mic chunk during a move, keeping the latest {@link HELD_MIC_AUDIO_SECONDS} seconds; older audio goes first. */
+    holdMicChunk(base64Pcm16) {
+        this.heldMicAudio.push(base64Pcm16);
+        this.heldMicAudioChars += base64Pcm16.length;
+        while (this.heldMicAudioChars > MAX_HELD_MIC_AUDIO_BASE64_CHARS && this.heldMicAudio.length > 1) {
+            this.heldMicAudioChars -= this.heldMicAudio.shift()?.length ?? 0;
+        }
+    }
+    /** Sends the mic audio held during a move on the connection now in use, oldest first. */
+    sendHeldMicAudio() {
+        const held = this.heldMicAudio;
+        this.heldMicAudio = [];
+        this.heldMicAudioChars = 0;
+        if (!this.session) {
+            return;
+        }
+        for (const chunk of held) {
+            this.sendAudio(this.session, chunk);
+        }
+    }
+    /**
+     * Surfaces a fatal websocket error and marks the session unusable. When the session can be
+     * resumed, the error is only logged: a websocket `error` is always followed by a `close`,
+     * and {@link handleTransportClose} resumes from there.
+     */
     handleTransportError(event) {
-        const detail = event.message || (event.error instanceof Error ? event.error.message : String(event.error ?? 'unknown'));
+        const detail = GeminiRealtimeClient_1.transportErrorDetail(event);
         RealtimeDiagLog(`[GeminiRealtimeClient] Transport error: ${detail}`);
+        if (this.resumption?.Handle) {
+            return;
+        }
         this.emitError({ Message: `Gemini Live transport error: ${detail}`, Fatal: true });
         this.setState('error');
     }
-    /** Reflects a provider-side close (unless the session already ended in error). */
+    /**
+     * Handles a close of the current connection that the consumer did not ask for. MJAPI's relay
+     * closing at the session's time limit ends the call with a `time-limit` error. Otherwise, when
+     * Google issued a resumption handle, the session reconnects with it; else the close ends the
+     * session, as an error when it was abnormal.
+     */
     handleTransportClose(event) {
         const code = event?.code;
         const reason = event?.reason;
         const wasClean = event?.wasClean;
         RealtimeDiagLog(`[GeminiRealtimeClient] Transport closed: code=${code} reason=${reason} wasClean=${wasClean}`);
+        if (this.currentState === 'error' || this.currentState === 'closed') {
+            return;
+        }
+        if (code === REALTIME_RELAY_TIME_LIMIT_CLOSE_CODE) {
+            // The relay session reached its time limit: it is gone, so a resume could only fail. End with its own code.
+            this.emitError({
+                Message: `Gemini Live session reached its time limit (${code}): ${reason || 'no reason given'}`,
+                Code: REALTIME_CLIENT_TIME_LIMIT_ERROR_CODE,
+                Fatal: true,
+            });
+            this.setState('error');
+            return;
+        }
+        if (this.resumption?.ConnectionLost()) {
+            return;
+        }
         const isAbnormal = (code !== undefined && code !== 0 && code !== 1000 && code !== 1005) || (wasClean === false && code !== 1000 && code !== 0 && code !== 1005 && code !== undefined);
-        if (isAbnormal && this.currentState !== 'error') {
+        if (isAbnormal) {
             this.emitError({
                 Message: `Gemini Live connection closed (${code}): ${reason || 'unexpected disconnect'}`,
                 Fatal: true,
@@ -586,9 +945,7 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             this.setState('error');
             return;
         }
-        if (this.currentState !== 'error' && this.currentState !== 'closed') {
-            this.setState('closed');
-        }
+        this.setState('closed');
     }
     // ── Inbound message translation ────────────────────────────────────────────
     /**
@@ -597,20 +954,14 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      */
     handleServerMessage(message) {
         this.checkInteractionStatus(message);
-        // Session continuity: track resumption token updates (F7)
+        // Session continuity (F7). `resumable: false` (mid-turn, mid-tool-call) arrives with no
+        // handle; an update without the flag is treated as resumable.
         if (message.sessionResumptionUpdate) {
-            if (message.sessionResumptionUpdate.resumable === false) {
-                this.resumptionHandle = null;
-            }
-            else if (message.sessionResumptionUpdate.newHandle) {
-                this.resumptionHandle = message.sessionResumptionUpdate.newHandle;
-            }
+            const update = message.sessionResumptionUpdate;
+            this.resumption?.RecordHandle(update.newHandle, update.resumable !== false);
         }
-        // Server approaching timeout / abort: reconnect seamlessly using resumption handle (F7)
         if (message.goAway) {
-            if (this.resumptionHandle) {
-                void this.resumeSession(this.resumptionHandle);
-            }
+            this.resumption?.ConnectionEnding(ParseDurationToMs(message.goAway.timeLeft));
         }
         if (message.serverContent) {
             this.handleServerContent(message.serverContent);
@@ -623,33 +974,151 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         }
     }
     /**
-     * Resumes the live session using a previously captured session resumption handle (F7).
+     * Opens a replacement connection that resumes the session from `handle`, switches to it once
+     * Google confirms its setup, and closes the old one. Until then the old connection stays in
+     * use, so what it still sends (the rest of a turn the deadline cut, its usage) is handled.
+     * Mic capture and playout carry over untouched. Rejects when the connection can't be opened or
+     * its setup fails, so {@link RealtimeSessionResumption} can retry.
+     *
+     * The resume reuses the session's ephemeral token, which Google accepts until the token's
+     * `expireTime` (the server driver mints 30 minutes). After that every attempt fails and the
+     * session ends with a fatal error.
+     *
+     * @param handle The resumption handle Google issued.
+     * @param attempt Marks the attempt abandoned after a timeout or a consumer close; a
+     *                connection that opens after that is closed instead of used.
      */
-    async resumeSession(handle) {
-        if (!this.lastConnectArgs) {
+    async resumeSession(handle, attempt) {
+        const target = this.connectTarget;
+        if (!target) {
+            throw new Error('there is no Gemini Live session to resume');
+        }
+        const resumeTarget = {
+            ...target,
+            Config: { ...target.Config, sessionResumption: { ...target.Config.sessionResumption, handle } },
+        };
+        const opened = await this.openConnection(resumeTarget);
+        await GeminiRealtimeClient_1.awaitSetup(opened);
+        if (attempt.Abandoned || this.connectTarget !== target) {
+            GeminiRealtimeClient_1.closeQuietly(opened.Session);
             return;
         }
+        const previous = this.session;
+        this.useConnection(opened);
+        this.connectTarget = resumeTarget;
+        if (previous) {
+            GeminiRealtimeClient_1.closeQuietly(previous);
+        }
+    }
+    /**
+     * Opens a connection through {@link connectLiveSession}. Its callbacks act only while it is the
+     * connection in use, so events from one that was replaced, closed, or is still opening are
+     * dropped; its `setupComplete`, and a close or error before it, settle its {@link GeminiOpenedConnection.Setup}
+     * whether or not it is in use yet. It becomes the connection in use through {@link useConnection},
+     * which lets the old connection keep delivering events while a planned move is in progress.
+     */
+    async openConnection(target) {
+        const number = ++this.issuedConnections;
+        const isCurrent = () => number === this.currentConnection;
+        const setup = new RealtimeConnectionSetup();
+        const session = await this.connectLiveSession({
+            ...target,
+            OnMessage: (message) => {
+                if (message.setupComplete) {
+                    setup.Confirm();
+                }
+                if (isCurrent()) {
+                    this.handleServerMessage(message);
+                }
+            },
+            OnError: (event) => {
+                setup.Fail(new Error(`Gemini Live connection failed before confirming its setup: ${GeminiRealtimeClient_1.transportErrorDetail(event)}`));
+                if (isCurrent()) {
+                    this.handleTransportError(event);
+                }
+            },
+            OnClose: (event) => {
+                setup.Fail(new Error(`Gemini Live closed the connection before confirming its setup (${event?.code}): ${event?.reason || 'no reason given'}`));
+                if (isCurrent()) {
+                    this.handleTransportClose(event);
+                }
+            },
+        });
+        return { Session: session, ConnectionNumber: number, Setup: setup };
+    }
+    /** Waits for Google to confirm a new connection's setup. On failure the connection is closed and the error rethrown. */
+    static async awaitSetup(opened) {
         try {
-            const reconnectArgs = {
-                ...this.lastConnectArgs,
-                Config: {
-                    ...this.lastConnectArgs.Config,
-                    sessionResumption: { handle },
-                },
-            };
-            const oldSession = this.session;
-            const newSession = await this.connectLiveSession(reconnectArgs);
-            this.session = newSession;
-            this.lastConnectArgs = reconnectArgs;
-            try {
-                oldSession?.close();
-            }
-            catch {
-                // The old socket has already been replaced by newSession, so a close failure on it cannot affect the new session
-            }
+            await opened.Setup.Wait();
+        }
+        catch (error) {
+            GeminiRealtimeClient_1.closeQuietly(opened.Session);
+            throw error;
+        }
+    }
+    /** What a websocket error event says went wrong. */
+    static transportErrorDetail(event) {
+        return event.message || (event.error instanceof Error ? event.error.message : String(event.error ?? 'unknown'));
+    }
+    /** Makes an opened connection the one in use. */
+    useConnection(opened) {
+        this.session = opened.Session;
+        this.currentConnection = opened.ConnectionNumber;
+    }
+    /**
+     * Builds the resumption helper for a newly connected session; see {@link resumption}. From the
+     * moment a move starts, input waits for the new connection ({@link connectionReady}).
+     */
+    createResumption() {
+        return new RealtimeSessionResumption({
+            Reconnect: (handle, attempt) => this.resumeSession(handle, attempt),
+            OnReconnecting: (reason) => {
+                RealtimeDiagLog(`[GeminiRealtimeClient] Resuming the session on a new connection (${reason})`);
+                this.connectionReady = false;
+                this.setState('connecting');
+            },
+            OnReconnected: () => this.handleResumed(),
+            OnReconnectFailed: (error) => this.handleResumeFailed(error),
+            Log: (message) => RealtimeDiagLog(message),
+        });
+    }
+    /**
+     * The session continues on a new connection, whose setup Google has confirmed. A turn cut off
+     * by a drop or the deadline never completes there, so the turn state is reset (its partial
+     * transcripts are emitted as final). The mic audio held during the move goes out, then the
+     * sends queued behind it. An avatar's video plays out what arrived of that turn and holds its
+     * last frame.
+     */
+    handleResumed() {
+        this.avatarOutput?.Resumed();
+        this.emitAvatarVideoSeconds();
+        this.finalizeUserTranscript();
+        this.finalizeAssistantTranscript();
+        this.responseActive = false;
+        this.interactionInProgress = false;
+        this.activeResponseKind = 'normal';
+        this.clearSafetyBackstop();
+        this.endTurn();
+        this.connectionReady = true;
+        this.setState('listening');
+        this.sendHeldMicAudio();
+        this.flushQueuedSends();
+    }
+    /** Every resume attempt failed: the session ends with a fatal error. */
+    handleResumeFailed(error) {
+        if (this.currentState === 'closed' || this.currentState === 'error') {
+            return;
+        }
+        this.emitError({ Message: `Gemini Live connection was lost and could not be resumed: ${error.message}`, Fatal: true });
+        this.setState('error');
+    }
+    /** Closes a socket the session no longer uses; it may already be closed. */
+    static closeQuietly(session) {
+        try {
+            session.close();
         }
         catch (err) {
-            RealtimeDiagLog(`[GeminiRealtimeClient] Session resumption failed: ${err instanceof Error ? err.message : String(err)}`);
+            RealtimeDiagLog(`[GeminiRealtimeClient] Closing a replaced connection failed: ${err instanceof Error ? err.message : String(err)}`);
         }
     }
     /**
@@ -671,6 +1140,8 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             this.interactionInProgress = true;
             this.responseActive = true;
             if (this.idleSignal === 'interactionStatus') {
+                // The turn ends at the true IDLE (handleIdleTerminal), so the status that opens it counts only here.
+                this.markTurnOpen();
                 this.scheduleSafetyBackstop();
             }
         }
@@ -695,24 +1166,9 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      * the server-bridged `GeminiRealtime` driver forwards the same payload to `IRealtimeSession.OnUsage`.
      */
     handleUsageMetadata(usageMetadata) {
-        let inputDetails;
-        if (usageMetadata.promptTokensDetails && Array.isArray(usageMetadata.promptTokensDetails)) {
-            for (const detail of usageMetadata.promptTokensDetails) {
-                if (typeof detail.tokenCount === 'number') {
-                    inputDetails = inputDetails ?? {};
-                    const mod = String(detail.modality ?? '').toUpperCase();
-                    if (mod === 'AUDIO') {
-                        inputDetails.AudioTokens = (inputDetails.AudioTokens ?? 0) + detail.tokenCount;
-                    }
-                    else if (mod === 'TEXT') {
-                        inputDetails.TextTokens = (inputDetails.TextTokens ?? 0) + detail.tokenCount;
-                    }
-                    else if (mod === 'IMAGE') {
-                        inputDetails.ImageTokens = (inputDetails.ImageTokens ?? 0) + detail.tokenCount;
-                    }
-                }
-            }
-        }
+        let inputDetails = GeminiRealtimeClient_1.modalityTokens(usageMetadata.promptTokensDetails);
+        // The response side: TEXT and AUDIO, and VIDEO for a generated avatar (when Google reports it).
+        const outputDetails = GeminiRealtimeClient_1.modalityTokens(usageMetadata.responseTokensDetails);
         /**
          * Cost Attribution Note (F6 & Reviewer Item 29):
          * Inbound video frames are sent as individual JPEG images (V5) and billed on the video pricing tier
@@ -730,14 +1186,13 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
          * and provider-received ImageTokens immediately surfaces frame drops or network throttling in production.
          */
         if (this.videoFramesSent > 0) {
-            inputDetails = inputDetails ?? {};
-            inputDetails.VideoFrames = this.videoFramesSent;
-            inputDetails.VideoSeconds = this.VideoSeconds;
+            inputDetails = { ...inputDetails, VideoFrames: this.videoFramesSent, VideoSeconds: this.VideoSeconds };
         }
         this.emitUsage({
             InputTokens: typeof usageMetadata.promptTokenCount === 'number' ? usageMetadata.promptTokenCount : undefined,
             OutputTokens: typeof usageMetadata.responseTokenCount === 'number' ? usageMetadata.responseTokenCount : undefined,
             ...(inputDetails ? { InputTokenDetails: inputDetails } : {}),
+            ...(outputDetails ? { OutputTokenDetails: outputDetails } : {}),
             ...(this.videoFramesSent > 0 ? {
                 VideoFrames: this.videoFramesSent,
                 VideoSeconds: this.VideoSeconds,
@@ -745,13 +1200,53 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             Raw: usageMetadata,
         });
     }
-    /** Translates one {@link LiveServerContent} frame in provider-documented signal order. */
+    /**
+     * Gemini's per-modality token counts (`promptTokensDetails` or `responseTokensDetails`) as a detail block: TEXT,
+     * AUDIO, IMAGE and VIDEO. `undefined` when Gemini sent no count for a modality it names.
+     */
+    static modalityTokens(details) {
+        if (!Array.isArray(details)) {
+            return undefined;
+        }
+        let block;
+        for (const detail of details) {
+            const field = GEMINI_MODALITY_TOKEN_FIELDS[String(detail.modality ?? '').toUpperCase()];
+            if (field && typeof detail.tokenCount === 'number') {
+                block = block ?? {};
+                block[field] = (block[field] ?? 0) + detail.tokenCount;
+            }
+        }
+        return block;
+    }
+    /**
+     * Emits the avatar video generated since the last emission as an update of its own
+     * (`OutputTokenDetails.VideoSeconds`, an amount): called where a turn's counting ends (generation complete,
+     * turn complete, resumed; an interrupted turn ends at its turn complete) and before the avatar output is
+     * disposed, so every second is emitted once, whenever Google's `usageMetadata` arrives.
+     */
+    emitAvatarVideoSeconds() {
+        const seconds = this.avatarOutput?.TakeVideoSeconds() ?? 0;
+        if (seconds > 0) {
+            this.emitUsage({ OutputTokenDetails: { VideoSeconds: seconds } });
+        }
+    }
+    /**
+     * Translates one {@link LiveServerContent} frame in provider-documented signal order. What the frame carries opens
+     * the turn before its `turnComplete` ends it.
+     */
     handleServerContent(content) {
+        if (this.contentOpensTurn(content)) {
+            this.markTurnOpen();
+        }
+        if (this.avatarOutput && GeminiRealtimeClient_1.hasText(content.outputTranscription)) {
+            // The model speaks: the avatar's video is its answer's from here, this message's parts included.
+            this.avatarOutput.AnswerStarted();
+        }
         if (content.interrupted) {
             this.handleInterruption();
         }
         if (content.modelTurn) {
-            this.handleModelAudio(content.modelTurn);
+            this.handleModelMedia(content.modelTurn);
             this.handleModelThoughts(content.modelTurn);
         }
         if (content.inputTranscription) {
@@ -773,8 +1268,12 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      *
      * Per Reviewer Item 20: Draining the queue happens on turnComplete or true IDLE, not prematurely
      * on generationComplete. We set responseActive = false so busy state reflects token completion.
+     * An avatar's video gets its end of turn here: playback runs to the true end and holds the
+     * last frame.
      */
     handleGenerationComplete() {
+        this.avatarOutput?.GenerationComplete();
+        this.emitAvatarVideoSeconds();
         if (this.idleSignal === 'turnComplete') {
             this.responseActive = false;
         }
@@ -785,16 +1284,23 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
      * client's audio queue), surface the TRUE barge-in to the host (Gemini only emits
      * `interrupted` when user input actually cut off in-flight generation — no extra gating
      * needed), and give the floor back. The interrupted turn's accumulated transcript is
-     * kept — the following `turnComplete` finalizes what WAS spoken.
+     * kept — the following `turnComplete` finalizes what WAS spoken. In an avatar session the
+     * video stops too, and the turn's late parts are dropped until its `turnComplete`.
      */
     handleInterruption() {
         this.playback?.Flush();
+        this.avatarOutput?.Interrupted();
         this.finalizeThoughtTranscript();
         this.emitInterruption();
         this.setState('listening');
     }
-    /** Decodes inline model-audio parts (base64 PCM16 @ 24 kHz) into the playout queue. */
-    handleModelAudio(modelTurn) {
+    /**
+     * Plays inline model parts. In an avatar session the avatar output routes each part to the
+     * video, to the PCM playback or nowhere; only the answer's parts mark generation started
+     * (and the turn open): the idle video between answers plays without making the agent
+     * speaking or busy. Otherwise base64 PCM16 @ 24 kHz parts go to the playout queue.
+     */
+    handleModelMedia(modelTurn) {
         if (!modelTurn.parts) {
             return;
         }
@@ -802,13 +1308,35 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             if (part.thought) {
                 continue; // Thoughts are reasoning summaries, never spoken audio
             }
-            const data = part.inlineData?.data;
-            if (data) {
-                this.markGenerationStarted();
-                this.playback?.Enqueue(Base64ToArrayBuffer(data));
+            const inline = part.inlineData;
+            if (!inline?.data) {
+                continue;
+            }
+            if (this.avatarOutput) {
+                if (this.avatarOutput.Accept(inline.mimeType, Base64ToArrayBuffer(inline.data)) === 'answer') {
+                    this.markTurnOpen();
+                    this.markGenerationStarted();
+                }
+                continue;
+            }
+            // A part that names a non-PCM type (e.g. video/mp4 avatar frames) must never reach PCM
+            // playback, where it would play as noise. A part with no type plays, as it always has.
+            if (inline.mimeType && !IsPcmAudioMimeType(inline.mimeType)) {
+                this.reportDroppedOutput(inline.mimeType);
+                continue;
             }
+            this.markGenerationStarted();
+            this.playback?.Enqueue(Base64ToArrayBuffer(inline.data));
         }
     }
+    /** Reports each MIME type of dropped model output once per session, not once per part. */
+    reportDroppedOutput(mimeType) {
+        if (this.droppedOutputTypes.has(mimeType)) {
+            return;
+        }
+        this.droppedOutputTypes.add(mimeType);
+        console.warn(`[GeminiRealtimeClient] Dropped model output of type ${mimeType}: only PCM audio is played on this session.`);
+    }
     /**
      * Extracts thought parts (`part.thought === true`) from model turns and emits them
      * as narration transcript deltas (`Kind: 'narration'`). Thought summaries are reasoning
@@ -882,6 +1410,8 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         if (!functionCalls || functionCalls.length === 0) {
             return;
         }
+        // The answer stops at the call: the avatar's video until the model speaks again is idle.
+        this.avatarOutput?.ToolCalled();
         if (!this.isNonBlocking) {
             if (this.currentState === 'speaking') {
                 this.currentState = 'connected';
@@ -895,21 +1425,27 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             this.toolBatchBarrier.TrackPendingCall(callID, () => {
                 this.handleToolBatchTimeout();
             });
+            // A pending call keeps its turn open until its result goes out (see IsTurnOpen).
+            this.markTurnOpen();
             this.emitToolCall({ CallID: callID, ToolName: toolName, ArgumentsJson: JSON.stringify(call.args ?? {}) });
         }
     }
     /**
      * Turn boundary: finalize any un-finished assistant transcript.
-     * Under 'turnComplete' idle signal, release the busy lock, reset response kind,
-     * drain queued sends, and return the floor to the user.
+     * Under 'turnComplete' idle signal, end the turn (see {@link IsTurnOpen}), release the busy
+     * lock, reset response kind, drain queued sends, and return the floor to the user.
      * Under 'interactionStatus' (Extended Thinking), turnComplete does NOT indicate idle:
-     * background reasoning or async tool calls may still be in flight, so the busy lock
+     * background reasoning or async tool calls may still be in flight, so the turn, the busy lock
      * and queued sends remain held until the true IDLE signal lands.
      */
     handleTurnComplete() {
+        this.avatarOutput?.TurnComplete();
+        this.emitAvatarVideoSeconds();
         this.finalizeAssistantTranscript();
         this.finalizeThoughtTranscript();
         if (this.idleSignal === 'turnComplete') {
+            // Before the drain: a queued send that starts a new turn opens it again.
+            this.endTurnUnlessToolCallPending();
             this.responseActive = false;
             this.activeResponseKind = 'normal';
             this.openClientTurn = false; // the completed generation consumed any open client content
@@ -925,11 +1461,12 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
     }
     /**
      * Reached when the server emits true IDLE (interactionStatus) or the safety backstop fires.
-     * Releases busy state, commits any deferred open client turns without cutting off generation,
-     * drains queued sends, and returns the floor.
+     * Ends the turn (see {@link IsTurnOpen}), releases busy state, commits any deferred open client
+     * turns without cutting off generation, drains queued sends, and returns the floor.
      */
     handleIdleTerminal() {
         this.clearSafetyBackstop();
+        this.endTurnUnlessToolCallPending();
         this.interactionInProgress = false;
         this.responseActive = false;
         this.activeResponseKind = 'normal';
@@ -944,9 +1481,9 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         }
     }
     /**
-     * First model output of a turn (audio part or transcription delta): the user's turn is
-     * over (finalize their pending transcript), the model is busy, and the client is audibly /
-     * imminently `'speaking'`.
+     * First model output of a turn (audio part, transcription delta, or a part of an avatar's
+     * answer): the user's turn is over (finalize their pending transcript), the model is busy,
+     * and the client is audibly / imminently `'speaking'`.
      */
     markGenerationStarted() {
         this.finalizeUserTranscript();
@@ -983,15 +1520,73 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             this.emitTranscript({ Role: 'Assistant', Text: text, IsFinal: true, Kind: 'narration', IsThought: true });
         }
     }
+    // ── Turn tracking (see IsTurnOpen) ─────────────────────────────────────────
+    /**
+     * Whether a server content message is a sign of a turn: the user's or the model's transcribed
+     * words, a thought, or (in a session without an avatar) the model's audio. An avatar's video
+     * never is: Vertex AI streams it between turns too.
+     */
+    contentOpensTurn(content) {
+        if (GeminiRealtimeClient_1.hasText(content.inputTranscription) || GeminiRealtimeClient_1.hasText(content.outputTranscription)) {
+            return true;
+        }
+        return (content.modelTurn?.parts ?? []).some((part) => this.partOpensTurn(part));
+    }
+    /**
+     * Whether a model part is a sign of a turn: a thought, or audio (as this client plays it) in a session without an
+     * avatar. In an avatar session, a voice part opens the turn once it plays ({@link handleModelMedia}).
+     */
+    partOpensTurn(part) {
+        if (part.thought) {
+            return !!part.text;
+        }
+        const inline = part.inlineData;
+        return !this.avatarOutput && !!inline?.data && (!inline.mimeType || IsPcmAudioMimeType(inline.mimeType));
+    }
+    /** Whether a transcription frame carries words (whitespace alone does not count). */
+    static hasText(transcription) {
+        return (transcription?.text?.trim().length ?? 0) > 0;
+    }
+    /** A turn started or continued: a planned move waits until it ends. */
+    markTurnOpen() {
+        this.turnOpen = true;
+        this.turnEndAwaitsToolResults = false;
+        this.resumption?.TurnStarted();
+    }
+    /** The turn's `turnComplete` (or true idle): it ends now, or once the results of the tool calls still pending go out. */
+    endTurnUnlessToolCallPending() {
+        if (this.pendingToolCallNames.size > 0) {
+            this.turnEndAwaitsToolResults = this.turnOpen;
+            return;
+        }
+        this.endTurn();
+    }
+    /** A tool result went out: a turn whose `turnComplete` waited on the pending calls ends once none is left. */
+    endTurnIfToolResultsIn() {
+        if (this.turnEndAwaitsToolResults && this.pendingToolCallNames.size === 0) {
+            this.endTurn();
+        }
+    }
+    /** The turn is over: a planned move happens at the next resumable handle Google issues. */
+    endTurn() {
+        this.turnOpen = false;
+        this.turnEndAwaitsToolResults = false;
+        this.resumption?.TurnEnded();
+    }
     // ── Collision-safe send machinery ──────────────────────────────────────────
     /**
      * Runs a send immediately when no turn is in flight; otherwise queues it for the next
      * `turnComplete`. This is Gemini's equivalent of the OpenAI driver's
      * queue-behind-active-response rule — stricter here because ANY client content interrupts
-     * in-flight generation on the Live API.
+     * in-flight generation on the Live API. Every send also waits while no connection is ready
+     * ({@link connectionReady}): during a move it goes out on the new connection.
+     *
+     * @param send The send.
+     * @param waitsForTurn Whether the send waits for a turn in flight (`false` for a non-blocking
+     *   tool result, which goes out while the model talks).
      */
-    enqueueOrRun(send) {
-        if (this.responseActive) {
+    enqueueOrRun(send, waitsForTurn = true) {
+        if (!this.connectionReady || (waitsForTurn && this.responseActive)) {
             this.queuedSends.push(send);
             return;
         }
@@ -999,10 +1594,11 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
     }
     /**
      * Drains queued sends in order on a turn boundary, stopping as soon as one starts a new
-     * turn (sets {@link responseActive}) — the rest wait for that turn's completion.
+     * turn (sets {@link responseActive}) — the rest wait for that turn's completion. Nothing
+     * drains while no connection is ready: {@link handleResumed} drains onto the new one.
      */
     flushQueuedSends() {
-        while (!this.responseActive && this.queuedSends.length > 0) {
+        while (!this.responseActive && this.connectionReady && this.queuedSends.length > 0) {
             const send = this.queuedSends.shift();
             send?.();
         }
@@ -1027,6 +1623,7 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
         // `openClientTurn` is left as-is — the model `turnComplete` that follows clears it.
         session.sendRealtimeInput({ text });
         this.responseActive = true;
+        this.markTurnOpen();
         if (this.idleSignal === 'interactionStatus') {
             this.interactionInProgress = true;
             this.scheduleSafetyBackstop();
@@ -1101,6 +1698,7 @@ let GeminiRealtimeClient = class GeminiRealtimeClient extends BaseRealtimeClient
             this.setState('speaking');
         }
         this.pendingToolCallNames.delete(callID);
+        this.endTurnIfToolResultsIn();
     }
     /**
      * Parses a JSON-stringified tool result into the structured object Gemini's