@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/drivers/geminiAvatarOutput.js

@@ -0,0 +1,310 @@
+/**
+ * @fileoverview The avatar half of a Gemini Live browser session: where each model part goes once the server has granted
+ * an avatar and the host shows it, and what each turn boundary does to the avatar's video and voice.
+ *
+ * - **Parts.** A part that opens with an MP4 box (`ftyp`, `moov`, `moof`, `styp`) goes to the video player whatever MIME
+ *   type it names, and so does a `video/*` part, each as a `RealtimeVideoFrame` (Core's `Fmp4PieceToVideoFrame`: an init
+ *   segment or a fragment, with its time and key-frame flag). A `video/*` part that is neither labelled `video/mp4` nor
+ *   opens with an MP4 box is dropped and reported once per type. A PCM part follows the voice rule below; a part with no
+ *   MIME type that is not MP4 is PCM. Any other part is dropped and reported once per type: unknown data never plays.
+ * - **Who carries the voice.** The grant says whether the avatar's MP4 carries the voice (`audioMuxed`); the session's
+ *   first readable init segment decides: an audio track means it does, none means the video plays muted and the voice
+ *   comes as PCM. A disagreement with the grant is logged once.
+ * - **The voice rule.** When the video carries the voice, a PCM part plays only while the turn has no video yet; the
+ *   turn's first video part flushes the PCM the turn queued, and later PCM in that turn is dropped (reported once), so
+ *   the voice never plays twice. Otherwise PCM always plays, not lip-synced to the video; one line per turn logs how far
+ *   apart the two started.
+ * - **Turns.** `generationComplete` ends the turn's video, so it plays to its true end and holds the last frame;
+ *   `turnComplete` is the fallback. `interrupted` stops the video at once and drops the turn's late parts until its
+ *   `turnComplete`. A resume lets a turn the drop cut off play out what arrived, without a flush.
+ * - **Answer or idle.** Vertex AI streams the avatar's video between answers too. The model's answer is under way from
+ *   its first spoken words (the client reports its output transcription with {@link GeminiAvatarOutput.AnswerStarted})
+ *   or a PCM voice part that plays, until its `generationComplete`, the turn's `turnComplete`, a barge-in or cancel, a
+ *   tool call, or a resume. Video inside the answer is the answer's; video outside it is idle: it plays (the avatar keeps
+ *   its face) but is not the agent speaking. {@link GeminiAvatarOutput.Accept} says which, so the client marks the agent
+ *   speaking and busy for the answer only, and {@link GeminiAvatarOutput.IsPlaying} counts the avatar only while some of
+ *   the answer's media is still ahead of the player's playhead.
+ * - **Usage.** The seconds of video the model generated count from the fragments' durations (Core's reader, timed by
+ *   the latest init segment): video that arrives inside a model turn, from its first part until `generationComplete`;
+ *   an interrupted turn counts what arrived before `interrupted`. Video after `generationComplete` (idle frames while
+ *   the avatar listens) does not count. A flush never subtracts: generated video is billed whether or not it played.
+ *   The count does not follow the answer above: all video from the session's start, a `turnComplete` or a resume until
+ *   the next `generationComplete` counts, idle or not, while how Google bills avatar video is open (#5312).
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+import { Fmp4PieceToVideoFrame, Fmp4VideoSeconds, IsPcmAudioMimeType, ReadFmp4Init, SniffFmp4Piece } from '@memberjunction/ai';
+function newTurn() {
+    return { HasVideo: false, HasVoice: false, VideoEnded: false, Dropping: false, GenerationComplete: false, VoiceStartedAt: null, VideoStartedAt: null };
+}
+/**
+ * Routes a Gemini Live avatar session's model parts to the avatar's video player and the session's PCM playback, applies
+ * each turn boundary to both, and tells the answer's media from the idle video between answers. The Gemini client
+ * creates one at connect, only when the server granted an avatar and the host established the outbound video track.
+ */
+export class GeminiAvatarOutput {
+    /**
+     * @param playout The avatar's video player, created with `CarriesVoice` from the grant.
+     * @param voice The session's PCM playback.
+     * @param grant What the server minted.
+     */
+    constructor(playout, voice, grant) {
+        this.playout = playout;
+        this.voice = voice;
+        this.grant = grant;
+        this.reported = new Set();
+        this.turn = newTurn();
+        this.voiceDecided = false;
+        /** The latest readable init segment: its video track's timescale times the fragments. */
+        this.init = null;
+        /** Seconds of video generated and not yet taken by {@link TakeVideoSeconds}. */
+        this.pendingVideoSeconds = 0;
+        /** Whether the model's answer is under way (see the module's "Answer or idle"): video outside it is idle. */
+        this.answering = false;
+        /** Frames handed to the player so far. */
+        this.framesHanded = 0;
+        /** {@link framesHanded} just after the latest frame of an answer: the answers' media ends there. 0 until one. */
+        this.answerEndsAtFrame = 0;
+        this.videoCarriesVoice = grant.AudioMuxed;
+    }
+    /** The avatar's video, for the host. */
+    get Source() {
+        return this.playout.Source;
+    }
+    /**
+     * Whether the avatar is audibly answering: its video plays with media buffered ahead of the playhead, and the playhead
+     * has not yet passed the last frame of an answer (the player's {@link IAvatarVideoPlayout.FramesAhead}). The idle
+     * video that follows an answer plays on, but does not count.
+     */
+    get IsPlaying() {
+        if (this.answerEndsAtFrame === 0 || !this.playout.IsPlaying) {
+            return false;
+        }
+        const played = this.framesHanded - this.playout.FramesAhead;
+        return played < this.answerEndsAtFrame;
+    }
+    /**
+     * Plays one model part, or drops it, and says which: the answer's video or voice (the caller then counts the turn's
+     * generation as started), idle video (it plays, but no answer is under way, so the agent is not speaking), or dropped.
+     *
+     * @param mimeType The part's `inlineData.mimeType`, when it names one.
+     * @param data The part's bytes.
+     */
+    Accept(mimeType, data) {
+        if (this.turn.Dropping) {
+            return 'dropped';
+        }
+        const kind = GeminiAvatarOutput.kindOf(mimeType, data);
+        if (kind === 'video') {
+            return this.acceptVideo(mimeType, data);
+        }
+        if (kind === 'voice') {
+            return this.acceptVoice(data);
+        }
+        this.reportOnce(`type:${mimeType}`, `[GeminiRealtimeClient] Dropped model output of type ${mimeType}: an avatar session plays its video and PCM audio only.`);
+        return 'dropped';
+    }
+    /**
+     * The model's answer is under way: its output transcription has words. The video that follows is the answer's until
+     * its generation completes, the turn completes, a barge-in or cancel stops it, or the model calls a tool.
+     */
+    AnswerStarted() {
+        this.answering = true;
+    }
+    /** The model called a tool: its answer stops there, and the video until it speaks again is idle. */
+    ToolCalled() {
+        this.answering = false;
+    }
+    /**
+     * Takes the seconds of avatar video generated since the last call, for the session's usage, and starts counting
+     * from 0 again: each second is taken once.
+     */
+    TakeVideoSeconds() {
+        const seconds = this.pendingVideoSeconds;
+        this.pendingVideoSeconds = 0;
+        return seconds;
+    }
+    /**
+     * `generationComplete`: the turn's media is all in, so its video plays to the true end and holds the last frame, and
+     * the answer is over. Video that still arrives in this turn is idle, and not counted as generated.
+     */
+    GenerationComplete() {
+        this.turn.GenerationComplete = true;
+        this.answering = false;
+        this.endTurnVideo();
+    }
+    /**
+     * `turnComplete`: ends the turn's video if `generationComplete` didn't, ends the answer and a barge-in's drop window,
+     * starts a new turn.
+     */
+    TurnComplete() {
+        this.endTurnVideo();
+        this.answering = false;
+        this.turn = newTurn();
+    }
+    /**
+     * `interrupted`: the user cut the turn off. Stops its video now (the last frame stays), ends the answer, and drops the
+     * turn's late parts until its `turnComplete`. The caller flushes the PCM.
+     */
+    Interrupted() {
+        this.playout.Flush();
+        this.answering = false;
+        this.turn.VideoEnded = true;
+        this.turn.Dropping = true;
+    }
+    /**
+     * The client cancelled the response (`CancelActiveResponse`): stops the video now and ends the answer; what still
+     * arrives of it is idle until the model speaks again. The caller flushes the PCM.
+     */
+    Cancel() {
+        this.playout.Flush();
+        this.answering = false;
+    }
+    /**
+     * The session resumed on a new connection, where a turn the drop cut off never completes: its video plays out what
+     * arrived and holds the last frame (no flush), the answer ends, and a new turn starts.
+     */
+    Resumed() {
+        this.endTurnVideo();
+        this.answering = false;
+        this.turn = newTurn();
+    }
+    /** Stops the video and releases its element. */
+    Dispose() {
+        this.playout.Dispose();
+    }
+    /**
+     * How a part plays. Its bytes decide first: a part that opens with an MP4 box is the avatar's whatever type it names,
+     * since Google does not document the type of avatar parts. Then its type: none means PCM; a `video/*` type is the
+     * avatar's (a `video/mp4` piece may start inside a box, which Media Source still plays; any other video type that
+     * isn't MP4 is then dropped); a PCM type is the voice; anything else never plays, as video or as voice.
+     */
+    static kindOf(mimeType, data) {
+        if (SniffFmp4Piece(data)) {
+            return 'video';
+        }
+        const type = mimeType?.trim() ?? '';
+        if (type.length === 0) {
+            return 'voice';
+        }
+        if (/^video\//i.test(type)) {
+            return 'video';
+        }
+        return IsPcmAudioMimeType(type) ? 'voice' : 'other';
+    }
+    /**
+     * Hands a video part to the player as a frame, the answer's while one is under way and idle otherwise; a part that
+     * isn't MP4 is dropped, reported once per type.
+     */
+    acceptVideo(mimeType, piece) {
+        const frame = Fmp4PieceToVideoFrame(piece, mimeType, this.init);
+        if (!frame) {
+            const message = `[GeminiRealtimeClient] Dropped model output of type ${mimeType}: it is not fragmented MP4, the only video an avatar plays.`;
+            this.reportOnce(`type:${mimeType}`, message);
+            return 'dropped';
+        }
+        if (frame.Piece === 'init') {
+            this.takeInit(piece);
+        }
+        if (!this.turn.HasVideo) {
+            this.startTurnVideo();
+        }
+        // More media came, so the turn's video is not over even if generationComplete said so.
+        this.turn.VideoEnded = false;
+        this.playout.Append(frame);
+        this.framesHanded++;
+        if (this.answering) {
+            this.answerEndsAtFrame = this.framesHanded;
+        }
+        this.countGeneratedVideo(piece);
+        return this.answering ? 'answer' : 'idle';
+    }
+    /** Keeps an init segment's tracks to time the fragments by; the session's first readable one also decides the voice. */
+    takeInit(piece) {
+        const init = ReadFmp4Init(piece);
+        if (!init) {
+            return;
+        }
+        this.init = init;
+        if (!this.voiceDecided) {
+            this.followInit(init);
+        }
+    }
+    /** Adds a video piece's seconds to the generated video, while its turn's generation has not completed. */
+    countGeneratedVideo(piece) {
+        if (this.turn.GenerationComplete) {
+            return;
+        }
+        const seconds = this.init ? Fmp4VideoSeconds(piece, this.init) : null;
+        if (seconds === null) {
+            this.reportOnce('uncounted-video', "[GeminiRealtimeClient] Could not read the avatar video's duration; that video is not counted in usage.");
+            return;
+        }
+        this.pendingVideoSeconds += seconds;
+    }
+    /** The turn's first video part: when the video carries the voice, PCM the turn queued would double it, so it stops. */
+    startTurnVideo() {
+        this.turn.HasVideo = true;
+        this.turn.VideoStartedAt = Date.now();
+        if (this.videoCarriesVoice && this.turn.HasVoice) {
+            this.voice.Flush();
+        }
+        this.logSeparateVoiceOffset();
+    }
+    /** Queues a PCM part as the voice, unless the turn's video carries it. Voice that plays means the answer is under way. */
+    acceptVoice(pcm) {
+        if (this.videoCarriesVoice && this.turn.HasVideo) {
+            this.reportOnce('pcm-with-video', '[GeminiRealtimeClient] Dropped PCM audio in a turn whose avatar video carries the voice.');
+            return 'dropped';
+        }
+        this.voice.Enqueue(pcm);
+        this.answering = true;
+        if (!this.turn.HasVoice) {
+            this.turn.HasVoice = true;
+            this.turn.VoiceStartedAt = Date.now();
+            this.logSeparateVoiceOffset();
+        }
+        return 'answer';
+    }
+    /**
+     * Follows the session's first readable init segment: an audio track means the video carries the voice; none means the
+     * video plays muted and the voice comes as PCM. Logged once when that disagrees with the grant.
+     */
+    followInit(init) {
+        this.voiceDecided = true;
+        const hasAudio = init.Tracks.some((track) => track.Handler === 'soun');
+        if (hasAudio === this.videoCarriesVoice) {
+            return;
+        }
+        this.videoCarriesVoice = hasAudio;
+        this.playout.CarriesVoice = hasAudio;
+        const following = hasAudio ? 'its audio is the voice' : 'it plays muted and the voice plays as PCM';
+        console.warn(`[GeminiRealtimeClient] The avatar's video ${hasAudio ? 'has an' : 'has no'} audio track, but the session config said ` +
+            `audioMuxed: ${this.grant.AudioMuxed}. Following the video: ${following}.`);
+    }
+    /** Separate voice: one line per turn with how far apart the voice and the video started (they are not lip-synced). */
+    logSeparateVoiceOffset() {
+        const voiceAt = this.turn.VoiceStartedAt;
+        const videoAt = this.turn.VideoStartedAt;
+        if (this.videoCarriesVoice || voiceAt === null || videoAt === null) {
+            return;
+        }
+        const offset = videoAt - voiceAt;
+        const order = offset >= 0 ? 'after' : 'before';
+        console.info(`[GeminiRealtimeClient] Avatar turn: the video started ${Math.abs(offset)} ms ${order} the voice (separate audio, not lip-synced).`);
+    }
+    endTurnVideo() {
+        if (this.turn.HasVideo && !this.turn.VideoEnded) {
+            this.turn.VideoEnded = true;
+            this.playout.EndOfTurn();
+        }
+    }
+    reportOnce(key, message) {
+        if (this.reported.has(key)) {
+            return;
+        }
+        this.reported.add(key);
+        console.warn(message);
+    }
+}
+//# sourceMappingURL=geminiAvatarOutput.js.map

package/dist/drivers/geminiAvatarOutput.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geminiAvatarOutput.js","sourceRoot":"","sources":["../../src/drivers/geminiAvatarOutput.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,YAAY,EAAE,cAAc,EAAiB,MAAM,oBAAoB,CAAC;AAyC9I,SAAS,OAAO;IACZ,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,kBAAkB,EAAE,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAC3J,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAiB3B;;;;OAIG;IACH,YACqB,OAA4B,EAC5B,KAA2B,EAC3B,KAAwB;QAFxB,YAAO,GAAP,OAAO,CAAqB;QAC5B,UAAK,GAAL,KAAK,CAAsB;QAC3B,UAAK,GAAL,KAAK,CAAmB;QAxB5B,aAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,SAAI,GAAe,OAAO,EAAE,CAAC;QAG7B,iBAAY,GAAG,KAAK,CAAC;QAC7B,yFAAyF;QACjF,SAAI,GAAoB,IAAI,CAAC;QACrC,gFAAgF;QACxE,wBAAmB,GAAG,CAAC,CAAC;QAChC,6GAA6G;QACrG,cAAS,GAAG,KAAK,CAAC;QAC1B,0CAA0C;QAClC,iBAAY,GAAG,CAAC,CAAC;QACzB,iHAAiH;QACzG,sBAAiB,GAAG,CAAC,CAAC;QAY1B,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,UAAU,CAAC;IAC9C,CAAC;IAED,wCAAwC;IACxC,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,IAAW,SAAS;QAChB,IAAI,IAAI,CAAC,iBAAiB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;QAC5D,OAAO,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC;IAC3C,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CAAC,QAA4B,EAAE,IAAiB;QACzD,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACrB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QACvD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACnB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,QAAQ,QAAQ,EAAE,EAAE,uDAAuD,QAAQ,yDAAyD,CAAC,CAAC;QAC9J,OAAO,SAAS,CAAC;IACrB,CAAC;IAED;;;OAGG;IACI,aAAa;QAChB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,oGAAoG;IAC7F,UAAU;QACb,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACI,gBAAgB;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,CAAC;QACzC,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC7B,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;OAGG;IACI,kBAAkB;QACrB,IAAI,CAAC,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QACpC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,YAAY,EAAE,CAAC;IACxB,CAAC;IAED;;;OAGG;IACI,YAAY;QACf,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACI,WAAW;QACd,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACI,MAAM;QACT,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACI,OAAO;QACV,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,gDAAgD;IACzC,OAAO;QACV,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,MAAM,CAAC,QAA4B,EAAE,IAAiB;QACjE,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACvB,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC;IACxD,CAAC;IAED;;;OAGG;IACK,WAAW,CAAC,QAA4B,EAAE,KAAkB;QAChE,MAAM,KAAK,GAAG,qBAAqB,CAAC,KAAK,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,MAAM,OAAO,GAAG,uDAAuD,QAAQ,6DAA6D,CAAC;YAC7I,IAAI,CAAC,UAAU,CAAC,QAAQ,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;QACD,uFAAuF;QACvF,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,YAAY,CAAC;QAC/C,CAAC;QACD,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED,wHAAwH;IAChH,QAAQ,CAAC,KAAkB;QAC/B,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,EAAE,CAAC;YACR,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,0GAA0G;IAClG,mBAAmB,CAAC,KAAkB;QAC1C,IAAI,IAAI,CAAC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,UAAU,CAAC,iBAAiB,EAAE,wGAAwG,CAAC,CAAC;YAC7I,OAAO;QACX,CAAC;QACD,IAAI,CAAC,mBAAmB,IAAI,OAAO,CAAC;IACxC,CAAC;IAED,uHAAuH;IAC/G,cAAc;QAClB,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,sBAAsB,EAAE,CAAC;IAClC,CAAC;IAED,0HAA0H;IAClH,WAAW,CAAC,GAAgB;QAChC,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC/C,IAAI,CAAC,UAAU,CAAC,gBAAgB,EAAE,0FAA0F,CAAC,CAAC;YAC9H,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtC,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAClC,CAAC;QACD,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,IAAc;QAC7B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;QACvE,IAAI,QAAQ,KAAK,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACtC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC;QAClC,IAAI,CAAC,OAAO,CAAC,YAAY,GAAG,QAAQ,CAAC;QACrC,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,2CAA2C,CAAC;QACpG,OAAO,CAAC,IAAI,CACR,6CAA6C,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,4CAA4C;YACnH,eAAe,IAAI,CAAC,KAAK,CAAC,UAAU,0BAA0B,SAAS,GAAG,CACjF,CAAC;IACN,CAAC;IAED,sHAAsH;IAC9G,sBAAsB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC;QACzC,IAAI,IAAI,CAAC,iBAAiB,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACjE,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,GAAG,OAAO,CAAC;QACjC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/C,OAAO,CAAC,IAAI,CAAC,yDAAyD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,8CAA8C,CAAC,CAAC;IACtJ,CAAC;IAEO,YAAY;QAChB,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YAC9C,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YAC5B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC7B,CAAC;IACL,CAAC;IAEO,UAAU,CAAC,GAAW,EAAE,OAAe;QAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC1B,CAAC;CACJ"}

package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts

@@ -0,0 +1,38 @@
+import { GeminiRealtimeClient, type GeminiClientConnectArgs, type GeminiLiveClientSession } from './geminiRealtimeClient.js';
+/**
+ * Gemini Live on **Gemini Enterprise**, in the browser: a session through MJAPI's realtime relay.
+ *
+ * Registered under `'gemini-enterprise'`, the `Provider` the server's `GeminiEnterpriseRealtime` driver stamps. Gemini
+ * Enterprise has no browser credential, so the mint makes a relay session: `Transport` `'relay'`, the relay URL as
+ * `RelayUrl` (the ticket is in its path) and no `EphemeralToken`. MJAPI holds the Google credential, writes the setup and
+ * filters what the browser sends. The client opens only relay sessions. Everything else is {@link GeminiRealtimeClient}'s:
+ * audio, transcripts, tools, the avatar playout, resumption (which reuses the URL).
+ *
+ * The web SDK runs in Vertex mode against the relay: `vertexai: true` (the browser build ignores `enterprise`), a
+ * placeholder API key (the build requires one; it never leaves the page), and the relay URL as the base URL, to which the
+ * SDK appends `/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`.
+ */
+export declare class GeminiEnterpriseRealtimeClient extends GeminiRealtimeClient {
+    /** Opts in to relay sessions: this client speaks Gemini Live to MJAPI's relay, which speaks it to Google. */
+    protected get SupportsRelayTransport(): boolean;
+    /**
+     * Opens one Live connection through the relay; a resume opens another with the same URL and the new handle.
+     *
+     * The config states the response modalities: the web SDK fills in AUDIO when it has none, and the relay reads an
+     * AUDIO-only setup as a request to drop the avatar. So a session that shows the avatar always asks for VIDEO.
+     *
+     * @throws When the session is not a relay session, or its relay URL is not a ws(s) URL (see {@link relayBaseUrl}).
+     */
+    protected connectLiveSession(args: GeminiClientConnectArgs): Promise<GeminiLiveClientSession>;
+    /**
+     * A copy of the config that states its response modalities: VIDEO while the avatar shows (the outbound video track is
+     * live), else what the config says (the minted modalities, or AUDIO when the host shows no avatar), else AUDIO.
+     */
+    private withStatedModalities;
+}
+/**
+ * Tree-shaking prevention: bundlers cannot see that {@link GeminiEnterpriseRealtimeClient} is instantiated through the
+ * ClassFactory, so a consumer calls this no-op to keep the `@RegisterClass` side effect alive.
+ */
+export declare function LoadGeminiEnterpriseRealtimeClient(): void;
+//# sourceMappingURL=geminiEnterpriseRealtimeClient.d.ts.map

package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"geminiEnterpriseRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/geminiEnterpriseRealtimeClient.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,oBAAoB,EAAE,KAAK,uBAAuB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AAc1H;;;;;;;;;;;;GAYG;AACH,qBACa,8BAA+B,SAAQ,oBAAoB;IACpE,6GAA6G;IAC7G,cAAuB,sBAAsB,IAAI,OAAO,CAEvD;IAED;;;;;;;OAOG;cACsB,kBAAkB,CAAC,IAAI,EAAE,uBAAuB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAiB5G;;;OAGG;IACH,OAAO,CAAC,oBAAoB;CAM/B;AA0BD;;;GAGG;AACH,wBAAgB,kCAAkC,IAAI,IAAI,CAEzD"}

package/dist/drivers/geminiEnterpriseRealtimeClient.js

@@ -0,0 +1,107 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+import { RegisterClass } from '@memberjunction/global';
+import { GoogleGenAI, Modality } from '@google/genai';
+import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
+import { GeminiRealtimeClient } from './geminiRealtimeClient.js';
+/**
+ * The API key the web SDK requires before it opens a socket. A placeholder: in Vertex mode the SDK puts a key only in a
+ * header, and a browser socket sends no headers, so it never leaves the page. The relay URL is the credential.
+ */
+const RELAY_PLACEHOLDER_API_KEY = 'mjapi-relay';
+/** The Vertex AI API version; MJAPI's relay opens the same one upstream. */
+const VERTEX_LIVE_API_VERSION = 'v1';
+/** A relay URL's scheme, and the rest of it. */
+const RELAY_URL_PATTERN = /^(wss?|https?):\/\/(.+)$/i;
+/**
+ * Gemini Live on **Gemini Enterprise**, in the browser: a session through MJAPI's realtime relay.
+ *
+ * Registered under `'gemini-enterprise'`, the `Provider` the server's `GeminiEnterpriseRealtime` driver stamps. Gemini
+ * Enterprise has no browser credential, so the mint makes a relay session: `Transport` `'relay'`, the relay URL as
+ * `RelayUrl` (the ticket is in its path) and no `EphemeralToken`. MJAPI holds the Google credential, writes the setup and
+ * filters what the browser sends. The client opens only relay sessions. Everything else is {@link GeminiRealtimeClient}'s:
+ * audio, transcripts, tools, the avatar playout, resumption (which reuses the URL).
+ *
+ * The web SDK runs in Vertex mode against the relay: `vertexai: true` (the browser build ignores `enterprise`), a
+ * placeholder API key (the build requires one; it never leaves the page), and the relay URL as the base URL, to which the
+ * SDK appends `/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent`.
+ */
+let GeminiEnterpriseRealtimeClient = class GeminiEnterpriseRealtimeClient extends GeminiRealtimeClient {
+    /** Opts in to relay sessions: this client speaks Gemini Live to MJAPI's relay, which speaks it to Google. */
+    get SupportsRelayTransport() {
+        return true;
+    }
+    /**
+     * Opens one Live connection through the relay; a resume opens another with the same URL and the new handle.
+     *
+     * The config states the response modalities: the web SDK fills in AUDIO when it has none, and the relay reads an
+     * AUDIO-only setup as a request to drop the avatar. So a session that shows the avatar always asks for VIDEO.
+     *
+     * @throws When the session is not a relay session, or its relay URL is not a ws(s) URL (see {@link relayBaseUrl}).
+     */
+    async connectLiveSession(args) {
+        const ai = new GoogleGenAI({
+            vertexai: true,
+            apiKey: RELAY_PLACEHOLDER_API_KEY,
+            httpOptions: { baseUrl: relayBaseUrl(args), apiVersion: VERTEX_LIVE_API_VERSION },
+        });
+        return ai.live.connect({
+            model: args.Model,
+            config: this.withStatedModalities(args.Config),
+            callbacks: {
+                onmessage: args.OnMessage,
+                onerror: args.OnError,
+                onclose: args.OnClose,
+            },
+        });
+    }
+    /**
+     * A copy of the config that states its response modalities: VIDEO while the avatar shows (the outbound video track is
+     * live), else what the config says (the minted modalities, or AUDIO when the host shows no avatar), else AUDIO.
+     */
+    withStatedModalities(config) {
+        if (this.IsTrackEstablished('video', 'outbound')) {
+            return { ...config, responseModalities: [Modality.VIDEO] };
+        }
+        return { ...config, responseModalities: config.responseModalities ?? [Modality.AUDIO] };
+    }
+};
+GeminiEnterpriseRealtimeClient = __decorate([
+    RegisterClass(BaseRealtimeClient, 'gemini-enterprise')
+], GeminiEnterpriseRealtimeClient);
+export { GeminiEnterpriseRealtimeClient };
+/**
+ * The relay URL as the web SDK's base URL. The SDK turns an `http:` base into `ws:` and any other into `wss:`, so a
+ * `ws://` relay URL (MJAPI without TLS, in development) goes in as `http://`, and a `wss://` one as `https://`.
+ *
+ * @param args The session's transport and relay URL, as minted.
+ * @throws When the session is not a relay session (a Gemini Enterprise session has no direct path), has no relay URL, or
+ *   its relay URL is not a ws(s) or http(s) URL. No message quotes the URL or the token: the URL carries the ticket.
+ */
+function relayBaseUrl(args) {
+    if (args.Transport !== 'relay') {
+        throw new Error("GeminiEnterpriseRealtimeClient: the session is not a relay session (Transport 'relay'); Gemini Enterprise runs only through MJAPI's relay.");
+    }
+    if (!args.RelayUrl) {
+        throw new Error('GeminiEnterpriseRealtimeClient: the relay session has no relay URL.');
+    }
+    const match = RELAY_URL_PATTERN.exec(args.RelayUrl);
+    if (!match) {
+        throw new Error('GeminiEnterpriseRealtimeClient: the relay URL is not a ws:// or wss:// URL.');
+    }
+    const scheme = match[1].toLowerCase();
+    const httpScheme = scheme === 'ws' || scheme === 'http' ? 'http' : 'https';
+    return `${httpScheme}://${match[2]}`;
+}
+/**
+ * Tree-shaking prevention: bundlers cannot see that {@link GeminiEnterpriseRealtimeClient} is instantiated through the
+ * ClassFactory, so a consumer calls this no-op to keep the `@RegisterClass` side effect alive.
+ */
+export function LoadGeminiEnterpriseRealtimeClient() {
+    // intentional no-op: the static import of this module is the point
+}
+//# sourceMappingURL=geminiEnterpriseRealtimeClient.js.map

package/dist/drivers/geminiEnterpriseRealtimeClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"geminiEnterpriseRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/geminiEnterpriseRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAA0B,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAA8D,MAAM,wBAAwB,CAAC;AAE1H;;;GAGG;AACH,MAAM,yBAAyB,GAAG,aAAa,CAAC;AAEhD,4EAA4E;AAC5E,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAErC,gDAAgD;AAChD,MAAM,iBAAiB,GAAG,2BAA2B,CAAC;AAEtD;;;;;;;;;;;;GAYG;AAEI,IAAM,8BAA8B,GAApC,MAAM,8BAA+B,SAAQ,oBAAoB;IACpE,6GAA6G;IAC7G,IAAuB,sBAAsB;QACzC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;;;OAOG;IACgB,KAAK,CAAC,kBAAkB,CAAC,IAA6B;QACrE,MAAM,EAAE,GAAG,IAAI,WAAW,CAAC;YACvB,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,yBAAyB;YACjC,WAAW,EAAE,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,uBAAuB,EAAE;SACpF,CAAC,CAAC;QACH,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9C,SAAS,EAAE;gBACP,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,OAAO,EAAE,IAAI,CAAC,OAAO;aACxB;SACJ,CAAC,CAAC;IACP,CAAC;IAED;;;OAGG;IACK,oBAAoB,CAAC,MAAyB;QAClD,IAAI,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,EAAE,CAAC;YAC/C,OAAO,EAAE,GAAG,MAAM,EAAE,kBAAkB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,CAAC;QACD,OAAO,EAAE,GAAG,MAAM,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;IAC5F,CAAC;CACJ,CAAA;AAzCY,8BAA8B;IAD1C,aAAa,CAAC,kBAAkB,EAAE,mBAAmB,CAAC;GAC1C,8BAA8B,CAyC1C;;AAED;;;;;;;GAOG;AACH,SAAS,YAAY,CAAC,IAA6D;IAC/E,IAAI,IAAI,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,4IAA4I,CAAC,CAAC;IAClK,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IAC3F,CAAC;IACD,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAC;IACnG,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,UAAU,GAAG,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IAC3E,OAAO,GAAG,UAAU,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kCAAkC;IAC9C,mEAAmE;AACvE,CAAC"}