@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/drivers/openAIRealtimeClient.js

@@ -6,7 +6,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
 };
 import { RegisterClass } from '@memberjunction/global';
 import { RealtimeAudioMeter } from '../audio/audioMeter.js';
-import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
+import { BaseRealtimeClient, ToProviderSessionConfig } from '../generic/baseRealtimeClient.js';
 import { OpenAIProtocolRealtimeClient, } from '../generic/openAIProtocolClient.js';
 /**
  * OpenAI implementation of {@link BaseRealtimeClient}: a **browser-direct** WebRTC
@@ -32,17 +32,11 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         super(...arguments);
         // ── Transport ──────────────────────────────────────────────────────────────
         this.peerConnection = null;
+        /** The senders carrying the mic tracks, kept so {@link ReplaceMicrophone} can move them to a new track. */
+        this.micSenders = [];
         /** Protected so test subclasses can inspect/inject; production code treats it as private. */
         this.dataChannel = null;
         this.remoteAudioEl = null;
-        /**
-         * The AGENT's remote-audio stream, captured from the peer connection's `ontrack` event
-         * (see {@link attachRemoteAudio}). Exposed via {@link GetRemoteMediaStream} so a host can
-         * mix the agent's voice into a browser-side recording. `null` until the remote track lands.
-         */
-        this.remoteStream = null;
-        /** Host handlers notified when the agent's remote-audio stream lands (or immediately, if already present). */
-        this.remoteStreamHandlers = [];
         /**
          * The server-built session config applied verbatim via `session.update` when the data
          * channel opens. Protected so test subclasses can seed it without a full Connect.
@@ -63,6 +57,7 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
      * data channel opens; the client reports `'listening'` at that point.
      */
     async Connect(config, micStream) {
+        this.AssertTransportSupported(config);
         this.sessionConfig = config.SessionConfig;
         this.micStream = micStream;
         this.setState('connecting');
@@ -78,6 +73,22 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         await this.performSdpHandshake(pc, config.EphemeralToken);
         this.setState('connected');
     }
+    /**
+     * Moves the mic senders and the input meter to the stream's current tracks (obligation #10).
+     * `replaceTrack` needs no renegotiation, so the call carries on.
+     */
+    async ReplaceMicrophone(micStream) {
+        if (this.micSenders.length === 0) {
+            return;
+        }
+        const tracks = micStream.getAudioTracks();
+        if (tracks.length === 0) {
+            throw new Error('The microphone stream has no audio track.');
+        }
+        await Promise.all(this.micSenders.map((sender, index) => sender.replaceTrack(tracks[index] ?? null)));
+        this.micStream = micStream;
+        this.attachInputAudioMeter(RealtimeAudioMeter.ForMicStream(micStream));
+    }
     /**
      * Tears down the channel, peer connection, mic tracks, and audio sink, resets the
      * response state machine, and emits a final `'closed'` (unless already `'error'`).
@@ -86,6 +97,7 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         this.closeAudioMeters();
         this.micStream?.getTracks().forEach((t) => t.stop());
         this.micStream = null;
+        this.micSenders = [];
         if (this.dataChannel) {
             try {
                 this.dataChannel.close();
@@ -105,9 +117,8 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
             this.remoteAudioEl.remove();
             this.remoteAudioEl = null;
         }
-        this.remoteStream = null;
-        // Session-scoped host handlers must not survive into a later Connect on a reused instance.
-        this.remoteStreamHandlers = [];
+        // Drops the stream and the session-scoped host handlers (must not survive a reused instance).
+        this.clearRemoteMediaStream();
         this.sessionConfig = null;
         this.resetResponseState();
         this.audioPlaying = false;
@@ -194,11 +205,9 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         return response.text();
     }
     // ── Connection internals ───────────────────────────────────────────────────
-    /** Streams the mic tracks to the provider. */
+    /** Streams the mic tracks to the provider, keeping their senders. */
     attachMicrophone(pc, micStream) {
-        for (const track of micStream.getAudioTracks()) {
-            pc.addTrack(track, micStream);
-        }
+        this.micSenders = micStream.getAudioTracks().map((track) => pc.addTrack(track, micStream));
     }
     /** Routes the provider's audio track into the hidden `<audio>` sink. */
     attachRemoteAudio(pc) {
@@ -206,52 +215,16 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         pc.ontrack = (e) => {
             if (this.remoteAudioEl && e.streams[0]) {
                 this.remoteAudioEl.srcObject = e.streams[0];
-                // Capture the agent's remote stream so a host can mix it into a recording
-                // (see GetRemoteMediaStream). Playback still flows through the <audio> element.
-                this.remoteStream = e.streams[0];
-                // Notify hosts (e.g. a browser recorder) that the agent's stream is now available
-                // so they can mix it in — the track typically lands AFTER recording already began.
-                this.notifyRemoteStream(e.streams[0]);
+                // Publish the agent's remote stream so a host can mix it into a recording
+                // (see GetRemoteMediaStream) — the track typically lands AFTER recording already
+                // began. Playback still flows through the <audio> element.
+                this.publishRemoteMediaStream(e.streams[0]);
                 // Agent-side audio meter taps the remote stream (obligation #9). The
                 // analyser sinks nowhere — playback still flows through the <audio> element.
                 this.attachOutputAudioMeter(RealtimeAudioMeter.ForStream(e.streams[0]));
             }
         };
     }
-    /**
-     * Returns the AGENT's remote-audio stream once the WebRTC `ontrack` event has delivered it,
-     * or `null` before the track lands. Lets a host mix the agent's voice into a browser-side
-     * recording alongside the mic.
-     */
-    GetRemoteMediaStream() {
-        return this.remoteStream;
-    }
-    /**
-     * Registers a handler invoked when the agent's remote-audio stream becomes available — either
-     * later via the WebRTC `ontrack`, or IMMEDIATELY if the track has already landed. Lets a host
-     * attach the agent voice to a recording that started (mic-only) before the track arrived.
-     */
-    OnRemoteMediaStream(handler) {
-        this.remoteStreamHandlers.push(handler);
-        if (this.remoteStream) {
-            this.invokeRemoteStreamHandler(handler, this.remoteStream);
-        }
-    }
-    /** Fans a freshly-landed remote stream out to all registered host handlers. */
-    notifyRemoteStream(stream) {
-        for (const handler of this.remoteStreamHandlers) {
-            this.invokeRemoteStreamHandler(handler, stream);
-        }
-    }
-    /** Invokes one remote-stream handler, isolating host errors so they never disturb the call. */
-    invokeRemoteStreamHandler(handler, stream) {
-        try {
-            handler(stream);
-        }
-        catch (error) {
-            console.warn('[OpenAIRealtimeClient] remote-stream handler threw:', error);
-        }
-    }
     /**
      * Adopts + wires the events data channel: applies the session config and reports
      * `'listening'` on open; translates inbound frames through the shared protocol brain;
@@ -284,6 +257,18 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
         const answerSdp = await this.postSdpOffer(offer.sdp ?? '', ephemeralToken);
         await pc.setRemoteDescription({ type: 'answer', sdp: answerSdp });
     }
+    /**
+     * @inheritdoc
+     *
+     * This transport keeps the pact in {@link sessionConfig} rather than in the protocol
+     * client's `sessionObject` — WebRTC applies it over the data channel, so the base field is
+     * never populated here. Without this override a spoken update would find no identity to
+     * carry and would send the caller's direction alone, which is the defect the base method
+     * documents (#397).
+     */
+    currentSessionInstructions() {
+        return OpenAIProtocolRealtimeClient.readInstructions(this.sessionConfig);
+    }
     /**
      * Sends the server-controlled session config (instructions + tools) as a
      * `session.update` so the co-agent's identity and tool set apply. Skipped when the
@@ -295,7 +280,7 @@ let OpenAIRealtimeClient = class OpenAIRealtimeClient extends OpenAIProtocolReal
             return;
         }
         if (channel.readyState === 'open') {
-            channel.send(JSON.stringify({ type: 'session.update', session: this.sessionConfig }));
+            channel.send(JSON.stringify({ type: 'session.update', session: ToProviderSessionConfig(this.sessionConfig) }));
         }
     }
 };

package/dist/drivers/openAIRealtimeClient.js.map

@@ -1 +1 @@
-{"version":3,"file":"openAIRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/openAIRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EACH,4BAA4B,GAE/B,MAAM,iCAAiC,CAAC;AAqCzC;;;;;;;;;;;;;;;;;;GAkBG;AAEI,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,4BAA4B;IAA/D;;QACH,8EAA8E;QACtE,mBAAc,GAAmC,IAAI,CAAC;QAC9D,6FAA6F;QACnF,gBAAW,GAAgC,IAAI,CAAC;QAClD,kBAAa,GAA8B,IAAI,CAAC;QACxD;;;;WAIG;QACK,iBAAY,GAAuB,IAAI,CAAC;QAChD,8GAA8G;QACtG,yBAAoB,GAAyC,EAAE,CAAC;QACxE;;;WAGG;QACO,kBAAa,GAAsB,IAAI,CAAC;QAClD;;;;WAIG;QACK,iBAAY,GAAG,KAAK,CAAC;IA0QjC,CAAC;IAxQG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,MAAM,EAAE,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1D,kFAAkF;QAClF,kFAAkF;QAClF,+EAA+E;QAC/E,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAEvE,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC;gBAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACpE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,CAAC,SAAS,GAAG,IAAI,CAAC;YACpC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,2FAA2F;QAC3F,IAAI,CAAC,oBAAoB,GAAG,EAAE,CAAC;QAE/B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E,sFAAsF;IAC5E,aAAa;QACnB,OAAO,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,UAAU,KAAK,MAAM,CAAC;IAC/E,CAAC;IAED,kBAAkB;IACR,iBAAiB,CAAC,KAAgC;QACxD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;OAMG;IACO,eAAe;QACrB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,uDAAuD;IACpC,0BAA0B;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,+DAA+D;IAC5C,0BAA0B;QACzC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,8EAA8E;IAE9E,kFAAkF;IACxE,oBAAoB;QAC1B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACnC,CAAC;IAED,2EAA2E;IACjE,eAAe;QACrB,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC;QACnB,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;OAYG;IACO,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,cAAsB;QACjE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,0CAA0C,EAAE;YACrE,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE;gBACL,aAAa,EAAE,UAAU,cAAc,EAAE;gBACzC,cAAc,EAAE,iBAAiB;aACpC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAE9E,8CAA8C;IACtC,gBAAgB,CAAC,EAA2B,EAAE,SAAsB;QACxE,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,cAAc,EAAE,EAAE,CAAC;YAC7C,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;QAClC,CAAC;IACL,CAAC;IAED,wEAAwE;IAChE,iBAAiB,CAAC,EAA2B;QACjD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAC5C,EAAE,CAAC,OAAO,GAAG,CAAC,CAAgB,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC,aAAa,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,0EAA0E;gBAC1E,gFAAgF;gBAChF,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBACjC,kFAAkF;gBAClF,mFAAmF;gBACnF,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACtC,qEAAqE;gBACrE,6EAA6E;gBAC7E,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED;;;;OAIG;IACI,oBAAoB;QACvB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED;;;;OAIG;IACI,mBAAmB,CAAC,OAAsC;QAC7D,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACxC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QAC/D,CAAC;IACL,CAAC;IAED,+EAA+E;IACvE,kBAAkB,CAAC,MAAmB;QAC1C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAC9C,IAAI,CAAC,yBAAyB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACpD,CAAC;IACL,CAAC;IAED,+FAA+F;IACvF,yBAAyB,CAAC,OAAsC,EAAE,MAAmB;QACzF,IAAI,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC;QAC/E,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACO,gBAAgB,CAAC,OAA6B;QACpD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;QAC3B,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE;YAClB,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC,CAAC;QACF,OAAO,CAAC,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE;YACpC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC;QACF,OAAO,CAAC,OAAO,GAAG,CAAC,CAAQ,EAAE,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,uBAAuB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC;QACF,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,mBAAmB,CAAC,EAA2B,EAAE,cAAsB;QACjF,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,OAA6B;QACpD,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO;QACX,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QAC1F,CAAC;IACL,CAAC;CACJ,CAAA;AAlSY,oBAAoB;IADhC,aAAa,CAAC,kBAAkB,EAAE,QAAQ,CAAC;GAC/B,oBAAoB,CAkShC;;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB;IACpC,oEAAoE;AACxE,CAAC"}
+{"version":3,"file":"openAIRealtimeClient.js","sourceRoot":"","sources":["../../src/drivers/openAIRealtimeClient.ts"],"names":[],"mappings":";;;;;;AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAC5F,OAAO,EACH,4BAA4B,GAE/B,MAAM,iCAAiC,CAAC;AA0CzC;;;;;;;;;;;;;;;;;;GAkBG;AAEI,IAAM,oBAAoB,GAA1B,MAAM,oBAAqB,SAAQ,4BAA4B;IAA/D;;QACH,8EAA8E;QACtE,mBAAc,GAAmC,IAAI,CAAC;QAC9D,2GAA2G;QACnG,eAAU,GAAyB,EAAE,CAAC;QAC9C,6FAA6F;QACnF,gBAAW,GAAgC,IAAI,CAAC;QAClD,kBAAa,GAA8B,IAAI,CAAC;QACxD;;;WAGG;QACO,kBAAa,GAAsB,IAAI,CAAC;QAClD;;;;WAIG;QACK,iBAAY,GAAG,KAAK,CAAC;IAgQjC,CAAC;IA9PG,8EAA8E;IAE9E;;;;;OAKG;IACI,KAAK,CAAC,OAAO,CAAC,MAAmC,EAAE,SAAsB;QAC5E,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC,CAAC;QACtC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAE5B,MAAM,EAAE,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;QACvC,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC;QACzB,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC,CAAC;QAC1D,kFAAkF;QAClF,kFAAkF;QAClF,+EAA+E;QAC/E,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;QAEvE,MAAM,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QAC1D,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,iBAAiB,CAAC,SAAsB;QACjD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,SAAS,CAAC,cAAc,EAAE,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QACtG,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAC3E,CAAC;IAED;;;OAGG;IACI,KAAK,CAAC,UAAU;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,SAAS,EAAE,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QAErB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC;gBAAC,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACjE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,qBAAqB,CAAC,CAAC;YACpE,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC/B,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrB,IAAI,CAAC,aAAa,CAAC,SAAS,GAAG,IAAI,CAAC;YACpC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC9B,CAAC;QACD,8FAA8F;QAC9F,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAE9B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,8EAA8E;IAE9E,sFAAsF;IAC5E,aAAa;QACnB,OAAO,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,UAAU,KAAK,MAAM,CAAC;IAC/E,CAAC;IAED,kBAAkB;IACR,iBAAiB,CAAC,KAAgC;QACxD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;IAClD,CAAC;IAED;;;;;;OAMG;IACO,eAAe;QACrB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,2BAA2B,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,uDAAuD;IACpC,0BAA0B;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,+DAA+D;IAC5C,0BAA0B;QACzC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,YAAY,KAAK,UAAU,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3D,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;IACL,CAAC;IAED,kBAAkB;IAClB,IAAW,cAAc;QACrB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IAED,8EAA8E;IAE9E,kFAAkF;IACxE,oBAAoB;QAC1B,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACnC,CAAC;IAED,2EAA2E;IACjE,eAAe;QACrB,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAC3C,EAAE,CAAC,QAAQ,GAAG,IAAI,CAAC;QACnB,EAAE,CAAC,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC;QAC1B,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAC9B,OAAO,EAAE,CAAC;IACd,CAAC;IAED;;;;;;;;;;;;OAYG;IACO,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,cAAsB;QACjE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,0CAA0C,EAAE;YACrE,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE;gBACL,aAAa,EAAE,UAAU,cAAc,EAAE;gBACzC,cAAc,EAAE,iBAAiB;aACpC;SACJ,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAE9E,qEAAqE;IAC7D,gBAAgB,CAAC,EAA2B,EAAE,SAAsB;QACxE,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/F,CAAC;IAED,wEAAwE;IAChE,iBAAiB,CAAC,EAA2B;QACjD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;QAC5C,EAAE,CAAC,OAAO,GAAG,CAAC,CAAgB,EAAE,EAAE;YAC9B,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrC,IAAI,CAAC,aAAa,CAAC,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;gBAC5C,0EAA0E;gBAC1E,iFAAiF;gBACjF,2DAA2D;gBAC3D,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5C,qEAAqE;gBACrE,6EAA6E;gBAC7E,IAAI,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5E,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED;;;;;OAKG;IACO,gBAAgB,CAAC,OAA6B;QACpD,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;QAC3B,OAAO,CAAC,MAAM,GAAG,GAAG,EAAE;YAClB,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC,CAAC;QACF,OAAO,CAAC,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE;YACpC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC;QACF,OAAO,CAAC,OAAO,GAAG,CAAC,CAAQ,EAAE,EAAE;YAC3B,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,uBAAuB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7E,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QAC3B,CAAC,CAAC;QACF,OAAO,CAAC,OAAO,GAAG,GAAG,EAAE;YACnB,IAAI,IAAI,CAAC,YAAY,KAAK,OAAO,EAAE,CAAC;gBAChC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED,gFAAgF;IACxE,KAAK,CAAC,mBAAmB,CAAC,EAA2B,EAAE,cAAsB;QACjF,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,EAAE,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;QAC3E,MAAM,EAAE,CAAC,oBAAoB,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;IACtE,CAAC;IAED;;;;;;;;OAQG;IACgB,0BAA0B;QACzC,OAAO,4BAA4B,CAAC,gBAAgB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC7E,CAAC;IAED;;;;;OAKG;IACK,kBAAkB,CAAC,OAA6B;QACpD,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtE,OAAO;QACX,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;YAChC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,OAAO,EAAE,uBAAuB,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC,CAAC;QACnH,CAAC;IACL,CAAC;CACJ,CAAA;AAlRY,oBAAoB;IADhC,aAAa,CAAC,kBAAkB,EAAE,QAAQ,CAAC;GAC/B,oBAAoB,CAkRhC;;AAED;;;;GAIG;AACH,MAAM,UAAU,wBAAwB;IACpC,oEAAoE;AACxE,CAAC"}

package/dist/generic/baseRealtimeClient.d.ts

@@ -1,5 +1,20 @@
-import { ClientRealtimeSessionConfig, RealtimeTrack, RealtimeTrackDescriptor, RealtimeTrackDirection, RealtimeUsageModalityDetail } from '@memberjunction/ai';
+import { ClientRealtimeSessionConfig, JSONObject, RealtimeTrack, RealtimeTrackDescriptor, RealtimeTrackDirection, RealtimeUsageModalityDetail } from '@memberjunction/ai';
 import { IRealtimeAudioMeter } from '../audio/audioMeter.js';
+import type { MediaVideoSource } from '../media/model.js';
+/**
+ * Session-config key under which the realtime runtime hands a driver the tracks to negotiate
+ * (`RealtimeSessionRuntime.BuildClientConfig` aggregates them from the active channels). It is a
+ * CLIENT-side negotiation hint, never a provider field: a driver reads it (Gemini) or ignores it.
+ */
+export declare const REQUESTED_TRACKS_SESSION_KEY = "requestedTracks";
+/**
+ * The session config as the PROVIDER must receive it: the server-built config minus the
+ * client-only hints the runtime adds. Drivers that send the config on the wire go through this —
+ * OpenAI's Realtime API rejects the WHOLE `session.update` on an unknown key
+ * (`unknown_parameter: 'session.requestedTracks'`), so one stray hint costs the session its
+ * instructions and every tool.
+ */
+export declare function ToProviderSessionConfig(sessionConfig: JSONObject): JSONObject;
 /**
  * A point-in-time snapshot of the session's audible activity, sampled by the host
  * (typically per animation frame) via {@link BaseRealtimeClient.GetAudioActivity} to drive
@@ -84,7 +99,9 @@ export interface RealtimeClientToolCall {
  * simply never emit):
  * - **OpenAI** — emits per-response deltas from the GA `response.done` frame's `usage` payload.
  * - **Gemini** — emits per-turn deltas from `LiveServerMessage.usageMetadata`
- *   (`promptTokenCount` / `responseTokenCount` are per-response counts, not cumulative).
+ *   (`promptTokenCount` / `responseTokenCount` are per-response counts, not cumulative), and,
+ *   in an avatar session, the seconds of avatar video each turn generated, in updates of their
+ *   own that carry only `OutputTokenDetails.VideoSeconds`.
  * - **ElevenLabs** — the Conversational AI socket exposes no usage events; never emits.
  * - **AssemblyAI** — the streaming STT socket exposes no token-usage events; never emits.
  */
@@ -100,7 +117,10 @@ export interface RealtimeClientUsage {
     DurationSeconds?: number;
     /** Per-modality breakdown of input tokens (text, audio, image/video). */
     InputTokenDetails?: RealtimeUsageModalityDetail;
-    /** Per-modality breakdown of output tokens (text, audio). */
+    /**
+     * Per-modality breakdown of output tokens (text, audio, video), and the seconds of avatar video
+     * generated since the last update (`VideoSeconds`, an amount like the tokens).
+     */
     OutputTokenDetails?: RealtimeUsageModalityDetail;
     /** Cumulative video frames processed or sent across inbound video tracks. */
     VideoFrames?: number;
@@ -123,6 +143,11 @@ export interface RealtimeClientUsage {
  * receiving a tool call and calling {@link BaseRealtimeClient.SendToolResult}.
  */
 export type RealtimeClientState = 'connecting' | 'connected' | 'listening' | 'speaking' | 'closed' | 'error';
+/**
+ * The {@link RealtimeClientError.Code} of a call that ended because its session reached its time limit (MJAPI's relay
+ * closed it with `REALTIME_RELAY_TIME_LIMIT_CLOSE_CODE`), so a host can say so instead of showing a connection error.
+ */
+export declare const REALTIME_CLIENT_TIME_LIMIT_ERROR_CODE = "time-limit";
 /**
  * An error surfaced by a realtime client. `Fatal: true` means the session is unusable
  * (transport failure); `Fatal: false` is a provider-reported, recoverable error frame.
@@ -212,6 +237,32 @@ export interface RealtimeClientError {
  *    client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
  *    Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
  *    tappable plane simply attaches nothing — hosts fall back to turn-state animation.
+ * 10. **Follow a replaced microphone track, when the transport can.** A device switch or a
+ *    lost device swaps the track inside the mic stream, and anything bound to the old track (a
+ *    WebRTC sender, a Web Audio source node, the input meter's clone) goes silent. A driver
+ *    implements {@link BaseRealtimeClient.ReplaceMicrophone} to rebind them: `replaceTrack` on
+ *    its senders, or `IPcmMicCapture.Rebind` on its PCM capture, and a new input meter.
+ * 11. **Refuse a transport you don't speak.** A session minted to run through MJAPI's realtime
+ *    relay (`ClientRealtimeSessionConfig.Transport` `'relay'`) has no provider token, only a relay
+ *    URL. Every driver calls {@link BaseRealtimeClient.AssertTransportSupported} first in
+ *    {@link BaseRealtimeClient.Connect}, so a driver that connects only to its provider refuses
+ *    such a session with a clear error instead of failing at the provider. A driver that speaks
+ *    its provider's protocol through the relay opts in by overriding
+ *    {@link BaseRealtimeClient.SupportsRelayTransport}.
+ * 12. **The agent's video keeps the voice's rules.** A driver whose model sends the agent's video
+ *    hands it over through {@link emitRemoteVideo} once per session, and only while its outbound
+ *    video track is live; an audio-only session hands over nothing and never plays a video part as
+ *    audio. Its player gets each frame as the model sent it (a `RealtimeVideoFrame`, in order).
+ *    Barge-in flushes the video with the voice (obligation #3) and the cut turn's late media never
+ *    plays; a turn's end lets its video play out; a video that carries the voice never plays it
+ *    twice; each second of generated video is reported once (`OutputTokenDetails.VideoSeconds`); the
+ *    video a model streams between answers (Vertex AI's avatar does) plays, but is not the agent
+ *    speaking: it reports no `'speaking'`, leaves {@link IsBusy} as it was, and does not keep
+ *    {@link IsAudioPlaying} true once the answer's media has played; a resumed session keeps its
+ *    video; `Disconnect` releases it. A driver whose voice and video share a media timeline queues
+ *    its PCM at its media times and gives the player its voice playback as the clock
+ *    (`VideoPlayoutOptions.Clock`). The video conformance kit
+ *    (`@memberjunction/ai-realtime-client/testing`) checks each rule.
  */
 export declare abstract class BaseRealtimeClient {
     private transcriptHandler?;
@@ -232,6 +283,32 @@ export declare abstract class BaseRealtimeClient {
      * All tracks evaluated for this session (including `'unsupported'` or `'denied'`).
      */
     get AllTracks(): readonly RealtimeTrack[];
+    /** The model's declared inbound video stream ceiling, recorded by {@link negotiateTracks}; `undefined` until a driver declares one. */
+    private inboundVideoStreamLimit;
+    /** What the driver supports, recorded by {@link negotiateTracks} so {@link AddTrack} can resolve against it later. */
+    private supportedTracks;
+    /** The index the next added track's id takes, so an added track never reuses an id. */
+    private nextTrackIndex;
+    /**
+     * How many concurrent inbound video streams the model accepts: the ceiling the driver declared when it
+     * negotiated tracks, else `1` when an inbound video track is live and `0` when it is not.
+     *
+     * A source arbiter reads this to decide whether several live video sources must be narrowed to one
+     * (with a note to the model on each switch) or passed through untouched.
+     */
+    get MaxInboundVideoStreams(): number;
+    /**
+     * The frame rate (frames per second) the live inbound video track was negotiated at — the model's
+     * ceiling or the session's request, whichever is lower — or `undefined` when no inbound video track
+     * is live or none declared a rate. Anything pacing a video feed reads THIS, never a constant of its own.
+     */
+    get InboundVideoRate(): number | undefined;
+    /**
+     * Whether the model takes inbound video on this session: the driver declared an inbound video track it supports when it
+     * negotiated tracks, and a stream limit above zero. No video track need be live; one is added when a capture starts
+     * ({@link AddTrack}). `false` before tracks are negotiated, and for a driver that does not negotiate them.
+     */
+    get SupportsInboundVideo(): boolean;
     /**
      * Returns whether a track of the requested modality and direction is established and `'live'`.
      */
@@ -241,7 +318,28 @@ export declare abstract class BaseRealtimeClient {
      * Audio (inbound + outbound) is the baseline floor of every realtime session;
      * any additional requested tracks (such as channel video) are unioned onto this baseline floor.
      */
-    protected negotiateTracks(requested: readonly RealtimeTrackDescriptor[] | undefined, supported: readonly RealtimeTrackDescriptor[] | undefined): RealtimeTrack[];
+    protected negotiateTracks(requested: readonly RealtimeTrackDescriptor[] | undefined, supported: readonly RealtimeTrackDescriptor[] | undefined, maxInboundVideoStreams?: number): RealtimeTrack[];
+    /**
+     * Adds a track to a running session, such as inbound video when the user starts their camera, resolved against
+     * what the driver supported when it negotiated at connect. The new track is `'live'`, or `'unsupported'` with the
+     * reason (the model takes no such track, or no more inbound video streams). A live track with the same
+     * direction, modality and source is returned as it is. Other tracks keep their ids.
+     *
+     * The base re-negotiates locally, which is all a driver needs when its provider accepts the media without a
+     * setup change (Gemini Live takes video frames at any time). A driver whose wire must change overrides this and
+     * calls it.
+     *
+     * @returns The track, or `null` when the session has not negotiated tracks: before it connects, or with a driver
+     *   that does not negotiate them (only the Gemini driver does today).
+     */
+    AddTrack(descriptor: RealtimeTrackDescriptor): RealtimeTrack | null;
+    /**
+     * Ends a track on a running session, such as inbound video when the user stops their camera: the track is
+     * reported `'ended'` and leaves {@link AllTracks}. Audio, the session's floor, cannot be removed.
+     *
+     * @returns Whether a track was ended.
+     */
+    RemoveTrack(descriptor: Pick<RealtimeTrackDescriptor, 'Modality' | 'Direction' | 'SourceID'>): boolean;
     /** Meter over the USER's microphone, when the driver attached one. */
     protected inputAudioMeter: IRealtimeAudioMeter | null;
     /** Meter over the AGENT's audio output, when the driver attached one. */
@@ -266,7 +364,8 @@ export declare abstract class BaseRealtimeClient {
      * private pact between this driver and the same-keyed server driver that minted it (see
      * driver obligation #8); how it is applied is entirely driver-specific.
      *
-     * @param config The server-minted client session config (provider, model, ephemeral token, session config).
+     * @param config The server-minted client session config (provider, model, ephemeral token or relay transport,
+     *   session config). A driver refuses a transport it does not speak first (see {@link AssertTransportSupported}).
      * @param micStream The user's microphone capture stream. The caller acquires it (so IT owns
      *   the permission prompt UX); the client attaches it to the transport and stops its tracks
      *   on {@link Disconnect}.
@@ -277,25 +376,56 @@ export declare abstract class BaseRealtimeClient {
      */
     abstract Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream, cameraStream?: MediaStream): Promise<void>;
     /**
-     * Returns the AGENT's remote-audio {@link MediaStream} when this driver owns a tappable
-     * remote-audio plane (e.g. a WebRTC peer-connection driver routes the model's audio track
-     * here), or `null` otherwise. Hosts use it to MIX the agent's voice into a browser-side
-     * recording alongside the mic; a `null` return (the default, and what every non-WebRTC
-     * driver gives) degrades gracefully to mic-only capture.
+     * Whether this driver can open a relay session (`ClientRealtimeSessionConfig.Transport` `'relay'`): connect to
+     * MJAPI's realtime relay at the session's `RelayUrl` rather than to its provider. `false` here; a driver that speaks
+     * its provider's protocol through the relay opts in by overriding it (the Gemini Enterprise client does).
+     */
+    protected get SupportsRelayTransport(): boolean;
+    /**
+     * Refuses a session this driver cannot open: a relay session, unless the driver opts in through
+     * {@link SupportsRelayTransport}. Every driver calls this first in {@link Connect}, before it touches the session's
+     * media or opens a connection (driver obligation #11). The message names the provider key and never quotes the
+     * relay URL, which carries the relay's ticket.
      *
-     * **Optional capability:** the method itself is optional — call sites must use
-     * `client.GetRemoteMediaStream?.() ?? null`. Drivers that can't expose a remote stream
-     * simply don't implement it (or return `null`).
+     * @param config The session the host asked this driver to open.
+     * @throws When the session runs through the relay and this driver does not support it.
+     */
+    protected AssertTransportSupported(config: ClientRealtimeSessionConfig): void;
+    /** The agent's current remote-audio stream, or `null` until a driver publishes one. */
+    private remoteMediaStream;
+    /** Host handlers notified when the agent's stream lands (or immediately, if already present). */
+    private remoteMediaStreamHandlers;
+    /**
+     * Returns the AGENT's remote-audio {@link MediaStream} when this driver owns a tappable
+     * agent-audio plane (a WebRTC driver's remote track, or a PCM driver's playout output),
+     * or `null` otherwise. Hosts use it to MIX the agent's voice into a browser-side recording
+     * alongside the mic; `null` (drivers with no tappable plane, or before the stream lands)
+     * degrades gracefully to mic-only capture. Hosts may still call it as `client.GetRemoteMediaStream?.()`.
+     * Pure read — never mutates.
+     */
+    GetRemoteMediaStream(): MediaStream | null;
+    /**
+     * Registers a handler invoked when the agent's remote-audio stream becomes available —
+     * IMMEDIATELY (synchronously) if it has already landed, otherwise when the driver publishes
+     * it (a WebRTC track typically lands AFTER {@link Connect} resolves). Lets a host attach the
+     * agent voice to a recording that began before the stream landed. Handler errors are
+     * isolated (logged) so a host bug never disturbs the call. Handlers are session-scoped:
+     * {@link clearRemoteMediaStream} (called from `Disconnect`) drops them.
      */
-    GetRemoteMediaStream?(): MediaStream | null;
+    OnRemoteMediaStream(handler: (stream: MediaStream) => void): void;
     /**
-     * **Optional capability:** registers a handler invoked when the agent's remote-audio stream
-     * becomes available — immediately if it has already landed, otherwise when the WebRTC track
-     * arrives (typically AFTER {@link Connect} resolves). Lets a host attach the agent voice to a
-     * recording that began before the track landed. Call as `client.OnRemoteMediaStream?.(cb)`;
-     * drivers without a remote stream simply don't implement it.
+     * Driver hook: records the agent's remote-audio stream and fans it out to every registered
+     * handler. Passing `null` (a driver with no tappable plane) just records `null` and
+     * notifies nobody.
      */
-    OnRemoteMediaStream?(handler: (stream: MediaStream) => void): void;
+    protected publishRemoteMediaStream(stream: MediaStream | null): void;
+    /**
+     * Driver hook for `Disconnect`: drops the stream AND the handlers. Handlers must not survive
+     * into a later `Connect` on a reused instance, or a stale host callback would receive the
+     * next session's stream.
+     */
+    protected clearRemoteMediaStream(): void;
+    private invokeRemoteMediaStreamHandler;
     /**
      * Injects typed text into the live session as a USER turn and asks the model to respond.
      * Implementations must route the reply through the same collision-safe path as tool
@@ -364,6 +494,24 @@ export declare abstract class BaseRealtimeClient {
      * @param muted `true` to mute the mic, `false` to unmute.
      */
     abstract SetMuted(muted: boolean): void;
+    /**
+     * Moves a connected session onto the microphone stream's current audio track: call it after the
+     * track changes (a device switch, or a lost device replaced by the default) or to use another
+     * stream. The connection stays up and the provider hears the new track. The driver now owns this
+     * stream as it owned the one given to {@link Connect}; the previous stream's tracks are the
+     * caller's to stop.
+     *
+     * Mute follows the track: the new track's `enabled` flag decides it, so a caller replacing a
+     * muted track carries `enabled` over (the `LocalMediaController` does).
+     *
+     * **Optional capability** (driver obligation #10): a driver that can't rebind its transport
+     * leaves it undefined. Does nothing when the session isn't connected.
+     *
+     * @param micStream The stream to follow, usually the one given to {@link Connect} after the
+     *   controller swapped its track.
+     * @throws (rejects) When the stream has no audio track; the session keeps its current microphone.
+     */
+    ReplaceMicrophone?(micStream: MediaStream): Promise<void>;
     /**
      * Tears down the provider connection and all client-held resources (control channel,
      * transport, mic tracks, audio sink) and emits a final `'closed'` state (unless the
@@ -433,17 +581,21 @@ export declare abstract class BaseRealtimeClient {
      */
     OnUsage(handler: (usage: RealtimeClientUsage) => void): void;
     /**
-     * Registers the (single) remote-VIDEO handler — the model/avatar's video track for a VIDEO session
-     * (a talking-head the host renders, e.g. as the agent's tile). Invoked once the provider publishes
-     * its video track.
+     * Registers the (single) remote-VIDEO handler — the model/avatar's video for a VIDEO session (a talking-head
+     * the host renders, e.g. as the agent's tile). Invoked once the provider's video is available.
+     *
+     * The video is a {@link MediaVideoSource}: a live stream (a WebRTC track), or a player that must own the
+     * `<video>` element (MSE or WebCodecs playout of encoded avatar video). Hosts show either with
+     * `AttachVideoSource`.
      *
      * **Optional capability:** audio-only drivers (the default) never emit — registering is always safe,
-     * but hosts must not assume a video track arrives. Video-capable drivers
-     * ({@link BaseRealtimeModel.SupportsVideo}) call {@link emitRemoteVideo} when the track is live.
+     * but hosts must not assume video arrives. A driver whose model sends video (the server driver declares an
+     * outbound video track in its session's `Capabilities.SupportedOutboundTracks`, or renders an avatar,
+     * `BaseRealtimeModel.SupportsAvatarOutput`) calls {@link emitRemoteVideo} when it is live.
      *
-     * @param handler Invoked with the remote video `MediaStream` when it becomes available.
+     * @param handler Invoked with the remote video when it becomes available.
      */
-    OnRemoteVideo(handler: (stream: MediaStream) => void): void;
+    OnRemoteVideo(handler: (video: MediaVideoSource) => void): void;
     /**
      * Registers the track-state-change handler — invoked when tracks transition in lifecycle
      * (e.g. established to `'live'`, `'denied'`, or `'unsupported'`).
@@ -455,9 +607,12 @@ export declare abstract class BaseRealtimeClient {
      *
      * @param base64Image Base64-encoded image data.
      * @param mimeType Image MIME type (defaults to 'image/jpeg').
+     * @param sourceId Which source this frame came from. A driver whose model takes ONE inbound video
+     *   stream ignores it (the source arbiter already chose what to send); a driver whose model takes
+     *   several routes the frame to the track bound to that `SourceID`.
      * @returns `true` if accepted and sent; `false` if dropped (throttled, unestablished, etc.).
      */
-    SendVideoFrame?(base64Image: string, mimeType?: string): boolean;
+    SendVideoFrame?(base64Image: string, mimeType?: string, sourceId?: string): boolean;
     /** Emits a track state change to the registered handler (if any). */
     protected emitTrackStateChange(track: RealtimeTrack): void;
     /** Emits a transcript event to the registered handler (if any). */
@@ -472,7 +627,10 @@ export declare abstract class BaseRealtimeClient {
     protected emitInterruption(): void;
     /** Emits a usage update (token deltas for a completed response/turn) to the registered handler (if any). */
     protected emitUsage(usage: RealtimeClientUsage): void;
-    /** Emits the model/avatar's remote video stream to the registered handler (video drivers only). */
-    protected emitRemoteVideo(stream: MediaStream): void;
+    /**
+     * Emits the model/avatar's remote video to the registered handler (video drivers only). A driver with a plain
+     * `MediaStream` passes it as it is; it is handed on as a `'stream'` source.
+     */
+    protected emitRemoteVideo(video: MediaVideoSource | MediaStream): void;
 }
 //# sourceMappingURL=baseRealtimeClient.d.ts.map

package/dist/generic/baseRealtimeClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"baseRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAE3B,aAAa,EACb,uBAAuB,EACvB,sBAAsB,EAEtB,2BAA2B,EAC9B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAA4B,MAAM,qBAAqB,CAAC;AAEpF;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAqB;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6FAA6F;IAC7F,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3B,sGAAsG;IACtG,UAAU,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,wBAAwB;IACrC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,gGAAgG;IAChG,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,mBAAmB;IAChC,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;IAChD,6DAA6D;IAC7D,kBAAkB,CAAC,EAAE,2BAA2B,CAAC;IACjD,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,GAAG,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE7G;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,8BAAsB,kBAAkB;IAEpC,OAAO,CAAC,iBAAiB,CAAC,CAAiD;IAC3E,OAAO,CAAC,eAAe,CAAC,CAAyC;IACjE,OAAO,CAAC,kBAAkB,CAAC,CAAuC;IAClE,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,kBAAkB,CAAC,CAAgC;IAC3D,OAAO,CAAC,YAAY,CAAC,CAAiC;IAGtD,gEAAgE;IAChE,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,CAAM;IAEvC;;OAEG;IACH,IAAW,iBAAiB,IAAI,SAAS,aAAa,EAAE,CAEvD;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,SAAS,aAAa,EAAE,CAE/C;IAED;;OAEG;IACI,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,sBAAsB,GAAG,OAAO;IAOvF;;;;OAIG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,SAAS,uBAAuB,EAAE,GAAG,SAAS,EACzD,SAAS,EAAE,SAAS,uBAAuB,EAAE,GAAG,SAAS,GAC1D,aAAa,EAAE;IA4BlB,sEAAsE;IACtE,SAAS,CAAC,eAAe,EAAE,mBAAmB,GAAG,IAAI,CAAQ;IAC7D,yEAAyE;IACzE,SAAS,CAAC,gBAAgB,EAAE,mBAAmB,GAAG,IAAI,CAAQ;IAE9D;;;;;OAKG;IACI,gBAAgB,IAAI,qBAAqB,GAAG,IAAI;IAYvD,8FAA8F;IAC9F,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKxE,2FAA2F;IAC3F,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKzE,wFAAwF;IACxF,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAKlC;;;;;;;;;;;;;;;OAeG;aACa,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/H;;;;;;;;;;OAUG;IACI,oBAAoB,CAAC,IAAI,WAAW,GAAG,IAAI;IAElD;;;;;;OAMG;IACI,mBAAmB,CAAC,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAEzE;;;;;;;;;;;;;;OAcG;aACa,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAE5C;;;;;;;;;OASG;aACa,oBAAoB,IAAI,IAAI;IAE5C;;;;;OAKG;aACa,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAEnD;;;;;;;;;;;;;;;OAeG;aACa,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAE/D;;;;;;;;OAQG;aACa,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAExE;;;;;OAKG;aACa,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAE9C;;;;OAIG;aACa,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAE3C;;;;OAIG;IACH,aAAoB,MAAM,IAAI,OAAO,CAAC;IAEtC;;;;;OAKG;IACH,aAAoB,cAAc,IAAI,OAAO,CAAC;IAI9C;;;;OAIG;IACI,YAAY,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,wBAAwB,KAAK,IAAI,GAAG,IAAI;IAIlF;;;OAGG;IACI,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,sBAAsB,KAAK,IAAI,GAAG,IAAI;IAIxE;;;OAGG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAIzE;;;OAGG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;OAUG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAIlE;;;OAGG;IACI,kBAAkB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIxE;;;;;;;OAOG;IACI,cAAc,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO;IAIvE,qEAAqE;IACrE,SAAS,CAAC,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAI1D,mEAAmE;IACnE,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,wBAAwB,GAAG,IAAI;IAIpE,oEAAoE;IACpE,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,sBAAsB,GAAG,IAAI;IAI1D,+DAA+D;IAC/D,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAI3D,yDAAyD;IACzD,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,6EAA6E;IAC7E,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAIlC,4GAA4G;IAC5G,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,mGAAmG;IACnG,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;CAGvD"}
+{"version":3,"file":"baseRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,2BAA2B,EAE3B,UAAU,EACV,aAAa,EACb,uBAAuB,EACvB,sBAAsB,EAGtB,2BAA2B,EAC9B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,mBAAmB,EAA4B,MAAM,qBAAqB,CAAC;AACpF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAEvD;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,oBAAoB,CAAC;AAE9D;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,aAAa,EAAE,UAAU,GAAG,UAAU,CAG7E;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,qBAAqB;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,+EAA+E;IAC/E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,6FAA6F;IAC7F,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC3B,sGAAsG;IACtG,UAAU,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,wBAAwB;IACrC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAC3B;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,gGAAgG;IAChG,OAAO,EAAE,OAAO,CAAC;IACjB;;;OAGG;IACH,IAAI,EAAE,QAAQ,GAAG,WAAW,CAAC;IAC7B;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,sBAAsB;IACnC,8FAA8F;IAC9F,MAAM,EAAE,MAAM,CAAC;IACf,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,CAAC;IACjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,WAAW,mBAAmB;IAChC,sFAAsF;IACtF,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uFAAuF;IACvF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yEAAyE;IACzE,iBAAiB,CAAC,EAAE,2BAA2B,CAAC;IAChD;;;OAGG;IACH,kBAAkB,CAAC,EAAE,2BAA2B,CAAC;IACjD,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oFAAoF;IACpF,GAAG,CAAC,EAAE,OAAO,CAAC;CACjB;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,mBAAmB,GAAG,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE7G;;;GAGG;AACH,eAAO,MAAM,qCAAqC,eAAe,CAAC;AAElE;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAChC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuGG;AACH,8BAAsB,kBAAkB;IAEpC,OAAO,CAAC,iBAAiB,CAAC,CAAiD;IAC3E,OAAO,CAAC,eAAe,CAAC,CAAyC;IACjE,OAAO,CAAC,kBAAkB,CAAC,CAAuC;IAClE,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAAuC;IAC5D,OAAO,CAAC,kBAAkB,CAAC,CAAoC;IAC/D,OAAO,CAAC,YAAY,CAAC,CAAiC;IAGtD,gEAAgE;IAChE,SAAS,CAAC,MAAM,EAAE,aAAa,EAAE,CAAM;IAEvC;;OAEG;IACH,IAAW,iBAAiB,IAAI,SAAS,aAAa,EAAE,CAEvD;IAED;;OAEG;IACH,IAAW,SAAS,IAAI,SAAS,aAAa,EAAE,CAE/C;IAED,uIAAuI;IACvI,OAAO,CAAC,uBAAuB,CAAqB;IAEpD,sHAAsH;IACtH,OAAO,CAAC,eAAe,CAAmD;IAE1E,uFAAuF;IACvF,OAAO,CAAC,cAAc,CAAK;IAE3B;;;;;;OAMG;IACH,IAAW,sBAAsB,IAAI,MAAM,CAK1C;IAED;;;;OAIG;IACH,IAAW,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAMhD;IAED;;;;OAIG;IACH,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IAED;;OAEG;IACI,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,sBAAsB,GAAG,OAAO;IAOvF;;;;OAIG;IACH,SAAS,CAAC,eAAe,CACrB,SAAS,EAAE,SAAS,uBAAuB,EAAE,GAAG,SAAS,EACzD,SAAS,EAAE,SAAS,uBAAuB,EAAE,GAAG,SAAS,EACzD,sBAAsB,CAAC,EAAE,MAAM,GAChC,aAAa,EAAE;IAkClB;;;;;;;;;;;;OAYG;IACI,QAAQ,CAAC,UAAU,EAAE,uBAAuB,GAAG,aAAa,GAAG,IAAI;IAyB1E;;;;;OAKG;IACI,WAAW,CAAC,UAAU,EAAE,IAAI,CAAC,uBAAuB,EAAE,UAAU,GAAG,WAAW,GAAG,UAAU,CAAC,GAAG,OAAO;IAe7G,sEAAsE;IACtE,SAAS,CAAC,eAAe,EAAE,mBAAmB,GAAG,IAAI,CAAQ;IAC7D,yEAAyE;IACzE,SAAS,CAAC,gBAAgB,EAAE,mBAAmB,GAAG,IAAI,CAAQ;IAE9D;;;;;OAKG;IACI,gBAAgB,IAAI,qBAAqB,GAAG,IAAI;IAYvD,8FAA8F;IAC9F,SAAS,CAAC,qBAAqB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKxE,2FAA2F;IAC3F,SAAS,CAAC,sBAAsB,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,GAAG,IAAI;IAKzE,wFAAwF;IACxF,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAKlC;;;;;;;;;;;;;;;;OAgBG;aACa,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/H;;;;OAIG;IACH,SAAS,KAAK,sBAAsB,IAAI,OAAO,CAE9C;IAED;;;;;;;;OAQG;IACH,SAAS,CAAC,wBAAwB,CAAC,MAAM,EAAE,2BAA2B,GAAG,IAAI;IAa7E,uFAAuF;IACvF,OAAO,CAAC,iBAAiB,CAA4B;IACrD,iGAAiG;IACjG,OAAO,CAAC,yBAAyB,CAA4C;IAE7E;;;;;;;OAOG;IACI,oBAAoB,IAAI,WAAW,GAAG,IAAI;IAIjD;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAO,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAOxE;;;;OAIG;IACH,SAAS,CAAC,wBAAwB,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI;IAQpE;;;;OAIG;IACH,SAAS,CAAC,sBAAsB,IAAI,IAAI;IAKxC,OAAO,CAAC,8BAA8B;IAQtC;;;;;;;;;;;;;;OAcG;aACa,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAE5C;;;;;;;;;OASG;aACa,oBAAoB,IAAI,IAAI;IAE5C;;;;;OAKG;aACa,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAEnD;;;;;;;;;;;;;;;OAeG;aACa,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAE/D;;;;;;;;OAQG;aACa,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAExE;;;;;OAKG;aACa,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAE9C;;;;;;;;;;;;;;;;OAgBG;IACI,iBAAiB,CAAC,CAAC,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAEhE;;;;OAIG;aACa,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAE3C;;;;OAIG;IACH,aAAoB,MAAM,IAAI,OAAO,CAAC;IAEtC;;;;;OAKG;IACH,aAAoB,cAAc,IAAI,OAAO,CAAC;IAI9C;;;;OAIG;IACI,YAAY,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,wBAAwB,KAAK,IAAI,GAAG,IAAI;IAIlF;;;OAGG;IACI,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,sBAAsB,KAAK,IAAI,GAAG,IAAI;IAIxE;;;OAGG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAIzE;;;OAGG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI;IAInE;;;;;;;;;;;;;;OAcG;IACI,aAAa,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAItE;;;OAGG;IACI,kBAAkB,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIxE;;;;;;;;;;OAUG;IACI,cAAc,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO;IAI1F,qEAAqE;IACrE,SAAS,CAAC,oBAAoB,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI;IAI1D,mEAAmE;IACnE,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,wBAAwB,GAAG,IAAI;IAIpE,oEAAoE;IACpE,SAAS,CAAC,YAAY,CAAC,IAAI,EAAE,sBAAsB,GAAG,IAAI;IAI1D,+DAA+D;IAC/D,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAI3D,yDAAyD;IACzD,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD,6EAA6E;IAC7E,SAAS,CAAC,gBAAgB,IAAI,IAAI;IAIlC,4GAA4G;IAC5G,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAIrD;;;OAGG;IACH,SAAS,CAAC,eAAe,CAAC,KAAK,EAAE,gBAAgB,GAAG,WAAW,GAAG,IAAI;CAGzE"}