@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/README.md +156 -29
  2. package/dist/audio/audioMeter.d.ts +71 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -1
  4. package/dist/audio/audioMeter.js +83 -0
  5. package/dist/audio/audioMeter.js.map +1 -1
  6. package/dist/audio/mediaElementAudioRouter.d.ts +41 -0
  7. package/dist/audio/mediaElementAudioRouter.d.ts.map +1 -0
  8. package/dist/audio/mediaElementAudioRouter.js +82 -0
  9. package/dist/audio/mediaElementAudioRouter.js.map +1 -0
  10. package/dist/audio/micCapture.d.ts +12 -3
  11. package/dist/audio/micCapture.d.ts.map +1 -1
  12. package/dist/audio/micCapture.js +15 -1
  13. package/dist/audio/micCapture.js.map +1 -1
  14. package/dist/audio/pcmPlayback.d.ts +72 -4
  15. package/dist/audio/pcmPlayback.d.ts.map +1 -1
  16. package/dist/audio/pcmPlayback.js +101 -1
  17. package/dist/audio/pcmPlayback.js.map +1 -1
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +2 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -1
  20. package/dist/drivers/assemblyAIRealtimeClient.js +14 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -1
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +2 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -1
  24. package/dist/drivers/elevenLabsRealtimeClient.js +14 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -1
  26. package/dist/drivers/geminiAvatarOutput.d.ts +132 -0
  27. package/dist/drivers/geminiAvatarOutput.d.ts.map +1 -0
  28. package/dist/drivers/geminiAvatarOutput.js +310 -0
  29. package/dist/drivers/geminiAvatarOutput.js.map +1 -0
  30. package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts +38 -0
  31. package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/geminiEnterpriseRealtimeClient.js +107 -0
  33. package/dist/drivers/geminiEnterpriseRealtimeClient.js.map +1 -0
  34. package/dist/drivers/geminiRealtimeClient.d.ts +314 -48
  35. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -1
  36. package/dist/drivers/geminiRealtimeClient.js +759 -176
  37. package/dist/drivers/geminiRealtimeClient.js.map +1 -1
  38. package/dist/drivers/openAILiveClient.d.ts +5 -8
  39. package/dist/drivers/openAILiveClient.d.ts.map +1 -1
  40. package/dist/drivers/openAILiveClient.js +19 -32
  41. package/dist/drivers/openAILiveClient.js.map +1 -1
  42. package/dist/drivers/openAIRealtimeClient.d.ts +23 -26
  43. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -1
  44. package/dist/drivers/openAIRealtimeClient.js +42 -57
  45. package/dist/drivers/openAIRealtimeClient.js.map +1 -1
  46. package/dist/generic/baseRealtimeClient.d.ts +188 -30
  47. package/dist/generic/baseRealtimeClient.d.ts.map +1 -1
  48. package/dist/generic/baseRealtimeClient.js +250 -14
  49. package/dist/generic/baseRealtimeClient.js.map +1 -1
  50. package/dist/generic/openAIProtocolClient.d.ts +57 -1
  51. package/dist/generic/openAIProtocolClient.d.ts.map +1 -1
  52. package/dist/generic/openAIProtocolClient.js +89 -4
  53. package/dist/generic/openAIProtocolClient.js.map +1 -1
  54. package/dist/index.d.ts +2 -3
  55. package/dist/index.d.ts.map +1 -1
  56. package/dist/index.js +3 -3
  57. package/dist/index.js.map +1 -1
  58. package/dist/media/attachVideoSource.d.ts +23 -0
  59. package/dist/media/attachVideoSource.d.ts.map +1 -0
  60. package/dist/media/attachVideoSource.js +49 -0
  61. package/dist/media/attachVideoSource.js.map +1 -0
  62. package/dist/media/channelVideoSource.d.ts +110 -14
  63. package/dist/media/channelVideoSource.d.ts.map +1 -1
  64. package/dist/media/channelVideoSource.js +183 -18
  65. package/dist/media/channelVideoSource.js.map +1 -1
  66. package/dist/media/decoders/frameScheduler.d.ts +119 -0
  67. package/dist/media/decoders/frameScheduler.d.ts.map +1 -0
  68. package/dist/media/decoders/frameScheduler.js +238 -0
  69. package/dist/media/decoders/frameScheduler.js.map +1 -0
  70. package/dist/media/decoders/imageFrameDecoder.d.ts +58 -0
  71. package/dist/media/decoders/imageFrameDecoder.d.ts.map +1 -0
  72. package/dist/media/decoders/imageFrameDecoder.js +110 -0
  73. package/dist/media/decoders/imageFrameDecoder.js.map +1 -0
  74. package/dist/media/decoders/mseFmp4Decoder.d.ts +130 -0
  75. package/dist/media/decoders/mseFmp4Decoder.d.ts.map +1 -0
  76. package/dist/media/decoders/mseFmp4Decoder.js +402 -0
  77. package/dist/media/decoders/mseFmp4Decoder.js.map +1 -0
  78. package/dist/media/decoders/streamCanvas.d.ts +46 -0
  79. package/dist/media/decoders/streamCanvas.d.ts.map +1 -0
  80. package/dist/media/decoders/streamCanvas.js +163 -0
  81. package/dist/media/decoders/streamCanvas.js.map +1 -0
  82. package/dist/media/decoders/videoMimeTypes.d.ts +29 -0
  83. package/dist/media/decoders/videoMimeTypes.d.ts.map +1 -0
  84. package/dist/media/decoders/videoMimeTypes.js +50 -0
  85. package/dist/media/decoders/videoMimeTypes.js.map +1 -0
  86. package/dist/media/decoders/webCodecsChunkDecoder.d.ts +99 -0
  87. package/dist/media/decoders/webCodecsChunkDecoder.d.ts.map +1 -0
  88. package/dist/media/decoders/webCodecsChunkDecoder.js +265 -0
  89. package/dist/media/decoders/webCodecsChunkDecoder.js.map +1 -0
  90. package/dist/media/displayCapture.d.ts +128 -0
  91. package/dist/media/displayCapture.d.ts.map +1 -0
  92. package/dist/media/displayCapture.js +245 -0
  93. package/dist/media/displayCapture.js.map +1 -0
  94. package/dist/media/frameCapture.d.ts +25 -8
  95. package/dist/media/frameCapture.d.ts.map +1 -1
  96. package/dist/media/frameCapture.js +42 -72
  97. package/dist/media/frameCapture.js.map +1 -1
  98. package/dist/media/frameSampler.d.ts +94 -0
  99. package/dist/media/frameSampler.d.ts.map +1 -0
  100. package/dist/media/frameSampler.js +134 -0
  101. package/dist/media/frameSampler.js.map +1 -0
  102. package/dist/media/index.d.ts +32 -0
  103. package/dist/media/index.d.ts.map +1 -0
  104. package/dist/media/index.js +32 -0
  105. package/dist/media/index.js.map +1 -0
  106. package/dist/media/localMediaController.d.ts +116 -0
  107. package/dist/media/localMediaController.d.ts.map +1 -0
  108. package/dist/media/localMediaController.js +281 -0
  109. package/dist/media/localMediaController.js.map +1 -0
  110. package/dist/media/mediaLayout.d.ts +93 -0
  111. package/dist/media/mediaLayout.d.ts.map +1 -0
  112. package/dist/media/mediaLayout.js +187 -0
  113. package/dist/media/mediaLayout.js.map +1 -0
  114. package/dist/media/mediaPreview.d.ts +94 -0
  115. package/dist/media/mediaPreview.d.ts.map +1 -0
  116. package/dist/media/mediaPreview.js +154 -0
  117. package/dist/media/mediaPreview.js.map +1 -0
  118. package/dist/media/mediaStage.d.ts +99 -0
  119. package/dist/media/mediaStage.d.ts.map +1 -0
  120. package/dist/media/mediaStage.js +171 -0
  121. package/dist/media/mediaStage.js.map +1 -0
  122. package/dist/media/model.d.ts +154 -0
  123. package/dist/media/model.d.ts.map +1 -0
  124. package/dist/media/model.js +9 -0
  125. package/dist/media/model.js.map +1 -0
  126. package/dist/media/playbackClock.d.ts +20 -0
  127. package/dist/media/playbackClock.d.ts.map +1 -0
  128. package/dist/media/playbackClock.js +13 -0
  129. package/dist/media/playbackClock.js.map +1 -0
  130. package/dist/media/videoFrameDecoder.d.ts +86 -0
  131. package/dist/media/videoFrameDecoder.d.ts.map +1 -0
  132. package/dist/media/videoFrameDecoder.js +2 -0
  133. package/dist/media/videoFrameDecoder.js.map +1 -0
  134. package/dist/media/videoFrameDecoderRegistry.d.ts +40 -0
  135. package/dist/media/videoFrameDecoderRegistry.d.ts.map +1 -0
  136. package/dist/media/videoFrameDecoderRegistry.js +69 -0
  137. package/dist/media/videoFrameDecoderRegistry.js.map +1 -0
  138. package/dist/media/videoPacing.d.ts +32 -0
  139. package/dist/media/videoPacing.d.ts.map +1 -0
  140. package/dist/media/videoPacing.js +37 -0
  141. package/dist/media/videoPacing.js.map +1 -0
  142. package/dist/media/videoPlayout.d.ts +168 -0
  143. package/dist/media/videoPlayout.d.ts.map +1 -0
  144. package/dist/media/videoPlayout.js +217 -0
  145. package/dist/media/videoPlayout.js.map +1 -0
  146. package/dist/media/videoSourceArbiter.d.ts +250 -0
  147. package/dist/media/videoSourceArbiter.d.ts.map +1 -0
  148. package/dist/media/videoSourceArbiter.js +423 -0
  149. package/dist/media/videoSourceArbiter.js.map +1 -0
  150. package/dist/testing/conformanceAssertions.d.ts +25 -0
  151. package/dist/testing/conformanceAssertions.d.ts.map +1 -0
  152. package/dist/testing/conformanceAssertions.js +64 -0
  153. package/dist/testing/conformanceAssertions.js.map +1 -0
  154. package/dist/testing/conformanceFixtures.d.ts +76 -0
  155. package/dist/testing/conformanceFixtures.d.ts.map +1 -0
  156. package/dist/testing/conformanceFixtures.js +169 -0
  157. package/dist/testing/conformanceFixtures.js.map +1 -0
  158. package/dist/testing/conformanceGates.d.ts +32 -0
  159. package/dist/testing/conformanceGates.d.ts.map +1 -0
  160. package/dist/testing/conformanceGates.js +35 -0
  161. package/dist/testing/conformanceGates.js.map +1 -0
  162. package/dist/testing/conformanceMicrophone.d.ts +10 -0
  163. package/dist/testing/conformanceMicrophone.d.ts.map +1 -0
  164. package/dist/testing/conformanceMicrophone.js +71 -0
  165. package/dist/testing/conformanceMicrophone.js.map +1 -0
  166. package/dist/testing/conformanceTimeline.d.ts +128 -0
  167. package/dist/testing/conformanceTimeline.d.ts.map +1 -0
  168. package/dist/testing/conformanceTimeline.js +32 -0
  169. package/dist/testing/conformanceTimeline.js.map +1 -0
  170. package/dist/testing/conformanceVideoStream.d.ts +29 -0
  171. package/dist/testing/conformanceVideoStream.d.ts.map +1 -0
  172. package/dist/testing/conformanceVideoStream.js +39 -0
  173. package/dist/testing/conformanceVideoStream.js.map +1 -0
  174. package/dist/testing/index.d.ts +29 -0
  175. package/dist/testing/index.d.ts.map +1 -0
  176. package/dist/testing/index.js +28 -0
  177. package/dist/testing/index.js.map +1 -0
  178. package/dist/testing/realtimeVideoConformance.d.ts +23 -0
  179. package/dist/testing/realtimeVideoConformance.d.ts.map +1 -0
  180. package/dist/testing/realtimeVideoConformance.js +139 -0
  181. package/dist/testing/realtimeVideoConformance.js.map +1 -0
  182. package/dist/testing/realtimeVideoConformanceChecks.d.ts +4 -0
  183. package/dist/testing/realtimeVideoConformanceChecks.d.ts.map +1 -0
  184. package/dist/testing/realtimeVideoConformanceChecks.js +164 -0
  185. package/dist/testing/realtimeVideoConformanceChecks.js.map +1 -0
  186. package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts +4 -0
  187. package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts.map +1 -0
  188. package/dist/testing/realtimeVideoConformanceFrameChecks.js +63 -0
  189. package/dist/testing/realtimeVideoConformanceFrameChecks.js.map +1 -0
  190. package/dist/testing/realtimeVideoConformanceSession.d.ts +108 -0
  191. package/dist/testing/realtimeVideoConformanceSession.d.ts.map +1 -0
  192. package/dist/testing/realtimeVideoConformanceSession.js +270 -0
  193. package/dist/testing/realtimeVideoConformanceSession.js.map +1 -0
  194. package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts +4 -0
  195. package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts.map +1 -0
  196. package/dist/testing/realtimeVideoConformanceTurnChecks.js +248 -0
  197. package/dist/testing/realtimeVideoConformanceTurnChecks.js.map +1 -0
  198. package/dist/testing/realtimeVideoConformanceTypes.d.ts +155 -0
  199. package/dist/testing/realtimeVideoConformanceTypes.d.ts.map +1 -0
  200. package/dist/testing/realtimeVideoConformanceTypes.js +2 -0
  201. package/dist/testing/realtimeVideoConformanceTypes.js.map +1 -0
  202. package/dist/testing/recordingPlayout.d.ts +84 -0
  203. package/dist/testing/recordingPlayout.d.ts.map +1 -0
  204. package/dist/testing/recordingPlayout.js +120 -0
  205. package/dist/testing/recordingPlayout.js.map +1 -0
  206. package/package.json +29 -4
@@ -1,5 +1,27 @@
1
- import { DEFAULT_REALTIME_AUDIO_TRACKS, ResolveRequestedTracks, } from '@memberjunction/ai';
1
+ import { DEFAULT_REALTIME_AUDIO_TRACKS, RealtimeTrackKey, ResolveRequestedTracks, } from '@memberjunction/ai';
2
2
  import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
3
+ /**
4
+ * Session-config key under which the realtime runtime hands a driver the tracks to negotiate
5
+ * (`RealtimeSessionRuntime.BuildClientConfig` aggregates them from the active channels). It is a
6
+ * CLIENT-side negotiation hint, never a provider field: a driver reads it (Gemini) or ignores it.
7
+ */
8
+ export const REQUESTED_TRACKS_SESSION_KEY = 'requestedTracks';
9
+ /**
10
+ * The session config as the PROVIDER must receive it: the server-built config minus the
11
+ * client-only hints the runtime adds. Drivers that send the config on the wire go through this —
12
+ * OpenAI's Realtime API rejects the WHOLE `session.update` on an unknown key
13
+ * (`unknown_parameter: 'session.requestedTracks'`), so one stray hint costs the session its
14
+ * instructions and every tool.
15
+ */
16
+ export function ToProviderSessionConfig(sessionConfig) {
17
+ const { [REQUESTED_TRACKS_SESSION_KEY]: _clientOnly, ...providerConfig } = sessionConfig;
18
+ return providerConfig;
19
+ }
20
+ /**
21
+ * The {@link RealtimeClientError.Code} of a call that ended because its session reached its time limit (MJAPI's relay
22
+ * closed it with `REALTIME_RELAY_TIME_LIMIT_CLOSE_CODE`), so a host can say so instead of showing a connection error.
23
+ */
24
+ export const REALTIME_CLIENT_TIME_LIMIT_ERROR_CODE = 'time-limit';
3
25
  /**
4
26
  * Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
5
27
  *
@@ -77,17 +99,55 @@ import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
77
99
  * client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
78
100
  * Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
79
101
  * tappable plane simply attaches nothing — hosts fall back to turn-state animation.
102
+ * 10. **Follow a replaced microphone track, when the transport can.** A device switch or a
103
+ * lost device swaps the track inside the mic stream, and anything bound to the old track (a
104
+ * WebRTC sender, a Web Audio source node, the input meter's clone) goes silent. A driver
105
+ * implements {@link BaseRealtimeClient.ReplaceMicrophone} to rebind them: `replaceTrack` on
106
+ * its senders, or `IPcmMicCapture.Rebind` on its PCM capture, and a new input meter.
107
+ * 11. **Refuse a transport you don't speak.** A session minted to run through MJAPI's realtime
108
+ * relay (`ClientRealtimeSessionConfig.Transport` `'relay'`) has no provider token, only a relay
109
+ * URL. Every driver calls {@link BaseRealtimeClient.AssertTransportSupported} first in
110
+ * {@link BaseRealtimeClient.Connect}, so a driver that connects only to its provider refuses
111
+ * such a session with a clear error instead of failing at the provider. A driver that speaks
112
+ * its provider's protocol through the relay opts in by overriding
113
+ * {@link BaseRealtimeClient.SupportsRelayTransport}.
114
+ * 12. **The agent's video keeps the voice's rules.** A driver whose model sends the agent's video
115
+ * hands it over through {@link emitRemoteVideo} once per session, and only while its outbound
116
+ * video track is live; an audio-only session hands over nothing and never plays a video part as
117
+ * audio. Its player gets each frame as the model sent it (a `RealtimeVideoFrame`, in order).
118
+ * Barge-in flushes the video with the voice (obligation #3) and the cut turn's late media never
119
+ * plays; a turn's end lets its video play out; a video that carries the voice never plays it
120
+ * twice; each second of generated video is reported once (`OutputTokenDetails.VideoSeconds`); the
121
+ * video a model streams between answers (Vertex AI's avatar does) plays, but is not the agent
122
+ * speaking: it reports no `'speaking'`, leaves {@link IsBusy} as it was, and does not keep
123
+ * {@link IsAudioPlaying} true once the answer's media has played; a resumed session keeps its
124
+ * video; `Disconnect` releases it. A driver whose voice and video share a media timeline queues
125
+ * its PCM at its media times and gives the player its voice playback as the clock
126
+ * (`VideoPlayoutOptions.Clock`). The video conformance kit
127
+ * (`@memberjunction/ai-realtime-client/testing`) checks each rule.
80
128
  */
81
129
  export class BaseRealtimeClient {
82
130
  constructor() {
83
131
  // ── Media-track state and negotiation ──────────────────────────────────────
84
132
  /** Tracks requested, established, or denied on this session. */
85
133
  this.tracks = [];
134
+ /** What the driver supports, recorded by {@link negotiateTracks} so {@link AddTrack} can resolve against it later. */
135
+ this.supportedTracks = null;
136
+ /** The index the next added track's id takes, so an added track never reuses an id. */
137
+ this.nextTrackIndex = 0;
86
138
  // ── Audio-activity metering (capability surface — see driver obligation #9) ─
87
139
  /** Meter over the USER's microphone, when the driver attached one. */
88
140
  this.inputAudioMeter = null;
89
141
  /** Meter over the AGENT's audio output, when the driver attached one. */
90
142
  this.outputAudioMeter = null;
143
+ // ── Remote (agent) audio slot ──────────────────────────────────────────────
144
+ // Lives here, not in each driver, so every driver that owns a tappable agent-audio plane
145
+ // (a WebRTC remote track, or a PCM playout engine's output node) shares ONE lifecycle:
146
+ // current stream + late-subscriber replay + isolated fan-out + reset on Disconnect.
147
+ /** The agent's current remote-audio stream, or `null` until a driver publishes one. */
148
+ this.remoteMediaStream = null;
149
+ /** Host handlers notified when the agent's stream lands (or immediately, if already present). */
150
+ this.remoteMediaStreamHandlers = [];
91
151
  }
92
152
  /**
93
153
  * Established tracks on this session (tracks in the `'live'` state).
@@ -101,6 +161,37 @@ export class BaseRealtimeClient {
101
161
  get AllTracks() {
102
162
  return this.tracks;
103
163
  }
164
+ /**
165
+ * How many concurrent inbound video streams the model accepts: the ceiling the driver declared when it
166
+ * negotiated tracks, else `1` when an inbound video track is live and `0` when it is not.
167
+ *
168
+ * A source arbiter reads this to decide whether several live video sources must be narrowed to one
169
+ * (with a note to the model on each switch) or passed through untouched.
170
+ */
171
+ get MaxInboundVideoStreams() {
172
+ if (typeof this.inboundVideoStreamLimit === 'number') {
173
+ return this.IsTrackEstablished('video', 'inbound') ? this.inboundVideoStreamLimit : 0;
174
+ }
175
+ return this.IsTrackEstablished('video', 'inbound') ? 1 : 0;
176
+ }
177
+ /**
178
+ * The frame rate (frames per second) the live inbound video track was negotiated at — the model's
179
+ * ceiling or the session's request, whichever is lower — or `undefined` when no inbound video track
180
+ * is live or none declared a rate. Anything pacing a video feed reads THIS, never a constant of its own.
181
+ */
182
+ get InboundVideoRate() {
183
+ const track = this.tracks.find((t) => t.State === 'live' && t.Descriptor.Direction === 'inbound' && String(t.Descriptor.Modality).trim().toLowerCase() === 'video');
184
+ const rate = track?.Descriptor.Rate;
185
+ return typeof rate === 'number' && rate > 0 ? rate : undefined;
186
+ }
187
+ /**
188
+ * Whether the model takes inbound video on this session: the driver declared an inbound video track it supports when it
189
+ * negotiated tracks, and a stream limit above zero. No video track need be live; one is added when a capture starts
190
+ * ({@link AddTrack}). `false` before tracks are negotiated, and for a driver that does not negotiate them.
191
+ */
192
+ get SupportsInboundVideo() {
193
+ return (this.inboundVideoStreamLimit ?? 1) > 0 && (this.supportedTracks ?? []).some(isInboundVideo);
194
+ }
104
195
  /**
105
196
  * Returns whether a track of the requested modality and direction is established and `'live'`.
106
197
  */
@@ -113,28 +204,85 @@ export class BaseRealtimeClient {
113
204
  * Audio (inbound + outbound) is the baseline floor of every realtime session;
114
205
  * any additional requested tracks (such as channel video) are unioned onto this baseline floor.
115
206
  */
116
- negotiateTracks(requested, supported) {
207
+ negotiateTracks(requested, supported, maxInboundVideoStreams) {
208
+ // Keyed by direction + modality + SOURCE, so two requested inbound video streams fed by
209
+ // different sources stay two tracks (a model that accepts several needs both), while two
210
+ // descriptors that name no source still collapse to the one stream they always did.
117
211
  const trackMap = new Map();
118
212
  for (const t of DEFAULT_REALTIME_AUDIO_TRACKS) {
119
- trackMap.set(`${t.Direction}:${t.Modality}`, t);
213
+ trackMap.set(RealtimeTrackKey(t), t);
120
214
  }
121
215
  if (requested) {
122
216
  for (const t of requested) {
123
- trackMap.set(`${t.Direction}:${t.Modality}`, t);
217
+ trackMap.set(RealtimeTrackKey(t), t);
124
218
  }
125
219
  }
126
220
  const effectiveRequested = Array.from(trackMap.values());
127
221
  const effectiveSupported = supported ?? DEFAULT_REALTIME_AUDIO_TRACKS;
128
- const resolved = ResolveRequestedTracks(effectiveRequested, effectiveSupported, (d, i) => `${d.Direction}:${String(d.Modality)}:${i}`);
222
+ this.inboundVideoStreamLimit = typeof maxInboundVideoStreams === 'number' ? maxInboundVideoStreams : undefined;
223
+ const resolved = ResolveRequestedTracks(effectiveRequested, effectiveSupported, (d, i) => `${d.Direction}:${String(d.Modality)}:${i}`, { MaxInboundVideoStreams: this.inboundVideoStreamLimit });
129
224
  this.tracks = resolved.map((t) => ({
130
225
  ...t,
131
226
  State: t.State === 'requested' ? 'live' : t.State,
132
227
  }));
228
+ this.supportedTracks = effectiveSupported;
229
+ this.nextTrackIndex = this.tracks.length;
133
230
  for (const track of this.tracks) {
134
231
  this.emitTrackStateChange(track);
135
232
  }
136
233
  return this.tracks;
137
234
  }
235
+ /**
236
+ * Adds a track to a running session, such as inbound video when the user starts their camera, resolved against
237
+ * what the driver supported when it negotiated at connect. The new track is `'live'`, or `'unsupported'` with the
238
+ * reason (the model takes no such track, or no more inbound video streams). A live track with the same
239
+ * direction, modality and source is returned as it is. Other tracks keep their ids.
240
+ *
241
+ * The base re-negotiates locally, which is all a driver needs when its provider accepts the media without a
242
+ * setup change (Gemini Live takes video frames at any time). A driver whose wire must change overrides this and
243
+ * calls it.
244
+ *
245
+ * @returns The track, or `null` when the session has not negotiated tracks: before it connects, or with a driver
246
+ * that does not negotiate them (only the Gemini driver does today).
247
+ */
248
+ AddTrack(descriptor) {
249
+ if (!this.supportedTracks) {
250
+ return null;
251
+ }
252
+ const key = RealtimeTrackKey(descriptor);
253
+ const existing = this.tracks.find((t) => t.State === 'live' && RealtimeTrackKey(t.Descriptor) === key);
254
+ if (existing) {
255
+ return existing;
256
+ }
257
+ // Resolve after the live inbound video streams, so the stream cap and its reason count them.
258
+ const liveVideo = this.tracks.filter((t) => t.State === 'live' && isInboundVideo(t.Descriptor)).map((t) => t.Descriptor);
259
+ const index = this.nextTrackIndex++;
260
+ const resolved = ResolveRequestedTracks([...liveVideo, descriptor], this.supportedTracks, (d) => `${d.Direction}:${String(d.Modality)}:${index}`, { MaxInboundVideoStreams: this.inboundVideoStreamLimit });
261
+ const added = resolved[resolved.length - 1];
262
+ const track = { ...added, State: added.State === 'requested' ? 'live' : added.State };
263
+ this.tracks = [...this.tracks.filter((t) => RealtimeTrackKey(t.Descriptor) !== key), track];
264
+ this.emitTrackStateChange(track);
265
+ return track;
266
+ }
267
+ /**
268
+ * Ends a track on a running session, such as inbound video when the user stops their camera: the track is
269
+ * reported `'ended'` and leaves {@link AllTracks}. Audio, the session's floor, cannot be removed.
270
+ *
271
+ * @returns Whether a track was ended.
272
+ */
273
+ RemoveTrack(descriptor) {
274
+ if (String(descriptor.Modality).trim().toLowerCase() === 'audio') {
275
+ return false;
276
+ }
277
+ const key = RealtimeTrackKey(descriptor);
278
+ const track = this.tracks.find((t) => RealtimeTrackKey(t.Descriptor) === key);
279
+ if (!track) {
280
+ return false;
281
+ }
282
+ this.tracks = this.tracks.filter((t) => t !== track);
283
+ this.emitTrackStateChange({ ...track, State: 'ended' });
284
+ return true;
285
+ }
138
286
  /**
139
287
  * The session's current audible activity, or `null` when this driver attached no
140
288
  * meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
@@ -167,6 +315,83 @@ export class BaseRealtimeClient {
167
315
  this.attachInputAudioMeter(null);
168
316
  this.attachOutputAudioMeter(null);
169
317
  }
318
+ /**
319
+ * Whether this driver can open a relay session (`ClientRealtimeSessionConfig.Transport` `'relay'`): connect to
320
+ * MJAPI's realtime relay at the session's `RelayUrl` rather than to its provider. `false` here; a driver that speaks
321
+ * its provider's protocol through the relay opts in by overriding it (the Gemini Enterprise client does).
322
+ */
323
+ get SupportsRelayTransport() {
324
+ return false;
325
+ }
326
+ /**
327
+ * Refuses a session this driver cannot open: a relay session, unless the driver opts in through
328
+ * {@link SupportsRelayTransport}. Every driver calls this first in {@link Connect}, before it touches the session's
329
+ * media or opens a connection (driver obligation #11). The message names the provider key and never quotes the
330
+ * relay URL, which carries the relay's ticket.
331
+ *
332
+ * @param config The session the host asked this driver to open.
333
+ * @throws When the session runs through the relay and this driver does not support it.
334
+ */
335
+ AssertTransportSupported(config) {
336
+ if (config.Transport === 'relay' && !this.SupportsRelayTransport) {
337
+ throw new Error(`The '${config.Provider}' realtime client cannot open a session through MJAPI's relay (Transport 'relay'); it connects only to its provider.`);
338
+ }
339
+ }
340
+ /**
341
+ * Returns the AGENT's remote-audio {@link MediaStream} when this driver owns a tappable
342
+ * agent-audio plane (a WebRTC driver's remote track, or a PCM driver's playout output),
343
+ * or `null` otherwise. Hosts use it to MIX the agent's voice into a browser-side recording
344
+ * alongside the mic; `null` (drivers with no tappable plane, or before the stream lands)
345
+ * degrades gracefully to mic-only capture. Hosts may still call it as `client.GetRemoteMediaStream?.()`.
346
+ * Pure read — never mutates.
347
+ */
348
+ GetRemoteMediaStream() {
349
+ return this.remoteMediaStream;
350
+ }
351
+ /**
352
+ * Registers a handler invoked when the agent's remote-audio stream becomes available —
353
+ * IMMEDIATELY (synchronously) if it has already landed, otherwise when the driver publishes
354
+ * it (a WebRTC track typically lands AFTER {@link Connect} resolves). Lets a host attach the
355
+ * agent voice to a recording that began before the stream landed. Handler errors are
356
+ * isolated (logged) so a host bug never disturbs the call. Handlers are session-scoped:
357
+ * {@link clearRemoteMediaStream} (called from `Disconnect`) drops them.
358
+ */
359
+ OnRemoteMediaStream(handler) {
360
+ this.remoteMediaStreamHandlers.push(handler);
361
+ if (this.remoteMediaStream) {
362
+ this.invokeRemoteMediaStreamHandler(handler, this.remoteMediaStream);
363
+ }
364
+ }
365
+ /**
366
+ * Driver hook: records the agent's remote-audio stream and fans it out to every registered
367
+ * handler. Passing `null` (a driver with no tappable plane) just records `null` and
368
+ * notifies nobody.
369
+ */
370
+ publishRemoteMediaStream(stream) {
371
+ this.remoteMediaStream = stream;
372
+ if (!stream)
373
+ return;
374
+ for (const handler of this.remoteMediaStreamHandlers) {
375
+ this.invokeRemoteMediaStreamHandler(handler, stream);
376
+ }
377
+ }
378
+ /**
379
+ * Driver hook for `Disconnect`: drops the stream AND the handlers. Handlers must not survive
380
+ * into a later `Connect` on a reused instance, or a stale host callback would receive the
381
+ * next session's stream.
382
+ */
383
+ clearRemoteMediaStream() {
384
+ this.remoteMediaStream = null;
385
+ this.remoteMediaStreamHandlers = [];
386
+ }
387
+ invokeRemoteMediaStreamHandler(handler, stream) {
388
+ try {
389
+ handler(stream);
390
+ }
391
+ catch (error) {
392
+ console.warn(`[${this.constructor.name}] remote-stream handler threw:`, error);
393
+ }
394
+ }
170
395
  // ── Handler registration ──────────────────────────────────────────────────
171
396
  /**
172
397
  * Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
@@ -230,15 +455,19 @@ export class BaseRealtimeClient {
230
455
  this.usageHandler = handler;
231
456
  }
232
457
  /**
233
- * Registers the (single) remote-VIDEO handler — the model/avatar's video track for a VIDEO session
234
- * (a talking-head the host renders, e.g. as the agent's tile). Invoked once the provider publishes
235
- * its video track.
458
+ * Registers the (single) remote-VIDEO handler — the model/avatar's video for a VIDEO session (a talking-head
459
+ * the host renders, e.g. as the agent's tile). Invoked once the provider's video is available.
460
+ *
461
+ * The video is a {@link MediaVideoSource}: a live stream (a WebRTC track), or a player that must own the
462
+ * `<video>` element (MSE or WebCodecs playout of encoded avatar video). Hosts show either with
463
+ * `AttachVideoSource`.
236
464
  *
237
465
  * **Optional capability:** audio-only drivers (the default) never emit — registering is always safe,
238
- * but hosts must not assume a video track arrives. Video-capable drivers
239
- * ({@link BaseRealtimeModel.SupportsVideo}) call {@link emitRemoteVideo} when the track is live.
466
+ * but hosts must not assume video arrives. A driver whose model sends video (the server driver declares an
467
+ * outbound video track in its session's `Capabilities.SupportedOutboundTracks`, or renders an avatar,
468
+ * `BaseRealtimeModel.SupportsAvatarOutput`) calls {@link emitRemoteVideo} when it is live.
240
469
  *
241
- * @param handler Invoked with the remote video `MediaStream` when it becomes available.
470
+ * @param handler Invoked with the remote video when it becomes available.
242
471
  */
243
472
  OnRemoteVideo(handler) {
244
473
  this.remoteVideoHandler = handler;
@@ -279,9 +508,16 @@ export class BaseRealtimeClient {
279
508
  emitUsage(usage) {
280
509
  this.usageHandler?.(usage);
281
510
  }
282
- /** Emits the model/avatar's remote video stream to the registered handler (video drivers only). */
283
- emitRemoteVideo(stream) {
284
- this.remoteVideoHandler?.(stream);
511
+ /**
512
+ * Emits the model/avatar's remote video to the registered handler (video drivers only). A driver with a plain
513
+ * `MediaStream` passes it as it is; it is handed on as a `'stream'` source.
514
+ */
515
+ emitRemoteVideo(video) {
516
+ this.remoteVideoHandler?.('Kind' in video ? video : { Kind: 'stream', Stream: video });
285
517
  }
286
518
  }
519
+ /** Whether a descriptor is an inbound video track. */
520
+ function isInboundVideo(descriptor) {
521
+ return descriptor.Direction === 'inbound' && String(descriptor.Modality).trim().toLowerCase() === 'video';
522
+ }
287
523
  //# sourceMappingURL=baseRealtimeClient.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,6BAA6B,EAI7B,sBAAsB,GAEzB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAuB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AA+IpF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6EG;AACH,MAAM,OAAgB,kBAAkB;IAAxC;QAWI,8EAA8E;QAC9E,gEAAgE;QACtD,WAAM,GAAoB,EAAE,CAAC;QA6DvC,+EAA+E;QAC/E,sEAAsE;QAC5D,oBAAe,GAA+B,IAAI,CAAC;QAC7D,yEAAyE;QAC/D,qBAAgB,GAA+B,IAAI,CAAC;IA6TlE,CAAC;IA5XG;;OAEG;IACH,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;OAEG;IACI,kBAAkB,CAAC,QAAgB,EAAE,SAAiC;QACzE,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,CAClI,CAAC;IACN,CAAC;IAED;;;;OAIG;IACO,eAAe,CACrB,SAAyD,EACzD,SAAyD;QAEzD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,KAAK,MAAM,CAAC,IAAI,6BAA6B,EAAE,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC;YACpD,CAAC;QACL,CAAC;QACD,MAAM,kBAAkB,GAAuC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GAAG,SAAS,IAAI,6BAA6B,CAAC;QACtE,MAAM,QAAQ,GAAG,sBAAsB,CACnC,kBAAkB,EAClB,kBAAkB,EAClB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CACxD,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;SACpD,CAAC,CAAC,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAQD;;;;;OAKG;IACI,gBAAgB;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5F,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG,CAAC;IACN,CAAC;IAED,8FAA8F;IACpF,qBAAqB,CAAC,KAAiC;QAC7D,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,2FAA2F;IACjF,sBAAsB,CAAC,KAAiC;QAC9D,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED,wFAAwF;IAC9E,gBAAgB;QACtB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IA0ID,6EAA6E;IAE7E;;;;OAIG;IACI,YAAY,CAAC,OAAuD;QACvE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAA+C;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,OAA6C;QAC9D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;OAUG;IACI,aAAa,CAAC,OAAsC;QACvD,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,kBAAkB,CAAC,OAAuC;QAC7D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAYD,6EAA6E;IAE7E,qEAAqE;IAC3D,oBAAoB,CAAC,KAAoB;QAC/C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,mEAAmE;IACzD,cAAc,CAAC,UAAoC;QACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,oEAAoE;IAC1D,YAAY,CAAC,IAA4B;QAC/C,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,+DAA+D;IACrD,eAAe,CAAC,KAA0B;QAChD,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,yDAAyD;IAC/C,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IACnE,gBAAgB;QACtB,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,4GAA4G;IAClG,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,mGAAmG;IACzF,eAAe,CAAC,MAAmB;QACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC,MAAM,CAAC,CAAC;IACtC,CAAC;CACJ"}
1
+ {"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,6BAA6B,EAK7B,gBAAgB,EAChB,sBAAsB,GAEzB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAuB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAGpF;;;;GAIG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,iBAAiB,CAAC;AAE9D;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,aAAyB;IAC7D,MAAM,EAAE,CAAC,4BAA4B,CAAC,EAAE,WAAW,EAAE,GAAG,cAAc,EAAE,GAAG,aAAa,CAAC;IACzF,OAAO,cAAc,CAAC;AAC1B,CAAC;AAuID;;;GAGG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAG,YAAY,CAAC;AAelE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuGG;AACH,MAAM,OAAgB,kBAAkB;IAAxC;QAWI,8EAA8E;QAC9E,gEAAgE;QACtD,WAAM,GAAoB,EAAE,CAAC;QAmBvC,sHAAsH;QAC9G,oBAAe,GAA8C,IAAI,CAAC;QAE1E,uFAAuF;QAC/E,mBAAc,GAAG,CAAC,CAAC;QAqJ3B,+EAA+E;QAC/E,sEAAsE;QAC5D,oBAAe,GAA+B,IAAI,CAAC;QAC7D,yEAAyE;QAC/D,qBAAgB,GAA+B,IAAI,CAAC;QAmF9D,8EAA8E;QAC9E,yFAAyF;QACzF,uFAAuF;QACvF,oFAAoF;QAEpF,uFAAuF;QAC/E,sBAAiB,GAAuB,IAAI,CAAC;QACrD,iGAAiG;QACzF,8BAAyB,GAAyC,EAAE,CAAC;IAwUjF,CAAC;IAjlBG;;OAEG;IACH,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAWD;;;;;;OAMG;IACH,IAAW,sBAAsB;QAC7B,IAAI,OAAO,IAAI,CAAC,uBAAuB,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED;;;;OAIG;IACH,IAAW,gBAAgB;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,CACtI,CAAC;QACF,MAAM,IAAI,GAAG,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;QACpC,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,CAAC;IAED;;;;OAIG;IACH,IAAW,oBAAoB;QAC3B,OAAO,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACxG,CAAC;IAED;;OAEG;IACI,kBAAkB,CAAC,QAAgB,EAAE,SAAiC;QACzE,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,CAClI,CAAC;IACN,CAAC;IAED;;;;OAIG;IACO,eAAe,CACrB,SAAyD,EACzD,SAAyD,EACzD,sBAA+B;QAE/B,wFAAwF;QACxF,yFAAyF;QACzF,oFAAoF;QACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,KAAK,MAAM,CAAC,IAAI,6BAA6B,EAAE,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QACD,MAAM,kBAAkB,GAAuC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GAAG,SAAS,IAAI,6BAA6B,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,OAAO,sBAAsB,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/G,MAAM,QAAQ,GAAG,sBAAsB,CACnC,kBAAkB,EAClB,kBAAkB,EAClB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EACrD,EAAE,sBAAsB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAC3D,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;SACpD,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,eAAe,GAAG,kBAAkB,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,QAAQ,CAAC,UAAmC;QAC/C,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;QACvG,IAAI,QAAQ,EAAE,CAAC;YACX,OAAO,QAAQ,CAAC;QACpB,CAAC;QACD,6FAA6F;QAC7F,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACzH,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,sBAAsB,CACnC,CAAC,GAAG,SAAS,EAAE,UAAU,CAAC,EAC1B,IAAI,CAAC,eAAe,EACpB,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,EACtD,EAAE,sBAAsB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAC3D,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAkB,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAC5F,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACI,WAAW,CAAC,UAAgF;QAC/F,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;YAC/D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;QAC9E,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,oBAAoB,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IAChB,CAAC;IAQD;;;;;OAKG;IACI,gBAAgB;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5F,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG,CAAC;IACN,CAAC;IAED,8FAA8F;IACpF,qBAAqB,CAAC,KAAiC;QAC7D,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,2FAA2F;IACjF,sBAAsB,CAAC,KAAiC;QAC9D,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED,wFAAwF;IAC9E,gBAAgB;QACtB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAqBD;;;;OAIG;IACH,IAAc,sBAAsB;QAChC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;OAQG;IACO,wBAAwB,CAAC,MAAmC;QAClE,IAAI,MAAM,CAAC,SAAS,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CACX,QAAQ,MAAM,CAAC,QAAQ,sHAAsH,CAChJ,CAAC;QACN,CAAC;IACL,CAAC;IAYD;;;;;;;OAOG;IACI,oBAAoB;QACvB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAsC;QAC7D,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,wBAAwB,CAAC,MAA0B;QACzD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;QAChC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnD,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,sBAAsB;QAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;IACxC,CAAC;IAEO,8BAA8B,CAAC,OAAsC,EAAE,MAAmB;QAC9F,IAAI,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAqHD,6EAA6E;IAE7E;;;;OAIG;IACI,YAAY,CAAC,OAAuD;QACvE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAA+C;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,OAA6C;QAC9D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACI,aAAa,CAAC,OAA0C;QAC3D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,kBAAkB,CAAC,OAAuC;QAC7D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAeD,6EAA6E;IAE7E,qEAAqE;IAC3D,oBAAoB,CAAC,KAAoB;QAC/C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,mEAAmE;IACzD,cAAc,CAAC,UAAoC;QACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,oEAAoE;IAC1D,YAAY,CAAC,IAA4B;QAC/C,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,+DAA+D;IACrD,eAAe,CAAC,KAA0B;QAChD,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,yDAAyD;IAC/C,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IACnE,gBAAgB;QACtB,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,4GAA4G;IAClG,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,KAAqC;QAC3D,IAAI,CAAC,kBAAkB,EAAE,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;CACJ;AAED,sDAAsD;AACtD,SAAS,cAAc,CAAC,UAAmC;IACvD,OAAO,UAAU,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;AAC9G,CAAC"}
@@ -304,8 +304,57 @@ export declare abstract class OpenAIProtocolRealtimeClient extends BaseRealtimeC
304
304
  * provider, and narration is disposable by contract, so the update is dropped with a debug
305
305
  * log rather than queued to come out late and stale. Hosts SHOULD still gate on
306
306
  * {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
307
+ *
308
+ * The wire frame is built by {@link buildSpokenUpdateEvent}, which is where the identity
309
+ * rule lives — see that method for why the caller's direction is never sent on its own.
307
310
  */
308
311
  RequestSpokenUpdate(instructions: string): void;
312
+ /**
313
+ * The `response.create` frame for one spoken update — **carrying the session identity, not
314
+ * replacing it.**
315
+ *
316
+ * ⚠️ `response.instructions` IS A FULL OVERRIDE OF THE SESSION SYSTEM PROMPT for that one
317
+ * response, not an addition to it. That is the whole reason this method exists. The
318
+ * server-side twin of this driver (`openAIRealtime.ts`) already states the rule and guards
319
+ * the BLANK case on it — forwarding `''` "would wipe the co-agent identity framing" — but the
320
+ * NON-blank case was never guarded, so every caller that supplied a real direction wiped it
321
+ * just as thoroughly and much less visibly.
322
+ *
323
+ * What that cost, live: the one turn that rides this method had no persona, so the model fell
324
+ * back to its vendor default and opened a hiring interview with *"I'm ChatGPT, your friendly
325
+ * voice companion"* while every other turn in the same session correctly said *"I'm Sam
326
+ * Rivera, Support Team Lead"* (bizapps-caliber#397 / MJ#4591). Every commentary-style caller
327
+ * is exposed the same way — an opening turn, a silence check-in, a delegation narration — and
328
+ * none of them should have to restate an identity the session already carries.
329
+ *
330
+ * Three cases, and the middle one is the fix:
331
+ *
332
+ * - **Blank direction** → a bare `response.create`. "Respond now, under the session prompt"
333
+ * (the meeting-mode bridge trigger passes `''`). Matches the server twin exactly.
334
+ * - **Direction + session instructions** → `identity, then direction`. The identity leads
335
+ * because it is context, and the direction is LAST because it is the thing being asked for
336
+ * now — the most recent line is the one a model weights hardest.
337
+ * - **Direction, no session instructions** → the direction alone. A session minted without
338
+ * instructions has no identity to lose, and prefixing a blank line would be noise.
339
+ */
340
+ protected buildSpokenUpdateEvent(instructions: string): OAIProtocolResponseCreateEvent;
341
+ /**
342
+ * The session-level instructions currently in force — the co-agent's identity, persona and
343
+ * standing directives, as applied by `session.update`.
344
+ *
345
+ * A SEAM, not a field read, because the pact lives in a different place per transport: the
346
+ * websocket subclass keeps it in `sessionObject`, the WebRTC driver in its own `sessionConfig`.
347
+ * The base holds no copy of either, so it answers "no identity on record" and each transport
348
+ * overrides with the one it actually applied. A transport that forgets to override loses only
349
+ * the carry — {@link buildSpokenUpdateEvent} still sends the caller's direction — which is the
350
+ * old behaviour rather than a new failure.
351
+ *
352
+ * Null rather than `''` for "this session carries no identity", so {@link buildSpokenUpdateEvent}
353
+ * can tell "nothing to preserve" from "preserve this".
354
+ */
355
+ protected currentSessionInstructions(): string | null;
356
+ /** Reads a session pact's `instructions`, or null when it carries none worth preserving. */
357
+ protected static readInstructions(pact: JSONObject | null | undefined): string | null;
309
358
  /**
310
359
  * Sends the tool result back as a `function_call_output` conversation item, then triggers
311
360
  * a reply — immediately if the model is idle, otherwise queued until the current response
@@ -450,7 +499,10 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
450
499
  private failPendingConnect;
451
500
  /** Opens the provider socket for this connection (drivers own URL/auth specifics). */
452
501
  protected abstract openProviderSocket(config: ClientRealtimeSessionConfig): IOpenAIProtocolClientSocket;
453
- /** Extracts the wire-shaped `session` object from the server pact. Default: the pact itself. */
502
+ /**
503
+ * Extracts the wire-shaped `session` object from the server pact. Default: the pact minus the
504
+ * client-only hints ({@link ToProviderSessionConfig}).
505
+ */
454
506
  protected resolveSessionObject(config: ClientRealtimeSessionConfig): JSONObject;
455
507
  /** Resolves the PCM sample rate for the audio plane (both directions). */
456
508
  protected abstract resolveSampleRate(config: ClientRealtimeSessionConfig): number;
@@ -473,6 +525,8 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
473
525
  * `'listening'` only after all of that (obligation #7).
474
526
  */
475
527
  Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
528
+ /** Rebinds the PCM capture and the input meter to the stream's current track (obligation #10); the socket stays open. */
529
+ ReplaceMicrophone(micStream: MediaStream): Promise<void>;
476
530
  /**
477
531
  * Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
478
532
  * state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call
@@ -511,6 +565,8 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
511
565
  * EMPTY `session.update` would be wrong).
512
566
  */
513
567
  protected applySessionConfig(): void;
568
+ /** @inheritdoc — this transport's pact is {@link sessionObject}, applied over the socket. */
569
+ protected currentSessionInstructions(): string | null;
514
570
  /** Streams one base64 PCM16 mic chunk as an `input_audio_buffer.append` frame. */
515
571
  protected sendMicChunk(base64Pcm16: string): void;
516
572
  /** Formats a socket-level error message for the host. Providers brand this (see xAI). */
@@ -1 +1 @@
1
- {"version":3,"file":"openAIProtocolClient.d.ts","sourceRoot":"","sources":["../../src/generic/openAIProtocolClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAA4B,MAAM,oBAAoB,CAAC;AACvG,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE/E,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAO1E,yFAAyF;AACzF,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,wCAAwC,GAAG,iCAAiC,CAAC;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,uCAAuC,GAAG,gCAAgC,CAAC;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,uFAAuF;AACvF,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,sBAAsB,GAAG,6BAA6B,CAAC;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,uDAAuD,CAAC;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,oCAAoC;IACjD,IAAI,EAAE,uCAAuC,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mEAAmE;AACnE,MAAM,WAAW,wBAAwB;IACrC,IAAI,EAAE,mCAAmC,CAAC;CAC7C;AAED,+FAA+F;AAC/F,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,kBAAkB,CAAC;CAC5B;AAED,mGAAmG;AACnG,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,oGAAoG;AACpG,MAAM,WAAW,gBAAgB;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,gBAAgB,CAAA;KAAE,CAAC;IACxC,uDAAuD;IACvD,KAAK,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,CAAC;CACvC;AACD,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,GAAG,6BAA6B,CAAC;CACvE;AAED,iCAAiC;AACjC,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,gFAAgF;AAChF,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,yBAAyB,GAC/B,0BAA0B,GAC1B,yBAAyB,GACzB,qBAAqB,GACrB,sCAAsC,GACtC,oCAAoC,GACpC,wBAAwB,GACxB,0BAA0B,GAC1B,yBAAyB,GACzB,uBAAuB,GACvB,mCAAmC,GACnC,mCAAmC,GACnC,qBAAqB,GACrB,uBAAuB,CAAC;AAI9B,0FAA0F;AAC1F,MAAM,WAAW,6BAA6B;IAC1C,IAAI,EAAE,gBAAgB,CAAC;IACvB,OAAO,EAAE,UAAU,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,+EAA+E;AAC/E,MAAM,WAAW,iCAAiC;IAC9C,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,4DAA4D;AAC5D,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,0BAA0B,CAAC;IACjC,IAAI,EAAE,sBAAsB,GAAG,iCAAiC,CAAC;CACpE;AAED,kFAAkF;AAClF,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAED,0FAA0F;AAC1F,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,wFAAwF;AACxF,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;CACrC;AAED,sGAAsG;AACtG,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,yBAAyB,GAC/B,6BAA6B,GAC7B,sCAAsC,GACtC,8BAA8B,GAC9B,8BAA8B,GAC9B,sCAAsC,GACtC,sCAAsC,CAAC;AAI7C;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,8BAAsB,4BAA6B,SAAQ,kBAAkB;IAEzE,0EAA0E;IAC1E,SAAS,CAAC,oBAAoB,SAAM;IACpC,+FAA+F;IAC/F,SAAS,CAAC,cAAc,UAAS;IACjC,kGAAkG;IAClG,SAAS,CAAC,qBAAqB,UAAS;IACxC;;;;;OAKG;IACH,SAAS,CAAC,2BAA2B,SAAK;IAC1C;;;;;;;;OAQG;IACH,SAAS,CAAC,uBAAuB,UAAS;IAC1C;;;;;;OAMG;IACH,SAAS,CAAC,oBAAoB,UAAS;IACvC;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,EAAE,QAAQ,GAAG,WAAW,CAAY;IAChE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,EAAE,mBAAmB,CAAY;IACvD,4FAA4F;IAC5F,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI,CAAQ;IAI/C,uFAAuF;IACvF,SAAS,CAAC,QAAQ,CAAC,aAAa,IAAI,OAAO;IAC3C,sGAAsG;IACtG,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAC5E;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;IAI1C,0DAA0D;IAC1D,SAAS,KAAK,kBAAkB,IAAI,MAAM,CAEzC;IAED;;;;OAIG;IACH,SAAS,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAI3D;;;;;OAKG;IACH,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAIzD;;;;;OAKG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAWtC;;;;;OAKG;IACH,SAAS,KAAK,0BAA0B,IAAI,OAAO,CAElD;IAED,+FAA+F;IAC/F,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAIvC,sEAAsE;IACtE,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAIlE,wEAAwE;IACxE,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5C,gEAAgE;IAChE,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAMnE;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYnC;;;;;;;;;OASG;IACI,oBAAoB,IAAI,IAAI;IAoBnC;;;;;;OAMG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU1C;;;;;;;;;;;;OAYG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IActD;;;;;;OAMG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/D;;;;OAIG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAMrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAID;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAelD,mFAAmF;IACnF,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAuF7D;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAInC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAG5C,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAI5C,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAUpB,yFAAyF;IACzF,OAAO,CAAC,gBAAgB;IAWxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,YAAY;IAgCpB;;;;;;OAMG;IACH,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAevC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAW5C,0EAA0E;IAC1E,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAYpC,4EAA4E;IAC5E,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAKpD,uFAAuF;IACvF,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;CAO9D;AAID;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC,uCAAuC;IACvC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,8BAAsB,qCAAsC,SAAQ,4BAA4B;IAC5F,0FAA0F;IAC1F,SAAS,CAAC,MAAM,EAAE,2BAA2B,GAAG,IAAI,CAAQ;IAC5D,2DAA2D;IAC3D,SAAS,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IACvD,mEAAmE;IACnE,SAAS,CAAC,UAAU,EAAE,cAAc,GAAG,IAAI,CAAQ;IACnD;;;OAGG;IACH,SAAS,CAAC,aAAa,EAAE,UAAU,CAAM;IACzC,qFAAqF;IACrF,SAAS,CAAC,gBAAgB,UAAS;IACnC,4FAA4F;IAC5F,SAAS,CAAC,UAAU,UAAS;IAC7B,8FAA8F;IAC9F,OAAO,CAAC,sBAAsB,CAA6B;IAC3D,mFAAmF;IACnF,OAAO,CAAC,kBAAkB,CAAyC;IAInE,sFAAsF;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,MAAM,EAAE,2BAA2B,GAAG,2BAA2B;IACvG,gGAAgG;IAChG,SAAS,CAAC,oBAAoB,CAAC,MAAM,EAAE,2BAA2B,GAAG,UAAU;IAG/E,0EAA0E;IAC1E,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,GAAG,MAAM;IACjF;;;;OAIG;IACH,SAAS,KAAK,sBAAsB,IAAI,OAAO,CAE9C;IAED;;;;OAIG;IACH,SAAS,KAAK,gBAAgB,IAAI,MAAM,CAEvC;IAID;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAgGhG;;;;OAIG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAgCxC,yFAAyF;IACzF,SAAS,CAAC,aAAa,IAAI,OAAO;IAIlC,kBAAkB;IAClB,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAInE,0FAA0F;IAC1F,SAAS,CAAC,eAAe,IAAI,IAAI;IAIjC,mFAAmF;cAChE,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAInE;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAED;;;OAGG;cACgB,oBAAoB,IAAI,IAAI;IAU/C,iGAAiG;IACjG,cAAuB,0BAA0B,IAAI,OAAO,CAE3D;IAED,2EAA2E;cACxD,qBAAqB,IAAI,IAAI;IAOhD;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAOpC,kFAAkF;IAClF,SAAS,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAMjD,yFAAyF;IACzF,SAAS,CAAC,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIvD,kGAAkG;IAClG,SAAS,KAAK,sBAAsB,IAAI,MAAM,CAE7C;IAED,oFAAoF;IACpF,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAQlD;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAUnC;;;;OAIG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B,uGAAuG;IACvG,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;CAGrE"}
1
+ {"version":3,"file":"openAIProtocolClient.d.ts","sourceRoot":"","sources":["../../src/generic/openAIProtocolClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAA4B,MAAM,oBAAoB,CAAC;AACvG,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAA2B,MAAM,sBAAsB,CAAC;AAExG,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAO1E,yFAAyF;AACzF,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,wCAAwC,GAAG,iCAAiC,CAAC;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,uCAAuC,GAAG,gCAAgC,CAAC;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,uFAAuF;AACvF,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,sBAAsB,GAAG,6BAA6B,CAAC;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,uDAAuD,CAAC;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,oCAAoC;IACjD,IAAI,EAAE,uCAAuC,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mEAAmE;AACnE,MAAM,WAAW,wBAAwB;IACrC,IAAI,EAAE,mCAAmC,CAAC;CAC7C;AAED,+FAA+F;AAC/F,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,kBAAkB,CAAC;CAC5B;AAED,mGAAmG;AACnG,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,oGAAoG;AACpG,MAAM,WAAW,gBAAgB;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,gBAAgB,CAAA;KAAE,CAAC;IACxC,uDAAuD;IACvD,KAAK,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,CAAC;CACvC;AACD,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,GAAG,6BAA6B,CAAC;CACvE;AAED,iCAAiC;AACjC,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,gFAAgF;AAChF,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,yBAAyB,GAC/B,0BAA0B,GAC1B,yBAAyB,GACzB,qBAAqB,GACrB,sCAAsC,GACtC,oCAAoC,GACpC,wBAAwB,GACxB,0BAA0B,GAC1B,yBAAyB,GACzB,uBAAuB,GACvB,mCAAmC,GACnC,mCAAmC,GACnC,qBAAqB,GACrB,uBAAuB,CAAC;AAI9B,0FAA0F;AAC1F,MAAM,WAAW,6BAA6B;IAC1C,IAAI,EAAE,gBAAgB,CAAC;IACvB,OAAO,EAAE,UAAU,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,+EAA+E;AAC/E,MAAM,WAAW,iCAAiC;IAC9C,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,4DAA4D;AAC5D,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,0BAA0B,CAAC;IACjC,IAAI,EAAE,sBAAsB,GAAG,iCAAiC,CAAC;CACpE;AAED,kFAAkF;AAClF,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAED,0FAA0F;AAC1F,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,wFAAwF;AACxF,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;CACrC;AAED,sGAAsG;AACtG,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,yBAAyB,GAC/B,6BAA6B,GAC7B,sCAAsC,GACtC,8BAA8B,GAC9B,8BAA8B,GAC9B,sCAAsC,GACtC,sCAAsC,CAAC;AAI7C;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,8BAAsB,4BAA6B,SAAQ,kBAAkB;IAEzE,0EAA0E;IAC1E,SAAS,CAAC,oBAAoB,SAAM;IACpC,+FAA+F;IAC/F,SAAS,CAAC,cAAc,UAAS;IACjC,kGAAkG;IAClG,SAAS,CAAC,qBAAqB,UAAS;IACxC;;;;;OAKG;IACH,SAAS,CAAC,2BAA2B,SAAK;IAC1C;;;;;;;;OAQG;IACH,SAAS,CAAC,uBAAuB,UAAS;IAC1C;;;;;;OAMG;IACH,SAAS,CAAC,oBAAoB,UAAS;IACvC;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,EAAE,QAAQ,GAAG,WAAW,CAAY;IAChE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,EAAE,mBAAmB,CAAY;IACvD,4FAA4F;IAC5F,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI,CAAQ;IAI/C,uFAAuF;IACvF,SAAS,CAAC,QAAQ,CAAC,aAAa,IAAI,OAAO;IAC3C,sGAAsG;IACtG,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAC5E;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;IAI1C,0DAA0D;IAC1D,SAAS,KAAK,kBAAkB,IAAI,MAAM,CAEzC;IAED;;;;OAIG;IACH,SAAS,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAI3D;;;;;OAKG;IACH,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAIzD;;;;;OAKG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAWtC;;;;;OAKG;IACH,SAAS,KAAK,0BAA0B,IAAI,OAAO,CAElD;IAED,+FAA+F;IAC/F,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAIvC,sEAAsE;IACtE,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAIlE,wEAAwE;IACxE,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5C,gEAAgE;IAChE,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAMnE;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYnC;;;;;;;;;OASG;IACI,oBAAoB,IAAI,IAAI;IAoBnC;;;;;;OAMG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU1C;;;;;;;;;;;;;;;OAeG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IActD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,8BAA8B;IAYtF;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,0BAA0B,IAAI,MAAM,GAAG,IAAI;IAIrD,4FAA4F;IAC5F,SAAS,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI;IAKrF;;;;;;OAMG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/D;;;;OAIG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAMrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAID;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAelD,mFAAmF;IACnF,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAuF7D;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAInC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAG5C,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAI5C,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAUpB,yFAAyF;IACzF,OAAO,CAAC,gBAAgB;IAWxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,YAAY;IAgCpB;;;;;;OAMG;IACH,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAevC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAW5C,0EAA0E;IAC1E,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAYpC,4EAA4E;IAC5E,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAKpD,uFAAuF;IACvF,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;CAO9D;AAID;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC,uCAAuC;IACvC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,8BAAsB,qCAAsC,SAAQ,4BAA4B;IAC5F,0FAA0F;IAC1F,SAAS,CAAC,MAAM,EAAE,2BAA2B,GAAG,IAAI,CAAQ;IAC5D,2DAA2D;IAC3D,SAAS,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IACvD,mEAAmE;IACnE,SAAS,CAAC,UAAU,EAAE,cAAc,GAAG,IAAI,CAAQ;IACnD;;;OAGG;IACH,SAAS,CAAC,aAAa,EAAE,UAAU,CAAM;IACzC,qFAAqF;IACrF,SAAS,CAAC,gBAAgB,UAAS;IACnC,4FAA4F;IAC5F,SAAS,CAAC,UAAU,UAAS;IAC7B,8FAA8F;IAC9F,OAAO,CAAC,sBAAsB,CAA6B;IAC3D,mFAAmF;IACnF,OAAO,CAAC,kBAAkB,CAAyC;IAInE,sFAAsF;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,MAAM,EAAE,2BAA2B,GAAG,2BAA2B;IACvG;;;OAGG;IACH,SAAS,CAAC,oBAAoB,CAAC,MAAM,EAAE,2BAA2B,GAAG,UAAU;IAG/E,0EAA0E;IAC1E,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,GAAG,MAAM;IACjF;;;;OAIG;IACH,SAAS,KAAK,sBAAsB,IAAI,OAAO,CAE9C;IAED;;;;OAIG;IACH,SAAS,KAAK,gBAAgB,IAAI,MAAM,CAEvC;IAID;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAoGhG,yHAAyH;IAC5G,iBAAiB,CAAC,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IASrE;;;;OAIG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiCxC,yFAAyF;IACzF,SAAS,CAAC,aAAa,IAAI,OAAO;IAIlC,kBAAkB;IAClB,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAInE,0FAA0F;IAC1F,SAAS,CAAC,eAAe,IAAI,IAAI;IAIjC,mFAAmF;cAChE,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAInE;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAED;;;OAGG;cACgB,oBAAoB,IAAI,IAAI;IAU/C,iGAAiG;IACjG,cAAuB,0BAA0B,IAAI,OAAO,CAE3D;IAED,2EAA2E;cACxD,qBAAqB,IAAI,IAAI;IAOhD;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAOpC,6FAA6F;cAC1E,0BAA0B,IAAI,MAAM,GAAG,IAAI;IAI9D,kFAAkF;IAClF,SAAS,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAMjD,yFAAyF;IACzF,SAAS,CAAC,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIvD,kGAAkG;IAClG,SAAS,KAAK,sBAAsB,IAAI,MAAM,CAE7C;IAED,oFAAoF;IACpF,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAQlD;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAUnC;;;;OAIG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B,uGAAuG;IACvG,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;CAGrE"}