@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -29
- package/dist/audio/audioMeter.d.ts +71 -0
- package/dist/audio/audioMeter.d.ts.map +1 -1
- package/dist/audio/audioMeter.js +83 -0
- package/dist/audio/audioMeter.js.map +1 -1
- package/dist/audio/mediaElementAudioRouter.d.ts +41 -0
- package/dist/audio/mediaElementAudioRouter.d.ts.map +1 -0
- package/dist/audio/mediaElementAudioRouter.js +82 -0
- package/dist/audio/mediaElementAudioRouter.js.map +1 -0
- package/dist/audio/micCapture.d.ts +12 -3
- package/dist/audio/micCapture.d.ts.map +1 -1
- package/dist/audio/micCapture.js +15 -1
- package/dist/audio/micCapture.js.map +1 -1
- package/dist/audio/pcmPlayback.d.ts +72 -4
- package/dist/audio/pcmPlayback.d.ts.map +1 -1
- package/dist/audio/pcmPlayback.js +101 -1
- package/dist/audio/pcmPlayback.js.map +1 -1
- package/dist/drivers/assemblyAIRealtimeClient.d.ts +2 -0
- package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/assemblyAIRealtimeClient.js +14 -0
- package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -1
- package/dist/drivers/elevenLabsRealtimeClient.d.ts +2 -0
- package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/elevenLabsRealtimeClient.js +14 -0
- package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -1
- package/dist/drivers/geminiAvatarOutput.d.ts +132 -0
- package/dist/drivers/geminiAvatarOutput.d.ts.map +1 -0
- package/dist/drivers/geminiAvatarOutput.js +310 -0
- package/dist/drivers/geminiAvatarOutput.js.map +1 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts +38 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.d.ts.map +1 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.js +107 -0
- package/dist/drivers/geminiEnterpriseRealtimeClient.js.map +1 -0
- package/dist/drivers/geminiRealtimeClient.d.ts +314 -48
- package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/geminiRealtimeClient.js +759 -176
- package/dist/drivers/geminiRealtimeClient.js.map +1 -1
- package/dist/drivers/openAILiveClient.d.ts +5 -8
- package/dist/drivers/openAILiveClient.d.ts.map +1 -1
- package/dist/drivers/openAILiveClient.js +19 -32
- package/dist/drivers/openAILiveClient.js.map +1 -1
- package/dist/drivers/openAIRealtimeClient.d.ts +23 -26
- package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/openAIRealtimeClient.js +42 -57
- package/dist/drivers/openAIRealtimeClient.js.map +1 -1
- package/dist/generic/baseRealtimeClient.d.ts +188 -30
- package/dist/generic/baseRealtimeClient.d.ts.map +1 -1
- package/dist/generic/baseRealtimeClient.js +250 -14
- package/dist/generic/baseRealtimeClient.js.map +1 -1
- package/dist/generic/openAIProtocolClient.d.ts +57 -1
- package/dist/generic/openAIProtocolClient.d.ts.map +1 -1
- package/dist/generic/openAIProtocolClient.js +89 -4
- package/dist/generic/openAIProtocolClient.js.map +1 -1
- package/dist/index.d.ts +2 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -3
- package/dist/index.js.map +1 -1
- package/dist/media/attachVideoSource.d.ts +23 -0
- package/dist/media/attachVideoSource.d.ts.map +1 -0
- package/dist/media/attachVideoSource.js +49 -0
- package/dist/media/attachVideoSource.js.map +1 -0
- package/dist/media/channelVideoSource.d.ts +110 -14
- package/dist/media/channelVideoSource.d.ts.map +1 -1
- package/dist/media/channelVideoSource.js +183 -18
- package/dist/media/channelVideoSource.js.map +1 -1
- package/dist/media/decoders/frameScheduler.d.ts +119 -0
- package/dist/media/decoders/frameScheduler.d.ts.map +1 -0
- package/dist/media/decoders/frameScheduler.js +238 -0
- package/dist/media/decoders/frameScheduler.js.map +1 -0
- package/dist/media/decoders/imageFrameDecoder.d.ts +58 -0
- package/dist/media/decoders/imageFrameDecoder.d.ts.map +1 -0
- package/dist/media/decoders/imageFrameDecoder.js +110 -0
- package/dist/media/decoders/imageFrameDecoder.js.map +1 -0
- package/dist/media/decoders/mseFmp4Decoder.d.ts +130 -0
- package/dist/media/decoders/mseFmp4Decoder.d.ts.map +1 -0
- package/dist/media/decoders/mseFmp4Decoder.js +402 -0
- package/dist/media/decoders/mseFmp4Decoder.js.map +1 -0
- package/dist/media/decoders/streamCanvas.d.ts +46 -0
- package/dist/media/decoders/streamCanvas.d.ts.map +1 -0
- package/dist/media/decoders/streamCanvas.js +163 -0
- package/dist/media/decoders/streamCanvas.js.map +1 -0
- package/dist/media/decoders/videoMimeTypes.d.ts +29 -0
- package/dist/media/decoders/videoMimeTypes.d.ts.map +1 -0
- package/dist/media/decoders/videoMimeTypes.js +50 -0
- package/dist/media/decoders/videoMimeTypes.js.map +1 -0
- package/dist/media/decoders/webCodecsChunkDecoder.d.ts +99 -0
- package/dist/media/decoders/webCodecsChunkDecoder.d.ts.map +1 -0
- package/dist/media/decoders/webCodecsChunkDecoder.js +265 -0
- package/dist/media/decoders/webCodecsChunkDecoder.js.map +1 -0
- package/dist/media/displayCapture.d.ts +128 -0
- package/dist/media/displayCapture.d.ts.map +1 -0
- package/dist/media/displayCapture.js +245 -0
- package/dist/media/displayCapture.js.map +1 -0
- package/dist/media/frameCapture.d.ts +25 -8
- package/dist/media/frameCapture.d.ts.map +1 -1
- package/dist/media/frameCapture.js +42 -72
- package/dist/media/frameCapture.js.map +1 -1
- package/dist/media/frameSampler.d.ts +94 -0
- package/dist/media/frameSampler.d.ts.map +1 -0
- package/dist/media/frameSampler.js +134 -0
- package/dist/media/frameSampler.js.map +1 -0
- package/dist/media/index.d.ts +32 -0
- package/dist/media/index.d.ts.map +1 -0
- package/dist/media/index.js +32 -0
- package/dist/media/index.js.map +1 -0
- package/dist/media/localMediaController.d.ts +116 -0
- package/dist/media/localMediaController.d.ts.map +1 -0
- package/dist/media/localMediaController.js +281 -0
- package/dist/media/localMediaController.js.map +1 -0
- package/dist/media/mediaLayout.d.ts +93 -0
- package/dist/media/mediaLayout.d.ts.map +1 -0
- package/dist/media/mediaLayout.js +187 -0
- package/dist/media/mediaLayout.js.map +1 -0
- package/dist/media/mediaPreview.d.ts +94 -0
- package/dist/media/mediaPreview.d.ts.map +1 -0
- package/dist/media/mediaPreview.js +154 -0
- package/dist/media/mediaPreview.js.map +1 -0
- package/dist/media/mediaStage.d.ts +99 -0
- package/dist/media/mediaStage.d.ts.map +1 -0
- package/dist/media/mediaStage.js +171 -0
- package/dist/media/mediaStage.js.map +1 -0
- package/dist/media/model.d.ts +154 -0
- package/dist/media/model.d.ts.map +1 -0
- package/dist/media/model.js +9 -0
- package/dist/media/model.js.map +1 -0
- package/dist/media/playbackClock.d.ts +20 -0
- package/dist/media/playbackClock.d.ts.map +1 -0
- package/dist/media/playbackClock.js +13 -0
- package/dist/media/playbackClock.js.map +1 -0
- package/dist/media/videoFrameDecoder.d.ts +86 -0
- package/dist/media/videoFrameDecoder.d.ts.map +1 -0
- package/dist/media/videoFrameDecoder.js +2 -0
- package/dist/media/videoFrameDecoder.js.map +1 -0
- package/dist/media/videoFrameDecoderRegistry.d.ts +40 -0
- package/dist/media/videoFrameDecoderRegistry.d.ts.map +1 -0
- package/dist/media/videoFrameDecoderRegistry.js +69 -0
- package/dist/media/videoFrameDecoderRegistry.js.map +1 -0
- package/dist/media/videoPacing.d.ts +32 -0
- package/dist/media/videoPacing.d.ts.map +1 -0
- package/dist/media/videoPacing.js +37 -0
- package/dist/media/videoPacing.js.map +1 -0
- package/dist/media/videoPlayout.d.ts +168 -0
- package/dist/media/videoPlayout.d.ts.map +1 -0
- package/dist/media/videoPlayout.js +217 -0
- package/dist/media/videoPlayout.js.map +1 -0
- package/dist/media/videoSourceArbiter.d.ts +250 -0
- package/dist/media/videoSourceArbiter.d.ts.map +1 -0
- package/dist/media/videoSourceArbiter.js +423 -0
- package/dist/media/videoSourceArbiter.js.map +1 -0
- package/dist/testing/conformanceAssertions.d.ts +25 -0
- package/dist/testing/conformanceAssertions.d.ts.map +1 -0
- package/dist/testing/conformanceAssertions.js +64 -0
- package/dist/testing/conformanceAssertions.js.map +1 -0
- package/dist/testing/conformanceFixtures.d.ts +76 -0
- package/dist/testing/conformanceFixtures.d.ts.map +1 -0
- package/dist/testing/conformanceFixtures.js +169 -0
- package/dist/testing/conformanceFixtures.js.map +1 -0
- package/dist/testing/conformanceGates.d.ts +32 -0
- package/dist/testing/conformanceGates.d.ts.map +1 -0
- package/dist/testing/conformanceGates.js +35 -0
- package/dist/testing/conformanceGates.js.map +1 -0
- package/dist/testing/conformanceMicrophone.d.ts +10 -0
- package/dist/testing/conformanceMicrophone.d.ts.map +1 -0
- package/dist/testing/conformanceMicrophone.js +71 -0
- package/dist/testing/conformanceMicrophone.js.map +1 -0
- package/dist/testing/conformanceTimeline.d.ts +128 -0
- package/dist/testing/conformanceTimeline.d.ts.map +1 -0
- package/dist/testing/conformanceTimeline.js +32 -0
- package/dist/testing/conformanceTimeline.js.map +1 -0
- package/dist/testing/conformanceVideoStream.d.ts +29 -0
- package/dist/testing/conformanceVideoStream.d.ts.map +1 -0
- package/dist/testing/conformanceVideoStream.js +39 -0
- package/dist/testing/conformanceVideoStream.js.map +1 -0
- package/dist/testing/index.d.ts +29 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/index.js +28 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/testing/realtimeVideoConformance.d.ts +23 -0
- package/dist/testing/realtimeVideoConformance.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformance.js +139 -0
- package/dist/testing/realtimeVideoConformance.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceChecks.js +164 -0
- package/dist/testing/realtimeVideoConformanceChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.js +63 -0
- package/dist/testing/realtimeVideoConformanceFrameChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceSession.d.ts +108 -0
- package/dist/testing/realtimeVideoConformanceSession.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceSession.js +270 -0
- package/dist/testing/realtimeVideoConformanceSession.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts +4 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.js +248 -0
- package/dist/testing/realtimeVideoConformanceTurnChecks.js.map +1 -0
- package/dist/testing/realtimeVideoConformanceTypes.d.ts +155 -0
- package/dist/testing/realtimeVideoConformanceTypes.d.ts.map +1 -0
- package/dist/testing/realtimeVideoConformanceTypes.js +2 -0
- package/dist/testing/realtimeVideoConformanceTypes.js.map +1 -0
- package/dist/testing/recordingPlayout.d.ts +84 -0
- package/dist/testing/recordingPlayout.d.ts.map +1 -0
- package/dist/testing/recordingPlayout.js +120 -0
- package/dist/testing/recordingPlayout.js.map +1 -0
- package/package.json +29 -4
|
@@ -1,5 +1,27 @@
|
|
|
1
|
-
import { DEFAULT_REALTIME_AUDIO_TRACKS, ResolveRequestedTracks, } from '@memberjunction/ai';
|
|
1
|
+
import { DEFAULT_REALTIME_AUDIO_TRACKS, RealtimeTrackKey, ResolveRequestedTracks, } from '@memberjunction/ai';
|
|
2
2
|
import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
|
|
3
|
+
/**
|
|
4
|
+
* Session-config key under which the realtime runtime hands a driver the tracks to negotiate
|
|
5
|
+
* (`RealtimeSessionRuntime.BuildClientConfig` aggregates them from the active channels). It is a
|
|
6
|
+
* CLIENT-side negotiation hint, never a provider field: a driver reads it (Gemini) or ignores it.
|
|
7
|
+
*/
|
|
8
|
+
export const REQUESTED_TRACKS_SESSION_KEY = 'requestedTracks';
|
|
9
|
+
/**
|
|
10
|
+
* The session config as the PROVIDER must receive it: the server-built config minus the
|
|
11
|
+
* client-only hints the runtime adds. Drivers that send the config on the wire go through this —
|
|
12
|
+
* OpenAI's Realtime API rejects the WHOLE `session.update` on an unknown key
|
|
13
|
+
* (`unknown_parameter: 'session.requestedTracks'`), so one stray hint costs the session its
|
|
14
|
+
* instructions and every tool.
|
|
15
|
+
*/
|
|
16
|
+
export function ToProviderSessionConfig(sessionConfig) {
|
|
17
|
+
const { [REQUESTED_TRACKS_SESSION_KEY]: _clientOnly, ...providerConfig } = sessionConfig;
|
|
18
|
+
return providerConfig;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The {@link RealtimeClientError.Code} of a call that ended because its session reached its time limit (MJAPI's relay
|
|
22
|
+
* closed it with `REALTIME_RELAY_TIME_LIMIT_CLOSE_CODE`), so a host can say so instead of showing a connection error.
|
|
23
|
+
*/
|
|
24
|
+
export const REALTIME_CLIENT_TIME_LIMIT_ERROR_CODE = 'time-limit';
|
|
3
25
|
/**
|
|
4
26
|
* Abstract base class for **browser-side, provider-direct** realtime (voice) clients.
|
|
5
27
|
*
|
|
@@ -77,17 +99,55 @@ import { REALTIME_AUDIO_BIN_COUNT } from '../audio/audioMeter.js';
|
|
|
77
99
|
* client-owned-audio drivers, the remote WebRTC stream on peer-connection drivers.
|
|
78
100
|
* Meters must be released on disconnect ({@link closeAudioMeters}). A driver with no
|
|
79
101
|
* tappable plane simply attaches nothing — hosts fall back to turn-state animation.
|
|
102
|
+
* 10. **Follow a replaced microphone track, when the transport can.** A device switch or a
|
|
103
|
+
* lost device swaps the track inside the mic stream, and anything bound to the old track (a
|
|
104
|
+
* WebRTC sender, a Web Audio source node, the input meter's clone) goes silent. A driver
|
|
105
|
+
* implements {@link BaseRealtimeClient.ReplaceMicrophone} to rebind them: `replaceTrack` on
|
|
106
|
+
* its senders, or `IPcmMicCapture.Rebind` on its PCM capture, and a new input meter.
|
|
107
|
+
* 11. **Refuse a transport you don't speak.** A session minted to run through MJAPI's realtime
|
|
108
|
+
* relay (`ClientRealtimeSessionConfig.Transport` `'relay'`) has no provider token, only a relay
|
|
109
|
+
* URL. Every driver calls {@link BaseRealtimeClient.AssertTransportSupported} first in
|
|
110
|
+
* {@link BaseRealtimeClient.Connect}, so a driver that connects only to its provider refuses
|
|
111
|
+
* such a session with a clear error instead of failing at the provider. A driver that speaks
|
|
112
|
+
* its provider's protocol through the relay opts in by overriding
|
|
113
|
+
* {@link BaseRealtimeClient.SupportsRelayTransport}.
|
|
114
|
+
* 12. **The agent's video keeps the voice's rules.** A driver whose model sends the agent's video
|
|
115
|
+
* hands it over through {@link emitRemoteVideo} once per session, and only while its outbound
|
|
116
|
+
* video track is live; an audio-only session hands over nothing and never plays a video part as
|
|
117
|
+
* audio. Its player gets each frame as the model sent it (a `RealtimeVideoFrame`, in order).
|
|
118
|
+
* Barge-in flushes the video with the voice (obligation #3) and the cut turn's late media never
|
|
119
|
+
* plays; a turn's end lets its video play out; a video that carries the voice never plays it
|
|
120
|
+
* twice; each second of generated video is reported once (`OutputTokenDetails.VideoSeconds`); the
|
|
121
|
+
* video a model streams between answers (Vertex AI's avatar does) plays, but is not the agent
|
|
122
|
+
* speaking: it reports no `'speaking'`, leaves {@link IsBusy} as it was, and does not keep
|
|
123
|
+
* {@link IsAudioPlaying} true once the answer's media has played; a resumed session keeps its
|
|
124
|
+
* video; `Disconnect` releases it. A driver whose voice and video share a media timeline queues
|
|
125
|
+
* its PCM at its media times and gives the player its voice playback as the clock
|
|
126
|
+
* (`VideoPlayoutOptions.Clock`). The video conformance kit
|
|
127
|
+
* (`@memberjunction/ai-realtime-client/testing`) checks each rule.
|
|
80
128
|
*/
|
|
81
129
|
export class BaseRealtimeClient {
|
|
82
130
|
constructor() {
|
|
83
131
|
// ── Media-track state and negotiation ──────────────────────────────────────
|
|
84
132
|
/** Tracks requested, established, or denied on this session. */
|
|
85
133
|
this.tracks = [];
|
|
134
|
+
/** What the driver supports, recorded by {@link negotiateTracks} so {@link AddTrack} can resolve against it later. */
|
|
135
|
+
this.supportedTracks = null;
|
|
136
|
+
/** The index the next added track's id takes, so an added track never reuses an id. */
|
|
137
|
+
this.nextTrackIndex = 0;
|
|
86
138
|
// ── Audio-activity metering (capability surface — see driver obligation #9) ─
|
|
87
139
|
/** Meter over the USER's microphone, when the driver attached one. */
|
|
88
140
|
this.inputAudioMeter = null;
|
|
89
141
|
/** Meter over the AGENT's audio output, when the driver attached one. */
|
|
90
142
|
this.outputAudioMeter = null;
|
|
143
|
+
// ── Remote (agent) audio slot ──────────────────────────────────────────────
|
|
144
|
+
// Lives here, not in each driver, so every driver that owns a tappable agent-audio plane
|
|
145
|
+
// (a WebRTC remote track, or a PCM playout engine's output node) shares ONE lifecycle:
|
|
146
|
+
// current stream + late-subscriber replay + isolated fan-out + reset on Disconnect.
|
|
147
|
+
/** The agent's current remote-audio stream, or `null` until a driver publishes one. */
|
|
148
|
+
this.remoteMediaStream = null;
|
|
149
|
+
/** Host handlers notified when the agent's stream lands (or immediately, if already present). */
|
|
150
|
+
this.remoteMediaStreamHandlers = [];
|
|
91
151
|
}
|
|
92
152
|
/**
|
|
93
153
|
* Established tracks on this session (tracks in the `'live'` state).
|
|
@@ -101,6 +161,37 @@ export class BaseRealtimeClient {
|
|
|
101
161
|
get AllTracks() {
|
|
102
162
|
return this.tracks;
|
|
103
163
|
}
|
|
164
|
+
/**
|
|
165
|
+
* How many concurrent inbound video streams the model accepts: the ceiling the driver declared when it
|
|
166
|
+
* negotiated tracks, else `1` when an inbound video track is live and `0` when it is not.
|
|
167
|
+
*
|
|
168
|
+
* A source arbiter reads this to decide whether several live video sources must be narrowed to one
|
|
169
|
+
* (with a note to the model on each switch) or passed through untouched.
|
|
170
|
+
*/
|
|
171
|
+
get MaxInboundVideoStreams() {
|
|
172
|
+
if (typeof this.inboundVideoStreamLimit === 'number') {
|
|
173
|
+
return this.IsTrackEstablished('video', 'inbound') ? this.inboundVideoStreamLimit : 0;
|
|
174
|
+
}
|
|
175
|
+
return this.IsTrackEstablished('video', 'inbound') ? 1 : 0;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* The frame rate (frames per second) the live inbound video track was negotiated at — the model's
|
|
179
|
+
* ceiling or the session's request, whichever is lower — or `undefined` when no inbound video track
|
|
180
|
+
* is live or none declared a rate. Anything pacing a video feed reads THIS, never a constant of its own.
|
|
181
|
+
*/
|
|
182
|
+
get InboundVideoRate() {
|
|
183
|
+
const track = this.tracks.find((t) => t.State === 'live' && t.Descriptor.Direction === 'inbound' && String(t.Descriptor.Modality).trim().toLowerCase() === 'video');
|
|
184
|
+
const rate = track?.Descriptor.Rate;
|
|
185
|
+
return typeof rate === 'number' && rate > 0 ? rate : undefined;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Whether the model takes inbound video on this session: the driver declared an inbound video track it supports when it
|
|
189
|
+
* negotiated tracks, and a stream limit above zero. No video track need be live; one is added when a capture starts
|
|
190
|
+
* ({@link AddTrack}). `false` before tracks are negotiated, and for a driver that does not negotiate them.
|
|
191
|
+
*/
|
|
192
|
+
get SupportsInboundVideo() {
|
|
193
|
+
return (this.inboundVideoStreamLimit ?? 1) > 0 && (this.supportedTracks ?? []).some(isInboundVideo);
|
|
194
|
+
}
|
|
104
195
|
/**
|
|
105
196
|
* Returns whether a track of the requested modality and direction is established and `'live'`.
|
|
106
197
|
*/
|
|
@@ -113,28 +204,85 @@ export class BaseRealtimeClient {
|
|
|
113
204
|
* Audio (inbound + outbound) is the baseline floor of every realtime session;
|
|
114
205
|
* any additional requested tracks (such as channel video) are unioned onto this baseline floor.
|
|
115
206
|
*/
|
|
116
|
-
negotiateTracks(requested, supported) {
|
|
207
|
+
negotiateTracks(requested, supported, maxInboundVideoStreams) {
|
|
208
|
+
// Keyed by direction + modality + SOURCE, so two requested inbound video streams fed by
|
|
209
|
+
// different sources stay two tracks (a model that accepts several needs both), while two
|
|
210
|
+
// descriptors that name no source still collapse to the one stream they always did.
|
|
117
211
|
const trackMap = new Map();
|
|
118
212
|
for (const t of DEFAULT_REALTIME_AUDIO_TRACKS) {
|
|
119
|
-
trackMap.set(
|
|
213
|
+
trackMap.set(RealtimeTrackKey(t), t);
|
|
120
214
|
}
|
|
121
215
|
if (requested) {
|
|
122
216
|
for (const t of requested) {
|
|
123
|
-
trackMap.set(
|
|
217
|
+
trackMap.set(RealtimeTrackKey(t), t);
|
|
124
218
|
}
|
|
125
219
|
}
|
|
126
220
|
const effectiveRequested = Array.from(trackMap.values());
|
|
127
221
|
const effectiveSupported = supported ?? DEFAULT_REALTIME_AUDIO_TRACKS;
|
|
128
|
-
|
|
222
|
+
this.inboundVideoStreamLimit = typeof maxInboundVideoStreams === 'number' ? maxInboundVideoStreams : undefined;
|
|
223
|
+
const resolved = ResolveRequestedTracks(effectiveRequested, effectiveSupported, (d, i) => `${d.Direction}:${String(d.Modality)}:${i}`, { MaxInboundVideoStreams: this.inboundVideoStreamLimit });
|
|
129
224
|
this.tracks = resolved.map((t) => ({
|
|
130
225
|
...t,
|
|
131
226
|
State: t.State === 'requested' ? 'live' : t.State,
|
|
132
227
|
}));
|
|
228
|
+
this.supportedTracks = effectiveSupported;
|
|
229
|
+
this.nextTrackIndex = this.tracks.length;
|
|
133
230
|
for (const track of this.tracks) {
|
|
134
231
|
this.emitTrackStateChange(track);
|
|
135
232
|
}
|
|
136
233
|
return this.tracks;
|
|
137
234
|
}
|
|
235
|
+
/**
|
|
236
|
+
* Adds a track to a running session, such as inbound video when the user starts their camera, resolved against
|
|
237
|
+
* what the driver supported when it negotiated at connect. The new track is `'live'`, or `'unsupported'` with the
|
|
238
|
+
* reason (the model takes no such track, or no more inbound video streams). A live track with the same
|
|
239
|
+
* direction, modality and source is returned as it is. Other tracks keep their ids.
|
|
240
|
+
*
|
|
241
|
+
* The base re-negotiates locally, which is all a driver needs when its provider accepts the media without a
|
|
242
|
+
* setup change (Gemini Live takes video frames at any time). A driver whose wire must change overrides this and
|
|
243
|
+
* calls it.
|
|
244
|
+
*
|
|
245
|
+
* @returns The track, or `null` when the session has not negotiated tracks: before it connects, or with a driver
|
|
246
|
+
* that does not negotiate them (only the Gemini driver does today).
|
|
247
|
+
*/
|
|
248
|
+
AddTrack(descriptor) {
|
|
249
|
+
if (!this.supportedTracks) {
|
|
250
|
+
return null;
|
|
251
|
+
}
|
|
252
|
+
const key = RealtimeTrackKey(descriptor);
|
|
253
|
+
const existing = this.tracks.find((t) => t.State === 'live' && RealtimeTrackKey(t.Descriptor) === key);
|
|
254
|
+
if (existing) {
|
|
255
|
+
return existing;
|
|
256
|
+
}
|
|
257
|
+
// Resolve after the live inbound video streams, so the stream cap and its reason count them.
|
|
258
|
+
const liveVideo = this.tracks.filter((t) => t.State === 'live' && isInboundVideo(t.Descriptor)).map((t) => t.Descriptor);
|
|
259
|
+
const index = this.nextTrackIndex++;
|
|
260
|
+
const resolved = ResolveRequestedTracks([...liveVideo, descriptor], this.supportedTracks, (d) => `${d.Direction}:${String(d.Modality)}:${index}`, { MaxInboundVideoStreams: this.inboundVideoStreamLimit });
|
|
261
|
+
const added = resolved[resolved.length - 1];
|
|
262
|
+
const track = { ...added, State: added.State === 'requested' ? 'live' : added.State };
|
|
263
|
+
this.tracks = [...this.tracks.filter((t) => RealtimeTrackKey(t.Descriptor) !== key), track];
|
|
264
|
+
this.emitTrackStateChange(track);
|
|
265
|
+
return track;
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Ends a track on a running session, such as inbound video when the user stops their camera: the track is
|
|
269
|
+
* reported `'ended'` and leaves {@link AllTracks}. Audio, the session's floor, cannot be removed.
|
|
270
|
+
*
|
|
271
|
+
* @returns Whether a track was ended.
|
|
272
|
+
*/
|
|
273
|
+
RemoveTrack(descriptor) {
|
|
274
|
+
if (String(descriptor.Modality).trim().toLowerCase() === 'audio') {
|
|
275
|
+
return false;
|
|
276
|
+
}
|
|
277
|
+
const key = RealtimeTrackKey(descriptor);
|
|
278
|
+
const track = this.tracks.find((t) => RealtimeTrackKey(t.Descriptor) === key);
|
|
279
|
+
if (!track) {
|
|
280
|
+
return false;
|
|
281
|
+
}
|
|
282
|
+
this.tracks = this.tracks.filter((t) => t !== track);
|
|
283
|
+
this.emitTrackStateChange({ ...track, State: 'ended' });
|
|
284
|
+
return true;
|
|
285
|
+
}
|
|
138
286
|
/**
|
|
139
287
|
* The session's current audible activity, or `null` when this driver attached no
|
|
140
288
|
* meters at all (hosts then keep their turn-state-driven visuals). Sampled by hosts
|
|
@@ -167,6 +315,83 @@ export class BaseRealtimeClient {
|
|
|
167
315
|
this.attachInputAudioMeter(null);
|
|
168
316
|
this.attachOutputAudioMeter(null);
|
|
169
317
|
}
|
|
318
|
+
/**
|
|
319
|
+
* Whether this driver can open a relay session (`ClientRealtimeSessionConfig.Transport` `'relay'`): connect to
|
|
320
|
+
* MJAPI's realtime relay at the session's `RelayUrl` rather than to its provider. `false` here; a driver that speaks
|
|
321
|
+
* its provider's protocol through the relay opts in by overriding it (the Gemini Enterprise client does).
|
|
322
|
+
*/
|
|
323
|
+
get SupportsRelayTransport() {
|
|
324
|
+
return false;
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Refuses a session this driver cannot open: a relay session, unless the driver opts in through
|
|
328
|
+
* {@link SupportsRelayTransport}. Every driver calls this first in {@link Connect}, before it touches the session's
|
|
329
|
+
* media or opens a connection (driver obligation #11). The message names the provider key and never quotes the
|
|
330
|
+
* relay URL, which carries the relay's ticket.
|
|
331
|
+
*
|
|
332
|
+
* @param config The session the host asked this driver to open.
|
|
333
|
+
* @throws When the session runs through the relay and this driver does not support it.
|
|
334
|
+
*/
|
|
335
|
+
AssertTransportSupported(config) {
|
|
336
|
+
if (config.Transport === 'relay' && !this.SupportsRelayTransport) {
|
|
337
|
+
throw new Error(`The '${config.Provider}' realtime client cannot open a session through MJAPI's relay (Transport 'relay'); it connects only to its provider.`);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
/**
|
|
341
|
+
* Returns the AGENT's remote-audio {@link MediaStream} when this driver owns a tappable
|
|
342
|
+
* agent-audio plane (a WebRTC driver's remote track, or a PCM driver's playout output),
|
|
343
|
+
* or `null` otherwise. Hosts use it to MIX the agent's voice into a browser-side recording
|
|
344
|
+
* alongside the mic; `null` (drivers with no tappable plane, or before the stream lands)
|
|
345
|
+
* degrades gracefully to mic-only capture. Hosts may still call it as `client.GetRemoteMediaStream?.()`.
|
|
346
|
+
* Pure read — never mutates.
|
|
347
|
+
*/
|
|
348
|
+
GetRemoteMediaStream() {
|
|
349
|
+
return this.remoteMediaStream;
|
|
350
|
+
}
|
|
351
|
+
/**
|
|
352
|
+
* Registers a handler invoked when the agent's remote-audio stream becomes available —
|
|
353
|
+
* IMMEDIATELY (synchronously) if it has already landed, otherwise when the driver publishes
|
|
354
|
+
* it (a WebRTC track typically lands AFTER {@link Connect} resolves). Lets a host attach the
|
|
355
|
+
* agent voice to a recording that began before the stream landed. Handler errors are
|
|
356
|
+
* isolated (logged) so a host bug never disturbs the call. Handlers are session-scoped:
|
|
357
|
+
* {@link clearRemoteMediaStream} (called from `Disconnect`) drops them.
|
|
358
|
+
*/
|
|
359
|
+
OnRemoteMediaStream(handler) {
|
|
360
|
+
this.remoteMediaStreamHandlers.push(handler);
|
|
361
|
+
if (this.remoteMediaStream) {
|
|
362
|
+
this.invokeRemoteMediaStreamHandler(handler, this.remoteMediaStream);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Driver hook: records the agent's remote-audio stream and fans it out to every registered
|
|
367
|
+
* handler. Passing `null` (a driver with no tappable plane) just records `null` and
|
|
368
|
+
* notifies nobody.
|
|
369
|
+
*/
|
|
370
|
+
publishRemoteMediaStream(stream) {
|
|
371
|
+
this.remoteMediaStream = stream;
|
|
372
|
+
if (!stream)
|
|
373
|
+
return;
|
|
374
|
+
for (const handler of this.remoteMediaStreamHandlers) {
|
|
375
|
+
this.invokeRemoteMediaStreamHandler(handler, stream);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* Driver hook for `Disconnect`: drops the stream AND the handlers. Handlers must not survive
|
|
380
|
+
* into a later `Connect` on a reused instance, or a stale host callback would receive the
|
|
381
|
+
* next session's stream.
|
|
382
|
+
*/
|
|
383
|
+
clearRemoteMediaStream() {
|
|
384
|
+
this.remoteMediaStream = null;
|
|
385
|
+
this.remoteMediaStreamHandlers = [];
|
|
386
|
+
}
|
|
387
|
+
invokeRemoteMediaStreamHandler(handler, stream) {
|
|
388
|
+
try {
|
|
389
|
+
handler(stream);
|
|
390
|
+
}
|
|
391
|
+
catch (error) {
|
|
392
|
+
console.warn(`[${this.constructor.name}] remote-stream handler threw:`, error);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
170
395
|
// ── Handler registration ──────────────────────────────────────────────────
|
|
171
396
|
/**
|
|
172
397
|
* Registers the (single) transcript handler. Receives interim deltas (`IsFinal: false`)
|
|
@@ -230,15 +455,19 @@ export class BaseRealtimeClient {
|
|
|
230
455
|
this.usageHandler = handler;
|
|
231
456
|
}
|
|
232
457
|
/**
|
|
233
|
-
* Registers the (single) remote-VIDEO handler — the model/avatar's video
|
|
234
|
-
*
|
|
235
|
-
*
|
|
458
|
+
* Registers the (single) remote-VIDEO handler — the model/avatar's video for a VIDEO session (a talking-head
|
|
459
|
+
* the host renders, e.g. as the agent's tile). Invoked once the provider's video is available.
|
|
460
|
+
*
|
|
461
|
+
* The video is a {@link MediaVideoSource}: a live stream (a WebRTC track), or a player that must own the
|
|
462
|
+
* `<video>` element (MSE or WebCodecs playout of encoded avatar video). Hosts show either with
|
|
463
|
+
* `AttachVideoSource`.
|
|
236
464
|
*
|
|
237
465
|
* **Optional capability:** audio-only drivers (the default) never emit — registering is always safe,
|
|
238
|
-
* but hosts must not assume
|
|
239
|
-
*
|
|
466
|
+
* but hosts must not assume video arrives. A driver whose model sends video (the server driver declares an
|
|
467
|
+
* outbound video track in its session's `Capabilities.SupportedOutboundTracks`, or renders an avatar,
|
|
468
|
+
* `BaseRealtimeModel.SupportsAvatarOutput`) calls {@link emitRemoteVideo} when it is live.
|
|
240
469
|
*
|
|
241
|
-
* @param handler Invoked with the remote video
|
|
470
|
+
* @param handler Invoked with the remote video when it becomes available.
|
|
242
471
|
*/
|
|
243
472
|
OnRemoteVideo(handler) {
|
|
244
473
|
this.remoteVideoHandler = handler;
|
|
@@ -279,9 +508,16 @@ export class BaseRealtimeClient {
|
|
|
279
508
|
emitUsage(usage) {
|
|
280
509
|
this.usageHandler?.(usage);
|
|
281
510
|
}
|
|
282
|
-
/**
|
|
283
|
-
|
|
284
|
-
|
|
511
|
+
/**
|
|
512
|
+
* Emits the model/avatar's remote video to the registered handler (video drivers only). A driver with a plain
|
|
513
|
+
* `MediaStream` passes it as it is; it is handed on as a `'stream'` source.
|
|
514
|
+
*/
|
|
515
|
+
emitRemoteVideo(video) {
|
|
516
|
+
this.remoteVideoHandler?.('Kind' in video ? video : { Kind: 'stream', Stream: video });
|
|
285
517
|
}
|
|
286
518
|
}
|
|
519
|
+
/** Whether a descriptor is an inbound video track. */
|
|
520
|
+
function isInboundVideo(descriptor) {
|
|
521
|
+
return descriptor.Direction === 'inbound' && String(descriptor.Modality).trim().toLowerCase() === 'video';
|
|
522
|
+
}
|
|
287
523
|
//# sourceMappingURL=baseRealtimeClient.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,6BAA6B,
|
|
1
|
+
{"version":3,"file":"baseRealtimeClient.js","sourceRoot":"","sources":["../../src/generic/baseRealtimeClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,6BAA6B,EAK7B,gBAAgB,EAChB,sBAAsB,GAEzB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAuB,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AAGpF;;;;GAIG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,iBAAiB,CAAC;AAE9D;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,aAAyB;IAC7D,MAAM,EAAE,CAAC,4BAA4B,CAAC,EAAE,WAAW,EAAE,GAAG,cAAc,EAAE,GAAG,aAAa,CAAC;IACzF,OAAO,cAAc,CAAC;AAC1B,CAAC;AAuID;;;GAGG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAG,YAAY,CAAC;AAelE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuGG;AACH,MAAM,OAAgB,kBAAkB;IAAxC;QAWI,8EAA8E;QAC9E,gEAAgE;QACtD,WAAM,GAAoB,EAAE,CAAC;QAmBvC,sHAAsH;QAC9G,oBAAe,GAA8C,IAAI,CAAC;QAE1E,uFAAuF;QAC/E,mBAAc,GAAG,CAAC,CAAC;QAqJ3B,+EAA+E;QAC/E,sEAAsE;QAC5D,oBAAe,GAA+B,IAAI,CAAC;QAC7D,yEAAyE;QAC/D,qBAAgB,GAA+B,IAAI,CAAC;QAmF9D,8EAA8E;QAC9E,yFAAyF;QACzF,uFAAuF;QACvF,oFAAoF;QAEpF,uFAAuF;QAC/E,sBAAiB,GAAuB,IAAI,CAAC;QACrD,iGAAiG;QACzF,8BAAyB,GAAyC,EAAE,CAAC;IAwUjF,CAAC;IAjlBG;;OAEG;IACH,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,CAAC;IACzD,CAAC;IAED;;OAEG;IACH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAWD;;;;;;OAMG;IACH,IAAW,sBAAsB;QAC7B,IAAI,OAAO,IAAI,CAAC,uBAAuB,KAAK,QAAQ,EAAE,CAAC;YACnD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1F,CAAC;QACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED;;;;OAIG;IACH,IAAW,gBAAgB;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,CACtI,CAAC;QACF,MAAM,IAAI,GAAG,KAAK,EAAE,UAAU,CAAC,IAAI,CAAC;QACpC,OAAO,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,CAAC;IAED;;;;OAIG;IACH,IAAW,oBAAoB;QAC3B,OAAO,CAAC,IAAI,CAAC,uBAAuB,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACxG,CAAC;IAED;;OAEG;IACI,kBAAkB,CAAC,QAAgB,EAAE,SAAiC;QACzE,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CACnB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC,SAAS,KAAK,SAAS,CAClI,CAAC;IACN,CAAC;IAED;;;;OAIG;IACO,eAAe,CACrB,SAAyD,EACzD,SAAyD,EACzD,sBAA+B;QAE/B,wFAAwF;QACxF,yFAAyF;QACzF,oFAAoF;QACpF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;QAC5D,KAAK,MAAM,CAAC,IAAI,6BAA6B,EAAE,CAAC;YAC5C,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzC,CAAC;QACD,IAAI,SAAS,EAAE,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACxB,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACzC,CAAC;QACL,CAAC;QACD,MAAM,kBAAkB,GAAuC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7F,MAAM,kBAAkB,GAAG,SAAS,IAAI,6BAA6B,CAAC;QACtE,IAAI,CAAC,uBAAuB,GAAG,OAAO,sBAAsB,KAAK,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/G,MAAM,QAAQ,GAAG,sBAAsB,CACnC,kBAAkB,EAClB,kBAAkB,EAClB,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EACrD,EAAE,sBAAsB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAC3D,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,GAAG,CAAC;YACJ,KAAK,EAAE,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;SACpD,CAAC,CAAC,CAAC;QACJ,IAAI,CAAC,eAAe,GAAG,kBAAkB,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACrC,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,QAAQ,CAAC,UAAmC;QAC/C,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;QACvG,IAAI,QAAQ,EAAE,CAAC;YACX,OAAO,QAAQ,CAAC;QACpB,CAAC;QACD,6FAA6F;QAC7F,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,IAAI,cAAc,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;QACzH,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,sBAAsB,CACnC,CAAC,GAAG,SAAS,EAAE,UAAU,CAAC,EAC1B,IAAI,CAAC,eAAe,EACpB,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,EACtD,EAAE,sBAAsB,EAAE,IAAI,CAAC,uBAAuB,EAAE,CAC3D,CAAC;QACF,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAkB,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACrG,IAAI,CAAC,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,CAAC;QAC5F,IAAI,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC;QACjC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;OAKG;IACI,WAAW,CAAC,UAAgF;QAC/F,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,EAAE,CAAC;YAC/D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC;QAC9E,IAAI,CAAC,KAAK,EAAE,CAAC;YACT,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,oBAAoB,CAAC,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC;IAChB,CAAC;IAQD;;;;;OAKG;IACI,gBAAgB;QACnB,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,OAAO;YACH,UAAU,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACtE,WAAW,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI;YACzE,SAAS,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;YAC5F,UAAU,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC,IAAI;SAClG,CAAC;IACN,CAAC;IAED,8FAA8F;IACpF,qBAAqB,CAAC,KAAiC;QAC7D,IAAI,CAAC,eAAe,EAAE,KAAK,EAAE,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IACjC,CAAC;IAED,2FAA2F;IACjF,sBAAsB,CAAC,KAAiC;QAC9D,IAAI,CAAC,gBAAgB,EAAE,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAClC,CAAC;IAED,wFAAwF;IAC9E,gBAAgB;QACtB,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;IACtC,CAAC;IAqBD;;;;OAIG;IACH,IAAc,sBAAsB;QAChC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;OAQG;IACO,wBAAwB,CAAC,MAAmC;QAClE,IAAI,MAAM,CAAC,SAAS,KAAK,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC/D,MAAM,IAAI,KAAK,CACX,QAAQ,MAAM,CAAC,QAAQ,sHAAsH,CAChJ,CAAC;QACN,CAAC;IACL,CAAC;IAYD;;;;;;;OAOG;IACI,oBAAoB;QACvB,OAAO,IAAI,CAAC,iBAAiB,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACI,mBAAmB,CAAC,OAAsC;QAC7D,IAAI,CAAC,yBAAyB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzE,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,wBAAwB,CAAC,MAA0B;QACzD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;QAChC,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,yBAAyB,EAAE,CAAC;YACnD,IAAI,CAAC,8BAA8B,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACzD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACO,sBAAsB;QAC5B,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,yBAAyB,GAAG,EAAE,CAAC;IACxC,CAAC;IAEO,8BAA8B,CAAC,OAAsC,EAAE,MAAmB;QAC9F,IAAI,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,gCAAgC,EAAE,KAAK,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IAqHD,6EAA6E;IAE7E;;;;OAIG;IACI,YAAY,CAAC,OAAuD;QACvE,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC;IACrC,CAAC;IAED;;;OAGG;IACI,UAAU,CAAC,OAA+C;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC;IACnC,CAAC;IAED;;;OAGG;IACI,aAAa,CAAC,OAA6C;QAC9D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACI,cAAc,CAAC,OAAmB;QACrC,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,OAAO,CAAC,OAA6C;QACxD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACI,aAAa,CAAC,OAA0C;QAC3D,IAAI,CAAC,kBAAkB,GAAG,OAAO,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,kBAAkB,CAAC,OAAuC;QAC7D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC;IAChC,CAAC;IAeD,6EAA6E;IAE7E,qEAAqE;IAC3D,oBAAoB,CAAC,KAAoB;QAC/C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,mEAAmE;IACzD,cAAc,CAAC,UAAoC;QACzD,IAAI,CAAC,iBAAiB,EAAE,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;IAED,oEAAoE;IAC1D,YAAY,CAAC,IAA4B;QAC/C,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,+DAA+D;IACrD,eAAe,CAAC,KAA0B;QAChD,IAAI,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IAED,yDAAyD;IAC/C,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED,6EAA6E;IACnE,gBAAgB;QACtB,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;IACjC,CAAC;IAED,4GAA4G;IAClG,SAAS,CAAC,KAA0B;QAC1C,IAAI,CAAC,YAAY,EAAE,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACO,eAAe,CAAC,KAAqC;QAC3D,IAAI,CAAC,kBAAkB,EAAE,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,CAAC;CACJ;AAED,sDAAsD;AACtD,SAAS,cAAc,CAAC,UAAmC;IACvD,OAAO,UAAU,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC;AAC9G,CAAC"}
|
|
@@ -304,8 +304,57 @@ export declare abstract class OpenAIProtocolRealtimeClient extends BaseRealtimeC
|
|
|
304
304
|
* provider, and narration is disposable by contract, so the update is dropped with a debug
|
|
305
305
|
* log rather than queued to come out late and stale. Hosts SHOULD still gate on
|
|
306
306
|
* {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
|
|
307
|
+
*
|
|
308
|
+
* The wire frame is built by {@link buildSpokenUpdateEvent}, which is where the identity
|
|
309
|
+
* rule lives — see that method for why the caller's direction is never sent on its own.
|
|
307
310
|
*/
|
|
308
311
|
RequestSpokenUpdate(instructions: string): void;
|
|
312
|
+
/**
|
|
313
|
+
* The `response.create` frame for one spoken update — **carrying the session identity, not
|
|
314
|
+
* replacing it.**
|
|
315
|
+
*
|
|
316
|
+
* ⚠️ `response.instructions` IS A FULL OVERRIDE OF THE SESSION SYSTEM PROMPT for that one
|
|
317
|
+
* response, not an addition to it. That is the whole reason this method exists. The
|
|
318
|
+
* server-side twin of this driver (`openAIRealtime.ts`) already states the rule and guards
|
|
319
|
+
* the BLANK case on it — forwarding `''` "would wipe the co-agent identity framing" — but the
|
|
320
|
+
* NON-blank case was never guarded, so every caller that supplied a real direction wiped it
|
|
321
|
+
* just as thoroughly and much less visibly.
|
|
322
|
+
*
|
|
323
|
+
* What that cost, live: the one turn that rides this method had no persona, so the model fell
|
|
324
|
+
* back to its vendor default and opened a hiring interview with *"I'm ChatGPT, your friendly
|
|
325
|
+
* voice companion"* while every other turn in the same session correctly said *"I'm Sam
|
|
326
|
+
* Rivera, Support Team Lead"* (bizapps-caliber#397 / MJ#4591). Every commentary-style caller
|
|
327
|
+
* is exposed the same way — an opening turn, a silence check-in, a delegation narration — and
|
|
328
|
+
* none of them should have to restate an identity the session already carries.
|
|
329
|
+
*
|
|
330
|
+
* Three cases, and the middle one is the fix:
|
|
331
|
+
*
|
|
332
|
+
* - **Blank direction** → a bare `response.create`. "Respond now, under the session prompt"
|
|
333
|
+
* (the meeting-mode bridge trigger passes `''`). Matches the server twin exactly.
|
|
334
|
+
* - **Direction + session instructions** → `identity, then direction`. The identity leads
|
|
335
|
+
* because it is context, and the direction is LAST because it is the thing being asked for
|
|
336
|
+
* now — the most recent line is the one a model weights hardest.
|
|
337
|
+
* - **Direction, no session instructions** → the direction alone. A session minted without
|
|
338
|
+
* instructions has no identity to lose, and prefixing a blank line would be noise.
|
|
339
|
+
*/
|
|
340
|
+
protected buildSpokenUpdateEvent(instructions: string): OAIProtocolResponseCreateEvent;
|
|
341
|
+
/**
|
|
342
|
+
* The session-level instructions currently in force — the co-agent's identity, persona and
|
|
343
|
+
* standing directives, as applied by `session.update`.
|
|
344
|
+
*
|
|
345
|
+
* A SEAM, not a field read, because the pact lives in a different place per transport: the
|
|
346
|
+
* websocket subclass keeps it in `sessionObject`, the WebRTC driver in its own `sessionConfig`.
|
|
347
|
+
* The base holds no copy of either, so it answers "no identity on record" and each transport
|
|
348
|
+
* overrides with the one it actually applied. A transport that forgets to override loses only
|
|
349
|
+
* the carry — {@link buildSpokenUpdateEvent} still sends the caller's direction — which is the
|
|
350
|
+
* old behaviour rather than a new failure.
|
|
351
|
+
*
|
|
352
|
+
* Null rather than `''` for "this session carries no identity", so {@link buildSpokenUpdateEvent}
|
|
353
|
+
* can tell "nothing to preserve" from "preserve this".
|
|
354
|
+
*/
|
|
355
|
+
protected currentSessionInstructions(): string | null;
|
|
356
|
+
/** Reads a session pact's `instructions`, or null when it carries none worth preserving. */
|
|
357
|
+
protected static readInstructions(pact: JSONObject | null | undefined): string | null;
|
|
309
358
|
/**
|
|
310
359
|
* Sends the tool result back as a `function_call_output` conversation item, then triggers
|
|
311
360
|
* a reply — immediately if the model is idle, otherwise queued until the current response
|
|
@@ -450,7 +499,10 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
|
|
|
450
499
|
private failPendingConnect;
|
|
451
500
|
/** Opens the provider socket for this connection (drivers own URL/auth specifics). */
|
|
452
501
|
protected abstract openProviderSocket(config: ClientRealtimeSessionConfig): IOpenAIProtocolClientSocket;
|
|
453
|
-
/**
|
|
502
|
+
/**
|
|
503
|
+
* Extracts the wire-shaped `session` object from the server pact. Default: the pact minus the
|
|
504
|
+
* client-only hints ({@link ToProviderSessionConfig}).
|
|
505
|
+
*/
|
|
454
506
|
protected resolveSessionObject(config: ClientRealtimeSessionConfig): JSONObject;
|
|
455
507
|
/** Resolves the PCM sample rate for the audio plane (both directions). */
|
|
456
508
|
protected abstract resolveSampleRate(config: ClientRealtimeSessionConfig): number;
|
|
@@ -473,6 +525,8 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
|
|
|
473
525
|
* `'listening'` only after all of that (obligation #7).
|
|
474
526
|
*/
|
|
475
527
|
Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
|
|
528
|
+
/** Rebinds the PCM capture and the input meter to the stream's current track (obligation #10); the socket stays open. */
|
|
529
|
+
ReplaceMicrophone(micStream: MediaStream): Promise<void>;
|
|
476
530
|
/**
|
|
477
531
|
* Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
|
|
478
532
|
* state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call
|
|
@@ -511,6 +565,8 @@ export declare abstract class OpenAIProtocolWebSocketRealtimeClient extends Open
|
|
|
511
565
|
* EMPTY `session.update` would be wrong).
|
|
512
566
|
*/
|
|
513
567
|
protected applySessionConfig(): void;
|
|
568
|
+
/** @inheritdoc — this transport's pact is {@link sessionObject}, applied over the socket. */
|
|
569
|
+
protected currentSessionInstructions(): string | null;
|
|
514
570
|
/** Streams one base64 PCM16 mic chunk as an `input_audio_buffer.append` frame. */
|
|
515
571
|
protected sendMicChunk(base64Pcm16: string): void;
|
|
516
572
|
/** Formats a socket-level error message for the host. Providers brand this (see xAI). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openAIProtocolClient.d.ts","sourceRoot":"","sources":["../../src/generic/openAIProtocolClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAA4B,MAAM,oBAAoB,CAAC;AACvG,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,
|
|
1
|
+
{"version":3,"file":"openAIProtocolClient.d.ts","sourceRoot":"","sources":["../../src/generic/openAIProtocolClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAA4B,MAAM,oBAAoB,CAAC;AACvG,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAA2B,MAAM,sBAAsB,CAAC;AAExG,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAO1E,yFAAyF;AACzF,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,wCAAwC,GAAG,iCAAiC,CAAC;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,uCAAuC,GAAG,gCAAgC,CAAC;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,uFAAuF;AACvF,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,sBAAsB,GAAG,6BAA6B,CAAC;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,uDAAuD,CAAC;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,oCAAoC;IACjD,IAAI,EAAE,uCAAuC,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mEAAmE;AACnE,MAAM,WAAW,wBAAwB;IACrC,IAAI,EAAE,mCAAmC,CAAC;CAC7C;AAED,+FAA+F;AAC/F,MAAM,WAAW,0BAA0B;IACvC,IAAI,EAAE,kBAAkB,CAAC;CAC5B;AAED,mGAAmG;AACnG,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,oGAAoG;AACpG,MAAM,WAAW,gBAAgB;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;CAC7B;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,gBAAgB,CAAA;KAAE,CAAC;IACxC,uDAAuD;IACvD,KAAK,CAAC,EAAE,gBAAgB,CAAC;CAC5B;AAED;;;GAGG;AACH,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,CAAC;CACvC;AACD,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,6BAA6B,GAAG,6BAA6B,CAAC;CACvE;AAED,iCAAiC;AACjC,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,gFAAgF;AAChF,MAAM,WAAW,uBAAuB;IACpC,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,yBAAyB,GAC/B,0BAA0B,GAC1B,yBAAyB,GACzB,qBAAqB,GACrB,sCAAsC,GACtC,oCAAoC,GACpC,wBAAwB,GACxB,0BAA0B,GAC1B,yBAAyB,GACzB,uBAAuB,GACvB,mCAAmC,GACnC,mCAAmC,GACnC,qBAAqB,GACrB,uBAAuB,CAAC;AAI9B,0FAA0F;AAC1F,MAAM,WAAW,6BAA6B;IAC1C,IAAI,EAAE,gBAAgB,CAAC;IACvB,OAAO,EAAE,UAAU,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,+EAA+E;AAC/E,MAAM,WAAW,iCAAiC;IAC9C,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,4DAA4D;AAC5D,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,0BAA0B,CAAC;IACjC,IAAI,EAAE,sBAAsB,GAAG,iCAAiC,CAAC;CACpE;AAED,kFAAkF;AAClF,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAED,0FAA0F;AAC1F,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,wFAAwF;AACxF,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;CACrC;AAED,sGAAsG;AACtG,MAAM,WAAW,sCAAsC;IACnD,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,yBAAyB,GAC/B,6BAA6B,GAC7B,sCAAsC,GACtC,8BAA8B,GAC9B,8BAA8B,GAC9B,sCAAsC,GACtC,sCAAsC,CAAC;AAI7C;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,8BAAsB,4BAA6B,SAAQ,kBAAkB;IAEzE,0EAA0E;IAC1E,SAAS,CAAC,oBAAoB,SAAM;IACpC,+FAA+F;IAC/F,SAAS,CAAC,cAAc,UAAS;IACjC,kGAAkG;IAClG,SAAS,CAAC,qBAAqB,UAAS;IACxC;;;;;OAKG;IACH,SAAS,CAAC,2BAA2B,SAAK;IAC1C;;;;;;;;OAQG;IACH,SAAS,CAAC,uBAAuB,UAAS;IAC1C;;;;;;OAMG;IACH,SAAS,CAAC,oBAAoB,UAAS;IACvC;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,EAAE,QAAQ,GAAG,WAAW,CAAY;IAChE;;;;;OAKG;IACH,SAAS,CAAC,YAAY,EAAE,mBAAmB,CAAY;IACvD,4FAA4F;IAC5F,SAAS,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI,CAAQ;IAI/C,uFAAuF;IACvF,SAAS,CAAC,QAAQ,CAAC,aAAa,IAAI,OAAO;IAC3C,sGAAsG;IACtG,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAC5E;;;OAGG;IACH,SAAS,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;IAI1C,0DAA0D;IAC1D,SAAS,KAAK,kBAAkB,IAAI,MAAM,CAEzC;IAED;;;;OAIG;IACH,SAAS,CAAC,qBAAqB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAI3D;;;;;OAKG;IACH,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAIzD;;;;;OAKG;IACH,SAAS,CAAC,oBAAoB,IAAI,IAAI;IAWtC;;;;;OAKG;IACH,SAAS,KAAK,0BAA0B,IAAI,OAAO,CAElD;IAED,+FAA+F;IAC/F,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAIvC,sEAAsE;IACtE,SAAS,CAAC,eAAe,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAIlE,wEAAwE;IACxE,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAI5C,gEAAgE;IAChE,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAMnE;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYnC;;;;;;;;;OASG;IACI,oBAAoB,IAAI,IAAI;IAoBnC;;;;;;OAMG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU1C;;;;;;;;;;;;;;;OAeG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IActD;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,SAAS,CAAC,sBAAsB,CAAC,YAAY,EAAE,MAAM,GAAG,8BAA8B;IAYtF;;;;;;;;;;;;;OAaG;IACH,SAAS,CAAC,0BAA0B,IAAI,MAAM,GAAG,IAAI;IAIrD,4FAA4F;IAC5F,SAAS,CAAC,MAAM,CAAC,gBAAgB,CAAC,IAAI,EAAE,UAAU,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI;IAKrF;;;;;;OAMG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/D;;;;OAIG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAMrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAID;;;OAGG;IACH,SAAS,CAAC,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAelD,mFAAmF;IACnF,SAAS,CAAC,WAAW,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAuF7D;;;OAGG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAInC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAG5C,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAI5C,6FAA6F;IAC7F,OAAO,CAAC,YAAY;IAUpB,yFAAyF;IACzF,OAAO,CAAC,gBAAgB;IAWxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAUvB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,YAAY;IAgCpB;;;;;;OAMG;IACH,SAAS,CAAC,qBAAqB,IAAI,IAAI;IAevC,0FAA0F;IAC1F,SAAS,CAAC,0BAA0B,IAAI,IAAI;IAW5C,0EAA0E;IAC1E,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAYpC,4EAA4E;IAC5E,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAKpD,uFAAuF;IACvF,SAAS,CAAC,SAAS,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;CAO9D;AAID;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC,uCAAuC;IACvC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH,8BAAsB,qCAAsC,SAAQ,4BAA4B;IAC5F,0FAA0F;IAC1F,SAAS,CAAC,MAAM,EAAE,2BAA2B,GAAG,IAAI,CAAQ;IAC5D,2DAA2D;IAC3D,SAAS,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAQ;IACvD,mEAAmE;IACnE,SAAS,CAAC,UAAU,EAAE,cAAc,GAAG,IAAI,CAAQ;IACnD;;;OAGG;IACH,SAAS,CAAC,aAAa,EAAE,UAAU,CAAM;IACzC,qFAAqF;IACrF,SAAS,CAAC,gBAAgB,UAAS;IACnC,4FAA4F;IAC5F,SAAS,CAAC,UAAU,UAAS;IAC7B,8FAA8F;IAC9F,OAAO,CAAC,sBAAsB,CAA6B;IAC3D,mFAAmF;IACnF,OAAO,CAAC,kBAAkB,CAAyC;IAInE,sFAAsF;IACtF,SAAS,CAAC,QAAQ,CAAC,kBAAkB,CAAC,MAAM,EAAE,2BAA2B,GAAG,2BAA2B;IACvG;;;OAGG;IACH,SAAS,CAAC,oBAAoB,CAAC,MAAM,EAAE,2BAA2B,GAAG,UAAU;IAG/E,0EAA0E;IAC1E,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,GAAG,MAAM;IACjF;;;;OAIG;IACH,SAAS,KAAK,sBAAsB,IAAI,OAAO,CAE9C;IAED;;;;OAIG;IACH,SAAS,KAAK,gBAAgB,IAAI,MAAM,CAEvC;IAID;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAoGhG,yHAAyH;IAC5G,iBAAiB,CAAC,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IASrE;;;;OAIG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAiCxC,yFAAyF;IACzF,SAAS,CAAC,aAAa,IAAI,OAAO;IAIlC,kBAAkB;IAClB,SAAS,CAAC,iBAAiB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAInE,0FAA0F;IAC1F,SAAS,CAAC,eAAe,IAAI,IAAI;IAIjC,mFAAmF;cAChE,qBAAqB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAInE;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAED;;;OAGG;cACgB,oBAAoB,IAAI,IAAI;IAU/C,iGAAiG;IACjG,cAAuB,0BAA0B,IAAI,OAAO,CAE3D;IAED,2EAA2E;cACxD,qBAAqB,IAAI,IAAI;IAOhD;;;;;OAKG;IACH,SAAS,CAAC,kBAAkB,IAAI,IAAI;IAOpC,6FAA6F;cAC1E,0BAA0B,IAAI,MAAM,GAAG,IAAI;IAI9D,kFAAkF;IAClF,SAAS,CAAC,YAAY,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAMjD,yFAAyF;IACzF,SAAS,CAAC,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIvD,kGAAkG;IAClG,SAAS,KAAK,sBAAsB,IAAI,MAAM,CAE7C;IAED,oFAAoF;IACpF,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAQlD;;;;;OAKG;IACH,SAAS,CAAC,iBAAiB,IAAI,IAAI;IAUnC;;;;OAIG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B,uGAAuG;IACvG,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;CAGrE"}
|