@memberjunction/ai-realtime-client 0.0.1 → 5.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/README.md +148 -28
  2. package/dist/audio/audioMeter.d.ts +101 -0
  3. package/dist/audio/audioMeter.d.ts.map +1 -0
  4. package/dist/audio/audioMeter.js +193 -0
  5. package/dist/audio/audioMeter.js.map +1 -0
  6. package/dist/audio/micCapture.d.ts +26 -0
  7. package/dist/audio/micCapture.d.ts.map +1 -0
  8. package/dist/audio/micCapture.js +69 -0
  9. package/dist/audio/micCapture.js.map +1 -0
  10. package/dist/audio/pcmPlayback.d.ts +73 -0
  11. package/dist/audio/pcmPlayback.d.ts.map +1 -0
  12. package/dist/audio/pcmPlayback.js +78 -0
  13. package/dist/audio/pcmPlayback.js.map +1 -0
  14. package/dist/audio/pcmUtils.d.ts +17 -0
  15. package/dist/audio/pcmUtils.d.ts.map +1 -0
  16. package/dist/audio/pcmUtils.js +46 -0
  17. package/dist/audio/pcmUtils.js.map +1 -0
  18. package/dist/drivers/assemblyAIRealtimeClient.d.ts +384 -0
  19. package/dist/drivers/assemblyAIRealtimeClient.d.ts.map +1 -0
  20. package/dist/drivers/assemblyAIRealtimeClient.js +732 -0
  21. package/dist/drivers/assemblyAIRealtimeClient.js.map +1 -0
  22. package/dist/drivers/elevenLabsRealtimeClient.d.ts +362 -0
  23. package/dist/drivers/elevenLabsRealtimeClient.d.ts.map +1 -0
  24. package/dist/drivers/elevenLabsRealtimeClient.js +686 -0
  25. package/dist/drivers/elevenLabsRealtimeClient.js.map +1 -0
  26. package/dist/drivers/geminiRealtimeClient.d.ts +406 -0
  27. package/dist/drivers/geminiRealtimeClient.d.ts.map +1 -0
  28. package/dist/drivers/geminiRealtimeClient.js +675 -0
  29. package/dist/drivers/geminiRealtimeClient.js.map +1 -0
  30. package/dist/drivers/openAIRealtimeClient.d.ts +381 -0
  31. package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -0
  32. package/dist/drivers/openAIRealtimeClient.js +602 -0
  33. package/dist/drivers/openAIRealtimeClient.js.map +1 -0
  34. package/dist/drivers/xaiRealtimeClient.d.ts +430 -0
  35. package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -0
  36. package/dist/drivers/xaiRealtimeClient.js +676 -0
  37. package/dist/drivers/xaiRealtimeClient.js.map +1 -0
  38. package/dist/generic/baseRealtimeClient.d.ts +382 -0
  39. package/dist/generic/baseRealtimeClient.d.ts.map +1 -0
  40. package/dist/generic/baseRealtimeClient.js +208 -0
  41. package/dist/generic/baseRealtimeClient.js.map +1 -0
  42. package/dist/index.d.ts +11 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +11 -0
  45. package/dist/index.js.map +1 -0
  46. package/package.json +28 -7
@@ -0,0 +1,430 @@
1
+ import { ClientRealtimeSessionConfig, JSONObject } from '@memberjunction/ai';
2
+ import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
3
+ import { IRealtimePcmPlayback } from '../audio/pcmPlayback.js';
4
+ import { IPcmMicCapture } from '../audio/micCapture.js';
5
+ /**
6
+ * The Grok Voice realtime websocket endpoint. The model is appended as a `?model=` query
7
+ * parameter, derived from `config.Model` (e.g. `grok-voice-latest`) — unlike OpenAI's GA
8
+ * browser flow (where the ephemeral secret encodes the model), xAI takes the model on the URL.
9
+ */
10
+ export declare const XAI_REALTIME_WS_URL = "wss://api.x.ai/v1/realtime";
11
+ /**
12
+ * Browser auth rides as a websocket SUBPROTOCOL with this prefix: the ephemeral client secret
13
+ * is passed as `"xai-client-secret." + token`. Browsers cannot set request headers on a
14
+ * websocket handshake, so an `Authorization` header is impossible — the subprotocol channel is
15
+ * how the server-minted one-time credential reaches xAI (no API key ever touches the browser).
16
+ */
17
+ export declare const XAI_CLIENT_SECRET_SUBPROTOCOL_PREFIX = "xai-client-secret.";
18
+ /**
19
+ * The Grok Voice wire audio format is FIXED: 16-bit signed little-endian PCM, mono, 24 kHz,
20
+ * base64-encoded, in BOTH directions (`input_audio_buffer.append` up, `response.audio.delta`
21
+ * down) — there is no per-session format negotiation on this provider.
22
+ */
23
+ export declare const XAI_PCM_SAMPLE_RATE = 24000;
24
+ /** Streaming delta of the assistant's spoken-text transcript (GA or beta event name). */
25
+ export interface XAIResponseAudioTranscriptDelta {
26
+ type: 'response.output_audio_transcript.delta' | 'response.audio_transcript.delta';
27
+ delta: string;
28
+ response_id?: string;
29
+ item_id?: string;
30
+ }
31
+ /** Final assistant transcript for a turn (GA or beta event name). */
32
+ export interface XAIResponseAudioTranscriptDone {
33
+ type: 'response.output_audio_transcript.done' | 'response.audio_transcript.done';
34
+ transcript: string;
35
+ response_id?: string;
36
+ item_id?: string;
37
+ }
38
+ /**
39
+ * One base64 PCM16 chunk of the agent's spoken output. The GA event is
40
+ * `response.audio.delta`; the GA-output alias `response.output_audio.delta` is also accepted
41
+ * so playback populates regardless of the model generation.
42
+ */
43
+ export interface XAIResponseAudioDelta {
44
+ type: 'response.audio.delta' | 'response.output_audio.delta';
45
+ delta: string;
46
+ response_id?: string;
47
+ item_id?: string;
48
+ }
49
+ /** Final transcription of the user's spoken input for a turn. */
50
+ export interface XAIInputAudioTranscriptionCompleted {
51
+ type: 'conversation.item.input_audio_transcription.completed';
52
+ transcript: string;
53
+ item_id?: string;
54
+ }
55
+ /** The model finished assembling a function (tool) call and wants it executed. */
56
+ export interface XAIFunctionCallArgumentsDone {
57
+ type: 'response.function_call_arguments.done';
58
+ call_id: string;
59
+ name: string;
60
+ /** JSON-encoded arguments. */
61
+ arguments: string;
62
+ }
63
+ /** The provider detected the user starting to speak (barge-in). */
64
+ export interface XAIInputAudioBufferSpeechStarted {
65
+ type: 'input_audio_buffer.speech_started';
66
+ }
67
+ /** A new response (turn) started — tracked so we never start a second overlapping response. */
68
+ export interface XAIResponseCreated {
69
+ type: 'response.created';
70
+ }
71
+ /**
72
+ * A full response (turn) completed — carries the usage payload for THIS response
73
+ * (`input_tokens` / `output_tokens`), i.e. per-response DELTAS, exactly the `OnUsage`
74
+ * contract shape.
75
+ */
76
+ export interface XAIResponseDone {
77
+ type: 'response.done';
78
+ response?: {
79
+ usage?: {
80
+ input_tokens?: number;
81
+ output_tokens?: number;
82
+ [detail: string]: unknown;
83
+ };
84
+ };
85
+ }
86
+ /** Provider-side error frame. */
87
+ export interface XAIErrorEvent {
88
+ type: 'error';
89
+ error?: {
90
+ message?: string;
91
+ code?: string;
92
+ };
93
+ }
94
+ /** Events whose `type` we don't explicitly handle still parse to this shape. */
95
+ export interface XAIUnknownEvent {
96
+ type: string;
97
+ }
98
+ export type XAIRealtimeEvent = XAIResponseAudioTranscriptDelta | XAIResponseAudioTranscriptDone | XAIResponseAudioDelta | XAIInputAudioTranscriptionCompleted | XAIFunctionCallArgumentsDone | XAIInputAudioBufferSpeechStarted | XAIResponseCreated | XAIResponseDone | XAIErrorEvent | XAIUnknownEvent;
99
+ /** Applies the server-built session config (instructions + tools) to the live session. */
100
+ export interface XAISessionUpdateEvent {
101
+ type: 'session.update';
102
+ session: JSONObject;
103
+ }
104
+ /** A user or system `message` conversation item. */
105
+ export interface XAIMessageItem {
106
+ type: 'message';
107
+ role: 'user' | 'system';
108
+ content: Array<{
109
+ type: 'input_text';
110
+ text: string;
111
+ }>;
112
+ }
113
+ /** The output of an executed function (tool) call, correlated by `call_id`. */
114
+ export interface XAIFunctionCallOutputItem {
115
+ type: 'function_call_output';
116
+ call_id: string;
117
+ output: string;
118
+ }
119
+ /** Creates a conversation item (message or tool output). */
120
+ export interface XAIConversationItemCreateEvent {
121
+ type: 'conversation.item.create';
122
+ item: XAIMessageItem | XAIFunctionCallOutputItem;
123
+ }
124
+ /** Asks the model to produce a response, optionally with one-off instructions. */
125
+ export interface XAIResponseCreateEvent {
126
+ type: 'response.create';
127
+ response?: {
128
+ instructions: string;
129
+ };
130
+ }
131
+ /** Cancels the model's in-flight response (generation stops; a response.done follows). */
132
+ export interface XAIResponseCancelEvent {
133
+ type: 'response.cancel';
134
+ }
135
+ /** Appends one base64 PCM16 mic chunk to the provider's input audio buffer. */
136
+ export interface XAIInputAudioBufferAppendEvent {
137
+ type: 'input_audio_buffer.append';
138
+ audio: string;
139
+ }
140
+ export type XAIRealtimeClientEvent = XAISessionUpdateEvent | XAIConversationItemCreateEvent | XAIResponseCreateEvent | XAIResponseCancelEvent | XAIInputAudioBufferAppendEvent;
141
+ /**
142
+ * The minimal websocket surface this client depends on: assignable lifecycle handlers plus
143
+ * `send`/`close`. Declaring the seam as an interface (rather than the platform `WebSocket`)
144
+ * lets unit tests inject a fully in-memory fake that captures outbound frames and drives the
145
+ * handlers with xAI-shaped events — no websocket, no network.
146
+ */
147
+ export interface IxAIClientSocket {
148
+ /** Invoked once the socket is open. */
149
+ onopen: (() => void) | null;
150
+ /** Invoked with each inbound frame's raw string payload. */
151
+ onmessage: ((data: string) => void) | null;
152
+ /** Invoked on a socket-level error (fatal). */
153
+ onerror: ((message: string) => void) | null;
154
+ /** Invoked when the socket closes (any reason). */
155
+ onclose: (() => void) | null;
156
+ /** Sends one JSON-serialized client frame. */
157
+ send(data: string): void;
158
+ /** Terminates the underlying connection. */
159
+ close(): void;
160
+ }
161
+ /**
162
+ * xAI Grok Voice implementation of {@link BaseRealtimeClient}: a **browser-direct** websocket
163
+ * connection to xAI's Grok Voice realtime API, authenticated with the server-minted ONE-TIME
164
+ * ephemeral client secret passed as a websocket SUBPROTOCOL (no API key ever reaches the
165
+ * browser, and browsers cannot set a handshake `Authorization` header).
166
+ *
167
+ * Registered with the ClassFactory under the key `'xai'` — the `Provider` string the server's
168
+ * matching Grok Voice driver stamps on its `ClientRealtimeSessionConfig` — so hosts resolve it
169
+ * without referencing this class directly.
170
+ *
171
+ * It is the structural CROSS of the two sibling drivers:
172
+ * - Like the AssemblyAI client driver, the transport is a WEBSOCKET and the audio plane is
173
+ * CLIENT-OWNED: mic PCM16 streams out as base64 frames via the shared {@link createPcmMicCapture}
174
+ * worklet, and the agent's base64 audio chunks feed the shared {@link RealtimePcmPlayback}
175
+ * engine ({@link IsAudioPlaying} comes from its playout clock). There is NO SDP handshake, NO
176
+ * peer connection, NO remote-audio sink element.
177
+ * - Like the OpenAI client driver, the EVENT PROTOCOL is OpenAI-Realtime-API compatible: the
178
+ * same client events (session.update, conversation.item.create, response.create,
179
+ * response.cancel, input_audio_buffer.append) and the same server events
180
+ * (response.output_audio_transcript.delta and .done, response.audio.delta,
181
+ * conversation.item.input_audio_transcription.completed,
182
+ * response.function_call_arguments.done, response.created, response.done,
183
+ * input_audio_buffer.speech_started, error). The response state machine, narration-kind
184
+ * tagging, and tool-result queueing are identical to the OpenAI driver.
185
+ *
186
+ * Connect handshake: open the model-on-URL endpoint with the subprotocol auth → on socket open,
187
+ * apply the server-authored `config.SessionConfig` via a `session.update` frame → build the
188
+ * audio plane at the provider's FIXED 24 kHz PCM16 format → report `'listening'`. The
189
+ * `'listening'` gate is the socket-open + session.update applied point (obligation #7): unlike
190
+ * AssemblyAI there is no separate `session.ready` confirmation in this protocol, so applying the
191
+ * config on open is the readiness boundary.
192
+ */
193
+ export declare class xAIRealtimeClient extends BaseRealtimeClient {
194
+ private socket;
195
+ private micStream;
196
+ private micCapture;
197
+ private playback;
198
+ /**
199
+ * The server-built session config applied verbatim via `session.update` once the socket
200
+ * opens. Protected so test subclasses can seed it without a full Connect.
201
+ */
202
+ protected sessionConfig: JSONObject | null;
203
+ /**
204
+ * Whether the CURRENT user turn has already emitted a transcription. Grok streams input
205
+ * transcription as repeated `.completed` events (each the full growing text), so the first emission
206
+ * of a turn appends a caption and the rest are flagged ReplacesPrevious — one in-place bubble, not a
207
+ * stack of growing duplicates. Reset on each `input_audio_buffer.speech_started` (new turn).
208
+ */
209
+ private userTurnTranscribed;
210
+ /** Accumulates the in-flight assistant transcript across delta frames. */
211
+ private pendingAssistantText;
212
+ /** True while the model has a response in flight; gates narration + queues the tool result. */
213
+ private responseActive;
214
+ /** Set when a tool result is ready while a response is active; sent on the next response.done. */
215
+ private pendingResultResponse;
216
+ /**
217
+ * Set by {@link RequestSpokenUpdate} just before it sends its `response.create`, and
218
+ * CONSUMED by the very next `response.created` frame, which stamps
219
+ * {@link activeResponseKind} for that turn.
220
+ */
221
+ private pendingNarrationKind;
222
+ /**
223
+ * The kind of the response currently in flight. Event ordering: `response.created` →
224
+ * transcript deltas → `*_audio_transcript.done` → `response.done`. The transcript-done
225
+ * frame therefore arrives while the kind is still set, letting {@link onAssistantDone}
226
+ * classify the turn; `response.done` then resets the kind to `'normal'`.
227
+ */
228
+ private activeResponseKind;
229
+ /**
230
+ * The client's own view of the session state — mirrors what {@link emitStateChange} last
231
+ * reported, EXCEPT after a tool call is emitted: the host typically shows its own busy
232
+ * indicator then, so the client silently leaves `'speaking'` (no emission) to preserve the
233
+ * host's indicator until the result reply starts (see {@link handleEvent}).
234
+ */
235
+ private currentState;
236
+ /** True once Disconnect ran — an expected socket close must not surface as fatal. */
237
+ private closedByConsumer;
238
+ /**
239
+ * Opens the client-direct Grok Voice websocket: the model-on-URL endpoint with the
240
+ * subprotocol auth, the server-authored `config.SessionConfig` applied via `session.update`
241
+ * once the socket opens, then the audio plane at the provider's fixed 24 kHz format. Reports
242
+ * `'listening'` only after all of that (obligation #7).
243
+ */
244
+ Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
245
+ /**
246
+ * Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
247
+ * state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call more
248
+ * than once.
249
+ */
250
+ Disconnect(): Promise<void>;
251
+ /**
252
+ * Injects typed text as a user-role `message` conversation item, then triggers a reply
253
+ * through the SAME collision-safe path tool results use ({@link requestResultResponse}).
254
+ * No-op when the socket isn't open.
255
+ *
256
+ * **SendText implies barge-in** (base-contract rule): an active spoken response is cancelled
257
+ * via {@link CancelActiveResponse} before the text is injected, so the typed turn takes the
258
+ * floor immediately instead of waiting behind stale speech. When nothing is active the cancel
259
+ * is a no-op and the reply triggers immediately.
260
+ */
261
+ SendText(text: string): void;
262
+ /**
263
+ * @inheritdoc
264
+ *
265
+ * Sends `response.cancel` (only when a response is actually in flight) and flushes the
266
+ * locally-owned playout queue so already-generated speech stops coming out of the speaker
267
+ * immediately. Resets the local response state machine (active flag, narration kind,
268
+ * accumulated transcript) but PRESERVES any queued tool-result trigger: delegated work is
269
+ * never affected by a floor-control cancel, and the queued trigger still fires on the
270
+ * cancelled response's trailing `response.done`. No-op when idle or when the socket is gone.
271
+ */
272
+ CancelActiveResponse(): void;
273
+ /**
274
+ * Injects a system-role context item the model can draw on the next time it speaks, WITHOUT
275
+ * forcing a reply. Item creation is always safe mid-response, so it is sent immediately even
276
+ * while a reply is in flight.
277
+ *
278
+ * NOTE: role must be 'system' — the OpenAI-compatible realtime API rejects 'developer' items.
279
+ */
280
+ SendContextNote(text: string): void;
281
+ /**
282
+ * Triggers ONE short spoken update with the given instructions. Marks the upcoming response
283
+ * as `'narration'` (flag consumed by the next `response.created`) so its transcripts are
284
+ * emitted with `Kind: 'narration'` — ephemeral by contract. Sets {@link responseActive}
285
+ * eagerly so a tool result landing mid-narration queues instead of colliding.
286
+ *
287
+ * **Skips when busy** (base-contract collision rule — drivers MUST queue or skip): a
288
+ * `response.create` sent while a response is in flight would be rejected/garbled by the
289
+ * provider, and narration is disposable by contract, so the update is dropped with a debug
290
+ * log rather than queued to come out late and stale. Hosts SHOULD still gate on
291
+ * {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
292
+ */
293
+ RequestSpokenUpdate(instructions: string): void;
294
+ /**
295
+ * Sends the tool result back as a `function_call_output` conversation item, then triggers a
296
+ * reply — immediately if the model is idle, otherwise queued until the current response
297
+ * (e.g. a progress narration) finishes. Without the queueing the result's `response.create`
298
+ * would collide with an in-flight narration and be dropped, leaving the model silent when
299
+ * delegated work comes back.
300
+ */
301
+ SendToolResult(callID: string, outputJson: string): void;
302
+ /**
303
+ * Mutes / unmutes by toggling the mic tracks' `enabled` flag: the capture pipeline stays up
304
+ * and streams SILENCE while muted (the provider's VAD sees a continuous stream and the
305
+ * un-mute is glitch-free — same policy as the other client drivers).
306
+ */
307
+ SetMuted(muted: boolean): void;
308
+ /** @inheritdoc */
309
+ get IsBusy(): boolean;
310
+ /**
311
+ * @inheritdoc
312
+ *
313
+ * Computed directly from the playout engine's playhead clock — this client OWNS the output
314
+ * buffer, so "audibly playing" is precisely "scheduled audio extends beyond the audio
315
+ * context's current time".
316
+ */
317
+ get IsAudioPlaying(): boolean;
318
+ /**
319
+ * Creation seam for the realtime websocket. Production wraps the platform-global `WebSocket`
320
+ * opened against the model-on-URL endpoint WITH the `xai-client-secret.<token>` subprotocol
321
+ * (browser auth — no handshake header is possible); unit tests override this to return an
322
+ * in-memory fake. Handlers are attached by {@link Connect} AFTER this returns, so the
323
+ * implementation must not require them at construction time.
324
+ */
325
+ protected createSocket(url: string, subprotocol: string): IxAIClientSocket;
326
+ /**
327
+ * Creation seam for the mic-capture pipeline at the provider's fixed 24 kHz rate. Production
328
+ * delegates to the shared {@link createPcmMicCapture}; unit tests override this with a no-op
329
+ * fake (and may capture `onPcmChunk` to simulate mic frames).
330
+ */
331
+ protected createMicCapture(micStream: MediaStream, sampleRate: number, onPcmChunk: (base64Pcm16: string) => void): Promise<IPcmMicCapture>;
332
+ /**
333
+ * Creation seam for the playout engine at the provider's fixed 24 kHz rate. Production
334
+ * returns the shared {@link RealtimePcmPlayback}.
335
+ */
336
+ protected createPlayback(sampleRate: number): IRealtimePcmPlayback;
337
+ /**
338
+ * Sends the server-controlled session config (instructions + tools) as a `session.update` so
339
+ * the co-agent's identity and tool set apply. Skipped when the host supplied no config (e.g.
340
+ * it failed to parse the server payload — the host already logged that; sending an EMPTY
341
+ * `session.update` would be wrong).
342
+ */
343
+ private applySessionConfig;
344
+ /** Streams one base64 PCM16 mic chunk as an `input_audio_buffer.append` frame. */
345
+ private sendMicChunk;
346
+ /** Surfaces a fatal socket error and marks the session unusable (obligation #6). */
347
+ private handleSocketError;
348
+ /**
349
+ * A socket close the CONSUMER didn't ask for is fatal: the provider hard-closes at token
350
+ * expiry and when it ends the session itself, so an unexpected close is how credential /
351
+ * session death reaches the host (obligation #6).
352
+ */
353
+ private handleSocketClose;
354
+ /** Parses one raw socket payload; non-JSON frames are ignored. */
355
+ private handleSocketMessage;
356
+ /** Dispatches a typed xAI realtime server event to the appropriate behavior. */
357
+ private handleEvent;
358
+ /** Appends an assistant transcript delta, reflects `'speaking'`, and emits the delta. */
359
+ private onAssistantDelta;
360
+ /**
361
+ * Finalizes the assistant turn: emits the final transcript tagged with the ACTIVE response
362
+ * kind (the transcript-done frame arrives BEFORE `response.done`, so
363
+ * {@link activeResponseKind} still reflects this turn), then returns to `'listening'`. Empty
364
+ * turns emit nothing.
365
+ */
366
+ private onAssistantDone;
367
+ /**
368
+ * Decodes one base64 PCM16 chunk of the agent's spoken output into the playout queue and
369
+ * reflects `'speaking'`. The client OWNS the audio plane on this websocket transport (unlike
370
+ * the WebRTC OpenAI driver where the peer connection plays the remote track), so agent audio
371
+ * arrives as these deltas and is scheduled by the shared playout engine.
372
+ */
373
+ private onAudioDelta;
374
+ /**
375
+ * Emits the user's spoken-input transcription. Grok STREAMS this — repeated
376
+ * `input_audio_transcription.completed` events, each carrying the full text so far — unlike OpenAI's
377
+ * single final. So the FIRST emission of a turn appends a fresh caption and every later one is flagged
378
+ * ReplacesPrevious, collapsing the stream into ONE in-place-updating user bubble. The per-turn flag
379
+ * resets on the next `speech_started` ({@link onSpeechStarted}).
380
+ */
381
+ private onUserTranscript;
382
+ /**
383
+ * Surfaces a completed tool call to the host. Two deliberate behaviors mirror the OpenAI
384
+ * driver: (1) the client silently leaves `'speaking'` (NO emission) so a host-rendered busy
385
+ * indicator isn't clobbered by this turn's trailing `response.done` / playback frames
386
+ * (obligation #1); (2) {@link responseActive} is CLEARED — the model has yielded the floor
387
+ * pending the result, so a queued {@link SendToolResult} can never deadlock (obligation #2).
388
+ */
389
+ private onToolCallFrame;
390
+ /**
391
+ * The user started speaking. This is a TRUE barge-in only when it cut off active model output
392
+ * (a response in flight or audio audibly playing) — a normal turn while the model is idle is
393
+ * NOT an interruption, so the emission is gated (base-contract rule). On a true barge-in the
394
+ * client OWNS the audio plane, so it flushes its own playout queue NOW (obligation #3),
395
+ * surfaces the interruption, and returns the floor; the provider cancels its own turn and
396
+ * emits a terminal `response.done`.
397
+ */
398
+ private onSpeechStarted;
399
+ /**
400
+ * Emits the completed response's usage to the host as a DELTA (the `response.done` usage
401
+ * payload covers exactly this response, so it is already incremental — the `OnUsage`
402
+ * contract's preferred shape). Frames without a usage payload emit nothing.
403
+ */
404
+ private emitResponseUsage;
405
+ /** Surfaces a provider error frame (non-fatal; the session continues). */
406
+ private onErrorFrame;
407
+ /**
408
+ * Asks the model to speak (a tool result or typed-text reply) — immediately if it's idle,
409
+ * otherwise queued until the current response finishes. An immediate trigger also CONSUMES
410
+ * any queued trigger debt: every payload item is already in the conversation, so one
411
+ * `response.create` voices everything (e.g. typed text barging in over a narration that had
412
+ * tool results queued behind it).
413
+ */
414
+ private requestResultResponse;
415
+ /** On a turn completing, fire any queued tool-result response so the answer is spoken. */
416
+ private flushPendingResultResponse;
417
+ /** Resets the per-session response state machine (used on Disconnect). */
418
+ private resetResponseState;
419
+ /** Updates the client's own state view and emits the change to the host. */
420
+ private setState;
421
+ /** JSON-serializes + sends a client event over the socket (only when open). */
422
+ private sendEvent;
423
+ }
424
+ /**
425
+ * Tree-shaking prevention: bundlers cannot see that {@link xAIRealtimeClient} is instantiated
426
+ * dynamically through the ClassFactory, so a consumer must call this no-op to create a static
427
+ * code path that keeps the `@RegisterClass` side effect alive.
428
+ */
429
+ export declare function LoadxAIRealtimeClient(): void;
430
+ //# sourceMappingURL=xaiRealtimeClient.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"xaiRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/xaiRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AAExF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1E;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,+BAA+B,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,oCAAoC,uBAAuB,CAAC;AAEzE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAQ,CAAC;AAOzC,yFAAyF;AACzF,MAAM,WAAW,+BAA+B;IAC5C,IAAI,EAAE,wCAAwC,GAAG,iCAAiC,CAAC;IACnF,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qEAAqE;AACrE,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,uCAAuC,GAAG,gCAAgC,CAAC;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,sBAAsB,GAAG,6BAA6B,CAAC;IAC7D,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,iEAAiE;AACjE,MAAM,WAAW,mCAAmC;IAChD,IAAI,EAAE,uDAAuD,CAAC;IAC9D,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,kFAAkF;AAClF,MAAM,WAAW,4BAA4B;IACzC,IAAI,EAAE,uCAAuC,CAAC;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,mEAAmE;AACnE,MAAM,WAAW,gCAAgC;IAC7C,IAAI,EAAE,mCAAmC,CAAC;CAC7C;AAED,+FAA+F;AAC/F,MAAM,WAAW,kBAAkB;IAC/B,IAAI,EAAE,kBAAkB,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE;YAAE,YAAY,CAAC,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;SAAE,CAAA;KAAE,CAAC;CACvG;AAED,iCAAiC;AACjC,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAED,gFAAgF;AAChF,MAAM,WAAW,eAAe;IAC5B,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,MAAM,gBAAgB,GACtB,+BAA+B,GAC/B,8BAA8B,GAC9B,qBAAqB,GACrB,mCAAmC,GACnC,4BAA4B,GAC5B,gCAAgC,GAChC,kBAAkB,GAClB,eAAe,GACf,aAAa,GACb,eAAe,CAAC;AAItB,0FAA0F;AAC1F,MAAM,WAAW,qBAAqB;IAClC,IAAI,EAAE,gBAAgB,CAAC;IACvB,OAAO,EAAE,UAAU,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC3B,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,YAAY,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACxD;AAED,+EAA+E;AAC/E,MAAM,WAAW,yBAAyB;IACtC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,4DAA4D;AAC5D,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,0BAA0B,CAAC;IACjC,IAAI,EAAE,cAAc,GAAG,yBAAyB,CAAC;CACpD;AAED,kFAAkF;AAClF,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,CAAC,EAAE;QAAE,YAAY,EAAE,MAAM,CAAA;KAAE,CAAC;CACvC;AAED,0FAA0F;AAC1F,MAAM,WAAW,sBAAsB;IACnC,IAAI,EAAE,iBAAiB,CAAC;CAC3B;AAED,+EAA+E;AAC/E,MAAM,WAAW,8BAA8B;IAC3C,IAAI,EAAE,2BAA2B,CAAC;IAClC,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,sBAAsB,GAC5B,qBAAqB,GACrB,8BAA8B,GAC9B,sBAAsB,GACtB,sBAAsB,GACtB,8BAA8B,CAAC;AAIrC;;;;;GAKG;AACH,MAAM,WAAW,gBAAgB;IAC7B,uCAAuC;IACvC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,4DAA4D;IAC5D,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,+CAA+C;IAC/C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,mDAAmD;IACnD,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,8CAA8C;IAC9C,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,4CAA4C;IAC5C,KAAK,IAAI,IAAI,CAAC;CACjB;AAkBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBACa,iBAAkB,SAAQ,kBAAkB;IAErD,OAAO,CAAC,MAAM,CAAiC;IAC/C,OAAO,CAAC,SAAS,CAA4B;IAC7C,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,QAAQ,CAAqC;IACrD;;;OAGG;IACH,SAAS,CAAC,aAAa,EAAE,UAAU,GAAG,IAAI,CAAQ;IAGlD;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB,CAAS;IACpC,0EAA0E;IAC1E,OAAO,CAAC,oBAAoB,CAAM;IAClC,+FAA+F;IAC/F,OAAO,CAAC,cAAc,CAAS;IAC/B,kGAAkG;IAClG,OAAO,CAAC,qBAAqB,CAAS;IACtC;;;;OAIG;IACH,OAAO,CAAC,oBAAoB,CAAS;IACrC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB,CAAoC;IAC9D;;;;;OAKG;IACH,OAAO,CAAC,YAAY,CAAiC;IACrD,qFAAqF;IACrF,OAAO,CAAC,gBAAgB,CAAS;IAIjC;;;;;OAKG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA+ChG;;;;OAIG;IACU,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BxC;;;;;;;;;OASG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAgBnC;;;;;;;;;OASG;IACI,oBAAoB,IAAI,IAAI;IAsBnC;;;;;;OAMG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAc1C;;;;;;;;;;;OAWG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAatD;;;;;;OAMG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAe/D;;;;OAIG;IACI,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAOrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED;;;;;;OAMG;IACH,IAAW,cAAc,IAAI,OAAO,CAEnC;IAID;;;;;;OAMG;IACH,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,gBAAgB;IAqB1E;;;;OAIG;cACa,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B;;;OAGG;IACH,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;IAMlE;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IAO1B,kFAAkF;IAClF,OAAO,CAAC,YAAY;IAMpB,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAQzB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAUzB,kEAAkE;IAClE,OAAO,CAAC,mBAAmB;IAmB3B,gFAAgF;IAChF,OAAO,CAAC,WAAW;IAoDnB,yFAAyF;IACzF,OAAO,CAAC,gBAAgB;IAQxB;;;;;OAKG;IACH,OAAO,CAAC,eAAe;IAWvB;;;;;OAKG;IACH,OAAO,CAAC,YAAY;IAUpB;;;;;;OAMG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAQvB;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IAevB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAYzB,0EAA0E;IAC1E,OAAO,CAAC,YAAY;IAUpB;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAc7B,0FAA0F;IAC1F,OAAO,CAAC,0BAA0B;IAUlC,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAU1B,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAKhB,+EAA+E;IAC/E,OAAO,CAAC,SAAS;CASpB;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,IAAI,CAE5C"}