@memberjunction/ai-realtime-client 5.48.0 → 5.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/drivers/huggingFaceRealtimeClient.d.ts +27 -148
- package/dist/drivers/huggingFaceRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/huggingFaceRealtimeClient.js +41 -388
- package/dist/drivers/huggingFaceRealtimeClient.js.map +1 -1
- package/dist/drivers/openAIRealtimeClient.d.ts +28 -270
- package/dist/drivers/openAIRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/openAIRealtimeClient.js +42 -379
- package/dist/drivers/openAIRealtimeClient.js.map +1 -1
- package/dist/drivers/xaiRealtimeClient.d.ts +57 -370
- package/dist/drivers/xaiRealtimeClient.d.ts.map +1 -1
- package/dist/drivers/xaiRealtimeClient.js +85 -554
- package/dist/drivers/xaiRealtimeClient.js.map +1 -1
- package/dist/generic/openAIProtocolClient.d.ts +527 -0
- package/dist/generic/openAIProtocolClient.d.ts.map +1 -0
- package/dist/generic/openAIProtocolClient.js +873 -0
- package/dist/generic/openAIProtocolClient.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { ClientRealtimeSessionConfig
|
|
2
|
-
import {
|
|
3
|
-
import { IRealtimePcmPlayback } from '../audio/pcmPlayback.js';
|
|
4
|
-
import { IPcmMicCapture } from '../audio/micCapture.js';
|
|
1
|
+
import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
|
|
2
|
+
import { OpenAIProtocolWebSocketRealtimeClient, IOpenAIProtocolClientSocket, OpenAIProtocolClientEvent, OpenAIProtocolServerEvent } from '../generic/openAIProtocolClient.js';
|
|
5
3
|
/**
|
|
6
4
|
* The Grok Voice realtime websocket endpoint. The model is appended as a `?model=` query
|
|
7
5
|
* parameter, derived from `config.Model` (e.g. `grok-voice-latest`) — unlike OpenAI's GA
|
|
@@ -21,143 +19,6 @@ export declare const XAI_CLIENT_SECRET_SUBPROTOCOL_PREFIX = "xai-client-secret."
|
|
|
21
19
|
* down) — there is no per-session format negotiation on this provider.
|
|
22
20
|
*/
|
|
23
21
|
export declare const XAI_PCM_SAMPLE_RATE = 24000;
|
|
24
|
-
/** Streaming delta of the assistant's spoken-text transcript (GA or beta event name). */
|
|
25
|
-
export interface XAIResponseAudioTranscriptDelta {
|
|
26
|
-
type: 'response.output_audio_transcript.delta' | 'response.audio_transcript.delta';
|
|
27
|
-
delta: string;
|
|
28
|
-
response_id?: string;
|
|
29
|
-
item_id?: string;
|
|
30
|
-
}
|
|
31
|
-
/** Final assistant transcript for a turn (GA or beta event name). */
|
|
32
|
-
export interface XAIResponseAudioTranscriptDone {
|
|
33
|
-
type: 'response.output_audio_transcript.done' | 'response.audio_transcript.done';
|
|
34
|
-
transcript: string;
|
|
35
|
-
response_id?: string;
|
|
36
|
-
item_id?: string;
|
|
37
|
-
}
|
|
38
|
-
/**
|
|
39
|
-
* One base64 PCM16 chunk of the agent's spoken output. The GA event is
|
|
40
|
-
* `response.audio.delta`; the GA-output alias `response.output_audio.delta` is also accepted
|
|
41
|
-
* so playback populates regardless of the model generation.
|
|
42
|
-
*/
|
|
43
|
-
export interface XAIResponseAudioDelta {
|
|
44
|
-
type: 'response.audio.delta' | 'response.output_audio.delta';
|
|
45
|
-
delta: string;
|
|
46
|
-
response_id?: string;
|
|
47
|
-
item_id?: string;
|
|
48
|
-
}
|
|
49
|
-
/** Final transcription of the user's spoken input for a turn. */
|
|
50
|
-
export interface XAIInputAudioTranscriptionCompleted {
|
|
51
|
-
type: 'conversation.item.input_audio_transcription.completed';
|
|
52
|
-
transcript: string;
|
|
53
|
-
item_id?: string;
|
|
54
|
-
}
|
|
55
|
-
/** The model finished assembling a function (tool) call and wants it executed. */
|
|
56
|
-
export interface XAIFunctionCallArgumentsDone {
|
|
57
|
-
type: 'response.function_call_arguments.done';
|
|
58
|
-
call_id: string;
|
|
59
|
-
name: string;
|
|
60
|
-
/** JSON-encoded arguments. */
|
|
61
|
-
arguments: string;
|
|
62
|
-
}
|
|
63
|
-
/** The provider detected the user starting to speak (barge-in). */
|
|
64
|
-
export interface XAIInputAudioBufferSpeechStarted {
|
|
65
|
-
type: 'input_audio_buffer.speech_started';
|
|
66
|
-
}
|
|
67
|
-
/** A new response (turn) started — tracked so we never start a second overlapping response. */
|
|
68
|
-
export interface XAIResponseCreated {
|
|
69
|
-
type: 'response.created';
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* A full response (turn) completed — carries the usage payload for THIS response
|
|
73
|
-
* (`input_tokens` / `output_tokens`), i.e. per-response DELTAS, exactly the `OnUsage`
|
|
74
|
-
* contract shape.
|
|
75
|
-
*/
|
|
76
|
-
export interface XAIResponseDone {
|
|
77
|
-
type: 'response.done';
|
|
78
|
-
response?: {
|
|
79
|
-
usage?: {
|
|
80
|
-
input_tokens?: number;
|
|
81
|
-
output_tokens?: number;
|
|
82
|
-
[detail: string]: unknown;
|
|
83
|
-
};
|
|
84
|
-
};
|
|
85
|
-
}
|
|
86
|
-
/** Provider-side error frame. */
|
|
87
|
-
export interface XAIErrorEvent {
|
|
88
|
-
type: 'error';
|
|
89
|
-
error?: {
|
|
90
|
-
message?: string;
|
|
91
|
-
code?: string;
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
/** Events whose `type` we don't explicitly handle still parse to this shape. */
|
|
95
|
-
export interface XAIUnknownEvent {
|
|
96
|
-
type: string;
|
|
97
|
-
}
|
|
98
|
-
export type XAIRealtimeEvent = XAIResponseAudioTranscriptDelta | XAIResponseAudioTranscriptDone | XAIResponseAudioDelta | XAIInputAudioTranscriptionCompleted | XAIFunctionCallArgumentsDone | XAIInputAudioBufferSpeechStarted | XAIResponseCreated | XAIResponseDone | XAIErrorEvent | XAIUnknownEvent;
|
|
99
|
-
/** Applies the server-built session config (instructions + tools) to the live session. */
|
|
100
|
-
export interface XAISessionUpdateEvent {
|
|
101
|
-
type: 'session.update';
|
|
102
|
-
session: JSONObject;
|
|
103
|
-
}
|
|
104
|
-
/** A user or system `message` conversation item. */
|
|
105
|
-
export interface XAIMessageItem {
|
|
106
|
-
type: 'message';
|
|
107
|
-
role: 'user' | 'system';
|
|
108
|
-
content: Array<{
|
|
109
|
-
type: 'input_text';
|
|
110
|
-
text: string;
|
|
111
|
-
}>;
|
|
112
|
-
}
|
|
113
|
-
/** The output of an executed function (tool) call, correlated by `call_id`. */
|
|
114
|
-
export interface XAIFunctionCallOutputItem {
|
|
115
|
-
type: 'function_call_output';
|
|
116
|
-
call_id: string;
|
|
117
|
-
output: string;
|
|
118
|
-
}
|
|
119
|
-
/** Creates a conversation item (message or tool output). */
|
|
120
|
-
export interface XAIConversationItemCreateEvent {
|
|
121
|
-
type: 'conversation.item.create';
|
|
122
|
-
item: XAIMessageItem | XAIFunctionCallOutputItem;
|
|
123
|
-
}
|
|
124
|
-
/** Asks the model to produce a response, optionally with one-off instructions. */
|
|
125
|
-
export interface XAIResponseCreateEvent {
|
|
126
|
-
type: 'response.create';
|
|
127
|
-
response?: {
|
|
128
|
-
instructions: string;
|
|
129
|
-
};
|
|
130
|
-
}
|
|
131
|
-
/** Cancels the model's in-flight response (generation stops; a response.done follows). */
|
|
132
|
-
export interface XAIResponseCancelEvent {
|
|
133
|
-
type: 'response.cancel';
|
|
134
|
-
}
|
|
135
|
-
/** Appends one base64 PCM16 mic chunk to the provider's input audio buffer. */
|
|
136
|
-
export interface XAIInputAudioBufferAppendEvent {
|
|
137
|
-
type: 'input_audio_buffer.append';
|
|
138
|
-
audio: string;
|
|
139
|
-
}
|
|
140
|
-
export type XAIRealtimeClientEvent = XAISessionUpdateEvent | XAIConversationItemCreateEvent | XAIResponseCreateEvent | XAIResponseCancelEvent | XAIInputAudioBufferAppendEvent;
|
|
141
|
-
/**
|
|
142
|
-
* The minimal websocket surface this client depends on: assignable lifecycle handlers plus
|
|
143
|
-
* `send`/`close`. Declaring the seam as an interface (rather than the platform `WebSocket`)
|
|
144
|
-
* lets unit tests inject a fully in-memory fake that captures outbound frames and drives the
|
|
145
|
-
* handlers with xAI-shaped events — no websocket, no network.
|
|
146
|
-
*/
|
|
147
|
-
export interface IxAIClientSocket {
|
|
148
|
-
/** Invoked once the socket is open. */
|
|
149
|
-
onopen: (() => void) | null;
|
|
150
|
-
/** Invoked with each inbound frame's raw string payload. */
|
|
151
|
-
onmessage: ((data: string) => void) | null;
|
|
152
|
-
/** Invoked on a socket-level error (fatal). */
|
|
153
|
-
onerror: ((message: string) => void) | null;
|
|
154
|
-
/** Invoked when the socket closes (any reason). */
|
|
155
|
-
onclose: (() => void) | null;
|
|
156
|
-
/** Sends one JSON-serialized client frame. */
|
|
157
|
-
send(data: string): void;
|
|
158
|
-
/** Terminates the underlying connection. */
|
|
159
|
-
close(): void;
|
|
160
|
-
}
|
|
161
22
|
/**
|
|
162
23
|
* xAI Grok Voice implementation of {@link BaseRealtimeClient}: a **browser-direct** websocket
|
|
163
24
|
* connection to xAI's Grok Voice realtime API, authenticated with the server-minted ONE-TIME
|
|
@@ -168,38 +29,19 @@ export interface IxAIClientSocket {
|
|
|
168
29
|
* matching Grok Voice driver stamps on its `ClientRealtimeSessionConfig` — so hosts resolve it
|
|
169
30
|
* without referencing this class directly.
|
|
170
31
|
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
176
|
-
*
|
|
177
|
-
* - Like the OpenAI client driver, the EVENT PROTOCOL is OpenAI-Realtime-API compatible: the
|
|
178
|
-
* same client events (session.update, conversation.item.create, response.create,
|
|
179
|
-
* response.cancel, input_audio_buffer.append) and the same server events
|
|
180
|
-
* (response.output_audio_transcript.delta and .done, response.audio.delta,
|
|
181
|
-
* conversation.item.input_audio_transcription.completed,
|
|
182
|
-
* response.function_call_arguments.done, response.created, response.done,
|
|
183
|
-
* input_audio_buffer.speech_started, error). The response state machine, narration-kind
|
|
184
|
-
* tagging, and tool-result queueing are identical to the OpenAI driver.
|
|
32
|
+
* Grok Voice speaks the OpenAI Realtime wire protocol over a websocket with a client-owned PCM
|
|
33
|
+
* audio plane, so nearly everything lives in the shared layers: the protocol brain
|
|
34
|
+
* (`OpenAIProtocolRealtimeClient`) and the websocket+PCM transport
|
|
35
|
+
* ({@link OpenAIProtocolWebSocketRealtimeClient}). This class supplies only the Grok
|
|
36
|
+
* specifics: the model-on-URL endpoint + subprotocol auth, the FIXED 24 kHz audio format,
|
|
37
|
+
* Grok's STREAMED input-transcription behavior, and wire diagnostics.
|
|
185
38
|
*
|
|
186
|
-
* Connect handshake: open the
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
*
|
|
190
|
-
* AssemblyAI there is no separate `session.ready` confirmation in this protocol, so applying the
|
|
191
|
-
* config on open is the readiness boundary.
|
|
39
|
+
* Connect handshake (shared skeleton): open the socket → apply the server-authored
|
|
40
|
+
* `config.SessionConfig` via `session.update` on OPEN (this protocol has no separate readiness
|
|
41
|
+
* ack, so applying the config on open is the readiness boundary — obligation #7) → build the
|
|
42
|
+
* audio plane at 24 kHz → report `'listening'`.
|
|
192
43
|
*/
|
|
193
|
-
export declare class xAIRealtimeClient extends
|
|
194
|
-
private socket;
|
|
195
|
-
private micStream;
|
|
196
|
-
private micCapture;
|
|
197
|
-
private playback;
|
|
198
|
-
/**
|
|
199
|
-
* The server-built session config applied verbatim via `session.update` once the socket
|
|
200
|
-
* opens. Protected so test subclasses can seed it without a full Connect.
|
|
201
|
-
*/
|
|
202
|
-
protected sessionConfig: JSONObject | null;
|
|
44
|
+
export declare class xAIRealtimeClient extends OpenAIProtocolWebSocketRealtimeClient {
|
|
203
45
|
/**
|
|
204
46
|
* Whether the CURRENT user turn has already emitted a transcription. Grok streams input
|
|
205
47
|
* transcription as repeated `.completed` events (each the full growing text), so the first emission
|
|
@@ -207,219 +49,64 @@ export declare class xAIRealtimeClient extends BaseRealtimeClient {
|
|
|
207
49
|
* stack of growing duplicates. Reset on each `input_audio_buffer.speech_started` (new turn).
|
|
208
50
|
*/
|
|
209
51
|
private userTurnTranscribed;
|
|
210
|
-
/** Accumulates the in-flight assistant transcript across delta frames. */
|
|
211
|
-
private pendingAssistantText;
|
|
212
|
-
/** True while the model has a response in flight; gates narration + queues the tool result. */
|
|
213
|
-
private responseActive;
|
|
214
|
-
/** Set when a tool result is ready while a response is active; sent on the next response.done. */
|
|
215
|
-
private pendingResultResponse;
|
|
216
|
-
/**
|
|
217
|
-
* Set by {@link RequestSpokenUpdate} just before it sends its `response.create`, and
|
|
218
|
-
* CONSUMED by the very next `response.created` frame, which stamps
|
|
219
|
-
* {@link activeResponseKind} for that turn.
|
|
220
|
-
*/
|
|
221
|
-
private pendingNarrationKind;
|
|
222
|
-
/**
|
|
223
|
-
* The kind of the response currently in flight. Event ordering: `response.created` →
|
|
224
|
-
* transcript deltas → `*_audio_transcript.done` → `response.done`. The transcript-done
|
|
225
|
-
* frame therefore arrives while the kind is still set, letting {@link onAssistantDone}
|
|
226
|
-
* classify the turn; `response.done` then resets the kind to `'normal'`.
|
|
227
|
-
*/
|
|
228
|
-
private activeResponseKind;
|
|
229
52
|
/**
|
|
230
|
-
*
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
*
|
|
53
|
+
* Text of the current user turn's latest streamed caption, used to recognize that a post-pause
|
|
54
|
+
* caption CONTINUES the same utterance (Grok re-emits the full accumulated text) rather than
|
|
55
|
+
* starting a new turn. Cleared in {@link onResponseStarted} — once the model replies, the user's
|
|
56
|
+
* turn is over and a later utterance must never merge into it. Mirrors the server session.
|
|
234
57
|
*/
|
|
235
|
-
private
|
|
236
|
-
/**
|
|
237
|
-
|
|
58
|
+
private lastUserTranscript;
|
|
59
|
+
/** @inheritdoc — used in the shared diagnostics + close messages. */
|
|
60
|
+
protected get providerDebugLabel(): string;
|
|
61
|
+
/** @inheritdoc — model on the URL, ephemeral secret as the subprotocol. */
|
|
62
|
+
protected openProviderSocket(config: ClientRealtimeSessionConfig): IOpenAIProtocolClientSocket;
|
|
63
|
+
/** @inheritdoc — the Grok Voice wire format is fixed at 24 kHz PCM16 both directions. */
|
|
64
|
+
protected resolveSampleRate(_config: ClientRealtimeSessionConfig): number;
|
|
238
65
|
/**
|
|
239
|
-
*
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
243
|
-
|
|
244
|
-
Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
|
|
245
|
-
/**
|
|
246
|
-
* Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
|
|
247
|
-
* state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call more
|
|
248
|
-
* than once.
|
|
249
|
-
*/
|
|
250
|
-
Disconnect(): Promise<void>;
|
|
251
|
-
/**
|
|
252
|
-
* Injects typed text as a user-role `message` conversation item, then triggers a reply
|
|
253
|
-
* through the SAME collision-safe path tool results use ({@link requestResultResponse}).
|
|
254
|
-
* No-op when the socket isn't open.
|
|
255
|
-
*
|
|
256
|
-
* **SendText implies barge-in** (base-contract rule): an active spoken response is cancelled
|
|
257
|
-
* via {@link CancelActiveResponse} before the text is injected, so the typed turn takes the
|
|
258
|
-
* floor immediately instead of waiting behind stale speech. When nothing is active the cancel
|
|
259
|
-
* is a no-op and the reply triggers immediately.
|
|
66
|
+
* Creation seam for the realtime websocket. Production wraps the platform-global `WebSocket`
|
|
67
|
+
* opened against the model-on-URL endpoint WITH the `xai-client-secret.<token>` subprotocol
|
|
68
|
+
* (browser auth — no handshake header is possible); unit tests override this to return an
|
|
69
|
+
* in-memory fake. Handlers are attached by the shared Connect AFTER this returns, so the
|
|
70
|
+
* implementation must not require them at construction time.
|
|
260
71
|
*/
|
|
261
|
-
|
|
72
|
+
protected createSocket(url: string, subprotocol: string): IOpenAIProtocolClientSocket;
|
|
262
73
|
/**
|
|
263
74
|
* @inheritdoc
|
|
264
75
|
*
|
|
265
|
-
*
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
270
|
-
* cancelled response's trailing `response.done`. No-op when idle or when the socket is gone.
|
|
76
|
+
* Grok STREAMS the input transcription — repeated `input_audio_transcription.completed`
|
|
77
|
+
* events, each carrying the full text so far — unlike OpenAI's single final. So the FIRST
|
|
78
|
+
* emission of a turn appends a fresh caption and every later one is flagged
|
|
79
|
+
* ReplacesPrevious, collapsing the stream into ONE in-place-updating user bubble. The
|
|
80
|
+
* per-turn flag resets on the next `speech_started` ({@link onSpeechStartedFrame}).
|
|
271
81
|
*/
|
|
272
|
-
|
|
82
|
+
protected onUserTranscriptFrame(transcript: string): void;
|
|
273
83
|
/**
|
|
274
|
-
*
|
|
275
|
-
* forcing a reply. Item creation is always safe mid-response, so it is sent immediately even
|
|
276
|
-
* while a reply is in flight.
|
|
277
|
-
*
|
|
278
|
-
* NOTE: role must be 'system' — the OpenAI-compatible realtime API rejects 'developer' items.
|
|
279
|
-
*/
|
|
280
|
-
SendContextNote(text: string): void;
|
|
281
|
-
/**
|
|
282
|
-
* Triggers ONE short spoken update with the given instructions. Marks the upcoming response
|
|
283
|
-
* as `'narration'` (flag consumed by the next `response.created`) so its transcripts are
|
|
284
|
-
* emitted with `Kind: 'narration'` — ephemeral by contract. Sets {@link responseActive}
|
|
285
|
-
* eagerly so a tool result landing mid-narration queues instead of colliding.
|
|
84
|
+
* @inheritdoc
|
|
286
85
|
*
|
|
287
|
-
*
|
|
288
|
-
*
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
* {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
|
|
292
|
-
*/
|
|
293
|
-
RequestSpokenUpdate(instructions: string): void;
|
|
294
|
-
/**
|
|
295
|
-
* Sends the tool result back as a `function_call_output` conversation item, then triggers a
|
|
296
|
-
* reply — immediately if the model is idle, otherwise queued until the current response
|
|
297
|
-
* (e.g. a progress narration) finishes. Without the queueing the result's `response.create`
|
|
298
|
-
* would collide with an in-flight narration and be dropped, leaving the model silent when
|
|
299
|
-
* delegated work comes back.
|
|
86
|
+
* On top of the shared websocket behavior (flush local playout on TRUE barge-in, gated
|
|
87
|
+
* interruption): a new user turn begins, so the streamed-transcription flag resets, and an
|
|
88
|
+
* interrupted response's busy flag is cleared eagerly (Grok cancels its own turn; clearing
|
|
89
|
+
* now lets a queued tool result fire without waiting on the trailing `response.done`).
|
|
300
90
|
*/
|
|
301
|
-
SendToolResult(callID: string, outputJson: string): void;
|
|
302
|
-
/**
|
|
303
|
-
* Mutes / unmutes by toggling the mic tracks' `enabled` flag: the capture pipeline stays up
|
|
304
|
-
* and streams SILENCE while muted (the provider's VAD sees a continuous stream and the
|
|
305
|
-
* un-mute is glitch-free — same policy as the other client drivers).
|
|
306
|
-
*/
|
|
307
|
-
SetMuted(muted: boolean): void;
|
|
308
|
-
/** @inheritdoc */
|
|
309
|
-
get IsBusy(): boolean;
|
|
310
91
|
/**
|
|
311
92
|
* @inheritdoc
|
|
312
93
|
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
315
|
-
*
|
|
316
|
-
*/
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
* fake (and may capture `onPcmChunk` to simulate mic frames).
|
|
330
|
-
*/
|
|
331
|
-
protected createMicCapture(micStream: MediaStream, sampleRate: number, onPcmChunk: (base64Pcm16: string) => void): Promise<IPcmMicCapture>;
|
|
332
|
-
/**
|
|
333
|
-
* Creation seam for the playout engine at the provider's fixed 24 kHz rate. Production
|
|
334
|
-
* returns the shared {@link RealtimePcmPlayback}.
|
|
335
|
-
*/
|
|
336
|
-
protected createPlayback(sampleRate: number): IRealtimePcmPlayback;
|
|
337
|
-
/**
|
|
338
|
-
* Sends the server-controlled session config (instructions + tools) as a `session.update` so
|
|
339
|
-
* the co-agent's identity and tool set apply. Skipped when the host supplied no config (e.g.
|
|
340
|
-
* it failed to parse the server payload — the host already logged that; sending an EMPTY
|
|
341
|
-
* `session.update` would be wrong).
|
|
342
|
-
*/
|
|
343
|
-
private applySessionConfig;
|
|
344
|
-
/** Streams one base64 PCM16 mic chunk as an `input_audio_buffer.append` frame. */
|
|
345
|
-
private sendMicChunk;
|
|
346
|
-
/** Surfaces a fatal socket error and marks the session unusable (obligation #6). */
|
|
347
|
-
private handleSocketError;
|
|
348
|
-
/**
|
|
349
|
-
* A socket close the CONSUMER didn't ask for is fatal: the provider hard-closes at token
|
|
350
|
-
* expiry and when it ends the session itself, so an unexpected close is how credential /
|
|
351
|
-
* session death reaches the host (obligation #6).
|
|
352
|
-
*/
|
|
353
|
-
private handleSocketClose;
|
|
354
|
-
/** Parses one raw socket payload; non-JSON frames are ignored. */
|
|
355
|
-
private handleSocketMessage;
|
|
356
|
-
/** Dispatches a typed xAI realtime server event to the appropriate behavior. */
|
|
357
|
-
private handleEvent;
|
|
358
|
-
/** Appends an assistant transcript delta, reflects `'speaking'`, and emits the delta. */
|
|
359
|
-
private onAssistantDelta;
|
|
360
|
-
/**
|
|
361
|
-
* Finalizes the assistant turn: emits the final transcript tagged with the ACTIVE response
|
|
362
|
-
* kind (the transcript-done frame arrives BEFORE `response.done`, so
|
|
363
|
-
* {@link activeResponseKind} still reflects this turn), then returns to `'listening'`. Empty
|
|
364
|
-
* turns emit nothing.
|
|
365
|
-
*/
|
|
366
|
-
private onAssistantDone;
|
|
367
|
-
/**
|
|
368
|
-
* Decodes one base64 PCM16 chunk of the agent's spoken output into the playout queue and
|
|
369
|
-
* reflects `'speaking'`. The client OWNS the audio plane on this websocket transport (unlike
|
|
370
|
-
* the WebRTC OpenAI driver where the peer connection plays the remote track), so agent audio
|
|
371
|
-
* arrives as these deltas and is scheduled by the shared playout engine.
|
|
372
|
-
*/
|
|
373
|
-
private onAudioDelta;
|
|
374
|
-
/**
|
|
375
|
-
* Emits the user's spoken-input transcription. Grok STREAMS this — repeated
|
|
376
|
-
* `input_audio_transcription.completed` events, each carrying the full text so far — unlike OpenAI's
|
|
377
|
-
* single final. So the FIRST emission of a turn appends a fresh caption and every later one is flagged
|
|
378
|
-
* ReplacesPrevious, collapsing the stream into ONE in-place-updating user bubble. The per-turn flag
|
|
379
|
-
* resets on the next `speech_started` ({@link onSpeechStarted}).
|
|
380
|
-
*/
|
|
381
|
-
private onUserTranscript;
|
|
382
|
-
/**
|
|
383
|
-
* Surfaces a completed tool call to the host. Two deliberate behaviors mirror the OpenAI
|
|
384
|
-
* driver: (1) the client silently leaves `'speaking'` (NO emission) so a host-rendered busy
|
|
385
|
-
* indicator isn't clobbered by this turn's trailing `response.done` / playback frames
|
|
386
|
-
* (obligation #1); (2) {@link responseActive} is CLEARED — the model has yielded the floor
|
|
387
|
-
* pending the result, so a queued {@link SendToolResult} can never deadlock (obligation #2).
|
|
388
|
-
*/
|
|
389
|
-
private onToolCallFrame;
|
|
390
|
-
/**
|
|
391
|
-
* The user started speaking. This is a TRUE barge-in only when it cut off active model output
|
|
392
|
-
* (a response in flight or audio audibly playing) — a normal turn while the model is idle is
|
|
393
|
-
* NOT an interruption, so the emission is gated (base-contract rule). On a true barge-in the
|
|
394
|
-
* client OWNS the audio plane, so it flushes its own playout queue NOW (obligation #3),
|
|
395
|
-
* surfaces the interruption, and returns the floor; the provider cancels its own turn and
|
|
396
|
-
* emits a terminal `response.done`.
|
|
397
|
-
*/
|
|
398
|
-
private onSpeechStarted;
|
|
399
|
-
/**
|
|
400
|
-
* Emits the completed response's usage to the host as a DELTA (the `response.done` usage
|
|
401
|
-
* payload covers exactly this response, so it is already incremental — the `OnUsage`
|
|
402
|
-
* contract's preferred shape). Frames without a usage payload emit nothing.
|
|
403
|
-
*/
|
|
404
|
-
private emitResponseUsage;
|
|
405
|
-
/** Surfaces a provider error frame (non-fatal; the session continues). */
|
|
406
|
-
private onErrorFrame;
|
|
407
|
-
/**
|
|
408
|
-
* Asks the model to speak (a tool result or typed-text reply) — immediately if it's idle,
|
|
409
|
-
* otherwise queued until the current response finishes. An immediate trigger also CONSUMES
|
|
410
|
-
* any queued trigger debt: every payload item is already in the conversation, so one
|
|
411
|
-
* `response.create` voices everything (e.g. typed text barging in over a narration that had
|
|
412
|
-
* tool results queued behind it).
|
|
413
|
-
*/
|
|
414
|
-
private requestResultResponse;
|
|
415
|
-
/** On a turn completing, fire any queued tool-result response so the answer is spoken. */
|
|
416
|
-
private flushPendingResultResponse;
|
|
417
|
-
/** Resets the per-session response state machine (used on Disconnect). */
|
|
418
|
-
private resetResponseState;
|
|
419
|
-
/** Updates the client's own state view and emits the change to the host. */
|
|
420
|
-
private setState;
|
|
421
|
-
/** JSON-serializes + sends a client event over the socket (only when open). */
|
|
422
|
-
private sendEvent;
|
|
94
|
+
* The model has taken the floor, so the user's turn is definitively over — drop the continuation
|
|
95
|
+
* anchor. Without this, a later utterance that happened to open with the same words could be
|
|
96
|
+
* merged into the previous turn instead of appending its own.
|
|
97
|
+
*/
|
|
98
|
+
protected onResponseStarted(): void;
|
|
99
|
+
protected onSpeechStartedFrame(): void;
|
|
100
|
+
/** @inheritdoc — provider-branded transport-error message (kept stable for hosts/logs). */
|
|
101
|
+
protected formatTransportError(message: string): string;
|
|
102
|
+
/** @inheritdoc — provider-branded unexpected-close message (kept stable for hosts/logs). */
|
|
103
|
+
protected get unexpectedCloseMessage(): string;
|
|
104
|
+
/** @inheritdoc — log every inbound event type; error frames include their payload. */
|
|
105
|
+
protected logInboundEvent(event: OpenAIProtocolServerEvent): void;
|
|
106
|
+
/** @inheritdoc */
|
|
107
|
+
protected onNonJsonFrame(raw: string): void;
|
|
108
|
+
/** @inheritdoc — log outbound control frames (skip the high-frequency mic audio). */
|
|
109
|
+
protected logOutboundEvent(event: OpenAIProtocolClientEvent): void;
|
|
423
110
|
}
|
|
424
111
|
/**
|
|
425
112
|
* Tree-shaking prevention: bundlers cannot see that {@link xAIRealtimeClient} is instantiated
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"xaiRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/xaiRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,
|
|
1
|
+
{"version":3,"file":"xaiRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/xaiRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAA4B,MAAM,oBAAoB,CAAC;AAE3F,OAAO,EACH,qCAAqC,EACrC,2BAA2B,EAC3B,yBAAyB,EACzB,yBAAyB,EAC5B,MAAM,iCAAiC,CAAC;AAIzC;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,+BAA+B,CAAC;AAEhE;;;;;GAKG;AACH,eAAO,MAAM,oCAAoC,uBAAuB,CAAC;AAEzE;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,QAAQ,CAAC;AAkBzC;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBACa,iBAAkB,SAAQ,qCAAqC;IACxE;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB,CAAS;IAEpC;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB,CAAM;IAEhC,qEAAqE;IACrE,cAAuB,kBAAkB,IAAI,MAAM,CAElD;IAID,2EAA2E;IAC3E,SAAS,CAAC,kBAAkB,CAAC,MAAM,EAAE,2BAA2B,GAAG,2BAA2B;IAM9F,yFAAyF;IACzF,SAAS,CAAC,iBAAiB,CAAC,OAAO,EAAE,2BAA2B,GAAG,MAAM;IAIzE;;;;;;OAMG;IACH,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,2BAA2B;IAuBrF;;;;;;;;OAQG;cACgB,qBAAqB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAkBlE;;;;;;;OAOG;IACH;;;;;;OAMG;cACgB,iBAAiB,IAAI,IAAI;cAIzB,oBAAoB,IAAI,IAAI;IAa/C,2FAA2F;cACxE,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAIhE,4FAA4F;IAC5F,cAAuB,sBAAsB,IAAI,MAAM,CAEtD;IAID,sFAAsF;cACnE,eAAe,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;IAK1E,kBAAkB;cACC,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIpD,qFAAqF;cAClE,gBAAgB,CAAC,KAAK,EAAE,yBAAyB,GAAG,IAAI;CAK9E;AAED;;;;GAIG;AACH,wBAAgB,qBAAqB,IAAI,IAAI,CAE5C"}
|