@memberjunction/ai-realtime-client 5.48.0 → 5.50.0

package/dist/generic/openAIProtocolClient.js

@@ -0,0 +1,873 @@
+import { BaseRealtimeClient } from './baseRealtimeClient.js';
+import { base64ToArrayBuffer } from '../audio/pcmUtils.js';
+import { RealtimePcmPlayback } from '../audio/pcmPlayback.js';
+import { RealtimeAudioMeter } from '../audio/audioMeter.js';
+import { createPcmMicCapture } from '../audio/micCapture.js';
+// ── Layer 1: the protocol brain ─────────────────────────────────────────────────
+/**
+ * The shared **OpenAI-protocol brain** for realtime CLIENT drivers — the transport-agnostic
+ * middle layer between {@link BaseRealtimeClient} and the per-provider drivers.
+ *
+ * Every OpenAI-protocol provider (OpenAI itself over WebRTC, xAI Grok Voice and self-hosted
+ * HuggingFace over websockets) shares the exact same event vocabulary and turn discipline, so
+ * this class owns — ONCE — the pieces that were previously cloned per driver:
+ *
+ * - **Inbound event dispatch** ({@link handleEvent}): GA + beta transcript names, input
+ *   transcription, tool calls, barge-in gating, playback-buffer events, provider error frames.
+ * - **The response state machine**: `responseActive` set on `response.created` / cleared on
+ *   `response.done`; tool-result `response.create`s queued while a response is in flight and
+ *   flushed on `response.done` so the model ALWAYS voices delegated results.
+ * - **Narration-kind tagging**: {@link RequestSpokenUpdate} marks the NEXT response as
+ *   `'narration'` so its transcripts are emitted with `Kind: 'narration'` (ephemeral).
+ * - **The outbound actions**: {@link SendText} (implies barge-in), {@link SendContextNote},
+ *   {@link SendToolResult}, {@link CancelActiveResponse}, {@link SetMuted}.
+ *
+ * Transport specifics stay in subclasses through a small set of seams: {@link canSendEvents} /
+ * {@link sendProtocolEvent} (the wire), {@link stopAudioOutput} (how already-generated speech is
+ * silenced), {@link handleAudioDeltaFrame} (websocket transports enqueue PCM; WebRTC ignores —
+ * audio rides the peer connection), and per-provider behavioral overrides
+ * ({@link onUserTranscriptFrame}, {@link onSpeechStartedFrame}, {@link releasesBusyFlagOnToolCall}).
+ */
+export class OpenAIProtocolRealtimeClient extends BaseRealtimeClient {
+    constructor() {
+        super(...arguments);
+        // ── Response state machine (shared verbatim across the protocol family) ────
+        /** Accumulates the in-flight assistant transcript across delta frames. */
+        this.pendingAssistantText = '';
+        /** True while the model has a response in flight; gates narration + queues the tool result. */
+        this.responseActive = false;
+        /** Set when a tool result is ready while a response is active; sent on the next response.done. */
+        this.pendingResultResponse = false;
+        /**
+         * Count of locally-initiated `response.create`s whose `response.created` echo has not arrived
+         * yet. While non-zero, a `response.done` belongs to an EARLIER (typically just-cancelled)
+         * response — it must not clear the busy flag, flush the queued trigger, or flip the state for
+         * the response we just started (usage is still emitted). Consumed by `response.created`.
+         */
+        this.pendingLocalResponseCreates = 0;
+        /**
+         * True while a response the provider has CONFIRMED (`response.created` seen, `response.done` not
+         * yet) is in flight. Distinct from {@link responseActive}, which is set EAGERLY before a local
+         * `response.create` is even sent: if the provider REJECTS that create (an `error` frame with no
+         * `response.created`), `responseActive` would otherwise stay stuck true forever. This flag lets
+         * {@link onErrorFrame} tell "the eager flag is a phantom for a rejected create" (no confirmed
+         * response) from "a real response is genuinely active" (e.g. a concurrent VAD turn), so it clears
+         * the phantom without disturbing a live turn. Set on `response.created`, cleared on `response.done`.
+         */
+        this.confirmedResponseActive = false;
+        /**
+         * Set by {@link RequestSpokenUpdate} just before it sends its `response.create`, and
+         * CONSUMED by the very next `response.created` frame, which stamps
+         * {@link activeResponseKind} for that turn. Narration is only requested while the model is
+         * idle (hosts gate on {@link IsBusy}), so under normal ordering the next `response.created`
+         * is ours.
+         */
+        this.pendingNarrationKind = false;
+        /**
+         * The kind of the response currently in flight. Event ordering (confirmed against the live
+         * OpenAI API): `response.created` → transcript deltas → `*_audio_transcript.done` →
+         * `response.done`. The transcript-done frame therefore arrives while the kind is still set,
+         * letting {@link onAssistantDone} classify the turn; `response.done` then resets it.
+         */
+        this.activeResponseKind = 'normal';
+        /**
+         * The client's own view of the session state — mirrors what `emitStateChange` last
+         * reported, EXCEPT after a tool call is emitted: the host typically shows its own busy
+         * state then, so the client silently leaves `'speaking'` (no emission) to preserve the
+         * host's indicator until the result reply starts (see {@link onToolCallFrame}).
+         */
+        this.currentState = 'closed';
+        /** The mic stream owned by the current connection (used by the shared {@link SetMuted}). */
+        this.micStream = null;
+    }
+    // ── Virtual per-provider behavior seams (sensible OpenAI defaults) ─────────
+    /** Debug label used in the shared console diagnostics. */
+    get providerDebugLabel() {
+        return this.constructor.name;
+    }
+    /**
+     * Handles one base64 PCM16 audio delta. Default: no-op — on the WebRTC transport the agent's
+     * audio rides the peer connection's remote track, not data-channel frames. Websocket
+     * transports override to enqueue into their local playout engine.
+     */
+    handleAudioDeltaFrame(_deltaBase64) {
+        // WebRTC: audio arrives on the remote media track, not as protocol frames.
+    }
+    /**
+     * Emits the user's spoken-input transcription. Default (OpenAI/HuggingFace): each
+     * `.completed` frame is one final caption. Providers that STREAM the completed event
+     * (Grok re-sends the full growing text each time) override to collapse the stream into a
+     * single in-place-updating bubble.
+     */
+    onUserTranscriptFrame(transcript) {
+        this.emitTranscript({ Role: 'User', Text: transcript, IsFinal: true, Kind: 'normal' });
+    }
+    /**
+     * The user started speaking. TRUE barge-in only when it cut off active model output (a
+     * response in flight or audio audibly playing) — a normal turn while the model is idle is
+     * NOT an interruption, so the emission is gated (base-contract rule). Transports that own
+     * their playback override to also flush the local playout queue.
+     */
+    onSpeechStartedFrame() {
+        if (this.responseActive || this.IsAudioPlaying) {
+            // The user took the floor. Drop the queued AUTO-trigger: the tool-result item is
+            // already in the conversation, so the user's next turn (server-VAD response) voices
+            // it contextually instead of a queued response.create stomping the fresh user turn.
+            this.pendingResultResponse = false;
+            this.emitInterruption();
+        }
+        this.setState('listening');
+    }
+    /**
+     * Whether a completed tool call CLEARS {@link responseActive}. WebRTC keeps the flag (the
+     * provider reliably follows with `response.done`); websocket transports clear it as a
+     * deadlock guard so a queued {@link SendToolResult} can never wedge if the endpoint skips
+     * the trailing frame.
+     */
+    get releasesBusyFlagOnToolCall() {
+        return false;
+    }
+    /** Hook for transports that gate on the endpoint's `session.created` frame. Default: no-op. */
+    onSessionCreatedFrame() {
+        // Only deferring transports care.
+    }
+    /** Diagnostic hook for each parsed inbound event. Default: silent. */
+    logInboundEvent(_event) {
+        // Overridden by drivers that need wire diagnostics.
+    }
+    /** Diagnostic hook for each non-JSON inbound frame. Default: silent. */
+    onNonJsonFrame(_raw) {
+        // Overridden by drivers that need wire diagnostics.
+    }
+    /** Diagnostic hook for each outbound event. Default: silent. */
+    logOutboundEvent(_event) {
+        // Overridden by drivers that need wire diagnostics.
+    }
+    // ── BaseRealtimeClient: shared outbound actions ────────────────────────────
+    /**
+     * Injects typed text as a user-role `message` conversation item, then triggers a reply
+     * through the SAME collision-safe path tool results use ({@link requestResultResponse}).
+     * No-op when the transport isn't open.
+     *
+     * **SendText implies barge-in** (base-contract rule): an active spoken response is
+     * cancelled via {@link CancelActiveResponse} before the text is injected, so the typed
+     * turn takes the floor immediately instead of waiting behind stale speech. When nothing is
+     * active the cancel is a no-op and the reply triggers immediately.
+     */
+    SendText(text) {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        this.CancelActiveResponse();
+        this.sendEvent({
+            type: 'conversation.item.create',
+            item: { type: 'message', role: 'user', content: [{ type: 'input_text', text }] },
+        });
+        this.requestResultResponse();
+    }
+    /**
+     * @inheritdoc
+     *
+     * Sends `response.cancel` (only when a response is actually in flight) and silences
+     * already-generated speech via the transport's {@link stopAudioOutput}. Resets the local
+     * response state machine (active flag, narration kind, accumulated transcript) but
+     * PRESERVES any queued tool-result trigger: delegated work is never affected by a
+     * floor-control cancel, and the queued trigger still fires on the cancelled response's
+     * trailing `response.done`. No-op when idle or when the transport is not open.
+     */
+    CancelActiveResponse() {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        if (!this.responseActive && !this.IsAudioPlaying) {
+            return; // nothing active — no-op by contract
+        }
+        if (this.responseActive) {
+            this.sendEvent({ type: 'response.cancel' });
+            this.responseActive = false;
+            this.pendingNarrationKind = false;
+            this.activeResponseKind = 'normal';
+            this.pendingAssistantText = '';
+        }
+        this.stopAudioOutput();
+        if (this.currentState === 'speaking') {
+            this.setState('listening');
+        }
+    }
+    /**
+     * Injects a system-role context item the model can draw on the next time it speaks,
+     * WITHOUT forcing a reply. Item creation is always safe mid-response.
+     *
+     * NOTE: role must be 'system' — gpt-realtime (and the compatible endpoints) reject
+     * 'developer' items ("Developer messages are only supported for quicksilver sessions").
+     */
+    SendContextNote(text) {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        this.sendEvent({
+            type: 'conversation.item.create',
+            item: { type: 'message', role: 'system', content: [{ type: 'input_text', text }] },
+        });
+    }
+    /**
+     * Triggers ONE short spoken update with the given instructions. Marks the upcoming
+     * response as `'narration'` (flag consumed by the next `response.created`) so its
+     * transcripts are emitted with `Kind: 'narration'` — ephemeral by contract. Sets
+     * {@link responseActive} eagerly so a tool result landing mid-narration queues instead of
+     * colliding.
+     *
+     * **Skips when busy** (base-contract collision rule — drivers MUST queue or skip): a
+     * `response.create` sent while a response is in flight would be rejected/garbled by the
+     * provider, and narration is disposable by contract, so the update is dropped with a debug
+     * log rather than queued to come out late and stale. Hosts SHOULD still gate on
+     * {@link IsBusy} / {@link IsAudioPlaying} for timing quality.
+     */
+    RequestSpokenUpdate(instructions) {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        if (this.responseActive) {
+            console.debug(`[${this.providerDebugLabel}] RequestSpokenUpdate skipped — a response is already in flight (narration is disposable).`);
+            return;
+        }
+        this.responseActive = true;
+        this.pendingNarrationKind = true;
+        this.pendingLocalResponseCreates++;
+        this.sendEvent({ type: 'response.create', response: { instructions } });
+    }
+    /**
+     * Sends the tool result back as a `function_call_output` conversation item, then triggers
+     * a reply — immediately if the model is idle, otherwise queued until the current response
+     * (e.g. a progress narration) finishes. Without the queueing the result's `response.create`
+     * would collide with an in-flight narration and be dropped, leaving the model silent when
+     * delegated work comes back.
+     */
+    SendToolResult(callID, outputJson) {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        this.sendEvent({
+            type: 'conversation.item.create',
+            item: { type: 'function_call_output', call_id: callID, output: outputJson },
+        });
+        this.requestResultResponse();
+    }
+    /**
+     * Mutes / unmutes by toggling the mic tracks' `enabled` flag: the transport stays up and
+     * streams SILENCE while muted (the provider's VAD sees a continuous stream and the un-mute
+     * is glitch-free — the same policy across the client driver family).
+     */
+    SetMuted(muted) {
+        for (const track of this.micStream?.getAudioTracks() ?? []) {
+            track.enabled = !muted;
+        }
+    }
+    /** @inheritdoc */
+    get IsBusy() {
+        return this.responseActive;
+    }
+    // ── Inbound event translation (the shared dispatcher) ──────────────────────
+    /**
+     * Parses one raw inbound payload and dispatches it. Non-JSON frames and non-object JSON
+     * values are ignored (with the {@link onNonJsonFrame} diagnostic hook for the former).
+     */
+    handleProtocolMessage(raw) {
+        let event;
+        try {
+            event = JSON.parse(raw);
+        }
+        catch {
+            this.onNonJsonFrame(raw);
+            return;
+        }
+        if (event === null || typeof event !== 'object') {
+            return; // valid JSON but not an event object (e.g. "null", a number) — ignore
+        }
+        this.logInboundEvent(event);
+        this.handleEvent(event);
+    }
+    /** Dispatches a typed OpenAI-protocol server event to the appropriate behavior. */
+    handleEvent(event) {
+        switch (event.type) {
+            case 'session.created':
+                this.onSessionCreatedFrame();
+                break;
+            case 'response.output_audio_transcript.delta':
+            case 'response.audio_transcript.delta':
+                this.onAssistantDelta(event.delta);
+                break;
+            case 'response.output_audio_transcript.done':
+            case 'response.audio_transcript.done':
+                this.onAssistantDone(event.transcript);
+                break;
+            case 'response.audio.delta':
+            case 'response.output_audio.delta':
+                this.onAudioDelta(event.delta);
+                break;
+            case 'conversation.item.input_audio_transcription.completed':
+                this.onUserTranscriptFrame(event.transcript);
+                break;
+            case 'response.function_call_arguments.done':
+                this.onToolCallFrame(event);
+                break;
+            case 'input_audio_buffer.speech_started':
+                this.onSpeechStartedFrame();
+                break;
+            case 'response.created': {
+                this.responseActive = true;
+                this.confirmedResponseActive = true; // a real response is now in flight
+                this.onResponseStarted();
+                // Only a LOCALLY-initiated create (counter-tracked) can be our narration. A VAD /
+                // unsolicited response.created arrives with the counter already at 0 — it must NOT
+                // consume the pending narration kind, or it would mistag a genuine user turn as
+                // ephemeral narration (and lose the flag meant for our own create).
+                const wasLocalCreate = this.pendingLocalResponseCreates > 0;
+                if (wasLocalCreate) {
+                    this.pendingLocalResponseCreates--;
+                }
+                // Stamp the kind of THIS response: 'narration' only when the flag was set by
+                // RequestSpokenUpdate immediately before ITS OWN local response.create (consumed here).
+                this.activeResponseKind = (wasLocalCreate && this.pendingNarrationKind) ? 'narration' : 'normal';
+                if (wasLocalCreate) {
+                    this.pendingNarrationKind = false;
+                }
+                break;
+            }
+            case 'output_audio_buffer.started':
+                this.onOutputAudioBufferStarted();
+                break;
+            case 'output_audio_buffer.stopped':
+            case 'output_audio_buffer.cleared':
+                this.onOutputAudioBufferStopped();
+                break;
+            case 'response.done':
+                // The confirmed response that was in flight has ended (whether this is its real done
+                // or the trailing done of a since-cancelled one) — no response.created is outstanding.
+                this.confirmedResponseActive = false;
+                // A STALE done (the trailing frame of a response we just cancelled while our own
+                // replacement response.create is in flight) must not release the lock, flush the
+                // queue, or flip the state out from under the response we just started — the real
+                // done for OUR response handles all of that. Usage is still emitted (the cancelled
+                // response consumed tokens).
+                if (this.pendingLocalResponseCreates > 0) {
+                    this.emitResponseUsage(event);
+                    break;
+                }
+                // A turn finished — release the lock and speak any queued tool result so the
+                // model always voices the answer when delegated work comes back. The
+                // transcript-done frame for this turn has already arrived (it precedes
+                // response.done), so it's safe to reset the response kind here.
+                this.responseActive = false;
+                this.activeResponseKind = 'normal';
+                this.emitResponseUsage(event);
+                this.flushPendingResultResponse();
+                if (this.currentState === 'speaking') {
+                    this.setState('listening');
+                }
+                break;
+            case 'error':
+                this.onErrorFrame(event);
+                break;
+            default:
+                // Unhandled event types are expected (the provider emits many); no-op.
+                break;
+        }
+    }
+    /**
+     * Called when the model takes the floor (`response.created`), i.e. the user's turn is definitively
+     * over. Drivers that track per-user-turn transcription state override this to reset it. No-op here.
+     */
+    onResponseStarted() {
+        // Only streamed-transcription drivers carry per-user-turn state worth clearing.
+    }
+    /** WebRTC-only playback-buffer hooks; websocket transports never receive these frames. */
+    onOutputAudioBufferStarted() {
+        // Only the WebRTC transport tracks provider-managed playback.
+    }
+    onOutputAudioBufferStopped() {
+        // Only the WebRTC transport tracks provider-managed playback.
+    }
+    /** Reflects `'speaking'` on the first audio delta, then hands the chunk to the transport. */
+    onAudioDelta(deltaBase64) {
+        if (!deltaBase64) {
+            return;
+        }
+        if (this.currentState !== 'speaking') {
+            this.setState('speaking');
+        }
+        this.handleAudioDeltaFrame(deltaBase64);
+    }
+    /** Appends an assistant transcript delta, reflects `'speaking'`, and emits the delta. */
+    onAssistantDelta(delta) {
+        if (!delta) {
+            return;
+        }
+        if (this.currentState !== 'speaking') {
+            this.setState('speaking');
+        }
+        this.pendingAssistantText += delta;
+        this.emitTranscript({ Role: 'Assistant', Text: delta, IsFinal: false, Kind: this.activeResponseKind });
+    }
+    /**
+     * Finalizes the assistant turn: emits the final transcript tagged with the ACTIVE response
+     * kind (the transcript-done frame arrives BEFORE `response.done`, so
+     * {@link activeResponseKind} still reflects this turn), then returns to `'listening'`.
+     * Empty turns emit nothing.
+     */
+    onAssistantDone(transcript) {
+        const finalText = transcript || this.pendingAssistantText;
+        this.pendingAssistantText = '';
+        if (finalText.trim().length > 0) {
+            this.emitTranscript({ Role: 'Assistant', Text: finalText, IsFinal: true, Kind: this.activeResponseKind });
+        }
+        if (this.currentState === 'speaking') {
+            this.setState('listening');
+        }
+    }
+    /**
+     * Surfaces a completed tool call to the host. The client silently leaves the `'speaking'`
+     * state (NO emission) so a host-rendered busy indicator (e.g. "thinking") isn't clobbered
+     * by this turn's trailing `response.done` / playback-stopped frames. Websocket transports
+     * additionally clear the busy flag ({@link releasesBusyFlagOnToolCall}) as a deadlock guard.
+     */
+    onToolCallFrame(call) {
+        if (this.currentState === 'speaking') {
+            this.currentState = 'connected';
+        }
+        if (this.releasesBusyFlagOnToolCall) {
+            this.responseActive = false;
+        }
+        this.emitToolCall({ CallID: call.call_id, ToolName: call.name, ArgumentsJson: call.arguments });
+    }
+    /**
+     * Emits the completed response's usage to the host as a DELTA (the `response.done` usage
+     * payload covers exactly this response, so it is already incremental — the `OnUsage`
+     * contract's preferred shape). Frames without a usage payload emit nothing.
+     */
+    emitResponseUsage(event) {
+        const usage = event.response?.usage;
+        if (!usage) {
+            return;
+        }
+        this.emitUsage({
+            InputTokens: typeof usage.input_tokens === 'number' ? usage.input_tokens : undefined,
+            OutputTokens: typeof usage.output_tokens === 'number' ? usage.output_tokens : undefined,
+            Raw: usage,
+        });
+    }
+    /**
+     * Surfaces a provider error frame (non-fatal; the session continues).
+     *
+     * SELF-HEAL for the stale-response counter (see {@link pendingLocalResponseCreates}): a
+     * `response.create` the provider REJECTS (overlap, invalid params) produces an `error` frame
+     * and **no** `response.created`, so the counter would otherwise never decrement and every
+     * future `response.done` would take the stale branch — wedging the client (`IsBusy` stuck true,
+     * narration + tool results silently dropped). Decrementing here (floored at 0) reverts to the
+     * pre-counter self-healing behavior for that turn rather than leaving the session dead. An
+     * error unrelated to a local create at worst under-protects one `done` (the old default), never
+     * wedges.
+     */
+    onErrorFrame(event) {
+        if (this.pendingLocalResponseCreates > 0) {
+            this.pendingLocalResponseCreates--;
+            // The counter dropping here means this error plausibly REJECTED one of our local
+            // response.creates (rejected creates emit an error and NO response.created). The rejected
+            // create's narration kind belongs to THAT create — it will never arrive, so drop the flag
+            // UNCONDITIONALLY, or it would arm the NEXT local create (e.g. a delegated tool-result
+            // reply) and mistag that durable turn as ephemeral narration. This must NOT be gated on
+            // confirmedResponseActive: a narration create can be rejected while a since-cancelled
+            // response is still draining (confirmedResponseActive still true), and the flag must die
+            // regardless.
+            this.pendingNarrationKind = false;
+            // The EAGER responseActive / 'speaking' state, by contrast, is only a phantom to clear
+            // when NO provider-CONFIRMED response is in flight; a genuinely-active response (concurrent
+            // VAD turn, or a cancelled one still draining) owns that state and its response.done clears
+            // it. Clearing it here would cut a live turn.
+            if (!this.confirmedResponseActive) {
+                this.responseActive = false;
+                if (this.currentState === 'speaking') {
+                    this.setState('listening');
+                }
+            }
+        }
+        this.emitError({
+            Message: event.error?.message ?? 'Unknown provider error',
+            Code: event.error?.code,
+            Fatal: false,
+        });
+    }
+    // ── Response state machine helpers ─────────────────────────────────────────
+    /**
+     * Asks the model to speak (a tool result or typed-text reply) — immediately if it's idle,
+     * otherwise queued until the current response finishes. An immediate trigger also
+     * CONSUMES any queued trigger debt: every payload item is already in the conversation, so
+     * one `response.create` voices everything (e.g. typed text barging in over a narration
+     * that had tool results queued behind it).
+     */
+    requestResultResponse() {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        if (this.responseActive) {
+            this.pendingResultResponse = true;
+            return;
+        }
+        this.pendingResultResponse = false;
+        this.responseActive = true;
+        this.pendingLocalResponseCreates++;
+        this.sendEvent({ type: 'response.create' });
+        this.setState('speaking');
+    }
+    /** On a turn completing, fire any queued tool-result response so the answer is spoken. */
+    flushPendingResultResponse() {
+        if (!this.pendingResultResponse || !this.canSendEvents()) {
+            return;
+        }
+        this.pendingResultResponse = false;
+        this.responseActive = true;
+        this.pendingLocalResponseCreates++;
+        this.sendEvent({ type: 'response.create' });
+        this.setState('speaking');
+    }
+    /** Resets the per-session response state machine (used on Disconnect). */
+    resetResponseState() {
+        this.pendingAssistantText = '';
+        this.responseActive = false;
+        this.confirmedResponseActive = false;
+        this.pendingResultResponse = false;
+        this.pendingLocalResponseCreates = 0;
+        this.pendingNarrationKind = false;
+        this.activeResponseKind = 'normal';
+    }
+    // ── Shared helpers ─────────────────────────────────────────────────────────
+    /** Updates the client's own state view and emits the change to the host. */
+    setState(state) {
+        this.currentState = state;
+        this.emitStateChange(state);
+    }
+    /** Logs (diagnostic hook) and delivers one client event when the transport is open. */
+    sendEvent(event) {
+        if (!this.canSendEvents()) {
+            return;
+        }
+        this.logOutboundEvent(event);
+        this.sendProtocolEvent(event);
+    }
+}
+/**
+ * The shared **websocket + client-owned-PCM transport** for OpenAI-protocol client drivers
+ * (xAI Grok Voice, self-hosted HuggingFace). Extends the protocol brain with everything the
+ * websocket drivers previously each owned:
+ *
+ * - The socket lifecycle: wire-up, open/created gating (profile-driven via
+ *   {@link waitsForSessionCreated}), fatal error / unexpected-close handling.
+ * - The PCM audio plane: shared mic-capture worklet streaming `input_audio_buffer.append`
+ *   frames up, shared playout engine scheduling base64 deltas down, audio meters both ways.
+ * - The transport implementations of the brain's seams: {@link sendProtocolEvent} /
+ *   {@link canSendEvents} over the socket, {@link stopAudioOutput} → local playout flush,
+ *   {@link handleAudioDeltaFrame} → playout enqueue, `IsAudioPlaying` → the playout clock.
+ *
+ * Concrete drivers supply only: the socket target ({@link openProviderSocket}), the session
+ * object + sample rate resolution from the server pact ({@link resolveSessionObject} /
+ * {@link resolveSampleRate}), and any provider behavior overrides (streamed user transcripts,
+ * close semantics).
+ */
+export class OpenAIProtocolWebSocketRealtimeClient extends OpenAIProtocolRealtimeClient {
+    constructor() {
+        super(...arguments);
+        /** The live socket (null when disconnected). Protected so test subclasses can inspect. */
+        this.socket = null;
+        /** The local playout engine (client-owned audio plane). */
+        this.playback = null;
+        /** The mic-capture pipeline streaming PCM16 up to the provider. */
+        this.micCapture = null;
+        /**
+         * The wire-shaped session object applied via `session.update` at the readiness boundary.
+         * Protected so test subclasses can seed it without a full Connect.
+         */
+        this.sessionObject = {};
+        /** True once Disconnect ran — an expected socket close must not surface as fatal. */
+        this.closedByConsumer = false;
+        /** True while the socket is OPEN — gates every send so a CONNECTING socket never throws. */
+        this.socketOpen = false;
+        /** Resolver for the in-flight Connect's `session.created` wait (deferring providers only). */
+        this.sessionCreatedResolver = null;
+        /** Rejects the in-flight Connect (open OR created phase); null outside Connect. */
+        this.failPendingConnect = null;
+    }
+    /** Extracts the wire-shaped `session` object from the server pact. Default: the pact itself. */
+    resolveSessionObject(config) {
+        return config.SessionConfig ?? {};
+    }
+    /**
+     * Whether the readiness boundary waits for the endpoint's `session.created` frame before
+     * applying the session config (HuggingFace) or applies it on socket open (xAI — the
+     * protocol has no separate readiness ack there).
+     */
+    get waitsForSessionCreated() {
+        return false;
+    }
+    /**
+     * Connect-phase deadline in milliseconds, covering socket-open AND (when the provider
+     * defers) the `session.created` wait. On expiry `Connect` rejects with a fatal error
+     * instead of hanging forever against a silent endpoint. Override per driver/deployment.
+     */
+    get connectTimeoutMs() {
+        return 15_000;
+    }
+    // ── Connection lifecycle ────────────────────────────────────────────────────
+    /**
+     * Opens the websocket, gates on open (and `session.created` when the provider defers),
+     * applies the server-authored session config as the FIRST frame (prompt + tool authority
+     * stay server-side), builds the PCM audio plane at the provider's rate, and reports
+     * `'listening'` only after all of that (obligation #7).
+     */
+    async Connect(config, micStream) {
+        this.sessionObject = this.resolveSessionObject(config);
+        this.micStream = micStream;
+        this.closedByConsumer = false;
+        this.setState('connecting');
+        let openSocket = null;
+        const opened = new Promise((resolve) => {
+            openSocket = resolve;
+        });
+        const created = this.waitsForSessionCreated
+            ? new Promise((resolve) => {
+                this.sessionCreatedResolver = resolve;
+            })
+            : null;
+        // ONE failure channel covers BOTH connect phases (open + created): socket error/close,
+        // a consumer Disconnect, and the connect deadline all reject the in-flight Connect —
+        // no phase can hang an awaited Connect() forever against a silent endpoint.
+        const failure = new Promise((_, reject) => {
+            this.failPendingConnect = reject;
+        });
+        failure.catch(() => undefined); // consumed via Promise.race; guard stray-rejection noise
+        const deadline = setTimeout(() => this.failPendingConnect?.(new Error(`${this.providerDebugLabel} connect timed out after ${this.connectTimeoutMs}ms`)), this.connectTimeoutMs);
+        // Node parity with the server readiness timer — never hold the event loop for the deadline.
+        deadline.unref?.();
+        // S2: build the socket + wire handlers INSIDE the try so a synchronous throw from
+        // openProviderSocket (no global WebSocket, malformed URL) still clears the deadline timer
+        // and nulls failPendingConnect via the finally.
+        try {
+            this.socketOpen = false; // S3: never inherit a prior connect's open state on reuse
+            const socket = this.openProviderSocket(config);
+            this.socket = socket;
+            // S3: identity-guard every handler — a PREVIOUS socket's late onerror/onclose (they
+            // fire asynchronously after a Disconnect+reconnect on a reused instance) must not
+            // corrupt the NEW socket's state or drive the fresh session to error/closed.
+            socket.onopen = () => {
+                if (this.socket !== socket) {
+                    return;
+                }
+                this.socketOpen = true;
+                openSocket?.();
+            };
+            socket.onmessage = (data) => {
+                if (this.socket !== socket) {
+                    return;
+                }
+                this.handleProtocolMessage(data);
+            };
+            socket.onerror = (message) => {
+                if (this.socket !== socket) {
+                    return;
+                }
+                this.socketOpen = false;
+                this.failPendingConnect?.(new Error(message));
+                this.handleSocketError(message);
+            };
+            socket.onclose = () => {
+                if (this.socket !== socket) {
+                    return;
+                }
+                this.socketOpen = false;
+                this.failPendingConnect?.(new Error(`${this.providerDebugLabel} socket closed during connect`));
+                this.handleSocketClose();
+            };
+            await Promise.race([opened, failure]);
+            this.setState('connected');
+            if (created) {
+                await Promise.race([created, failure]);
+            }
+        }
+        catch (error) {
+            // Deadline expiry reaches the host ONLY through this rejection — surface it as a
+            // fatal error + error state (socket error/close paths already emitted their own).
+            const message = error instanceof Error ? error.message : String(error);
+            if (message.includes('timed out')) {
+                this.emitError({ Message: message, Fatal: true });
+                this.setState('error');
+                try {
+                    this.socket?.close();
+                }
+                catch { /* already closing */ }
+            }
+            throw error;
+        }
+        finally {
+            clearTimeout(deadline);
+            this.failPendingConnect = null;
+            this.sessionCreatedResolver = null;
+        }
+        // The server-authored session config (the SessionConfig pact) is applied as the FIRST
+        // frame — prompt and tool authority stay server-side (obligation #8).
+        this.applySessionConfig();
+        const sampleRate = this.resolveSampleRate(config);
+        this.playback = this.createPlayback(sampleRate);
+        this.micCapture = await this.createMicCapture(micStream, sampleRate, (base64Pcm16) => this.sendMicChunk(base64Pcm16));
+        // Audio-activity capability (base obligation #9): agent side taps the playout engine's
+        // master gain; user side meters the mic stream. Null-safe — test fakes / no-WebAudio
+        // environments simply leave the session un-metered.
+        this.attachOutputAudioMeter(this.playback?.CreateMeter?.() ?? null);
+        this.attachInputAudioMeter(RealtimeAudioMeter.ForMicStream(micStream));
+        this.setState('listening');
+    }
+    /**
+     * Tears down the socket, mic capture, mic tracks, and playout engine, resets the response
+     * state machine, and emits a final `'closed'` (unless already `'error'`). Safe to call
+     * more than once.
+     */
+    async Disconnect() {
+        this.closedByConsumer = true;
+        // Release an in-flight Connect (open or created phase) so the awaited promise cannot
+        // outlive the session the consumer just tore down.
+        this.failPendingConnect?.(new Error(`${this.providerDebugLabel} disconnected during connect`));
+        this.failPendingConnect = null;
+        this.sessionCreatedResolver = null;
+        this.socketOpen = false;
+        this.closeAudioMeters();
+        this.micStream?.getTracks().forEach((track) => track.stop());
+        this.micStream = null;
+        this.micCapture?.Stop();
+        this.micCapture = null;
+        this.playback?.Close();
+        this.playback = null;
+        if (this.socket) {
+            try {
+                this.socket.close();
+            }
+            catch {
+                /* already closing */
+            }
+            this.socket = null;
+        }
+        this.sessionObject = {};
+        this.resetResponseState();
+        if (this.currentState !== 'error') {
+            this.setState('closed');
+        }
+    }
+    // ── Brain seam implementations ─────────────────────────────────────────────
+    /** @inheritdoc — requires an OPEN socket (a CONNECTING raw WebSocket throws on send). */
+    canSendEvents() {
+        return this.socket !== null && this.socketOpen;
+    }
+    /** @inheritdoc */
+    sendProtocolEvent(event) {
+        this.socket?.send(JSON.stringify(event));
+    }
+    /** @inheritdoc — the client OWNS the audio plane, so a cancel flushes the local queue. */
+    stopAudioOutput() {
+        this.playback?.Flush();
+    }
+    /** @inheritdoc — enqueues one base64 PCM16 chunk into the local playout engine. */
+    handleAudioDeltaFrame(deltaBase64) {
+        this.playback?.Enqueue(base64ToArrayBuffer(deltaBase64));
+    }
+    /**
+     * @inheritdoc
+     *
+     * Computed directly from the playout engine's playhead clock — this client OWNS the output
+     * buffer, so "audibly playing" is precisely "scheduled audio extends beyond the audio
+     * context's current time".
+     */
+    get IsAudioPlaying() {
+        return this.playback?.IsPlaying ?? false;
+    }
+    /**
+     * @inheritdoc — websocket transports flush their local playout on TRUE barge-in
+     * (obligation #3); the provider cancels its own turn and emits a terminal `response.done`.
+     */
+    onSpeechStartedFrame() {
+        if (this.responseActive || this.IsAudioPlaying) {
+            this.playback?.Flush();
+            // Floor to the user — drop the queued auto-trigger (see the brain's docstring).
+            this.pendingResultResponse = false;
+            this.emitInterruption();
+        }
+        this.setState('listening');
+    }
+    /** @inheritdoc — deadlock guard: compat endpoints may skip `response.done` after a tool call. */
+    get releasesBusyFlagOnToolCall() {
+        return true;
+    }
+    /** @inheritdoc — releases a deferring Connect's `session.created` gate. */
+    onSessionCreatedFrame() {
+        this.sessionCreatedResolver?.();
+        this.sessionCreatedResolver = null;
+    }
+    // ── Transport internals ─────────────────────────────────────────────────────
+    /**
+     * Sends the server-controlled session config (instructions + tools) as a `session.update`
+     * so the co-agent's identity and tool set apply. Skipped when the pact carried no config
+     * (e.g. the host failed to parse the server payload — it already logged that; sending an
+     * EMPTY `session.update` would be wrong).
+     */
+    applySessionConfig() {
+        if (Object.keys(this.sessionObject).length === 0) {
+            return;
+        }
+        this.sendEvent({ type: 'session.update', session: this.sessionObject });
+    }
+    /** Streams one base64 PCM16 mic chunk as an `input_audio_buffer.append` frame. */
+    sendMicChunk(base64Pcm16) {
+        if (this.socket) {
+            this.sendEvent({ type: 'input_audio_buffer.append', audio: base64Pcm16 });
+        }
+    }
+    /** Formats a socket-level error message for the host. Providers brand this (see xAI). */
+    formatTransportError(message) {
+        return message;
+    }
+    /** The fatal-error message surfaced when the socket closes unexpectedly. Providers brand this. */
+    get unexpectedCloseMessage() {
+        return `${this.providerDebugLabel} connection closed unexpectedly`;
+    }
+    /** Surfaces a fatal socket error and marks the session unusable (obligation #6). */
+    handleSocketError(message) {
+        if (this.currentState === 'error' || this.currentState === 'closed') {
+            return;
+        }
+        this.emitError({ Message: this.formatTransportError(message), Fatal: true });
+        this.setState('error');
+    }
+    /**
+     * A socket close the CONSUMER didn't ask for. Default: FATAL — providers hard-close at
+     * token expiry and when they end the session themselves, so an unexpected close is how
+     * credential / session death reaches the host (obligation #6). Providers with benign
+     * close semantics (a self-hosted proxy hop) override.
+     */
+    handleSocketClose() {
+        if (this.closedByConsumer || this.currentState === 'error' || this.currentState === 'closed') {
+            return;
+        }
+        this.emitError({ Message: this.unexpectedCloseMessage, Fatal: true });
+        this.setState('error');
+    }
+    // ── Overridable creation seams (tests inject fakes — no audio stack) ───────
+    /**
+     * Creation seam for the mic-capture pipeline at the provider's rate. Production delegates
+     * to the shared {@link createPcmMicCapture}; unit tests override this with a no-op fake
+     * (and may capture `onPcmChunk` to simulate mic frames).
+     */
+    async createMicCapture(micStream, sampleRate, onPcmChunk) {
+        return createPcmMicCapture(micStream, sampleRate, onPcmChunk);
+    }
+    /** Creation seam for the playout engine. Production returns the shared {@link RealtimePcmPlayback}. */
+    createPlayback(sampleRate) {
+        return new RealtimePcmPlayback(sampleRate);
+    }
+}
+//# sourceMappingURL=openAIProtocolClient.js.map