@memberjunction/ai-realtime-client 5.48.0 → 5.49.0

package/README.md

@@ -163,3 +163,14 @@ If you write a new driver, follow the same shape: every wire/hardware boundary b
 - `@memberjunction/ai-agents` — `RealtimeSessionRunner`, `RealtimeToolBroker`, `RealtimeClientSessionService`
 - `@memberjunction/ng-conversations` — the Angular host (overlay, channels, session review)
 - `@memberjunction/ng-whiteboard` — the generic whiteboard the Whiteboard channel surfaces
+
+## OpenAI-protocol client family architecture
+
+The OpenAI-protocol drivers share two stacked layers in `generic/openAIProtocolClient.ts`:
+
+- **`OpenAIProtocolRealtimeClient`** — the transport-agnostic protocol brain: inbound event dispatch (GA + beta frame names), the response state machine, narration-kind tagging, tool-result queueing, and the outbound actions (`SendText`, `SendToolResult`, `SendContextNote`, `RequestSpokenUpdate`, `CancelActiveResponse`, `SetMuted`).
+- **`OpenAIProtocolWebSocketRealtimeClient`** — the websocket + client-owned PCM transport: socket lifecycle with a **connect-phase deadline** (`connectTimeoutMs`, default 15s, covering open AND the optional `session.created` wait — socket death, consumer `Disconnect`, and the deadline all reject the awaited `Connect`), the shared mic-capture/playout plumbing, and open-gated sends.
+
+`OpenAIRealtimeClient` (WebRTC) extends the brain directly; `xAIRealtimeClient` and `HuggingFaceRealtimeClient` are thin subclasses of the websocket layer.
+
+**Turn-discipline guarantees** (all drivers): a cancelled turn's trailing `response.done` cannot release the busy lock out from under a locally-initiated replacement response (stale-done protection); a TRUE user barge-in drops the queued tool-result auto-trigger — the result item is already in the conversation and the user's next turn voices it, so the model never speaks over a user who just took the floor.

package/dist/drivers/huggingFaceRealtimeClient.d.ts

@@ -1,48 +1,11 @@
-import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
-import { BaseRealtimeClient } from '../generic/baseRealtimeClient.js';
-import { IRealtimePcmPlayback } from '../audio/pcmPlayback.js';
-import { IPcmMicCapture } from '../audio/micCapture.js';
+import { ClientRealtimeSessionConfig, JSONObject } from '@memberjunction/ai';
+import { OpenAIProtocolWebSocketRealtimeClient, IOpenAIProtocolClientSocket } from '../generic/openAIProtocolClient.js';
 /**
  * Default PCM16 sample rate (mono) both directions, used when the server pact omits `sampleRate`.
  * HuggingFace's speech-to-speech cascade is natively **16 kHz**, so capture AND playout default to that
- * (a wrong rate pitch/speed-distorts audio). Matches the server driver's {@link HUGGINGFACE_DEFAULT_PCM_SAMPLE_RATE}.
+ * (a wrong rate pitch/speed-distorts audio). Matches the server driver's `HUGGINGFACE_DEFAULT_PCM_SAMPLE_RATE`.
  */
 export declare const HUGGINGFACE_DEFAULT_PCM_SAMPLE_RATE = 16000;
-/** A parsed inbound frame from the OpenAI-Realtime-compatible endpoint (discriminated by `type`). */
-export interface HuggingFaceClientServerEvent {
-    type?: string;
-    /** `response.output_audio.delta` / `*_audio_transcript.delta` / input-transcription delta. */
-    delta?: string;
-    /** `*_audio_transcript.done` — the finalized assistant transcript. */
-    transcript?: string;
-    /** `response.function_call_arguments.done` — the tool name. */
-    name?: string;
-    /** `response.function_call_arguments.done` — correlation id the result must echo. */
-    call_id?: string;
-    /** `response.function_call_arguments.done` — JSON-encoded tool arguments (a STRING). */
-    arguments?: string;
-    /** `response.done` — usage payload for the completed response. */
-    response?: {
-        usage?: {
-            input_tokens?: number;
-            output_tokens?: number;
-        };
-    };
-    /** `error` — the provider error payload. */
-    error?: {
-        message?: string;
-        code?: string;
-    };
-}
-/** The minimal websocket surface this client depends on (mirrors the other websocket drivers). */
-export interface IHuggingFaceClientSocket {
-    onopen: (() => void) | null;
-    onmessage: ((data: string) => void) | null;
-    onerror: ((message: string) => void) | null;
-    onclose: (() => void) | null;
-    send(data: string): void;
-    close(): void;
-}
 /**
  * HuggingFace speech-to-speech implementation of {@link BaseRealtimeClient}: a **browser-direct**
  * websocket that speaks the OpenAI-Realtime wire protocol over the shared PCM audio plane.
@@ -54,118 +17,34 @@ export interface IHuggingFaceClientSocket {
  * — the credential IS the URL): the browser connects to MJAPI's realtime proxy, which tunnels
  * transparently to the internal self-hosted endpoint. The client never learns the internal endpoint.
  *
- * OpenAI ordering: the session config is applied via `session.update` only AFTER the endpoint's
- * `session.created` frame (sending it earlier is dropped), and `'listening'` is reported at that point
- * (obligation #7). Because the audio plane is a raw websocket (not WebRTC), the client OWNS playback —
- * `IsAudioPlaying` reflects the local playout clock, and barge-in/cancel flush it locally.
+ * Because HuggingFace speaks the OpenAI wire protocol over a websocket with a client-owned PCM
+ * plane, nearly everything lives in the shared layers (protocol brain + websocket transport in
+ * {@link OpenAIProtocolWebSocketRealtimeClient}). This class supplies only the HuggingFace
+ * specifics: the proxy-URL connect, the `{ session, sampleRate }` server-pact parsing, the
+ * `session.created` readiness gate (OpenAI ordering — a `session.update` sent earlier is
+ * dropped), and the benign close semantics of a self-hosted proxy hop.
  */
-export declare class HuggingFaceRealtimeClient extends BaseRealtimeClient {
-    protected socket: IHuggingFaceClientSocket | null;
-    private playback;
-    private micCapture;
-    private micStream;
-    /** The OpenAI-Realtime `session` object applied via `session.update` on `session.created`. */
-    private sessionObject;
-    /** Resolver for the in-flight Connect's `session.created` wait (null outside Connect). */
-    private onSessionCreated;
-    private pendingAssistantText;
-    private responseActive;
-    private pendingResultResponse;
-    private pendingNarrationKind;
-    private activeResponseKind;
-    private currentState;
-    private closedByConsumer;
-    /**
-     * Opens the browser-direct websocket to the proxy URL (`config.EphemeralToken`), waits for the
-     * endpoint's `session.created`, applies the server-built session config, builds the PCM audio plane
-     * at the pact's sample rate, and reports `'listening'`.
-     */
-    Connect(config: ClientRealtimeSessionConfig, micStream: MediaStream): Promise<void>;
-    /** Tears down the audio plane + socket and resets the response state machine. */
-    Disconnect(): Promise<void>;
-    /**
-     * Injects typed text as a user-role message, then triggers a reply through the same collision-safe
-     * path tool results use. SendText implies barge-in — an active spoken response is cancelled first.
-     */
-    SendText(text: string): void;
-    /**
-     * Cancels the model's in-flight response (only when one is active) and flushes local playback so
-     * already-buffered speech stops immediately. Preserves any queued tool-result trigger. No-op when idle.
-     */
-    CancelActiveResponse(): void;
-    /** Injects a system-role context item the model can draw on next time it speaks, WITHOUT a reply. */
-    SendContextNote(text: string): void;
-    /**
-     * Triggers ONE short spoken update; marks the upcoming response `'narration'` so its transcripts are
-     * ephemeral. Skipped while a response is in flight (narration is disposable by contract).
-     */
-    RequestSpokenUpdate(instructions: string): void;
+export declare class HuggingFaceRealtimeClient extends OpenAIProtocolWebSocketRealtimeClient {
+    /** @inheritdoc — used in the shared diagnostics + close messages. */
+    protected get providerDebugLabel(): string;
+    /** @inheritdoc — the proxy URL IS the credential (one-time ticket in the query string). */
+    protected openProviderSocket(config: ClientRealtimeSessionConfig): IOpenAIProtocolClientSocket;
+    /** @inheritdoc — the endpoint drops `session.update` until `session.created` confirms the session. */
+    protected get waitsForSessionCreated(): boolean;
+    /** @inheritdoc — extracts the wire-shaped `session` object from the `{ session, sampleRate }` pact. */
+    protected resolveSessionObject(config: ClientRealtimeSessionConfig): JSONObject;
+    /** @inheritdoc — the pact's `sampleRate`, defaulting to the HF-native 16 kHz. */
+    protected resolveSampleRate(config: ClientRealtimeSessionConfig): number;
     /**
-     * Sends the tool result as a `function_call_output` item, then triggers a reply — immediately if idle,
-     * otherwise queued until the current response finishes (so the model always voices delegated results).
+     * @inheritdoc
+     *
+     * A close the consumer didn't ask for is reported as a terminal `'closed'` — NOT the fatal
+     * error the cloud providers surface: the self-hosted proxy hop closes benignly (ticket
+     * expiry, upstream restart), and the host treats `'closed'` as the session-over signal.
      */
-    SendToolResult(callID: string, outputJson: string): void;
-    /** Mute / unmute by toggling the mic tracks' `enabled` flag (transport stays up). */
-    SetMuted(muted: boolean): void;
-    /** @inheritdoc */
-    get IsBusy(): boolean;
-    /** @inheritdoc */
-    get IsAudioPlaying(): boolean;
+    protected handleSocketClose(): void;
     /** Creates the websocket to the given URL (the proxy URL). Production wraps the platform `WebSocket`. */
-    protected createSocket(url: string): IHuggingFaceClientSocket;
-    /** Creation seam for the mic-capture pipeline. Production delegates to the shared {@link createPcmMicCapture}. */
-    protected createMicCapture(micStream: MediaStream, sampleRate: number, onPcmChunk: (base64Pcm16: string) => void): Promise<IPcmMicCapture>;
-    /** Creation seam for the playout engine. Production returns the shared {@link RealtimePcmPlayback}. */
-    protected createPlayback(sampleRate: number): IRealtimePcmPlayback;
-    /** Resolver for the in-flight Connect's socket-open wait. */
-    private onOpen;
-    /** Extracts the wire-shaped `session` object from the server pact (`{ session, sampleRate }`). */
-    private static parseSessionObject;
-    /** Extracts the PCM sample rate from the server pact, defaulting to the HF-native 16 kHz. */
-    private static parseSampleRate;
-    /** Creates a one-shot signal promise, handing its resolver to the supplied setter. */
-    private buildSignal;
-    /** Sends the server-built session config once the session exists (skipped when empty). */
-    private applySessionConfig;
-    /** Streams one base64 PCM16 mic frame as an `input_audio_buffer.append`. */
-    private sendInput;
-    /** Parses an inbound frame and dispatches it. */
-    private handleSocketMessage;
-    /** Dispatches a typed OpenAI-Realtime server event to the appropriate behavior. */
-    private handleEvent;
-    /** Enqueues one base64 audio chunk for playout and reflects `'speaking'`. */
-    private handleAudioDelta;
-    /** Appends an assistant transcript delta and emits it (tagged with the active response kind). */
-    private onAssistantDelta;
-    /** Finalizes the assistant turn (empty turns emit nothing). */
-    private onAssistantDone;
-    /**
-     * Surfaces a completed tool call. Silently leaves `'speaking'` (no emission) so a host busy indicator
-     * isn't clobbered, and clears the busy flag (deadlock guard — obligation #2). `arguments` is already a
-     * JSON string on the wire.
-     */
-    private onToolCallFrame;
-    /**
-     * True barge-in ONLY: the user started speaking over active model output (a response in flight or audio
-     * audibly playing). Flushes local playback, emits the interruption, returns to `'listening'`.
-     */
-    private handleSpeechStarted;
-    /** Response boundary: release the lock, emit usage, fire any queued tool-result reply, return to listening. */
-    private handleResponseDone;
-    /** Triggers a reply immediately when idle, else queues it until the current response finishes. */
-    private requestResultResponse;
-    /** On a turn completing, fire any queued tool-result response so the answer is spoken. */
-    private flushPendingResultResponse;
-    /** Resets the per-session response state machine (used on Disconnect). */
-    private resetResponseState;
-    /** A socket-level error is fatal — the transport is gone (obligation #6). */
-    private handleSocketError;
-    /** A close is silent when consumer-initiated; otherwise reported as a terminal `'closed'`. */
-    private handleSocketClose;
-    /** Updates the client's own state view and emits the change to the host. */
-    private setState;
-    /** JSON-serializes and sends one client frame (no-op when the socket is closed). */
-    private sendFrame;
+    protected createSocket(url: string): IOpenAIProtocolClientSocket;
 }
 /**
  * Tree-shaking prevention: bundlers cannot see that {@link HuggingFaceRealtimeClient} is instantiated

package/dist/drivers/huggingFaceRealtimeClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"huggingFaceRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/huggingFaceRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAc,MAAM,oBAAoB,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAuB,MAAM,+BAA+B,CAAC;AAExF,OAAO,EAAE,oBAAoB,EAAuB,MAAM,sBAAsB,CAAC;AAEjF,OAAO,EAAuB,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1E;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAQ,CAAC;AAIzD,qGAAqG;AACrG,MAAM,WAAW,4BAA4B;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8FAA8F;IAC9F,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+DAA+D;IAC/D,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,qFAAqF;IACrF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wFAAwF;IACxF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,QAAQ,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE;YAAE,YAAY,CAAC,EAAE,MAAM,CAAC;YAAC,aAAa,CAAC,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC;IACzE,4CAA4C;IAC5C,KAAK,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/C;AAID,kGAAkG;AAClG,MAAM,WAAW,wBAAwB;IACrC,MAAM,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC3C,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC;IAC5C,OAAO,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,IAAI,CAAC;IAC7B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,IAAI,IAAI,CAAC;CACjB;AAcD;;;;;;;;;;;;;;;GAeG;AACH,qBACa,yBAA0B,SAAQ,kBAAkB;IAE7D,SAAS,CAAC,MAAM,EAAE,wBAAwB,GAAG,IAAI,CAAQ;IACzD,OAAO,CAAC,QAAQ,CAAqC;IACrD,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,SAAS,CAA4B;IAE7C,8FAA8F;IAC9F,OAAO,CAAC,aAAa,CAAkB;IACvC,0FAA0F;IAC1F,OAAO,CAAC,gBAAgB,CAA6B;IAGrD,OAAO,CAAC,oBAAoB,CAAM;IAClC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,kBAAkB,CAAoC;IAC9D,OAAO,CAAC,YAAY,CAAiC;IACrD,OAAO,CAAC,gBAAgB,CAAS;IAIjC;;;;OAIG;IACU,OAAO,CAAC,MAAM,EAAE,2BAA2B,EAAE,SAAS,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IA2BhG,iFAAiF;IACpE,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAqBxC;;;OAGG;IACI,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAYnC;;;OAGG;IACI,oBAAoB,IAAI,IAAI;IAqBnC,qGAAqG;IAC9F,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAU1C;;;OAGG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI;IAStD;;;OAGG;IACI,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI;IAW/D,qFAAqF;IAC9E,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAMrC,kBAAkB;IAClB,IAAW,MAAM,IAAI,OAAO,CAE3B;IAED,kBAAkB;IAClB,IAAW,cAAc,IAAI,OAAO,CAEnC;IAID,yGAAyG;IACzG,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,wBAAwB;IAqB7D,kHAAkH;cAClG,gBAAgB,CAC5B,SAAS,EAAE,WAAW,EACtB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,CAAC,WAAW,EAAE,MAAM,KAAK,IAAI,GAC1C,OAAO,CAAC,cAAc,CAAC;IAI1B,uGAAuG;IACvG,SAAS,CAAC,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,oBAAoB;IAMlE,6DAA6D;IAC7D,OAAO,CAAC,MAAM,CAA6B;IAE3C,kGAAkG;IAClG,OAAO,CAAC,MAAM,CAAC,kBAAkB;IAMjC,6FAA6F;IAC7F,OAAO,CAAC,MAAM,CAAC,eAAe;IAK9B,sFAAsF;IACtF,OAAO,CAAC,WAAW;IAInB,0FAA0F;IAC1F,OAAO,CAAC,kBAAkB;IAO1B,4EAA4E;IAC5E,OAAO,CAAC,SAAS;IAMjB,iDAAiD;IACjD,OAAO,CAAC,mBAAmB;IAa3B,mFAAmF;IACnF,OAAO,CAAC,WAAW;IA2CnB,6EAA6E;IAC7E,OAAO,CAAC,gBAAgB;IAUxB,iGAAiG;IACjG,OAAO,CAAC,gBAAgB;IAWxB,+DAA+D;IAC/D,OAAO,CAAC,eAAe;IAQvB;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAYvB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAS3B,+GAA+G;IAC/G,OAAO,CAAC,kBAAkB;IAe1B,kGAAkG;IAClG,OAAO,CAAC,qBAAqB;IAc7B,0FAA0F;IAC1F,OAAO,CAAC,0BAA0B;IAUlC,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAU1B,6EAA6E;IAC7E,OAAO,CAAC,iBAAiB;IAKzB,8FAA8F;IAC9F,OAAO,CAAC,iBAAiB;IAWzB,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAKhB,oFAAoF;IACpF,OAAO,CAAC,SAAS;CAGpB;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,IAAI,IAAI,CAEpD"}
+{"version":3,"file":"huggingFaceRealtimeClient.d.ts","sourceRoot":"","sources":["../../src/drivers/huggingFaceRealtimeClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EACH,qCAAqC,EACrC,2BAA2B,EAC9B,MAAM,iCAAiC,CAAC;AAIzC;;;;GAIG;AACH,eAAO,MAAM,mCAAmC,QAAQ,CAAC;AAczD;;;;;;;;;;;;;;;;;GAiBG;AACH,qBACa,yBAA0B,SAAQ,qCAAqC;IAChF,qEAAqE;IACrE,cAAuB,kBAAkB,IAAI,MAAM,CAElD;IAID,2FAA2F;IAC3F,SAAS,CAAC,kBAAkB,CAAC,MAAM,EAAE,2BAA2B,GAAG,2BAA2B;IAI9F,sGAAsG;IACtG,cAAuB,sBAAsB,IAAI,OAAO,CAEvD;IAED,uGAAuG;cACpF,oBAAoB,CAAC,MAAM,EAAE,2BAA2B,GAAG,UAAU;IAMxF,iFAAiF;IACjF,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE,2BAA2B,GAAG,MAAM;IAKxE;;;;;;OAMG;cACgB,iBAAiB,IAAI,IAAI;IAS5C,yGAAyG;IACzG,SAAS,CAAC,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,2BAA2B;CAoBnE;AAED;;;;GAIG;AACH,wBAAgB,6BAA6B,IAAI,IAAI,CAEpD"}