@memberjunction/ai-openai 5.47.0 → 5.49.0

package/dist/models/openAIRealtime.js

@@ -8,7 +8,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import { RegisterClass } from '@memberjunction/global';
-import { BaseRealtimeModel, RealtimeDiagLog, } from '@memberjunction/ai';
+import { BaseRealtimeModel, RealtimeDiagLog, IsTranscriptContinuation, } from '@memberjunction/ai';
 import { OpenAI } from 'openai';
 import { OpenAIRealtimeWebSocket } from 'openai/realtime/websocket';
 /**
@@ -19,6 +19,213 @@ import { OpenAIRealtimeWebSocket } from 'openai/realtime/websocket';
  * contract's promise of both-role transcripts holds everywhere.
  */
 const OPENAI_INPUT_TRANSCRIPTION_MODEL = 'gpt-4o-mini-transcribe';
+/** Runtime validation set for {@link RealtimeReasoningEffort} values arriving via the untyped Config bag. */
+const REALTIME_REASONING_EFFORTS = new Set(['minimal', 'low', 'medium', 'high', 'xhigh']);
+/**
+ * Maps MJ's NORMALIZED effort level (the same `ChatParams.effortLevel` vocabulary the LLM drivers
+ * consume: a numeric 1–100 value, or a named level) onto OpenAI's realtime
+ * {@link RealtimeReasoningEffort} union. This is the OpenAI implementation of the
+ * {@link OpenAIRealtimeProfile.mapEffortLevel} seam — providers with a DIFFERENT effort vocabulary
+ * override the profile function rather than the protocol code.
+ *
+ * Numeric mapping is quintile-based across OpenAI's five levels: ≤20 → `minimal`, ≤40 → `low`,
+ * ≤60 → `medium`, ≤80 → `high`, >80 → `xhigh`. Named values already in the union pass through.
+ * Unmappable values return `undefined` (dropped with a diag log — never sent raw).
+ *
+ * @param effortLevel The MJ-normalized effort level (numeric string/number 1–100 or named level).
+ * @returns The provider effort literal, or `undefined` when the value cannot be mapped.
+ */
+export function MapEffortLevelToOpenAIRealtime(effortLevel) {
+    const named = effortLevel.trim().toLowerCase();
+    if (REALTIME_REASONING_EFFORTS.has(named)) {
+        return named;
+    }
+    const numValue = Number.parseInt(named, 10);
+    // A non-numeric OR non-positive value is nonsensical for a 1–100 scale — drop it (no override)
+    // rather than silently flooring 0/negatives to a real 'minimal' reasoning setting.
+    if (Number.isNaN(numValue) || numValue <= 0) {
+        return undefined;
+    }
+    if (numValue <= 20)
+        return 'minimal';
+    if (numValue <= 40)
+        return 'low';
+    if (numValue <= 60)
+        return 'medium';
+    if (numValue <= 80)
+        return 'high';
+    return 'xhigh';
+}
+/**
+ * Maps a GA per-modality usage-detail block onto the Core {@link RealtimeUsageModalityDetail}
+ * shape. Returns `undefined` when the provider reported no detail block (totals-only flows).
+ */
+export function MapUsageModalityDetail(detail) {
+    if (!detail) {
+        return undefined;
+    }
+    const mapped = {};
+    if (typeof detail.text_tokens === 'number')
+        mapped.TextTokens = detail.text_tokens;
+    if (typeof detail.audio_tokens === 'number')
+        mapped.AudioTokens = detail.audio_tokens;
+    if (typeof detail.image_tokens === 'number')
+        mapped.ImageTokens = detail.image_tokens;
+    if (typeof detail.cached_tokens === 'number')
+        mapped.CachedTokens = detail.cached_tokens;
+    return Object.keys(mapped).length > 0 ? mapped : undefined;
+}
+/** The OpenAI provider profile — the defaults every OpenAI-compatible subclass overrides from. */
+export const OPENAI_REALTIME_PROFILE = {
+    providerKey: 'openai',
+    inputTranscriptionModel: OPENAI_INPUT_TRANSCRIPTION_MODEL,
+    deferInitialConfigUntilSessionCreated: true,
+    foldInitialContextIntoPrompt: false,
+    supportsReasoningEffort: true,
+    supportsParallelToolCalls: true,
+    supportsMcpTools: true,
+    supportsVoiceOutput: true,
+    supportsLiveReconfigure: true,
+    unexpectedCloseMessage: 'OpenAI realtime connection closed unexpectedly',
+    // OpenAI's default turn detection (server VAD with auto-response) is correct for 1:1 calls, so
+    // the block is only sent when meeting mode needs create_response disabled.
+    buildTurnDetection: (disableAutoResponse) => disableAutoResponse ? { type: 'server_vad', create_response: false, interrupt_response: true } : undefined,
+    mapEffortLevel: MapEffortLevelToOpenAIRealtime,
+};
+/**
+ * Pulls the MJ-idiomatic feature keys OUT of the open Config bag so they are (a) translated to
+ * their provider-native session fields only when the profile confirms support, and (b) NEVER
+ * leaked raw into a provider payload that would reject unknown fields.
+ *
+ * Recognized bag keys: `effortLevel` (MJ-normalized: numeric 1–100 or named), `reasoningEffort`
+ * (provider-native literal — wins over `effortLevel` when both are present), `parallelToolCalls`,
+ * `mcpTools`, `voice`, `disableAutoResponse`. Everything else passes through in `rest`
+ * (provider-native keys like `tool_choice` or `output_modalities` can be set directly by config
+ * authors).
+ *
+ * @param config The open session Config bag (may be undefined).
+ * @returns The extracted features plus the residual bag.
+ */
+export function ExtractRealtimeFeatures(config) {
+    const rest = { ...(config ?? {}) };
+    // The provider-native key is an explicit override; the normalized key is the standard channel.
+    // Both are scrubbed either way so neither ever leaks raw into a provider payload. Numbers are
+    // accepted on effortLevel (ChatParams.effortLevel is a string, but config authors write JSON).
+    const rawNative = rest.reasoningEffort;
+    delete rest.reasoningEffort;
+    const rawNormalized = rest.effortLevel;
+    delete rest.effortLevel;
+    let effortLevel;
+    if (typeof rawNative === 'string' && rawNative.trim().length > 0) {
+        effortLevel = rawNative.trim();
+    }
+    else if (typeof rawNormalized === 'string' && rawNormalized.trim().length > 0) {
+        effortLevel = rawNormalized.trim();
+    }
+    else if (typeof rawNormalized === 'number' && Number.isFinite(rawNormalized)) {
+        effortLevel = String(rawNormalized);
+    }
+    const rawParallel = rest.parallelToolCalls;
+    delete rest.parallelToolCalls;
+    const parallelToolCalls = typeof rawParallel === 'boolean' ? rawParallel : undefined;
+    const rawMcp = rest.mcpTools;
+    delete rest.mcpTools;
+    const mcpTools = Array.isArray(rawMcp) && rawMcp.length > 0 ? rawMcp : undefined;
+    const rawVoice = rest.voice;
+    delete rest.voice;
+    const trimmedVoice = typeof rawVoice === 'string' ? rawVoice.trim() : '';
+    const voice = trimmedVoice.length > 0 ? trimmedVoice : undefined;
+    const disableAutoResponse = rest.disableAutoResponse === true;
+    delete rest.disableAutoResponse;
+    // PROTECTED WIRE FIELDS — never overridable through the open bag. `type` is the GA session
+    // discriminator (a clobbered value makes strict endpoints reject the WHOLE session.update,
+    // silently dropping the prompt AND tools); `instructions` is the server-authored co-agent
+    // identity; `tools` is the server-authored tool authority. `audio` remains an intentional,
+    // documented override channel.
+    const protectedBag = rest;
+    if (protectedBag.type !== undefined || protectedBag.instructions !== undefined || protectedBag.tools !== undefined || protectedBag.model !== undefined) {
+        RealtimeDiagLog('[OpenAIRealtime][diag] Scrubbing protected wire field(s) (type/instructions/tools/model) from the session Config bag — these are server-authored and cannot be overridden per session');
+    }
+    delete protectedBag.type;
+    delete protectedBag.instructions;
+    delete protectedBag.tools;
+    // `model` is server-authoritative on the client-direct minted session (set from params.Model) —
+    // a bag override would let a browser pin a different model in the ephemeral pact.
+    delete protectedBag.model;
+    // Per-session transcription-model override + MJ-side transport settings. All scrubbed
+    // unconditionally — none of these are wire fields on ANY provider in the family.
+    const bag = rest;
+    const rawItm = bag.inputTranscriptionModel;
+    delete bag.inputTranscriptionModel;
+    const inputTranscriptionModel = typeof rawItm === 'string' && rawItm.trim().length > 0 ? rawItm.trim() : undefined;
+    const rawEndpoint = bag.endpoint;
+    delete bag.endpoint;
+    const endpoint = typeof rawEndpoint === 'string' && rawEndpoint.trim().length > 0 ? rawEndpoint.trim() : undefined;
+    const rawRate = bag.sampleRate;
+    delete bag.sampleRate;
+    const sampleRate = typeof rawRate === 'number' && rawRate > 0 ? rawRate : undefined;
+    const rawProxy = bag.proxyBaseUrl;
+    delete bag.proxyBaseUrl;
+    const proxyBaseUrl = typeof rawProxy === 'string' && rawProxy.trim().length > 0 ? rawProxy.trim() : undefined;
+    return { effortLevel, parallelToolCalls, mcpTools, voice, disableAutoResponse, inputTranscriptionModel, endpoint, sampleRate, proxyBaseUrl, rest };
+}
+/**
+ * Applies the profile-gated GA features onto a session payload. Features a provider has not
+ * confirmed are silently dropped (already scrubbed from the bag by {@link ExtractRealtimeFeatures})
+ * rather than sent and rejected. Effort levels run through the profile's `mapEffortLevel` seam so
+ * each provider translates MJ's normalized vocabulary to its own literals.
+ *
+ * @param session The session payload under construction.
+ * @param features The features extracted from the Config bag.
+ * @param profile The provider profile gating each feature.
+ */
+function applyGAFeatures(session, features, profile) {
+    if (profile.supportsReasoningEffort && features.effortLevel) {
+        const mapped = profile.mapEffortLevel(features.effortLevel);
+        if (mapped) {
+            session.reasoning = { effort: mapped };
+        }
+        else {
+            RealtimeDiagLog(`[${profile.providerKey}Realtime][diag] Ignoring unmappable effort level '${features.effortLevel}'`);
+        }
+    }
+    if (profile.supportsParallelToolCalls && features.parallelToolCalls !== undefined) {
+        session.parallel_tool_calls = features.parallelToolCalls;
+    }
+    if (profile.supportsMcpTools && features.mcpTools && features.mcpTools.length > 0) {
+        // MCP server tools ride ALONGSIDE the function tools — the GA tools array is a union of both.
+        // NOTE: the driver has no MCP approval UX yet, so config authors should declare servers with
+        // `require_approval: 'never'`; an mcp_approval_request arriving mid-session is surfaced as a
+        // recoverable session error (see OpenAIRealtimeSession.dispatch) rather than silently stalling.
+        session.tools = [...(session.tools ?? []), ...features.mcpTools];
+    }
+}
+/**
+ * Assembles the session `audio` block from the profile + extracted features, or `undefined` when
+ * every part is empty (compat endpoints reject/ignore hollow blocks). The transcription model is
+ * the per-session bag override when present, else the profile's default (which may be undefined
+ * for natively-transcribing providers).
+ *
+ * @param profile The provider profile.
+ * @param features The extracted Config-bag features.
+ * @param turnDetection The already-built turn-detection block, if any.
+ * @returns The audio block, or `undefined` to omit it.
+ */
+function BuildAudioBlock(profile, features, turnDetection) {
+    const transcriptionModel = features.inputTranscriptionModel ?? profile.inputTranscriptionModel;
+    const input = {
+        ...(transcriptionModel ? { transcription: { model: transcriptionModel } } : {}),
+        ...(turnDetection ? { turn_detection: turnDetection } : {}),
+    };
+    const output = profile.supportsVoiceOutput && features.voice ? { voice: features.voice } : undefined;
+    if (Object.keys(input).length === 0 && !output) {
+        return undefined;
+    }
+    return {
+        ...(Object.keys(input).length > 0 ? { input } : {}),
+        ...(output ? { output } : {}),
+    };
+}
 /**
  * Maps Core {@link RealtimeToolDefinition}s up to OpenAI's native function-tool schema.
  *
@@ -46,31 +253,65 @@ function mapRealtimeTools(tools) {
  * context), and translates the provider's server-event stream into the modality-agnostic
  * {@link IRealtimeSession} contract.
  *
+ * **This class is also the shared implementation for OpenAI-Realtime-compatible providers.**
+ * Compatible providers (e.g. xAI Grok Voice) subclass it, pass their base URL to the constructor,
+ * and override {@link OpenAIRealtime.Profile} — inheriting the whole protocol implementation and
+ * every future GA feature (gated per-provider by the profile) instead of maintaining a clone.
+ *
+ * **GA features** (gpt-realtime-2 / 2.1 era) are driven from the open
+ * {@link RealtimeSessionParams.Config} bag with MJ-idiomatic keys, translated to provider-native
+ * session fields only when the profile confirms support:
+ * - `reasoningEffort: 'minimal'|'low'|'medium'|'high'|'xhigh'` → `reasoning.effort`
+ * - `parallelToolCalls: boolean` → `parallel_tool_calls`
+ * - `mcpTools: [{ type:'mcp', server_label, server_url|connector_id, ... }]` → appended to `session.tools`
+ *
  * **Tool results** complete the tool-call loop: the returned session implements the Core
  * `IRealtimeSession.SendToolResult` contract method, which the agent layer calls after executing a
  * tool to feed its result back to the model. See {@link OpenAIRealtimeSession.SendToolResult}.
  */
 let OpenAIRealtime = class OpenAIRealtime extends BaseRealtimeModel {
-    constructor(apiKey) {
+    /**
+     * @param apiKey The provider API key.
+     * @param baseURL Optional override for OpenAI-compatible providers (subclasses pass their own
+     * endpoint; the SDK's `buildRealtimeURL()` derives the wss:// realtime endpoint from it).
+     */
+    constructor(apiKey, baseURL) {
         super(apiKey);
-        this._openAI = new OpenAI({ apiKey });
+        this._openAI = baseURL ? new OpenAI({ apiKey, baseURL }) : new OpenAI({ apiKey });
     }
     /** Read-only accessor for the underlying OpenAI SDK client. */
     get OpenAI() {
         return this._openAI;
     }
+    /**
+     * The provider profile driving per-provider knobs and GA feature gates. Subclasses override
+     * this single seam instead of re-implementing the protocol.
+     */
+    get Profile() {
+        return OPENAI_REALTIME_PROFILE;
+    }
     /**
      * Creates the realtime connection for a model. Overridable seam for testing.
      *
      * Production returns a real `OpenAIRealtimeWebSocket`. Unit tests override this to return a
      * fake {@link IOpenAIRealtimeConnection} that emits OpenAI-shaped events and captures sends.
      *
-     * @param model The provider realtime model id (e.g. `gpt-realtime`).
+     * @param model The provider realtime model id (e.g. `gpt-realtime-2.1`).
      * @returns A connection implementing {@link IOpenAIRealtimeConnection}.
      */
     createConnection(model) {
         return new OpenAIRealtimeWebSocket({ model }, this._openAI);
     }
+    /**
+     * Creates the session wrapper for a freshly-opened connection. Overridable seam so subclasses
+     * can return their own session subclass while {@link StartSession} stays shared.
+     *
+     * @param connection The open provider connection.
+     * @returns The session bound to this driver's {@link Profile}.
+     */
+    createSessionInstance(connection) {
+        return new OpenAIRealtimeSession(connection, this.Profile);
+    }
     /**
      * Opens a duplex realtime session, applies the session config, and returns the live handle.
      *
@@ -79,7 +320,7 @@ let OpenAIRealtime = class OpenAIRealtime extends BaseRealtimeModel {
      */
     async StartSession(params) {
         const connection = this.createConnection(params.Model);
-        const session = new OpenAIRealtimeSession(connection);
+        const session = this.createSessionInstance(connection);
         session.applyInitialConfig(params);
         return session;
     }
@@ -108,11 +349,12 @@ let OpenAIRealtime = class OpenAIRealtime extends BaseRealtimeModel {
         ];
     }
     /**
-     * Mints the ephemeral client secret via OpenAI's Realtime client-secrets API. Overridable
-     * seam for testing — unit tests return a fake response so no network call is made.
+     * Mints the ephemeral client secret via the provider's Realtime client-secrets API (resolved
+     * from the SDK client's base URL, so OpenAI-compatible subclasses target their own endpoint).
+     * Overridable seam for testing — unit tests return a fake response so no network call is made.
      *
      * @param body The client-secret create request (carries the realtime session config).
-     * @returns The OpenAI client-secret create response (token value + expiry + echoed session).
+     * @returns The client-secret create response (token value + expiry + echoed session).
      */
     async mintClientSecret(body) {
         return this._openAI.realtime.clientSecrets.create(body);
@@ -123,32 +365,42 @@ let OpenAIRealtime = class OpenAIRealtime extends BaseRealtimeModel {
      * (system prompt + tools + model) so it retains control of behavior even though the browser
      * owns the socket.
      *
+     * The GA features (reasoning effort, parallel tool calls, MCP tools) and the output voice are
+     * extracted from the Config bag and applied here exactly as on the server-bridged path, so the
+     * two topologies stay behaviorally identical — the browser applies the minted SessionConfig
+     * verbatim with no client-side changes needed.
+     *
      * @param params Session configuration (model, system prompt, tools).
      * @returns The minted {@link ClientRealtimeSessionConfig} the browser authenticates + applies.
      */
     async CreateClientSession(params) {
+        const profile = this.Profile;
+        const features = ExtractRealtimeFeatures(params.Config);
+        // Enable transcription of the user's mic input so BOTH sides of the conversation are
+        // captured (live captions + persisted ConversationDetail turns). Realtime models accept
+        // audio natively, so input transcription is a separate ASR pass that must be opted into.
+        // The OUTPUT voice comes from the effective config's per-provider voice (`params.Config.voice`,
+        // shaped by GetProviderVoiceSettings) — this is what lets a co-agent's configured voice OR a
+        // per-session override actually take effect in the client-direct topology.
+        const turnDetection = profile.buildTurnDetection(features.disableAutoResponse);
+        const audio = BuildAudioBlock(profile, features, turnDetection);
         const session = {
             type: 'realtime',
             model: params.Model,
             instructions: params.SystemPrompt,
+            ...(audio ? { audio } : {}),
+            // The residual (feature-scrubbed, wire-field-protected) Config bag applies here EXACTLY
+            // as on the server-bridged session.update — same construction ORDER too, so a raw
+            // `audio` override behaves identically on both topologies.
+            ...features.rest,
         };
         if (params.Tools && params.Tools.length > 0) {
             session.tools = mapRealtimeTools(params.Tools);
         }
-        // Enable transcription of the user's mic input so BOTH sides of the conversation are
-        // captured (live captions + persisted ConversationDetail turns). Realtime models accept
-        // audio natively, so input transcription is a separate ASR pass that must be opted into.
-        // The OUTPUT voice comes from the effective config's per-provider voice (`params.Config.voice`,
-        // shaped by GetProviderVoiceSettings) — this is what lets a co-agent's configured voice OR a
-        // per-session override actually take effect in the client-direct topology.
-        const voice = params.Config?.voice;
-        session.audio = {
-            input: { transcription: { model: OPENAI_INPUT_TRANSCRIPTION_MODEL } },
-            ...(voice && voice.trim().length > 0 ? { output: { voice: voice.trim() } } : {}),
-        };
+        applyGAFeatures(session, features, profile);
         const response = await this.mintClientSecret({ session });
         return {
-            Provider: 'openai',
+            Provider: profile.providerKey,
             Model: params.Model,
             EphemeralToken: response.value,
             ExpiresAt: new Date(response.expires_at * 1000).toISOString(),
@@ -159,7 +411,7 @@ let OpenAIRealtime = class OpenAIRealtime extends BaseRealtimeModel {
 };
 OpenAIRealtime = __decorate([
     RegisterClass(BaseRealtimeModel, 'OpenAIRealtime'),
-    __metadata("design:paramtypes", [String])
+    __metadata("design:paramtypes", [String, String])
 ], OpenAIRealtime);
 export { OpenAIRealtime };
 /**
@@ -167,11 +419,24 @@ export { OpenAIRealtime };
  *
  * Holds the registered handlers and the single `'event'` listener that fans the provider's
  * server-event stream out to the contract handlers via {@link OpenAIRealtimeSession.dispatch}.
+ *
+ * The session is profile-parameterized (see {@link OpenAIRealtimeProfile}) so OpenAI-compatible
+ * provider subclasses reuse it verbatim with their own knobs.
  */
 export class OpenAIRealtimeSession {
-    constructor(connection) {
+    /**
+     * @param connection The injectable provider-connection seam.
+     * @param profile The provider profile (defaults to OpenAI's so existing direct construction keeps working).
+     */
+    constructor(connection, profile = OPENAI_REALTIME_PROFILE) {
         /** Set by {@link Close} so a consumer-initiated teardown never reports an "unexpected" close. */
         this.closedByConsumer = false;
+        this.resolveConfigApplied = null;
+        this.rejectConfigApplied = null;
+        /** The deferred-config listener awaiting `session.created`, tracked so teardown can remove it. */
+        this.pendingConfigListener = null;
+        /** Deadline timer for the deferred-config readiness wait (see {@link configReadinessTimeoutMs}). */
+        this.configReadinessTimer = null;
         /**
          * Whether a model response is currently in flight. Minimal response tracking that mirrors the
          * client driver's state machine: set on `response.created` (and eagerly whenever this session
@@ -180,9 +445,46 @@ export class OpenAIRealtimeSession {
          * `cancelled` after barge-in, so the flag can never stick. Consumed by
          * {@link OpenAIRealtimeSession.RequestSpokenUpdate} to skip (not collide with) an active
          * response, since the API rejects overlapping `response.create` requests.
+         *
+         * Protected (not private) so compat-endpoint session subclasses can apply provider-specific
+         * robustness tweaks (e.g. HuggingFace marks a response active on the first audio delta and
+         * releases the flag when a tool call yields the floor).
          */
         this.responseActive = false;
+        /**
+         * Whether the CURRENT user turn has already produced at least one finalized input transcription.
+         * Streamed-transcription providers (Grok) emit `input_audio_transcription.completed` REPEATEDLY
+         * for one utterance, each carrying the full growing text; without this flag every repeat lands as
+         * a fresh non-replacing final and the persistence layer mints a duplicate `ConversationDetail`
+         * row per caption. The second-and-later completeds are flagged {@link RealtimeTranscript.ReplacesPrevious}
+         * so they REPLACE the turn's row in place — exactly the client-direct driver's behavior, kept in
+         * sync here so the two topologies persist identically. Reset on each `speech_started` (new turn).
+         * Harmless for single-completed providers (OpenAI): the flag is always false on the one completed.
+         */
+        this.userTurnTranscribed = false;
+        /**
+         * Text of the CURRENT user turn's most recent finalized transcription, used to detect that a new
+         * `completed` is the same utterance continuing rather than a new turn.
+         *
+         * Streaming-transcription providers re-emit the FULL accumulated utterance on every `completed`,
+         * and their VAD fires `speech_started` on ordinary mid-sentence pauses. Keying the turn boundary
+         * solely off `speech_started` therefore splits one spoken thought into several turns, each a longer
+         * copy of the last. Comparing against this text (via {@link IsTranscriptContinuation}, which
+         * normalizes punctuation because ASR re-punctuates as a sentence grows) lets a post-pause caption
+         * be recognized as a continuation and REPLACE the turn in place instead.
+         *
+         * Cleared when the model starts responding (`response.created`), which is the real end of the
+         * user's turn — so two genuinely separate utterances can never be merged across a model reply.
+         */
+        this.lastUserTranscript = '';
         this.connection = connection;
+        this.profile = profile;
+        this.configAppliedPromise = new Promise((resolve, reject) => {
+            this.resolveConfigApplied = resolve;
+            this.rejectConfigApplied = reject;
+        });
+        // Not every consumer awaits WaitForConfigApplied — guard unhandled-rejection noise.
+        this.configAppliedPromise.catch(() => undefined);
         this.eventListener = (event) => this.dispatch(event);
         this.connection.on('event', this.eventListener);
         this.errorListener = (error) => this.handleConnectionError(error);
@@ -195,17 +497,37 @@ export class OpenAIRealtimeSession {
      * Applies the initial session config: system prompt + tools via `session.update`, optional initial
      * context as a user message. Called once by {@link OpenAIRealtime.StartSession}.
      *
-     * **Deferred to `session.created`.** When `StartSession` returns, the realtime WebSocket is NOT open yet
-     * — sending `session.update` synchronously races the handshake and the instructions (the **system
-     * prompt + tools**) are silently dropped, so the model runs with NO prompt (no identity, no companion
-     * framing). We therefore wait for the server's `session.created` frame — the first event once the socket
-     * is open and the session exists, and the canonical moment to configure a realtime session — exactly the
-     * point the browser/client-direct path applies its config. Idempotent (a re-emitted `session.created`
-     * can't double-apply); the listener removes itself once it fires.
+     * **Deferral is profile-driven.** On OpenAI the realtime WebSocket is NOT open when `StartSession`
+     * returns — sending `session.update` synchronously races the handshake and the instructions (the
+     * **system prompt + tools**) are silently dropped, so the model runs with NO prompt (no identity, no
+     * companion framing). We therefore wait for the server's `session.created` frame — the first event once
+     * the socket is open and the session exists, and the canonical moment to configure a realtime session —
+     * exactly the point the browser/client-direct path applies its config. Idempotent (a re-emitted
+     * `session.created` can't double-apply); the listener removes itself once it fires. Providers whose
+     * socket accepts config immediately (xAI) set the profile flag false and send synchronously.
      *
      * @param params The session parameters.
      */
     applyInitialConfig(params) {
+        // Compat endpoints with no history-seeding channel fold the prior context into the system
+        // prompt instead of seeding a separate user message (profile-driven).
+        const fold = this.profile.foldInitialContextIntoPrompt;
+        const context = params.InitialContext?.trim();
+        const systemPrompt = fold && context ? `${params.SystemPrompt}\n\n## Prior context\n${context}` : params.SystemPrompt;
+        const applyConfig = () => {
+            this.sendSessionUpdate(systemPrompt, params.Tools, params.Config);
+            if (!fold && context && context.length > 0) {
+                this.sendInitialContext(context);
+            }
+            this.clearConfigReadinessTimer();
+            this.resolveConfigApplied?.();
+            this.resolveConfigApplied = null;
+            this.rejectConfigApplied = null;
+        };
+        if (!this.profile.deferInitialConfigUntilSessionCreated) {
+            applyConfig();
+            return;
+        }
         let applied = false;
         const applyWhenReady = (event) => {
             if (applied || event.type !== 'session.created') {
@@ -213,12 +535,77 @@ export class OpenAIRealtimeSession {
             }
             applied = true;
             this.connection.off('event', applyWhenReady);
-            this.sendSessionUpdate(params.SystemPrompt, params.Tools, params.Config);
-            if (params.InitialContext && params.InitialContext.length > 0) {
-                this.sendInitialContext(params.InitialContext);
-            }
+            this.pendingConfigListener = null;
+            applyConfig();
         };
+        this.pendingConfigListener = applyWhenReady;
         this.connection.on('event', applyWhenReady);
+        // Readiness deadline: a silent endpoint (socket open, no session.created) must not hang a
+        // driver that AWAITS WaitForConfigApplied (HuggingFace) forever. The timeout rejects the
+        // WAIT only — the deferred listener stays registered, so a late session.created on a
+        // fire-and-forget flow (OpenAI's non-awaiting StartSession) still applies the config.
+        this.configReadinessTimer = setTimeout(() => {
+            this.configReadinessTimer = null;
+            this.failConfigWaitOnly(`session.created not received within ${this.configReadinessTimeoutMs}ms — endpoint silent during startup`);
+        }, this.configReadinessTimeoutMs);
+        // Node-only nicety: never let a readiness timer keep the process alive (browser bundles
+        // of this server package don't exist; unref is feature-detected anyway).
+        this.configReadinessTimer.unref?.();
+    }
+    /**
+     * Readiness deadline in milliseconds for the deferred-config wait. Only affects consumers of
+     * {@link WaitForConfigApplied}; the deferred apply itself is not cancelled. Overridable.
+     */
+    get configReadinessTimeoutMs() {
+        return 15_000;
+    }
+    /** Rejects a pending config wait WITHOUT removing the deferred listener (timeout semantics). */
+    failConfigWaitOnly(message) {
+        if (this.rejectConfigApplied) {
+            const reject = this.rejectConfigApplied;
+            this.rejectConfigApplied = null;
+            this.resolveConfigApplied = null;
+            reject(new Error(message));
+        }
+    }
+    /** Clears the readiness-deadline timer (config applied, or session torn down). */
+    clearConfigReadinessTimer() {
+        if (this.configReadinessTimer) {
+            clearTimeout(this.configReadinessTimer);
+            this.configReadinessTimer = null;
+        }
+    }
+    /** Removes a still-pending deferred-config listener (teardown before `session.created`). */
+    clearPendingConfigListener() {
+        if (this.pendingConfigListener) {
+            this.connection.off('event', this.pendingConfigListener);
+            this.pendingConfigListener = null;
+        }
+    }
+    /**
+     * Resolves once the initial session config has been APPLIED (sent on the socket) — immediately
+     * for providers that configure synchronously, or on the server's `session.created` frame for
+     * deferring providers. Rejects if the transport dies (fatal error or unexpected close) or the
+     * consumer closes the session before the config went out.
+     *
+     * The base {@link OpenAIRealtime.StartSession} deliberately does NOT await this (OpenAI
+     * semantics: the session handle is returned while the handshake completes). Drivers whose
+     * contract promises "ready only after config is applied" (HuggingFace) await it in their
+     * `StartSession` override.
+     */
+    WaitForConfigApplied() {
+        return this.configAppliedPromise;
+    }
+    /** Rejects a still-pending {@link WaitForConfigApplied} (transport death / early consumer close). */
+    failConfigWait(message) {
+        this.clearConfigReadinessTimer();
+        this.clearPendingConfigListener();
+        if (this.rejectConfigApplied) {
+            const reject = this.rejectConfigApplied;
+            this.rejectConfigApplied = null;
+            this.resolveConfigApplied = null;
+            reject(new Error(message));
+        }
     }
     // ---- IRealtimeSession outbound ----
     /** @inheritdoc */
@@ -295,11 +682,11 @@ export class OpenAIRealtimeSession {
      */
     RequestSpokenUpdate(instructions) {
         if (this.responseActive) {
-            RealtimeDiagLog('[OpenAIRealtime][diag] RequestSpokenUpdate SKIPPED — a response is already active (interim updates are disposable)');
+            RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] RequestSpokenUpdate SKIPPED — a response is already active (interim updates are disposable)`);
             return false; // NOT sent — the caller (bridge) releases the floor instead of wedging on it
         }
         this.responseActive = true;
-        RealtimeDiagLog(`[OpenAIRealtime][diag] RequestSpokenUpdate → sending response.create (perResponseInstructions=${typeof instructions === 'string' && instructions.trim().length > 0 ? 'yes' : 'none → session prompt governs'})`);
+        RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] RequestSpokenUpdate → sending response.create (perResponseInstructions=${typeof instructions === 'string' && instructions.trim().length > 0 ? 'yes' : 'none → session prompt governs'})`);
         // CRITICAL: only set per-response `instructions` when the caller actually supplied some. OpenAI's
         // `response.create` treats `response.instructions` as a FULL override of the session system prompt for
         // that response — so forwarding `''` would wipe the co-agent identity framing (incl. the
@@ -309,9 +696,9 @@ export class OpenAIRealtimeSession {
         this.connection.send(hasInstructions ? { type: 'response.create', response: { instructions } } : { type: 'response.create' });
         return true; // a response.create was issued — the bridge may hold the floor for this turn
     }
-    /** @inheritdoc — OpenAI's `session.update` is runtime-mutable, so a live turn-mode change is supported. */
+    /** @inheritdoc — profile-gated: only providers whose endpoint honors a live partial `session.update`. */
     get Capabilities() {
-        return { CanReconfigureTurnMode: true };
+        return { CanReconfigureTurnMode: this.profile.supportsLiveReconfigure };
     }
     /**
      * @inheritdoc
@@ -321,17 +708,29 @@ export class OpenAIRealtimeSession {
      * transcription block is re-sent alongside so the partial update can't drop it.
      */
     Reconfigure(params) {
+        if (!this.profile.supportsLiveReconfigure) {
+            // The profile declares no live-reconfigure support — advertising Capabilities false is
+            // the primary guard; this no-op is defense-in-depth against callers that skip the check.
+            RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] Reconfigure ignored — profile declares no live turn-mode support`);
+            return;
+        }
         const disable = params.DisableAutoResponse === true;
         const turnDetection = {
             type: 'server_vad',
             create_response: !disable,
             interrupt_response: true,
         };
+        // Re-send the transcription block alongside ONLY when this profile transcribes via an
+        // opt-in model — a partial update must not fabricate `transcription: { model: undefined }`
+        // for natively-transcribing providers.
+        const transcription = this.profile.inputTranscriptionModel
+            ? { transcription: { model: this.profile.inputTranscriptionModel } }
+            : {};
         this.connection.send({
             type: 'session.update',
             session: {
                 type: 'realtime',
-                audio: { input: { transcription: { model: OPENAI_INPUT_TRANSCRIPTION_MODEL }, turn_detection: turnDetection } },
+                audio: { input: { ...transcription, turn_detection: turnDetection } },
             },
         });
     }
@@ -367,6 +766,8 @@ export class OpenAIRealtimeSession {
     /** @inheritdoc */
     async Close() {
         this.closedByConsumer = true;
+        this.failConfigWait('session closed by consumer before the initial config was applied');
+        this.clearPendingConfigListener();
         this.connection.off('event', this.eventListener);
         this.connection.off('error', this.errorListener);
         this.connection.close();
@@ -376,6 +777,9 @@ export class OpenAIRealtimeSession {
      * Routes a provider server event to the matching contract handler. Each branch delegates to a
      * small, single-purpose handler to keep this dispatcher flat.
      *
+     * Protected (not private) so OpenAI-compatible session subclasses can pre-translate legacy /
+     * beta frame aliases before delegating here.
+     *
      * @param event The OpenAI realtime server event.
      */
     dispatch(event) {
@@ -388,13 +792,35 @@ export class OpenAIRealtimeSession {
                 return this.emitTranscript('assistant', event.transcript, true);
             case 'conversation.item.input_audio_transcription.delta':
                 return this.emitTranscript('user', event.delta ?? '', false);
-            case 'conversation.item.input_audio_transcription.completed':
-                return this.emitTranscript('user', event.transcript, true);
+            case 'conversation.item.input_audio_transcription.completed': {
+                // Streamed transcription (Grok): the 2nd+ completed of a turn REPLACES the turn's row
+                // in place rather than appending a duplicate. Two ways a completed can be a replacement:
+                //   1. another completed already landed in THIS turn (userTurnTranscribed); or
+                //   2. it CONTINUES the previous utterance — the provider re-emitted the whole thing
+                //      with more words on the end. This second case is what rescues a mid-sentence
+                //      pause: the VAD fires speech_started (clearing the flag) even though the user
+                //      never stopped talking, and without the continuation test that one spoken thought
+                //      persists as several rows, each a longer copy of the last.
+                const text = event.transcript;
+                const replacesPrevious = this.userTurnTranscribed || IsTranscriptContinuation(this.lastUserTranscript, text);
+                this.userTurnTranscribed = true;
+                this.lastUserTranscript = text;
+                return this.emitTranscript('user', text, true, replacesPrevious);
+            }
             case 'response.function_call_arguments.done':
                 return this.handleFunctionCall(event.call_id, event.name, event.arguments);
             case 'input_audio_buffer.speech_started':
+                // A new user turn begins — reset the streamed-transcription flag so its first
+                // completed is a fresh (non-replacing) final. Do this UNCONDITIONALLY (not only on
+                // true barge-in): handleInterruption gates its handler on responseActive, but the
+                // turn boundary is real regardless of whether the model was mid-response.
+                this.userTurnTranscribed = false;
                 return this.handleInterruption();
             case 'response.created':
+                // The model has taken the floor — the user's turn is definitively over. Clear the
+                // continuation anchor so a LATER utterance can never be merged into it just because it
+                // happens to start with the same words.
+                this.lastUserTranscript = '';
                 // A response is in flight (whether server-VAD-triggered or locally triggered).
                 this.responseActive = true;
                 return;
@@ -403,6 +829,55 @@ export class OpenAIRealtimeSession {
                 this.responseActive = false;
                 return this.handleResponseDone(event.response.usage);
             default:
+                return this.dispatchMcpEvent(event);
+        }
+    }
+    /**
+     * Handles the MCP slice of the server-event stream. MCP tool calls execute SERVER-SIDE at the
+     * provider (no MJ round-trip like function tools), so most lifecycle frames are diag-only. The
+     * one that needs action — `mcp_approval_request` — cannot be satisfied yet (no approval UX in
+     * the agent layer), so it is surfaced as a RECOVERABLE session error instead of silently
+     * stalling the session; config authors should declare MCP servers with `require_approval: 'never'`.
+     *
+     * The frames are matched by type string because the pinned SDK's `RealtimeServerEvent` union
+     * carries them with dedicated interfaces already (`McpListToolsFailed`, `ResponseMcpCallFailed`, etc.).
+     *
+     * @param event The (possibly MCP-related) server event.
+     */
+    dispatchMcpEvent(event) {
+        switch (event.type) {
+            case 'response.mcp_call.failed':
+                RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] MCP tool call FAILED`);
+                this.errorHandler?.({ Message: 'A remote MCP tool call failed at the provider', Fatal: false });
+                return;
+            case 'mcp_list_tools.failed':
+                RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] MCP server tool listing FAILED`);
+                this.errorHandler?.({ Message: 'Listing tools from a remote MCP server failed at the provider', Fatal: false });
+                return;
+            default:
+                // mcp_approval_request arrives as a conversation item add — detect it structurally.
+                if (event.type === 'conversation.item.added' && event.item?.type === 'mcp_approval_request') {
+                    // DEFENSIVE AUTO-DENY: no approval UX exists yet, and the model BLOCKS forever
+                    // awaiting an mcp_approval_response — dead air from the user's perspective. A
+                    // denial lets the model continue and voice the refusal instead of wedging the
+                    // turn. Config authors who want silent MCP flow declare require_approval:'never'.
+                    const approvalRequestId = event.item.id;
+                    if (approvalRequestId) {
+                        this.connection.send({
+                            type: 'conversation.item.create',
+                            item: {
+                                type: 'mcp_approval_response',
+                                approval_request_id: approvalRequestId,
+                                approve: false,
+                            },
+                        });
+                        RealtimeDiagLog(`[${this.profile.providerKey}Realtime][diag] MCP approval request AUTO-DENIED (no approval UX yet) — request ${approvalRequestId}`);
+                    }
+                    this.errorHandler?.({
+                        Message: "An MCP server requested tool approval; no approval UX exists yet, so it was automatically DENIED (the model continues and voices the refusal). Declare the server with require_approval: 'never' to avoid the round-trip.",
+                        Fatal: false,
+                    });
+                }
                 return;
         }
     }
@@ -410,9 +885,18 @@ export class OpenAIRealtimeSession {
     handleAudioDelta(deltaBase64) {
         this.outputHandler?.(this.decodeBase64(deltaBase64));
     }
-    /** Emits a transcript event to the transcript handler. */
-    emitTranscript(role, text, isFinal) {
-        this.transcriptHandler?.({ Role: role, Text: text, IsFinal: isFinal });
+    /**
+     * Emits a transcript event, skipping empty/whitespace text — empty captions are pure noise.
+     *
+     * @param replacesPrevious When true, this final REPLACES the current turn's persisted row in
+     *   place (streamed-transcription providers whose repeated completeds carry the full growing
+     *   text) rather than appending a new turn. Defaults to false (append/normal final).
+     */
+    emitTranscript(role, text, isFinal, replacesPrevious = false) {
+        if (!text || text.trim().length === 0) {
+            return;
+        }
+        this.transcriptHandler?.({ Role: role, Text: text, IsFinal: isFinal, ReplacesPrevious: replacesPrevious });
     }
     /** Forwards a completed function call to the tool-call handler. */
     handleFunctionCall(callId, name, args) {
@@ -441,6 +925,9 @@ export class OpenAIRealtimeSession {
      */
     handleConnectionError(error) {
         const isProviderFrame = error.error != null;
+        if (!isProviderFrame) {
+            this.failConfigWait(error.message);
+        }
         this.errorHandler?.({
             Message: error.message,
             Code: error.error?.code ?? undefined,
@@ -456,53 +943,62 @@ export class OpenAIRealtimeSession {
         if (this.closedByConsumer) {
             return;
         }
-        this.errorHandler?.({ Message: 'OpenAI realtime connection closed unexpectedly', Fatal: true });
+        this.failConfigWait(this.profile.unexpectedCloseMessage);
+        this.errorHandler?.({ Message: this.profile.unexpectedCloseMessage, Fatal: true });
         this.closeHandler?.();
     }
-    /** Translates a response's usage block into a {@link RealtimeUsage} update. */
+    /**
+     * Translates a response's usage block into a {@link RealtimeUsage} update, INCLUDING the
+     * per-modality token details the GA API reports — realtime cost attribution is impossible
+     * without the audio/text/cached split (audio-in bills ~8x text-in on GPT Realtime 2.1).
+     */
     handleResponseDone(usage) {
         if (!usage) {
             return;
         }
-        this.usageHandler?.({
+        const update = {
             InputTokens: usage.input_tokens ?? 0,
             OutputTokens: usage.output_tokens ?? 0,
-        });
+        };
+        const input = MapUsageModalityDetail(usage.input_token_details);
+        if (input) {
+            update.InputTokenDetails = input;
+        }
+        const output = MapUsageModalityDetail(usage.output_token_details);
+        if (output) {
+            update.OutputTokenDetails = output;
+        }
+        this.usageHandler?.(update);
     }
     // ---- Config helpers ----
-    /** Sends the `session.update` that establishes instructions, input transcription, and tools. */
+    /** Sends the `session.update` that establishes instructions, input transcription, tools, and GA features. */
     sendSessionUpdate(systemPrompt, tools, config) {
-        // Pull the host-neutral meeting flag OUT of the open Config bag so it is never sent raw to the API.
-        // In a multi-agent meeting the BRIDGE (after its turn policy gates on addressing), not the model,
-        // decides WHEN to speak — so we disable server-VAD auto-response while KEEPING detection so input
-        // transcription and barge-in still work. A 1:1 call (flag absent) keeps the default auto-response.
-        const cfg = { ...(config ?? {}) };
-        const disableAutoResponse = cfg.disableAutoResponse === true;
-        delete cfg.disableAutoResponse;
-        // Pull the OUTPUT voice out of the bag too — OpenAI's realtime session takes it at
-        // `audio.output.voice` (NOT top-level), so letting it spread via `...cfg` would silently no-op (the
-        // co-agent's configured voice / a per-session dev override would be ignored on the server-bridged path).
-        const voice = typeof cfg.voice === 'string' ? cfg.voice.trim() : '';
-        delete cfg.voice;
-        const turnDetection = disableAutoResponse
-            ? { type: 'server_vad', create_response: false, interrupt_response: true }
-            : undefined;
+        // Pull the MJ-idiomatic feature keys OUT of the open Config bag: the host-neutral meeting flag
+        // (disableAutoResponse), the output voice, and the GA features (reasoningEffort/parallelToolCalls/
+        // mcpTools) — each translated to its provider-native field only when the profile confirms support,
+        // and never sent raw to the API. In a multi-agent meeting the BRIDGE (after its turn policy gates
+        // on addressing), not the model, decides WHEN to speak — so we disable server-VAD auto-response
+        // while KEEPING detection so input transcription and barge-in still work. A 1:1 call (flag absent)
+        // keeps the provider's default auto-response.
+        const features = ExtractRealtimeFeatures(config);
+        const turnDetection = this.profile.buildTurnDetection(features.disableAutoResponse);
+        // Opt into USER input transcription — the same opt-in CreateClientSession applies for the
+        // client-direct topology — so user-role transcripts flow server-bridged too (the contract
+        // promises BOTH roles). Providers that transcribe natively (profile model undefined, no bag
+        // override) get no transcription block; an all-empty audio block is omitted entirely. The
+        // residual config bag spreads AFTER the built block so a per-conversation raw `audio`
+        // override can still replace it wholesale.
+        const audio = BuildAudioBlock(this.profile, features, turnDetection);
         const session = {
             type: 'realtime',
             instructions: systemPrompt,
-            // Opt into USER input transcription — the same opt-in CreateClientSession applies for
-            // the client-direct topology — so user-role transcripts flow server-bridged too (the
-            // contract promises BOTH roles). The config bag spreads after this so a
-            // per-conversation override can still replace the audio block.
-            audio: {
-                input: { transcription: { model: OPENAI_INPUT_TRANSCRIPTION_MODEL }, ...(turnDetection ? { turn_detection: turnDetection } : {}) },
-                ...(voice ? { output: { voice } } : {}),
-            },
-            ...cfg,
+            ...(audio ? { audio } : {}),
+            ...features.rest,
         };
         if (tools && tools.length > 0) {
             session.tools = this.mapTools(tools);
         }
+        applyGAFeatures(session, features, this.profile);
         this.connection.send({ type: 'session.update', session });
     }
     /** Seeds the conversation with initial context as a user text message. */