@memberjunction/ai-openai 5.47.0 → 5.49.0

package/dist/models/openAIRealtime.d.ts

@@ -1,9 +1,151 @@
-import { BaseRealtimeModel, IRealtimeSession, RealtimeSessionCapabilities, RealtimeReconfigureParams, RealtimeSessionParams, RealtimeToolDefinition, RealtimeTranscript, RealtimeToolCall, RealtimeUsage, RealtimeSessionError, RealtimeVoiceOption } from '@memberjunction/ai';
+import { BaseRealtimeModel, IRealtimeSession, RealtimeSessionCapabilities, RealtimeReconfigureParams, RealtimeSessionParams, RealtimeToolDefinition, RealtimeTranscript, RealtimeToolCall, RealtimeUsage, RealtimeUsageModalityDetail, RealtimeSessionError, RealtimeVoiceOption, JSONObject } from '@memberjunction/ai';
 import { ClientRealtimeSessionConfig } from '@memberjunction/ai';
 import { OpenAI } from 'openai';
 import type { OpenAIRealtimeError } from 'openai/realtime/index';
-import type { RealtimeClientEvent, RealtimeServerEvent } from 'openai/resources/realtime/realtime';
+import type { RealtimeClientEvent, RealtimeServerEvent, RealtimeAudioInputTurnDetection, RealtimeToolsConfigUnion } from 'openai/resources/realtime/realtime';
 import type { ClientSecretCreateParams, ClientSecretCreateResponse } from 'openai/resources/realtime/client-secrets';
+/**
+ * The reasoning effort levels the GA Realtime API accepts for reasoning-capable realtime models
+ * (gpt-realtime-2 / gpt-realtime-2.1 line). `low` is the provider default — it keeps latency down
+ * for voice; raise only when task complexity justifies the added latency and reasoning tokens.
+ */
+export type RealtimeReasoningEffort = 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
+/**
+ * Maps MJ's NORMALIZED effort level (the same `ChatParams.effortLevel` vocabulary the LLM drivers
+ * consume: a numeric 1–100 value, or a named level) onto OpenAI's realtime
+ * {@link RealtimeReasoningEffort} union. This is the OpenAI implementation of the
+ * {@link OpenAIRealtimeProfile.mapEffortLevel} seam — providers with a DIFFERENT effort vocabulary
+ * override the profile function rather than the protocol code.
+ *
+ * Numeric mapping is quintile-based across OpenAI's five levels: ≤20 → `minimal`, ≤40 → `low`,
+ * ≤60 → `medium`, ≤80 → `high`, >80 → `xhigh`. Named values already in the union pass through.
+ * Unmappable values return `undefined` (dropped with a diag log — never sent raw).
+ *
+ * @param effortLevel The MJ-normalized effort level (numeric string/number 1–100 or named level).
+ * @returns The provider effort literal, or `undefined` when the value cannot be mapped.
+ */
+export declare function MapEffortLevelToOpenAIRealtime(effortLevel: string): RealtimeReasoningEffort | undefined;
+/** Wire shape of a per-modality usage-detail block on the GA API. */
+interface GARealtimeUsageDetail {
+    text_tokens?: number;
+    audio_tokens?: number;
+    image_tokens?: number;
+    cached_tokens?: number;
+}
+/**
+ * Maps a GA per-modality usage-detail block onto the Core {@link RealtimeUsageModalityDetail}
+ * shape. Returns `undefined` when the provider reported no detail block (totals-only flows).
+ */
+export declare function MapUsageModalityDetail(detail: GARealtimeUsageDetail | undefined): RealtimeUsageModalityDetail | undefined;
+/**
+ * Provider profile for the OpenAI-Realtime-protocol driver family.
+ *
+ * `OpenAIRealtime` implements the full OpenAI Realtime wire protocol once; OpenAI-compatible
+ * providers (e.g. xAI Grok Voice) subclass it and supply their own profile instead of cloning the
+ * driver. The profile carries the per-provider knobs — transcription model, turn detection, GA
+ * feature gates — so protocol/feature work lands here once and flows to every compatible provider,
+ * while providers that have NOT confirmed a GA feature keep it gated off (a one-line flip later).
+ */
+export interface OpenAIRealtimeProfile {
+    /** The {@link ClientRealtimeSessionConfig.Provider} key the browser uses to pick its client driver. */
+    providerKey: string;
+    /**
+     * The ASR model for USER input transcription (opt-in pass; see {@link OPENAI_INPUT_TRANSCRIPTION_MODEL}).
+     * `undefined` means the provider transcribes natively (e.g. a cascaded STT stage) and no
+     * transcription block is sent unless the Config bag supplies `inputTranscriptionModel`.
+     */
+    inputTranscriptionModel?: string;
+    /**
+     * Whether the initial `session.update` must wait for the server's `session.created` frame.
+     * OpenAI's socket drops config sent during the handshake; xAI's accepts it immediately.
+     */
+    deferInitialConfigUntilSessionCreated: boolean;
+    /**
+     * When true, `InitialContext` is folded into the system prompt under a "Prior context" heading
+     * instead of being seeded as a separate user conversation item — for compat endpoints with no
+     * guaranteed history-seeding channel (HuggingFace speech-to-speech).
+     */
+    foldInitialContextIntoPrompt: boolean;
+    /** Whether the provider accepts the GA `reasoning.effort` session field. */
+    supportsReasoningEffort: boolean;
+    /** Whether the provider accepts the GA `parallel_tool_calls` session field. */
+    supportsParallelToolCalls: boolean;
+    /** Whether the provider accepts remote MCP server tools (`type: "mcp"`) in `session.tools`. */
+    supportsMcpTools: boolean;
+    /** Whether the provider accepts an output voice at `audio.output.voice`. */
+    supportsVoiceOutput: boolean;
+    /**
+     * Whether the provider supports LIVE turn-mode reconfiguration via a partial `session.update`
+     * (drives both the session's `Capabilities.CanReconfigureTurnMode` and whether `Reconfigure`
+     * emits anything). Compat endpoints without `create_response` gating set false.
+     */
+    supportsLiveReconfigure: boolean;
+    /** The fatal-error message surfaced when the socket closes unexpectedly. */
+    unexpectedCloseMessage: string;
+    /**
+     * Builds the `audio.input.turn_detection` block. `disableAutoResponse` comes from the Config
+     * bag (meeting mode: the bridge, not server VAD, decides when the model speaks). Return
+     * `undefined` to omit the block and accept the provider default.
+     */
+    buildTurnDetection(disableAutoResponse: boolean): RealtimeAudioInputTurnDetection | undefined;
+    /**
+     * Maps MJ's NORMALIZED effort level (numeric 1–100 or named — the same vocabulary as
+     * `ChatParams.effortLevel`) onto THIS provider's realtime effort literals. Providers whose
+     * endpoint uses a different level set override this seam; return `undefined` to drop an
+     * unmappable value (it is never sent raw). Only consulted when `supportsReasoningEffort` is on.
+     */
+    mapEffortLevel(effortLevel: string): string | undefined;
+}
+/** The OpenAI provider profile — the defaults every OpenAI-compatible subclass overrides from. */
+export declare const OPENAI_REALTIME_PROFILE: OpenAIRealtimeProfile;
+/**
+ * The realtime feature values extracted (and removed) from the open {@link RealtimeSessionParams.Config}
+ * bag before the remainder is spread into the provider session payload.
+ */
+interface ExtractedRealtimeFeatures {
+    /**
+     * The raw effort value awaiting the profile's `mapEffortLevel` translation. Sourced from the
+     * provider-native `reasoningEffort` bag key when present (explicit override), else from the
+     * MJ-normalized `effortLevel` key (numeric 1–100 or named — `ChatParams.effortLevel` vocabulary).
+     */
+    effortLevel?: string;
+    /** `parallelToolCalls` bag value, if present. */
+    parallelToolCalls?: boolean;
+    /** Remote MCP server tool declarations from the `mcpTools` bag value, if present. */
+    mcpTools?: RealtimeToolsConfigUnion.Mcp[];
+    /** Trimmed `voice` bag value, if present and non-blank. */
+    voice?: string;
+    /** The host-neutral meeting flag (`disableAutoResponse`) — never sent raw to a provider. */
+    disableAutoResponse: boolean;
+    /** Per-session input-transcription model override (`inputTranscriptionModel` bag key). */
+    inputTranscriptionModel?: string;
+    /**
+     * MJ-side transport settings (`endpoint`, `sampleRate`, `proxyBaseUrl` bag keys) consumed by
+     * self-hosted/proxied drivers — ALWAYS scrubbed so they never leak into a provider payload.
+     */
+    endpoint?: string;
+    /** See {@link ExtractedRealtimeFeatures.endpoint}. */
+    sampleRate?: number;
+    /** See {@link ExtractedRealtimeFeatures.endpoint}. */
+    proxyBaseUrl?: string;
+    /** The remaining bag entries, safe to spread into the session payload. */
+    rest: JSONObject;
+}
+/**
+ * Pulls the MJ-idiomatic feature keys OUT of the open Config bag so they are (a) translated to
+ * their provider-native session fields only when the profile confirms support, and (b) NEVER
+ * leaked raw into a provider payload that would reject unknown fields.
+ *
+ * Recognized bag keys: `effortLevel` (MJ-normalized: numeric 1–100 or named), `reasoningEffort`
+ * (provider-native literal — wins over `effortLevel` when both are present), `parallelToolCalls`,
+ * `mcpTools`, `voice`, `disableAutoResponse`. Everything else passes through in `rest`
+ * (provider-native keys like `tool_choice` or `output_modalities` can be set directly by config
+ * authors).
+ *
+ * @param config The open session Config bag (may be undefined).
+ * @returns The extracted features plus the residual bag.
+ */
+export declare function ExtractRealtimeFeatures(config: JSONObject | undefined): ExtractedRealtimeFeatures;
 /**
  * Minimal connection surface the {@link OpenAIRealtime} driver depends on.
  *
@@ -57,25 +199,55 @@ export interface IOpenAIRealtimeConnection {
  * context), and translates the provider's server-event stream into the modality-agnostic
  * {@link IRealtimeSession} contract.
  *
+ * **This class is also the shared implementation for OpenAI-Realtime-compatible providers.**
+ * Compatible providers (e.g. xAI Grok Voice) subclass it, pass their base URL to the constructor,
+ * and override {@link OpenAIRealtime.Profile} — inheriting the whole protocol implementation and
+ * every future GA feature (gated per-provider by the profile) instead of maintaining a clone.
+ *
+ * **GA features** (gpt-realtime-2 / 2.1 era) are driven from the open
+ * {@link RealtimeSessionParams.Config} bag with MJ-idiomatic keys, translated to provider-native
+ * session fields only when the profile confirms support:
+ * - `reasoningEffort: 'minimal'|'low'|'medium'|'high'|'xhigh'` → `reasoning.effort`
+ * - `parallelToolCalls: boolean` → `parallel_tool_calls`
+ * - `mcpTools: [{ type:'mcp', server_label, server_url|connector_id, ... }]` → appended to `session.tools`
+ *
  * **Tool results** complete the tool-call loop: the returned session implements the Core
  * `IRealtimeSession.SendToolResult` contract method, which the agent layer calls after executing a
  * tool to feed its result back to the model. See {@link OpenAIRealtimeSession.SendToolResult}.
  */
 export declare class OpenAIRealtime extends BaseRealtimeModel {
     private _openAI;
-    constructor(apiKey: string);
+    /**
+     * @param apiKey The provider API key.
+     * @param baseURL Optional override for OpenAI-compatible providers (subclasses pass their own
+     * endpoint; the SDK's `buildRealtimeURL()` derives the wss:// realtime endpoint from it).
+     */
+    constructor(apiKey: string, baseURL?: string);
     /** Read-only accessor for the underlying OpenAI SDK client. */
     get OpenAI(): OpenAI;
+    /**
+     * The provider profile driving per-provider knobs and GA feature gates. Subclasses override
+     * this single seam instead of re-implementing the protocol.
+     */
+    protected get Profile(): OpenAIRealtimeProfile;
     /**
      * Creates the realtime connection for a model. Overridable seam for testing.
      *
      * Production returns a real `OpenAIRealtimeWebSocket`. Unit tests override this to return a
      * fake {@link IOpenAIRealtimeConnection} that emits OpenAI-shaped events and captures sends.
      *
-     * @param model The provider realtime model id (e.g. `gpt-realtime`).
+     * @param model The provider realtime model id (e.g. `gpt-realtime-2.1`).
      * @returns A connection implementing {@link IOpenAIRealtimeConnection}.
      */
     protected createConnection(model: string): IOpenAIRealtimeConnection;
+    /**
+     * Creates the session wrapper for a freshly-opened connection. Overridable seam so subclasses
+     * can return their own session subclass while {@link StartSession} stays shared.
+     *
+     * @param connection The open provider connection.
+     * @returns The session bound to this driver's {@link Profile}.
+     */
+    protected createSessionInstance(connection: IOpenAIRealtimeConnection): OpenAIRealtimeSession;
     /**
      * Opens a duplex realtime session, applies the session config, and returns the live handle.
      *
@@ -95,11 +267,12 @@ export declare class OpenAIRealtime extends BaseRealtimeModel {
      */
     get SupportedVoices(): RealtimeVoiceOption[];
     /**
-     * Mints the ephemeral client secret via OpenAI's Realtime client-secrets API. Overridable
-     * seam for testing — unit tests return a fake response so no network call is made.
+     * Mints the ephemeral client secret via the provider's Realtime client-secrets API (resolved
+     * from the SDK client's base URL, so OpenAI-compatible subclasses target their own endpoint).
+     * Overridable seam for testing — unit tests return a fake response so no network call is made.
      *
      * @param body The client-secret create request (carries the realtime session config).
-     * @returns The OpenAI client-secret create response (token value + expiry + echoed session).
+     * @returns The client-secret create response (token value + expiry + echoed session).
      */
     protected mintClientSecret(body: ClientSecretCreateParams): Promise<ClientSecretCreateResponse>;
     /**
@@ -108,6 +281,11 @@ export declare class OpenAIRealtime extends BaseRealtimeModel {
      * (system prompt + tools + model) so it retains control of behavior even though the browser
      * owns the socket.
      *
+     * The GA features (reasoning effort, parallel tool calls, MCP tools) and the output voice are
+     * extracted from the Config bag and applied here exactly as on the server-bridged path, so the
+     * two topologies stay behaviorally identical — the browser applies the minted SessionConfig
+     * verbatim with no client-side changes needed.
+     *
      * @param params Session configuration (model, system prompt, tools).
      * @returns The minted {@link ClientRealtimeSessionConfig} the browser authenticates + applies.
      */
@@ -118,9 +296,13 @@ export declare class OpenAIRealtime extends BaseRealtimeModel {
  *
  * Holds the registered handlers and the single `'event'` listener that fans the provider's
  * server-event stream out to the contract handlers via {@link OpenAIRealtimeSession.dispatch}.
+ *
+ * The session is profile-parameterized (see {@link OpenAIRealtimeProfile}) so OpenAI-compatible
+ * provider subclasses reuse it verbatim with their own knobs.
  */
 export declare class OpenAIRealtimeSession implements IRealtimeSession {
     private connection;
+    private profile;
     private outputHandler?;
     private transcriptHandler?;
     private toolCallHandler?;
@@ -132,6 +314,14 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
     private errorListener;
     /** Set by {@link Close} so a consumer-initiated teardown never reports an "unexpected" close. */
     private closedByConsumer;
+    /** Backing promise for {@link WaitForConfigApplied}; resolve/reject handles null once settled. */
+    private configAppliedPromise;
+    private resolveConfigApplied;
+    private rejectConfigApplied;
+    /** The deferred-config listener awaiting `session.created`, tracked so teardown can remove it. */
+    private pendingConfigListener;
+    /** Deadline timer for the deferred-config readiness wait (see {@link configReadinessTimeoutMs}). */
+    private configReadinessTimer;
     /**
      * Whether a model response is currently in flight. Minimal response tracking that mirrors the
      * client driver's state machine: set on `response.created` (and eagerly whenever this session
@@ -140,24 +330,84 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
      * `cancelled` after barge-in, so the flag can never stick. Consumed by
      * {@link OpenAIRealtimeSession.RequestSpokenUpdate} to skip (not collide with) an active
      * response, since the API rejects overlapping `response.create` requests.
+     *
+     * Protected (not private) so compat-endpoint session subclasses can apply provider-specific
+     * robustness tweaks (e.g. HuggingFace marks a response active on the first audio delta and
+     * releases the flag when a tool call yields the floor).
+     */
+    protected responseActive: boolean;
+    /**
+     * Whether the CURRENT user turn has already produced at least one finalized input transcription.
+     * Streamed-transcription providers (Grok) emit `input_audio_transcription.completed` REPEATEDLY
+     * for one utterance, each carrying the full growing text; without this flag every repeat lands as
+     * a fresh non-replacing final and the persistence layer mints a duplicate `ConversationDetail`
+     * row per caption. The second-and-later completeds are flagged {@link RealtimeTranscript.ReplacesPrevious}
+     * so they REPLACE the turn's row in place — exactly the client-direct driver's behavior, kept in
+     * sync here so the two topologies persist identically. Reset on each `speech_started` (new turn).
+     * Harmless for single-completed providers (OpenAI): the flag is always false on the one completed.
+     */
+    private userTurnTranscribed;
+    /**
+     * Text of the CURRENT user turn's most recent finalized transcription, used to detect that a new
+     * `completed` is the same utterance continuing rather than a new turn.
+     *
+     * Streaming-transcription providers re-emit the FULL accumulated utterance on every `completed`,
+     * and their VAD fires `speech_started` on ordinary mid-sentence pauses. Keying the turn boundary
+     * solely off `speech_started` therefore splits one spoken thought into several turns, each a longer
+     * copy of the last. Comparing against this text (via {@link IsTranscriptContinuation}, which
+     * normalizes punctuation because ASR re-punctuates as a sentence grows) lets a post-pause caption
+     * be recognized as a continuation and REPLACE the turn in place instead.
+     *
+     * Cleared when the model starts responding (`response.created`), which is the real end of the
+     * user's turn — so two genuinely separate utterances can never be merged across a model reply.
      */
-    private responseActive;
-    constructor(connection: IOpenAIRealtimeConnection);
+    private lastUserTranscript;
+    /**
+     * @param connection The injectable provider-connection seam.
+     * @param profile The provider profile (defaults to OpenAI's so existing direct construction keeps working).
+     */
+    constructor(connection: IOpenAIRealtimeConnection, profile?: OpenAIRealtimeProfile);
     /**
      * Applies the initial session config: system prompt + tools via `session.update`, optional initial
      * context as a user message. Called once by {@link OpenAIRealtime.StartSession}.
      *
-     * **Deferred to `session.created`.** When `StartSession` returns, the realtime WebSocket is NOT open yet
-     * — sending `session.update` synchronously races the handshake and the instructions (the **system
-     * prompt + tools**) are silently dropped, so the model runs with NO prompt (no identity, no companion
-     * framing). We therefore wait for the server's `session.created` frame — the first event once the socket
-     * is open and the session exists, and the canonical moment to configure a realtime session — exactly the
-     * point the browser/client-direct path applies its config. Idempotent (a re-emitted `session.created`
-     * can't double-apply); the listener removes itself once it fires.
+     * **Deferral is profile-driven.** On OpenAI the realtime WebSocket is NOT open when `StartSession`
+     * returns — sending `session.update` synchronously races the handshake and the instructions (the
+     * **system prompt + tools**) are silently dropped, so the model runs with NO prompt (no identity, no
+     * companion framing). We therefore wait for the server's `session.created` frame — the first event once
+     * the socket is open and the session exists, and the canonical moment to configure a realtime session —
+     * exactly the point the browser/client-direct path applies its config. Idempotent (a re-emitted
+     * `session.created` can't double-apply); the listener removes itself once it fires. Providers whose
+     * socket accepts config immediately (xAI) set the profile flag false and send synchronously.
      *
      * @param params The session parameters.
      */
     applyInitialConfig(params: RealtimeSessionParams): void;
+    /**
+     * Readiness deadline in milliseconds for the deferred-config wait. Only affects consumers of
+     * {@link WaitForConfigApplied}; the deferred apply itself is not cancelled. Overridable.
+     */
+    protected get configReadinessTimeoutMs(): number;
+    /** Rejects a pending config wait WITHOUT removing the deferred listener (timeout semantics). */
+    private failConfigWaitOnly;
+    /** Clears the readiness-deadline timer (config applied, or session torn down). */
+    private clearConfigReadinessTimer;
+    /** Removes a still-pending deferred-config listener (teardown before `session.created`). */
+    private clearPendingConfigListener;
+    /**
+     * Resolves once the initial session config has been APPLIED (sent on the socket) — immediately
+     * for providers that configure synchronously, or on the server's `session.created` frame for
+     * deferring providers. Rejects if the transport dies (fatal error or unexpected close) or the
+     * consumer closes the session before the config went out.
+     *
+     * The base {@link OpenAIRealtime.StartSession} deliberately does NOT await this (OpenAI
+     * semantics: the session handle is returned while the handshake completes). Drivers whose
+     * contract promises "ready only after config is applied" (HuggingFace) await it in their
+     * `StartSession` override.
+     */
+    WaitForConfigApplied(): Promise<void>;
+    /** Rejects a still-pending {@link WaitForConfigApplied} (transport death / early consumer close). */
+    private failConfigWait;
     /** @inheritdoc */
     SendInput(chunk: ArrayBuffer): void;
     /** @inheritdoc */
@@ -203,7 +453,7 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
      *   forwarded as a per-response override.
      */
     RequestSpokenUpdate(instructions: string): boolean;
-    /** @inheritdoc — OpenAI's `session.update` is runtime-mutable, so a live turn-mode change is supported. */
+    /** @inheritdoc — profile-gated: only providers whose endpoint honors a live partial `session.update`. */
     get Capabilities(): RealtimeSessionCapabilities;
     /**
      * @inheritdoc
@@ -233,12 +483,34 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
      * Routes a provider server event to the matching contract handler. Each branch delegates to a
      * small, single-purpose handler to keep this dispatcher flat.
      *
+     * Protected (not private) so OpenAI-compatible session subclasses can pre-translate legacy /
+     * beta frame aliases before delegating here.
+     *
      * @param event The OpenAI realtime server event.
      */
-    private dispatch;
+    protected dispatch(event: RealtimeServerEvent): void;
+    /**
+     * Handles the MCP slice of the server-event stream. MCP tool calls execute SERVER-SIDE at the
+     * provider (no MJ round-trip like function tools), so most lifecycle frames are diag-only. The
+     * one that needs action — `mcp_approval_request` — cannot be satisfied yet (no approval UX in
+     * the agent layer), so it is surfaced as a RECOVERABLE session error instead of silently
+     * stalling the session; config authors should declare MCP servers with `require_approval: 'never'`.
+     *
+     * The frames are matched by type string because the pinned SDK's `RealtimeServerEvent` union
+     * carries them with dedicated interfaces already (`McpListToolsFailed`, `ResponseMcpCallFailed`, etc.).
+     *
+     * @param event The (possibly MCP-related) server event.
+     */
+    private dispatchMcpEvent;
     /** Decodes a base64 audio delta and forwards it to the output handler. */
     private handleAudioDelta;
-    /** Emits a transcript event to the transcript handler. */
+    /**
+     * Emits a transcript event, skipping empty/whitespace text — empty captions are pure noise.
+     *
+     * @param replacesPrevious When true, this final REPLACES the current turn's persisted row in
+     *   place (streamed-transcription providers whose repeated completeds carry the full growing
+     *   text) rather than appending a new turn. Defaults to false (append/normal final).
+     */
     private emitTranscript;
     /** Forwards a completed function call to the tool-call handler. */
     private handleFunctionCall;
@@ -265,9 +537,13 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
      * so consumers finalize instead of idling on a dead socket — followed by the close handler.
      */
     private handleSocketClose;
-    /** Translates a response's usage block into a {@link RealtimeUsage} update. */
+    /**
+     * Translates a response's usage block into a {@link RealtimeUsage} update, INCLUDING the
+     * per-modality token details the GA API reports — realtime cost attribution is impossible
+     * without the audio/text/cached split (audio-in bills ~8x text-in on GPT Realtime 2.1).
+     */
     private handleResponseDone;
-    /** Sends the `session.update` that establishes instructions, input transcription, and tools. */
+    /** Sends the `session.update` that establishes instructions, input transcription, tools, and GA features. */
     private sendSessionUpdate;
     /** Seeds the conversation with initial context as a user text message. */
     private sendInitialContext;
@@ -278,4 +554,5 @@ export declare class OpenAIRealtimeSession implements IRealtimeSession {
     /** Decodes a base64 audio delta into a freshly-allocated `ArrayBuffer`. */
     private decodeBase64;
 }
+export {};
 //# sourceMappingURL=openAIRealtime.d.ts.map

package/dist/models/openAIRealtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openAIRealtime.d.ts","sourceRoot":"","sources":["../../src/models/openAIRealtime.ts"],"names":[],"mappings":"AACA,OAAO,EACH,iBAAiB,EAEjB,gBAAgB,EAChB,2BAA2B,EAC3B,yBAAyB,EACzB,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EACb,oBAAoB,EACpB,mBAAmB,EAEtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAIhC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EACR,mBAAmB,EACnB,mBAAmB,EAOtB,MAAM,oCAAoC,CAAC;AAC5C,OAAO,KAAK,EACR,wBAAwB,EACxB,0BAA0B,EAC7B,MAAM,0CAA0C,CAAC;AA+BlD;;;;;;;;GAQG;AACH,MAAM,WAAW,yBAAyB;IACtC;;;;;;OAMG;IACH,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE;;;;;OAKG;IACH,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE,0GAA0G;IAC1G,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,sDAAsD;IACtD,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,gDAAgD;IAChD,IAAI,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACvC,oCAAoC;IACpC,KAAK,CAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACtD;;;;OAIG;IACH,MAAM,CAAC,EAAE;QAAE,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,IAAI,CAAA;KAAE,CAAC;CAC5E;AAED;;;;;;;;;;;GAWG;AACH,qBACa,cAAe,SAAQ,iBAAiB;IACjD,OAAO,CAAC,OAAO,CAAS;gBAEZ,MAAM,EAAE,MAAM;IAK1B,+DAA+D;IAC/D,IAAW,MAAM,IAAI,MAAM,CAE1B;IAED;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,yBAAyB;IAIpE;;;;;OAKG;IACU,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAOnF;;;;OAIG;IACH,IAAoB,oBAAoB,IAAI,OAAO,CAElD;IAED;;;OAGG;IACH,IAAoB,eAAe,IAAI,mBAAmB,EAAE,CAW3D;IAED;;;;;;OAMG;cACa,gBAAgB,CAAC,IAAI,EAAE,wBAAwB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAIrG;;;;;;;;OAQG;IACmB,mBAAmB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CA8BjH;AAED;;;;;GAKG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC1D,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,aAAa,CAAC,CAA+B;IACrD,OAAO,CAAC,iBAAiB,CAAC,CAAkC;IAC5D,OAAO,CAAC,eAAe,CAAC,CAAmC;IAC3D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAA6B;IAClD,OAAO,CAAC,YAAY,CAAC,CAAwC;IAC7D,OAAO,CAAC,YAAY,CAAC,CAAa;IAClC,OAAO,CAAC,aAAa,CAAuC;IAC5D,OAAO,CAAC,aAAa,CAAuC;IAC5D,iGAAiG;IACjG,OAAO,CAAC,gBAAgB,CAAS;IAEjC;;;;;;;;OAQG;IACH,OAAO,CAAC,cAAc,CAAS;gBAEnB,UAAU,EAAE,yBAAyB;IAWjD;;;;;;;;;;;;;OAaG;IACI,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI;IAkB9D,kBAAkB;IACX,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAO1C,kBAAkB;IACL,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO1E;;;;;;;;;OASG;IACU,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1E;;;;;;;;;;;;OAYG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAS1C;;;;;;;;;;;;;;OAcG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAmBzD,2GAA2G;IAC3G,IAAW,YAAY,IAAI,2BAA2B,CAErD;IAED;;;;;;OAMG;IACI,WAAW,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IAkB3D,kBAAkB;IACX,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAI5D,kBAAkB;IACX,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI;IAInE,kBAAkB;IACX,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAIlE,kBAAkB;IACX,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIzD,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI;IAIpE,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIzC,kBAAkB;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IASnC;;;;;OAKG;IACH,OAAO,CAAC,QAAQ;IA6BhB,0EAA0E;IAC1E,OAAO,CAAC,gBAAgB;IAIxB,0DAA0D;IAC1D,OAAO,CAAC,cAAc;IAItB,mEAAmE;IACnE,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAQzB,+EAA+E;IAC/E,OAAO,CAAC,kBAAkB;IAY1B,gGAAgG;IAChG,OAAO,CAAC,iBAAiB;IAqCzB,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAS1B,8FAA8F;IAC9F,OAAO,CAAC,QAAQ;IAMhB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;IAIpB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;CAMvB"}
+{"version":3,"file":"openAIRealtime.d.ts","sourceRoot":"","sources":["../../src/models/openAIRealtime.ts"],"names":[],"mappings":"AACA,OAAO,EACH,iBAAiB,EAEjB,gBAAgB,EAChB,2BAA2B,EAC3B,yBAAyB,EACzB,qBAAqB,EACrB,sBAAsB,EACtB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EACb,2BAA2B,EAC3B,oBAAoB,EACpB,mBAAmB,EAEnB,UAAU,EACb,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,2BAA2B,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAIhC,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AACjE,OAAO,KAAK,EACR,mBAAmB,EACnB,mBAAmB,EAMnB,+BAA+B,EAC/B,wBAAwB,EAC3B,MAAM,oCAAoC,CAAC;AAC5C,OAAO,KAAK,EACR,wBAAwB,EACxB,0BAA0B,EAC7B,MAAM,0CAA0C,CAAC;AAWlD;;;;GAIG;AACH,MAAM,MAAM,uBAAuB,GAAG,SAAS,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;AAKtF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,8BAA8B,CAAC,WAAW,EAAE,MAAM,GAAG,uBAAuB,GAAG,SAAS,CAgBvG;AA8BD,qEAAqE;AACrE,UAAU,qBAAqB;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,GAAG,2BAA2B,GAAG,SAAS,CAUzH;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB;IAClC,uGAAuG;IACvG,WAAW,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,qCAAqC,EAAE,OAAO,CAAC;IAC/C;;;;OAIG;IACH,4BAA4B,EAAE,OAAO,CAAC;IACtC,4EAA4E;IAC5E,uBAAuB,EAAE,OAAO,CAAC;IACjC,+EAA+E;IAC/E,yBAAyB,EAAE,OAAO,CAAC;IACnC,+FAA+F;IAC/F,gBAAgB,EAAE,OAAO,CAAC;IAC1B,4EAA4E;IAC5E,mBAAmB,EAAE,OAAO,CAAC;IAC7B;;;;OAIG;IACH,uBAAuB,EAAE,OAAO,CAAC;IACjC,4EAA4E;IAC5E,sBAAsB,EAAE,MAAM,CAAC;IAC/B;;;;OAIG;IACH,kBAAkB,CAAC,mBAAmB,EAAE,OAAO,GAAG,+BAA+B,GAAG,SAAS,CAAC;IAC9F;;;;;OAKG;IACH,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;CAC3D;AAED,kGAAkG;AAClG,eAAO,MAAM,uBAAuB,EAAE,qBAgBrC,CAAC;AAEF;;;GAGG;AACH,UAAU,yBAAyB;IAC/B;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,qFAAqF;IACrF,QAAQ,CAAC,EAAE,wBAAwB,CAAC,GAAG,EAAE,CAAC;IAC1C,2DAA2D;IAC3D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,4FAA4F;IAC5F,mBAAmB,EAAE,OAAO,CAAC;IAC7B,0FAA0F;IAC1F,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0EAA0E;IAC1E,IAAI,EAAE,UAAU,CAAC;CACpB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,UAAU,GAAG,SAAS,GAAG,yBAAyB,CA6EjG;AAyFD;;;;;;;;GAQG;AACH,MAAM,WAAW,yBAAyB;IACtC;;;;;;OAMG;IACH,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE;;;;;OAKG;IACH,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE,0GAA0G;IAC1G,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,sDAAsD;IACtD,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,KAAK,EAAE,mBAAmB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,gDAAgD;IAChD,IAAI,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI,CAAC;IACvC,oCAAoC;IACpC,KAAK,CAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACtD;;;;OAIG;IACH,MAAM,CAAC,EAAE;QAAE,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,IAAI,CAAA;KAAE,CAAC;CAC5E;AAED;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBACa,cAAe,SAAQ,iBAAiB;IACjD,OAAO,CAAC,OAAO,CAAS;IAExB;;;;OAIG;gBACS,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM;IAK5C,+DAA+D;IAC/D,IAAW,MAAM,IAAI,MAAM,CAE1B;IAED;;;OAGG;IACH,SAAS,KAAK,OAAO,IAAI,qBAAqB,CAE7C;IAED;;;;;;;;OAQG;IACH,SAAS,CAAC,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,yBAAyB;IAIpE;;;;;;OAMG;IACH,SAAS,CAAC,qBAAqB,CAAC,UAAU,EAAE,yBAAyB,GAAG,qBAAqB;IAI7F;;;;;OAKG;IACU,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAOnF;;;;OAIG;IACH,IAAoB,oBAAoB,IAAI,OAAO,CAElD;IAED;;;OAGG;IACH,IAAoB,eAAe,IAAI,mBAAmB,EAAE,CAW3D;IAED;;;;;;;OAOG;cACa,gBAAgB,CAAC,IAAI,EAAE,wBAAwB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAIrG;;;;;;;;;;;;;OAaG;IACmB,mBAAmB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAmCjH;AAED;;;;;;;;GAQG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAC1D,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,aAAa,CAAC,CAA+B;IACrD,OAAO,CAAC,iBAAiB,CAAC,CAAkC;IAC5D,OAAO,CAAC,eAAe,CAAC,CAAmC;IAC3D,OAAO,CAAC,mBAAmB,CAAC,CAAa;IACzC,OAAO,CAAC,YAAY,CAAC,CAA6B;IAClD,OAAO,CAAC,YAAY,CAAC,CAAwC;IAC7D,OAAO,CAAC,YAAY,CAAC,CAAa;IAClC,OAAO,CAAC,aAAa,CAAuC;IAC5D,OAAO,CAAC,aAAa,CAAuC;IAC5D,iGAAiG;IACjG,OAAO,CAAC,gBAAgB,CAAS;IAEjC,kGAAkG;IAClG,OAAO,CAAC,oBAAoB,CAAgB;IAC5C,OAAO,CAAC,oBAAoB,CAA6B;IACzD,OAAO,CAAC,mBAAmB,CAAyC;IACpE,kGAAkG;IAClG,OAAO,CAAC,qBAAqB,CAAuD;IACpF,oGAAoG;IACpG,OAAO,CAAC,oBAAoB,CAA8C;IAE1E;;;;;;;;;;;;OAYG;IACH,SAAS,CAAC,cAAc,UAAS;IAEjC;;;;;;;;;OASG;IACH,OAAO,CAAC,mBAAmB,CAAS;IAEpC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,kBAAkB,CAAM;IAEhC;;;OAGG;gBACS,UAAU,EAAE,yBAAyB,EAAE,OAAO,GAAE,qBAA+C;IAkB3G;;;;;;;;;;;;;;OAcG;IACI,kBAAkB,CAAC,MAAM,EAAE,qBAAqB,GAAG,IAAI;IA6C9D;;;OAGG;IACH,SAAS,KAAK,wBAAwB,IAAI,MAAM,CAE/C;IAED,gGAAgG;IAChG,OAAO,CAAC,kBAAkB;IAS1B,kFAAkF;IAClF,OAAO,CAAC,yBAAyB;IAOjC,4FAA4F;IAC5F,OAAO,CAAC,0BAA0B;IAOlC;;;;;;;;;;OAUG;IACI,oBAAoB,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5C,qGAAqG;IACrG,OAAO,CAAC,cAAc;IAatB,kBAAkB;IACX,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI;IAO1C,kBAAkB;IACL,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO1E;;;;;;;;;OASG;IACU,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAa1E;;;;;;;;;;;;OAYG;IACI,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAS1C;;;;;;;;;;;;;;OAcG;IACI,mBAAmB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO;IAmBzD,yGAAyG;IACzG,IAAW,YAAY,IAAI,2BAA2B,CAErD;IAED;;;;;;OAMG;IACI,WAAW,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI;IA8B3D,kBAAkB;IACX,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI;IAI5D,kBAAkB;IACX,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI;IAInE,kBAAkB;IACX,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI;IAIlE,kBAAkB;IACX,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIhD,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI;IAIzD,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI;IAIpE,kBAAkB;IACX,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI;IAIzC,kBAAkB;IACL,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAWnC;;;;;;;;OAQG;IACH,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,mBAAmB,GAAG,IAAI;IAmDpD;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,gBAAgB;IAsCxB,0EAA0E;IAC1E,OAAO,CAAC,gBAAgB;IAIxB;;;;;;OAMG;IACH,OAAO,CAAC,cAAc;IAOtB,mEAAmE;IACnE,OAAO,CAAC,kBAAkB;IAI1B;;;;;;;OAOG;IACH,OAAO,CAAC,kBAAkB;IAO1B;;;;;;OAMG;IACH,OAAO,CAAC,qBAAqB;IAY7B;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IASzB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAqB1B,6GAA6G;IAC7G,OAAO,CAAC,iBAAiB;IA+BzB,0EAA0E;IAC1E,OAAO,CAAC,kBAAkB;IAS1B,8FAA8F;IAC9F,OAAO,CAAC,QAAQ;IAMhB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;IAIpB,2EAA2E;IAC3E,OAAO,CAAC,YAAY;CAMvB"}