@memberjunction/ai 5.41.0 → 5.43.0

package/dist/generic/baseRealtime.d.ts

@@ -75,6 +75,18 @@ export type JSONObject = {
  *
  * @abstract
  */
+/**
+ * Emits a realtime-driver diagnostic line, but ONLY when verbose logging is enabled
+ * (`MJ_VERBOSE=true|1|yes`). These traces — turn boundaries, activity windows, response gating,
+ * barge-in — are invaluable when debugging a live session but far too chatty for normal operation,
+ * so they stay dark unless verbose mode is explicitly turned on. Shared here so every realtime driver
+ * and its session twin (OpenAI, Gemini, …) gate diagnostics through one consistent switch. The truthy
+ * set matches `@memberjunction/core`'s `IsVerboseLoggingEnabled` so a single `MJ_VERBOSE` flag governs
+ * verbose output across the whole stack.
+ *
+ * @param message The diagnostic message, already prefixed by the caller (e.g. `[GeminiRealtime][diag] …`).
+ */
+export declare function RealtimeDiagLog(message: string): void;
 export declare abstract class BaseRealtimeModel extends BaseModel {
     /**
      * Opens a stateful duplex session with the provider.
@@ -121,7 +133,47 @@ export declare abstract class BaseRealtimeModel extends BaseModel {
      * @throws Always, unless overridden by a provider that supports client-direct sessions.
      */
     CreateClientSession(_params: RealtimeSessionParams): Promise<ClientRealtimeSessionConfig>;
+    /**
+     * Whether this driver's sessions carry a **video** track in addition to audio — i.e. the model
+     * accepts video input (it can "see" the user's camera) and/or emits video output (a talking-head
+     * avatar / generated video), in sync with audio.
+     *
+     * Defaults to `false` (audio-only — today's realtime models). Video-capable drivers (a native
+     * multimodal realtime model, or an avatar provider) override this to `true`. The session's media
+     * plane is media-tagged ({@link IRealtimeSession.SendInput} takes a {@link RealtimeMediaKind};
+     * {@link IRealtimeSession.OnVideoOutput} delivers video-out), so a video session reuses the entire
+     * realtime contract — only the media frames gain a `video` kind. Resolution prefers a video-capable
+     * model when an agent requests video, and degrades to audio-only otherwise.
+     *
+     * @returns `true` if sessions can carry video; `false` (audio-only) otherwise.
+     */
+    get SupportsVideo(): boolean;
+    /**
+     * The provider-native voice ids this model can speak with (e.g. OpenAI `alloy`/`echo`/`shimmer`). The
+     * model/driver is the authoritative owner of "what voices do I support", so each driver declares its
+     * own — used to populate the dev voice picker. Default empty (a driver that hasn't declared voices
+     * yields no picker options, falling back to the configured/default voice).
+     *
+     * NOTE: this is the near-term, driver-owned source of truth. Long term this should move to metadata so
+     * providers that let users add their OWN voices (e.g. ElevenLabs) can be enumerated dynamically.
+     *
+     * @returns The supported voice ids (id + human label), or `[]` when none are declared.
+     */
+    get SupportedVoices(): RealtimeVoiceOption[];
+}
+/** A selectable provider-native voice — `ID` is sent to the provider, `Name` is the human label. */
+export interface RealtimeVoiceOption {
+    /** The provider-native voice id (e.g. `echo`) — what gets written to the session config. */
+    ID: string;
+    /** The human-friendly label for the picker (e.g. `Echo`). */
+    Name: string;
 }
+/**
+ * The media plane a realtime frame belongs to. The realtime contract is otherwise media-agnostic — a
+ * `video` session reuses every method (tools, transcript, usage, turn-taking); only the media frames
+ * carry this tag so audio and video can be disambiguated on the same session.
+ */
+export type RealtimeMediaKind = 'audio' | 'video';
 /**
  * The server-minted configuration a browser needs to open a **client-direct** realtime session.
  *
@@ -165,6 +217,24 @@ export interface ClientRealtimeSessionConfig {
      */
     SessionConfig: JSONObject;
 }
+/**
+ * Static capability flags of a live {@link IRealtimeSession}, for container introspection (the realtime-
+ * session analogue of `IBridgeProviderFeatures`). Grow this as providers gain runtime abilities — each new
+ * flag defaults to "unsupported" for any driver that hasn't declared it, so the container stays safe.
+ */
+export interface RealtimeSessionCapabilities {
+    /**
+     * Whether the session can change its turn-taking / auto-response mode on a **live** socket (no
+     * reconnect) via {@link IRealtimeSession.Reconfigure}. `true` for providers with a runtime-mutable
+     * session config (OpenAI `session.update`); `false` where it's fixed at connect (Gemini Live).
+     */
+    CanReconfigureTurnMode: boolean;
+}
+/** Parameters for {@link IRealtimeSession.Reconfigure} — a live turn-taking change. */
+export interface RealtimeReconfigureParams {
+    /** Switch the model's blind auto-response OFF (meeting mode) or ON (1:1). */
+    DisableAutoResponse?: boolean;
+}
 /**
  * A long-lived, full-duplex session handle returned by {@link BaseRealtimeModel.StartSession}.
  *
@@ -174,13 +244,31 @@ export interface ClientRealtimeSessionConfig {
  */
 export interface IRealtimeSession {
     /**
-     * Sends a client media frame to the model (audio now, video later).
+     * The PCM sample rate (Hz) this model **consumes** on {@link IRealtimeSession.SendInput} — its audio
+     * INPUT format. Optional; consumers default to 24000 (OpenAI Realtime). **Gemini Live = 16000.** A
+     * server-bridged host (LiveKit/Zoom/Teams) MUST resample inbound room audio to this rate or the model
+     * receives mis-rated audio it can't parse (the symptom: the agent never responds on the bridge while
+     * the same model works client-direct, where the browser negotiates the rate itself).
+     */
+    InputSampleRate?: number;
+    /**
+     * The PCM sample rate (Hz) this model **emits** on {@link IRealtimeSession.OnOutput} — its audio OUTPUT
+     * format. Optional; consumers default to 24000 (both OpenAI and Gemini Live emit 24 kHz today).
+     */
+    OutputSampleRate?: number;
+    /**
+     * Sends a client media frame to the model.
      *
-     * Fire-and-forget: frames are streamed straight to the provider with no JSON intermediation.
+     * Fire-and-forget: frames are streamed straight to the provider with no JSON intermediation. The
+     * optional `kind` tags the media plane — `'audio'` (default, back-compatible: existing callers and
+     * audio-only drivers need not pass or read it) or `'video'` for a camera frame to a video-capable
+     * model (one that {@link BaseRealtimeModel.SupportsVideo}). Audio-only drivers ignore `'video'`
+     * frames.
      *
      * @param chunk A raw media frame as an `ArrayBuffer`.
+     * @param kind The media plane the frame belongs to. Defaults to `'audio'`.
      */
-    SendInput(chunk: ArrayBuffer): void;
+    SendInput(chunk: ArrayBuffer, kind?: RealtimeMediaKind): void;
     /**
      * Registers the set of tools the model may call, translating them into the provider's
      * native function-calling format.
@@ -207,11 +295,23 @@ export interface IRealtimeSession {
      */
     RegisterTools(tools: RealtimeToolDefinition[]): Promise<void>;
     /**
-     * Registers a handler for model media output frames (the media plane).
+     * Registers a handler for model **audio** output frames (the audio media plane).
      *
-     * @param handler Invoked with each output media frame as an `ArrayBuffer`.
+     * @param handler Invoked with each output audio frame as an `ArrayBuffer`.
      */
     OnOutput(handler: (chunk: ArrayBuffer) => void): void;
+    /**
+     * Registers a handler for model **video** output frames — the talking-head avatar / generated
+     * video a video-capable model emits, in sync with {@link IRealtimeSession.OnOutput}'s audio.
+     *
+     * Optional: audio-only drivers (the default) don't implement it, and consumers must call it
+     * null-safely (`session.OnVideoOutput?.(...)`). A video-capable driver
+     * ({@link BaseRealtimeModel.SupportsVideo}) implements it; the consumer (bridge / client) maps these
+     * frames onto its `video-out` track exactly as it maps audio.
+     *
+     * @param handler Invoked with each output video frame as an `ArrayBuffer`.
+     */
+    OnVideoOutput?(handler: (chunk: ArrayBuffer) => void): void;
     /**
      * Registers a handler for transcript events (the text stream).
      *
@@ -277,8 +377,32 @@ export interface IRealtimeSession {
      * mid-session omit the member, and callers must feature-detect before invoking.
      *
      * @param instructions Instructions for the single spoken update (tone, brevity, content).
+     * @returns `true` when a response was actually triggered, `false` when it was skipped (e.g. a response
+     *   is already in flight). A bridge that claimed the speaking floor for this turn uses this to release the
+     *   floor immediately on a skip — otherwise a skipped trigger would wedge the room until the safety timer.
+     *   `void`/`undefined` from legacy drivers is treated as "triggered" for backward compatibility.
+     */
+    RequestSpokenUpdate?(instructions: string): boolean | void;
+    /**
+     * **Capability introspection.** A small, static description of what THIS live session can do, so the
+     * container can ask "is it safe to call X?" instead of invoking optional methods that silently no-op (or
+     * can't be supported) on some providers — the same role `IBridgeProviderFeatures` plays for bridges and
+     * {@link BaseRealtimeModel.SupportsClientDirect} plays for minting. Optional: a driver that hasn't
+     * declared its capabilities is treated **conservatively** (everything unsupported). As models gain
+     * abilities, drivers just flip a flag — no container changes.
+     */
+    Capabilities?: RealtimeSessionCapabilities;
+    /**
+     * **Optional capability** (gate on {@link RealtimeSessionCapabilities.CanReconfigureTurnMode}) —
+     * reconfigures a **live** session's turn-taking without reconnecting: e.g. switch a 1:1 agent to
+     * meeting mode (auto-response off) when its room becomes multi-agent. Providers whose runtime config is
+     * mutable mid-socket (OpenAI: `session.update`) implement this and report the capability `true`;
+     * providers whose turn config is fixed at connect (Gemini Live's activity detection) report `false` and
+     * omit the method. The container **must** check the capability before calling — never blind-invoke.
+     *
+     * @param params The reconfiguration to apply (e.g. `DisableAutoResponse`).
      */
-    RequestSpokenUpdate?(instructions: string): void;
+    Reconfigure?(params: RealtimeReconfigureParams): void;
     /**
      * Registers a handler for provider-detected interruptions (barge-in).
      *

package/dist/generic/baseRealtime.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"baseRealtime.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC;;;GAGG;AACH,MAAM,MAAM,SAAS,GACf,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,8BAAsB,iBAAkB,SAAQ,SAAS;IACrD;;;;;;;;;;OAUG;aACa,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAEtF;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACU,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;CAGzG;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,2BAA2B;IACxC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,aAAa,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAEpC;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;IAEtD;;;;;;;OAOG;IACH,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE7D;;;;;;;OAOG;IACH,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE5D;;;;;;;;OAQG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAErC;;;;;;;;;;;;;;;;;OAiBG;IACH,mBAAmB,CAAC,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjD;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAE1C;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAEpC;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAEnD;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,KAAK,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEjC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAE3B;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACjC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAEhB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACnC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,gBAAgB,EAAE,UAAU,CAAC;CAChC"}
+{"version":3,"file":"baseRealtime.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC;;;GAGG;AACH,MAAM,MAAM,SAAS,GACf,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAEH;;;;;;;;;;GAUG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAMrD;AAED,8BAAsB,iBAAkB,SAAQ,SAAS;IACrD;;;;;;;;;;OAUG;aACa,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAEtF;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACU,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;IAItG;;;;;;;;;;;;;OAaG;IACH,IAAW,aAAa,IAAI,OAAO,CAElC;IAED;;;;;;;;;;OAUG;IACH,IAAW,eAAe,IAAI,mBAAmB,EAAE,CAElD;CACJ;AAED,oGAAoG;AACpG,MAAM,WAAW,mBAAmB;IAChC,4FAA4F;IAC5F,EAAE,EAAE,MAAM,CAAC;IACX,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,OAAO,CAAC;AAElD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,2BAA2B;IACxC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,aAAa,EAAE,UAAU,CAAC;CAC7B;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IACxC;;;;OAIG;IACH,sBAAsB,EAAE,OAAO,CAAC;CACnC;AAED,uFAAuF;AACvF,MAAM,WAAW,yBAAyB;IACtC,6EAA6E;IAC7E,mBAAmB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;IAEtD;;;;;;;;;;OAUG;IACH,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;IAE5D;;;;;;;OAOG;IACH,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE7D;;;;;;;OAOG;IACH,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE5D;;;;;;;;OAQG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAErC;;;;;;;;;;;;;;;;;;;;;OAqBG;IACH,mBAAmB,CAAC,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;IAE3D;;;;;;;OAOG;IACH,YAAY,CAAC,EAAE,2BAA2B,CAAC;IAE3C;;;;;;;;;OASG;IACH,WAAW,CAAC,CAAC,MAAM,EAAE,yBAAyB,GAAG,IAAI,CAAC;IAEtD;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAE1C;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAEpC;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAEnD;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,KAAK,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEjC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAE3B;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACjC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAEhB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACnC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,gBAAgB,EAAE,UAAU,CAAC;CAChC"}

package/dist/generic/baseRealtime.js

@@ -61,6 +61,24 @@ import { BaseModel } from "./baseModel.js";
  *
  * @abstract
  */
+/**
+ * Emits a realtime-driver diagnostic line, but ONLY when verbose logging is enabled
+ * (`MJ_VERBOSE=true|1|yes`). These traces — turn boundaries, activity windows, response gating,
+ * barge-in — are invaluable when debugging a live session but far too chatty for normal operation,
+ * so they stay dark unless verbose mode is explicitly turned on. Shared here so every realtime driver
+ * and its session twin (OpenAI, Gemini, …) gate diagnostics through one consistent switch. The truthy
+ * set matches `@memberjunction/core`'s `IsVerboseLoggingEnabled` so a single `MJ_VERBOSE` flag governs
+ * verbose output across the whole stack.
+ *
+ * @param message The diagnostic message, already prefixed by the caller (e.g. `[GeminiRealtime][diag] …`).
+ */
+export function RealtimeDiagLog(message) {
+    const v = (process.env.MJ_VERBOSE ?? '').toLowerCase();
+    if (v === 'true' || v === '1' || v === 'yes') {
+        // eslint-disable-next-line no-console
+        console.log(message);
+    }
+}
 export class BaseRealtimeModel extends BaseModel {
     /**
      * Whether this driver can mint an ephemeral, server-scoped client credential for a
@@ -99,5 +117,36 @@ export class BaseRealtimeModel extends BaseModel {
     async CreateClientSession(_params) {
         throw new Error(`${this.constructor.name} does not support client-direct realtime sessions`);
     }
+    /**
+     * Whether this driver's sessions carry a **video** track in addition to audio — i.e. the model
+     * accepts video input (it can "see" the user's camera) and/or emits video output (a talking-head
+     * avatar / generated video), in sync with audio.
+     *
+     * Defaults to `false` (audio-only — today's realtime models). Video-capable drivers (a native
+     * multimodal realtime model, or an avatar provider) override this to `true`. The session's media
+     * plane is media-tagged ({@link IRealtimeSession.SendInput} takes a {@link RealtimeMediaKind};
+     * {@link IRealtimeSession.OnVideoOutput} delivers video-out), so a video session reuses the entire
+     * realtime contract — only the media frames gain a `video` kind. Resolution prefers a video-capable
+     * model when an agent requests video, and degrades to audio-only otherwise.
+     *
+     * @returns `true` if sessions can carry video; `false` (audio-only) otherwise.
+     */
+    get SupportsVideo() {
+        return false;
+    }
+    /**
+     * The provider-native voice ids this model can speak with (e.g. OpenAI `alloy`/`echo`/`shimmer`). The
+     * model/driver is the authoritative owner of "what voices do I support", so each driver declares its
+     * own — used to populate the dev voice picker. Default empty (a driver that hasn't declared voices
+     * yields no picker options, falling back to the configured/default voice).
+     *
+     * NOTE: this is the near-term, driver-owned source of truth. Long term this should move to metadata so
+     * providers that let users add their OWN voices (e.g. ElevenLabs) can be enumerated dynamically.
+     *
+     * @returns The supported voice ids (id + human label), or `[]` when none are declared.
+     */
+    get SupportedVoices() {
+        return [];
+    }
 }
 //# sourceMappingURL=baseRealtime.js.map

package/dist/generic/baseRealtime.js.map

@@ -1 +1 @@
-{"version":3,"file":"baseRealtime.js","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAoBxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,MAAM,OAAgB,iBAAkB,SAAQ,SAAS;IAcrD;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB;QAC3B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,KAAK,CAAC,mBAAmB,CAAC,OAA8B;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,mDAAmD,CAAC,CAAC;IACjG,CAAC;CACJ"}
+{"version":3,"file":"baseRealtime.js","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAoBxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AAEH;;;;;;;;;;GAUG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC3C,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACvD,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;QAC3C,sCAAsC;QACtC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;AACL,CAAC;AAED,MAAM,OAAgB,iBAAkB,SAAQ,SAAS;IAcrD;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB;QAC3B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,KAAK,CAAC,mBAAmB,CAAC,OAA8B;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,mDAAmD,CAAC,CAAC;IACjG,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,IAAW,aAAa;QACpB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;OAUG;IACH,IAAW,eAAe;QACtB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@memberjunction/ai",
   "type": "module",
-  "version": "5.41.0",
+  "version": "5.43.0",
   "description": "MemberJunction: AI - core components for abstracting LLMs and other AI model types that are usable anywhere without ANY other MJ dependencies past @memberjunction/global which itself has zero additional dependencies.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -17,7 +17,7 @@
   "author": "MemberJunction.com",
   "license": "ISC",
   "dependencies": {
-    "@memberjunction/global": "5.41.0",
+    "@memberjunction/global": "5.43.0",
     "dotenv": "^17.2.4",
     "rxjs": "^7.8.2"
   },