@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/media/model.d.ts

@@ -0,0 +1,154 @@
+/**
+ * @fileoverview Shared types for the `/media` entry point: the local devices and capture state that the
+ * media controllers and the UI exchange.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** A kind of local device the user captures from. */
+export type LocalMediaKind = 'camera' | 'microphone';
+/** What a device does: a camera or a microphone the user captures from, or a speaker that plays sound. */
+export type MediaDeviceKind = LocalMediaKind | 'speaker';
+/** One device. */
+export interface MediaDevice {
+    /** The browser's id for the device, stable for this site until the user clears its data. */
+    DeviceID: string;
+    Kind: MediaDeviceKind;
+    /** The device's name. Browsers leave it empty until the user has allowed this site that kind of device. */
+    Label: string;
+    /** Devices on the same physical product (a webcam's camera and microphone) share a group id. */
+    GroupID: string;
+}
+/** Why a local device could not be used. */
+export type LocalMediaFailure = 
+/** The user, the operating system or the page's permissions policy refused access. */
+'denied'
+/** There is no device of that kind, or not the one asked for. */
+ | 'not-found'
+/** The device exists, but another application is using it. */
+ | 'in-use'
+/** The browser cannot capture at all (no `getUserMedia`). */
+ | 'unsupported'
+/** Anything else; see the message. */
+ | 'error';
+/** One kind of local capture. */
+export interface LocalTrackState {
+    Status: 'off' | 'starting' | 'on' | 'failed';
+    /** The device in use, while `on`. */
+    DeviceID?: string;
+    /** That device's name, while `on`. */
+    Label?: string;
+    /** Why capture failed, while `failed`. */
+    Failure?: LocalMediaFailure;
+    /** The browser's message for the failure. */
+    Message?: string;
+}
+/** Everything captured locally, as the `LocalMediaController` reports it. */
+export interface LocalMediaState {
+    Camera: LocalTrackState;
+    Microphone: LocalTrackState;
+    /** The input devices the browser lists now. */
+    Devices: MediaDevice[];
+}
+/** The outcome of starting or switching a local device. It always resolves; it never rejects. */
+export type LocalMediaResult = {
+    Status: 'started';
+    Stream: MediaStream;
+} | {
+    Status: 'failed';
+    Reason: LocalMediaFailure;
+    Message: string;
+};
+/**
+ * A source of video to show: a live `MediaStream` (a camera, a shared screen, a WebRTC track), or a player that
+ * must own the `<video>` element (MSE or WebCodecs playout of encoded video, such as an avatar). Renderers show
+ * either with `AttachVideoSource`.
+ */
+export type MediaVideoSource = 
+/** A live stream, shown muted: its audio is played elsewhere. */
+{
+    Kind: 'stream';
+    Stream: MediaStream;
+}
+/** A player that takes over the element and returns a function that releases it. */
+ | {
+    Kind: 'element';
+    Attach(element: HTMLVideoElement): () => void;
+};
+/**
+ * What a participant's video shows. Open vocabulary. These are on-screen kinds; the video source arbiter's
+ * `VideoSourceKind` is a different list, of what the model is shown.
+ */
+export type MediaSourceKind = 'camera' | 'screen' | 'avatar' | (string & {});
+/** Where a surface appears: the main stage, a picture-in-picture tile, a tab, or nowhere. */
+export type MediaPlacement = 'stage' | 'pip' | 'tab' | 'hidden';
+/** Every placement, in the order a displaced surface falls back through them. */
+export declare const MEDIA_PLACEMENTS: readonly MediaPlacement[];
+/** One person or agent in a session, as the media stage and its tiles see them. */
+export interface MediaParticipant {
+    /** Stable identity within the session. */
+    Identity: string;
+    DisplayName: string;
+    /** `'self'` is the local user. */
+    Role: 'self' | 'agent' | 'host' | 'participant';
+    /** Whether the participant is speaking now. */
+    IsSpeaking: boolean;
+    /** The participant's videos, by kind: a camera, a shared screen, an avatar. */
+    Video: Partial<Record<MediaSourceKind, MediaVideoSource>>;
+    /** Which video a tile shows when there are several. Defaults to a shared screen, then the camera, then the avatar. */
+    PreferredVideo?: MediaSourceKind;
+    /** Whether the participant's microphone is off. */
+    IsMuted?: boolean;
+    /** The connection's quality, when the platform reports one. */
+    ConnectionQuality?: MediaConnectionQuality;
+    /** Reads the participant's current audio level, 0..1, for a meter. Called on every animation frame. */
+    GetAudioLevel?: () => number;
+    /**
+     * Whether an agent can see this participant's camera and shared screen: they allowed it, and an agent is there to
+     * watch. A tile shows "Agent can see". Absent means no.
+     */
+    AgentCanSee?: boolean;
+}
+/** A participant's connection quality. */
+export type MediaConnectionQuality = 'excellent' | 'good' | 'poor' | 'lost' | 'unknown';
+/** The agent's conversational state, for an indicator. */
+export type MediaAgentState = 'idle' | 'listening' | 'thinking' | 'speaking';
+/** The state of a session's connection, for an overlay. */
+export type MediaConnectionStatus = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'disconnected' | 'error';
+/** Why a session ended. */
+export type MediaDisconnectReason = 'client-initiated' | 'server-shutdown' | 'participant-removed' | 'room-deleted' | 'connection-lost' | 'duplicate-identity' | 'unknown';
+/** A device the user picked in a device menu. */
+export interface MediaDeviceSelection {
+    Kind: MediaDeviceKind;
+    DeviceID: string;
+}
+/**
+ * Something that can be placed: a channel's surface (a whiteboard, a remote browser) or a participant's video
+ * (the avatar, the user's camera, a shared screen).
+ */
+export interface MediaSurface {
+    /** Stable key, usually the channel key. The user's moves are recorded against it. */
+    Key: string;
+    Label: string;
+    /** Where it goes unless the user moved it. */
+    DefaultPlacement: MediaPlacement;
+    /** Where the user may move it. Defaults to every placement. */
+    AllowedPlacements?: readonly MediaPlacement[];
+    /** The participant video it shows, when it is one; absent for a component surface such as a whiteboard. */
+    Video?: {
+        ParticipantIdentity: string;
+        Kind: MediaSourceKind;
+    };
+}
+/** One move the user made: a surface to a placement. */
+export interface MediaPlacementMove {
+    SurfaceKey: string;
+    Placement: MediaPlacement;
+}
+/** A picture-in-picture box as fractions of the stage (0 to 1), so it survives a resized window: what a host saves. */
+export interface MediaPipRect {
+    X: number;
+    Y: number;
+    W: number;
+    H: number;
+}
+//# sourceMappingURL=model.d.ts.map

package/dist/media/model.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../src/media/model.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,qDAAqD;AACrD,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,YAAY,CAAC;AAErD,0GAA0G;AAC1G,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,SAAS,CAAC;AAEzD,kBAAkB;AAClB,MAAM,WAAW,WAAW;IACxB,4FAA4F;IAC5F,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,eAAe,CAAC;IACtB,2GAA2G;IAC3G,KAAK,EAAE,MAAM,CAAC;IACd,gGAAgG;IAChG,OAAO,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB;AACzB,sFAAsF;AACpF,QAAQ;AACV,iEAAiE;GAC/D,WAAW;AACb,8DAA8D;GAC5D,QAAQ;AACV,6DAA6D;GAC3D,aAAa;AACf,sCAAsC;GACpC,OAAO,CAAC;AAEd,iCAAiC;AACjC,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,KAAK,GAAG,UAAU,GAAG,IAAI,GAAG,QAAQ,CAAC;IAC7C,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,OAAO,CAAC,EAAE,iBAAiB,CAAC;IAC5B,6CAA6C;IAC7C,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,eAAe,CAAC;IACxB,UAAU,EAAE,eAAe,CAAC;IAC5B,+CAA+C;IAC/C,OAAO,EAAE,WAAW,EAAE,CAAC;CAC1B;AAED,iGAAiG;AACjG,MAAM,MAAM,gBAAgB,GACtB;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE,GAC1C;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvE;;;;GAIG;AACH,MAAM,MAAM,gBAAgB;AACxB,iEAAiE;AAC/D;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,WAAW,CAAA;CAAE;AACzC,oFAAoF;GAClF;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,MAAM,IAAI,CAAA;CAAE,CAAC;AAEzE;;;GAGG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAE7E,6FAA6F;AAC7F,MAAM,MAAM,cAAc,GAAG,OAAO,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,CAAC;AAEhE,iFAAiF;AACjF,eAAO,MAAM,gBAAgB,EAAE,SAAS,cAAc,EAAsC,CAAC;AAE7F,mFAAmF;AACnF,MAAM,WAAW,gBAAgB;IAC7B,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,CAAC;IAChD,+CAA+C;IAC/C,UAAU,EAAE,OAAO,CAAC;IACpB,+EAA+E;IAC/E,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAC,CAAC;IAC1D,sHAAsH;IACtH,cAAc,CAAC,EAAE,eAAe,CAAC;IACjC,mDAAmD;IACnD,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,sBAAsB,CAAC;IAC3C,uGAAuG;IACvG,aAAa,CAAC,EAAE,MAAM,MAAM,CAAC;IAC7B;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;CACzB;AAED,0CAA0C;AAC1C,MAAM,MAAM,sBAAsB,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAExF,0DAA0D;AAC1D,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,UAAU,CAAC;AAE7E,2DAA2D;AAC3D,MAAM,MAAM,qBAAqB,GAAG,MAAM,GAAG,YAAY,GAAG,WAAW,GAAG,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC;AAEpH,2BAA2B;AAC3B,MAAM,MAAM,qBAAqB,GAC3B,kBAAkB,GAClB,iBAAiB,GACjB,qBAAqB,GACrB,cAAc,GACd,iBAAiB,GACjB,oBAAoB,GACpB,SAAS,CAAC;AAEhB,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,eAAe,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,MAAM,WAAW,YAAY;IACzB,qFAAqF;IACrF,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,8CAA8C;IAC9C,gBAAgB,EAAE,cAAc,CAAC;IACjC,+DAA+D;IAC/D,iBAAiB,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C,2GAA2G;IAC3G,KAAK,CAAC,EAAE;QAAE,mBAAmB,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,eAAe,CAAA;KAAE,CAAC;CAClE;AAED,wDAAwD;AACxD,MAAM,WAAW,kBAAkB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,cAAc,CAAC;CAC7B;AAED,uHAAuH;AACvH,MAAM,WAAW,YAAY;IACzB,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;IACV,CAAC,EAAE,MAAM,CAAC;CACb"}

package/dist/media/model.js

@@ -0,0 +1,9 @@
+/**
+ * @fileoverview Shared types for the `/media` entry point: the local devices and capture state that the
+ * media controllers and the UI exchange.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** Every placement, in the order a displaced surface falls back through them. */
+export const MEDIA_PLACEMENTS = ['stage', 'pip', 'tab', 'hidden'];
+//# sourceMappingURL=model.js.map

package/dist/media/model.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/media/model.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AA8EH,iFAAiF;AACjF,MAAM,CAAC,MAAM,gBAAgB,GAA8B,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC"}

package/dist/media/playbackClock.d.ts

@@ -0,0 +1,20 @@
+/**
+ * @fileoverview PLAYBACK CLOCK: where the voice is, so a video player can show each frame when the voice reaches it and
+ * the face moves with the voice.
+ *
+ * The voice is the master clock, as in the meeting bot: a driver that times both its PCM and its video frames on one media
+ * timeline gives the player its PCM playback (`RealtimePcmPlayback`) as `VideoPlayoutOptions.Clock`. The chunk and image
+ * decoders then show a frame when the clock reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the
+ * voice plays voice and face together on its own timestamps.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** Where the voice is: the clock a video player shows frames against, so the face moves with the voice. */
+export interface IPlaybackClock {
+    /**
+     * The media time, in milliseconds on the stream's timeline, of the audio heard now; `null` while no timed audio plays
+     * (nothing is queued, or the audio queued carries no media times).
+     */
+    readonly CurrentTimeMs: number | null;
+}
+//# sourceMappingURL=playbackClock.d.ts.map

package/dist/media/playbackClock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbackClock.d.ts","sourceRoot":"","sources":["../../src/media/playbackClock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,2GAA2G;AAC3G,MAAM,WAAW,cAAc;IAC3B;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC"}

package/dist/media/playbackClock.js

@@ -0,0 +1,13 @@
+/**
+ * @fileoverview PLAYBACK CLOCK: where the voice is, so a video player can show each frame when the voice reaches it and
+ * the face moves with the voice.
+ *
+ * The voice is the master clock, as in the meeting bot: a driver that times both its PCM and its video frames on one media
+ * timeline gives the player its PCM playback (`RealtimePcmPlayback`) as `VideoPlayoutOptions.Clock`. The chunk and image
+ * decoders then show a frame when the clock reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the
+ * voice plays voice and face together on its own timestamps.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+export {};
+//# sourceMappingURL=playbackClock.js.map

package/dist/media/playbackClock.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbackClock.js","sourceRoot":"","sources":["../../src/media/playbackClock.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG"}

package/dist/media/videoFrameDecoder.d.ts

@@ -0,0 +1,86 @@
+/**
+ * @fileoverview VIDEO FRAME DECODERS: what plays one kind of `RealtimeVideoFrame` into a `<video>` element for
+ * {@link VideoPlayout}, and how a decoder is offered to the player.
+ *
+ * The player keeps one decoder at a time. For each frame it picks, from the {@link VideoFrameDecoderRegistry}, the first
+ * registration of the frame's kind that can play its MIME type, and hands the decoder a {@link VideoFrameDecoderContext}:
+ * the player's settings and where to report. A decoder plays into the host's element, whatever it decodes with, so the
+ * host shows every kind of video the same way.
+ *
+ * Built in: `'mse-fmp4'` (fragmented MP4 through Media Source Extensions), `'webcodecs'` (encoded chunks through a
+ * WebCodecs `VideoDecoder`) and `'image'` (still images through `createImageBitmap`).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { RealtimeVideoFrame, RealtimeVideoFrameKind } from '@memberjunction/ai';
+import type { IPlaybackClock } from './playbackClock.js';
+import type { VideoPlayoutProblem } from './videoPlayout.js';
+/** What a decoder plays with and reports through. The player gives one to each decoder it creates. */
+export interface VideoFrameDecoderContext {
+    /**
+     * The player's configured type (`VideoPlayoutOptions.MimeType`), with its codecs: what a decoder that opens by type
+     * (MSE) opens with until a frame says more.
+     */
+    readonly MimeType: string;
+    /** Seconds of played media a buffering decoder keeps behind the playhead (`VideoPlayoutOptions.BackBufferSeconds`). */
+    readonly BackBufferSeconds: number;
+    /** Whether the element plays the media's own audio (an avatar MP4's voice). The player keeps it current. */
+    readonly CarriesVoice: boolean;
+    /**
+     * The element hook from `VideoPlayoutOptions.OnElementAttached`: the voice's route into Web Audio. A decoder whose media
+     * carries audio calls it with each element it takes over, before playback starts.
+     */
+    OnElementAttached?(element: HTMLVideoElement): void;
+    /**
+     * The voice's playback clock (`VideoPlayoutOptions.Clock`), when the player has one: a decoder that paces its own
+     * frames shows each when the clock reaches its `PresentationTimeMs`. A decoder whose media carries the voice ignores it.
+     */
+    readonly Clock?: IPlaybackClock;
+    /** Reports a problem. The player passes each kind on once. */
+    Report(problem: VideoPlayoutProblem, message: string): void;
+    /**
+     * The decoder can't go on (its browser decoder closed for good, or the browser can't decode the stream's codec). The
+     * player disposes it and picks the next decoder that can play the frame's type; out of decoders, frames of that type are
+     * dropped.
+     */
+    Failed(message: string): void;
+}
+/** Plays one kind of frame into a `<video>` element. The player creates one at a time and moves it between elements. */
+export interface IVideoFrameDecoder {
+    /** Takes the element over and shows its video there, keeping the frames it holds and the last one shown. */
+    Attach(element: HTMLVideoElement): void;
+    /** Lets the element go (the tile moved or closed). The decoder keeps what it holds for the next element. */
+    Detach(): void;
+    /** Takes one frame of its kind, in the order they arrived. */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last frame has arrived: everything that came plays to the end, then the last frame holds. */
+    EndOfTurn(): void;
+    /** Barge-in: drops what isn't shown yet. The last frame shown stays on screen. */
+    Flush(): void;
+    /** Whether the decoder's own audio is audibly playing (an MP4 that carries the voice); `false` for decoders without audio. */
+    readonly IsPlaying: boolean;
+    /**
+     * How many of the frames it took are still to play: waiting, being decoded or appended, or buffered ahead of the
+     * playhead. A frame it dropped (a barge-in's flush, an overflow, a decode error) or played past is not counted.
+     */
+    readonly FramesAhead: number;
+    /** Stops, releases the element and frees what it holds. */
+    Dispose(): void;
+}
+/** One decoder the registry can create. */
+export interface VideoFrameDecoderRegistration {
+    /** Its name, unique in the registry: registering the same name again replaces it. Built in: `'mse-fmp4'`, `'webcodecs'`, `'image'`. */
+    Name: string;
+    /** The kind of frame it plays. */
+    Kind: RealtimeVideoFrameKind;
+    /** Among the decoders that can play a type, a higher priority is tried first. The built-ins have 0. */
+    Priority: number;
+    /**
+     * Whether it can play frames of this MIME type in this browser, answered at once. A decoder whose exact answer needs an
+     * asynchronous check answers from what it can see now, and calls `Failed` later if the check says no.
+     */
+    CanPlay(mimeType: string): boolean;
+    /** A new decoder, for one player. */
+    Create(context: VideoFrameDecoderContext): IVideoFrameDecoder;
+}
+//# sourceMappingURL=videoFrameDecoder.d.ts.map

package/dist/media/videoFrameDecoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoder.d.ts","sourceRoot":"","sources":["../../src/media/videoFrameDecoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AACH,OAAO,KAAK,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAE1D,sGAAsG;AACtG,MAAM,WAAW,wBAAwB;IACrC;;;OAGG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,uHAAuH;IACvH,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;IACnC,4GAA4G;IAC5G,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B;;;OAGG;IACH,iBAAiB,CAAC,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACpD;;;OAGG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC;IAChC,8DAA8D;IAC9D,MAAM,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5D;;;;OAIG;IACH,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,wHAAwH;AACxH,MAAM,WAAW,kBAAkB;IAC/B,4GAA4G;IAC5G,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACxC,4GAA4G;IAC5G,MAAM,IAAI,IAAI,CAAC;IACf,8DAA8D;IAC9D,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,2GAA2G;IAC3G,SAAS,IAAI,IAAI,CAAC;IAClB,kFAAkF;IAClF,KAAK,IAAI,IAAI,CAAC;IACd,8HAA8H;IAC9H,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,2DAA2D;IAC3D,OAAO,IAAI,IAAI,CAAC;CACnB;AAED,2CAA2C;AAC3C,MAAM,WAAW,6BAA6B;IAC1C,uIAAuI;IACvI,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,IAAI,EAAE,sBAAsB,CAAC;IAC7B,uGAAuG;IACvG,QAAQ,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,qCAAqC;IACrC,MAAM,CAAC,OAAO,EAAE,wBAAwB,GAAG,kBAAkB,CAAC;CACjE"}

package/dist/media/videoFrameDecoder.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=videoFrameDecoder.js.map

package/dist/media/videoFrameDecoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoder.js","sourceRoot":"","sources":["../../src/media/videoFrameDecoder.ts"],"names":[],"mappings":""}

package/dist/media/videoFrameDecoderRegistry.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @fileoverview VIDEO FRAME DECODER REGISTRY: the page's video frame decoders, which `VideoPlayout` chooses from for each
+ * frame type. The three built-in decoders (`'mse-fmp4'`, `'webcodecs'`, `'image'`) are registered when the registry is
+ * first used; an app adds its own with {@link VideoFrameDecoderRegistry.Register}, or replaces a built-in by registering
+ * the same name.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { BaseSingleton } from '@memberjunction/global';
+import type { RealtimeVideoFrameKind } from '@memberjunction/ai';
+import type { VideoFrameDecoderRegistration } from './videoFrameDecoder.js';
+/** The page's video frame decoders, by name. */
+export declare class VideoFrameDecoderRegistry extends BaseSingleton<VideoFrameDecoderRegistry> {
+    private readonly registrations;
+    protected constructor();
+    /** The page's registry, with the built-in decoders registered. */
+    static get Instance(): VideoFrameDecoderRegistry;
+    /**
+     * Adds a decoder. One with the same name is replaced; players that already use the old one keep it until they choose a
+     * decoder again.
+     *
+     * @param registration The decoder: its name, frame kind, priority, support check and factory.
+     */
+    Register(registration: VideoFrameDecoderRegistration): void;
+    /**
+     * Removes a decoder by name. Players that already use it keep it until they choose a decoder again.
+     *
+     * @param name The decoder's name.
+     */
+    Unregister(name: string): void;
+    /**
+     * The decoders that can play frames of `kind` and `mimeType` in this browser, highest priority first (in registration
+     * order among equals). A decoder whose support check throws is left out.
+     *
+     * @param kind The frame kind.
+     * @param mimeType The frame's MIME type.
+     */
+    Candidates(kind: RealtimeVideoFrameKind, mimeType: string): readonly VideoFrameDecoderRegistration[];
+}
+//# sourceMappingURL=videoFrameDecoderRegistry.d.ts.map

package/dist/media/videoFrameDecoderRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoderRegistry.d.ts","sourceRoot":"","sources":["../../src/media/videoFrameDecoderRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAQzE,gDAAgD;AAChD,qBAAa,yBAA0B,SAAQ,aAAa,CAAC,yBAAyB,CAAC;IACnF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoD;IAElF,SAAS;IAOT,kEAAkE;IAClE,WAAkB,QAAQ,IAAI,yBAAyB,CAEtD;IAED;;;;;OAKG;IACI,QAAQ,CAAC,YAAY,EAAE,6BAA6B,GAAG,IAAI;IAIlE;;;;OAIG;IACI,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIrC;;;;;;OAMG;IACI,UAAU,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,6BAA6B,EAAE;CAK9G"}

package/dist/media/videoFrameDecoderRegistry.js

@@ -0,0 +1,69 @@
+/**
+ * @fileoverview VIDEO FRAME DECODER REGISTRY: the page's video frame decoders, which `VideoPlayout` chooses from for each
+ * frame type. The three built-in decoders (`'mse-fmp4'`, `'webcodecs'`, `'image'`) are registered when the registry is
+ * first used; an app adds its own with {@link VideoFrameDecoderRegistry.Register}, or replaces a built-in by registering
+ * the same name.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { BaseSingleton } from '@memberjunction/global';
+import { IMAGE_FRAME_DECODER } from './decoders/imageFrameDecoder.js';
+import { MSE_FMP4_DECODER } from './decoders/mseFmp4Decoder.js';
+import { WEBCODECS_CHUNK_DECODER } from './decoders/webCodecsChunkDecoder.js';
+/** The decoders every page starts with. */
+const BUILT_IN_DECODERS = [MSE_FMP4_DECODER, WEBCODECS_CHUNK_DECODER, IMAGE_FRAME_DECODER];
+/** The page's video frame decoders, by name. */
+export class VideoFrameDecoderRegistry extends BaseSingleton {
+    constructor() {
+        super();
+        this.registrations = new Map();
+        for (const registration of BUILT_IN_DECODERS) {
+            this.Register(registration);
+        }
+    }
+    /** The page's registry, with the built-in decoders registered. */
+    static get Instance() {
+        return super.getInstance();
+    }
+    /**
+     * Adds a decoder. One with the same name is replaced; players that already use the old one keep it until they choose a
+     * decoder again.
+     *
+     * @param registration The decoder: its name, frame kind, priority, support check and factory.
+     */
+    Register(registration) {
+        this.registrations.set(registration.Name, registration);
+    }
+    /**
+     * Removes a decoder by name. Players that already use it keep it until they choose a decoder again.
+     *
+     * @param name The decoder's name.
+     */
+    Unregister(name) {
+        this.registrations.delete(name);
+    }
+    /**
+     * The decoders that can play frames of `kind` and `mimeType` in this browser, highest priority first (in registration
+     * order among equals). A decoder whose support check throws is left out.
+     *
+     * @param kind The frame kind.
+     * @param mimeType The frame's MIME type.
+     */
+    Candidates(kind, mimeType) {
+        return [...this.registrations.values()]
+            .filter((registration) => registration.Kind === kind && canPlay(registration, mimeType))
+            .sort((a, b) => b.Priority - a.Priority);
+    }
+}
+/** A registration's support check, with a check that throws read as "no" (logged). */
+function canPlay(registration, mimeType) {
+    try {
+        return registration.CanPlay(mimeType);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        console.warn(`[VideoPlayout] The ${registration.Name} decoder's support check failed for ${mimeType}: ${reason}`);
+        return false;
+    }
+}
+//# sourceMappingURL=videoFrameDecoderRegistry.js.map

package/dist/media/videoFrameDecoderRegistry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoderRegistry.js","sourceRoot":"","sources":["../../src/media/videoFrameDecoderRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAGvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAE3E,2CAA2C;AAC3C,MAAM,iBAAiB,GAA6C,CAAC,gBAAgB,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,CAAC;AAErI,gDAAgD;AAChD,MAAM,OAAO,yBAA0B,SAAQ,aAAwC;IAGnF;QACI,KAAK,EAAE,CAAC;QAHK,kBAAa,GAAG,IAAI,GAAG,EAAyC,CAAC;QAI9E,KAAK,MAAM,YAAY,IAAI,iBAAiB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,kEAAkE;IAC3D,MAAM,KAAK,QAAQ;QACtB,OAAO,KAAK,CAAC,WAAW,EAA6B,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACI,QAAQ,CAAC,YAA2C;QACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACI,UAAU,CAAC,IAAY;QAC1B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;OAMG;IACI,UAAU,CAAC,IAA4B,EAAE,QAAgB;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;aAClC,MAAM,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;aACvF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;CACJ;AAED,sFAAsF;AACtF,SAAS,OAAO,CAAC,YAA2C,EAAE,QAAgB;IAC1E,IAAI,CAAC;QACD,OAAO,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,IAAI,CAAC,sBAAsB,YAAY,CAAC,IAAI,uCAAuC,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC;QAClH,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}

package/dist/media/videoPacing.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @fileoverview Frame pacing derived from the NEGOTIATED inbound video rate.
+ *
+ * The rate a model accepts is per-model data (`MaxInboundVideoRate` in the provider's profile table),
+ * carried into the live track descriptor by track negotiation. Every layer that paces frames — the
+ * source arbiter, the driver's own backstop — asks THIS module what that rate means in milliseconds, so
+ * there is one answer to "how often may a frame go out" rather than a constant per layer.
+ *
+ * The math itself lives in `@memberjunction/ai` (`realtimeVideoFrames`), shared with the meeting bot that also sends
+ * frames to the model; these names are thin wrappers over it.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+/**
+ * The share of the nominal frame interval a pacer enforces as a MINIMUM spacing. Pacers upstream of
+ * the driver run on timers that jitter; enforcing exactly `1000 / rate` would drop a frame that arrived a
+ * few milliseconds early. At 1 fps this is the 750 ms backstop the Gemini driver has always applied.
+ */
+export declare const VIDEO_FRAME_JITTER_HEADROOM = 0.75;
+/** The rate assumed when none was negotiated: 1 frame per second, the Live API ceiling today. */
+export declare const DEFAULT_INBOUND_VIDEO_RATE = 1;
+/**
+ * The nominal interval between frames, in milliseconds, for a negotiated rate (frames per second).
+ * An absent, zero, negative or non-finite rate falls back to {@link DEFAULT_INBOUND_VIDEO_RATE}.
+ */
+export declare function NominalVideoFrameIntervalMs(rate: number | undefined | null): number;
+/**
+ * The minimum spacing, in milliseconds, a pacer should enforce between two frames to the model: the
+ * nominal interval less jitter headroom ({@link VIDEO_FRAME_JITTER_HEADROOM}).
+ */
+export declare function MinVideoFrameSpacingMs(rate: number | undefined | null): number;
+//# sourceMappingURL=videoPacing.d.ts.map

package/dist/media/videoPacing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPacing.d.ts","sourceRoot":"","sources":["../../src/media/videoPacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,OAAuC,CAAC;AAEhF,iGAAiG;AACjG,eAAO,MAAM,0BAA0B,IAAsC,CAAC;AAE9E;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAEnF;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAE9E"}

package/dist/media/videoPacing.js

@@ -0,0 +1,37 @@
+/**
+ * @fileoverview Frame pacing derived from the NEGOTIATED inbound video rate.
+ *
+ * The rate a model accepts is per-model data (`MaxInboundVideoRate` in the provider's profile table),
+ * carried into the live track descriptor by track negotiation. Every layer that paces frames — the
+ * source arbiter, the driver's own backstop — asks THIS module what that rate means in milliseconds, so
+ * there is one answer to "how often may a frame go out" rather than a constant per layer.
+ *
+ * The math itself lives in `@memberjunction/ai` (`realtimeVideoFrames`), shared with the meeting bot that also sends
+ * frames to the model; these names are thin wrappers over it.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+import { REALTIME_DEFAULT_INBOUND_VIDEO_RATE, REALTIME_VIDEO_FRAME_JITTER_HEADROOM, RealtimeMinVideoFrameSpacingMs, RealtimeVideoFrameIntervalMs, } from '@memberjunction/ai';
+/**
+ * The share of the nominal frame interval a pacer enforces as a MINIMUM spacing. Pacers upstream of
+ * the driver run on timers that jitter; enforcing exactly `1000 / rate` would drop a frame that arrived a
+ * few milliseconds early. At 1 fps this is the 750 ms backstop the Gemini driver has always applied.
+ */
+export const VIDEO_FRAME_JITTER_HEADROOM = REALTIME_VIDEO_FRAME_JITTER_HEADROOM;
+/** The rate assumed when none was negotiated: 1 frame per second, the Live API ceiling today. */
+export const DEFAULT_INBOUND_VIDEO_RATE = REALTIME_DEFAULT_INBOUND_VIDEO_RATE;
+/**
+ * The nominal interval between frames, in milliseconds, for a negotiated rate (frames per second).
+ * An absent, zero, negative or non-finite rate falls back to {@link DEFAULT_INBOUND_VIDEO_RATE}.
+ */
+export function NominalVideoFrameIntervalMs(rate) {
+    return RealtimeVideoFrameIntervalMs(rate);
+}
+/**
+ * The minimum spacing, in milliseconds, a pacer should enforce between two frames to the model: the
+ * nominal interval less jitter headroom ({@link VIDEO_FRAME_JITTER_HEADROOM}).
+ */
+export function MinVideoFrameSpacingMs(rate) {
+    return RealtimeMinVideoFrameSpacingMs(rate);
+}
+//# sourceMappingURL=videoPacing.js.map

package/dist/media/videoPacing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPacing.js","sourceRoot":"","sources":["../../src/media/videoPacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACH,mCAAmC,EACnC,oCAAoC,EACpC,8BAA8B,EAC9B,4BAA4B,GAC/B,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,oCAAoC,CAAC;AAEhF,iGAAiG;AACjG,MAAM,CAAC,MAAM,0BAA0B,GAAG,mCAAmC,CAAC;AAE9E;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CAAC,IAA+B;IACvE,OAAO,4BAA4B,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAA+B;IAClE,OAAO,8BAA8B,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC"}