@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/media/videoPlayout.d.ts

@@ -0,0 +1,168 @@
+/**
+ * @fileoverview VIDEO PLAYOUT: plays a model's video into one `<video>` element at a time, whatever form it arrives in.
+ * A provider hands each frame to {@link VideoPlayout.Append} as a `RealtimeVideoFrame` from `@memberjunction/ai`; a
+ * renderer shows {@link VideoPlayout.Source} with `AttachVideoSource`.
+ *
+ * - **A decoder per frame type.** The player keeps one decoder at a time, chosen from the page's
+ *   {@link VideoFrameDecoderRegistry} by the frame's kind and MIME type: `'mse-fmp4'` plays fragmented MP4 (a Gemini Live
+ *   avatar) through Media Source Extensions, `'webcodecs'` plays encoded chunks (H.264, VP8, VP9, AV1) through WebCodecs,
+ *   `'image'` plays still images. A frame of another type switches decoders; a type no decoder plays is dropped and
+ *   reported once (`'no-decoder'`).
+ * - **Ready before the first frame.** When an element is attached first, the player starts the decoder for its
+ *   configured type ({@link VideoPlayoutOptions.MimeType}), so MSE has its media source open when the first piece comes.
+ *   A configured type no decoder plays reports `'unsupported'` and leaves the element alone.
+ * - **Fallback.** A decoder that gives up (`Failed`) is disposed and not chosen again for that type; the next one that can
+ *   play it starts at the next init segment or key frame. Out of decoders, frames of that type are dropped; the voice is
+ *   unaffected, and the tile's stall fallback shows in place of the video.
+ * - **The voice.** An avatar's MP4 carries its voice, so the MSE decoder unmutes the element and
+ *   {@link VideoPlayout.IsPlaying} says whether the agent is audibly speaking. When the voice plays elsewhere,
+ *   {@link VideoPlayout.CarriesVoice} `false` mutes the element. Chunk and image decoders carry no voice.
+ *   {@link VideoPlayoutOptions.OnElementAttached} lets the realtime client route the element's audio into its own Web
+ *   Audio graph, so its meter and its recording carry the avatar's voice.
+ * - **Lip sync.** With {@link VideoPlayoutOptions.Clock}, the voice's playback clock, the chunk and image decoders show
+ *   each frame when the voice reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the voice keeps
+ *   voice and face together on its own timestamps.
+ * - **End of turn, barge-in, gaps.** {@link VideoPlayout.EndOfTurn} lets playback run to the true end of the turn;
+ *   {@link VideoPlayout.Flush} drops everything not yet played; between turns, or while a connection resumes, the element
+ *   holds the last frame. Each decoder does this its own way.
+ * - **Browsers.** {@link VideoPlayout.IsSupported} says, at once, whether some decoder can play a type here.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { type RealtimeVideoFrame } from '@memberjunction/ai';
+import type { MediaVideoSource } from './model.js';
+import type { IPlaybackClock } from './playbackClock.js';
+/** The type of a Gemini Live avatar: H.264 Constrained Baseline 3.1 video, AAC-LC audio. */
+export declare const GEMINI_AVATAR_MP4_TYPE = "video/mp4; codecs=\"avc1.42c01f, mp4a.40.2\"";
+export interface VideoPlayoutOptions {
+    /**
+     * The type, with codecs, the player expects: the decoder for it starts when an element is attached, before any frame
+     * arrives, and the MSE source buffer opens with it until an init segment names its own codecs (or names one the reader
+     * doesn't know). Defaults to {@link GEMINI_AVATAR_MP4_TYPE}.
+     */
+    MimeType?: string;
+    /** Seconds of played media to keep behind the playhead. Defaults to 10. */
+    BackBufferSeconds?: number;
+    /**
+     * Whether the element plays the media's audio. Defaults to `true`: an avatar's MP4 carries its voice. `false`
+     * mutes the element, for a voice that plays elsewhere. {@link VideoPlayout.CarriesVoice} changes it later.
+     */
+    CarriesVoice?: boolean;
+    /**
+     * Called with each element the player takes over, before playback starts, such as to route the element's audio
+     * into a Web Audio graph. A player that moves to another element calls it again with that one.
+     */
+    OnElementAttached?: (element: HTMLVideoElement) => void;
+    /**
+     * The voice's playback clock, for lip sync: the chunk and image decoders show each frame when the voice reaches its
+     * `PresentationTimeMs`. Give it when the driver times its PCM (`IRealtimePcmPlayback.Enqueue(pcm16, mediaTimeMs)`) and
+     * its frames on one media timeline; `RealtimePcmPlayback` is such a clock. Without it, frames go by their own times.
+     * MSE ignores it.
+     */
+    Clock?: IPlaybackClock;
+}
+/** Why playout reported a problem. */
+export type VideoPlayoutProblem = 
+/** The browser cannot play this type: no decoder plays the configured type, or a decoder can't decode the stream's codec. */
+'unsupported'
+/** No decoder in this browser plays the frame's kind and type; it was dropped. */
+ | 'no-decoder'
+/** A media fragment arrived before any init segment, or an encoded chunk before any key frame, and was dropped. */
+ | 'fragment-before-init'
+/** The browser rejected a piece (decode error, or out of buffer space after trimming). */
+ | 'append-failed'
+/** Media arrived faster than it could be played, or while no element was attached, and the oldest was dropped. */
+ | 'pending-overflow';
+/** Whether a piece is an MP4 init segment: its first box is `ftyp`. */
+export declare function IsMp4InitSegment(piece: ArrayBuffer): boolean;
+/**
+ * What a realtime driver needs from an avatar's video player: {@link VideoPlayout} in a browser, a fake in tests.
+ */
+export interface IAvatarVideoPlayout {
+    /** The video to show. A driver hands it to the host once. */
+    readonly Source: MediaVideoSource;
+    /** Whether it plays forward with media buffered ahead of the playhead: the avatar is audibly speaking. */
+    readonly IsPlaying: boolean;
+    /**
+     * Where the playhead is, in frames: how many of the frames handed to {@link Append} are still to play (waiting,
+     * being decoded or appended, or buffered ahead of the playhead). A frame that was dropped, or that no decoder plays,
+     * is not counted. Frames play in the order they were handed over, so the frames ahead are the latest ones handed
+     * over, less any of those that were dropped: a driver that counts what it hands over can tell whether a given frame
+     * has played.
+     */
+    readonly FramesAhead: number;
+    /** Whether the element plays the media's audio; `false` mutes it. */
+    CarriesVoice: boolean;
+    /**
+     * Hands over one frame, in the order they arrived: a piece of fragmented MP4 (an init segment or a media fragment),
+     * an encoded chunk or an image. A frame the player has no decoder for is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last piece has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn(): void;
+    /** Barge-in: drops everything not yet played and stops at once, holding the last frame. */
+    Flush(): void;
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler: (problem: VideoPlayoutProblem, message: string) => void): () => void;
+    /** Stops playout and releases the element. */
+    Dispose(): void;
+}
+/** Plays a model's video frames into one `<video>` element at a time, through the decoder each frame type needs. */
+export declare class VideoPlayout implements IAvatarVideoPlayout {
+    private readonly mimeType;
+    private readonly backBufferSeconds;
+    private readonly onElementAttached;
+    private readonly clock;
+    private readonly problemHandlers;
+    private readonly reported;
+    /** Decoders that gave up, by frame type: they are not chosen again for it. */
+    private readonly failed;
+    private carriesVoice;
+    private element;
+    private active;
+    private disposed;
+    /** Whether some decoder can play `mimeType` in this browser, answered at once (for negotiation, before connect). */
+    static IsSupported(mimeType?: string): boolean;
+    constructor(options?: VideoPlayoutOptions);
+    /** The video to show: the player takes over the element it is attached to. */
+    get Source(): MediaVideoSource;
+    /**
+     * Whether the element plays the media's audio. `false` mutes it: the voice plays elsewhere, such as separate PCM
+     * audio when the MP4 has no audio track. Takes effect at once on an attached element.
+     */
+    get CarriesVoice(): boolean;
+    set CarriesVoice(value: boolean);
+    /** Whether the decoder's own audio is audibly playing: an MP4 that carries the voice, buffered ahead of the playhead. */
+    get IsPlaying(): boolean;
+    /**
+     * How many of the frames handed over are still to play, as the decoder in use counts them. Frames a decoder that
+     * gave up held, and frames no decoder plays, never play, so they are not counted.
+     */
+    get FramesAhead(): number;
+    /**
+     * Hands over one frame, in the order they arrived, to the decoder for its type: the one in use, or the first the
+     * registry offers. A frame no decoder plays is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last frame has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn(): void;
+    /** Barge-in: drops everything not yet played and stops at once. The last frame stays on screen. */
+    Flush(): void;
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler: (problem: VideoPlayoutProblem, message: string) => void): () => void;
+    /** Stops playout, releases the element and disposes the decoder. */
+    Dispose(): void;
+    private attach;
+    private detachElement;
+    /** The decoder for the configured type, started before any frame so it is ready when the first one comes. */
+    private startConfiguredDecoder;
+    /** The decoder for a frame type: the one in use when it plays that type, else a new one from the registry. */
+    private decoderFor;
+    /** What a decoder chosen for `type` plays with and reports through. */
+    private contextFor;
+    /** A decoder gave up: it is disposed and not chosen again for its type. The next frame of the type picks another. */
+    private decoderFailed;
+    private releaseDecoder;
+    private report;
+}
+//# sourceMappingURL=videoPlayout.d.ts.map

package/dist/media/videoPlayout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPlayout.d.ts","sourceRoot":"","sources":["../../src/media/videoPlayout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAkB,KAAK,kBAAkB,EAA+B,MAAM,oBAAoB,CAAC;AAC1G,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAKtD,4FAA4F;AAC5F,eAAO,MAAM,sBAAsB,iDAA+C,CAAC;AAKnF,MAAM,WAAW,mBAAmB;IAChC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACxD;;;;;OAKG;IACH,KAAK,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,sCAAsC;AACtC,MAAM,MAAM,mBAAmB;AAC3B,6HAA6H;AAC3H,aAAa;AACf,kFAAkF;GAChF,YAAY;AACd,mHAAmH;GACjH,sBAAsB;AACxB,0FAA0F;GACxF,eAAe;AACjB,kHAAkH;GAChH,kBAAkB,CAAC;AAEzB,uEAAuE;AACvE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAE5D;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,0GAA0G;IAC1G,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,qEAAqE;IACrE,YAAY,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,iGAAiG;IACjG,SAAS,IAAI,IAAI,CAAC;IAClB,2FAA2F;IAC3F,KAAK,IAAI,IAAI,CAAC;IACd,oFAAoF;IACpF,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IACxF,8CAA8C;IAC9C,OAAO,IAAI,IAAI,CAAC;CACnB;AAWD,oHAAoH;AACpH,qBAAa,YAAa,YAAW,mBAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoD;IACtF,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsE;IACtG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,QAAQ,CAAS;IAEzB,oHAAoH;WACtG,WAAW,CAAC,QAAQ,GAAE,MAA+B,GAAG,OAAO;gBAKjE,OAAO,GAAE,mBAAwB;IAQ7C,8EAA8E;IAC9E,IAAW,MAAM,IAAI,gBAAgB,CAEpC;IAED;;;OAGG;IACH,IAAW,YAAY,IAAI,OAAO,CAEjC;IACD,IAAW,YAAY,CAAC,KAAK,EAAE,OAAO,EAKrC;IAED,yHAAyH;IACzH,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,WAAW,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACI,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAa9C,iGAAiG;IAC1F,SAAS,IAAI,IAAI;IAIxB,mGAAmG;IAC5F,KAAK,IAAI,IAAI;IAIpB,oFAAoF;IAC7E,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI;IAK9F,oEAAoE;IAC7D,OAAO,IAAI,IAAI;IAOtB,OAAO,CAAC,MAAM;IAmBd,OAAO,CAAC,aAAa;IAOrB,6GAA6G;IAC7G,OAAO,CAAC,sBAAsB;IAK9B,8GAA8G;IAC9G,OAAO,CAAC,UAAU;IAmBlB,uEAAuE;IACvE,OAAO,CAAC,UAAU;IAelB,qHAAqH;IACrH,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,MAAM;CAUjB"}

package/dist/media/videoPlayout.js

@@ -0,0 +1,217 @@
+/**
+ * @fileoverview VIDEO PLAYOUT: plays a model's video into one `<video>` element at a time, whatever form it arrives in.
+ * A provider hands each frame to {@link VideoPlayout.Append} as a `RealtimeVideoFrame` from `@memberjunction/ai`; a
+ * renderer shows {@link VideoPlayout.Source} with `AttachVideoSource`.
+ *
+ * - **A decoder per frame type.** The player keeps one decoder at a time, chosen from the page's
+ *   {@link VideoFrameDecoderRegistry} by the frame's kind and MIME type: `'mse-fmp4'` plays fragmented MP4 (a Gemini Live
+ *   avatar) through Media Source Extensions, `'webcodecs'` plays encoded chunks (H.264, VP8, VP9, AV1) through WebCodecs,
+ *   `'image'` plays still images. A frame of another type switches decoders; a type no decoder plays is dropped and
+ *   reported once (`'no-decoder'`).
+ * - **Ready before the first frame.** When an element is attached first, the player starts the decoder for its
+ *   configured type ({@link VideoPlayoutOptions.MimeType}), so MSE has its media source open when the first piece comes.
+ *   A configured type no decoder plays reports `'unsupported'` and leaves the element alone.
+ * - **Fallback.** A decoder that gives up (`Failed`) is disposed and not chosen again for that type; the next one that can
+ *   play it starts at the next init segment or key frame. Out of decoders, frames of that type are dropped; the voice is
+ *   unaffected, and the tile's stall fallback shows in place of the video.
+ * - **The voice.** An avatar's MP4 carries its voice, so the MSE decoder unmutes the element and
+ *   {@link VideoPlayout.IsPlaying} says whether the agent is audibly speaking. When the voice plays elsewhere,
+ *   {@link VideoPlayout.CarriesVoice} `false` mutes the element. Chunk and image decoders carry no voice.
+ *   {@link VideoPlayoutOptions.OnElementAttached} lets the realtime client route the element's audio into its own Web
+ *   Audio graph, so its meter and its recording carry the avatar's voice.
+ * - **Lip sync.** With {@link VideoPlayoutOptions.Clock}, the voice's playback clock, the chunk and image decoders show
+ *   each frame when the voice reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the voice keeps
+ *   voice and face together on its own timestamps.
+ * - **End of turn, barge-in, gaps.** {@link VideoPlayout.EndOfTurn} lets playback run to the true end of the turn;
+ *   {@link VideoPlayout.Flush} drops everything not yet played; between turns, or while a connection resumes, the element
+ *   holds the last frame. Each decoder does this its own way.
+ * - **Browsers.** {@link VideoPlayout.IsSupported} says, at once, whether some decoder can play a type here.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { SniffFmp4Piece } from '@memberjunction/ai';
+import { VideoFrameDecoderRegistry } from './videoFrameDecoderRegistry.js';
+import { FrameKindOfMimeType, FrameTypeKey } from './decoders/videoMimeTypes.js';
+/** The type of a Gemini Live avatar: H.264 Constrained Baseline 3.1 video, AAC-LC audio. */
+export const GEMINI_AVATAR_MP4_TYPE = 'video/mp4; codecs="avc1.42c01f, mp4a.40.2"';
+/** Seconds of already-played media kept behind the playhead before it is removed. */
+const DEFAULT_BACK_BUFFER_SECONDS = 10;
+/** Whether a piece is an MP4 init segment: its first box is `ftyp`. */
+export function IsMp4InitSegment(piece) {
+    return SniffFmp4Piece(piece) === 'init';
+}
+/** Plays a model's video frames into one `<video>` element at a time, through the decoder each frame type needs. */
+export class VideoPlayout {
+    /** Whether some decoder can play `mimeType` in this browser, answered at once (for negotiation, before connect). */
+    static IsSupported(mimeType = GEMINI_AVATAR_MP4_TYPE) {
+        const kind = FrameKindOfMimeType(mimeType);
+        return kind !== null && VideoFrameDecoderRegistry.Instance.Candidates(kind, mimeType).length > 0;
+    }
+    constructor(options = {}) {
+        this.problemHandlers = new Set();
+        this.reported = new Set();
+        /** Decoders that gave up, by frame type: they are not chosen again for it. */
+        this.failed = new Map();
+        this.element = null;
+        this.active = null;
+        this.disposed = false;
+        this.mimeType = options.MimeType ?? GEMINI_AVATAR_MP4_TYPE;
+        this.backBufferSeconds = options.BackBufferSeconds ?? DEFAULT_BACK_BUFFER_SECONDS;
+        this.carriesVoice = options.CarriesVoice ?? true;
+        this.onElementAttached = options.OnElementAttached;
+        this.clock = options.Clock;
+    }
+    /** The video to show: the player takes over the element it is attached to. */
+    get Source() {
+        return { Kind: 'element', Attach: (element) => this.attach(element) };
+    }
+    /**
+     * Whether the element plays the media's audio. `false` mutes it: the voice plays elsewhere, such as separate PCM
+     * audio when the MP4 has no audio track. Takes effect at once on an attached element.
+     */
+    get CarriesVoice() {
+        return this.carriesVoice;
+    }
+    set CarriesVoice(value) {
+        this.carriesVoice = value;
+        if (this.element) {
+            this.element.muted = !value;
+        }
+    }
+    /** Whether the decoder's own audio is audibly playing: an MP4 that carries the voice, buffered ahead of the playhead. */
+    get IsPlaying() {
+        return this.active?.Decoder.IsPlaying ?? false;
+    }
+    /**
+     * How many of the frames handed over are still to play, as the decoder in use counts them. Frames a decoder that
+     * gave up held, and frames no decoder plays, never play, so they are not counted.
+     */
+    get FramesAhead() {
+        return this.active?.Decoder.FramesAhead ?? 0;
+    }
+    /**
+     * Hands over one frame, in the order they arrived, to the decoder for its type: the one in use, or the first the
+     * registry offers. A frame no decoder plays is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame) {
+        if (this.disposed) {
+            return;
+        }
+        const decoder = this.decoderFor(frame.Kind, frame.MimeType);
+        if (!decoder) {
+            // A frame it can't play adds no media: the decoder in use, and a turn it was about to end, carry on.
+            this.report('no-decoder', `No decoder in this browser plays ${frame.Kind} frames of type ${frame.MimeType}; the frame was dropped.`);
+            return;
+        }
+        decoder.Append(frame);
+    }
+    /** The turn's last frame has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn() {
+        this.active?.Decoder.EndOfTurn();
+    }
+    /** Barge-in: drops everything not yet played and stops at once. The last frame stays on screen. */
+    Flush() {
+        this.active?.Decoder.Flush();
+    }
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler) {
+        this.problemHandlers.add(handler);
+        return () => this.problemHandlers.delete(handler);
+    }
+    /** Stops playout, releases the element and disposes the decoder. */
+    Dispose() {
+        this.disposed = true;
+        this.releaseDecoder();
+        this.element = null;
+        this.problemHandlers.clear();
+    }
+    attach(element) {
+        if (this.disposed) {
+            return () => undefined;
+        }
+        this.detachElement();
+        const decoder = this.active?.Decoder ?? this.startConfiguredDecoder();
+        if (!decoder) {
+            this.report('unsupported', `This browser cannot play ${this.mimeType}: no decoder plays it.`);
+            return () => undefined;
+        }
+        this.element = element;
+        decoder.Attach(element);
+        return () => {
+            if (this.element === element) {
+                this.detachElement();
+            }
+        };
+    }
+    detachElement() {
+        if (this.element) {
+            this.active?.Decoder.Detach();
+            this.element = null;
+        }
+    }
+    /** The decoder for the configured type, started before any frame so it is ready when the first one comes. */
+    startConfiguredDecoder() {
+        const kind = FrameKindOfMimeType(this.mimeType);
+        return kind ? this.decoderFor(kind, this.mimeType) : null;
+    }
+    /** The decoder for a frame type: the one in use when it plays that type, else a new one from the registry. */
+    decoderFor(kind, mimeType) {
+        const type = FrameTypeKey(kind, mimeType);
+        if (this.active?.Type === type) {
+            return this.active.Decoder;
+        }
+        const failed = this.failed.get(type);
+        const registration = VideoFrameDecoderRegistry.Instance.Candidates(kind, mimeType).find((candidate) => !failed?.has(candidate.Name));
+        if (!registration) {
+            return null;
+        }
+        this.releaseDecoder();
+        const decoder = registration.Create(this.contextFor(type, registration.Name));
+        this.active = { Type: type, Name: registration.Name, Decoder: decoder };
+        if (this.element) {
+            decoder.Attach(this.element);
+        }
+        return decoder;
+    }
+    /** What a decoder chosen for `type` plays with and reports through. */
+    contextFor(type, name) {
+        const carriesVoice = () => this.carriesVoice;
+        return {
+            MimeType: this.mimeType,
+            BackBufferSeconds: this.backBufferSeconds,
+            get CarriesVoice() {
+                return carriesVoice();
+            },
+            OnElementAttached: this.onElementAttached,
+            Clock: this.clock,
+            Report: (problem, message) => this.report(problem, message),
+            Failed: (message) => this.decoderFailed(type, name, message),
+        };
+    }
+    /** A decoder gave up: it is disposed and not chosen again for its type. The next frame of the type picks another. */
+    decoderFailed(type, name, message) {
+        console.warn(`[VideoPlayout] The ${name} decoder stopped: ${message}`);
+        const failed = this.failed.get(type) ?? new Set();
+        failed.add(name);
+        this.failed.set(type, failed);
+        if (this.active?.Type === type && this.active.Name === name) {
+            this.releaseDecoder();
+        }
+    }
+    releaseDecoder() {
+        const active = this.active;
+        this.active = null;
+        active?.Decoder.Dispose();
+    }
+    report(problem, message) {
+        if (this.reported.has(problem)) {
+            return;
+        }
+        this.reported.add(problem);
+        console.warn(`[VideoPlayout] ${message}`);
+        for (const handler of [...this.problemHandlers]) {
+            handler(problem, message);
+        }
+    }
+}
+//# sourceMappingURL=videoPlayout.js.map

package/dist/media/videoPlayout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPlayout.js","sourceRoot":"","sources":["../../src/media/videoPlayout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,cAAc,EAAwD,MAAM,oBAAoB,CAAC;AAI1G,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9E,4FAA4F;AAC5F,MAAM,CAAC,MAAM,sBAAsB,GAAG,4CAA4C,CAAC;AAEnF,qFAAqF;AACrF,MAAM,2BAA2B,GAAG,EAAE,CAAC;AA2CvC,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAAC,KAAkB;IAC/C,OAAO,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC;AAC5C,CAAC;AA4CD,oHAAoH;AACpH,MAAM,OAAO,YAAY;IAcrB,oHAAoH;IAC7G,MAAM,CAAC,WAAW,CAAC,WAAmB,sBAAsB;QAC/D,MAAM,IAAI,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAC3C,OAAO,IAAI,KAAK,IAAI,IAAI,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACrG,CAAC;IAED,YAAY,UAA+B,EAAE;QAf5B,oBAAe,GAAG,IAAI,GAAG,EAA2D,CAAC;QACrF,aAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC3D,8EAA8E;QAC7D,WAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;QAEjD,YAAO,GAA4B,IAAI,CAAC;QACxC,WAAM,GAAyB,IAAI,CAAC;QACpC,aAAQ,GAAG,KAAK,CAAC;QASrB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,sBAAsB,CAAC;QAC3D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,8EAA8E;IAC9E,IAAW,MAAM;QACb,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IACD,IAAW,YAAY,CAAC,KAAc;QAClC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC;QAChC,CAAC;IACL,CAAC;IAED,yHAAyH;IACzH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,KAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,qGAAqG;YACrG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,oCAAoC,KAAK,CAAC,IAAI,mBAAmB,KAAK,CAAC,QAAQ,0BAA0B,CAAC,CAAC;YACrI,OAAO;QACX,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,iGAAiG;IAC1F,SAAS;QACZ,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;IACrC,CAAC;IAED,mGAAmG;IAC5F,KAAK;QACR,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,oFAAoF;IAC7E,SAAS,CAAC,OAAgE;QAC7E,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,CAAC;IAED,oEAAoE;IAC7D,OAAO;QACV,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAEO,MAAM,CAAC,OAAyB;QACpC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,4BAA4B,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAC;YAC9F,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,GAAG,EAAE;YACR,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAEO,aAAa;QACjB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC;IAED,6GAA6G;IACrG,sBAAsB;QAC1B,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,8GAA8G;IACtG,UAAU,CAAC,IAA4B,EAAE,QAAgB;QAC7D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QAC/B,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACrI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,uEAAuE;IAC/D,UAAU,CAAC,IAAY,EAAE,IAAY;QACzC,MAAM,YAAY,GAAG,GAAY,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC;QACtD,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,IAAI,YAAY;gBACZ,OAAO,YAAY,EAAE,CAAC;YAC1B,CAAC;YACD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC;YAC3D,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;SAC/D,CAAC;IACN,CAAC;IAED,qHAAqH;IAC7G,aAAa,CAAC,IAAY,EAAE,IAAY,EAAE,OAAe;QAC7D,OAAO,CAAC,IAAI,CAAC,sBAAsB,IAAI,qBAAqB,OAAO,EAAE,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QAC1D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;IAEO,cAAc;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAEO,MAAM,CAAC,OAA4B,EAAE,OAAe;QACxD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,kBAAkB,OAAO,EAAE,CAAC,CAAC;QAC1C,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;CACJ"}

package/dist/media/videoSourceArbiter.d.ts

@@ -0,0 +1,250 @@
+/**
+ * @fileoverview The VIDEO SOURCE ARBITER: the single writer of a session's inbound video.
+ *
+ * A realtime session can have many things worth showing the model at once: a shared screen, the camera,
+ * a whiteboard, a remote browser. Today's video models accept ONE inbound stream. Before this existed,
+ * every source wrote to that one stream directly: whichever happened to send last "won", and the model
+ * was never told what it was looking at. The arbiter owns the decision:
+ *
+ * - **Sources register** (`RegisterSource`) with an id, a label and a kind, and push frames to the
+ *   arbiter (`PushFrame`) rather than to the client.
+ * - **The arbiter maps N live sources onto the model's `MaxInboundVideoStreams`.** With enough streams
+ *   for every enabled source (a future multi-stream model) frames pass through untouched, each tagged
+ *   with its source id. Otherwise it ranks the sources by a POLICY (data, overridable) and forwards only
+ *   the top N.
+ * - **The model is told when the choice changes** (`[You can now see: Camera]`), because a model
+ *   that silently starts seeing something else will describe the old thing.
+ * - **The user can switch a source off.** A disabled source is not a candidate, sends nothing, and the
+ *   model is told it was turned off (`[You can no longer see: Camera (the user turned it off)]`).
+ * - **The rate comes only from the negotiated track.** Pacing is read from the sink
+ *   (`InboundVideoRate`), never a constant here.
+ *
+ * The notes' wording comes from `@memberjunction/ai` ({@link VideoSourceSeenNote}, {@link VideoSourceTurnedOffNote},
+ * {@link VideoSourceTurnedOnNote}), where a meeting's bridge engine gets its notes too, so the model reads one style of
+ * note in a call and in a meeting.
+ *
+ * It deliberately depends only on the narrow {@link IVideoFrameSink} interface (which
+ * `BaseRealtimeClient` satisfies) and imports nothing from `generic/` or the drivers, so the media code
+ * can be split into its own subpath without untangling it.
+ *
+ * The default policy and the rationale for its ordering are on {@link DEFAULT_VIDEO_SOURCE_POLICY}.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+import { type RealtimeTrackDirection } from '@memberjunction/ai';
+/**
+ * The slice of a realtime client the arbiter writes to. `BaseRealtimeClient` satisfies it structurally,
+ * and tests satisfy it with a plain object.
+ */
+export interface IVideoFrameSink {
+    /**
+     * How many concurrent inbound video streams the model accepts. `0` when none is live (no video track
+     * negotiated), `1` for every model shipped so far.
+     */
+    readonly MaxInboundVideoStreams: number;
+    /** The negotiated inbound video rate in frames per second, or `undefined` when none was negotiated. */
+    readonly InboundVideoRate: number | undefined;
+    /** Whether a track of the given modality and direction is established and live. */
+    IsTrackEstablished(modality: string, direction: RealtimeTrackDirection): boolean;
+    /**
+     * Streams one frame to the model, tagged with the source that produced it. Optional because most
+     * drivers have no video; a sink without it makes the arbiter inert.
+     *
+     * @returns `true` if the frame was sent.
+     */
+    SendVideoFrame?(base64Image: string, mimeType?: string, sourceId?: string): boolean;
+    /** Injects a context note into the conversation without triggering a reply. */
+    SendContextNote(text: string): void;
+}
+/**
+ * What kind of source this is. Open vocabulary: `'camera'` and `'screen'` are CAPTURES (the user chose to
+ * share them), `'surface'` is a channel surface (a whiteboard, a remote browser); anything else is treated
+ * as a surface unless the policy says otherwise.
+ */
+export type VideoSourceKind = 'camera' | 'screen' | 'surface' | (string & {});
+/** Whether a source kind is a capture (as opposed to a channel surface). */
+export declare function IsVideoCaptureKind(kind: VideoSourceKind | undefined): boolean;
+/** What a source registers itself as. */
+export interface VideoSourceDescriptor {
+    /** Stable id of the source within the session. Registering an existing id updates it in place. */
+    SourceID: string;
+    /** Human-readable name, used in the notes sent to the model and in the UI's "agent can see" list. */
+    Label: string;
+    /** What kind of source this is. Defaults to `'surface'`. */
+    Kind?: VideoSourceKind;
+    /**
+     * The channel this source belongs to, when it comes from one. The UI maps a source to the user's
+     * persisted per-channel visual-perception choice by this key.
+     */
+    ChannelKey?: string;
+    /**
+     * Whether a newly registered source starts enabled. Default `true`. A source the agent may not see yet registers
+     * disabled, so the model is never told it is looking at it. Ignored for an id that is already registered; use
+     * {@link VideoSourceArbiter.SetSourceEnabled} for that.
+     */
+    Enabled?: boolean;
+}
+/** A source as the arbiter currently sees it. */
+export interface VideoSourceState extends VideoSourceDescriptor {
+    /** `false` when the user (or exposure policy) turned the source off: it is not a candidate and sends nothing. */
+    Enabled: boolean;
+    /** Whether the arbiter is currently forwarding this source's frames to the model. */
+    Active: boolean;
+    /** Frames forwarded to the model so far. */
+    FramesSent: number;
+    /** Epoch ms of the last forwarded frame; `undefined` before the first. */
+    LastFrameAt?: number;
+    /**
+     * `true` while it is the user's pick ({@link VideoSourceArbiter.SelectSource}), which beats every other rule. Absent
+     * otherwise, and while nothing is picked the policy decides.
+     */
+    Picked?: boolean;
+}
+/**
+ * One rule of the source-selection policy. Rules apply in order and each contributes the sources it
+ * prefers; the arbiter forwards the first `MaxInboundVideoStreams` of the combined ranking.
+ *
+ * - `'user-pick'`: the source the user explicitly chose ({@link VideoSourceArbiter.SelectSource}).
+ * - `'recent-capture'`: captures (camera, screen), the most recently started first.
+ * - `'focused'`: the source belonging to the surface the user is looking at
+ *   ({@link VideoSourceArbiter.SetFocusedSource}).
+ * - `'recent'`: any source at all, the most recently started first. The catch-all, so a session whose
+ *   only source is a surface with no focus still shows it.
+ */
+export type VideoSourcePolicyRule = 'user-pick' | 'recent-capture' | 'focused' | 'recent';
+/**
+ * The default selection policy, as DATA so an agent or host can reorder or trim it.
+ *
+ * Order: an explicit user pick beats everything (the user is the authority on what the agent sees);
+ * then the most recently started CAPTURE, because starting a camera or share is a deliberate "look at
+ * this" and a surface is always there to come back to; then the focused surface; then simply the newest
+ * source, so a lone unfocused surface is never invisible.
+ *
+ * "Most recently started" is applied to captures rather than to every source. Applied to everything it
+ * would always produce a winner and the focused-surface rule could never be reached.
+ */
+export declare const DEFAULT_VIDEO_SOURCE_POLICY: readonly VideoSourcePolicyRule[];
+/** Options for a {@link VideoSourceArbiter}. */
+export interface VideoSourceArbiterOptions {
+    /** The selection policy. Defaults to {@link DEFAULT_VIDEO_SOURCE_POLICY}. */
+    Policy?: readonly VideoSourcePolicyRule[];
+    /**
+     * The note sent when the set of sources the model sees changes while sources are being arbitrated.
+     * Receives the labels now in view. Return `null` to send nothing. Default: {@link VideoSourceSeenNote}
+     * with every label in view.
+     */
+    FormatSwitchNote?: (labels: readonly string[]) => string | null;
+    /** The note sent when a source is turned off. Return `null` to send nothing. Default: {@link VideoSourceTurnedOffNote}. */
+    FormatDisabledNote?: (label: string) => string | null;
+    /** The note sent when a source is turned back on. Return `null` to send nothing. Default: {@link VideoSourceTurnedOnNote}. */
+    FormatEnabledNote?: (label: string) => string | null;
+    /** Reports a problem (a throwing sink). Defaults to `console.error`; the arbiter never throws into a caller's timer. */
+    OnError?: (context: string, error: unknown) => void;
+    /** Clock, injectable for tests. Defaults to `Date.now`. */
+    Now?: () => number;
+}
+/** A sink or a way to find the current one (the client changes across a reconnect). */
+export type VideoFrameSinkSource = IVideoFrameSink | (() => IVideoFrameSink | null | undefined) | null | undefined;
+/**
+ * The per-session arbiter. Obtain the one for a client with {@link VideoSourceArbiter.ForSink} so every
+ * writer shares it; constructing a second one for the same client would make two writers again.
+ */
+export declare class VideoSourceArbiter {
+    private readonly sinkSource;
+    private readonly options;
+    private static readonly bySink;
+    /**
+     * The arbiter for a sink, created on first use. One arbiter per sink is the whole point (a single
+     * writer), so every bridge and channel that gets its arbiter this way shares it.
+     *
+     * @param sink The client (anything satisfying {@link IVideoFrameSink}).
+     * @param options Options applied only when the arbiter is created.
+     */
+    static ForSink(sink: IVideoFrameSink, options?: VideoSourceArbiterOptions): VideoSourceArbiter;
+    private readonly sources;
+    private readonly policy;
+    private readonly handlers;
+    private nextSeq;
+    private userPick;
+    private focused;
+    private focusedChannel;
+    /** The ids forwarded as of the last reconcile — the baseline a change is detected against. */
+    private lastActive;
+    /** Whether the previous reconcile had more enabled sources than streams (arbitration in effect). */
+    private lastArbitrating;
+    constructor(sinkSource: VideoFrameSinkSource, options?: VideoSourceArbiterOptions);
+    private get sink();
+    /**
+     * Registers a source, or updates its label/kind if the id is already registered (its recency, enabled
+     * state and counters are kept). A newly registered source counts as the most recently STARTED.
+     */
+    RegisterSource(descriptor: VideoSourceDescriptor): void;
+    /** Removes a source. Its explicit pick or focus, if it held one, is cleared. */
+    UnregisterSource(sourceId: string): void;
+    /** Whether a source id is registered. */
+    HasSource(sourceId: string): boolean;
+    /**
+     * The user explicitly picks the source the agent sees (`null` clears the pick). A pick beats every
+     * other rule. Returns `false` for an unknown or disabled source (nothing changes).
+     */
+    SelectSource(sourceId: string | null): boolean;
+    /**
+     * Marks the CHANNEL whose surface the user is looking at (`null` clears it): every source registered for that
+     * channel counts as focused, including one that registers after this call (a surface the user is already on
+     * before its first frame). Used by the `'focused'` rule alongside {@link SetFocusedSource}.
+     */
+    SetFocusedChannel(channelKey: string | null): void;
+    /** Marks the source whose surface the user is looking at (`null` clears it). Used by the `'focused'` rule. */
+    SetFocusedSource(sourceId: string | null): void;
+    /**
+     * Turns a source on or off. A disabled source is not a candidate and forwards nothing.
+     *
+     * @param sourceId The source.
+     * @param enabled The new state.
+     * @param notify Whether to tell the model. Default `true`. A caller that sends its own note about the
+     *   same change (a channel lowering its exposure) passes `false` so the model is told once.
+     * @returns `false` for an unknown source; `true` otherwise (including "already in that state").
+     */
+    SetSourceEnabled(sourceId: string, enabled: boolean, notify?: boolean): boolean;
+    /** A snapshot of every registered source and whether it is being forwarded, in registration order. */
+    GetSources(): VideoSourceState[];
+    /** The ids the arbiter would forward right now, best first. */
+    GetActiveSourceIDs(): string[];
+    /**
+     * Subscribes to structural changes (a source registered, removed, enabled, disabled, picked, focused,
+     * or the forwarded set changing). Not called per frame.
+     *
+     * @returns An unsubscribe function.
+     */
+    OnChange(handler: () => void): () => void;
+    /**
+     * A source offers a frame. It is forwarded to the model only if the source is enabled, among the
+     * sources the model should see, and not inside the negotiated pacing interval.
+     *
+     * @param sourceId The producing source (it must be registered).
+     * @param base64Image The frame, base64-encoded.
+     * @param mimeType Image MIME type. Defaults to `'image/jpeg'`.
+     * @returns `true` if the frame was sent to the model.
+     */
+    PushFrame(sourceId: string, base64Image: string, mimeType?: string): boolean;
+    /** Drops every source and subscription. The arbiter stays usable (it simply has no sources). */
+    Dispose(): void;
+    /** The enabled sources, ranked by policy, best first. */
+    private rank;
+    /** The ids to forward: the top N of the ranking, N being the model's inbound video stream count. */
+    private computeActive;
+    /**
+     * Re-derives the forwarded set and, when it changed while sources are being arbitrated, tells the model.
+     *
+     * @param quiet Update the baseline without sending a switch note (the caller sends its own).
+     */
+    private reconcile;
+    private formatDisabledNote;
+    private formatEnabledNote;
+    private now;
+    private send;
+    private sendNote;
+    private emitChange;
+    private reportError;
+}
+//# sourceMappingURL=videoSourceArbiter.d.ts.map