@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/videoFrameDecoderRegistry.d.ts

@@ -0,0 +1,40 @@
+/**
+ * @fileoverview VIDEO FRAME DECODER REGISTRY: the page's video frame decoders, which `VideoPlayout` chooses from for each
+ * frame type. The three built-in decoders (`'mse-fmp4'`, `'webcodecs'`, `'image'`) are registered when the registry is
+ * first used; an app adds its own with {@link VideoFrameDecoderRegistry.Register}, or replaces a built-in by registering
+ * the same name.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { BaseSingleton } from '@memberjunction/global';
+import type { RealtimeVideoFrameKind } from '@memberjunction/ai';
+import type { VideoFrameDecoderRegistration } from './videoFrameDecoder.js';
+/** The page's video frame decoders, by name. */
+export declare class VideoFrameDecoderRegistry extends BaseSingleton<VideoFrameDecoderRegistry> {
+    private readonly registrations;
+    protected constructor();
+    /** The page's registry, with the built-in decoders registered. */
+    static get Instance(): VideoFrameDecoderRegistry;
+    /**
+     * Adds a decoder. One with the same name is replaced; players that already use the old one keep it until they choose a
+     * decoder again.
+     *
+     * @param registration The decoder: its name, frame kind, priority, support check and factory.
+     */
+    Register(registration: VideoFrameDecoderRegistration): void;
+    /**
+     * Removes a decoder by name. Players that already use it keep it until they choose a decoder again.
+     *
+     * @param name The decoder's name.
+     */
+    Unregister(name: string): void;
+    /**
+     * The decoders that can play frames of `kind` and `mimeType` in this browser, highest priority first (in registration
+     * order among equals). A decoder whose support check throws is left out.
+     *
+     * @param kind The frame kind.
+     * @param mimeType The frame's MIME type.
+     */
+    Candidates(kind: RealtimeVideoFrameKind, mimeType: string): readonly VideoFrameDecoderRegistration[];
+}
+//# sourceMappingURL=videoFrameDecoderRegistry.d.ts.map

package/dist/media/videoFrameDecoderRegistry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoderRegistry.d.ts","sourceRoot":"","sources":["../../src/media/videoFrameDecoderRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AACjE,OAAO,KAAK,EAAE,6BAA6B,EAAE,MAAM,qBAAqB,CAAC;AAQzE,gDAAgD;AAChD,qBAAa,yBAA0B,SAAQ,aAAa,CAAC,yBAAyB,CAAC;IACnF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoD;IAElF,SAAS;IAOT,kEAAkE;IAClE,WAAkB,QAAQ,IAAI,yBAAyB,CAEtD;IAED;;;;;OAKG;IACI,QAAQ,CAAC,YAAY,EAAE,6BAA6B,GAAG,IAAI;IAIlE;;;;OAIG;IACI,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIrC;;;;;;OAMG;IACI,UAAU,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,GAAG,SAAS,6BAA6B,EAAE;CAK9G"}

package/dist/media/videoFrameDecoderRegistry.js

@@ -0,0 +1,69 @@
+/**
+ * @fileoverview VIDEO FRAME DECODER REGISTRY: the page's video frame decoders, which `VideoPlayout` chooses from for each
+ * frame type. The three built-in decoders (`'mse-fmp4'`, `'webcodecs'`, `'image'`) are registered when the registry is
+ * first used; an app adds its own with {@link VideoFrameDecoderRegistry.Register}, or replaces a built-in by registering
+ * the same name.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { BaseSingleton } from '@memberjunction/global';
+import { IMAGE_FRAME_DECODER } from './decoders/imageFrameDecoder.js';
+import { MSE_FMP4_DECODER } from './decoders/mseFmp4Decoder.js';
+import { WEBCODECS_CHUNK_DECODER } from './decoders/webCodecsChunkDecoder.js';
+/** The decoders every page starts with. */
+const BUILT_IN_DECODERS = [MSE_FMP4_DECODER, WEBCODECS_CHUNK_DECODER, IMAGE_FRAME_DECODER];
+/** The page's video frame decoders, by name. */
+export class VideoFrameDecoderRegistry extends BaseSingleton {
+    constructor() {
+        super();
+        this.registrations = new Map();
+        for (const registration of BUILT_IN_DECODERS) {
+            this.Register(registration);
+        }
+    }
+    /** The page's registry, with the built-in decoders registered. */
+    static get Instance() {
+        return super.getInstance();
+    }
+    /**
+     * Adds a decoder. One with the same name is replaced; players that already use the old one keep it until they choose a
+     * decoder again.
+     *
+     * @param registration The decoder: its name, frame kind, priority, support check and factory.
+     */
+    Register(registration) {
+        this.registrations.set(registration.Name, registration);
+    }
+    /**
+     * Removes a decoder by name. Players that already use it keep it until they choose a decoder again.
+     *
+     * @param name The decoder's name.
+     */
+    Unregister(name) {
+        this.registrations.delete(name);
+    }
+    /**
+     * The decoders that can play frames of `kind` and `mimeType` in this browser, highest priority first (in registration
+     * order among equals). A decoder whose support check throws is left out.
+     *
+     * @param kind The frame kind.
+     * @param mimeType The frame's MIME type.
+     */
+    Candidates(kind, mimeType) {
+        return [...this.registrations.values()]
+            .filter((registration) => registration.Kind === kind && canPlay(registration, mimeType))
+            .sort((a, b) => b.Priority - a.Priority);
+    }
+}
+/** A registration's support check, with a check that throws read as "no" (logged). */
+function canPlay(registration, mimeType) {
+    try {
+        return registration.CanPlay(mimeType);
+    }
+    catch (error) {
+        const reason = error instanceof Error ? error.message : String(error);
+        console.warn(`[VideoPlayout] The ${registration.Name} decoder's support check failed for ${mimeType}: ${reason}`);
+        return false;
+    }
+}
+//# sourceMappingURL=videoFrameDecoderRegistry.js.map

package/dist/media/videoFrameDecoderRegistry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoFrameDecoderRegistry.js","sourceRoot":"","sources":["../../src/media/videoFrameDecoderRegistry.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAGvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAE3E,2CAA2C;AAC3C,MAAM,iBAAiB,GAA6C,CAAC,gBAAgB,EAAE,uBAAuB,EAAE,mBAAmB,CAAC,CAAC;AAErI,gDAAgD;AAChD,MAAM,OAAO,yBAA0B,SAAQ,aAAwC;IAGnF;QACI,KAAK,EAAE,CAAC;QAHK,kBAAa,GAAG,IAAI,GAAG,EAAyC,CAAC;QAI9E,KAAK,MAAM,YAAY,IAAI,iBAAiB,EAAE,CAAC;YAC3C,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,kEAAkE;IAC3D,MAAM,KAAK,QAAQ;QACtB,OAAO,KAAK,CAAC,WAAW,EAA6B,CAAC;IAC1D,CAAC;IAED;;;;;OAKG;IACI,QAAQ,CAAC,YAA2C;QACvD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IAC5D,CAAC;IAED;;;;OAIG;IACI,UAAU,CAAC,IAAY;QAC1B,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;OAMG;IACI,UAAU,CAAC,IAA4B,EAAE,QAAgB;QAC5D,OAAO,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;aAClC,MAAM,CAAC,CAAC,YAAY,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,KAAK,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;aACvF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;CACJ;AAED,sFAAsF;AACtF,SAAS,OAAO,CAAC,YAA2C,EAAE,QAAgB;IAC1E,IAAI,CAAC;QACD,OAAO,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,OAAO,CAAC,IAAI,CAAC,sBAAsB,YAAY,CAAC,IAAI,uCAAuC,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC;QAClH,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}

package/dist/media/videoPacing.d.ts

@@ -0,0 +1,32 @@
+/**
+ * @fileoverview Frame pacing derived from the NEGOTIATED inbound video rate.
+ *
+ * The rate a model accepts is per-model data (`MaxInboundVideoRate` in the provider's profile table),
+ * carried into the live track descriptor by track negotiation. Every layer that paces frames — the
+ * source arbiter, the driver's own backstop — asks THIS module what that rate means in milliseconds, so
+ * there is one answer to "how often may a frame go out" rather than a constant per layer.
+ *
+ * The math itself lives in `@memberjunction/ai` (`realtimeVideoFrames`), shared with the meeting bot that also sends
+ * frames to the model; these names are thin wrappers over it.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+/**
+ * The share of the nominal frame interval a pacer enforces as a MINIMUM spacing. Pacers upstream of
+ * the driver run on timers that jitter; enforcing exactly `1000 / rate` would drop a frame that arrived a
+ * few milliseconds early. At 1 fps this is the 750 ms backstop the Gemini driver has always applied.
+ */
+export declare const VIDEO_FRAME_JITTER_HEADROOM = 0.75;
+/** The rate assumed when none was negotiated: 1 frame per second, the Live API ceiling today. */
+export declare const DEFAULT_INBOUND_VIDEO_RATE = 1;
+/**
+ * The nominal interval between frames, in milliseconds, for a negotiated rate (frames per second).
+ * An absent, zero, negative or non-finite rate falls back to {@link DEFAULT_INBOUND_VIDEO_RATE}.
+ */
+export declare function NominalVideoFrameIntervalMs(rate: number | undefined | null): number;
+/**
+ * The minimum spacing, in milliseconds, a pacer should enforce between two frames to the model: the
+ * nominal interval less jitter headroom ({@link VIDEO_FRAME_JITTER_HEADROOM}).
+ */
+export declare function MinVideoFrameSpacingMs(rate: number | undefined | null): number;
+//# sourceMappingURL=videoPacing.d.ts.map

package/dist/media/videoPacing.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPacing.d.ts","sourceRoot":"","sources":["../../src/media/videoPacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,OAAuC,CAAC;AAEhF,iGAAiG;AACjG,eAAO,MAAM,0BAA0B,IAAsC,CAAC;AAE9E;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAEnF;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAE9E"}

package/dist/media/videoPacing.js

@@ -0,0 +1,37 @@
+/**
+ * @fileoverview Frame pacing derived from the NEGOTIATED inbound video rate.
+ *
+ * The rate a model accepts is per-model data (`MaxInboundVideoRate` in the provider's profile table),
+ * carried into the live track descriptor by track negotiation. Every layer that paces frames — the
+ * source arbiter, the driver's own backstop — asks THIS module what that rate means in milliseconds, so
+ * there is one answer to "how often may a frame go out" rather than a constant per layer.
+ *
+ * The math itself lives in `@memberjunction/ai` (`realtimeVideoFrames`), shared with the meeting bot that also sends
+ * frames to the model; these names are thin wrappers over it.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+import { REALTIME_DEFAULT_INBOUND_VIDEO_RATE, REALTIME_VIDEO_FRAME_JITTER_HEADROOM, RealtimeMinVideoFrameSpacingMs, RealtimeVideoFrameIntervalMs, } from '@memberjunction/ai';
+/**
+ * The share of the nominal frame interval a pacer enforces as a MINIMUM spacing. Pacers upstream of
+ * the driver run on timers that jitter; enforcing exactly `1000 / rate` would drop a frame that arrived a
+ * few milliseconds early. At 1 fps this is the 750 ms backstop the Gemini driver has always applied.
+ */
+export const VIDEO_FRAME_JITTER_HEADROOM = REALTIME_VIDEO_FRAME_JITTER_HEADROOM;
+/** The rate assumed when none was negotiated: 1 frame per second, the Live API ceiling today. */
+export const DEFAULT_INBOUND_VIDEO_RATE = REALTIME_DEFAULT_INBOUND_VIDEO_RATE;
+/**
+ * The nominal interval between frames, in milliseconds, for a negotiated rate (frames per second).
+ * An absent, zero, negative or non-finite rate falls back to {@link DEFAULT_INBOUND_VIDEO_RATE}.
+ */
+export function NominalVideoFrameIntervalMs(rate) {
+    return RealtimeVideoFrameIntervalMs(rate);
+}
+/**
+ * The minimum spacing, in milliseconds, a pacer should enforce between two frames to the model: the
+ * nominal interval less jitter headroom ({@link VIDEO_FRAME_JITTER_HEADROOM}).
+ */
+export function MinVideoFrameSpacingMs(rate) {
+    return RealtimeMinVideoFrameSpacingMs(rate);
+}
+//# sourceMappingURL=videoPacing.js.map

package/dist/media/videoPacing.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPacing.js","sourceRoot":"","sources":["../../src/media/videoPacing.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EACH,mCAAmC,EACnC,oCAAoC,EACpC,8BAA8B,EAC9B,4BAA4B,GAC/B,MAAM,oBAAoB,CAAC;AAE5B;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAG,oCAAoC,CAAC;AAEhF,iGAAiG;AACjG,MAAM,CAAC,MAAM,0BAA0B,GAAG,mCAAmC,CAAC;AAE9E;;;GAGG;AACH,MAAM,UAAU,2BAA2B,CAAC,IAA+B;IACvE,OAAO,4BAA4B,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAA+B;IAClE,OAAO,8BAA8B,CAAC,IAAI,CAAC,CAAC;AAChD,CAAC"}

package/dist/media/videoPlayout.d.ts

@@ -0,0 +1,168 @@
+/**
+ * @fileoverview VIDEO PLAYOUT: plays a model's video into one `<video>` element at a time, whatever form it arrives in.
+ * A provider hands each frame to {@link VideoPlayout.Append} as a `RealtimeVideoFrame` from `@memberjunction/ai`; a
+ * renderer shows {@link VideoPlayout.Source} with `AttachVideoSource`.
+ *
+ * - **A decoder per frame type.** The player keeps one decoder at a time, chosen from the page's
+ *   {@link VideoFrameDecoderRegistry} by the frame's kind and MIME type: `'mse-fmp4'` plays fragmented MP4 (a Gemini Live
+ *   avatar) through Media Source Extensions, `'webcodecs'` plays encoded chunks (H.264, VP8, VP9, AV1) through WebCodecs,
+ *   `'image'` plays still images. A frame of another type switches decoders; a type no decoder plays is dropped and
+ *   reported once (`'no-decoder'`).
+ * - **Ready before the first frame.** When an element is attached first, the player starts the decoder for its
+ *   configured type ({@link VideoPlayoutOptions.MimeType}), so MSE has its media source open when the first piece comes.
+ *   A configured type no decoder plays reports `'unsupported'` and leaves the element alone.
+ * - **Fallback.** A decoder that gives up (`Failed`) is disposed and not chosen again for that type; the next one that can
+ *   play it starts at the next init segment or key frame. Out of decoders, frames of that type are dropped; the voice is
+ *   unaffected, and the tile's stall fallback shows in place of the video.
+ * - **The voice.** An avatar's MP4 carries its voice, so the MSE decoder unmutes the element and
+ *   {@link VideoPlayout.IsPlaying} says whether the agent is audibly speaking. When the voice plays elsewhere,
+ *   {@link VideoPlayout.CarriesVoice} `false` mutes the element. Chunk and image decoders carry no voice.
+ *   {@link VideoPlayoutOptions.OnElementAttached} lets the realtime client route the element's audio into its own Web
+ *   Audio graph, so its meter and its recording carry the avatar's voice.
+ * - **Lip sync.** With {@link VideoPlayoutOptions.Clock}, the voice's playback clock, the chunk and image decoders show
+ *   each frame when the voice reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the voice keeps
+ *   voice and face together on its own timestamps.
+ * - **End of turn, barge-in, gaps.** {@link VideoPlayout.EndOfTurn} lets playback run to the true end of the turn;
+ *   {@link VideoPlayout.Flush} drops everything not yet played; between turns, or while a connection resumes, the element
+ *   holds the last frame. Each decoder does this its own way.
+ * - **Browsers.** {@link VideoPlayout.IsSupported} says, at once, whether some decoder can play a type here.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { type RealtimeVideoFrame } from '@memberjunction/ai';
+import type { MediaVideoSource } from './model.js';
+import type { IPlaybackClock } from './playbackClock.js';
+/** The type of a Gemini Live avatar: H.264 Constrained Baseline 3.1 video, AAC-LC audio. */
+export declare const GEMINI_AVATAR_MP4_TYPE = "video/mp4; codecs=\"avc1.42c01f, mp4a.40.2\"";
+export interface VideoPlayoutOptions {
+    /**
+     * The type, with codecs, the player expects: the decoder for it starts when an element is attached, before any frame
+     * arrives, and the MSE source buffer opens with it until an init segment names its own codecs (or names one the reader
+     * doesn't know). Defaults to {@link GEMINI_AVATAR_MP4_TYPE}.
+     */
+    MimeType?: string;
+    /** Seconds of played media to keep behind the playhead. Defaults to 10. */
+    BackBufferSeconds?: number;
+    /**
+     * Whether the element plays the media's audio. Defaults to `true`: an avatar's MP4 carries its voice. `false`
+     * mutes the element, for a voice that plays elsewhere. {@link VideoPlayout.CarriesVoice} changes it later.
+     */
+    CarriesVoice?: boolean;
+    /**
+     * Called with each element the player takes over, before playback starts, such as to route the element's audio
+     * into a Web Audio graph. A player that moves to another element calls it again with that one.
+     */
+    OnElementAttached?: (element: HTMLVideoElement) => void;
+    /**
+     * The voice's playback clock, for lip sync: the chunk and image decoders show each frame when the voice reaches its
+     * `PresentationTimeMs`. Give it when the driver times its PCM (`IRealtimePcmPlayback.Enqueue(pcm16, mediaTimeMs)`) and
+     * its frames on one media timeline; `RealtimePcmPlayback` is such a clock. Without it, frames go by their own times.
+     * MSE ignores it.
+     */
+    Clock?: IPlaybackClock;
+}
+/** Why playout reported a problem. */
+export type VideoPlayoutProblem = 
+/** The browser cannot play this type: no decoder plays the configured type, or a decoder can't decode the stream's codec. */
+'unsupported'
+/** No decoder in this browser plays the frame's kind and type; it was dropped. */
+ | 'no-decoder'
+/** A media fragment arrived before any init segment, or an encoded chunk before any key frame, and was dropped. */
+ | 'fragment-before-init'
+/** The browser rejected a piece (decode error, or out of buffer space after trimming). */
+ | 'append-failed'
+/** Media arrived faster than it could be played, or while no element was attached, and the oldest was dropped. */
+ | 'pending-overflow';
+/** Whether a piece is an MP4 init segment: its first box is `ftyp`. */
+export declare function IsMp4InitSegment(piece: ArrayBuffer): boolean;
+/**
+ * What a realtime driver needs from an avatar's video player: {@link VideoPlayout} in a browser, a fake in tests.
+ */
+export interface IAvatarVideoPlayout {
+    /** The video to show. A driver hands it to the host once. */
+    readonly Source: MediaVideoSource;
+    /** Whether it plays forward with media buffered ahead of the playhead: the avatar is audibly speaking. */
+    readonly IsPlaying: boolean;
+    /**
+     * Where the playhead is, in frames: how many of the frames handed to {@link Append} are still to play (waiting,
+     * being decoded or appended, or buffered ahead of the playhead). A frame that was dropped, or that no decoder plays,
+     * is not counted. Frames play in the order they were handed over, so the frames ahead are the latest ones handed
+     * over, less any of those that were dropped: a driver that counts what it hands over can tell whether a given frame
+     * has played.
+     */
+    readonly FramesAhead: number;
+    /** Whether the element plays the media's audio; `false` mutes it. */
+    CarriesVoice: boolean;
+    /**
+     * Hands over one frame, in the order they arrived: a piece of fragmented MP4 (an init segment or a media fragment),
+     * an encoded chunk or an image. A frame the player has no decoder for is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last piece has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn(): void;
+    /** Barge-in: drops everything not yet played and stops at once, holding the last frame. */
+    Flush(): void;
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler: (problem: VideoPlayoutProblem, message: string) => void): () => void;
+    /** Stops playout and releases the element. */
+    Dispose(): void;
+}
+/** Plays a model's video frames into one `<video>` element at a time, through the decoder each frame type needs. */
+export declare class VideoPlayout implements IAvatarVideoPlayout {
+    private readonly mimeType;
+    private readonly backBufferSeconds;
+    private readonly onElementAttached;
+    private readonly clock;
+    private readonly problemHandlers;
+    private readonly reported;
+    /** Decoders that gave up, by frame type: they are not chosen again for it. */
+    private readonly failed;
+    private carriesVoice;
+    private element;
+    private active;
+    private disposed;
+    /** Whether some decoder can play `mimeType` in this browser, answered at once (for negotiation, before connect). */
+    static IsSupported(mimeType?: string): boolean;
+    constructor(options?: VideoPlayoutOptions);
+    /** The video to show: the player takes over the element it is attached to. */
+    get Source(): MediaVideoSource;
+    /**
+     * Whether the element plays the media's audio. `false` mutes it: the voice plays elsewhere, such as separate PCM
+     * audio when the MP4 has no audio track. Takes effect at once on an attached element.
+     */
+    get CarriesVoice(): boolean;
+    set CarriesVoice(value: boolean);
+    /** Whether the decoder's own audio is audibly playing: an MP4 that carries the voice, buffered ahead of the playhead. */
+    get IsPlaying(): boolean;
+    /**
+     * How many of the frames handed over are still to play, as the decoder in use counts them. Frames a decoder that
+     * gave up held, and frames no decoder plays, never play, so they are not counted.
+     */
+    get FramesAhead(): number;
+    /**
+     * Hands over one frame, in the order they arrived, to the decoder for its type: the one in use, or the first the
+     * registry offers. A frame no decoder plays is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last frame has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn(): void;
+    /** Barge-in: drops everything not yet played and stops at once. The last frame stays on screen. */
+    Flush(): void;
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler: (problem: VideoPlayoutProblem, message: string) => void): () => void;
+    /** Stops playout, releases the element and disposes the decoder. */
+    Dispose(): void;
+    private attach;
+    private detachElement;
+    /** The decoder for the configured type, started before any frame so it is ready when the first one comes. */
+    private startConfiguredDecoder;
+    /** The decoder for a frame type: the one in use when it plays that type, else a new one from the registry. */
+    private decoderFor;
+    /** What a decoder chosen for `type` plays with and reports through. */
+    private contextFor;
+    /** A decoder gave up: it is disposed and not chosen again for its type. The next frame of the type picks another. */
+    private decoderFailed;
+    private releaseDecoder;
+    private report;
+}
+//# sourceMappingURL=videoPlayout.d.ts.map

package/dist/media/videoPlayout.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPlayout.d.ts","sourceRoot":"","sources":["../../src/media/videoPlayout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAkB,KAAK,kBAAkB,EAA+B,MAAM,oBAAoB,CAAC;AAC1G,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAKtD,4FAA4F;AAC5F,eAAO,MAAM,sBAAsB,iDAA+C,CAAC;AAKnF,MAAM,WAAW,mBAAmB;IAChC;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;OAGG;IACH,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACxD;;;;;OAKG;IACH,KAAK,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,sCAAsC;AACtC,MAAM,MAAM,mBAAmB;AAC3B,6HAA6H;AAC3H,aAAa;AACf,kFAAkF;GAChF,YAAY;AACd,mHAAmH;GACjH,sBAAsB;AACxB,0FAA0F;GACxF,eAAe;AACjB,kHAAkH;GAChH,kBAAkB,CAAC;AAEzB,uEAAuE;AACvE,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAE5D;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,0GAA0G;IAC1G,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;IAC5B;;;;;;OAMG;IACH,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,qEAAqE;IACrE,YAAY,EAAE,OAAO,CAAC;IACtB;;;OAGG;IACH,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,iGAAiG;IACjG,SAAS,IAAI,IAAI,CAAC;IAClB,2FAA2F;IAC3F,KAAK,IAAI,IAAI,CAAC;IACd,oFAAoF;IACpF,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI,CAAC;IACxF,8CAA8C;IAC9C,OAAO,IAAI,IAAI,CAAC;CACnB;AAWD,oHAAoH;AACpH,qBAAa,YAAa,YAAW,mBAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAoD;IACtF,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAsE;IACtG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkC;IACzD,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,MAAM,CAA8B;IAC5C,OAAO,CAAC,QAAQ,CAAS;IAEzB,oHAAoH;WACtG,WAAW,CAAC,QAAQ,GAAE,MAA+B,GAAG,OAAO;gBAKjE,OAAO,GAAE,mBAAwB;IAQ7C,8EAA8E;IAC9E,IAAW,MAAM,IAAI,gBAAgB,CAEpC;IAED;;;OAGG;IACH,IAAW,YAAY,IAAI,OAAO,CAEjC;IACD,IAAW,YAAY,CAAC,KAAK,EAAE,OAAO,EAKrC;IAED,yHAAyH;IACzH,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;OAGG;IACH,IAAW,WAAW,IAAI,MAAM,CAE/B;IAED;;;OAGG;IACI,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAa9C,iGAAiG;IAC1F,SAAS,IAAI,IAAI;IAIxB,mGAAmG;IAC5F,KAAK,IAAI,IAAI;IAIpB,oFAAoF;IAC7E,SAAS,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,mBAAmB,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI;IAK9F,oEAAoE;IAC7D,OAAO,IAAI,IAAI;IAOtB,OAAO,CAAC,MAAM;IAmBd,OAAO,CAAC,aAAa;IAOrB,6GAA6G;IAC7G,OAAO,CAAC,sBAAsB;IAK9B,8GAA8G;IAC9G,OAAO,CAAC,UAAU;IAmBlB,uEAAuE;IACvE,OAAO,CAAC,UAAU;IAelB,qHAAqH;IACrH,OAAO,CAAC,aAAa;IAUrB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,MAAM;CAUjB"}

package/dist/media/videoPlayout.js

@@ -0,0 +1,217 @@
+/**
+ * @fileoverview VIDEO PLAYOUT: plays a model's video into one `<video>` element at a time, whatever form it arrives in.
+ * A provider hands each frame to {@link VideoPlayout.Append} as a `RealtimeVideoFrame` from `@memberjunction/ai`; a
+ * renderer shows {@link VideoPlayout.Source} with `AttachVideoSource`.
+ *
+ * - **A decoder per frame type.** The player keeps one decoder at a time, chosen from the page's
+ *   {@link VideoFrameDecoderRegistry} by the frame's kind and MIME type: `'mse-fmp4'` plays fragmented MP4 (a Gemini Live
+ *   avatar) through Media Source Extensions, `'webcodecs'` plays encoded chunks (H.264, VP8, VP9, AV1) through WebCodecs,
+ *   `'image'` plays still images. A frame of another type switches decoders; a type no decoder plays is dropped and
+ *   reported once (`'no-decoder'`).
+ * - **Ready before the first frame.** When an element is attached first, the player starts the decoder for its
+ *   configured type ({@link VideoPlayoutOptions.MimeType}), so MSE has its media source open when the first piece comes.
+ *   A configured type no decoder plays reports `'unsupported'` and leaves the element alone.
+ * - **Fallback.** A decoder that gives up (`Failed`) is disposed and not chosen again for that type; the next one that can
+ *   play it starts at the next init segment or key frame. Out of decoders, frames of that type are dropped; the voice is
+ *   unaffected, and the tile's stall fallback shows in place of the video.
+ * - **The voice.** An avatar's MP4 carries its voice, so the MSE decoder unmutes the element and
+ *   {@link VideoPlayout.IsPlaying} says whether the agent is audibly speaking. When the voice plays elsewhere,
+ *   {@link VideoPlayout.CarriesVoice} `false` mutes the element. Chunk and image decoders carry no voice.
+ *   {@link VideoPlayoutOptions.OnElementAttached} lets the realtime client route the element's audio into its own Web
+ *   Audio graph, so its meter and its recording carry the avatar's voice.
+ * - **Lip sync.** With {@link VideoPlayoutOptions.Clock}, the voice's playback clock, the chunk and image decoders show
+ *   each frame when the voice reaches its `PresentationTimeMs`. MSE needs no clock: an MP4 that carries the voice keeps
+ *   voice and face together on its own timestamps.
+ * - **End of turn, barge-in, gaps.** {@link VideoPlayout.EndOfTurn} lets playback run to the true end of the turn;
+ *   {@link VideoPlayout.Flush} drops everything not yet played; between turns, or while a connection resumes, the element
+ *   holds the last frame. Each decoder does this its own way.
+ * - **Browsers.** {@link VideoPlayout.IsSupported} says, at once, whether some decoder can play a type here.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { SniffFmp4Piece } from '@memberjunction/ai';
+import { VideoFrameDecoderRegistry } from './videoFrameDecoderRegistry.js';
+import { FrameKindOfMimeType, FrameTypeKey } from './decoders/videoMimeTypes.js';
+/** The type of a Gemini Live avatar: H.264 Constrained Baseline 3.1 video, AAC-LC audio. */
+export const GEMINI_AVATAR_MP4_TYPE = 'video/mp4; codecs="avc1.42c01f, mp4a.40.2"';
+/** Seconds of already-played media kept behind the playhead before it is removed. */
+const DEFAULT_BACK_BUFFER_SECONDS = 10;
+/** Whether a piece is an MP4 init segment: its first box is `ftyp`. */
+export function IsMp4InitSegment(piece) {
+    return SniffFmp4Piece(piece) === 'init';
+}
+/** Plays a model's video frames into one `<video>` element at a time, through the decoder each frame type needs. */
+export class VideoPlayout {
+    /** Whether some decoder can play `mimeType` in this browser, answered at once (for negotiation, before connect). */
+    static IsSupported(mimeType = GEMINI_AVATAR_MP4_TYPE) {
+        const kind = FrameKindOfMimeType(mimeType);
+        return kind !== null && VideoFrameDecoderRegistry.Instance.Candidates(kind, mimeType).length > 0;
+    }
+    constructor(options = {}) {
+        this.problemHandlers = new Set();
+        this.reported = new Set();
+        /** Decoders that gave up, by frame type: they are not chosen again for it. */
+        this.failed = new Map();
+        this.element = null;
+        this.active = null;
+        this.disposed = false;
+        this.mimeType = options.MimeType ?? GEMINI_AVATAR_MP4_TYPE;
+        this.backBufferSeconds = options.BackBufferSeconds ?? DEFAULT_BACK_BUFFER_SECONDS;
+        this.carriesVoice = options.CarriesVoice ?? true;
+        this.onElementAttached = options.OnElementAttached;
+        this.clock = options.Clock;
+    }
+    /** The video to show: the player takes over the element it is attached to. */
+    get Source() {
+        return { Kind: 'element', Attach: (element) => this.attach(element) };
+    }
+    /**
+     * Whether the element plays the media's audio. `false` mutes it: the voice plays elsewhere, such as separate PCM
+     * audio when the MP4 has no audio track. Takes effect at once on an attached element.
+     */
+    get CarriesVoice() {
+        return this.carriesVoice;
+    }
+    set CarriesVoice(value) {
+        this.carriesVoice = value;
+        if (this.element) {
+            this.element.muted = !value;
+        }
+    }
+    /** Whether the decoder's own audio is audibly playing: an MP4 that carries the voice, buffered ahead of the playhead. */
+    get IsPlaying() {
+        return this.active?.Decoder.IsPlaying ?? false;
+    }
+    /**
+     * How many of the frames handed over are still to play, as the decoder in use counts them. Frames a decoder that
+     * gave up held, and frames no decoder plays, never play, so they are not counted.
+     */
+    get FramesAhead() {
+        return this.active?.Decoder.FramesAhead ?? 0;
+    }
+    /**
+     * Hands over one frame, in the order they arrived, to the decoder for its type: the one in use, or the first the
+     * registry offers. A frame no decoder plays is dropped and reported once (`'no-decoder'`).
+     */
+    Append(frame) {
+        if (this.disposed) {
+            return;
+        }
+        const decoder = this.decoderFor(frame.Kind, frame.MimeType);
+        if (!decoder) {
+            // A frame it can't play adds no media: the decoder in use, and a turn it was about to end, carry on.
+            this.report('no-decoder', `No decoder in this browser plays ${frame.Kind} frames of type ${frame.MimeType}; the frame was dropped.`);
+            return;
+        }
+        decoder.Append(frame);
+    }
+    /** The turn's last frame has arrived: playback runs to its true end and holds the last frame. */
+    EndOfTurn() {
+        this.active?.Decoder.EndOfTurn();
+    }
+    /** Barge-in: drops everything not yet played and stops at once. The last frame stays on screen. */
+    Flush() {
+        this.active?.Decoder.Flush();
+    }
+    /** Reports problems, once per kind. Returns a function that removes the handler. */
+    OnProblem(handler) {
+        this.problemHandlers.add(handler);
+        return () => this.problemHandlers.delete(handler);
+    }
+    /** Stops playout, releases the element and disposes the decoder. */
+    Dispose() {
+        this.disposed = true;
+        this.releaseDecoder();
+        this.element = null;
+        this.problemHandlers.clear();
+    }
+    attach(element) {
+        if (this.disposed) {
+            return () => undefined;
+        }
+        this.detachElement();
+        const decoder = this.active?.Decoder ?? this.startConfiguredDecoder();
+        if (!decoder) {
+            this.report('unsupported', `This browser cannot play ${this.mimeType}: no decoder plays it.`);
+            return () => undefined;
+        }
+        this.element = element;
+        decoder.Attach(element);
+        return () => {
+            if (this.element === element) {
+                this.detachElement();
+            }
+        };
+    }
+    detachElement() {
+        if (this.element) {
+            this.active?.Decoder.Detach();
+            this.element = null;
+        }
+    }
+    /** The decoder for the configured type, started before any frame so it is ready when the first one comes. */
+    startConfiguredDecoder() {
+        const kind = FrameKindOfMimeType(this.mimeType);
+        return kind ? this.decoderFor(kind, this.mimeType) : null;
+    }
+    /** The decoder for a frame type: the one in use when it plays that type, else a new one from the registry. */
+    decoderFor(kind, mimeType) {
+        const type = FrameTypeKey(kind, mimeType);
+        if (this.active?.Type === type) {
+            return this.active.Decoder;
+        }
+        const failed = this.failed.get(type);
+        const registration = VideoFrameDecoderRegistry.Instance.Candidates(kind, mimeType).find((candidate) => !failed?.has(candidate.Name));
+        if (!registration) {
+            return null;
+        }
+        this.releaseDecoder();
+        const decoder = registration.Create(this.contextFor(type, registration.Name));
+        this.active = { Type: type, Name: registration.Name, Decoder: decoder };
+        if (this.element) {
+            decoder.Attach(this.element);
+        }
+        return decoder;
+    }
+    /** What a decoder chosen for `type` plays with and reports through. */
+    contextFor(type, name) {
+        const carriesVoice = () => this.carriesVoice;
+        return {
+            MimeType: this.mimeType,
+            BackBufferSeconds: this.backBufferSeconds,
+            get CarriesVoice() {
+                return carriesVoice();
+            },
+            OnElementAttached: this.onElementAttached,
+            Clock: this.clock,
+            Report: (problem, message) => this.report(problem, message),
+            Failed: (message) => this.decoderFailed(type, name, message),
+        };
+    }
+    /** A decoder gave up: it is disposed and not chosen again for its type. The next frame of the type picks another. */
+    decoderFailed(type, name, message) {
+        console.warn(`[VideoPlayout] The ${name} decoder stopped: ${message}`);
+        const failed = this.failed.get(type) ?? new Set();
+        failed.add(name);
+        this.failed.set(type, failed);
+        if (this.active?.Type === type && this.active.Name === name) {
+            this.releaseDecoder();
+        }
+    }
+    releaseDecoder() {
+        const active = this.active;
+        this.active = null;
+        active?.Decoder.Dispose();
+    }
+    report(problem, message) {
+        if (this.reported.has(problem)) {
+            return;
+        }
+        this.reported.add(problem);
+        console.warn(`[VideoPlayout] ${message}`);
+        for (const handler of [...this.problemHandlers]) {
+            handler(problem, message);
+        }
+    }
+}
+//# sourceMappingURL=videoPlayout.js.map

package/dist/media/videoPlayout.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoPlayout.js","sourceRoot":"","sources":["../../src/media/videoPlayout.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,cAAc,EAAwD,MAAM,oBAAoB,CAAC;AAI1G,OAAO,EAAE,yBAAyB,EAAE,MAAM,6BAA6B,CAAC;AACxE,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAE9E,4FAA4F;AAC5F,MAAM,CAAC,MAAM,sBAAsB,GAAG,4CAA4C,CAAC;AAEnF,qFAAqF;AACrF,MAAM,2BAA2B,GAAG,EAAE,CAAC;AA2CvC,uEAAuE;AACvE,MAAM,UAAU,gBAAgB,CAAC,KAAkB;IAC/C,OAAO,cAAc,CAAC,KAAK,CAAC,KAAK,MAAM,CAAC;AAC5C,CAAC;AA4CD,oHAAoH;AACpH,MAAM,OAAO,YAAY;IAcrB,oHAAoH;IAC7G,MAAM,CAAC,WAAW,CAAC,WAAmB,sBAAsB;QAC/D,MAAM,IAAI,GAAG,mBAAmB,CAAC,QAAQ,CAAC,CAAC;QAC3C,OAAO,IAAI,KAAK,IAAI,IAAI,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACrG,CAAC;IAED,YAAY,UAA+B,EAAE;QAf5B,oBAAe,GAAG,IAAI,GAAG,EAA2D,CAAC;QACrF,aAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC3D,8EAA8E;QAC7D,WAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;QAEjD,YAAO,GAA4B,IAAI,CAAC;QACxC,WAAM,GAAyB,IAAI,CAAC;QACpC,aAAQ,GAAG,KAAK,CAAC;QASrB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,sBAAsB,CAAC;QAC3D,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,2BAA2B,CAAC;QAClF,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QACjD,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;QACnD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC/B,CAAC;IAED,8EAA8E;IAC9E,IAAW,MAAM;QACb,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IAC1E,CAAC;IAED;;;OAGG;IACH,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,YAAY,CAAC;IAC7B,CAAC;IACD,IAAW,YAAY,CAAC,KAAc;QAClC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,KAAK,GAAG,CAAC,KAAK,CAAC;QAChC,CAAC;IACL,CAAC;IAED,yHAAyH;IACzH,IAAW,SAAS;QAChB,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IACnD,CAAC;IAED;;;OAGG;IACH,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;IACjD,CAAC;IAED;;;OAGG;IACI,MAAM,CAAC,KAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC5D,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,qGAAqG;YACrG,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,oCAAoC,KAAK,CAAC,IAAI,mBAAmB,KAAK,CAAC,QAAQ,0BAA0B,CAAC,CAAC;YACrI,OAAO;QACX,CAAC;QACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC;IAED,iGAAiG;IAC1F,SAAS;QACZ,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;IACrC,CAAC;IAED,mGAAmG;IAC5F,KAAK;QACR,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAED,oFAAoF;IAC7E,SAAS,CAAC,OAAgE;QAC7E,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtD,CAAC;IAED,oEAAoE;IAC7D,OAAO;QACV,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAEO,MAAM,CAAC,OAAyB;QACpC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACtE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,4BAA4B,IAAI,CAAC,QAAQ,wBAAwB,CAAC,CAAC;YAC9F,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACxB,OAAO,GAAG,EAAE;YACR,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;gBAC3B,IAAI,CAAC,aAAa,EAAE,CAAC;YACzB,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAEO,aAAa;QACjB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;IACL,CAAC;IAED,6GAA6G;IACrG,sBAAsB;QAC1B,MAAM,IAAI,GAAG,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,8GAA8G;IACtG,UAAU,CAAC,IAA4B,EAAE,QAAgB;QAC7D,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,IAAI,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC;QAC/B,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,yBAAyB,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;QACrI,IAAI,CAAC,YAAY,EAAE,CAAC;YAChB,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9E,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,uEAAuE;IAC/D,UAAU,CAAC,IAAY,EAAE,IAAY;QACzC,MAAM,YAAY,GAAG,GAAY,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC;QACtD,OAAO;YACH,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,IAAI,YAAY;gBACZ,OAAO,YAAY,EAAE,CAAC;YAC1B,CAAC;YACD,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;YACzC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC;YAC3D,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC;SAC/D,CAAC;IACN,CAAC;IAED,qHAAqH;IAC7G,aAAa,CAAC,IAAY,EAAE,IAAY,EAAE,OAAe;QAC7D,OAAO,CAAC,IAAI,CAAC,sBAAsB,IAAI,qBAAqB,OAAO,EAAE,CAAC,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;QAC1D,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9B,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YAC1D,IAAI,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;IACL,CAAC;IAEO,cAAc;QAClB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9B,CAAC;IAEO,MAAM,CAAC,OAA4B,EAAE,OAAe;QACxD,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,CAAC,IAAI,CAAC,kBAAkB,OAAO,EAAE,CAAC,CAAC;QAC1C,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;YAC9C,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;CACJ"}