@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.4

package/dist/media/decoders/mseFmp4Decoder.d.ts

@@ -0,0 +1,130 @@
+/**
+ * @fileoverview MSE FMP4 DECODER: plays fragmented MP4 frames (`Kind: 'fmp4'`), such as a Gemini Live avatar, through
+ * Media Source Extensions into the host's `<video>` element. The `'mse-fmp4'` decoder of `VideoPlayout`.
+ *
+ * - **Format.** The first piece of a turn is an init segment (`Piece: 'init'`, `ftyp` + `moov`); each later piece is a
+ *   media fragment (`moof` + `mdat`). A turn may send a fresh init segment and restart its timestamps; the source buffer
+ *   runs in `sequence` mode, so playback stays continuous either way.
+ * - **Codecs come from the init segment.** The source buffer takes the codecs the init segment's tracks name
+ *   (`avc1.42c01f` alone for video-only MP4), read by `ReadFmp4Init` from `@memberjunction/ai`; until an init arrives, or
+ *   when it names a codec the reader doesn't know, it takes the player's configured type.
+ * - **The video carries the voice.** An avatar's audio is muxed into the MP4, so the element is unmuted and lips and voice
+ *   stay in sync on the media's own timestamps. While media is buffered ahead of the playhead, {@link MseFmp4Decoder.IsPlaying}
+ *   is true: the agent is audibly speaking. When the voice plays elsewhere (separate PCM), the element is muted. The
+ *   element-attached hook lets the realtime client route the element's audio into its own Web Audio graph.
+ * - **End of turn** ends the stream once the turn's pieces are in, so playback runs to the true end and holds the last
+ *   frame. **Barge-in** drops everything not yet played and pauses. **Gaps** (between turns, or while a connection
+ *   resumes) hold the last frame.
+ * - **Browsers.** `MediaSource` (Chrome, Edge, Firefox, desktop Safari), or `ManagedMediaSource` (iOS Safari 17.1+).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { type RealtimeVideoFrame } from '@memberjunction/ai';
+import type { IVideoFrameDecoder, VideoFrameDecoderContext, VideoFrameDecoderRegistration } from '../videoFrameDecoder.js';
+/** Plays fragmented MP4 frames through MSE into one `<video>` element at a time. */
+export declare class MseFmp4Decoder implements IVideoFrameDecoder {
+    private readonly context;
+    private readonly mimeType;
+    private readonly backBufferSeconds;
+    private readonly pending;
+    private element;
+    private mediaSource;
+    private objectUrl;
+    private sourceBuffer;
+    /** The type the source buffer was opened with, or last changed to. */
+    private bufferType;
+    /** The latest init segment, replayed into a new source buffer when the decoder moves to another element. */
+    private lastInit;
+    /** Set by Flush: the next append starts at the playhead instead of after what was dropped. */
+    private restartAtPlayhead;
+    private retriedAfterQuota;
+    /** Where the last back-buffer trim ended, in media time. */
+    private trimmedTo;
+    /** Set by EndOfTurn: end the stream once every pending piece is in. */
+    private endPending;
+    /** Where each appended piece ends on the element's timeline (seconds), oldest first, until the playhead passes it. */
+    private readonly appendedEnds;
+    /** Whether the source buffer is appending a piece: its `updateend` records where the piece ends. */
+    private appending;
+    private disposed;
+    /** Whether this browser can play `mimeType` through MSE. */
+    static CanPlay(mimeType: string): boolean;
+    /** @param context The player this decoder plays for. */
+    constructor(context: VideoFrameDecoderContext);
+    /**
+     * Whether the element is playing forward with media buffered ahead of the playhead: the agent is audibly
+     * speaking. False while it waits for data, which it does just short of the end of a turn that hasn't
+     * been ended with {@link EndOfTurn}.
+     */
+    get IsPlaying(): boolean;
+    /**
+     * How many pieces are still to play: pending (waiting for the source buffer, or for an element), being appended, or
+     * appended and ending past the playhead. The source buffer lays pieces end to end (`sequence` mode), so a piece ends
+     * where the buffered media ended once it was in. A piece a {@link Flush} removed, or the browser rejected, is not
+     * counted; nor is media of an element the decoder let go.
+     */
+    get FramesAhead(): number;
+    /** Hands over one frame, in the order they arrived: an init segment or a media fragment. */
+    Append(frame: RealtimeVideoFrame): void;
+    /**
+     * The turn's last piece has arrived. Ends the stream once it is all in, so playback runs to the true end of
+     * the turn; the next {@link Append} reopens it and playback continues from there.
+     */
+    EndOfTurn(): void;
+    /**
+     * Barge-in: drops everything not yet played and pauses, so the voice stops now instead of after the audio the
+     * browser has already decoded. The last frame stays on screen; the next turn's media resumes playback.
+     */
+    Flush(): void;
+    /** Stops playout and releases the element and the media source. */
+    Dispose(): void;
+    /** Takes the element over: a new media source, the latest init segment replayed into it, then what is pending. */
+    Attach(element: HTMLVideoElement): void;
+    /** Lets the element go: its source is removed and the media source released. */
+    Detach(): void;
+    /** Readies an element before it plays: its audio on or off, the attach hook, and what iOS's managed MSE requires. */
+    private prepareElement;
+    private openSourceBuffer;
+    /** Appends the next pending frame when the buffer is free. */
+    private pump;
+    /** The type for a source buffer that takes `init`: the init's own codecs when this browser plays them, else the player's type. */
+    private typeFor;
+    /**
+     * Switches the source buffer to an init segment's codecs before the init is appended, when they differ from the
+     * buffer's: it opened with the player's type before the first init arrived, or a later init changed tracks. A browser
+     * without `changeType`, or an init whose codecs can't be read or played, leaves the buffer as it is.
+     */
+    private matchBufferType;
+    /** After each append or removal: trims played media, appends the next piece, and ends a finished turn. */
+    private afterUpdate;
+    /**
+     * An append finished: the piece ends where the buffered media now ends, since the buffer lays pieces end to end.
+     * Pieces the playhead has passed are forgotten here too, so the list holds only what is ahead.
+     */
+    private recordAppendedEnd;
+    /** Forgets the appended pieces the playhead has reached the end of (to within {@link PLAYING_EPSILON_SECONDS}). */
+    private forgetPlayedPieces;
+    /** Ends the stream for a finished turn once nothing is pending or in flight. */
+    private endStreamWhenDrained;
+    /**
+     * A turn that ran to its end, or a Flush, leaves the element paused. Once new media reaches past the playhead,
+     * play on from there. An ended element is first sought to where it stopped: play() on an ended element starts
+     * over from the beginning (Chrome does), which would replay the previous turn.
+     */
+    private resumeWhenMediaArrives;
+    /**
+     * Removes played media more than the back buffer's seconds behind the playhead, so a long session stays within the
+     * browser's buffer quota. A trim waits until {@link TRIM_STEP_SECONDS} more have played since the last one, so a
+     * removal that doesn't shrink the buffer can't repeat on every `updateend`. When `bufferFull`, keeps only
+     * {@link QUOTA_BACK_BUFFER_SECONDS} and skips the wait.
+     *
+     * @returns Whether a removal started (the next append waits for its `updateend`).
+     */
+    private trimBackBuffer;
+    /** Out of buffer space: trim harder and retry once. Anything else drops the piece. */
+    private handleAppendError;
+    private detach;
+}
+/** The `'mse-fmp4'` decoder: fragmented MP4 frames whose type this browser's Media Source Extensions play. */
+export declare const MSE_FMP4_DECODER: VideoFrameDecoderRegistration;
+//# sourceMappingURL=mseFmp4Decoder.d.ts.map

package/dist/media/decoders/mseFmp4Decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mseFmp4Decoder.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/mseFmp4Decoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAA6C,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxG,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AA2CxH,oFAAoF;AACpF,qBAAa,cAAe,YAAW,kBAAkB;IA+BzC,OAAO,CAAC,QAAQ,CAAC,OAAO;IA9BpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IACxD,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,sEAAsE;IACtE,OAAO,CAAC,UAAU,CAAuB;IACzC,4GAA4G;IAC5G,OAAO,CAAC,QAAQ,CAAuC;IACvD,8FAA8F;IAC9F,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,iBAAiB,CAAS;IAClC,4DAA4D;IAC5D,OAAO,CAAC,SAAS,CAAK;IACtB,uEAAuE;IACvE,OAAO,CAAC,UAAU,CAAS;IAC3B,sHAAsH;IACtH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,oGAAoG;IACpG,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;IAEzB,4DAA4D;WAC9C,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIhD,wDAAwD;gBAC3B,OAAO,EAAE,wBAAwB;IAK9D;;;;OAIG;IACH,IAAW,SAAS,IAAI,OAAO,CAO9B;IAED;;;;;OAKG;IACH,IAAW,WAAW,IAAI,MAAM,CAG/B;IAED,4FAA4F;IACrF,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAoB9C;;;OAGG;IACI,SAAS,IAAI,IAAI;IAKxB;;;OAGG;IACI,KAAK,IAAI,IAAI;IAsBpB,mEAAmE;IAC5D,OAAO,IAAI,IAAI;IAMtB,kHAAkH;IAC3G,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAsB9C,gFAAgF;IACzE,MAAM,IAAI,IAAI;IAIrB,qHAAqH;IACrH,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,gBAAgB;IAmBxB,8DAA8D;IAC9D,OAAO,CAAC,IAAI;IAuBZ,kIAAkI;IAClI,OAAO,CAAC,OAAO;IAKf;;;;OAIG;IACH,OAAO,CAAC,eAAe;IASvB,0GAA0G;IAC1G,OAAO,CAAC,WAAW;IAUnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAUzB,mHAAmH;IACnH,OAAO,CAAC,kBAAkB;IAU1B,gFAAgF;IAChF,OAAO,CAAC,oBAAoB;IAa5B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAgBtB,sFAAsF;IACtF,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,MAAM;CAoBjB;AAED,8GAA8G;AAC9G,eAAO,MAAM,gBAAgB,EAAE,6BAM9B,CAAC"}

package/dist/media/decoders/mseFmp4Decoder.js

@@ -0,0 +1,402 @@
+/**
+ * @fileoverview MSE FMP4 DECODER: plays fragmented MP4 frames (`Kind: 'fmp4'`), such as a Gemini Live avatar, through
+ * Media Source Extensions into the host's `<video>` element. The `'mse-fmp4'` decoder of `VideoPlayout`.
+ *
+ * - **Format.** The first piece of a turn is an init segment (`Piece: 'init'`, `ftyp` + `moov`); each later piece is a
+ *   media fragment (`moof` + `mdat`). A turn may send a fresh init segment and restart its timestamps; the source buffer
+ *   runs in `sequence` mode, so playback stays continuous either way.
+ * - **Codecs come from the init segment.** The source buffer takes the codecs the init segment's tracks name
+ *   (`avc1.42c01f` alone for video-only MP4), read by `ReadFmp4Init` from `@memberjunction/ai`; until an init arrives, or
+ *   when it names a codec the reader doesn't know, it takes the player's configured type.
+ * - **The video carries the voice.** An avatar's audio is muxed into the MP4, so the element is unmuted and lips and voice
+ *   stay in sync on the media's own timestamps. While media is buffered ahead of the playhead, {@link MseFmp4Decoder.IsPlaying}
+ *   is true: the agent is audibly speaking. When the voice plays elsewhere (separate PCM), the element is muted. The
+ *   element-attached hook lets the realtime client route the element's audio into its own Web Audio graph.
+ * - **End of turn** ends the stream once the turn's pieces are in, so playback runs to the true end and holds the last
+ *   frame. **Barge-in** drops everything not yet played and pauses. **Gaps** (between turns, or while a connection
+ *   resumes) hold the last frame.
+ * - **Browsers.** `MediaSource` (Chrome, Edge, Firefox, desktop Safari), or `ManagedMediaSource` (iOS Safari 17.1+).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { ReadFmp4Init } from '@memberjunction/ai';
+/** Back-buffer trims wait until this many more seconds have played, so a removal can never repeat in a loop. */
+const TRIM_STEP_SECONDS = 5;
+/** Seconds of played media kept when the browser's buffer is full. */
+const QUOTA_BACK_BUFFER_SECONDS = 2;
+/** Pieces held while no element is attached; older ones are dropped beyond this. */
+const MAX_PENDING_PIECES = 600;
+/** How far ahead of the playhead counts as "still playing", in seconds. */
+const PLAYING_EPSILON_SECONDS = 0.05;
+/** `HTMLMediaElement.HAVE_FUTURE_DATA`: the element has data to play beyond the current position. */
+const HAVE_FUTURE_DATA = 3;
+/** The MSE implementation this browser offers: the classic one, else the managed one. */
+function mediaSourceClass() {
+    const scope = globalThis;
+    if (scope.MediaSource) {
+        return { Class: scope.MediaSource, Managed: false };
+    }
+    return scope.ManagedMediaSource ? { Class: scope.ManagedMediaSource, Managed: true } : null;
+}
+/**
+ * The MSE type an init segment's video and audio tracks need, from their codecs: `video/mp4; codecs="avc1.42c01f,
+ * mp4a.40.2"` for a muxed avatar, `video/mp4; codecs="avc1.42c01f"` for video alone. `null` when the init can't be read,
+ * declares neither, or names a codec the reader doesn't know.
+ */
+function mseTypeOf(init) {
+    const tracks = ReadFmp4Init(init)?.Tracks.filter((track) => track.Handler === 'vide' || track.Handler === 'soun') ?? [];
+    if (tracks.length === 0 || tracks.some((track) => !track.Codec)) {
+        return null;
+    }
+    return `video/mp4; codecs="${tracks.map((track) => track.Codec).join(', ')}"`;
+}
+/** Plays fragmented MP4 frames through MSE into one `<video>` element at a time. */
+export class MseFmp4Decoder {
+    /** Whether this browser can play `mimeType` through MSE. */
+    static CanPlay(mimeType) {
+        return mediaSourceClass()?.Class.isTypeSupported(mimeType) ?? false;
+    }
+    /** @param context The player this decoder plays for. */
+    constructor(context) {
+        this.context = context;
+        this.pending = [];
+        this.element = null;
+        this.mediaSource = null;
+        this.objectUrl = null;
+        this.sourceBuffer = null;
+        /** The type the source buffer was opened with, or last changed to. */
+        this.bufferType = null;
+        /** The latest init segment, replayed into a new source buffer when the decoder moves to another element. */
+        this.lastInit = null;
+        /** Set by Flush: the next append starts at the playhead instead of after what was dropped. */
+        this.restartAtPlayhead = false;
+        this.retriedAfterQuota = false;
+        /** Where the last back-buffer trim ended, in media time. */
+        this.trimmedTo = 0;
+        /** Set by EndOfTurn: end the stream once every pending piece is in. */
+        this.endPending = false;
+        /** Where each appended piece ends on the element's timeline (seconds), oldest first, until the playhead passes it. */
+        this.appendedEnds = [];
+        /** Whether the source buffer is appending a piece: its `updateend` records where the piece ends. */
+        this.appending = false;
+        this.disposed = false;
+        this.mimeType = context.MimeType;
+        this.backBufferSeconds = context.BackBufferSeconds;
+    }
+    /**
+     * Whether the element is playing forward with media buffered ahead of the playhead: the agent is audibly
+     * speaking. False while it waits for data, which it does just short of the end of a turn that hasn't
+     * been ended with {@link EndOfTurn}.
+     */
+    get IsPlaying() {
+        const element = this.element;
+        const buffered = this.sourceBuffer?.buffered;
+        if (!element || element.paused || element.readyState < HAVE_FUTURE_DATA || !buffered || buffered.length === 0) {
+            return false;
+        }
+        return buffered.end(buffered.length - 1) - element.currentTime > PLAYING_EPSILON_SECONDS;
+    }
+    /**
+     * How many pieces are still to play: pending (waiting for the source buffer, or for an element), being appended, or
+     * appended and ending past the playhead. The source buffer lays pieces end to end (`sequence` mode), so a piece ends
+     * where the buffered media ended once it was in. A piece a {@link Flush} removed, or the browser rejected, is not
+     * counted; nor is media of an element the decoder let go.
+     */
+    get FramesAhead() {
+        this.forgetPlayedPieces();
+        return this.pending.length + (this.appending ? 1 : 0) + this.appendedEnds.length;
+    }
+    /** Hands over one frame, in the order they arrived: an init segment or a media fragment. */
+    Append(frame) {
+        if (this.disposed || frame.Kind !== 'fmp4') {
+            return;
+        }
+        // More media is coming, so a turn that was about to end continues instead.
+        this.endPending = false;
+        if (frame.Piece === 'init') {
+            this.lastInit = frame;
+        }
+        else if (!this.lastInit) {
+            this.context.Report('fragment-before-init', 'A video fragment arrived before its init segment and was dropped.');
+            return;
+        }
+        this.pending.push(frame);
+        if (this.pending.length > MAX_PENDING_PIECES) {
+            this.pending.splice(0, this.pending.length - MAX_PENDING_PIECES);
+            this.context.Report('pending-overflow', 'Video arrived with no element attached; the oldest pieces were dropped.');
+        }
+        this.pump();
+    }
+    /**
+     * The turn's last piece has arrived. Ends the stream once it is all in, so playback runs to the true end of
+     * the turn; the next {@link Append} reopens it and playback continues from there.
+     */
+    EndOfTurn() {
+        this.endPending = true;
+        this.endStreamWhenDrained();
+    }
+    /**
+     * Barge-in: drops everything not yet played and pauses, so the voice stops now instead of after the audio the
+     * browser has already decoded. The last frame stays on screen; the next turn's media resumes playback.
+     */
+    Flush() {
+        this.pending.length = 0;
+        this.endPending = false;
+        // What is buffered ahead of the playhead is removed, and an append under way is aborted: none of it plays.
+        this.appendedEnds.length = 0;
+        this.appending = false;
+        const buffer = this.sourceBuffer;
+        const element = this.element;
+        if (!buffer || !element) {
+            return;
+        }
+        element.pause();
+        if (buffer.updating) {
+            buffer.abort();
+        }
+        const buffered = buffer.buffered;
+        if (buffered.length > 0 && buffered.end(buffered.length - 1) > element.currentTime) {
+            buffer.remove(element.currentTime, Infinity);
+        }
+        this.restartAtPlayhead = true;
+    }
+    /** Stops playout and releases the element and the media source. */
+    Dispose() {
+        this.disposed = true;
+        this.pending.length = 0;
+        this.detach();
+    }
+    /** Takes the element over: a new media source, the latest init segment replayed into it, then what is pending. */
+    Attach(element) {
+        if (this.disposed) {
+            return;
+        }
+        this.detach();
+        const mse = mediaSourceClass();
+        if (!mse || !mse.Class.isTypeSupported(this.mimeType)) {
+            this.context.Report('unsupported', `This browser cannot play ${this.mimeType} through Media Source Extensions.`);
+            return;
+        }
+        const mediaSource = new mse.Class();
+        this.mediaSource = mediaSource;
+        this.element = element;
+        this.prepareElement(element, mse.Managed);
+        mediaSource.addEventListener('sourceopen', () => this.openSourceBuffer(mediaSource), { once: true });
+        this.objectUrl = URL.createObjectURL(mediaSource);
+        element.src = this.objectUrl;
+        void element.play().catch(() => {
+            // Playback starts when the first media arrives, or on the next user gesture if the browser waits.
+        });
+    }
+    /** Lets the element go: its source is removed and the media source released. */
+    Detach() {
+        this.detach();
+    }
+    /** Readies an element before it plays: its audio on or off, the attach hook, and what iOS's managed MSE requires. */
+    prepareElement(element, managed) {
+        if (managed) {
+            // ManagedMediaSource plays only when remote playback (AirPlay) is off or has an alternative source.
+            element.disableRemotePlayback = true;
+        }
+        // Set either way: a host may hand over an element it muted (a media tile never plays audio itself).
+        element.muted = !this.context.CarriesVoice;
+        try {
+            this.context.OnElementAttached?.(element);
+        }
+        catch (err) {
+            console.warn(`[VideoPlayout] The element-attached hook failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    openSourceBuffer(mediaSource) {
+        if (this.mediaSource !== mediaSource) {
+            return;
+        }
+        const type = this.typeFor(this.lastInit?.Data ?? null);
+        const buffer = mediaSource.addSourceBuffer(type);
+        this.bufferType = type;
+        // Sequence mode lays pieces end to end, so a turn that restarts its timestamps still plays on.
+        buffer.mode = 'sequence';
+        buffer.addEventListener('updateend', () => this.afterUpdate());
+        buffer.addEventListener('error', () => this.context.Report('append-failed', 'The browser could not decode a video piece.'));
+        this.sourceBuffer = buffer;
+        // A new source buffer knows no tracks yet: lead with the latest init segment unless one is already next.
+        if (this.lastInit && !(this.pending.length > 0 && this.pending[0].Piece === 'init')) {
+            this.pending.unshift(this.lastInit);
+        }
+        this.pump();
+    }
+    /** Appends the next pending frame when the buffer is free. */
+    pump() {
+        const buffer = this.sourceBuffer;
+        if (!buffer || buffer.updating || this.pending.length === 0) {
+            return;
+        }
+        if (this.restartAtPlayhead && this.element) {
+            buffer.timestampOffset = this.element.currentTime;
+            this.restartAtPlayhead = false;
+        }
+        const frame = this.pending[0];
+        try {
+            if (frame.Piece === 'init') {
+                this.matchBufferType(buffer, frame.Data);
+            }
+            buffer.appendBuffer(frame.Data);
+            this.pending.shift();
+            this.appending = true;
+            this.retriedAfterQuota = false;
+        }
+        catch (err) {
+            this.handleAppendError(err);
+        }
+    }
+    /** The type for a source buffer that takes `init`: the init's own codecs when this browser plays them, else the player's type. */
+    typeFor(init) {
+        const fromInit = init ? mseTypeOf(init) : null;
+        return fromInit && mediaSourceClass()?.Class.isTypeSupported(fromInit) ? fromInit : this.mimeType;
+    }
+    /**
+     * Switches the source buffer to an init segment's codecs before the init is appended, when they differ from the
+     * buffer's: it opened with the player's type before the first init arrived, or a later init changed tracks. A browser
+     * without `changeType`, or an init whose codecs can't be read or played, leaves the buffer as it is.
+     */
+    matchBufferType(buffer, init) {
+        const type = mseTypeOf(init);
+        if (!type || type === this.bufferType || typeof buffer.changeType !== 'function' || !mediaSourceClass()?.Class.isTypeSupported(type)) {
+            return;
+        }
+        buffer.changeType(type);
+        this.bufferType = type;
+    }
+    /** After each append or removal: trims played media, appends the next piece, and ends a finished turn. */
+    afterUpdate() {
+        this.recordAppendedEnd();
+        this.resumeWhenMediaArrives();
+        if (this.trimBackBuffer()) {
+            return;
+        }
+        this.pump();
+        this.endStreamWhenDrained();
+    }
+    /**
+     * An append finished: the piece ends where the buffered media now ends, since the buffer lays pieces end to end.
+     * Pieces the playhead has passed are forgotten here too, so the list holds only what is ahead.
+     */
+    recordAppendedEnd() {
+        if (!this.appending) {
+            return;
+        }
+        this.appending = false;
+        const buffered = this.sourceBuffer?.buffered;
+        this.appendedEnds.push(buffered && buffered.length > 0 ? buffered.end(buffered.length - 1) : 0);
+        this.forgetPlayedPieces();
+    }
+    /** Forgets the appended pieces the playhead has reached the end of (to within {@link PLAYING_EPSILON_SECONDS}). */
+    forgetPlayedPieces() {
+        const element = this.element;
+        if (!element) {
+            return;
+        }
+        while (this.appendedEnds.length > 0 && this.appendedEnds[0] - element.currentTime <= PLAYING_EPSILON_SECONDS) {
+            this.appendedEnds.shift();
+        }
+    }
+    /** Ends the stream for a finished turn once nothing is pending or in flight. */
+    endStreamWhenDrained() {
+        const mediaSource = this.mediaSource;
+        if (!this.endPending || !mediaSource || mediaSource.readyState !== 'open' || this.sourceBuffer?.updating || this.pending.length > 0) {
+            return;
+        }
+        this.endPending = false;
+        try {
+            mediaSource.endOfStream();
+        }
+        catch (err) {
+            this.context.Report('append-failed', `Could not end the turn's stream: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    /**
+     * A turn that ran to its end, or a Flush, leaves the element paused. Once new media reaches past the playhead,
+     * play on from there. An ended element is first sought to where it stopped: play() on an ended element starts
+     * over from the beginning (Chrome does), which would replay the previous turn.
+     */
+    resumeWhenMediaArrives() {
+        const element = this.element;
+        const buffered = this.sourceBuffer?.buffered;
+        // After a Flush, wait for media appended after it: the abort's own `updateend` arrives before the removal
+        // has shrunk the buffer, and resuming then would play the voice the Flush just cut off.
+        if (!element || !element.paused || this.restartAtPlayhead || !buffered || buffered.length === 0) {
+            return;
+        }
+        const position = element.currentTime;
+        if (buffered.end(buffered.length - 1) - position > PLAYING_EPSILON_SECONDS) {
+            if (element.ended) {
+                element.currentTime = position;
+            }
+            void element.play().catch(() => undefined);
+        }
+    }
+    /**
+     * Removes played media more than the back buffer's seconds behind the playhead, so a long session stays within the
+     * browser's buffer quota. A trim waits until {@link TRIM_STEP_SECONDS} more have played since the last one, so a
+     * removal that doesn't shrink the buffer can't repeat on every `updateend`. When `bufferFull`, keeps only
+     * {@link QUOTA_BACK_BUFFER_SECONDS} and skips the wait.
+     *
+     * @returns Whether a removal started (the next append waits for its `updateend`).
+     */
+    trimBackBuffer(bufferFull = false) {
+        const buffer = this.sourceBuffer;
+        const element = this.element;
+        if (!buffer || !element || buffer.updating || buffer.buffered.length === 0) {
+            return false;
+        }
+        const keepFrom = element.currentTime - (bufferFull ? QUOTA_BACK_BUFFER_SECONDS : this.backBufferSeconds);
+        const removableFrom = Math.max(buffer.buffered.start(0), bufferFull ? 0 : this.trimmedTo);
+        if (keepFrom <= 0 || keepFrom - removableFrom < (bufferFull ? 0.001 : TRIM_STEP_SECONDS)) {
+            return false;
+        }
+        buffer.remove(0, keepFrom);
+        this.trimmedTo = keepFrom;
+        return true;
+    }
+    /** Out of buffer space: trim harder and retry once. Anything else drops the piece. */
+    handleAppendError(err) {
+        const quota = err instanceof Error && err.name === 'QuotaExceededError';
+        if (quota && !this.retriedAfterQuota && this.trimBackBuffer(true)) {
+            this.retriedAfterQuota = true;
+            return;
+        }
+        this.pending.shift();
+        this.retriedAfterQuota = false;
+        this.context.Report('append-failed', err instanceof Error ? err.message : String(err));
+        this.pump();
+    }
+    detach() {
+        if (this.element) {
+            this.element.removeAttribute('src');
+            this.element.load();
+        }
+        if (this.objectUrl) {
+            URL.revokeObjectURL(this.objectUrl);
+        }
+        this.element = null;
+        this.mediaSource = null;
+        this.sourceBuffer = null;
+        this.bufferType = null;
+        this.objectUrl = null;
+        this.restartAtPlayhead = false;
+        this.trimmedTo = 0;
+        this.endPending = false;
+        // The media source and what it buffered are gone; the pending pieces stay for the next element.
+        this.appendedEnds.length = 0;
+        this.appending = false;
+    }
+}
+/** The `'mse-fmp4'` decoder: fragmented MP4 frames whose type this browser's Media Source Extensions play. */
+export const MSE_FMP4_DECODER = {
+    Name: 'mse-fmp4',
+    Kind: 'fmp4',
+    Priority: 0,
+    CanPlay: (mimeType) => MseFmp4Decoder.CanPlay(mimeType),
+    Create: (context) => new MseFmp4Decoder(context),
+};
+//# sourceMappingURL=mseFmp4Decoder.js.map

package/dist/media/decoders/mseFmp4Decoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mseFmp4Decoder.js","sourceRoot":"","sources":["../../../src/media/decoders/mseFmp4Decoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAAE,YAAY,EAAwD,MAAM,oBAAoB,CAAC;AAGxG,gHAAgH;AAChH,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,sEAAsE;AACtE,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAEpC,oFAAoF;AACpF,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,2EAA2E;AAC3E,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAErC,qGAAqG;AACrG,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAM3B,yFAAyF;AACzF,SAAS,gBAAgB;IACrB,MAAM,KAAK,GAAG,UAAsB,CAAC;IACrC,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;QACpB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACxD,CAAC;IACD,OAAO,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,kBAAkB,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAChG,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CAAC,IAAiB;IAChC,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,MAAM,IAAI,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;IACxH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC9D,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,sBAAsB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;AAClF,CAAC;AAED,oFAAoF;AACpF,MAAM,OAAO,cAAc;IAyBvB,4DAA4D;IACrD,MAAM,CAAC,OAAO,CAAC,QAAgB;QAClC,OAAO,gBAAgB,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC;IACxE,CAAC;IAED,wDAAwD;IACxD,YAA6B,OAAiC;QAAjC,YAAO,GAAP,OAAO,CAA0B;QA5B7C,YAAO,GAA6B,EAAE,CAAC;QAChD,YAAO,GAA4B,IAAI,CAAC;QACxC,gBAAW,GAAuB,IAAI,CAAC;QACvC,cAAS,GAAkB,IAAI,CAAC;QAChC,iBAAY,GAAwB,IAAI,CAAC;QACjD,sEAAsE;QAC9D,eAAU,GAAkB,IAAI,CAAC;QACzC,4GAA4G;QACpG,aAAQ,GAAkC,IAAI,CAAC;QACvD,8FAA8F;QACtF,sBAAiB,GAAG,KAAK,CAAC;QAC1B,sBAAiB,GAAG,KAAK,CAAC;QAClC,4DAA4D;QACpD,cAAS,GAAG,CAAC,CAAC;QACtB,uEAAuE;QAC/D,eAAU,GAAG,KAAK,CAAC;QAC3B,sHAAsH;QACrG,iBAAY,GAAa,EAAE,CAAC;QAC7C,oGAAoG;QAC5F,cAAS,GAAG,KAAK,CAAC;QAClB,aAAQ,GAAG,KAAK,CAAC;QASrB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACvD,CAAC;IAED;;;;OAIG;IACH,IAAW,SAAS;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC;QAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,GAAG,gBAAgB,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5G,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,GAAG,uBAAuB,CAAC;IAC7F,CAAC;IAED;;;;;OAKG;IACH,IAAW,WAAW;QAClB,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IACrF,CAAC;IAED,4FAA4F;IACrF,MAAM,CAAC,KAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzC,OAAO;QACX,CAAC;QACD,2EAA2E;QAC3E,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACzB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QAC1B,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,sBAAsB,EAAE,mEAAmE,CAAC,CAAC;YACjH,OAAO;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,yEAAyE,CAAC,CAAC;QACvH,CAAC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED;;;OAGG;IACI,SAAS;QACZ,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACI,KAAK;QACR,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,2GAA2G;QAC3G,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO;QACX,CAAC;QACD,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YACjF,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QACjD,CAAC;QACD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAClC,CAAC;IAED,mEAAmE;IAC5D,OAAO;QACV,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;IAED,kHAAkH;IAC3G,MAAM,CAAC,OAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,GAAG,GAAG,gBAAgB,EAAE,CAAC;QAC/B,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,4BAA4B,IAAI,CAAC,QAAQ,mCAAmC,CAAC,CAAC;YACjH,OAAO;QACX,CAAC;QACD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACpC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1C,WAAW,CAAC,gBAAgB,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACrG,IAAI,CAAC,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;YAC3B,kGAAkG;QACtG,CAAC,CAAC,CAAC;IACP,CAAC;IAED,gFAAgF;IACzE,MAAM;QACT,IAAI,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;IAED,qHAAqH;IAC7G,cAAc,CAAC,OAAyB,EAAE,OAAgB;QAC9D,IAAI,OAAO,EAAE,CAAC;YACV,oGAAoG;YACpG,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;QACzC,CAAC;QACD,oGAAoG;QACpG,OAAO,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;QAC3C,IAAI,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,IAAI,CAAC,oDAAoD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzH,CAAC;IACL,CAAC;IAEO,gBAAgB,CAAC,WAAwB;QAC7C,IAAI,IAAI,CAAC,WAAW,KAAK,WAAW,EAAE,CAAC;YACnC,OAAO;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,IAAI,IAAI,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,+FAA+F;QAC/F,MAAM,CAAC,IAAI,GAAG,UAAU,CAAC;QACzB,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,6CAA6C,CAAC,CAAC,CAAC;QAC5H,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC;QAC3B,yGAAyG;QACzG,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,MAAM,CAAC,EAAE,CAAC;YAClF,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,8DAA8D;IACtD,IAAI;QACR,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1D,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACzC,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YAClD,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACnC,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC9B,IAAI,CAAC;YACD,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;gBACzB,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QACnC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAED,kIAAkI;IAC1H,OAAO,CAAC,IAAwB;QACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/C,OAAO,QAAQ,IAAI,gBAAgB,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;IACtG,CAAC;IAED;;;;OAIG;IACK,eAAe,CAAC,MAAoB,EAAE,IAAiB;QAC3D,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,UAAU,IAAI,OAAO,MAAM,CAAC,UAAU,KAAK,UAAU,IAAI,CAAC,gBAAgB,EAAE,EAAE,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACnI,OAAO;QACX,CAAC;QACD,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,0GAA0G;IAClG,WAAW;QACf,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,cAAc,EAAE,EAAE,CAAC;YACxB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC,oBAAoB,EAAE,CAAC;IAChC,CAAC;IAED;;;OAGG;IACK,iBAAiB;QACrB,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAClB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC;QAC7C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChG,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC9B,CAAC;IAED,mHAAmH;IAC3G,kBAAkB;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO;QACX,CAAC;QACD,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,WAAW,IAAI,uBAAuB,EAAE,CAAC;YAC3G,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;IACL,CAAC;IAED,gFAAgF;IACxE,oBAAoB;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,UAAU,KAAK,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClI,OAAO;QACX,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC;YACD,WAAW,CAAC,WAAW,EAAE,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,oCAAoC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjI,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,sBAAsB;QAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC;QAC7C,0GAA0G;QAC1G,wFAAwF;QACxF,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,iBAAiB,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9F,OAAO;QACX,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;QACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,uBAAuB,EAAE,CAAC;YACzE,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,CAAC,WAAW,GAAG,QAAQ,CAAC;YACnC,CAAC;YACD,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACL,CAAC;IAED;;;;;;;OAOG;IACK,cAAc,CAAC,UAAU,GAAG,KAAK;QACrC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;QACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,MAAM,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzE,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACzG,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC1F,IAAI,QAAQ,IAAI,CAAC,IAAI,QAAQ,GAAG,aAAa,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC;YACvF,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC3B,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;QAC1B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,sFAAsF;IAC9E,iBAAiB,CAAC,GAAY;QAClC,MAAM,KAAK,GAAG,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,CAAC;QACxE,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,iBAAiB,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAC/B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAEO,MAAM;QACV,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,gGAAgG;QAChG,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;CACJ;AAED,8GAA8G;AAC9G,MAAM,CAAC,MAAM,gBAAgB,GAAkC;IAC3D,IAAI,EAAE,UAAU;IAChB,IAAI,EAAE,MAAM;IACZ,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,QAAQ,CAAC;IACvD,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,cAAc,CAAC,OAAO,CAAC;CACnD,CAAC"}

package/dist/media/decoders/streamCanvas.d.ts

@@ -0,0 +1,46 @@
+/**
+ * @fileoverview STREAM CANVAS: turns decoded pictures (WebCodecs `VideoFrame`s, `ImageBitmap`s) into a live
+ * `MediaStream` that the host's `<video>` element shows, so a decoder that isn't MSE still plays into the element: the
+ * tile, its stall watch (`requestVideoFrameCallback` fires once per picture) and its placements work as for any video.
+ *
+ * - **Chromium:** a `MediaStreamTrackGenerator` takes each picture as a `VideoFrame`, with no copy.
+ * - **Elsewhere:** a canvas captured at 0 fps (`captureStream(0)`) is drawn on, and a frame is requested after each draw.
+ * - **The last picture holds.** Between pictures the element keeps showing the last one, and an element attached later
+ *   is shown it: the generator is written a copy again, or the canvas is captured again. A picture that reaches an element
+ *   before its media pipeline is connected is lost (in Chromium that takes a task or two), so the copy is offered every
+ *   {@link REPLAY_INTERVAL_MS} until the element has a frame, at most {@link MAX_REPLAYS} times.
+ * - **Time moves forward.** Each picture written to the generator is stamped with the wall clock, after the one before, so a
+ *   stream that restarts its media time doesn't feed the track frames that go back in time.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** A decoded picture. The stream canvas takes it over and closes it. */
+export type StreamPicture = VideoFrame | ImageBitmap;
+/** How often the last picture is offered again to an element just attached, until it has a frame, in milliseconds. */
+export declare const REPLAY_INTERVAL_MS = 50;
+/** How many times the last picture is offered to an element just attached. */
+export declare const MAX_REPLAYS = 20;
+/** Shows decoded pictures in `<video>` elements through one live stream. */
+export declare class StreamCanvas {
+    private readonly sink;
+    private element;
+    private replayTimer;
+    /** Whether this browser can turn pictures into a stream: a track generator, or a canvas it can capture. */
+    static IsAvailable(): boolean;
+    constructor();
+    /** Shows a picture in the attached element (and any attached later), and closes it. */
+    Show(picture: StreamPicture): void;
+    /** Shows the stream in an element, with the last picture. The element plays muted: the stream has no audio. */
+    Attach(element: HTMLVideoElement): void;
+    /** Takes the stream out of the element it was attached to, unless another source replaced it there. */
+    Detach(): void;
+    /** Detaches, stops the stream's track and closes the last picture. */
+    Dispose(): void;
+    /**
+     * Offers the last picture to a just-attached element until it has a frame, every {@link REPLAY_INTERVAL_MS}. Letting
+     * the element go cancels the offers.
+     */
+    private replayUntilShown;
+    private cancelReplay;
+}
+//# sourceMappingURL=streamCanvas.d.ts.map

package/dist/media/decoders/streamCanvas.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamCanvas.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/streamCanvas.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,wEAAwE;AACxE,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,WAAW,CAAC;AAErD,sHAAsH;AACtH,eAAO,MAAM,kBAAkB,KAAK,CAAC;AAErC,8EAA8E;AAC9E,eAAO,MAAM,WAAW,KAAK,CAAC;AA0H9B,4EAA4E;AAC5E,qBAAa,YAAY;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAc;IACnC,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,WAAW,CAA8C;IAEjE,2GAA2G;WAC7F,WAAW,IAAI,OAAO;;IASpC,uFAAuF;IAChF,IAAI,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAIzC,+GAA+G;IACxG,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAY9C,uGAAuG;IAChG,MAAM,IAAI,IAAI;IAUrB,sEAAsE;IAC/D,OAAO,IAAI,IAAI;IAKtB;;;OAGG;IACH,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,YAAY;CAMvB"}