@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/decoders/videoMimeTypes.d.ts

@@ -0,0 +1,29 @@
+/**
+ * @fileoverview VIDEO MIME TYPES: what `VideoPlayout` and its decoders read from a frame's MIME type: its essence
+ * (`type/subtype`), its `codecs` parameter, the frame kind it names, and an encoded chunk's WebCodecs codec string.
+ *
+ * Chunk types follow the `RealtimeChunkVideoFrame` contract: `video/h264`, `video/vp9` and `video/av1` carry the WebCodecs
+ * codec string in their `codecs` parameter (`video/h264; codecs="avc1.42e01f"`); `video/vp8` needs none.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { RealtimeVideoFrameKind } from '@memberjunction/ai';
+/** The image types `createImageBitmap` decodes in every browser that has it. */
+export declare const IMAGE_FRAME_TYPES: ReadonlySet<string>;
+/** A MIME type's essence: `type/subtype` in lower case, without parameters. */
+export declare function MimeEssence(mimeType: string): string;
+/** The first entry of a MIME type's `codecs` parameter, unquoted; `undefined` when it has none. */
+export declare function MimeCodec(mimeType: string): string | undefined;
+/**
+ * The frame kind a MIME type names: `video/mp4` is fragmented MP4, the chunk types above are encoded chunks, and any
+ * `image/*` is an image. `null` for anything else.
+ */
+export declare function FrameKindOfMimeType(mimeType: string): RealtimeVideoFrameKind | null;
+/**
+ * The WebCodecs codec string for an encoded chunk type: its `codecs` parameter, or `'vp8'` for `video/vp8` without one.
+ * `null` for another type, or a `codecs` parameter of another codec family.
+ */
+export declare function WebCodecsCodecOf(mimeType: string): string | null;
+/** A frame's type as the player tells decoders apart: its kind and its MIME essence. */
+export declare function FrameTypeKey(kind: RealtimeVideoFrameKind, mimeType: string): string;
+//# sourceMappingURL=videoMimeTypes.d.ts.map

package/dist/media/decoders/videoMimeTypes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoMimeTypes.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/videoMimeTypes.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAgBjE,gFAAgF;AAChF,eAAO,MAAM,iBAAiB,EAAE,WAAW,CAAC,MAAM,CAAsD,CAAC;AAEzG,+EAA+E;AAC/E,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED,mGAAmG;AACnG,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAI9D;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,sBAAsB,GAAG,IAAI,CASnF;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAOhE;AAED,wFAAwF;AACxF,wBAAgB,YAAY,CAAC,IAAI,EAAE,sBAAsB,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEnF"}

package/dist/media/decoders/videoMimeTypes.js

@@ -0,0 +1,50 @@
+/** The chunk types a WebCodecs decoder can be configured for, by MIME essence. */
+const CHUNK_CODEC_FAMILIES = new Map([
+    ['video/h264', { Prefixes: ['avc1.', 'avc3.'] }],
+    ['video/vp8', { Prefixes: ['vp8'], Default: 'vp8' }],
+    ['video/vp9', { Prefixes: ['vp09.'] }],
+    ['video/av1', { Prefixes: ['av01.'] }],
+]);
+/** The image types `createImageBitmap` decodes in every browser that has it. */
+export const IMAGE_FRAME_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']);
+/** A MIME type's essence: `type/subtype` in lower case, without parameters. */
+export function MimeEssence(mimeType) {
+    return mimeType.split(';')[0].trim().toLowerCase();
+}
+/** The first entry of a MIME type's `codecs` parameter, unquoted; `undefined` when it has none. */
+export function MimeCodec(mimeType) {
+    const match = /;\s*codecs\s*=\s*(?:"([^"]*)"|([^;\s]+))/i.exec(mimeType);
+    const codec = (match?.[1] ?? match?.[2])?.split(',')[0].trim();
+    return codec || undefined;
+}
+/**
+ * The frame kind a MIME type names: `video/mp4` is fragmented MP4, the chunk types above are encoded chunks, and any
+ * `image/*` is an image. `null` for anything else.
+ */
+export function FrameKindOfMimeType(mimeType) {
+    const essence = MimeEssence(mimeType);
+    if (essence === 'video/mp4') {
+        return 'fmp4';
+    }
+    if (CHUNK_CODEC_FAMILIES.has(essence)) {
+        return 'chunk';
+    }
+    return essence.startsWith('image/') ? 'image' : null;
+}
+/**
+ * The WebCodecs codec string for an encoded chunk type: its `codecs` parameter, or `'vp8'` for `video/vp8` without one.
+ * `null` for another type, or a `codecs` parameter of another codec family.
+ */
+export function WebCodecsCodecOf(mimeType) {
+    const family = CHUNK_CODEC_FAMILIES.get(MimeEssence(mimeType));
+    const codec = MimeCodec(mimeType) ?? family?.Default;
+    if (!family || !codec) {
+        return null;
+    }
+    return family.Prefixes.some((prefix) => codec.toLowerCase().startsWith(prefix)) ? codec : null;
+}
+/** A frame's type as the player tells decoders apart: its kind and its MIME essence. */
+export function FrameTypeKey(kind, mimeType) {
+    return `${kind} ${MimeEssence(mimeType)}`;
+}
+//# sourceMappingURL=videoMimeTypes.js.map

package/dist/media/decoders/videoMimeTypes.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoMimeTypes.js","sourceRoot":"","sources":["../../../src/media/decoders/videoMimeTypes.ts"],"names":[],"mappings":"AAiBA,kFAAkF;AAClF,MAAM,oBAAoB,GAA0C,IAAI,GAAG,CAAC;IACxE,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;IAChD,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpD,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;IACtC,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;CACzC,CAAC,CAAC;AAEH,gFAAgF;AAChF,MAAM,CAAC,MAAM,iBAAiB,GAAwB,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC,CAAC;AAEzG,+EAA+E;AAC/E,MAAM,UAAU,WAAW,CAAC,QAAgB;IACxC,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AACvD,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,SAAS,CAAC,QAAgB;IACtC,MAAM,KAAK,GAAG,2CAA2C,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACzE,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,OAAO,KAAK,IAAI,SAAS,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAChD,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,OAAO,KAAK,WAAW,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAClB,CAAC;IACD,IAAI,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,OAAO,OAAO,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;AACzD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC7C,MAAM,MAAM,GAAG,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,MAAM,EAAE,OAAO,CAAC;IACrD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACpB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACnG,CAAC;AAED,wFAAwF;AACxF,MAAM,UAAU,YAAY,CAAC,IAA4B,EAAE,QAAgB;IACvE,OAAO,GAAG,IAAI,IAAI,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC9C,CAAC"}

package/dist/media/decoders/webCodecsChunkDecoder.d.ts

@@ -0,0 +1,99 @@
+/**
+ * @fileoverview WEBCODECS CHUNK DECODER: plays encoded video chunks (`Kind: 'chunk'`: H.264 in Annex B form, VP8, VP9,
+ * AV1) through a WebCodecs `VideoDecoder`, into the host's `<video>` element through a {@link StreamCanvas}. The
+ * `'webcodecs'` decoder of `VideoPlayout`.
+ *
+ * - **Configured at a key frame.** The decoder is configured from the chunk's MIME type (its `codecs` parameter is the
+ *   WebCodecs codec string; H.264 in Annex B needs no `description`) at the first key frame, after the browser confirms
+ *   it decodes the codec (`VideoDecoder.isConfigSupported`, asynchronous). A "no" reports `'unsupported'` and gives up,
+ *   so the player tries the next decoder. Until a key frame, delta chunks are dropped.
+ * - **Paced by presentation time.** Decoded frames show on the {@link FrameScheduler} by their `PresentationTimeMs`: on
+ *   the voice's clock when the player has one (`VideoPlayoutOptions.Clock`), else by their own times.
+ *   Chunks wait, encoded, while {@link MAX_FRAMES_AHEAD} frames are decoding or waiting to show: a browser decoder stalls
+ *   when its output frames aren't closed.
+ * - **End of turn** flushes the decoder once the waiting chunks are decoded, so the turn's last frames show; a chunk
+ *   that arrives first continues the turn instead. After the flush the next chunk must be a key frame (WebCodecs requires
+ *   one). **Barge-in** resets the decoder, drops what is waiting and what hasn't shown, and waits for the next key frame;
+ *   the last frame shown stays.
+ * - **Errors.** A decode error is reported (`'append-failed'`); the browser has closed that decoder, so a new one starts
+ *   at the next key frame. {@link MAX_DECODE_ERRORS} errors within {@link DECODE_ERROR_WINDOW_MS} give up.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { RealtimeVideoFrame } from '@memberjunction/ai';
+import type { IVideoFrameDecoder, VideoFrameDecoderContext, VideoFrameDecoderRegistration } from '../videoFrameDecoder.js';
+/** Frames decoding or waiting to show, at most; further chunks wait encoded. */
+export declare const MAX_FRAMES_AHEAD = 6;
+/** Encoded chunks waiting to be decoded, at most; beyond this they are dropped and decoding restarts at a key frame. */
+export declare const MAX_WAITING_CHUNKS = 600;
+/** Decode errors within {@link DECODE_ERROR_WINDOW_MS} that make the decoder give up. */
+export declare const MAX_DECODE_ERRORS = 3;
+/** The window decode errors are counted in, in milliseconds. */
+export declare const DECODE_ERROR_WINDOW_MS = 60000;
+/** Plays encoded chunks through a WebCodecs `VideoDecoder`. */
+export declare class WebCodecsChunkDecoder implements IVideoFrameDecoder {
+    private readonly context;
+    private readonly canvas;
+    private readonly scheduler;
+    /** Chunks from a key frame on, waiting to be decoded. */
+    private readonly waiting;
+    /** Codecs this browser said it decodes. */
+    private readonly supported;
+    /** When the recent decode errors happened, for the give-up rule. */
+    private readonly errors;
+    private decoder;
+    /** The codec the decoder is configured for; `null` until a key frame configures it, and after a reset. */
+    private configured;
+    /** The codec whose support check is under way. */
+    private checking;
+    /** Delta chunks are dropped until a key frame. */
+    private awaitingKeyFrame;
+    /** Set by EndOfTurn: flush the decoder once nothing waits. */
+    private endPending;
+    /** Whether a key frame ever arrived: a stream that starts on a delta chunk is reported once. */
+    private started;
+    private disposed;
+    /** Whether this browser can decode chunks of `mimeType` with WebCodecs and show them, as far as it can tell at once. */
+    static CanPlay(mimeType: string): boolean;
+    /** @param context The player this decoder plays for. */
+    constructor(context: VideoFrameDecoderContext);
+    /** Always `false`: a chunk carries no voice. */
+    get IsPlaying(): boolean;
+    /** Frames still to show: chunks waiting to be decoded, chunks the decoder is decoding, and frames waiting for their time. */
+    get FramesAhead(): number;
+    Attach(element: HTMLVideoElement): void;
+    Detach(): void;
+    /** Takes one chunk: from a key frame on, it waits its turn to be decoded. */
+    Append(frame: RealtimeVideoFrame): void;
+    /** The turn's last chunk has arrived: the decoder is flushed once the waiting chunks are decoded, so its last frames show. */
+    EndOfTurn(): void;
+    /** Barge-in: resets the decoder and drops what is waiting and what hasn't shown. The last frame shown stays. */
+    Flush(): void;
+    Dispose(): void;
+    /**
+     * A key frame: decoding starts here, configured for its codec (after the browser confirms it, the first time).
+     * Returns `false` when the chunk names no codec WebCodecs knows: it is dropped.
+     */
+    private startAt;
+    private dropBeforeKeyFrame;
+    private enqueue;
+    /** Asks the browser whether it decodes `codec`; a "yes" configures the decoder, a "no" gives up. */
+    private checkSupport;
+    private supportChecked;
+    /** Configures the decoder for `codec`, creating one when none is open. */
+    private configure;
+    private createDecoder;
+    /** Decodes waiting chunks while fewer than {@link MAX_FRAMES_AHEAD} frames are ahead, then flushes a finished turn. */
+    private feed;
+    /** Flushes the decoder for a finished turn once nothing waits. WebCodecs then needs a key frame to go on. */
+    private flushWhenDrained;
+    private decode;
+    /** A decoded frame: it waits on the scheduler for its time. A frame from a decoder since replaced is closed. */
+    private decoded;
+    /** A decode error: the browser closed the decoder. A new one starts at the next key frame, unless errors keep coming. */
+    private decodeFailed;
+    private closeDecoder;
+}
+/** The `'webcodecs'` decoder: encoded chunks whose codec this browser's WebCodecs decodes. */
+export declare const WEBCODECS_CHUNK_DECODER: VideoFrameDecoderRegistration;
+//# sourceMappingURL=webCodecsChunkDecoder.d.ts.map

package/dist/media/decoders/webCodecsChunkDecoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"webCodecsChunkDecoder.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/webCodecsChunkDecoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,KAAK,EAA2B,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACtF,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AAKxH,gFAAgF;AAChF,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,wHAAwH;AACxH,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,yFAAyF;AACzF,eAAO,MAAM,iBAAiB,IAAI,CAAC;AAEnC,gEAAgE;AAChE,eAAO,MAAM,sBAAsB,QAAS,CAAC;AAE7C,+DAA+D;AAC/D,qBAAa,qBAAsB,YAAW,kBAAkB;IA6BhD,OAAO,CAAC,QAAQ,CAAC,OAAO;IA5BpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiB;IAC3C,yDAAyD;IACzD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,2CAA2C;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,OAAO,CAA6B;IAC5C,0GAA0G;IAC1G,OAAO,CAAC,UAAU,CAAuB;IACzC,kDAAkD;IAClD,OAAO,CAAC,QAAQ,CAAuB;IACvC,kDAAkD;IAClD,OAAO,CAAC,gBAAgB,CAAQ;IAChC,8DAA8D;IAC9D,OAAO,CAAC,UAAU,CAAS;IAC3B,gGAAgG;IAChG,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,QAAQ,CAAS;IAEzB,wHAAwH;WAC1G,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAKhD,wDAAwD;gBAC3B,OAAO,EAAE,wBAAwB;IAI9D,gDAAgD;IAChD,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED,6HAA6H;IAC7H,IAAW,WAAW,IAAI,MAAM,CAG/B;IAEM,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAMvC,MAAM,IAAI,IAAI;IAIrB,6EAA6E;IACtE,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAkB9C,8HAA8H;IACvH,SAAS,IAAI,IAAI;IAKxB,gHAAgH;IACzG,KAAK,IAAI,IAAI;IAWb,OAAO,IAAI,IAAI;IAQtB;;;OAGG;IACH,OAAO,CAAC,OAAO;IAwBf,OAAO,CAAC,kBAAkB;IAM1B,OAAO,CAAC,OAAO;IAUf,oGAAoG;IACpG,OAAO,CAAC,YAAY;IAUpB,OAAO,CAAC,cAAc;IAetB,0EAA0E;IAC1E,OAAO,CAAC,SAAS;IAWjB,OAAO,CAAC,aAAa;IASrB,uHAAuH;IACvH,OAAO,CAAC,IAAI;IAWZ,6GAA6G;IAC7G,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,MAAM;IASd,gHAAgH;IAChH,OAAO,CAAC,OAAO;IASf,yHAAyH;IACzH,OAAO,CAAC,YAAY;IAkBpB,OAAO,CAAC,YAAY;CAQvB;AAOD,8FAA8F;AAC9F,eAAO,MAAM,uBAAuB,EAAE,6BAMrC,CAAC"}

package/dist/media/decoders/webCodecsChunkDecoder.js

@@ -0,0 +1,265 @@
+import { FrameScheduler } from './frameScheduler.js';
+import { StreamCanvas } from './streamCanvas.js';
+import { WebCodecsCodecOf } from './videoMimeTypes.js';
+/** Frames decoding or waiting to show, at most; further chunks wait encoded. */
+export const MAX_FRAMES_AHEAD = 6;
+/** Encoded chunks waiting to be decoded, at most; beyond this they are dropped and decoding restarts at a key frame. */
+export const MAX_WAITING_CHUNKS = 600;
+/** Decode errors within {@link DECODE_ERROR_WINDOW_MS} that make the decoder give up. */
+export const MAX_DECODE_ERRORS = 3;
+/** The window decode errors are counted in, in milliseconds. */
+export const DECODE_ERROR_WINDOW_MS = 60_000;
+/** Plays encoded chunks through a WebCodecs `VideoDecoder`. */
+export class WebCodecsChunkDecoder {
+    /** Whether this browser can decode chunks of `mimeType` with WebCodecs and show them, as far as it can tell at once. */
+    static CanPlay(mimeType) {
+        const canDecode = typeof VideoDecoder === 'function' && typeof EncodedVideoChunk === 'function';
+        return canDecode && StreamCanvas.IsAvailable() && WebCodecsCodecOf(mimeType) !== null;
+    }
+    /** @param context The player this decoder plays for. */
+    constructor(context) {
+        this.context = context;
+        this.canvas = new StreamCanvas();
+        /** Chunks from a key frame on, waiting to be decoded. */
+        this.waiting = [];
+        /** Codecs this browser said it decodes. */
+        this.supported = new Set();
+        /** When the recent decode errors happened, for the give-up rule. */
+        this.errors = [];
+        this.decoder = null;
+        /** The codec the decoder is configured for; `null` until a key frame configures it, and after a reset. */
+        this.configured = null;
+        /** The codec whose support check is under way. */
+        this.checking = null;
+        /** Delta chunks are dropped until a key frame. */
+        this.awaitingKeyFrame = true;
+        /** Set by EndOfTurn: flush the decoder once nothing waits. */
+        this.endPending = false;
+        /** Whether a key frame ever arrived: a stream that starts on a delta chunk is reported once. */
+        this.started = false;
+        this.disposed = false;
+        this.scheduler = new FrameScheduler(this.canvas, () => this.feed(), { Clock: context.Clock });
+    }
+    /** Always `false`: a chunk carries no voice. */
+    get IsPlaying() {
+        return false;
+    }
+    /** Frames still to show: chunks waiting to be decoded, chunks the decoder is decoding, and frames waiting for their time. */
+    get FramesAhead() {
+        const decoding = this.decoder && this.decoder.state !== 'closed' ? this.decoder.decodeQueueSize : 0;
+        return this.waiting.length + decoding + this.scheduler.Size;
+    }
+    Attach(element) {
+        if (!this.disposed) {
+            this.canvas.Attach(element);
+        }
+    }
+    Detach() {
+        this.canvas.Detach();
+    }
+    /** Takes one chunk: from a key frame on, it waits its turn to be decoded. */
+    Append(frame) {
+        if (this.disposed || frame.Kind !== 'chunk') {
+            return;
+        }
+        // More media is coming, so a turn that was about to end continues instead.
+        this.endPending = false;
+        if (frame.KeyFrame) {
+            if (!this.startAt(frame) || this.disposed) {
+                return;
+            }
+        }
+        else if (this.awaitingKeyFrame) {
+            this.dropBeforeKeyFrame();
+            return;
+        }
+        this.enqueue(frame);
+        this.feed();
+    }
+    /** The turn's last chunk has arrived: the decoder is flushed once the waiting chunks are decoded, so its last frames show. */
+    EndOfTurn() {
+        this.endPending = true;
+        this.flushWhenDrained();
+    }
+    /** Barge-in: resets the decoder and drops what is waiting and what hasn't shown. The last frame shown stays. */
+    Flush() {
+        this.waiting.length = 0;
+        this.endPending = false;
+        this.awaitingKeyFrame = true;
+        this.scheduler.Clear();
+        if (this.decoder && this.decoder.state !== 'closed') {
+            this.decoder.reset();
+        }
+        this.configured = null;
+    }
+    Dispose() {
+        this.disposed = true;
+        this.waiting.length = 0;
+        this.scheduler.Dispose();
+        this.closeDecoder();
+        this.canvas.Dispose();
+    }
+    /**
+     * A key frame: decoding starts here, configured for its codec (after the browser confirms it, the first time).
+     * Returns `false` when the chunk names no codec WebCodecs knows: it is dropped.
+     */
+    startAt(frame) {
+        const codec = WebCodecsCodecOf(frame.MimeType);
+        if (codec === null) {
+            this.context.Report('unsupported', `A video chunk of type ${frame.MimeType} names no codec WebCodecs knows; it was dropped.`);
+            return false;
+        }
+        this.awaitingKeyFrame = false;
+        this.started = true;
+        if (codec === this.configured || codec === this.checking) {
+            return true;
+        }
+        // A new codec: chunks of the old one still waiting can't be decoded with it, nothing decodes until it is configured,
+        // and a check under way for another codec no longer applies.
+        this.waiting.length = 0;
+        this.configured = null;
+        this.checking = null;
+        if (this.supported.has(codec)) {
+            this.configure(codec);
+        }
+        else {
+            this.checkSupport(codec);
+        }
+        return true;
+    }
+    dropBeforeKeyFrame() {
+        if (!this.started) {
+            this.context.Report('fragment-before-init', 'An encoded video chunk arrived before any key frame and was dropped.');
+        }
+    }
+    enqueue(frame) {
+        this.waiting.push(frame);
+        if (this.waiting.length > MAX_WAITING_CHUNKS) {
+            this.waiting.length = 0;
+            this.awaitingKeyFrame = true;
+            const message = 'Encoded video chunks piled up faster than they could be decoded; they were dropped until the next key frame.';
+            this.context.Report('pending-overflow', message);
+        }
+    }
+    /** Asks the browser whether it decodes `codec`; a "yes" configures the decoder, a "no" gives up. */
+    checkSupport(codec) {
+        this.checking = codec;
+        const canCheck = typeof VideoDecoder.isConfigSupported === 'function';
+        const check = canCheck ? VideoDecoder.isConfigSupported({ codec }) : Promise.resolve({ supported: true });
+        check.then((result) => this.supportChecked(codec, result.supported === true), () => this.supportChecked(codec, false));
+    }
+    supportChecked(codec, supported) {
+        if (this.disposed || this.checking !== codec) {
+            return;
+        }
+        this.checking = null;
+        if (!supported) {
+            this.context.Report('unsupported', `This browser cannot decode ${codec} video with WebCodecs.`);
+            this.context.Failed(`the browser does not decode ${codec}`);
+            return;
+        }
+        this.supported.add(codec);
+        this.configure(codec);
+        this.feed();
+    }
+    /** Configures the decoder for `codec`, creating one when none is open. */
+    configure(codec) {
+        try {
+            const decoder = this.decoder && this.decoder.state !== 'closed' ? this.decoder : this.createDecoder();
+            decoder.configure({ codec, optimizeForLatency: true });
+            this.configured = codec;
+        }
+        catch (error) {
+            this.context.Report('unsupported', `This browser cannot decode ${codec} video with WebCodecs: ${messageOf(error)}`);
+            this.context.Failed(`the decoder could not be configured for ${codec}`);
+        }
+    }
+    createDecoder() {
+        const decoder = new VideoDecoder({
+            output: (frame) => this.decoded(decoder, frame),
+            error: (error) => this.decodeFailed(decoder, error.message),
+        });
+        this.decoder = decoder;
+        return decoder;
+    }
+    /** Decodes waiting chunks while fewer than {@link MAX_FRAMES_AHEAD} frames are ahead, then flushes a finished turn. */
+    feed() {
+        const decoder = this.decoder;
+        while (decoder && this.decoder === decoder && this.configured !== null && this.waiting.length > 0) {
+            if (decoder.decodeQueueSize + this.scheduler.Size >= MAX_FRAMES_AHEAD) {
+                return;
+            }
+            this.decode(decoder, this.waiting.shift());
+        }
+        this.flushWhenDrained();
+    }
+    /** Flushes the decoder for a finished turn once nothing waits. WebCodecs then needs a key frame to go on. */
+    flushWhenDrained() {
+        const decoder = this.decoder;
+        if (!this.endPending || this.waiting.length > 0 || !decoder || this.configured === null) {
+            return;
+        }
+        this.endPending = false;
+        this.awaitingKeyFrame = true;
+        decoder.flush().catch(() => {
+            // A reset or an error since: what the flush waited for was dropped.
+        });
+    }
+    decode(decoder, frame) {
+        try {
+            const timestamp = Math.round(frame.PresentationTimeMs * 1000);
+            decoder.decode(new EncodedVideoChunk({ type: frame.KeyFrame ? 'key' : 'delta', timestamp, data: frame.Data }));
+        }
+        catch (error) {
+            this.decodeFailed(decoder, messageOf(error));
+        }
+    }
+    /** A decoded frame: it waits on the scheduler for its time. A frame from a decoder since replaced is closed. */
+    decoded(decoder, frame) {
+        if (this.disposed || this.decoder !== decoder) {
+            frame.close();
+            return;
+        }
+        this.scheduler.Push(frame, frame.timestamp / 1000);
+        this.feed();
+    }
+    /** A decode error: the browser closed the decoder. A new one starts at the next key frame, unless errors keep coming. */
+    decodeFailed(decoder, message) {
+        if (this.disposed || this.decoder !== decoder) {
+            return;
+        }
+        this.context.Report('append-failed', `The browser could not decode a video chunk: ${message}`);
+        this.closeDecoder();
+        this.waiting.length = 0;
+        this.awaitingKeyFrame = true;
+        const now = Date.now();
+        this.errors.push(now);
+        while (this.errors.length > 0 && now - this.errors[0] > DECODE_ERROR_WINDOW_MS) {
+            this.errors.shift();
+        }
+        if (this.errors.length >= MAX_DECODE_ERRORS) {
+            this.context.Failed(`${this.errors.length} decode errors within ${DECODE_ERROR_WINDOW_MS / 1000} s; the last: ${message}`);
+        }
+    }
+    closeDecoder() {
+        const decoder = this.decoder;
+        this.decoder = null;
+        this.configured = null;
+        if (decoder && decoder.state !== 'closed') {
+            decoder.close();
+        }
+    }
+}
+/** An error's message, whatever was thrown. */
+function messageOf(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/** The `'webcodecs'` decoder: encoded chunks whose codec this browser's WebCodecs decodes. */
+export const WEBCODECS_CHUNK_DECODER = {
+    Name: 'webcodecs',
+    Kind: 'chunk',
+    Priority: 0,
+    CanPlay: (mimeType) => WebCodecsChunkDecoder.CanPlay(mimeType),
+    Create: (context) => new WebCodecsChunkDecoder(context),
+};
+//# sourceMappingURL=webCodecsChunkDecoder.js.map

package/dist/media/decoders/webCodecsChunkDecoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"webCodecsChunkDecoder.js","sourceRoot":"","sources":["../../../src/media/decoders/webCodecsChunkDecoder.ts"],"names":[],"mappings":"AAwBA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEpD,gFAAgF;AAChF,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,wHAAwH;AACxH,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,yFAAyF;AACzF,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAEnC,gEAAgE;AAChE,MAAM,CAAC,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAE7C,+DAA+D;AAC/D,MAAM,OAAO,qBAAqB;IAsB9B,wHAAwH;IACjH,MAAM,CAAC,OAAO,CAAC,QAAgB;QAClC,MAAM,SAAS,GAAG,OAAO,YAAY,KAAK,UAAU,IAAI,OAAO,iBAAiB,KAAK,UAAU,CAAC;QAChG,OAAO,SAAS,IAAI,YAAY,CAAC,WAAW,EAAE,IAAI,gBAAgB,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IAC1F,CAAC;IAED,wDAAwD;IACxD,YAA6B,OAAiC;QAAjC,YAAO,GAAP,OAAO,CAA0B;QA5B7C,WAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAE7C,yDAAyD;QACxC,YAAO,GAA8B,EAAE,CAAC;QACzD,2CAA2C;QAC1B,cAAS,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/C,oEAAoE;QACnD,WAAM,GAAa,EAAE,CAAC;QAC/B,YAAO,GAAwB,IAAI,CAAC;QAC5C,0GAA0G;QAClG,eAAU,GAAkB,IAAI,CAAC;QACzC,kDAAkD;QAC1C,aAAQ,GAAkB,IAAI,CAAC;QACvC,kDAAkD;QAC1C,qBAAgB,GAAG,IAAI,CAAC;QAChC,8DAA8D;QACtD,eAAU,GAAG,KAAK,CAAC;QAC3B,gGAAgG;QACxF,YAAO,GAAG,KAAK,CAAC;QAChB,aAAQ,GAAG,KAAK,CAAC;QAUrB,IAAI,CAAC,SAAS,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,gDAAgD;IAChD,IAAW,SAAS;QAChB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,6HAA6H;IAC7H,IAAW,WAAW;QAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QACpG,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAChE,CAAC;IAEM,MAAM,CAAC,OAAyB;QACnC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAEM,MAAM;QACT,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED,6EAA6E;IACtE,MAAM,CAAC,KAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;QACX,CAAC;QACD,2EAA2E;QAC3E,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACxC,OAAO;YACX,CAAC;QACL,CAAC;aAAM,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC/B,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAC1B,OAAO;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,8HAA8H;IACvH,SAAS;QACZ,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,gHAAgH;IACzG,KAAK;QACR,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAClD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;IAC3B,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACK,OAAO,CAAC,KAA8B;QAC1C,MAAM,KAAK,GAAG,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/C,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,yBAAyB,KAAK,CAAC,QAAQ,kDAAkD,CAAC,CAAC;YAC9H,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,KAAK,KAAK,IAAI,CAAC,UAAU,IAAI,KAAK,KAAK,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,qHAAqH;QACrH,6DAA6D;QAC7D,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAEO,kBAAkB;QACtB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,sBAAsB,EAAE,sEAAsE,CAAC,CAAC;QACxH,CAAC;IACL,CAAC;IAEO,OAAO,CAAC,KAA8B;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,MAAM,OAAO,GAAG,8GAA8G,CAAC;YAC/H,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IAED,oGAAoG;IAC5F,YAAY,CAAC,KAAa;QAC9B,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,QAAQ,GAAG,OAAO,YAAY,CAAC,iBAAiB,KAAK,UAAU,CAAC;QACtE,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,iBAAiB,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1G,KAAK,CAAC,IAAI,CACN,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,EACjE,GAAG,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAC1C,CAAC;IACN,CAAC;IAEO,cAAc,CAAC,KAAa,EAAE,SAAkB;QACpD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;YAC3C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,8BAA8B,KAAK,wBAAwB,CAAC,CAAC;YAChG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAC;YAC5D,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC1B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,0EAA0E;IAClE,SAAS,CAAC,KAAa;QAC3B,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACtG,OAAO,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,CAAC;YACvD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,EAAE,8BAA8B,KAAK,0BAA0B,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACpH,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,2CAA2C,KAAK,EAAE,CAAC,CAAC;QAC5E,CAAC;IACL,CAAC;IAEO,aAAa;QACjB,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC;YAC7B,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC;YAC/C,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC;SAC9D,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,uHAAuH;IAC/G,IAAI;QACR,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,OAAO,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChG,IAAI,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,gBAAgB,EAAE,CAAC;gBACpE,OAAO;YACX,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAA6B,CAAC,CAAC;QAC1E,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;IAC5B,CAAC;IAED,6GAA6G;IACrG,gBAAgB;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAAE,CAAC;YACtF,OAAO;QACX,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;YACvB,oEAAoE;QACxE,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,MAAM,CAAC,OAAqB,EAAE,KAA8B;QAChE,IAAI,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC;YAC9D,OAAO,CAAC,MAAM,CAAC,IAAI,iBAAiB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACnH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,CAAC;IACL,CAAC;IAED,gHAAgH;IACxG,OAAO,CAAC,OAAqB,EAAE,KAAiB;QACpD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC5C,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,yHAAyH;IACjH,YAAY,CAAC,OAAqB,EAAE,OAAe;QACvD,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC5C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,+CAA+C,OAAO,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,YAAY,EAAE,CAAC;QACpB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,sBAAsB,EAAE,CAAC;YAC7E,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,iBAAiB,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,yBAAyB,sBAAsB,GAAG,IAAI,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC/H,CAAC;IACL,CAAC;IAEO,YAAY;QAChB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,OAAO,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACxC,OAAO,CAAC,KAAK,EAAE,CAAC;QACpB,CAAC;IACL,CAAC;CACJ;AAED,+CAA+C;AAC/C,SAAS,SAAS,CAAC,KAAc;IAC7B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC;AAED,8FAA8F;AAC9F,MAAM,CAAC,MAAM,uBAAuB,GAAkC;IAClE,IAAI,EAAE,WAAW;IACjB,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC9D,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,qBAAqB,CAAC,OAAO,CAAC;CAC1D,CAAC"}

package/dist/media/displayCapture.d.ts

@@ -0,0 +1,128 @@
+/**
+ * @fileoverview DISPLAY CAPTURE: asks the browser to share a screen, a window, a tab or one panel of this
+ * page, and reports what the user actually shared.
+ *
+ * Sharing works wherever `getDisplayMedia` exists: desktop Chrome, Edge, Firefox and Safari (no mobile browser
+ * has it). The picker hints sent with it (`displaySurface`, `selfBrowserSurface`, `surfaceSwitching`,
+ * `preferCurrentTab`, `monitorTypeSurfaces`, `systemAudio`) are read only by Chromium browsers; others ignore
+ * them.
+ *
+ * A PANEL capture shares one element of this page. The user picks this tab in the picker, and the stream is
+ * then narrowed to the element with Element Capture (`restrictTo`, Chromium 132+: the element alone, without
+ * anything drawn over it) or, failing that, Region Capture (`cropTo`, Chromium 104+: the element's rectangle,
+ * including anything over it). Element Capture emits frames only while the element is its own stacking
+ * context, so the element gets `isolation: isolate` for the share and its own value back when the share ends.
+ * Firefox and Safari have neither; {@link GetDisplayCaptureSupport} says so, and the caller shows the panel to
+ * the agent another way (its channel frames). If the user shares something other than this tab, the capture is
+ * stopped rather than sharing more than the panel.
+ *
+ * The browser's picker is the user's consent: it opens on every request, with no remembered grant. Like the
+ * picker itself, {@link RequestDisplayCapture} must be called from a user gesture (a click handler).
+ *
+ * Capture runs at the surface's native frame rate; pace what the model receives with a `FrameSampler`.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** What the user is asked to share. */
+export type DisplayCaptureSurface = 'screen' | 'window' | 'tab';
+/** What a capture turned out to be; `'unknown'` when the browser doesn't say. */
+export type CapturedDisplaySurface = DisplayCaptureSurface | 'unknown';
+/** How a panel capture was narrowed to its element. */
+export type PanelCaptureMethod = 'element' | 'region';
+/** What this browser can capture. */
+export interface DisplayCaptureSupport {
+    /** A screen, window or tab can be shared (`getDisplayMedia`). */
+    Display: boolean;
+    /** Element Capture (`RestrictionTarget`, Chromium 132+). */
+    ElementCapture: boolean;
+    /** Region Capture (`CropTarget`, Chromium 104+). */
+    RegionCapture: boolean;
+}
+export interface DisplayCaptureOptions {
+    /**
+     * The kind of surface the picker offers first. A hint: the user can still pick another kind, and only
+     * Chromium browsers read it. Ignored for a panel capture, which asks for this tab.
+     */
+    PreferredSurface?: DisplayCaptureSurface;
+    /**
+     * Share only this element of the page. Needs Element or Region Capture (see
+     * {@link GetDisplayCaptureSupport}); Element Capture is used where the browser has it. Element Capture emits
+     * frames only while the element is its own stacking context, so unless it already is one, the element's
+     * inline style gets `isolation: isolate` while it is shared and its own inline value back when the share
+     * ends. Meanwhile its descendants stack inside it: one with a high `z-index` no longer paints over content
+     * outside the panel.
+     */
+    Panel?: Element;
+    /**
+     * The panel's name, such as "Whiteboard", carried on the capture ({@link DisplayCapture.PanelLabel}) so the
+     * UI and the agent can name what is shared. Read only with {@link Panel}.
+     */
+    PanelLabel?: string;
+    /**
+     * Also capture the shared surface's audio, where the browser offers it (Chromium: tab or system audio).
+     * Defaults to `false`: the agent already hears the microphone.
+     */
+    IncludeAudio?: boolean;
+}
+/** A live share. */
+export interface DisplayCapture {
+    readonly Stream: MediaStream;
+    /** The shared video track. */
+    readonly Track: MediaStreamTrack;
+    /** What is being shared now. It can change if the user switches surfaces mid-share. */
+    readonly Surface: CapturedDisplaySurface;
+    /** The browser's label for the shared surface, such as a window or tab title. */
+    readonly Label: string;
+    /** How the stream was narrowed, for a panel capture. */
+    readonly PanelMethod?: PanelCaptureMethod;
+    /** The shared panel's name, for a panel capture that was given one ({@link DisplayCaptureOptions.PanelLabel}). */
+    readonly PanelLabel?: string;
+    /**
+     * Calls `handler` once when the share ends: the user pressed the browser's "Stop sharing", the shared
+     * surface closed, or {@link DisplayCapture.Stop} was called. If the share has already ended, `handler` is
+     * called at once.
+     *
+     * @returns A function that removes the handler.
+     */
+    OnEnded(handler: () => void): () => void;
+    /** Stops sharing. Calling it again does nothing. */
+    Stop(): void;
+}
+/** Why a capture failed. */
+export type DisplayCaptureFailure = 
+/** The browser cannot share a screen at all (mobile browsers, or no DOM). */
+'unsupported'
+/** A panel was asked for, but the browser has neither Element nor Region Capture (Firefox, Safari). */
+ | 'panel-unsupported'
+/** A panel was asked for, but the user shared a screen, a window or another tab. The capture was stopped. */
+ | 'panel-wrong-surface'
+/** The operating system or the page's permissions policy blocked sharing. */
+ | 'denied'
+/** Anything else; see the message. */
+ | 'error';
+/** The outcome of {@link RequestDisplayCapture}. It always resolves; it never rejects. */
+export type DisplayCaptureResult = {
+    Status: 'started';
+    Capture: DisplayCapture;
+} | {
+    Status: 'cancelled';
+} | {
+    Status: 'failed';
+    Reason: DisplayCaptureFailure;
+    Message: string;
+};
+/** What this browser can capture. */
+export declare function GetDisplayCaptureSupport(): DisplayCaptureSupport;
+/**
+ * Opens the browser's share picker and starts sharing what the user picks. Call it from a user gesture.
+ *
+ * @param options What to offer, whether to share only a panel, and whether to include audio.
+ * @returns `started` with the live capture, `cancelled` when the user closed the picker, or `failed` with a reason.
+ */
+export declare function RequestDisplayCapture(options?: DisplayCaptureOptions): Promise<DisplayCaptureResult>;
+/**
+ * The kind of surface a captured track shows, from the browser's `displaySurface` setting. It reads any display
+ * track, including one captured elsewhere, such as a LiveKit screen share.
+ */
+export declare function CapturedSurfaceOf(track: MediaStreamTrack): CapturedDisplaySurface;
+//# sourceMappingURL=displayCapture.d.ts.map

package/dist/media/displayCapture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"displayCapture.d.ts","sourceRoot":"","sources":["../../src/media/displayCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,uCAAuC;AACvC,MAAM,MAAM,qBAAqB,GAAG,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEhE,iFAAiF;AACjF,MAAM,MAAM,sBAAsB,GAAG,qBAAqB,GAAG,SAAS,CAAC;AAEvE,uDAAuD;AACvD,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEtD,qCAAqC;AACrC,MAAM,WAAW,qBAAqB;IAClC,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,4DAA4D;IAC5D,cAAc,EAAE,OAAO,CAAC;IACxB,oDAAoD;IACpD,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,qBAAqB,CAAC;IACzC;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,oBAAoB;AACpB,MAAM,WAAW,cAAc;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,8BAA8B;IAC9B,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,uFAAuF;IACvF,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAC;IACzC,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;IAC1C,kHAAkH;IAClH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC;IACzC,oDAAoD;IACpD,IAAI,IAAI,IAAI,CAAC;CAChB;AAED,4BAA4B;AAC5B,MAAM,MAAM,qBAAqB;AAC7B,6EAA6E;AAC3E,aAAa;AACf,uGAAuG;GACrG,mBAAmB;AACrB,6GAA6G;GAC3G,qBAAqB;AACvB,6EAA6E;GAC3E,QAAQ;AACV,sCAAsC;GACpC,OAAO,CAAC;AAEd,0FAA0F;AAC1F,MAAM,MAAM,oBAAoB,GAC1B;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,GAC9C;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA0C3E,qCAAqC;AACrC,wBAAgB,wBAAwB,IAAI,qBAAqB,CAOhE;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA6B9G;AAoHD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,GAAG,sBAAsB,CAWjF"}