@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/decoders/frameScheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"frameScheduler.js","sourceRoot":"","sources":["../../../src/media/decoders/frameScheduler.ts"],"names":[],"mappings":"AA2BA,gHAAgH;AAChH,MAAM,CAAC,MAAM,WAAW,GAAG,GAAG,CAAC;AAE/B,yHAAyH;AACzH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC;AAEhC,wGAAwG;AACxG,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC,uHAAuH;AACvH,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAEjC,oFAAoF;AACpF,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAsBhC,6GAA6G;AAC7G,MAAM,OAAO,cAAc;IAiBvB;;;;OAIG;IACH,YACqB,MAAoB,EACpB,OAAmB,EACpC,UAAiC,EAAE;QAFlB,WAAM,GAAN,MAAM,CAAc;QACpB,YAAO,GAAP,OAAO,CAAY;QAvBvB,UAAK,GAAuB,EAAE,CAAC;QAGhD,qFAAqF;QAC7E,WAAM,GAAuB,IAAI,CAAC;QAC1C,yEAAyE;QACjE,gBAAW,GAAkB,IAAI,CAAC;QAC1C,qFAAqF;QAC7E,gBAAW,GAAuB,IAAI,CAAC;QAC/C,4FAA4F;QACpF,eAAU,GAAG,KAAK,CAAC;QAC3B,qEAAqE;QAC7D,iBAAY,GAAkB,IAAI,CAAC;QACnC,UAAK,GAAyC,IAAI,CAAC;QACnD,aAAQ,GAAG,KAAK,CAAC;QAYrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;IACxD,CAAC;IAED,0CAA0C;IAC1C,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;;;;OAKG;IACI,IAAI,CAAC,OAAsB,EAAE,MAAe;QAC/C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO,CAAC,KAAK,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,IAAI,CAAC,IAAI,EAAE,CAAC;QAChB,CAAC;IACL,CAAC;IAED,iHAAiH;IAC1G,KAAK;QACR,IAAI,CAAC,WAAW,EAAE,CAAC;QACnB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3C,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;IAC7B,CAAC;IAED,qDAAqD;IAC9C,OAAO;QACV,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;IAED,8HAA8H;IACtH,IAAI;QACR,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,EAAE,aAAa,IAAI,IAAI,CAAC;QAClD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;YACnB,IAAI,CAAC,WAAW,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;YACrD,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;aAAM,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAC7B,OAAO;QACX,CAAC;aAAM,CAAC;YACJ,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QACxF,CAAC;QACD,IAAI,KAAK,EAAE,CAAC;YACR,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,YAAY,CAAC,GAAW;QAC5B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,mBAAmB,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACnB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;YACxB,IAAI,CAAC,YAAY,GAAG,GAAG,CAAC;QAC5B,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,YAAY,KAAK,GAAG,CAAC;QAC1B,OAAO,GAAG,GAAG,IAAI,CAAC,YAAY,GAAG,aAAa,CAAC;IACnD,CAAC;IAED,2FAA2F;IACnF,mBAAmB;QACvB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC;QACnC,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC;IACtF,CAAC;IAED,gHAAgH;IACxG,WAAW,CAAC,OAAe;QAC/B,IAAI,GAAG,GAA4B,IAAI,CAAC;QACxC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACjE,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;gBACrB,MAAM;YACV,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAsB,CAAC;YACpD,IAAI,OAAO,KAAK,MAAM,EAAE,CAAC;gBACrB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBACrB,SAAS;YACb,CAAC;YACD,GAAG,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;YACrB,GAAG,GAAG,IAAI,CAAC;QACf,CAAC;QACD,IAAI,GAAG,EAAE,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,OAAO,GAAG,KAAK,IAAI,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,YAAY,CAAC,MAA0B,EAAE,OAAe;QAC5D,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;YACxC,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,IAAI,OAAO,GAAG,MAAM,GAAG,iBAAiB,IAAI,MAAM,GAAG,OAAO,GAAG,WAAW,EAAE,CAAC;YACzE,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,OAAO,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9C,CAAC;IAED;;;OAGG;IACK,eAAe,CAAC,MAAc,EAAE,OAAe;QACnD,OAAO,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,WAAW,CAAC;IACjG,CAAC;IAED,kFAAkF;IAC1E,aAAa,CAAC,GAAW;QAC7B,IAAI,GAAG,GAA0D,IAAI,CAAC;QACtE,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAC7C,IAAI,KAAK,GAAG,GAAG,EAAE,CAAC;gBACd,MAAM;YACV,CAAC;YACD,GAAG,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC/B,GAAG,GAAG,EAAE,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAsB,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,CAAC;YACP,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC;QAClC,IAAI,IAAI,KAAK,SAAS,IAAI,GAAG,GAAG,GAAG,CAAC,KAAK,GAAG,WAAW,EAAE,CAAC;YACtD,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,4HAA4H;IACpH,KAAK,CAAC,SAA2B,EAAE,GAAW;QAClD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,OAAO,GAAG,CAAC;QACf,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;QACjD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7D,CAAC;IAED,sGAAsG;IAC9F,eAAe,CAAC,MAAc,EAAE,GAAW;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAqB,CAAC;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,KAAK,IAAI,IAAI,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC;QACzE,OAAO,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,WAAW,CAAC;IACtF,CAAC;IAEO,IAAI,CAAC,SAA2B;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,SAAS,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC;QACxC,CAAC;IACL,CAAC;IAED,kDAAkD;IAC1C,YAAY,CAAC,GAAW;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IAC7D,CAAC;IAED,gHAAgH;IACxG,UAAU,CAAC,OAAe;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAClC,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;IAC1G,CAAC;IAEO,QAAQ,CAAC,OAAe;QAC5B,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;QACxD,CAAC;IACL,CAAC;IAEO,WAAW;QACf,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACtB,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,CAAC;IACL,CAAC;CACJ"}

package/dist/media/decoders/imageFrameDecoder.d.ts

@@ -0,0 +1,58 @@
+/**
+ * @fileoverview IMAGE FRAME DECODER: plays still images (`Kind: 'image'`: JPEG, PNG, WebP) through `createImageBitmap`,
+ * into the host's `<video>` element through a {@link StreamCanvas}. The `'image'` decoder of `VideoPlayout`.
+ *
+ * - **In order.** Images are decoded one at a time, in the order they arrived, while fewer than {@link MAX_IMAGES_AHEAD}
+ *   decoded images wait to show; the rest wait encoded.
+ * - **Paced by presentation time when an image has one**, on the {@link FrameScheduler}: on the voice's clock when the
+ *   player has one (`VideoPlayoutOptions.Clock`), else by its own time. An image without a time shows as soon as it is
+ *   decoded.
+ * - **End of turn** needs nothing: every image stands alone. **Barge-in** drops the images waiting, decoded or not, and a
+ *   decode under way; the last image shown stays.
+ * - An image the browser can't decode is dropped and reported (`'append-failed'`); the next one plays.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { RealtimeVideoFrame } from '@memberjunction/ai';
+import type { IVideoFrameDecoder, VideoFrameDecoderContext, VideoFrameDecoderRegistration } from '../videoFrameDecoder.js';
+/** Decoded images waiting to show, at most; further images wait encoded. */
+export declare const MAX_IMAGES_AHEAD = 4;
+/** Encoded images waiting to be decoded, at most; beyond this the oldest are dropped. */
+export declare const MAX_WAITING_IMAGES = 120;
+/** Plays still images through `createImageBitmap`. */
+export declare class ImageFrameDecoder implements IVideoFrameDecoder {
+    private readonly context;
+    private readonly canvas;
+    private readonly scheduler;
+    /** Images waiting to be decoded, in arrival order. */
+    private readonly waiting;
+    /** Whether a decode is under way: images decode one at a time, so they show in order. */
+    private decoding;
+    /** Bumped by Flush and Dispose: a decode that finishes after it is dropped. */
+    private generation;
+    private disposed;
+    /** Whether this browser can decode images of `mimeType` and show them. */
+    static CanPlay(mimeType: string): boolean;
+    /** @param context The player this decoder plays for. */
+    constructor(context: VideoFrameDecoderContext);
+    /** Always `false`: an image carries no voice. */
+    get IsPlaying(): boolean;
+    /** Images still to show: waiting to be decoded, decoding, or decoded and waiting for their time. */
+    get FramesAhead(): number;
+    Attach(element: HTMLVideoElement): void;
+    Detach(): void;
+    /** Takes one image: it is decoded in its turn and shown at its time, or at once when it has none. */
+    Append(frame: RealtimeVideoFrame): void;
+    /** Nothing to drain: every image stands alone, and those waiting still show. */
+    EndOfTurn(): void;
+    /** Barge-in: drops the images waiting, decoded or not, and the decode under way. The last image shown stays. */
+    Flush(): void;
+    Dispose(): void;
+    /** Decodes the next waiting image, when none is decoding and fewer than {@link MAX_IMAGES_AHEAD} wait to show. */
+    private feed;
+    private decoded;
+    private decodeFailed;
+}
+/** The `'image'` decoder: JPEG, PNG and WebP images, where this browser has `createImageBitmap`. */
+export declare const IMAGE_FRAME_DECODER: VideoFrameDecoderRegistration;
+//# sourceMappingURL=imageFrameDecoder.d.ts.map

package/dist/media/decoders/imageFrameDecoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"imageFrameDecoder.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/imageFrameDecoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,OAAO,KAAK,EAA2B,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACtF,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AAKxH,4EAA4E;AAC5E,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAElC,yFAAyF;AACzF,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,sDAAsD;AACtD,qBAAa,iBAAkB,YAAW,kBAAkB;IAkB5C,OAAO,CAAC,QAAQ,CAAC,OAAO;IAjBpC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiB;IAC3C,sDAAsD;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAiC;IACzD,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAS;IACzB,+EAA+E;IAC/E,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,QAAQ,CAAS;IAEzB,0EAA0E;WAC5D,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAKhD,wDAAwD;gBAC3B,OAAO,EAAE,wBAAwB;IAI9D,iDAAiD;IACjD,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED,oGAAoG;IACpG,IAAW,WAAW,IAAI,MAAM,CAE/B;IAEM,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAMvC,MAAM,IAAI,IAAI;IAIrB,qGAAqG;IAC9F,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAY9C,gFAAgF;IACzE,SAAS,IAAI,IAAI;IAIxB,gHAAgH;IACzG,KAAK,IAAI,IAAI;IAOb,OAAO,IAAI,IAAI;IAOtB,kHAAkH;IAClH,OAAO,CAAC,IAAI;IAaZ,OAAO,CAAC,OAAO;IAUf,OAAO,CAAC,YAAY;CASvB;AAED,oGAAoG;AACpG,eAAO,MAAM,mBAAmB,EAAE,6BAMjC,CAAC"}

package/dist/media/decoders/imageFrameDecoder.js

@@ -0,0 +1,110 @@
+import { FrameScheduler } from './frameScheduler.js';
+import { StreamCanvas } from './streamCanvas.js';
+import { IMAGE_FRAME_TYPES, MimeEssence } from './videoMimeTypes.js';
+/** Decoded images waiting to show, at most; further images wait encoded. */
+export const MAX_IMAGES_AHEAD = 4;
+/** Encoded images waiting to be decoded, at most; beyond this the oldest are dropped. */
+export const MAX_WAITING_IMAGES = 120;
+/** Plays still images through `createImageBitmap`. */
+export class ImageFrameDecoder {
+    /** Whether this browser can decode images of `mimeType` and show them. */
+    static CanPlay(mimeType) {
+        const canDecode = typeof createImageBitmap === 'function' && typeof Blob === 'function';
+        return canDecode && StreamCanvas.IsAvailable() && IMAGE_FRAME_TYPES.has(MimeEssence(mimeType));
+    }
+    /** @param context The player this decoder plays for. */
+    constructor(context) {
+        this.context = context;
+        this.canvas = new StreamCanvas();
+        /** Images waiting to be decoded, in arrival order. */
+        this.waiting = [];
+        /** Whether a decode is under way: images decode one at a time, so they show in order. */
+        this.decoding = false;
+        /** Bumped by Flush and Dispose: a decode that finishes after it is dropped. */
+        this.generation = 0;
+        this.disposed = false;
+        this.scheduler = new FrameScheduler(this.canvas, () => this.feed(), { Clock: context.Clock });
+    }
+    /** Always `false`: an image carries no voice. */
+    get IsPlaying() {
+        return false;
+    }
+    /** Images still to show: waiting to be decoded, decoding, or decoded and waiting for their time. */
+    get FramesAhead() {
+        return this.waiting.length + (this.decoding ? 1 : 0) + this.scheduler.Size;
+    }
+    Attach(element) {
+        if (!this.disposed) {
+            this.canvas.Attach(element);
+        }
+    }
+    Detach() {
+        this.canvas.Detach();
+    }
+    /** Takes one image: it is decoded in its turn and shown at its time, or at once when it has none. */
+    Append(frame) {
+        if (this.disposed || frame.Kind !== 'image') {
+            return;
+        }
+        this.waiting.push(frame);
+        if (this.waiting.length > MAX_WAITING_IMAGES) {
+            this.waiting.splice(0, this.waiting.length - MAX_WAITING_IMAGES);
+            this.context.Report('pending-overflow', 'Images arrived faster than they could be decoded; the oldest were dropped.');
+        }
+        this.feed();
+    }
+    /** Nothing to drain: every image stands alone, and those waiting still show. */
+    EndOfTurn() {
+        // Images need no flush.
+    }
+    /** Barge-in: drops the images waiting, decoded or not, and the decode under way. The last image shown stays. */
+    Flush() {
+        this.generation++;
+        this.decoding = false;
+        this.waiting.length = 0;
+        this.scheduler.Clear();
+    }
+    Dispose() {
+        this.Flush();
+        this.disposed = true;
+        this.scheduler.Dispose();
+        this.canvas.Dispose();
+    }
+    /** Decodes the next waiting image, when none is decoding and fewer than {@link MAX_IMAGES_AHEAD} wait to show. */
+    feed() {
+        if (this.disposed || this.decoding || this.waiting.length === 0 || this.scheduler.Size >= MAX_IMAGES_AHEAD) {
+            return;
+        }
+        const frame = this.waiting.shift();
+        const generation = this.generation;
+        this.decoding = true;
+        createImageBitmap(new Blob([frame.Data], { type: frame.MimeType })).then((image) => this.decoded(generation, image, frame.PresentationTimeMs), (error) => this.decodeFailed(generation, frame, error));
+    }
+    decoded(generation, image, timeMs) {
+        if (generation !== this.generation) {
+            image.close();
+            return;
+        }
+        this.decoding = false;
+        this.scheduler.Push(image, timeMs);
+        this.feed();
+    }
+    decodeFailed(generation, frame, error) {
+        if (generation !== this.generation) {
+            return;
+        }
+        this.decoding = false;
+        const reason = error instanceof Error ? error.message : String(error);
+        this.context.Report('append-failed', `The browser could not decode an image of type ${frame.MimeType}: ${reason}`);
+        this.feed();
+    }
+}
+/** The `'image'` decoder: JPEG, PNG and WebP images, where this browser has `createImageBitmap`. */
+export const IMAGE_FRAME_DECODER = {
+    Name: 'image',
+    Kind: 'image',
+    Priority: 0,
+    CanPlay: (mimeType) => ImageFrameDecoder.CanPlay(mimeType),
+    Create: (context) => new ImageFrameDecoder(context),
+};
+//# sourceMappingURL=imageFrameDecoder.js.map

package/dist/media/decoders/imageFrameDecoder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"imageFrameDecoder.js","sourceRoot":"","sources":["../../../src/media/decoders/imageFrameDecoder.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAElE,4EAA4E;AAC5E,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAElC,yFAAyF;AACzF,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAEtC,sDAAsD;AACtD,MAAM,OAAO,iBAAiB;IAW1B,0EAA0E;IACnE,MAAM,CAAC,OAAO,CAAC,QAAgB;QAClC,MAAM,SAAS,GAAG,OAAO,iBAAiB,KAAK,UAAU,IAAI,OAAO,IAAI,KAAK,UAAU,CAAC;QACxF,OAAO,SAAS,IAAI,YAAY,CAAC,WAAW,EAAE,IAAI,iBAAiB,CAAC,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IACnG,CAAC;IAED,wDAAwD;IACxD,YAA6B,OAAiC;QAAjC,YAAO,GAAP,OAAO,CAA0B;QAjB7C,WAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAE7C,sDAAsD;QACrC,YAAO,GAA8B,EAAE,CAAC;QACzD,yFAAyF;QACjF,aAAQ,GAAG,KAAK,CAAC;QACzB,+EAA+E;QACvE,eAAU,GAAG,CAAC,CAAC;QACf,aAAQ,GAAG,KAAK,CAAC;QAUrB,IAAI,CAAC,SAAS,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,iDAAiD;IACjD,IAAW,SAAS;QAChB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,oGAAoG;IACpG,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;IAC/E,CAAC;IAEM,MAAM,CAAC,OAAyB;QACnC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChC,CAAC;IACL,CAAC;IAEM,MAAM;QACT,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;IACzB,CAAC;IAED,qGAAqG;IAC9F,MAAM,CAAC,KAAyB;QACnC,IAAI,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;QACX,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,kBAAkB,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,4EAA4E,CAAC,CAAC;QAC1H,CAAC;QACD,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAED,gFAAgF;IACzE,SAAS;QACZ,wBAAwB;IAC5B,CAAC;IAED,gHAAgH;IACzG,KAAK;QACR,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;QACxB,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IAC1B,CAAC;IAED,kHAAkH;IAC1G,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,gBAAgB,EAAE,CAAC;YACzG,OAAO;QACX,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAA6B,CAAC;QAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,iBAAiB,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,IAAI,CACpE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,kBAAkB,CAAC,EACpE,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,CAClE,CAAC;IACN,CAAC;IAEO,OAAO,CAAC,UAAkB,EAAE,KAAkB,EAAE,MAA0B;QAC9E,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,UAAkB,EAAE,KAA8B,EAAE,KAAc;QACnF,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACtE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,iDAAiD,KAAK,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC;QACnH,IAAI,CAAC,IAAI,EAAE,CAAC;IAChB,CAAC;CACJ;AAED,oGAAoG;AACpG,MAAM,CAAC,MAAM,mBAAmB,GAAkC;IAC9D,IAAI,EAAE,OAAO;IACb,IAAI,EAAE,OAAO;IACb,QAAQ,EAAE,CAAC;IACX,OAAO,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,QAAQ,CAAC;IAC1D,MAAM,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,iBAAiB,CAAC,OAAO,CAAC;CACtD,CAAC"}

package/dist/media/decoders/mseFmp4Decoder.d.ts

@@ -0,0 +1,130 @@
+/**
+ * @fileoverview MSE FMP4 DECODER: plays fragmented MP4 frames (`Kind: 'fmp4'`), such as a Gemini Live avatar, through
+ * Media Source Extensions into the host's `<video>` element. The `'mse-fmp4'` decoder of `VideoPlayout`.
+ *
+ * - **Format.** The first piece of a turn is an init segment (`Piece: 'init'`, `ftyp` + `moov`); each later piece is a
+ *   media fragment (`moof` + `mdat`). A turn may send a fresh init segment and restart its timestamps; the source buffer
+ *   runs in `sequence` mode, so playback stays continuous either way.
+ * - **Codecs come from the init segment.** The source buffer takes the codecs the init segment's tracks name
+ *   (`avc1.42c01f` alone for video-only MP4), read by `ReadFmp4Init` from `@memberjunction/ai`; until an init arrives, or
+ *   when it names a codec the reader doesn't know, it takes the player's configured type.
+ * - **The video carries the voice.** An avatar's audio is muxed into the MP4, so the element is unmuted and lips and voice
+ *   stay in sync on the media's own timestamps. While media is buffered ahead of the playhead, {@link MseFmp4Decoder.IsPlaying}
+ *   is true: the agent is audibly speaking. When the voice plays elsewhere (separate PCM), the element is muted. The
+ *   element-attached hook lets the realtime client route the element's audio into its own Web Audio graph.
+ * - **End of turn** ends the stream once the turn's pieces are in, so playback runs to the true end and holds the last
+ *   frame. **Barge-in** drops everything not yet played and pauses. **Gaps** (between turns, or while a connection
+ *   resumes) hold the last frame.
+ * - **Browsers.** `MediaSource` (Chrome, Edge, Firefox, desktop Safari), or `ManagedMediaSource` (iOS Safari 17.1+).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import { type RealtimeVideoFrame } from '@memberjunction/ai';
+import type { IVideoFrameDecoder, VideoFrameDecoderContext, VideoFrameDecoderRegistration } from '../videoFrameDecoder.js';
+/** Plays fragmented MP4 frames through MSE into one `<video>` element at a time. */
+export declare class MseFmp4Decoder implements IVideoFrameDecoder {
+    private readonly context;
+    private readonly mimeType;
+    private readonly backBufferSeconds;
+    private readonly pending;
+    private element;
+    private mediaSource;
+    private objectUrl;
+    private sourceBuffer;
+    /** The type the source buffer was opened with, or last changed to. */
+    private bufferType;
+    /** The latest init segment, replayed into a new source buffer when the decoder moves to another element. */
+    private lastInit;
+    /** Set by Flush: the next append starts at the playhead instead of after what was dropped. */
+    private restartAtPlayhead;
+    private retriedAfterQuota;
+    /** Where the last back-buffer trim ended, in media time. */
+    private trimmedTo;
+    /** Set by EndOfTurn: end the stream once every pending piece is in. */
+    private endPending;
+    /** Where each appended piece ends on the element's timeline (seconds), oldest first, until the playhead passes it. */
+    private readonly appendedEnds;
+    /** Whether the source buffer is appending a piece: its `updateend` records where the piece ends. */
+    private appending;
+    private disposed;
+    /** Whether this browser can play `mimeType` through MSE. */
+    static CanPlay(mimeType: string): boolean;
+    /** @param context The player this decoder plays for. */
+    constructor(context: VideoFrameDecoderContext);
+    /**
+     * Whether the element is playing forward with media buffered ahead of the playhead: the agent is audibly
+     * speaking. False while it waits for data, which it does just short of the end of a turn that hasn't
+     * been ended with {@link EndOfTurn}.
+     */
+    get IsPlaying(): boolean;
+    /**
+     * How many pieces are still to play: pending (waiting for the source buffer, or for an element), being appended, or
+     * appended and ending past the playhead. The source buffer lays pieces end to end (`sequence` mode), so a piece ends
+     * where the buffered media ended once it was in. A piece a {@link Flush} removed, or the browser rejected, is not
+     * counted; nor is media of an element the decoder let go.
+     */
+    get FramesAhead(): number;
+    /** Hands over one frame, in the order they arrived: an init segment or a media fragment. */
+    Append(frame: RealtimeVideoFrame): void;
+    /**
+     * The turn's last piece has arrived. Ends the stream once it is all in, so playback runs to the true end of
+     * the turn; the next {@link Append} reopens it and playback continues from there.
+     */
+    EndOfTurn(): void;
+    /**
+     * Barge-in: drops everything not yet played and pauses, so the voice stops now instead of after the audio the
+     * browser has already decoded. The last frame stays on screen; the next turn's media resumes playback.
+     */
+    Flush(): void;
+    /** Stops playout and releases the element and the media source. */
+    Dispose(): void;
+    /** Takes the element over: a new media source, the latest init segment replayed into it, then what is pending. */
+    Attach(element: HTMLVideoElement): void;
+    /** Lets the element go: its source is removed and the media source released. */
+    Detach(): void;
+    /** Readies an element before it plays: its audio on or off, the attach hook, and what iOS's managed MSE requires. */
+    private prepareElement;
+    private openSourceBuffer;
+    /** Appends the next pending frame when the buffer is free. */
+    private pump;
+    /** The type for a source buffer that takes `init`: the init's own codecs when this browser plays them, else the player's type. */
+    private typeFor;
+    /**
+     * Switches the source buffer to an init segment's codecs before the init is appended, when they differ from the
+     * buffer's: it opened with the player's type before the first init arrived, or a later init changed tracks. A browser
+     * without `changeType`, or an init whose codecs can't be read or played, leaves the buffer as it is.
+     */
+    private matchBufferType;
+    /** After each append or removal: trims played media, appends the next piece, and ends a finished turn. */
+    private afterUpdate;
+    /**
+     * An append finished: the piece ends where the buffered media now ends, since the buffer lays pieces end to end.
+     * Pieces the playhead has passed are forgotten here too, so the list holds only what is ahead.
+     */
+    private recordAppendedEnd;
+    /** Forgets the appended pieces the playhead has reached the end of (to within {@link PLAYING_EPSILON_SECONDS}). */
+    private forgetPlayedPieces;
+    /** Ends the stream for a finished turn once nothing is pending or in flight. */
+    private endStreamWhenDrained;
+    /**
+     * A turn that ran to its end, or a Flush, leaves the element paused. Once new media reaches past the playhead,
+     * play on from there. An ended element is first sought to where it stopped: play() on an ended element starts
+     * over from the beginning (Chrome does), which would replay the previous turn.
+     */
+    private resumeWhenMediaArrives;
+    /**
+     * Removes played media more than the back buffer's seconds behind the playhead, so a long session stays within the
+     * browser's buffer quota. A trim waits until {@link TRIM_STEP_SECONDS} more have played since the last one, so a
+     * removal that doesn't shrink the buffer can't repeat on every `updateend`. When `bufferFull`, keeps only
+     * {@link QUOTA_BACK_BUFFER_SECONDS} and skips the wait.
+     *
+     * @returns Whether a removal started (the next append waits for its `updateend`).
+     */
+    private trimBackBuffer;
+    /** Out of buffer space: trim harder and retry once. Anything else drops the piece. */
+    private handleAppendError;
+    private detach;
+}
+/** The `'mse-fmp4'` decoder: fragmented MP4 frames whose type this browser's Media Source Extensions play. */
+export declare const MSE_FMP4_DECODER: VideoFrameDecoderRegistration;
+//# sourceMappingURL=mseFmp4Decoder.d.ts.map

package/dist/media/decoders/mseFmp4Decoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mseFmp4Decoder.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/mseFmp4Decoder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,OAAO,EAA6C,KAAK,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxG,OAAO,KAAK,EAAE,kBAAkB,EAAE,wBAAwB,EAAE,6BAA6B,EAAE,MAAM,sBAAsB,CAAC;AA2CxH,oFAAoF;AACpF,qBAAa,cAAe,YAAW,kBAAkB;IA+BzC,OAAO,CAAC,QAAQ,CAAC,OAAO;IA9BpC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IACxD,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,sEAAsE;IACtE,OAAO,CAAC,UAAU,CAAuB;IACzC,4GAA4G;IAC5G,OAAO,CAAC,QAAQ,CAAuC;IACvD,8FAA8F;IAC9F,OAAO,CAAC,iBAAiB,CAAS;IAClC,OAAO,CAAC,iBAAiB,CAAS;IAClC,4DAA4D;IAC5D,OAAO,CAAC,SAAS,CAAK;IACtB,uEAAuE;IACvE,OAAO,CAAC,UAAU,CAAS;IAC3B,sHAAsH;IACtH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,oGAAoG;IACpG,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAS;IAEzB,4DAA4D;WAC9C,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIhD,wDAAwD;gBAC3B,OAAO,EAAE,wBAAwB;IAK9D;;;;OAIG;IACH,IAAW,SAAS,IAAI,OAAO,CAO9B;IAED;;;;;OAKG;IACH,IAAW,WAAW,IAAI,MAAM,CAG/B;IAED,4FAA4F;IACrF,MAAM,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI;IAoB9C;;;OAGG;IACI,SAAS,IAAI,IAAI;IAKxB;;;OAGG;IACI,KAAK,IAAI,IAAI;IAsBpB,mEAAmE;IAC5D,OAAO,IAAI,IAAI;IAMtB,kHAAkH;IAC3G,MAAM,CAAC,OAAO,EAAE,gBAAgB,GAAG,IAAI;IAsB9C,gFAAgF;IACzE,MAAM,IAAI,IAAI;IAIrB,qHAAqH;IACrH,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,gBAAgB;IAmBxB,8DAA8D;IAC9D,OAAO,CAAC,IAAI;IAuBZ,kIAAkI;IAClI,OAAO,CAAC,OAAO;IAKf;;;;OAIG;IACH,OAAO,CAAC,eAAe;IASvB,0GAA0G;IAC1G,OAAO,CAAC,WAAW;IAUnB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAUzB,mHAAmH;IACnH,OAAO,CAAC,kBAAkB;IAU1B,gFAAgF;IAChF,OAAO,CAAC,oBAAoB;IAa5B;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;;;;;;OAOG;IACH,OAAO,CAAC,cAAc;IAgBtB,sFAAsF;IACtF,OAAO,CAAC,iBAAiB;IAYzB,OAAO,CAAC,MAAM;CAoBjB;AAED,8GAA8G;AAC9G,eAAO,MAAM,gBAAgB,EAAE,6BAM9B,CAAC"}