@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/channelVideoSource.js

@@ -1,22 +1,39 @@
 /**
  * @fileoverview SHARED CHANNEL INBOUND VIDEO BRIDGE (Phase F5).
  *
- * Single shared implementation used by both Whiteboard and Remote Browser channels
- * to source inbound video frames into a realtime session when the model supports it.
+ * Single shared implementation used by the Whiteboard, Remote Browser and any other channel that
+ * shows the model pixels. It no longer writes to the model directly: it REGISTERS its channel as a
+ * source with the session's {@link VideoSourceArbiter} and pushes frames to it, so the arbiter stays
+ * the single writer of inbound video and decides which source the model sees when several are live.
  *
  * Enforces:
  *  1. Model capability & track gate: if the session has not established an inbound video track,
  *     falls back to tool-only behaviour — NO frames sent and NO errors raised.
- *  2. Cadence throttling: capped at 1 frame per second maximum (1 fps ceiling per Live API capabilities guide).
- *  3. Clean teardown on Stop().
+ *  2. Cadence: the poller (when used) runs at the NEGOTIATED inbound rate, read from the client;
+ *     there is no rate constant here. The arbiter and the driver enforce the same rate as a spacing.
+ *  3. Clean teardown on Stop(), which also unregisters the source.
+ *
+ * An owner that sends frames only when its picture changes learns from {@link ChannelInboundVideoBridge.OnSourceActivated}
+ * when the arbiter starts sending its source to the model again, so it can send the picture the model missed.
  *
  * @module @memberjunction/ai-realtime-client
  * @author MemberJunction.com
  */
+import { NominalVideoFrameIntervalMs } from './videoPacing.js';
+import { VideoSourceArbiter } from './videoSourceArbiter.js';
+/** Counter behind the generated source ids. */
+let generatedSourceCounter = 0;
 /**
- * Shared bridge connecting a visual channel surface to the session's inbound video track.
+ * Shared bridge connecting a visual channel surface to the session's inbound video track, by way of
+ * the session's {@link VideoSourceArbiter}.
  */
 export class ChannelInboundVideoBridge {
+    /**
+     * @param clientOrGetter The session's client, or a getter for it (a reconnect replaces the client). Any
+     *   {@link IVideoFrameSink} works; a `BaseRealtimeClient` is one.
+     * @param provider Produces the frames.
+     * @param options Source identity, MIME type and the deprecated rate cap.
+     */
     constructor(clientOrGetter, provider, options = {}) {
         this.clientOrGetter = clientOrGetter;
         this.provider = provider;
@@ -24,10 +41,131 @@ export class ChannelInboundVideoBridge {
         this.timer = null;
         this.active = false;
         this.frameErrorLogged = false;
+        /** The arbiter this bridge is currently registered with; `null` when it is not registered. */
+        this.registeredWith = null;
+        /** Whether the owner switched this source off (exposure policy or the user). Survives re-registration. */
+        this.sourceEnabled = true;
+        /** The owner's handlers for {@link OnSourceActivated}. They survive re-registration. */
+        this.activatedHandlers = new Set();
+        /** Stops following the arbiter's changes; `null` while not following (not registered, or no handler). */
+        this.stopFollowingArbiter = null;
+        /** Whether the arbiter was sending this source to the model as of its last change. */
+        this.forwarded = false;
+        this.sourceId = options.SourceID ?? `channel-video-${++generatedSourceCounter}`;
     }
     get client() {
         return typeof this.clientOrGetter === 'function' ? this.clientOrGetter() : this.clientOrGetter;
     }
+    /** The id this bridge's source is registered under. */
+    get SourceID() {
+        return this.sourceId;
+    }
+    /** The arbiter for the current client, or `null` when there is no client yet. */
+    resolveArbiter() {
+        if (this.options.Arbiter) {
+            return this.options.Arbiter;
+        }
+        const client = this.client;
+        return client ? VideoSourceArbiter.ForSink(client) : null;
+    }
+    /**
+     * Registers this bridge's source with the session's arbiter (idempotent). A new session means a new
+     * client and so a new arbiter: the source is re-registered there and removed from the old one, so a
+     * reconnect never leaves a ghost source behind.
+     *
+     * @returns The arbiter, or `null` when there is no client to register with.
+     */
+    ensureRegistered() {
+        const arbiter = this.resolveArbiter();
+        if (!arbiter) {
+            return null;
+        }
+        if (this.registeredWith !== arbiter) {
+            this.unfollowArbiter();
+            this.registeredWith?.UnregisterSource(this.sourceId);
+            arbiter.RegisterSource({
+                SourceID: this.sourceId,
+                Label: this.options.Label ?? this.sourceId,
+                Kind: this.options.Kind ?? 'surface',
+                ChannelKey: this.options.ChannelKey,
+            });
+            this.registeredWith = arbiter;
+            if (!this.sourceEnabled) {
+                arbiter.SetSourceEnabled(this.sourceId, false, false);
+            }
+            // Followed from here, once registered as it will stay: the registration itself is not an activation.
+            this.followArbiter();
+        }
+        return arbiter;
+    }
+    /**
+     * Calls `handler` each time the arbiter starts sending this source to the model after it was not: another source
+     * that held the model's only stream stopped or was set aside, or this source was switched back on. The source's
+     * first registration does not count. A frame offered while the source was not sent never reached the model, so an
+     * owner that sends frames only when its picture changes sends its current picture here.
+     *
+     * The handler is kept across a re-registration (a reconnect's new client) and called until removed.
+     *
+     * @param handler Called synchronously from the arbiter's change event; it should not throw.
+     * @returns A function that removes the handler.
+     */
+    OnSourceActivated(handler) {
+        this.activatedHandlers.add(handler);
+        this.followArbiter();
+        return () => {
+            this.activatedHandlers.delete(handler);
+            if (this.activatedHandlers.size === 0) {
+                this.unfollowArbiter();
+            }
+        };
+    }
+    /** Starts following the arbiter this source is registered with, when an owner asked to hear of activations. */
+    followArbiter() {
+        const arbiter = this.registeredWith;
+        if (!arbiter || this.stopFollowingArbiter || this.activatedHandlers.size === 0) {
+            return;
+        }
+        this.forwarded = arbiter.GetActiveSourceIDs().includes(this.sourceId);
+        this.stopFollowingArbiter = arbiter.OnChange(() => {
+            const forwarded = arbiter.GetActiveSourceIDs().includes(this.sourceId);
+            const activated = forwarded && !this.forwarded;
+            this.forwarded = forwarded;
+            if (activated) {
+                this.notifyActivated();
+            }
+        });
+    }
+    /** Stops following the arbiter's changes. */
+    unfollowArbiter() {
+        this.stopFollowingArbiter?.();
+        this.stopFollowingArbiter = null;
+        this.forwarded = false;
+    }
+    /** Calls each activation handler, containing a handler that throws. */
+    notifyActivated() {
+        for (const handler of [...this.activatedHandlers]) {
+            try {
+                handler();
+            }
+            catch (err) {
+                console.error('[ChannelInboundVideoBridge] Error in a source-activated handler:', err);
+            }
+        }
+    }
+    /**
+     * Registers this bridge's source with the session's arbiter now, without starting any poll — so the
+     * source is LISTED (and can be switched on or off) before it has pushed a frame. A source that is
+     * switched off never pushes one, so without this it could never be switched back on.
+     *
+     * @returns `true` if registered; `false` when there is no client or no inbound video track (nothing to register with).
+     */
+    Register() {
+        const client = this.client;
+        if (!client || !client.IsTrackEstablished('video', 'inbound')) {
+            return false;
+        }
+        return this.ensureRegistered() !== null;
+    }
     /**
      * Starts the periodic frame pump if the session has an established inbound video track.
      *
@@ -38,43 +176,66 @@ export class ChannelInboundVideoBridge {
         if (this.active) {
             return true;
         }
-        if (!this.client || !this.client.IsTrackEstablished('video', 'inbound')) {
+        const client = this.client;
+        if (!client || !client.IsTrackEstablished('video', 'inbound')) {
             // Non-video model or unrequested video track: graceful fallback with no error
             return false;
         }
+        this.ensureRegistered();
         this.active = true;
-        const rate = Math.min(Math.max(this.options.Rate ?? 1, 0.1), 1);
-        const intervalMs = Math.floor(1000 / rate);
         this.timer = setInterval(() => {
             void this.pumpFrame();
-        }, intervalMs);
+        }, this.pollIntervalMs(client));
         return true;
     }
+    /** The poll interval: the negotiated rate, capped by the (deprecated) `Rate` option when one was given. */
+    pollIntervalMs(client) {
+        const negotiated = client.InboundVideoRate;
+        const requested = this.options.Rate;
+        const rate = typeof requested === 'number' && requested > 0
+            ? Math.max(0.1, typeof negotiated === 'number' ? Math.min(requested, negotiated) : requested)
+            : negotiated;
+        return NominalVideoFrameIntervalMs(rate);
+    }
     /**
-     * Directly sends a frame through the client when the inbound video track is established
-     * and the client implements `SendVideoFrame`.
+     * Offers a frame to the arbiter, which forwards it only if this source is among those the model
+     * should see (and enabled, and inside the negotiated pacing interval).
      *
-     * @returns `true` if dispatched to the client; `false` otherwise.
+     * @returns `true` if dispatched to the model; `false` otherwise.
      */
     sendFrameDirect(base64Jpeg) {
-        if (!this.client || !this.client.IsTrackEstablished('video', 'inbound')) {
+        const client = this.client;
+        if (!client || !client.IsTrackEstablished('video', 'inbound')) {
             return false;
         }
-        if (typeof this.client.SendVideoFrame !== 'function') {
+        if (typeof client.SendVideoFrame !== 'function') {
             return false;
         }
-        return Boolean(this.client.SendVideoFrame(base64Jpeg, this.options.MimeType ?? 'image/jpeg'));
+        const arbiter = this.ensureRegistered();
+        return arbiter ? arbiter.PushFrame(this.sourceId, base64Jpeg, this.options.MimeType ?? 'image/jpeg') : false;
     }
     /**
      * Pushes an event-driven frame (e.g. screencast push from RemoteBrowser or stroke completion).
-     * Dispatches directly to the client's authoritative throttle gate.
+     * Registers the source on first use, so a purely event-driven channel never needs {@link Start}.
      *
      * @param base64Jpeg The base64-encoded image frame.
-     * @returns `true` if the frame was sent; `false` if dropped (throttled, track unestablished, or driver lacks video support).
+     * @returns `true` if the frame was sent; `false` if dropped (throttled, not the source the model is
+     *   viewing, turned off, track unestablished, or driver lacks video support).
      */
     PushFrame(base64Jpeg) {
         return this.sendFrameDirect(base64Jpeg);
     }
+    /**
+     * Turns this source on or off at the arbiter (a no-op before the first registration other than
+     * remembering the choice). A disabled source is not a candidate and forwards nothing.
+     *
+     * @param enabled Whether the model may see this source.
+     * @param notify Whether the arbiter tells the model. Pass `false` when the caller sends its own note.
+     */
+    SetSourceEnabled(enabled, notify = true) {
+        this.sourceEnabled = enabled;
+        this.registeredWith?.SetSourceEnabled(this.sourceId, enabled, notify);
+    }
     /**
      * Periodic poll tick that fetches the latest frame from the provider and pushes it.
      * Uses `sendFrameDirect` so periodic polling paced by `setInterval` does not fight
@@ -99,7 +260,8 @@ export class ChannelInboundVideoBridge {
         }
     }
     /**
-     * Stops the bridge and clears any active polling timer.
+     * Stops the bridge: clears the poll timer and removes this source from the arbiter. Activation handlers stay, and
+     * are called again if the source registers again.
      */
     Stop() {
         this.active = false;
@@ -108,6 +270,9 @@ export class ChannelInboundVideoBridge {
             clearInterval(this.timer);
             this.timer = null;
         }
+        this.unfollowArbiter();
+        this.registeredWith?.UnregisterSource(this.sourceId);
+        this.registeredWith = null;
     }
     /** Whether the video bridge is currently actively pumping frames. */
     get IsActive() {

package/dist/media/channelVideoSource.js.map

@@ -1 +1 @@
-{"version":3,"file":"channelVideoSource.js","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAyBH;;GAEG;AACH,MAAM,OAAO,yBAAyB;IAKlC,YACqB,cAAqG,EACrG,QAA+B,EAC/B,UAA4C,EAAE;QAF9C,mBAAc,GAAd,cAAc,CAAuF;QACrG,aAAQ,GAAR,QAAQ,CAAuB;QAC/B,YAAO,GAAP,OAAO,CAAuC;QAP3D,UAAK,GAA0C,IAAI,CAAC;QACpD,WAAM,GAAG,KAAK,CAAC;QACf,qBAAgB,GAAG,KAAK,CAAC;IAM9B,CAAC;IAEJ,IAAY,MAAM;QACd,OAAO,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;IACnG,CAAC;IAED;;;;;OAKG;IACI,KAAK;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YACtE,8EAA8E;YAC9E,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1B,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QAC1B,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,UAAkB;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YACtE,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YACnD,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC;IAClG,CAAC;IAED;;;;;;OAMG;IACI,SAAS,CAAC,UAAkB;QAC/B,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,CAAC,KAAK,CAAC,0GAA0G,EAAE,GAAG,CAAC,CAAC;YACnI,CAAC;QACL,CAAC;IACL,CAAC;IAED;;OAEG;IACI,IAAI;QACP,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,CAAC;IACL,CAAC;IAED,qEAAqE;IACrE,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ"}
+{"version":3,"file":"channelVideoSource.js","sourceRoot":"","sources":["../../src/media/channelVideoSource.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAA8C,MAAM,sBAAsB,CAAC;AA6CtG,+CAA+C;AAC/C,IAAI,sBAAsB,GAAG,CAAC,CAAC;AAE/B;;;GAGG;AACH,MAAM,OAAO,yBAAyB;IAgBlC;;;;;OAKG;IACH,YACqB,cAA+F,EAC/F,QAA+B,EAC/B,UAA4C,EAAE;QAF9C,mBAAc,GAAd,cAAc,CAAiF;QAC/F,aAAQ,GAAR,QAAQ,CAAuB;QAC/B,YAAO,GAAP,OAAO,CAAuC;QAxB3D,UAAK,GAA0C,IAAI,CAAC;QACpD,WAAM,GAAG,KAAK,CAAC;QACf,qBAAgB,GAAG,KAAK,CAAC;QAEjC,8FAA8F;QACtF,mBAAc,GAA8B,IAAI,CAAC;QACzD,0GAA0G;QAClG,kBAAa,GAAG,IAAI,CAAC;QAC7B,wFAAwF;QACvE,sBAAiB,GAAG,IAAI,GAAG,EAAc,CAAC;QAC3D,yGAAyG;QACjG,yBAAoB,GAAwB,IAAI,CAAC;QACzD,sFAAsF;QAC9E,cAAS,GAAG,KAAK,CAAC;QAatB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,EAAE,sBAAsB,EAAE,CAAC;IACpF,CAAC;IAED,IAAY,MAAM;QACd,OAAO,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;IACnG,CAAC;IAED,uDAAuD;IACvD,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC;IACzB,CAAC;IAED,iFAAiF;IACzE,cAAc;QAClB,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;QAChC,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,OAAO,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED;;;;;;OAMG;IACK,gBAAgB;QACpB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACtC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,KAAK,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrD,OAAO,CAAC,cAAc,CAAC;gBACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ;gBAC1C,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,SAAS;gBACpC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;aACtC,CAAC,CAAC;YACH,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;gBACtB,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;YACD,qGAAqG;YACrG,IAAI,CAAC,aAAa,EAAE,CAAC;QACzB,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACI,iBAAiB,CAAC,OAAmB;QACxC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,OAAO,GAAG,EAAE;YACR,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACvC,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3B,CAAC;QACL,CAAC,CAAC;IACN,CAAC;IAED,+GAA+G;IACvG,aAAa;QACjB,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC;QACpC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,oBAAoB,IAAI,IAAI,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC7E,OAAO;QACX,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACtE,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;YAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvE,MAAM,SAAS,GAAG,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;YAC/C,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;YAC3B,IAAI,SAAS,EAAE,CAAC;gBACZ,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3B,CAAC;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAED,6CAA6C;IACrC,eAAe;QACnB,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC9B,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAED,uEAAuE;IAC/D,eAAe;QACnB,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC;gBACD,OAAO,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,kEAAkE,EAAE,GAAG,CAAC,CAAC;YAC3F,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACI,QAAQ;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,EAAE,KAAK,IAAI,CAAC;IAC5C,CAAC;IAED;;;;;OAKG;IACI,KAAK;QACR,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACd,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,8EAA8E;YAC9E,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC1B,KAAK,IAAI,CAAC,SAAS,EAAE,CAAC;QAC1B,CAAC,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,2GAA2G;IACnG,cAAc,CAAC,MAAuB;QAC1C,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QACpC,MAAM,IAAI,GACN,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,GAAG,CAAC;YAC1C,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7F,CAAC,CAAC,UAAU,CAAC;QACrB,OAAO,2BAA2B,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC;IAED;;;;;OAKG;IACK,eAAe,CAAC,UAAkB;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC5D,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YAC9C,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;IACjH,CAAC;IAED;;;;;;;OAOG;IACI,SAAS,CAAC,UAAkB;QAC/B,OAAO,IAAI,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC;IAC5C,CAAC;IAED;;;;;;OAMG;IACI,gBAAgB,CAAC,OAAgB,EAAE,MAAM,GAAG,IAAI;QACnD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC;QAC7B,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,SAAS;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACf,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC;YACnD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3C,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACzB,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,CAAC,KAAK,CAAC,0GAA0G,EAAE,GAAG,CAAC,CAAC;YACnI,CAAC;QACL,CAAC;IACL,CAAC;IAED;;;OAGG;IACI,IAAI;QACP,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,eAAe,EAAE,CAAC;QACvB,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;IAC/B,CAAC;IAED,qEAAqE;IACrE,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;CACJ"}

package/dist/media/decoders/frameScheduler.d.ts

@@ -0,0 +1,119 @@
+/**
+ * @fileoverview FRAME SCHEDULER: shows decoded pictures on a {@link StreamCanvas} when their presentation time comes,
+ * for the decoders that aren't MSE (WebCodecs chunks, images). MSE paces itself on the element's clock.
+ *
+ * - **On the voice's clock, when the player has one** ({@link FrameSchedulerOptions.Clock}): the meeting bot's rules, on
+ *   media time. A picture shows when the clock reaches its time; one more than {@link MAX_CLOCK_LATE_MS} behind the clock
+ *   is dropped, never shown; one ahead of it waits, and one more than {@link MAX_LEAD_MS} ahead is dropped (it belongs to
+ *   a stream the voice has left). A picture timed before the last one shown starts a new stream: it waits while the clock
+ *   still plays the old stream, instead of counting as late.
+ * - **When the clock reads nothing.** After a reading, pictures carry on by their own times from the clock's last reading,
+ *   as if the voice went on (the tail of a turn whose video outlasts its voice, or idle video between turns); a new stream
+ *   then waits for its voice. Before any reading (at the start, or after a barge-in) pictures wait up to
+ *   {@link CLOCK_WAIT_MS} for the voice to start, then go by their own times.
+ * - **By their own times, without a clock.** The first timed picture after a start, a flush or a new stream anchors media
+ *   time to the wall clock; each later picture shows when its time comes. When several are due, the newest shows and the
+ *   older ones are dropped. A picture shown more than {@link MAX_LATE_MS} late anchors the pacing again; so does a
+ *   picture timed before the last one shown, or more than {@link MAX_LEAD_MS} ahead.
+ * - **Untimed pictures** (an image that carries no time) show as soon as they arrive.
+ * - **Bounded.** The decoders hold encoded frames back while {@link FrameScheduler.Size} pictures wait, and the scheduler
+ *   calls back each time it shows one, so a decoder never holds many decoded pictures (a browser decoder stalls when its
+ *   output frames aren't closed).
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+import type { IPlaybackClock } from '../playbackClock.js';
+import type { StreamCanvas, StreamPicture } from './streamCanvas.js';
+/** Without a clock, a picture shown this much later than its time anchors the pacing again, in milliseconds. */
+export declare const MAX_LATE_MS = 250;
+/** A picture due this much ahead starts a new stream (without a clock) or is dropped (on the clock), in milliseconds. */
+export declare const MAX_LEAD_MS = 2000;
+/** On the voice's clock, a picture this far behind it is dropped rather than shown, in milliseconds. */
+export declare const MAX_CLOCK_LATE_MS = 500;
+/** How long pictures wait for a clock that has read nothing yet before they go by their own times, in milliseconds. */
+export declare const CLOCK_WAIT_MS = 500;
+/** How often waiting pictures look at the clock again, at most, in milliseconds. */
+export declare const CLOCK_POLL_MS = 20;
+/** What a scheduler paces by, besides the pictures' own times. */
+export interface FrameSchedulerOptions {
+    /** The voice's playback clock: pictures show when it reaches their times. */
+    Clock?: IPlaybackClock;
+    /** The wall clock in milliseconds; injectable for tests. */
+    Now?: () => number;
+}
+/** Shows pictures on a stream canvas at their presentation times: on the voice's clock when there is one. */
+export declare class FrameScheduler {
+    private readonly canvas;
+    private readonly onShown;
+    private readonly queue;
+    private readonly clock;
+    private readonly now;
+    /** Own-time pacing: where the current stream's media time sits on the wall clock. */
+    private anchor;
+    /** The time of the last timed picture shown, to see a stream restart. */
+    private lastShownMs;
+    /** The clock's last reading and when it was taken, until the clock stops reading. */
+    private lastReading;
+    /** Whether own-time pacing carries on from the clock's last reading (the voice stopped). */
+    private continuing;
+    /** When the pictures began waiting for the voice's clock to read. */
+    private waitingSince;
+    private timer;
+    private disposed;
+    /**
+     * @param canvas Where pictures show.
+     * @param onShown Called after each picture shown, so the decoder can decode more.
+     * @param options The voice's clock, and a wall clock for tests.
+     */
+    constructor(canvas: StreamCanvas, onShown: () => void, options?: FrameSchedulerOptions);
+    /** How many pictures wait to be shown. */
+    get Size(): number;
+    /**
+     * Queues a picture: it shows when its time comes, or at once when it has no time.
+     *
+     * @param picture The decoded picture; the scheduler closes it, shown or dropped.
+     * @param timeMs Its presentation time on the stream's timeline, when it has one.
+     */
+    Push(picture: StreamPicture, timeMs?: number): void;
+    /** Drops every waiting picture and forgets both clocks: the pacing starts over. The last picture shown stays. */
+    Clear(): void;
+    /** Clears, and closes every picture pushed later. */
+    Dispose(): void;
+    /** Shows the picture that is due now, if any, by the voice's clock or by the pictures' own times; then waits for the next. */
+    private tick;
+    /**
+     * Whether the pictures wait for the voice's clock, which reads nothing now. A clock that read before stopped: pacing
+     * carries on from its last reading, but a picture that starts a new stream waits for its voice. Pictures wait up to
+     * {@link CLOCK_WAIT_MS}, then go by their own times. Never without a clock.
+     */
+    private holdForClock;
+    /** Whether the next picture is timed before the last one shown: it starts a new stream. */
+    private headStartsNewStream;
+    /** Shows the newest picture the clock has reached, dropping the stale ones before it ({@link clockVerdict}). */
+    private showByClock;
+    /**
+     * What the clock says about a picture: it waits when it starts a new stream whose voice hasn't started, or is ahead of
+     * the clock; it is dropped when more than {@link MAX_CLOCK_LATE_MS} behind the clock or more than {@link MAX_LEAD_MS}
+     * ahead of it; otherwise it is due. An untimed picture is due at once.
+     */
+    private clockVerdict;
+    /**
+     * Whether a picture starts a new stream whose voice hasn't started: it is timed before the last picture shown, and the
+     * clock still reads the old stream (at or after that picture).
+     */
+    private awaitsNewStream;
+    /** Shows the newest picture whose own time has come (dropping older due ones). */
+    private showByOwnTime;
+    /** When a picture is due on the wall clock. A first picture, or one that starts a new stream, anchors the pacing to now. */
+    private dueAt;
+    /** A picture timed before the last one shown, or far ahead of the pacing, belongs to a new stream. */
+    private startsNewStream;
+    private show;
+    /** By own time: until the next picture is due. */
+    private ownTimeDelay;
+    /** On the clock: until the clock should reach the next picture, but never longer than {@link CLOCK_POLL_MS}. */
+    private clockDelay;
+    private schedule;
+    private cancelTimer;
+}
+//# sourceMappingURL=frameScheduler.d.ts.map

package/dist/media/decoders/frameScheduler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frameScheduler.d.ts","sourceRoot":"","sources":["../../../src/media/decoders/frameScheduler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAElE,gHAAgH;AAChH,eAAO,MAAM,WAAW,MAAM,CAAC;AAE/B,yHAAyH;AACzH,eAAO,MAAM,WAAW,OAAO,CAAC;AAEhC,wGAAwG;AACxG,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,uHAAuH;AACvH,eAAO,MAAM,aAAa,MAAM,CAAC;AAEjC,oFAAoF;AACpF,eAAO,MAAM,aAAa,KAAK,CAAC;AAchC,kEAAkE;AAClE,MAAM,WAAW,qBAAqB;IAClC,6EAA6E;IAC7E,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,4DAA4D;IAC5D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACtB;AAED,6GAA6G;AAC7G,qBAAa,cAAc;IAuBnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAvB5B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA0B;IAChD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA6B;IACnD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAe;IACnC,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAA4B;IAC1C,yEAAyE;IACzE,OAAO,CAAC,WAAW,CAAuB;IAC1C,qFAAqF;IACrF,OAAO,CAAC,WAAW,CAA4B;IAC/C,4FAA4F;IAC5F,OAAO,CAAC,UAAU,CAAS;IAC3B,qEAAqE;IACrE,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,KAAK,CAA8C;IAC3D,OAAO,CAAC,QAAQ,CAAS;IAEzB;;;;OAIG;gBAEkB,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,MAAM,IAAI,EACpC,OAAO,GAAE,qBAA0B;IAMvC,0CAA0C;IAC1C,IAAW,IAAI,IAAI,MAAM,CAExB;IAED;;;;;OAKG;IACI,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAW1D,iHAAiH;IAC1G,KAAK,IAAI,IAAI;IAYpB,qDAAqD;IAC9C,OAAO,IAAI,IAAI;IAKtB,8HAA8H;IAC9H,OAAO,CAAC,IAAI;IAsBZ;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAqBpB,2FAA2F;IAC3F,OAAO,CAAC,mBAAmB;IAK3B,gHAAgH;IAChH,OAAO,CAAC,WAAW;IAqBnB;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAapB;;;OAGG;IACH,OAAO,CAAC,eAAe;IAIvB,kFAAkF;IAClF,OAAO,CAAC,aAAa;IAqBrB,4HAA4H;IAC5H,OAAO,CAAC,KAAK;IAWb,sGAAsG;IACtG,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,IAAI;IAOZ,kDAAkD;IAClD,OAAO,CAAC,YAAY;IAIpB,gHAAgH;IAChH,OAAO,CAAC,UAAU;IAKlB,OAAO,CAAC,QAAQ;IAMhB,OAAO,CAAC,WAAW;CAMtB"}

package/dist/media/decoders/frameScheduler.js

@@ -0,0 +1,238 @@
+/** Without a clock, a picture shown this much later than its time anchors the pacing again, in milliseconds. */
+export const MAX_LATE_MS = 250;
+/** A picture due this much ahead starts a new stream (without a clock) or is dropped (on the clock), in milliseconds. */
+export const MAX_LEAD_MS = 2000;
+/** On the voice's clock, a picture this far behind it is dropped rather than shown, in milliseconds. */
+export const MAX_CLOCK_LATE_MS = 500;
+/** How long pictures wait for a clock that has read nothing yet before they go by their own times, in milliseconds. */
+export const CLOCK_WAIT_MS = 500;
+/** How often waiting pictures look at the clock again, at most, in milliseconds. */
+export const CLOCK_POLL_MS = 20;
+/** Shows pictures on a stream canvas at their presentation times: on the voice's clock when there is one. */
+export class FrameScheduler {
+    /**
+     * @param canvas Where pictures show.
+     * @param onShown Called after each picture shown, so the decoder can decode more.
+     * @param options The voice's clock, and a wall clock for tests.
+     */
+    constructor(canvas, onShown, options = {}) {
+        this.canvas = canvas;
+        this.onShown = onShown;
+        this.queue = [];
+        /** Own-time pacing: where the current stream's media time sits on the wall clock. */
+        this.anchor = null;
+        /** The time of the last timed picture shown, to see a stream restart. */
+        this.lastShownMs = null;
+        /** The clock's last reading and when it was taken, until the clock stops reading. */
+        this.lastReading = null;
+        /** Whether own-time pacing carries on from the clock's last reading (the voice stopped). */
+        this.continuing = false;
+        /** When the pictures began waiting for the voice's clock to read. */
+        this.waitingSince = null;
+        this.timer = null;
+        this.disposed = false;
+        this.clock = options.Clock;
+        this.now = options.Now ?? (() => performance.now());
+    }
+    /** How many pictures wait to be shown. */
+    get Size() {
+        return this.queue.length;
+    }
+    /**
+     * Queues a picture: it shows when its time comes, or at once when it has no time.
+     *
+     * @param picture The decoded picture; the scheduler closes it, shown or dropped.
+     * @param timeMs Its presentation time on the stream's timeline, when it has one.
+     */
+    Push(picture, timeMs) {
+        if (this.disposed) {
+            picture.close();
+            return;
+        }
+        this.queue.push({ Picture: picture, TimeMs: timeMs });
+        if (this.timer === null) {
+            this.tick();
+        }
+    }
+    /** Drops every waiting picture and forgets both clocks: the pacing starts over. The last picture shown stays. */
+    Clear() {
+        this.cancelTimer();
+        for (const scheduled of this.queue.splice(0)) {
+            scheduled.Picture.close();
+        }
+        this.anchor = null;
+        this.lastShownMs = null;
+        this.lastReading = null;
+        this.continuing = false;
+        this.waitingSince = null;
+    }
+    /** Clears, and closes every picture pushed later. */
+    Dispose() {
+        this.disposed = true;
+        this.Clear();
+    }
+    /** Shows the picture that is due now, if any, by the voice's clock or by the pictures' own times; then waits for the next. */
+    tick() {
+        this.timer = null;
+        const now = this.now();
+        const clockMs = this.clock?.CurrentTimeMs ?? null;
+        let shown = false;
+        if (clockMs !== null) {
+            this.lastReading = { WallMs: now, MediaMs: clockMs };
+            shown = this.showByClock(clockMs);
+        }
+        else if (this.holdForClock(now)) {
+            this.schedule(CLOCK_POLL_MS);
+            return;
+        }
+        else {
+            shown = this.showByOwnTime(now);
+        }
+        if (this.queue.length > 0) {
+            this.schedule(clockMs === null ? this.ownTimeDelay(now) : this.clockDelay(clockMs));
+        }
+        if (shown) {
+            this.onShown();
+        }
+    }
+    /**
+     * Whether the pictures wait for the voice's clock, which reads nothing now. A clock that read before stopped: pacing
+     * carries on from its last reading, but a picture that starts a new stream waits for its voice. Pictures wait up to
+     * {@link CLOCK_WAIT_MS}, then go by their own times. Never without a clock.
+     */
+    holdForClock(now) {
+        if (!this.clock) {
+            return false;
+        }
+        if (this.lastReading) {
+            this.anchor = this.lastReading;
+            this.lastReading = null;
+            this.continuing = true;
+        }
+        if (this.continuing && this.headStartsNewStream()) {
+            this.anchor = null;
+            this.continuing = false;
+            this.waitingSince = now;
+        }
+        if (this.anchor) {
+            return false;
+        }
+        this.waitingSince ??= now;
+        return now - this.waitingSince < CLOCK_WAIT_MS;
+    }
+    /** Whether the next picture is timed before the last one shown: it starts a new stream. */
+    headStartsNewStream() {
+        const time = this.queue[0]?.TimeMs;
+        return time !== undefined && this.lastShownMs !== null && time < this.lastShownMs;
+    }
+    /** Shows the newest picture the clock has reached, dropping the stale ones before it ({@link clockVerdict}). */
+    showByClock(clockMs) {
+        let due = null;
+        while (this.queue.length > 0) {
+            const verdict = this.clockVerdict(this.queue[0].TimeMs, clockMs);
+            if (verdict === 'wait') {
+                break;
+            }
+            const head = this.queue.shift();
+            if (verdict === 'drop') {
+                head.Picture.close();
+                continue;
+            }
+            due?.Picture.close();
+            due = head;
+        }
+        if (due) {
+            this.show(due);
+        }
+        return due !== null;
+    }
+    /**
+     * What the clock says about a picture: it waits when it starts a new stream whose voice hasn't started, or is ahead of
+     * the clock; it is dropped when more than {@link MAX_CLOCK_LATE_MS} behind the clock or more than {@link MAX_LEAD_MS}
+     * ahead of it; otherwise it is due. An untimed picture is due at once.
+     */
+    clockVerdict(timeMs, clockMs) {
+        if (timeMs === undefined) {
+            return 'show';
+        }
+        if (this.awaitsNewStream(timeMs, clockMs)) {
+            return 'wait';
+        }
+        if (clockMs - timeMs > MAX_CLOCK_LATE_MS || timeMs - clockMs > MAX_LEAD_MS) {
+            return 'drop';
+        }
+        return timeMs > clockMs ? 'wait' : 'show';
+    }
+    /**
+     * Whether a picture starts a new stream whose voice hasn't started: it is timed before the last picture shown, and the
+     * clock still reads the old stream (at or after that picture).
+     */
+    awaitsNewStream(timeMs, clockMs) {
+        return this.lastShownMs !== null && timeMs < this.lastShownMs && clockMs >= this.lastShownMs;
+    }
+    /** Shows the newest picture whose own time has come (dropping older due ones). */
+    showByOwnTime(now) {
+        let due = null;
+        while (this.queue.length > 0) {
+            const dueMs = this.dueAt(this.queue[0], now);
+            if (dueMs > now) {
+                break;
+            }
+            due?.Scheduled.Picture.close();
+            due = { Scheduled: this.queue.shift(), DueMs: dueMs };
+        }
+        if (!due) {
+            return false;
+        }
+        this.show(due.Scheduled);
+        const time = due.Scheduled.TimeMs;
+        if (time !== undefined && now - due.DueMs > MAX_LATE_MS) {
+            this.anchor = { WallMs: now, MediaMs: time };
+        }
+        return true;
+    }
+    /** When a picture is due on the wall clock. A first picture, or one that starts a new stream, anchors the pacing to now. */
+    dueAt(scheduled, now) {
+        const time = scheduled.TimeMs;
+        if (time === undefined) {
+            return now;
+        }
+        if (!this.anchor || this.startsNewStream(time, now)) {
+            this.anchor = { WallMs: now, MediaMs: time };
+        }
+        return this.anchor.WallMs + (time - this.anchor.MediaMs);
+    }
+    /** A picture timed before the last one shown, or far ahead of the pacing, belongs to a new stream. */
+    startsNewStream(timeMs, now) {
+        const anchor = this.anchor;
+        const backwards = this.lastShownMs !== null && timeMs < this.lastShownMs;
+        return backwards || anchor.WallMs + (timeMs - anchor.MediaMs) - now > MAX_LEAD_MS;
+    }
+    show(scheduled) {
+        this.canvas.Show(scheduled.Picture);
+        if (scheduled.TimeMs !== undefined) {
+            this.lastShownMs = scheduled.TimeMs;
+        }
+    }
+    /** By own time: until the next picture is due. */
+    ownTimeDelay(now) {
+        return Math.max(0, this.dueAt(this.queue[0], now) - now);
+    }
+    /** On the clock: until the clock should reach the next picture, but never longer than {@link CLOCK_POLL_MS}. */
+    clockDelay(clockMs) {
+        const time = this.queue[0].TimeMs;
+        return time !== undefined && time > clockMs ? Math.min(CLOCK_POLL_MS, time - clockMs) : CLOCK_POLL_MS;
+    }
+    schedule(delayMs) {
+        if (this.timer === null && !this.disposed) {
+            this.timer = setTimeout(() => this.tick(), delayMs);
+        }
+    }
+    cancelTimer() {
+        if (this.timer !== null) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+}
+//# sourceMappingURL=frameScheduler.js.map