@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/media/videoSourceArbiter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoSourceArbiter.d.ts","sourceRoot":"","sources":["../../src/media/videoSourceArbiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAA0E,KAAK,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AAGzI;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC5B;;;OAGG;IACH,QAAQ,CAAC,sBAAsB,EAAE,MAAM,CAAC;IAExC,uGAAuG;IACvG,QAAQ,CAAC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IAE9C,mFAAmF;IACnF,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,sBAAsB,GAAG,OAAO,CAAC;IAEjF;;;;;OAKG;IACH,cAAc,CAAC,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IAEpF,+EAA+E;IAC/E,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACvC;AAED;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC;AAK9E,4EAA4E;AAC5E,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,CAE7E;AAED,yCAAyC;AACzC,MAAM,WAAW,qBAAqB;IAClC,kGAAkG;IAClG,QAAQ,EAAE,MAAM,CAAC;IACjB,qGAAqG;IACrG,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,IAAI,CAAC,EAAE,eAAe,CAAC;IACvB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACrB;AAED,iDAAiD;AACjD,MAAM,WAAW,gBAAiB,SAAQ,qBAAqB;IAC3D,iHAAiH;IACjH,OAAO,EAAE,OAAO,CAAC;IACjB,qFAAqF;IACrF,MAAM,EAAE,OAAO,CAAC;IAChB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,qBAAqB,GAAG,WAAW,GAAG,gBAAgB,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE1F;;;;;;;;;;GAUG;AACH,eAAO,MAAM,2BAA2B,EAAE,SAAS,qBAAqB,EAK7D,CAAC;AAEZ,gDAAgD;AAChD,MAAM,WAAW,yBAAyB;IACtC,6EAA6E;IAC7E,MAAM,CAAC,EAAE,SAAS,qBAAqB,EAAE,CAAC;IAC1C;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,KAAK,MAAM,GAAG,IAAI,CAAC;IAChE,2HAA2H;IAC3H,kBAAkB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACtD,8HAA8H;IAC9H,iBAAiB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,IAAI,CAAC;IACrD,wHAAwH;IACxH,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;IACpD,2DAA2D;IAC3D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACtB;AA2BD,uFAAuF;AACvF,MAAM,MAAM,oBAAoB,GAAG,eAAe,GAAG,CAAC,MAAM,eAAe,GAAG,IAAI,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,SAAS,CAAC;AAEnH;;;GAGG;AACH,qBAAa,kBAAkB;IAgCvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAhC5B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAA6C;IAE3E;;;;;;OAMG;WACW,OAAO,CAAC,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,yBAAyB,GAAG,kBAAkB;IASrG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmC;IAC1D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyB;IAClD,OAAO,CAAC,OAAO,CAAK;IACpB,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,OAAO,CAAuB;IACtC,OAAO,CAAC,cAAc,CAAuB;IAC7C,8FAA8F;IAC9F,OAAO,CAAC,UAAU,CAAyB;IAC3C,oGAAoG;IACpG,OAAO,CAAC,eAAe,CAAS;gBAGX,UAAU,EAAE,oBAAoB,EAChC,OAAO,GAAE,yBAA8B;IAK5D,OAAO,KAAK,IAAI,GAGf;IAED;;;OAGG;IACI,cAAc,CAAC,UAAU,EAAE,qBAAqB,GAAG,IAAI;IAuB9D,gFAAgF;IACzE,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAc/C,yCAAyC;IAClC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAI3C;;;OAGG;IACI,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO;IAarD;;;;OAIG;IACI,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IASzD,8GAA8G;IACvG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,GAAG,IAAI;IAMtD;;;;;;;;OAQG;IACI,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,UAAO,GAAG,OAAO;IAsBnF,sGAAsG;IAC/F,UAAU,IAAI,gBAAgB,EAAE;IAiBvC,+DAA+D;IACxD,kBAAkB,IAAI,MAAM,EAAE;IAIrC;;;;;OAKG;IACI,QAAQ,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI;IAOhD;;;;;;;;OAQG;IACI,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,SAAe,GAAG,OAAO;IA4BzF,gGAAgG;IACzF,OAAO,IAAI,IAAI;IAYtB,yDAAyD;IACzD,OAAO,CAAC,IAAI;IAgCZ,oGAAoG;IACpG,OAAO,CAAC,aAAa;IAQrB;;;;OAIG;IACH,OAAO,CAAC,SAAS;IA6BjB,OAAO,CAAC,kBAAkB;IAI1B,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,GAAG;IAIX,OAAO,CAAC,IAAI;IASZ,OAAO,CAAC,QAAQ;IAYhB,OAAO,CAAC,UAAU;IAUlB,OAAO,CAAC,WAAW;CAOtB"}

package/dist/media/videoSourceArbiter.js

@@ -0,0 +1,423 @@
+/**
+ * @fileoverview The VIDEO SOURCE ARBITER: the single writer of a session's inbound video.
+ *
+ * A realtime session can have many things worth showing the model at once: a shared screen, the camera,
+ * a whiteboard, a remote browser. Today's video models accept ONE inbound stream. Before this existed,
+ * every source wrote to that one stream directly: whichever happened to send last "won", and the model
+ * was never told what it was looking at. The arbiter owns the decision:
+ *
+ * - **Sources register** (`RegisterSource`) with an id, a label and a kind, and push frames to the
+ *   arbiter (`PushFrame`) rather than to the client.
+ * - **The arbiter maps N live sources onto the model's `MaxInboundVideoStreams`.** With enough streams
+ *   for every enabled source (a future multi-stream model) frames pass through untouched, each tagged
+ *   with its source id. Otherwise it ranks the sources by a POLICY (data, overridable) and forwards only
+ *   the top N.
+ * - **The model is told when the choice changes** (`[You can now see: Camera]`), because a model
+ *   that silently starts seeing something else will describe the old thing.
+ * - **The user can switch a source off.** A disabled source is not a candidate, sends nothing, and the
+ *   model is told it was turned off (`[You can no longer see: Camera (the user turned it off)]`).
+ * - **The rate comes only from the negotiated track.** Pacing is read from the sink
+ *   (`InboundVideoRate`), never a constant here.
+ *
+ * The notes' wording comes from `@memberjunction/ai` ({@link VideoSourceSeenNote}, {@link VideoSourceTurnedOffNote},
+ * {@link VideoSourceTurnedOnNote}), where a meeting's bridge engine gets its notes too, so the model reads one style of
+ * note in a call and in a meeting.
+ *
+ * It deliberately depends only on the narrow {@link IVideoFrameSink} interface (which
+ * `BaseRealtimeClient` satisfies) and imports nothing from `generic/` or the drivers, so the media code
+ * can be split into its own subpath without untangling it.
+ *
+ * The default policy and the rationale for its ordering are on {@link DEFAULT_VIDEO_SOURCE_POLICY}.
+ *
+ * @module @memberjunction/ai-realtime-client
+ */
+import { VideoSourceSeenNote, VideoSourceTurnedOffNote, VideoSourceTurnedOnNote } from '@memberjunction/ai';
+import { MinVideoFrameSpacingMs } from './videoPacing.js';
+/** Source kinds the default ranking treats as captures: things a person started sharing on purpose. */
+const CAPTURE_KINDS = new Set(['camera', 'screen']);
+/** Whether a source kind is a capture (as opposed to a channel surface). */
+export function IsVideoCaptureKind(kind) {
+    return kind !== undefined && CAPTURE_KINDS.has(kind);
+}
+/**
+ * The default selection policy, as DATA so an agent or host can reorder or trim it.
+ *
+ * Order: an explicit user pick beats everything (the user is the authority on what the agent sees);
+ * then the most recently started CAPTURE, because starting a camera or share is a deliberate "look at
+ * this" and a surface is always there to come back to; then the focused surface; then simply the newest
+ * source, so a lone unfocused surface is never invisible.
+ *
+ * "Most recently started" is applied to captures rather than to every source. Applied to everything it
+ * would always produce a winner and the focused-surface rule could never be reached.
+ */
+export const DEFAULT_VIDEO_SOURCE_POLICY = Object.freeze([
+    'user-pick',
+    'recent-capture',
+    'focused',
+    'recent',
+]);
+/**
+ * How many streams a sink can take right now. A sink that predates `MaxInboundVideoStreams` (a custom
+ * client, a test double) is read the way every model was until the field existed: one stream while an
+ * inbound video track is live, none otherwise.
+ */
+function streamCapacity(sink) {
+    if (!sink) {
+        return 0;
+    }
+    if (typeof sink.MaxInboundVideoStreams === 'number') {
+        return sink.MaxInboundVideoStreams;
+    }
+    return sink.IsTrackEstablished('video', 'inbound') ? 1 : 0;
+}
+/**
+ * The per-session arbiter. Obtain the one for a client with {@link VideoSourceArbiter.ForSink} so every
+ * writer shares it; constructing a second one for the same client would make two writers again.
+ */
+export class VideoSourceArbiter {
+    static { this.bySink = new WeakMap(); }
+    /**
+     * The arbiter for a sink, created on first use. One arbiter per sink is the whole point (a single
+     * writer), so every bridge and channel that gets its arbiter this way shares it.
+     *
+     * @param sink The client (anything satisfying {@link IVideoFrameSink}).
+     * @param options Options applied only when the arbiter is created.
+     */
+    static ForSink(sink, options) {
+        let arbiter = VideoSourceArbiter.bySink.get(sink);
+        if (!arbiter) {
+            arbiter = new VideoSourceArbiter(sink, options);
+            VideoSourceArbiter.bySink.set(sink, arbiter);
+        }
+        return arbiter;
+    }
+    constructor(sinkSource, options = {}) {
+        this.sinkSource = sinkSource;
+        this.options = options;
+        this.sources = new Map();
+        this.handlers = new Set();
+        this.nextSeq = 1;
+        this.userPick = null;
+        this.focused = null;
+        this.focusedChannel = null;
+        /** The ids forwarded as of the last reconcile — the baseline a change is detected against. */
+        this.lastActive = [];
+        /** Whether the previous reconcile had more enabled sources than streams (arbitration in effect). */
+        this.lastArbitrating = false;
+        this.policy = options.Policy ?? DEFAULT_VIDEO_SOURCE_POLICY;
+    }
+    get sink() {
+        const resolved = typeof this.sinkSource === 'function' ? this.sinkSource() : this.sinkSource;
+        return resolved ?? null;
+    }
+    /**
+     * Registers a source, or updates its label/kind if the id is already registered (its recency, enabled
+     * state and counters are kept). A newly registered source counts as the most recently STARTED.
+     */
+    RegisterSource(descriptor) {
+        if (!descriptor.SourceID) {
+            throw new Error('VideoSourceArbiter.RegisterSource requires a SourceID.');
+        }
+        const existing = this.sources.get(descriptor.SourceID);
+        if (existing) {
+            existing.Label = descriptor.Label;
+            existing.Kind = descriptor.Kind ?? existing.Kind;
+            existing.ChannelKey = descriptor.ChannelKey ?? existing.ChannelKey;
+        }
+        else {
+            this.sources.set(descriptor.SourceID, {
+                ...descriptor,
+                Kind: descriptor.Kind ?? 'surface',
+                Enabled: descriptor.Enabled ?? true,
+                Seq: this.nextSeq++,
+                FramesSent: 0,
+                LastSentAt: 0,
+            });
+        }
+        this.reconcile();
+        this.emitChange();
+    }
+    /** Removes a source. Its explicit pick or focus, if it held one, is cleared. */
+    UnregisterSource(sourceId) {
+        if (!this.sources.delete(sourceId)) {
+            return;
+        }
+        if (this.userPick === sourceId) {
+            this.userPick = null;
+        }
+        if (this.focused === sourceId) {
+            this.focused = null;
+        }
+        this.reconcile();
+        this.emitChange();
+    }
+    /** Whether a source id is registered. */
+    HasSource(sourceId) {
+        return this.sources.has(sourceId);
+    }
+    /**
+     * The user explicitly picks the source the agent sees (`null` clears the pick). A pick beats every
+     * other rule. Returns `false` for an unknown or disabled source (nothing changes).
+     */
+    SelectSource(sourceId) {
+        if (sourceId !== null) {
+            const source = this.sources.get(sourceId);
+            if (!source || !source.Enabled) {
+                return false;
+            }
+        }
+        this.userPick = sourceId;
+        this.reconcile();
+        this.emitChange();
+        return true;
+    }
+    /**
+     * Marks the CHANNEL whose surface the user is looking at (`null` clears it): every source registered for that
+     * channel counts as focused, including one that registers after this call (a surface the user is already on
+     * before its first frame). Used by the `'focused'` rule alongside {@link SetFocusedSource}.
+     */
+    SetFocusedChannel(channelKey) {
+        if (this.focusedChannel === channelKey) {
+            return;
+        }
+        this.focusedChannel = channelKey;
+        this.reconcile();
+        this.emitChange();
+    }
+    /** Marks the source whose surface the user is looking at (`null` clears it). Used by the `'focused'` rule. */
+    SetFocusedSource(sourceId) {
+        this.focused = sourceId !== null && this.sources.has(sourceId) ? sourceId : null;
+        this.reconcile();
+        this.emitChange();
+    }
+    /**
+     * Turns a source on or off. A disabled source is not a candidate and forwards nothing.
+     *
+     * @param sourceId The source.
+     * @param enabled The new state.
+     * @param notify Whether to tell the model. Default `true`. A caller that sends its own note about the
+     *   same change (a channel lowering its exposure) passes `false` so the model is told once.
+     * @returns `false` for an unknown source; `true` otherwise (including "already in that state").
+     */
+    SetSourceEnabled(sourceId, enabled, notify = true) {
+        const source = this.sources.get(sourceId);
+        if (!source) {
+            return false;
+        }
+        if (source.Enabled === enabled) {
+            return true;
+        }
+        source.Enabled = enabled;
+        if (!enabled && this.userPick === sourceId) {
+            this.userPick = null;
+        }
+        // Reconcile FIRST and quietly: the toggle has its own note, so a switch note on top of it
+        // would tell the model the same thing twice.
+        this.reconcile(true);
+        if (notify) {
+            this.sendNote(enabled ? this.formatEnabledNote(source.Label) : this.formatDisabledNote(source.Label));
+        }
+        this.emitChange();
+        return true;
+    }
+    /** A snapshot of every registered source and whether it is being forwarded, in registration order. */
+    GetSources() {
+        const active = new Set(this.computeActive());
+        return [...this.sources.values()]
+            .sort((a, b) => a.Seq - b.Seq)
+            .map((s) => ({
+            SourceID: s.SourceID,
+            Label: s.Label,
+            Kind: s.Kind,
+            ChannelKey: s.ChannelKey,
+            Enabled: s.Enabled,
+            Active: active.has(s.SourceID),
+            FramesSent: s.FramesSent,
+            LastFrameAt: s.LastFrameAt,
+            ...(this.userPick === s.SourceID ? { Picked: true } : {}),
+        }));
+    }
+    /** The ids the arbiter would forward right now, best first. */
+    GetActiveSourceIDs() {
+        return this.computeActive();
+    }
+    /**
+     * Subscribes to structural changes (a source registered, removed, enabled, disabled, picked, focused,
+     * or the forwarded set changing). Not called per frame.
+     *
+     * @returns An unsubscribe function.
+     */
+    OnChange(handler) {
+        this.handlers.add(handler);
+        return () => {
+            this.handlers.delete(handler);
+        };
+    }
+    /**
+     * A source offers a frame. It is forwarded to the model only if the source is enabled, among the
+     * sources the model should see, and not inside the negotiated pacing interval.
+     *
+     * @param sourceId The producing source (it must be registered).
+     * @param base64Image The frame, base64-encoded.
+     * @param mimeType Image MIME type. Defaults to `'image/jpeg'`.
+     * @returns `true` if the frame was sent to the model.
+     */
+    PushFrame(sourceId, base64Image, mimeType = 'image/jpeg') {
+        const source = this.sources.get(sourceId);
+        const sink = this.sink;
+        if (!source || !source.Enabled || !sink || typeof sink.SendVideoFrame !== 'function') {
+            return false;
+        }
+        if (!sink.IsTrackEstablished('video', 'inbound')) {
+            return false;
+        }
+        // The stream count can change once the track is negotiated, so the active set is re-derived
+        // here rather than trusted from the last registration.
+        this.reconcile();
+        if (!this.computeActive().includes(sourceId)) {
+            return false;
+        }
+        const now = this.now();
+        if (source.LastSentAt > 0 && now - source.LastSentAt < MinVideoFrameSpacingMs(sink.InboundVideoRate)) {
+            return false;
+        }
+        const sent = this.send(sink, base64Image, mimeType, sourceId);
+        if (sent) {
+            source.LastSentAt = now;
+            source.LastFrameAt = now;
+            source.FramesSent++;
+        }
+        return sent;
+    }
+    /** Drops every source and subscription. The arbiter stays usable (it simply has no sources). */
+    Dispose() {
+        this.sources.clear();
+        this.userPick = null;
+        this.focused = null;
+        this.focusedChannel = null;
+        this.lastActive = [];
+        this.lastArbitrating = false;
+        this.handlers.clear();
+    }
+    // ── selection ───────────────────────────────────────────────────────────
+    /** The enabled sources, ranked by policy, best first. */
+    rank() {
+        const enabled = [...this.sources.values()].filter((s) => s.Enabled);
+        const newestFirst = (list) => [...list].sort((a, b) => b.Seq - a.Seq);
+        const ranked = [];
+        const add = (list) => {
+            for (const s of list) {
+                if (!ranked.includes(s)) {
+                    ranked.push(s);
+                }
+            }
+        };
+        for (const rule of this.policy) {
+            switch (rule) {
+                case 'user-pick':
+                    add(enabled.filter((s) => s.SourceID === this.userPick));
+                    break;
+                case 'recent-capture':
+                    add(newestFirst(enabled.filter((s) => IsVideoCaptureKind(s.Kind))));
+                    break;
+                case 'focused':
+                    add(enabled.filter((s) => s.SourceID === this.focused || (this.focusedChannel !== null && s.ChannelKey === this.focusedChannel)));
+                    break;
+                case 'recent':
+                    add(newestFirst(enabled));
+                    break;
+            }
+        }
+        // A policy without the catch-all must still be total over the enabled sources' ORDER, but it is
+        // allowed to leave some out: that is what trimming the policy means.
+        return ranked;
+    }
+    /** The ids to forward: the top N of the ranking, N being the model's inbound video stream count. */
+    computeActive() {
+        const streams = streamCapacity(this.sink);
+        if (streams <= 0) {
+            return [];
+        }
+        return this.rank().slice(0, streams).map((s) => s.SourceID);
+    }
+    /**
+     * Re-derives the forwarded set and, when it changed while sources are being arbitrated, tells the model.
+     *
+     * @param quiet Update the baseline without sending a switch note (the caller sends its own).
+     */
+    reconcile(quiet = false) {
+        const sink = this.sink;
+        const streams = streamCapacity(sink);
+        const enabledCount = [...this.sources.values()].filter((s) => s.Enabled).length;
+        const arbitrating = streams > 0 && enabledCount > streams;
+        const active = this.computeActive();
+        const changed = active.length !== this.lastActive.length || active.some((id, i) => id !== this.lastActive[i]);
+        const wasArbitrating = this.lastArbitrating;
+        this.lastActive = active;
+        this.lastArbitrating = arbitrating;
+        if (!changed || quiet || !sink) {
+            return;
+        }
+        // Notes only matter when the model could be confused about what it sees: more sources than
+        // streams now, or just before. A lone source appearing or leaving is not a "switch".
+        if (!arbitrating && !wasArbitrating) {
+            return;
+        }
+        if (active.length === 0) {
+            // Nothing left to switch TO. A source the user turned off has its own note, and a session whose
+            // last source stopped has nothing the model could be confused about.
+            return;
+        }
+        const labels = active.map((id) => this.sources.get(id)?.Label ?? id);
+        this.sendNote(this.options.FormatSwitchNote ? this.options.FormatSwitchNote(labels) : VideoSourceSeenNote(labels));
+    }
+    // ── plumbing ────────────────────────────────────────────────────────────
+    formatDisabledNote(label) {
+        return this.options.FormatDisabledNote ? this.options.FormatDisabledNote(label) : VideoSourceTurnedOffNote(label);
+    }
+    formatEnabledNote(label) {
+        return this.options.FormatEnabledNote ? this.options.FormatEnabledNote(label) : VideoSourceTurnedOnNote(label);
+    }
+    now() {
+        return this.options.Now ? this.options.Now() : Date.now();
+    }
+    send(sink, base64Image, mimeType, sourceId) {
+        try {
+            return Boolean(sink.SendVideoFrame?.(base64Image, mimeType, sourceId));
+        }
+        catch (error) {
+            this.reportError('sending a video frame', error);
+            return false;
+        }
+    }
+    sendNote(note) {
+        const sink = this.sink;
+        if (!note || !sink) {
+            return;
+        }
+        try {
+            sink.SendContextNote(note);
+        }
+        catch (error) {
+            this.reportError('sending a source note', error);
+        }
+    }
+    emitChange() {
+        for (const handler of [...this.handlers]) {
+            try {
+                handler();
+            }
+            catch (error) {
+                this.reportError('notifying a change handler', error);
+            }
+        }
+    }
+    reportError(context, error) {
+        if (this.options.OnError) {
+            this.options.OnError(context, error);
+            return;
+        }
+        console.error(`[VideoSourceArbiter] Error ${context}:`, error);
+    }
+}
+//# sourceMappingURL=videoSourceArbiter.js.map

package/dist/media/videoSourceArbiter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"videoSourceArbiter.js","sourceRoot":"","sources":["../../src/media/videoSourceArbiter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,mBAAmB,EAAE,wBAAwB,EAAE,uBAAuB,EAA+B,MAAM,oBAAoB,CAAC;AACzI,OAAO,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAsCvD,uGAAuG;AACvG,MAAM,aAAa,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEzE,4EAA4E;AAC5E,MAAM,UAAU,kBAAkB,CAAC,IAAiC;IAChE,OAAO,IAAI,KAAK,SAAS,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AACzD,CAAC;AAqDD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAqC,MAAM,CAAC,MAAM,CAAC;IACvF,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,QAAQ;CACF,CAAC,CAAC;AAgCZ;;;;GAIG;AACH,SAAS,cAAc,CAAC,IAA4B;IAChD,IAAI,CAAC,IAAI,EAAE,CAAC;QACR,OAAO,CAAC,CAAC;IACb,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,sBAAsB,KAAK,QAAQ,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC,sBAAsB,CAAC;IACvC,CAAC;IACD,OAAO,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAKD;;;GAGG;AACH,MAAM,OAAO,kBAAkB;aACH,WAAM,GAAG,IAAI,OAAO,EAA8B,AAA5C,CAA6C;IAE3E;;;;;;OAMG;IACI,MAAM,CAAC,OAAO,CAAC,IAAqB,EAAE,OAAmC;QAC5E,IAAI,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,GAAG,IAAI,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAChD,kBAAkB,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;IAcD,YACqB,UAAgC,EAChC,UAAqC,EAAE;QADvC,eAAU,GAAV,UAAU,CAAsB;QAChC,YAAO,GAAP,OAAO,CAAgC;QAd3C,YAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;QAE1C,aAAQ,GAAG,IAAI,GAAG,EAAc,CAAC;QAC1C,YAAO,GAAG,CAAC,CAAC;QACZ,aAAQ,GAAkB,IAAI,CAAC;QAC/B,YAAO,GAAkB,IAAI,CAAC;QAC9B,mBAAc,GAAkB,IAAI,CAAC;QAC7C,8FAA8F;QACtF,eAAU,GAAsB,EAAE,CAAC;QAC3C,oGAAoG;QAC5F,oBAAe,GAAG,KAAK,CAAC;QAM5B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,2BAA2B,CAAC;IAChE,CAAC;IAED,IAAY,IAAI;QACZ,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;QAC7F,OAAO,QAAQ,IAAI,IAAI,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACI,cAAc,CAAC,UAAiC;QACnD,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,QAAQ,EAAE,CAAC;YACX,QAAQ,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;YAClC,QAAQ,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC;YACjD,QAAQ,CAAC,UAAU,GAAG,UAAU,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,CAAC;QACvE,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,QAAQ,EAAE;gBAClC,GAAG,UAAU;gBACb,IAAI,EAAE,UAAU,CAAC,IAAI,IAAI,SAAS;gBAClC,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,IAAI;gBACnC,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;gBACnB,UAAU,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;aAChB,CAAC,CAAC;QACP,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED,gFAAgF;IACzE,gBAAgB,CAAC,QAAgB;QACpC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO;QACX,CAAC;QACD,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC7B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED,yCAAyC;IAClC,SAAS,CAAC,QAAgB;QAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,CAAC;IAED;;;OAGG;IACI,YAAY,CAAC,QAAuB;QACvC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC7B,OAAO,KAAK,CAAC;YACjB,CAAC;QACL,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;OAIG;IACI,iBAAiB,CAAC,UAAyB;QAC9C,IAAI,IAAI,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YACrC,OAAO;QACX,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;QACjC,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED,8GAA8G;IACvG,gBAAgB,CAAC,QAAuB;QAC3C,IAAI,CAAC,OAAO,GAAG,QAAQ,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;QACjF,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,UAAU,EAAE,CAAC;IACtB,CAAC;IAED;;;;;;;;OAQG;IACI,gBAAgB,CAAC,QAAgB,EAAE,OAAgB,EAAE,MAAM,GAAG,IAAI;QACrE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACV,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,MAAM,CAAC,OAAO,KAAK,OAAO,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC;QACzB,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACzC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,0FAA0F;QAC1F,6CAA6C;QAC7C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACrB,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1G,CAAC;QACD,IAAI,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,sGAAsG;IAC/F,UAAU;QACb,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;aAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;aAC7B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACT,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC;YAC9B,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,GAAG,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC5D,CAAC,CAAC,CAAC;IACZ,CAAC;IAED,+DAA+D;IACxD,kBAAkB;QACrB,OAAO,IAAI,CAAC,aAAa,EAAE,CAAC;IAChC,CAAC;IAED;;;;;OAKG;IACI,QAAQ,CAAC,OAAmB;QAC/B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC3B,OAAO,GAAG,EAAE;YACR,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAClC,CAAC,CAAC;IACN,CAAC;IAED;;;;;;;;OAQG;IACI,SAAS,CAAC,QAAgB,EAAE,WAAmB,EAAE,QAAQ,GAAG,YAAY;QAC3E,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,cAAc,KAAK,UAAU,EAAE,CAAC;YACnF,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,4FAA4F;QAC5F,uDAAuD;QACvD,IAAI,CAAC,SAAS,EAAE,CAAC;QACjB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3C,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,IAAI,GAAG,GAAG,MAAM,CAAC,UAAU,GAAG,sBAAsB,CAAC,IAAI,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACnG,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC9D,IAAI,IAAI,EAAE,CAAC;YACP,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC;YACxB,MAAM,CAAC,WAAW,GAAG,GAAG,CAAC;YACzB,MAAM,CAAC,UAAU,EAAE,CAAC;QACxB,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,gGAAgG;IACzF,OAAO;QACV,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IAED,2EAA2E;IAE3E,yDAAyD;IACjD,IAAI;QACR,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACpE,MAAM,WAAW,GAAG,CAAC,IAAoB,EAAkB,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACtG,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,CAAC,IAAoB,EAAQ,EAAE;YACvC,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACnB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACnB,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QACF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC7B,QAAQ,IAAI,EAAE,CAAC;gBACX,KAAK,WAAW;oBACZ,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;oBACzD,MAAM;gBACV,KAAK,gBAAgB;oBACjB,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBACpE,MAAM;gBACV,KAAK,SAAS;oBACV,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,KAAK,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;oBAClI,MAAM;gBACV,KAAK,QAAQ;oBACT,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;oBAC1B,MAAM;YACd,CAAC;QACL,CAAC;QACD,gGAAgG;QAChG,qEAAqE;QACrE,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,oGAAoG;IAC5F,aAAa;QACjB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACd,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChE,CAAC;IAED;;;;OAIG;IACK,SAAS,CAAC,KAAK,GAAG,KAAK;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QAChF,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,IAAI,YAAY,GAAG,OAAO,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9G,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC;QACzB,IAAI,CAAC,eAAe,GAAG,WAAW,CAAC;QACnC,IAAI,CAAC,OAAO,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC7B,OAAO;QACX,CAAC;QACD,2FAA2F;QAC3F,qFAAqF;QACrF,IAAI,CAAC,WAAW,IAAI,CAAC,cAAc,EAAE,CAAC;YAClC,OAAO;QACX,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,gGAAgG;YAChG,qEAAqE;YACrE,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC;QACrE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC;IACvH,CAAC;IAED,2EAA2E;IAEnE,kBAAkB,CAAC,KAAa;QACpC,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC;IACtH,CAAC;IAEO,iBAAiB,CAAC,KAAa;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;IACnH,CAAC;IAEO,GAAG;QACP,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9D,CAAC;IAEO,IAAI,CAAC,IAAqB,EAAE,WAAmB,EAAE,QAAgB,EAAE,QAAgB;QACvF,IAAI,CAAC;YACD,OAAO,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,WAAW,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC3E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,WAAW,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC;QACjB,CAAC;IACL,CAAC;IAEO,QAAQ,CAAC,IAAmB;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO;QACX,CAAC;QACD,IAAI,CAAC;YACD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,WAAW,CAAC,uBAAuB,EAAE,KAAK,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IAEO,UAAU;QACd,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC;gBACD,OAAO,EAAE,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,WAAW,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YAC1D,CAAC;QACL,CAAC;IACL,CAAC;IAEO,WAAW,CAAC,OAAe,EAAE,KAAc;QAC/C,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACrC,OAAO;QACX,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,8BAA8B,OAAO,GAAG,EAAE,KAAK,CAAC,CAAC;IACnE,CAAC"}

package/dist/testing/conformanceAssertions.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @fileoverview The kit's plain assertions: no test framework. Each failure names the rule the driver broke and what
+ * was expected and seen; the runner prefixes the check's id.
+ *
+ * @module @memberjunction/ai-realtime-client/testing
+ */
+/** Thrown when a driver breaks a rule of the video conformance kit; the message names the check and the rule. */
+export declare class RealtimeVideoConformanceError extends Error {
+    constructor(message: string);
+}
+/** Fails the check with `<rule>: <detail>`. */
+export declare function Fail(rule: string, detail: string): never;
+/** Fails unless `condition` holds. `seen` says what was seen instead. */
+export declare function AssertTrue(condition: boolean, rule: string, seen: string): void;
+/** Fails unless a count is what the rule says. */
+export declare function AssertCount(actual: number, expected: number, rule: string): void;
+/** Fails unless a value (a state, a track state, a kind) is what the rule says. */
+export declare function AssertValue(actual: string | boolean | null | undefined, expected: string | boolean | null, rule: string): void;
+/** Fails unless the pieces seen are the pieces sent, in order, byte for byte. */
+export declare function AssertSameBytes(actual: readonly ArrayBuffer[], expected: readonly ArrayBuffer[], rule: string): void;
+/** Fails unless two amounts of seconds agree to the nanosecond: sums of 1/24 s are inexact in floating point. */
+export declare function AssertSeconds(actual: number, expected: number, rule: string): void;
+/** Seconds rounded to the nanosecond. */
+export declare function ToNanoseconds(seconds: number): number;
+//# sourceMappingURL=conformanceAssertions.d.ts.map

package/dist/testing/conformanceAssertions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformanceAssertions.d.ts","sourceRoot":"","sources":["../../src/testing/conformanceAssertions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,iHAAiH;AACjH,qBAAa,6BAA8B,SAAQ,KAAK;gBACxC,OAAO,EAAE,MAAM;CAI9B;AAED,+CAA+C;AAC/C,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,KAAK,CAExD;AAED,yEAAyE;AACzE,wBAAgB,UAAU,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAI/E;AAED,kDAAkD;AAClD,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAIhF;AAED,mFAAmF;AACnF,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAI9H;AAED,iFAAiF;AACjF,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,EAAE,QAAQ,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAQpH;AAED,iHAAiH;AACjH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAIlF;AAED,yCAAyC;AACzC,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAErD"}

package/dist/testing/conformanceAssertions.js

@@ -0,0 +1,64 @@
+/**
+ * @fileoverview The kit's plain assertions: no test framework. Each failure names the rule the driver broke and what
+ * was expected and seen; the runner prefixes the check's id.
+ *
+ * @module @memberjunction/ai-realtime-client/testing
+ */
+/** Thrown when a driver breaks a rule of the video conformance kit; the message names the check and the rule. */
+export class RealtimeVideoConformanceError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = 'RealtimeVideoConformanceError';
+    }
+}
+/** Fails the check with `<rule>: <detail>`. */
+export function Fail(rule, detail) {
+    throw new RealtimeVideoConformanceError(`${rule}: ${detail}`);
+}
+/** Fails unless `condition` holds. `seen` says what was seen instead. */
+export function AssertTrue(condition, rule, seen) {
+    if (!condition) {
+        Fail(rule, seen);
+    }
+}
+/** Fails unless a count is what the rule says. */
+export function AssertCount(actual, expected, rule) {
+    if (actual !== expected) {
+        Fail(rule, `expected ${expected}, got ${actual}`);
+    }
+}
+/** Fails unless a value (a state, a track state, a kind) is what the rule says. */
+export function AssertValue(actual, expected, rule) {
+    if (actual !== expected) {
+        Fail(rule, `expected ${String(expected)}, got ${String(actual)}`);
+    }
+}
+/** Fails unless the pieces seen are the pieces sent, in order, byte for byte. */
+export function AssertSameBytes(actual, expected, rule) {
+    if (actual.length !== expected.length) {
+        Fail(rule, `expected ${expected.length} pieces, got ${actual.length}`);
+    }
+    const different = expected.findIndex((piece, index) => !sameBytes(actual[index], piece));
+    if (different >= 0) {
+        Fail(rule, `piece ${different + 1} of ${expected.length} differs from what the model sent (or came out of order)`);
+    }
+}
+/** Fails unless two amounts of seconds agree to the nanosecond: sums of 1/24 s are inexact in floating point. */
+export function AssertSeconds(actual, expected, rule) {
+    if (ToNanoseconds(actual) !== ToNanoseconds(expected)) {
+        Fail(rule, `expected ${ToNanoseconds(expected)} s, got ${ToNanoseconds(actual)} s`);
+    }
+}
+/** Seconds rounded to the nanosecond. */
+export function ToNanoseconds(seconds) {
+    return Math.round(seconds * 1e9) / 1e9;
+}
+function sameBytes(a, b) {
+    if (!a || a.byteLength !== b.byteLength) {
+        return false;
+    }
+    const left = new Uint8Array(a);
+    const right = new Uint8Array(b);
+    return left.every((byte, index) => byte === right[index]);
+}
+//# sourceMappingURL=conformanceAssertions.js.map

package/dist/testing/conformanceAssertions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformanceAssertions.js","sourceRoot":"","sources":["../../src/testing/conformanceAssertions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,iHAAiH;AACjH,MAAM,OAAO,6BAA8B,SAAQ,KAAK;IACpD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAChD,CAAC;CACJ;AAED,+CAA+C;AAC/C,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,MAAc;IAC7C,MAAM,IAAI,6BAA6B,CAAC,GAAG,IAAI,KAAK,MAAM,EAAE,CAAC,CAAC;AAClE,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,UAAU,CAAC,SAAkB,EAAE,IAAY,EAAE,IAAY;IACrE,IAAI,CAAC,SAAS,EAAE,CAAC;QACb,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACrB,CAAC;AACL,CAAC;AAED,kDAAkD;AAClD,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,QAAgB,EAAE,IAAY;IACtE,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,EAAE,YAAY,QAAQ,SAAS,MAAM,EAAE,CAAC,CAAC;IACtD,CAAC;AACL,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,MAA2C,EAAE,QAAiC,EAAE,IAAY;IACpH,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtB,IAAI,CAAC,IAAI,EAAE,YAAY,MAAM,CAAC,QAAQ,CAAC,SAAS,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;AACL,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,eAAe,CAAC,MAA8B,EAAE,QAAgC,EAAE,IAAY;IAC1G,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,IAAI,CAAC,IAAI,EAAE,YAAY,QAAQ,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACzF,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;QACjB,IAAI,CAAC,IAAI,EAAE,SAAS,SAAS,GAAG,CAAC,OAAO,QAAQ,CAAC,MAAM,0DAA0D,CAAC,CAAC;IACvH,CAAC;AACL,CAAC;AAED,iHAAiH;AACjH,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,QAAgB,EAAE,IAAY;IACxE,IAAI,aAAa,CAAC,MAAM,CAAC,KAAK,aAAa,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpD,IAAI,CAAC,IAAI,EAAE,YAAY,aAAa,CAAC,QAAQ,CAAC,WAAW,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxF,CAAC;AACL,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,aAAa,CAAC,OAAe;IACzC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AAC3C,CAAC;AAED,SAAS,SAAS,CAAC,CAA0B,EAAE,CAAc;IACzD,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;QACtC,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC9D,CAAC"}

package/dist/testing/conformanceFixtures.d.ts

@@ -0,0 +1,76 @@
+/**
+ * @fileoverview The media the video conformance kit sends as the model: tiny, synthetic, deterministic video in each
+ * form a model's video comes in (Core's `RealtimeVideoFrame` kinds), and PCM chunks. Nothing here decodes; the frames
+ * only need to be read (by `@memberjunction/ai`'s fMP4 reader) and routed.
+ *
+ * - **Fragmented MP4**, as a realtime avatar sends it: an init segment (`ftyp` + `moov`) with an H.264 `avc1.42c01f` video
+ *   track at a 90 kHz timescale and an AAC-LC `mp4a.40.2` audio track at 24 kHz unless the video comes without the
+ *   voice; video fragments (`moof` + `mdat`) of one sample per 24 fps frame, so each frame is
+ *   {@link CONFORMANCE_FMP4_FRAME_SECONDS}; audio-only fragments of a muxed avatar, which carry no video seconds.
+ * - **Encoded chunks** ({@link ConformanceChunkFrames}): opaque bytes labelled H.264, a key frame every 24, presentation
+ *   times 24 to the second.
+ * - **Images** ({@link ConformanceImageFrame}): JPEG start and end markers around a few bytes, 24 to the second.
+ * - **PCM** chunks of 4 bytes, too short to ever be read as an MP4 box.
+ *
+ * Every kind is spaced 1/24 s apart, so a frame is {@link CONFORMANCE_FMP4_FRAME_SECONDS} whatever its kind. The box
+ * builders are exported for this package's own tests; the `/testing` entry exports only the fixtures.
+ *
+ * @module @memberjunction/ai-realtime-client/testing
+ */
+import { type RealtimeChunkVideoFrame, type RealtimeFmp4VideoFrame, type RealtimeImageVideoFrame } from '@memberjunction/ai';
+/** The seconds of video one frame carries, of any kind: 1/24. A {@link ConformanceFmp4VideoFragment} frame lasts this long. */
+export declare const CONFORMANCE_FMP4_FRAME_SECONDS: number;
+/** Joins byte arrays end to end. */
+export declare function ConcatBytes(...parts: Uint8Array[]): Uint8Array;
+/** A big-endian 32-bit unsigned integer. */
+export declare function Uint32Bytes(value: number): Uint8Array;
+/** The bytes of an ASCII string, such as a box type. */
+export declare function AsciiBytes(text: string): Uint8Array;
+/** An MP4 box: its size, its four-character type, then its payload. */
+export declare function Mp4Box(type: string, ...payload: Uint8Array[]): Uint8Array;
+/** A full box: version 0, no flags, then the payload. */
+export declare function Mp4FullBox(type: string, ...payload: Uint8Array[]): Uint8Array;
+/**
+ * An init segment (`ftyp` + `moov`): an H.264 video track at `videoTimescale` units a second, and an AAC-LC audio track
+ * (the avatar's voice) unless `withAudio` is false.
+ *
+ * @param withAudio Whether the video carries the voice: an audio track beside the video. Default `true`.
+ * @param videoTimescale The video track's units per second. Default 90 000, which times each frame at 1/24 s.
+ */
+export declare function ConformanceFmp4InitSegment(withAudio?: boolean, videoTimescale?: number): ArrayBuffer;
+/**
+ * A video fragment (`moof` + `mdat`) of `frames` 24 fps frames, {@link CONFORMANCE_FMP4_FRAME_SECONDS} each under the
+ * default init segment. `sequence` makes each fragment's bytes distinct.
+ */
+export declare function ConformanceFmp4VideoFragment(frames?: number, sequence?: number): ArrayBuffer;
+/** An audio-only fragment of a muxed avatar: one AAC frame and no video, so it adds no video seconds. */
+export declare function ConformanceFmp4AudioFragment(sequence?: number): ArrayBuffer;
+/**
+ * A 4-byte PCM16 chunk of the agent's voice. Too short to be read as an MP4 box, so a driver that sniffs bytes never
+ * takes it for video. `seed` makes each chunk's bytes distinct.
+ */
+export declare function ConformancePcm(seed?: number): ArrayBuffer;
+/** How to number and label {@link ConformanceChunkFrames}. */
+export interface ConformanceChunkOptions {
+    /** The first chunk's number in the stream, from 0: it sets its time, its bytes and whether it is a key frame. Default 0. */
+    FirstIndex?: number;
+    /** The chunks' type. Default `video/h264; codecs="avc1.42e01f"`. */
+    MimeType?: string;
+}
+/**
+ * `count` encoded chunks as a raw-frame provider sends them: each with distinct opaque bytes, its presentation time on a
+ * 24 fps timeline (chunk `n` at `n` × 1000/24 ms), and a key frame every 24 (chunk 0, 24, 48 …). Nothing decodes them.
+ */
+export declare function ConformanceChunkFrames(count: number, options?: ConformanceChunkOptions): RealtimeChunkVideoFrame[];
+/** Image `index` of a 24 fps stream: JPEG start and end markers around distinct bytes, timed at `index` × 1000/24 ms. */
+export declare function ConformanceImageFrame(index?: number): RealtimeImageVideoFrame;
+/**
+ * A piece of fragmented MP4 as the frame a provider's driver builds from it (Core's `Fmp4PieceToVideoFrame`), labelled
+ * `video/mp4`. For this package's own use; a harness gets the frames the kit sends.
+ *
+ * @throws When the piece is not fragmented MP4.
+ */
+export declare function Fmp4Frame(piece: ArrayBuffer): RealtimeFmp4VideoFrame;
+/** Frame `index`'s presentation time on the kit's 24 fps timeline, in ms. */
+export declare function FrameTimeMs(index: number): number;
+//# sourceMappingURL=conformanceFixtures.d.ts.map

package/dist/testing/conformanceFixtures.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformanceFixtures.d.ts","sourceRoot":"","sources":["../../src/testing/conformanceFixtures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AACH,OAAO,EAEH,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC/B,MAAM,oBAAoB,CAAC;AAW5B,+HAA+H;AAC/H,eAAO,MAAM,8BAA8B,QAAsC,CAAC;AAWlF,oCAAoC;AACpC,wBAAgB,WAAW,CAAC,GAAG,KAAK,EAAE,UAAU,EAAE,GAAG,UAAU,CAQ9D;AAED,4CAA4C;AAC5C,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAIrD;AAED,wDAAwD;AACxD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,CAEnD;AAED,uEAAuE;AACvE,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,UAAU,CAGzE;AAED,yDAAyD;AACzD,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,GAAG,UAAU,CAE7E;AAqCD;;;;;;GAMG;AACH,wBAAgB,0BAA0B,CAAC,SAAS,GAAE,OAAc,EAAE,cAAc,GAAE,MAAwB,GAAG,WAAW,CAI3H;AAED;;;GAGG;AACH,wBAAgB,4BAA4B,CAAC,MAAM,GAAE,MAAU,EAAE,QAAQ,GAAE,MAAU,GAAG,WAAW,CAGlG;AAED,yGAAyG;AACzG,wBAAgB,4BAA4B,CAAC,QAAQ,GAAE,MAAU,GAAG,WAAW,CAG9E;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,GAAE,MAAU,GAAG,WAAW,CAE5D;AAED,8DAA8D;AAC9D,MAAM,WAAW,uBAAuB;IACpC,4HAA4H;IAC5H,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,uBAA4B,GAAG,uBAAuB,EAAE,CAYtH;AAED,yHAAyH;AACzH,wBAAgB,qBAAqB,CAAC,KAAK,GAAE,MAAU,GAAG,uBAAuB,CAQhF;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,WAAW,GAAG,sBAAsB,CAMpE;AAED,6EAA6E;AAC7E,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEjD"}