@memberjunction/realtime-runtime 6.2.0-edge.3 → 6.2.0-edge.4

package/dist/session/realtime-captures.js

@@ -0,0 +1,383 @@
+/**
+ * @fileoverview The camera and the screen share of one realtime session: what the user chooses to show the agent
+ * besides the channels' own surfaces.
+ *
+ * A capture starts only when the user asks (a click in the host's UI) and in this order:
+ * 0. the session's policy must allow it ({@link RealtimeCapturesOptions.Admit}: the runtime admits a capture only while
+ *    the channel that fronts it is in the session and lets the agent see pixels);
+ * 1. the model must be able to take it: an inbound video track is already live, or one is added now
+ *    (`AddTrack`; a session never requests video before anyone shares, so it is not billed or limited for it);
+ * 2. the device or surface is opened: the camera through the host's `ILocalMediaController`, a screen, window or
+ *    tab through the host's `RequestDisplayCapture`;
+ * 3. frames are sampled at the rate the track negotiated, by the host's sampler or the DOM default, and it becomes a
+ *    source of the session's video source arbiter, which decides what the model sees and tells it when that changes.
+ *    A sampler that cannot start fails the capture as `unsupported`: the capture does not report on, and the model is
+ *    never told it can see it.
+ *
+ * With the camera check on ({@link RealtimeCapturesOptions.CameraCheck}), the first camera start waits between 2 and 3:
+ * the camera is open for the user to look at, and becomes a source only once the user confirms
+ * ({@link RealtimeCaptures.ConfirmCamera}).
+ *
+ * Stopping undoes all three. The track is removed again only when this class added it, no capture is left, and the
+ * arbiter has no other source: a video track the session was minted with (a channel's) stays, and so does one a channel
+ * started sending on while a capture was on. A camera that goes away (unplugged) or a share the user ends from the
+ * browser's own bar stops the capture the same way.
+ *
+ * @module @memberjunction/realtime-runtime
+ */
+import { BehaviorSubject } from 'rxjs';
+import { FrameSampler, VideoSourceArbiter, } from '@memberjunction/ai-realtime-client';
+/** Neither capture offered: what a host shows outside a call. */
+export const REALTIME_CAPTURE_OFFERS_NONE = Object.freeze({ Camera: false, Screen: false });
+const OFF = { Status: 'off' };
+/** What a capture gets when the session sets no policy. */
+const ADMITTED = { Admitted: true, VisibleToAgent: true };
+/** Both captures off: what a host shows outside a session. */
+export const REALTIME_CAPTURES_OFF = Object.freeze({ Camera: OFF, Screen: OFF });
+/**
+ * The ids the camera and the screen share go by on the session's video source arbiter, and so on the runtime's
+ * `VideoSources$`; prefixed so they never meet a channel's (`Key#instance`). A host or channel finds a capture's source
+ * by its id there, to tell whether the model is being sent its frames (`Enabled` and `Active`).
+ */
+export const REALTIME_CAPTURE_SOURCE_IDS = Object.freeze({
+    camera: 'capture:camera',
+    screen: 'capture:screen',
+});
+/** The names the model is told it is looking at, and the "agent can see" list shows. A shared panel goes by its own. */
+const LABELS = { camera: 'Camera', screen: 'Shared screen' };
+/** What the user is told when frames cannot be taken from a capture for the agent: its sampler could not start. */
+const CANNOT_SAMPLE = {
+    camera: 'This app cannot show the agent your camera.',
+    screen: 'This app cannot show the agent your screen.',
+};
+/** What the model and the "agent can see" list call a capture: a named shared panel by its name, as "Whiteboard (shared panel)". */
+function sourceLabel(kind, panelLabel) {
+    return panelLabel ? `${panelLabel} (shared panel)` : LABELS[kind];
+}
+/**
+ * The inbound video track a capture needs. No rate: the model's own ceiling applies. The user's click is the
+ * consent the track asks for.
+ */
+const CAPTURE_VIDEO_TRACK = {
+    Modality: 'video',
+    Direction: 'inbound',
+    Encoding: 'image/jpeg',
+    UsageBasis: ['tokens', 'frames'],
+    RequiresConsent: true,
+};
+/** The camera and the screen share of one session. Created once the session is connected; disposed at its end. */
+export class RealtimeCaptures {
+    constructor(options) {
+        this.options = options;
+        this.states = new BehaviorSubject(REALTIME_CAPTURES_OFF);
+        this.live = { camera: null, screen: null };
+        /** Bumped by every Stop, so a start that resolves after it knows it was cancelled. */
+        this.generation = { camera: 0, screen: 0 };
+        /** Whether this class added the session's inbound video track, and so may remove it. */
+        this.addedVideoTrack = false;
+        /** Whether the user has confirmed the camera check; later camera starts skip it. */
+        this.cameraChecked = false;
+        this.disposed = false;
+        /** How each capture is shown to the agent: set at its start, kept up to date by {@link SetVisibleToAgent}. */
+        this.showing = {
+            camera: { VisibleToAgent: true },
+            screen: { VisibleToAgent: true },
+        };
+    }
+    /** Both captures now. */
+    get States() {
+        return this.states.value;
+    }
+    /** Both captures, now and on every change. Completes on {@link Dispose}. */
+    get States$() {
+        return this.states.asObservable();
+    }
+    async Start(kind, options) {
+        const current = this.stateOf(kind);
+        if (this.disposed || current.Status === 'starting' || current.Status === 'on') {
+            return current;
+        }
+        const admission = this.options.Admit?.(kind) ?? ADMITTED;
+        if (!admission.Admitted) {
+            return this.fail(kind, 'policy', admission.Message);
+        }
+        this.showing[kind] = { ChannelKey: admission.ChannelKey, VisibleToAgent: admission.VisibleToAgent };
+        const generation = ++this.generation[kind];
+        this.setState(kind, { Status: 'starting' });
+        const refusal = this.ensureVideoTrack();
+        if (refusal) {
+            return this.fail(kind, 'unsupported', refusal);
+        }
+        const acquired = kind === 'camera' ? await this.openCamera(options) : await this.openScreen(options);
+        if (generation !== this.generation[kind]) {
+            // Stopped (or disposed) while opening: let go of what just opened.
+            if (acquired.Status === 'started') {
+                acquired.Release();
+            }
+            return this.stateOf(kind);
+        }
+        if (acquired.Status === 'failed') {
+            // Failed first, so this capture no longer counts as starting when the track is given back.
+            const failed = this.fail(kind, acquired.Failure, acquired.Message);
+            this.releaseVideoTrack();
+            return failed;
+        }
+        if (kind === 'camera' && this.options.CameraCheck && !this.cameraChecked) {
+            this.live.camera = { Stream: acquired.Stream, Sampler: null, Release: acquired.Release };
+            return this.setState('camera', { Status: 'starting', Checking: true, Stream: acquired.Stream, ...this.cameraDevices() });
+        }
+        const sampler = this.showToAgent(kind, acquired.Stream, sourceLabel(kind, acquired.PanelLabel));
+        if (!sampler) {
+            return this.failToShow(kind, acquired.Release);
+        }
+        this.live[kind] = { Stream: acquired.Stream, Sampler: sampler, Release: acquired.Release };
+        return this.setState(kind, {
+            Status: 'on',
+            Stream: acquired.Stream,
+            ...(acquired.Surface ? { Surface: acquired.Surface } : {}),
+            ...(acquired.PanelLabel ? { PanelLabel: acquired.PanelLabel } : {}),
+            ...(kind === 'camera' ? this.cameraDevices() : {}),
+        });
+    }
+    /**
+     * The user checked the camera and turned it on: it is shown to the agent from now on, and later camera starts skip the
+     * check. Returns the camera's state; unless the camera is waiting for its check, nothing changes. When frames cannot be
+     * taken from the camera, it fails as `unsupported` and the camera is released.
+     */
+    ConfirmCamera() {
+        const live = this.live.camera;
+        if (!live || !this.States.Camera.Checking) {
+            return this.States.Camera;
+        }
+        const sampler = this.showToAgent('camera', live.Stream, LABELS.camera);
+        if (!sampler) {
+            this.live.camera = null;
+            return this.failToShow('camera', live.Release);
+        }
+        this.cameraChecked = true;
+        live.Sampler = sampler;
+        return this.setState('camera', { Status: 'on', Stream: live.Stream, ...this.cameraDevices() });
+    }
+    /**
+     * Moves the open camera (being checked or on) to another of its {@link RealtimeCaptureState.Devices}. The stream stays
+     * the same, so the preview and the agent's frames carry on from the new camera. When the new camera cannot open, the
+     * host's controller goes back to the one in use; when that fails too, the camera stops. Resolves with the camera's
+     * state; nothing changes while the camera is not open.
+     */
+    async SwitchCamera(deviceId) {
+        const controller = this.options.LocalMedia;
+        if (this.live.camera && controller) {
+            await controller.SwitchDevice('camera', deviceId);
+        }
+        return this.States.Camera;
+    }
+    /**
+     * Lets the agent see a capture, or hides it from the agent while it keeps running for the user (the policy of the
+     * channel that fronts it changed). Applies to a capture that is still starting too. Quiet: the channel tells the model.
+     */
+    SetVisibleToAgent(kind, visible) {
+        this.showing[kind] = { ...this.showing[kind], VisibleToAgent: visible };
+        if (this.live[kind]) {
+            VideoSourceArbiter.ForSink(this.options.Client).SetSourceEnabled(REALTIME_CAPTURE_SOURCE_IDS[kind], visible, false);
+        }
+    }
+    /** Stops a capture: no more frames, out of the arbiter, the device or share released. Safe when it is off. */
+    Stop(kind) {
+        this.generation[kind]++;
+        const live = this.live[kind];
+        this.live[kind] = null;
+        if (live) {
+            live.Sampler?.Stop();
+            VideoSourceArbiter.ForSink(this.options.Client).UnregisterSource(REALTIME_CAPTURE_SOURCE_IDS[kind]);
+            live.Release();
+        }
+        if (this.stateOf(kind).Status !== 'off') {
+            this.setState(kind, OFF);
+        }
+        this.releaseVideoTrack();
+    }
+    /** Stops both captures and completes {@link States$}. */
+    Dispose() {
+        if (this.disposed) {
+            return;
+        }
+        this.Stop('camera');
+        this.Stop('screen');
+        this.disposed = true;
+        this.states.complete();
+    }
+    stateOf(kind) {
+        return kind === 'camera' ? this.states.value.Camera : this.states.value.Screen;
+    }
+    setState(kind, state) {
+        if (!this.disposed) {
+            this.states.next(kind === 'camera' ? { ...this.states.value, Camera: state } : { ...this.states.value, Screen: state });
+        }
+        return state;
+    }
+    fail(kind, failure, message) {
+        return this.setState(kind, { Status: 'failed', Failure: failure, Message: message });
+    }
+    /**
+     * Fails a capture whose device or share opened but whose frames cannot be taken for the agent (its sampler could not
+     * start): lets go of what opened, reports `unsupported`, and gives the track back.
+     */
+    failToShow(kind, release) {
+        release();
+        // Failed first, so this capture no longer counts as starting when the track is given back.
+        const failed = this.fail(kind, 'unsupported', CANNOT_SAMPLE[kind]);
+        this.releaseVideoTrack();
+        return failed;
+    }
+    /** The camera in use and the cameras to choose from, as the host's controller reports them now. */
+    cameraDevices() {
+        return cameraDevicesOf(this.options.LocalMedia?.State);
+    }
+    /**
+     * Keeps the open camera's device and list current as the controller reports them: a switch, a camera plugged in or
+     * removed, names that appear once the user allows the camera. While a switch is under way the device in use stays.
+     */
+    followCamera(media) {
+        const current = this.States.Camera;
+        const next = cameraDevicesOf(media, current.DeviceID);
+        if (next.DeviceID !== current.DeviceID || !sameDevices(next.Devices, current.Devices ?? [])) {
+            this.setState('camera', { ...current, ...next });
+        }
+    }
+    /**
+     * Makes sure an inbound video track is live: the one the session already has, or one added now.
+     *
+     * @returns Why the model cannot take video, or `null` when it can.
+     */
+    ensureVideoTrack() {
+        const client = this.options.Client;
+        if (client.IsTrackEstablished('video', 'inbound')) {
+            return null;
+        }
+        const track = client.AddTrack(CAPTURE_VIDEO_TRACK);
+        if (!track || track.State !== 'live') {
+            return track?.Reason ?? 'This model cannot see video.';
+        }
+        this.addedVideoTrack = true;
+        return null;
+    }
+    /**
+     * Removes the video track this class added, once no capture is on or starting and the arbiter has no other source.
+     * A channel that started sending on the track while a capture was on (one opened mid-call before it had a track of its
+     * own) is such a source: removing the track would leave it sending nothing, right after the model was told it can see
+     * that channel. The track then stays for the session, as a channel's own does.
+     */
+    releaseVideoTrack() {
+        const busy = [this.states.value.Camera, this.states.value.Screen].some((s) => s.Status === 'on' || s.Status === 'starting');
+        if (!this.addedVideoTrack || busy || VideoSourceArbiter.ForSink(this.options.Client).GetSources().length > 0) {
+            return;
+        }
+        this.addedVideoTrack = false;
+        this.options.Client.RemoveTrack(CAPTURE_VIDEO_TRACK);
+    }
+    /**
+     * Samples the capture at the negotiated rate and registers it with the arbiter under `label`, as its channel's and as
+     * visible as the policy allows. The sampler starts first: one that cannot start (no DOM, and no sampler from the host)
+     * leaves nothing registered, so the model is never told it can see a capture that would send it nothing.
+     *
+     * @returns The running sampler, or `null` when sampling could not start.
+     */
+    showToAgent(kind, stream, label) {
+        const arbiter = VideoSourceArbiter.ForSink(this.options.Client);
+        const sourceId = REALTIME_CAPTURE_SOURCE_IDS[kind];
+        const create = this.options.CreateSampler ?? createDomSampler;
+        const sampler = create(stream, {
+            Rate: this.options.Client.InboundVideoRate ?? 1,
+            OnFrame: (frame) => arbiter.PushFrame(sourceId, frame.Data, frame.MimeType),
+        });
+        if (!sampler.Start()) {
+            sampler.Stop();
+            return null;
+        }
+        const { ChannelKey, VisibleToAgent } = this.showing[kind];
+        arbiter.RegisterSource({ SourceID: sourceId, Label: label, Kind: kind, Enabled: VisibleToAgent, ...(ChannelKey ? { ChannelKey } : {}) });
+        return sampler;
+    }
+    /** Opens the camera through the host's controller, and stops the capture if the camera goes away. */
+    async openCamera(options) {
+        const controller = this.options.LocalMedia;
+        if (!controller) {
+            return { Status: 'failed', Failure: 'unsupported', Message: 'This app cannot open a camera.' };
+        }
+        const result = await controller.Start('camera', options?.DeviceID);
+        if (result.Status === 'failed') {
+            return { Status: 'failed', Failure: result.Reason, Message: result.Message };
+        }
+        const watch = controller.State$.subscribe((state) => {
+            if (state.Camera.Status === 'off' || state.Camera.Status === 'failed') {
+                this.Stop('camera');
+            }
+            else if (this.live.camera) {
+                this.followCamera(state);
+            }
+        });
+        return {
+            Status: 'started',
+            Stream: result.Stream,
+            Release: () => {
+                watch.unsubscribe();
+                controller.Stop('camera');
+            },
+        };
+    }
+    /** Asks the host for a screen, window, tab or panel, and stops the capture when the share ends. */
+    async openScreen(options) {
+        const host = this.options.Host;
+        if (!host.RequestDisplayCapture) {
+            return { Status: 'failed', Failure: 'unsupported', Message: 'This app cannot share a screen.' };
+        }
+        const result = await host.RequestDisplayCapture(options);
+        if (result.Status === 'cancelled') {
+            return { Status: 'failed', Failure: 'cancelled', Message: 'Sharing was cancelled.' };
+        }
+        if (result.Status === 'failed') {
+            return { Status: 'failed', Failure: screenFailure(result.Reason), Message: result.Message };
+        }
+        const capture = result.Capture;
+        const stopWatching = capture.OnEnded(() => this.Stop('screen'));
+        return {
+            Status: 'started',
+            Stream: capture.Stream,
+            Surface: capture.Surface,
+            ...(capture.PanelLabel ? { PanelLabel: capture.PanelLabel } : {}),
+            Release: () => {
+                stopWatching();
+                capture.Stop();
+            },
+        };
+    }
+}
+/**
+ * The camera in use and the cameras among the devices, as a controller reports them. The camera in use is known while
+ * the controller reports it on; otherwise it is `inUse`, the one the capture already names.
+ */
+function cameraDevicesOf(media, inUse) {
+    const deviceId = (media?.Camera.Status === 'on' ? media.Camera.DeviceID : undefined) ?? inUse;
+    return { ...(deviceId ? { DeviceID: deviceId } : {}), Devices: media?.Devices.filter((d) => d.Kind === 'camera') ?? [] };
+}
+/** Whether two lists name the same devices in the same order. */
+function sameDevices(a, b) {
+    return a.length === b.length && a.every((d, i) => d.DeviceID === b[i].DeviceID && d.Label === b[i].Label);
+}
+/** The default sampler: a DOM {@link FrameSampler}, whose `Start()` returns `false` without a DOM. */
+function createDomSampler(stream, options) {
+    return new FrameSampler(stream, options);
+}
+/** A share failure as a capture failure: a panel the browser cannot narrow to is unsupported; a wrong one an error. */
+function screenFailure(reason) {
+    switch (reason) {
+        case 'unsupported':
+        case 'panel-unsupported':
+            return 'unsupported';
+        case 'denied':
+            return 'denied';
+        default:
+            return 'error';
+    }
+}
+//# sourceMappingURL=realtime-captures.js.map

package/dist/session/realtime-captures.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-captures.js","sourceRoot":"","sources":["../../src/session/realtime-captures.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EAAE,eAAe,EAAsC,MAAM,MAAM,CAAC;AAE3E,OAAO,EACH,YAAY,EACZ,kBAAkB,GASrB,MAAM,oCAAoC,CAAC;AAsE5C,iEAAiE;AACjE,MAAM,CAAC,MAAM,4BAA4B,GAA0B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;AA2CnH,MAAM,GAAG,GAAyB,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAEpD,2DAA2D;AAC3D,MAAM,QAAQ,GAA6B,EAAE,QAAQ,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAEpF,8DAA8D;AAC9D,MAAM,CAAC,MAAM,qBAAqB,GAA0B,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;AAExG;;;;GAIG;AACH,MAAM,CAAC,MAAM,2BAA2B,GAAkD,MAAM,CAAC,MAAM,CAAC;IACpG,MAAM,EAAE,gBAAgB;IACxB,MAAM,EAAE,gBAAgB;CAC3B,CAAC,CAAC;AAEH,wHAAwH;AACxH,MAAM,MAAM,GAAwC,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;AAElG,mHAAmH;AACnH,MAAM,aAAa,GAAwC;IACvD,MAAM,EAAE,6CAA6C;IACrD,MAAM,EAAE,6CAA6C;CACxD,CAAC;AAEF,oIAAoI;AACpI,SAAS,WAAW,CAAC,IAAyB,EAAE,UAA8B;IAC1E,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACtE,CAAC;AAED;;;GAGG;AACH,MAAM,mBAAmB,GAA4B;IACjD,QAAQ,EAAE,OAAO;IACjB,SAAS,EAAE,SAAS;IACpB,QAAQ,EAAE,YAAY;IACtB,UAAU,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAChC,eAAe,EAAE,IAAI;CACxB,CAAC;AAeF,kHAAkH;AAClH,MAAM,OAAO,gBAAgB;IAgBzB,YAA6B,OAAgC;QAAhC,YAAO,GAAP,OAAO,CAAyB;QAf5C,WAAM,GAAG,IAAI,eAAe,CAAwB,qBAAqB,CAAC,CAAC;QAC3E,SAAI,GAAoD,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QACxG,sFAAsF;QACrE,eAAU,GAAwC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAC5F,wFAAwF;QAChF,oBAAe,GAAG,KAAK,CAAC;QAChC,oFAAoF;QAC5E,kBAAa,GAAG,KAAK,CAAC;QACtB,aAAQ,GAAG,KAAK,CAAC;QACzB,8GAA8G;QAC7F,YAAO,GAAkF;YACtG,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;YAChC,MAAM,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE;SACnC,CAAC;IAE8D,CAAC;IAEjE,yBAAyB;IACzB,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,4EAA4E;IAC5E,IAAW,OAAO;QACd,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;IACtC,CAAC;IAYM,KAAK,CAAC,KAAK,CAAC,IAAyB,EAAE,OAAuD;QACjG,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,IAAI,OAAO,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;YAC5E,OAAO,OAAO,CAAC;QACnB,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;QACzD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,SAAS,CAAC,UAAU,EAAE,cAAc,EAAE,SAAS,CAAC,cAAc,EAAE,CAAC;QACpG,MAAM,UAAU,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxC,IAAI,OAAO,EAAE,CAAC;YACV,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAA4C,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAA4C,CAAC,CAAC;QAC/K,IAAI,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,mEAAmE;YACnE,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,QAAQ,CAAC,OAAO,EAAE,CAAC;YACvB,CAAC;YACD,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC/B,2FAA2F;YAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,OAAO,MAAM,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACvE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;YACzF,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;QAC7H,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC;QAChG,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;QAC3F,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;YACvB,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrD,CAAC,CAAC;IACP,CAAC;IAED;;;;OAIG;IACI,aAAa;QAChB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YACxC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC9B,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACvE,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACxB,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;IACnG,CAAC;IAED;;;;;OAKG;IACI,KAAK,CAAC,YAAY,CAAC,QAAgB;QACtC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3C,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM,UAAU,CAAC,YAAY,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED;;;OAGG;IACI,iBAAiB,CAAC,IAAyB,EAAE,OAAgB;QAChE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,OAAO,EAAE,CAAC;QACxE,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAClB,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;QACxH,CAAC;IACL,CAAC;IAED,8GAA8G;IACvG,IAAI,CAAC,IAAyB;QACjC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QACvB,IAAI,IAAI,EAAE,CAAC;YACP,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC;YACrB,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,gBAAgB,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;YACpG,IAAI,CAAC,OAAO,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;YACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC7B,CAAC;IAED,yDAAyD;IAClD,OAAO;QACV,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;IAC3B,CAAC;IAEO,OAAO,CAAC,IAAyB;QACrC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC;IACnF,CAAC;IAEO,QAAQ,CAAC,IAAyB,EAAE,KAA2B;QACnE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACjB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5H,CAAC;QACD,OAAO,KAAK,CAAC;IACjB,CAAC;IAEO,IAAI,CAAC,IAAyB,EAAE,OAA+B,EAAE,OAAe;QACpF,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC;IACzF,CAAC;IAED;;;OAGG;IACK,UAAU,CAAC,IAAyB,EAAE,OAAmB;QAC7D,OAAO,EAAE,CAAC;QACV,2FAA2F;QAC3F,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnE,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,mGAAmG;IAC3F,aAAa;QACjB,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;IAC3D,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,KAAsB;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QACnC,MAAM,IAAI,GAAG,eAAe,CAAC,KAAK,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1F,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,GAAG,OAAO,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;QACrD,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,gBAAgB;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;QACnC,IAAI,MAAM,CAAC,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;QACnD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YACnC,OAAO,KAAK,EAAE,MAAM,IAAI,8BAA8B,CAAC;QAC3D,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;;;OAKG;IACK,iBAAiB;QACrB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;QAC5H,IAAI,CAAC,IAAI,CAAC,eAAe,IAAI,IAAI,IAAI,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3G,OAAO;QACX,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;IACzD,CAAC;IAED;;;;;;OAMG;IACK,WAAW,CAAC,IAAyB,EAAE,MAAmB,EAAE,KAAa;QAC7E,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,2BAA2B,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,gBAAgB,CAAC;QAC9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE;YAC3B,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,IAAI,CAAC;YAC/C,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;SAC9E,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QAChB,CAAC;QACD,MAAM,EAAE,UAAU,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1D,OAAO,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACzI,OAAO,OAAO,CAAC;IACnB,CAAC;IAED,qGAAqG;IAC7F,KAAK,CAAC,UAAU,CAAC,OAA0C;QAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC;QAC3C,IAAI,CAAC,UAAU,EAAE,CAAC;YACd,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC;QACnG,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACnE,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QACjF,CAAC;QACD,MAAM,KAAK,GAAiB,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE;YAC9D,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;gBACpE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC1B,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;QACL,CAAC,CAAC,CAAC;QACH,OAAO;YACH,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,OAAO,EAAE,GAAG,EAAE;gBACV,KAAK,CAAC,WAAW,EAAE,CAAC;gBACpB,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC9B,CAAC;SACJ,CAAC;IACN,CAAC;IAED,mGAAmG;IAC3F,KAAK,CAAC,UAAU,CAAC,OAA0C;QAC/D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC9B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,iCAAiC,EAAE,CAAC;QACpG,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QACzD,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,wBAAwB,EAAE,CAAC;QACzF,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;QAChG,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,OAAO;YACH,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjE,OAAO,EAAE,GAAG,EAAE;gBACV,YAAY,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,EAAE,CAAC;YACnB,CAAC;SACJ,CAAC;IACN,CAAC;CACJ;AAKD;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAkC,EAAE,KAAc;IACvE,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC;IAC9F,OAAO,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;AAC7H,CAAC;AAED,iEAAiE;AACjE,SAAS,WAAW,CAAC,CAAyB,EAAE,CAAyB;IACrE,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAC9G,CAAC;AAED,sGAAsG;AACtG,SAAS,gBAAgB,CAAC,MAAmB,EAAE,OAA4B;IACvE,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC;AAED,uHAAuH;AACvH,SAAS,aAAa,CAAC,MAA6B;IAChD,QAAQ,MAAM,EAAE,CAAC;QACb,KAAK,aAAa,CAAC;QACnB,KAAK,mBAAmB;YACpB,OAAO,aAAa,CAAC;QACzB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB;YACI,OAAO,OAAO,CAAC;IACvB,CAAC;AACL,CAAC"}

package/dist/session/realtime-microphone.d.ts

@@ -0,0 +1,50 @@
+/**
+ * @fileoverview The microphone of one realtime session, as a host's device menu shows it: the one the call uses, the
+ * ones the user can move it to (`RealtimeSessionRuntime.SwitchMicrophone`), and why none works once the call lost it.
+ *
+ * The call opens the microphone through the host's camera-and-microphone controller, and the controller reports the
+ * device in use and the devices the browser lists. This module reads the microphone's part of that report. It shows
+ * nothing outside a call and nothing on a host without a controller, which opens the microphone itself
+ * (`IRealtimeMediaHost.AcquireMicrophone`) and so has no list to offer.
+ *
+ * @module @memberjunction/realtime-runtime
+ */
+import type { LocalMediaFailure, LocalMediaState, MediaDevice } from '@memberjunction/ai-realtime-client';
+/** The call's microphone: the one in use, the ones the user can move it to, and why none works once the call lost it. */
+export interface RealtimeMicrophoneState {
+    /**
+     * The microphone in use. Kept while a switch is under way, until the new one is on. Absent outside a call, on a host
+     * without a controller, and when the controller has no microphone open (it stopped, or no device could open).
+     */
+    DeviceID?: string;
+    /**
+     * The microphones the browser lists while the call is connected: what `SwitchMicrophone` can move to. The call has
+     * opened the microphone, so the browser gives their names. Empty outside a call and on a host without a controller.
+     */
+    Devices: readonly MediaDevice[];
+    /**
+     * Why no microphone works: the call lost its microphone (the one in use failed and so did the fallback to the default,
+     * or the last one went away), so the agent hears nothing until the user picks one with `SwitchMicrophone`. Kept while
+     * the picked one opens, and gone once a microphone is on. Absent while the microphone works or is being switched,
+     * outside a call, and on a host without a controller.
+     */
+    Failure?: LocalMediaFailure;
+}
+/** No microphone to show or switch: what a host gets outside a call and on a host without a controller. */
+export declare const REALTIME_MICROPHONE_NONE: RealtimeMicrophoneState;
+/**
+ * The call's microphone as a controller reports it. The device in use is the controller's while its microphone is on.
+ * While the controller is starting a microphone (a switch, a lost device replaced by the default, or a pick after the call
+ * lost its microphone), the call keeps what it shows now: the microphone in use, so a device menu keeps showing it until
+ * the new one is on, or why none works. Failed, there is none, and the failure says why. Stopped, there is none.
+ *
+ * @param media The controller's state.
+ * @param current What the call shows now.
+ */
+export declare function ReadMicrophoneState(media: LocalMediaState, current?: RealtimeMicrophoneState): RealtimeMicrophoneState;
+/**
+ * Whether two states name the same microphone in use, list the same microphones (by id and name, in the same order) and
+ * give the same failure.
+ */
+export declare function SameMicrophoneState(a: RealtimeMicrophoneState, b: RealtimeMicrophoneState): boolean;
+//# sourceMappingURL=realtime-microphone.d.ts.map

package/dist/session/realtime-microphone.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-microphone.d.ts","sourceRoot":"","sources":["../../src/session/realtime-microphone.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AAE1G,yHAAyH;AACzH,MAAM,WAAW,uBAAuB;IACpC;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,OAAO,EAAE,SAAS,WAAW,EAAE,CAAC;IAChC;;;;;OAKG;IACH,OAAO,CAAC,EAAE,iBAAiB,CAAC;CAC/B;AAED,2GAA2G;AAC3G,eAAO,MAAM,wBAAwB,EAAE,uBAAuE,CAAC;AAE/G;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,GAAE,uBAAkD,GAAG,uBAAuB,CAiBhJ;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,uBAAuB,EAAE,CAAC,EAAE,uBAAuB,GAAG,OAAO,CAOnG"}

package/dist/session/realtime-microphone.js

@@ -0,0 +1,51 @@
+/**
+ * @fileoverview The microphone of one realtime session, as a host's device menu shows it: the one the call uses, the
+ * ones the user can move it to (`RealtimeSessionRuntime.SwitchMicrophone`), and why none works once the call lost it.
+ *
+ * The call opens the microphone through the host's camera-and-microphone controller, and the controller reports the
+ * device in use and the devices the browser lists. This module reads the microphone's part of that report. It shows
+ * nothing outside a call and nothing on a host without a controller, which opens the microphone itself
+ * (`IRealtimeMediaHost.AcquireMicrophone`) and so has no list to offer.
+ *
+ * @module @memberjunction/realtime-runtime
+ */
+/** No microphone to show or switch: what a host gets outside a call and on a host without a controller. */
+export const REALTIME_MICROPHONE_NONE = Object.freeze({ Devices: Object.freeze([]) });
+/**
+ * The call's microphone as a controller reports it. The device in use is the controller's while its microphone is on.
+ * While the controller is starting a microphone (a switch, a lost device replaced by the default, or a pick after the call
+ * lost its microphone), the call keeps what it shows now: the microphone in use, so a device menu keeps showing it until
+ * the new one is on, or why none works. Failed, there is none, and the failure says why. Stopped, there is none.
+ *
+ * @param media The controller's state.
+ * @param current What the call shows now.
+ */
+export function ReadMicrophoneState(media, current = REALTIME_MICROPHONE_NONE) {
+    const microphone = media.Microphone;
+    const devices = media.Devices.filter((d) => d.Kind === 'microphone');
+    switch (microphone.Status) {
+        case 'on':
+            return { ...(microphone.DeviceID ? { DeviceID: microphone.DeviceID } : {}), Devices: devices };
+        case 'starting':
+            return {
+                ...(current.DeviceID ? { DeviceID: current.DeviceID } : {}),
+                ...(current.Failure ? { Failure: current.Failure } : {}),
+                Devices: devices,
+            };
+        case 'failed':
+            return { Failure: microphone.Failure ?? 'error', Devices: devices };
+        default:
+            return { Devices: devices };
+    }
+}
+/**
+ * Whether two states name the same microphone in use, list the same microphones (by id and name, in the same order) and
+ * give the same failure.
+ */
+export function SameMicrophoneState(a, b) {
+    return (a.DeviceID === b.DeviceID &&
+        a.Failure === b.Failure &&
+        a.Devices.length === b.Devices.length &&
+        a.Devices.every((d, i) => d.DeviceID === b.Devices[i].DeviceID && d.Label === b.Devices[i].Label));
+}
+//# sourceMappingURL=realtime-microphone.js.map

package/dist/session/realtime-microphone.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"realtime-microphone.js","sourceRoot":"","sources":["../../src/session/realtime-microphone.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAyBH,2GAA2G;AAC3G,MAAM,CAAC,MAAM,wBAAwB,GAA4B,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAE/G;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAsB,EAAE,UAAmC,wBAAwB;IACnH,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAC,CAAC;IACrE,QAAQ,UAAU,CAAC,MAAM,EAAE,CAAC;QACxB,KAAK,IAAI;YACL,OAAO,EAAE,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACnG,KAAK,UAAU;YACX,OAAO;gBACH,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC3D,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,OAAO,EAAE,OAAO;aACnB,CAAC;QACN,KAAK,QAAQ;YACT,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACxE;YACI,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IACpC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,CAA0B,EAAE,CAA0B;IACtF,OAAO,CACH,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ;QACzB,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;QACvB,CAAC,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM;QACrC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CACpG,CAAC;AACN,CAAC"}

package/dist/session/session-event-hub.d.ts

@@ -0,0 +1,72 @@
+import type { Observable } from 'rxjs';
+import type { RealtimeSessionEvent, UnknownRealtimeSessionEvent } from '@memberjunction/ai-core-plus';
+/** An event a session delivers: a known, typed one, or an app-defined one the runtime does not interpret. */
+export type RealtimeSessionStreamEvent = RealtimeSessionEvent | UnknownRealtimeSessionEvent;
+/** The slice of a verification status read the hub needs to recover a missed `identity.verified`. */
+export interface RealtimeSessionVerificationSnapshot {
+    VerificationState: 'unverified' | 'pending' | 'verified';
+    VerifiedEmail?: string;
+    VerifiedAt?: string;
+    MaxSessionDeadlineIso?: string;
+}
+/**
+ * Where session events come from — the transport seam. `GraphQLRealtimeSessionClient`
+ * (`@memberjunction/graphql-dataprovider`) satisfies this structurally; a host on another transport
+ * supplies its own.
+ */
+export interface IRealtimeSessionEventSource {
+    /** Opens the session's event stream. It completes when the transport recycles (e.g. a token refresh). */
+    SubscribeToSessionEvents(agentSessionId: string): Observable<RealtimeSessionStreamEvent>;
+    /** Reads the session's verification state once — the durable backstop for an event missed while away. Never throws. */
+    GetVerificationStatus(agentSessionId: string): Promise<RealtimeSessionVerificationSnapshot>;
+}
+/** Tuning for {@link RealtimeSessionEventHub}'s reconnect loop. */
+export interface RealtimeSessionEventHubOptions {
+    /** Consecutive failed/short-lived subscriptions tolerated before the hub gives up. Default 6. */
+    MaxReconnectAttempts?: number;
+    /** First reconnect delay; doubles per consecutive attempt up to {@link MaxBackoffMs}. Default 1000. */
+    BaseBackoffMs?: number;
+    /** Ceiling of the reconnect delay. Default 15000. */
+    MaxBackoffMs?: number;
+    /** A subscription that lived at least this long counts as healthy and resets the attempt counter. Default 30000. */
+    StableAfterMs?: number;
+}
+/**
+ * Keeps one session's event stream open for the session's life and delivers every event to a sink.
+ *
+ * The stream is a long-lived subscription that ends when the transport recycles its connection (token
+ * refresh, network blip) — and delivery has no replay, so an event published during the gap is gone.
+ * The hub therefore, on every re-subscribe: opens the new subscription FIRST (so nothing published
+ * from now on is missed), THEN reads the verification status once, and if the person verified while the
+ * hub was away, delivers the `identity.verified` event it missed (marked `Recovered`).
+ *
+ * Reconnects are capped: a stream that keeps failing is given up on (once, loudly) rather than retried
+ * forever; a stream that stayed up long enough counts as healthy and resets the cap.
+ *
+ * Framework-agnostic and clock-driven by `setTimeout`, so tests drive it with fake timers.
+ */
+export declare class RealtimeSessionEventHub {
+    private readonly source;
+    private readonly deliver;
+    private readonly options;
+    private sessionId;
+    private subscription;
+    private reconnectTimer;
+    private attempts;
+    private verifiedDelivered;
+    /** Bumped by every Start/Stop so a late callback from a previous run can tell it is stale. */
+    private generation;
+    constructor(source: IRealtimeSessionEventSource, deliver: (event: RealtimeSessionStreamEvent) => void, options?: RealtimeSessionEventHubOptions);
+    /** Whether the hub currently has (or is about to re-open) a subscription. */
+    get IsRunning(): boolean;
+    /** Opens the stream for `agentSessionId`. Replaces any run in progress. */
+    Start(agentSessionId: string): void;
+    /** Closes the stream and cancels any pending reconnect. Safe to call when not running. */
+    Stop(): void;
+    private connect;
+    private onEvent;
+    private onStreamEnded;
+    /** Reads the verification status once and delivers the `identity.verified` event the hub missed, if any. */
+    private recoverMissedVerification;
+}
+//# sourceMappingURL=session-event-hub.d.ts.map

package/dist/session/session-event-hub.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-event-hub.d.ts","sourceRoot":"","sources":["../../src/session/session-event-hub.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAgB,MAAM,MAAM,CAAC;AACrD,OAAO,KAAK,EAAE,oBAAoB,EAAE,2BAA2B,EAAE,MAAM,8BAA8B,CAAC;AAEtG,6GAA6G;AAC7G,MAAM,MAAM,0BAA0B,GAAG,oBAAoB,GAAG,2BAA2B,CAAC;AAE5F,qGAAqG;AACrG,MAAM,WAAW,mCAAmC;IAClD,iBAAiB,EAAE,YAAY,GAAG,SAAS,GAAG,UAAU,CAAC;IACzD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;GAIG;AACH,MAAM,WAAW,2BAA2B;IAC1C,yGAAyG;IACzG,wBAAwB,CAAC,cAAc,EAAE,MAAM,GAAG,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACzF,uHAAuH;IACvH,qBAAqB,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,mCAAmC,CAAC,CAAC;CAC7F;AAED,mEAAmE;AACnE,MAAM,WAAW,8BAA8B;IAC7C,iGAAiG;IACjG,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,uGAAuG;IACvG,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oHAAoH;IACpH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AASD;;;;;;;;;;;;;GAaG;AACH,qBAAa,uBAAuB;IAWhC,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAX1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;IACnE,OAAO,CAAC,SAAS,CAAuB;IACxC,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,cAAc,CAA8C;IACpE,OAAO,CAAC,QAAQ,CAAK;IACrB,OAAO,CAAC,iBAAiB,CAAS;IAClC,8FAA8F;IAC9F,OAAO,CAAC,UAAU,CAAK;gBAGJ,MAAM,EAAE,2BAA2B,EACnC,OAAO,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,EACrE,OAAO,GAAE,8BAAmC;IAK9C,6EAA6E;IAC7E,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED,2EAA2E;IACpE,KAAK,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI;IAQ1C,0FAA0F;IACnF,IAAI,IAAI,IAAI;IAWnB,OAAO,CAAC,OAAO;IA2Bf,OAAO,CAAC,OAAO;IAUf,OAAO,CAAC,aAAa;IA0BrB,4GAA4G;YAC9F,yBAAyB;CAgCxC"}