@memberjunction/ai-realtime-client 6.2.0-edge.2 → 6.2.0-edge.4

package/dist/media/displayCapture.js

@@ -0,0 +1,245 @@
+/**
+ * @fileoverview DISPLAY CAPTURE: asks the browser to share a screen, a window, a tab or one panel of this
+ * page, and reports what the user actually shared.
+ *
+ * Sharing works wherever `getDisplayMedia` exists: desktop Chrome, Edge, Firefox and Safari (no mobile browser
+ * has it). The picker hints sent with it (`displaySurface`, `selfBrowserSurface`, `surfaceSwitching`,
+ * `preferCurrentTab`, `monitorTypeSurfaces`, `systemAudio`) are read only by Chromium browsers; others ignore
+ * them.
+ *
+ * A PANEL capture shares one element of this page. The user picks this tab in the picker, and the stream is
+ * then narrowed to the element with Element Capture (`restrictTo`, Chromium 132+: the element alone, without
+ * anything drawn over it) or, failing that, Region Capture (`cropTo`, Chromium 104+: the element's rectangle,
+ * including anything over it). Element Capture emits frames only while the element is its own stacking
+ * context, so the element gets `isolation: isolate` for the share and its own value back when the share ends.
+ * Firefox and Safari have neither; {@link GetDisplayCaptureSupport} says so, and the caller shows the panel to
+ * the agent another way (its channel frames). If the user shares something other than this tab, the capture is
+ * stopped rather than sharing more than the panel.
+ *
+ * The browser's picker is the user's consent: it opens on every request, with no remembered grant. Like the
+ * picker itself, {@link RequestDisplayCapture} must be called from a user gesture (a click handler).
+ *
+ * Capture runs at the surface's native frame rate; pace what the model receives with a `FrameSampler`.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** The picker's `displaySurface` value for each surface. */
+const PICKER_SURFACE = { screen: 'monitor', window: 'window', tab: 'browser' };
+/** What the user is told when a panel share got something other than this tab (without the closing period). */
+const WRONG_SURFACE_MESSAGE = "To share only the panel, choose this tab in the browser's picker";
+/** What this browser can capture. */
+export function GetDisplayCaptureSupport() {
+    const scope = globalThis;
+    return {
+        Display: typeof navigator !== 'undefined' && typeof navigator.mediaDevices?.getDisplayMedia === 'function',
+        ElementCapture: typeof scope.RestrictionTarget?.fromElement === 'function',
+        RegionCapture: typeof scope.CropTarget?.fromElement === 'function',
+    };
+}
+/**
+ * Opens the browser's share picker and starts sharing what the user picks. Call it from a user gesture.
+ *
+ * @param options What to offer, whether to share only a panel, and whether to include audio.
+ * @returns `started` with the live capture, `cancelled` when the user closed the picker, or `failed` with a reason.
+ */
+export async function RequestDisplayCapture(options = {}) {
+    const support = GetDisplayCaptureSupport();
+    if (!support.Display) {
+        return failure('unsupported', 'This browser cannot share a screen.');
+    }
+    if (options.Panel && !support.ElementCapture && !support.RegionCapture) {
+        return failure('panel-unsupported', 'This browser cannot share a single panel; show it through its channel instead.');
+    }
+    let stream;
+    try {
+        stream = await navigator.mediaDevices.getDisplayMedia(pickerOptions(options));
+    }
+    catch (err) {
+        return pickerFailure(err);
+    }
+    const track = stream.getVideoTracks()[0];
+    if (!track) {
+        stopTracks(stream);
+        return failure('error', 'The shared stream has no video track.');
+    }
+    if (!options.Panel) {
+        return { Status: 'started', Capture: new ActiveDisplayCapture(stream, track) };
+    }
+    const narrowed = await narrowToPanel(track, options.Panel, support);
+    if ('Status' in narrowed) {
+        stopTracks(stream);
+        return narrowed;
+    }
+    const panel = options.PanelLabel ? { ...narrowed, Label: options.PanelLabel } : narrowed;
+    return { Status: 'started', Capture: new ActiveDisplayCapture(stream, track, panel) };
+}
+/** The picker request: this tab for a panel; otherwise any surface but this tab, with the preferred kind first. */
+function pickerOptions(options) {
+    const audio = options.IncludeAudio ?? false;
+    if (options.Panel) {
+        return {
+            video: { displaySurface: PICKER_SURFACE.tab },
+            audio,
+            preferCurrentTab: true,
+            selfBrowserSurface: 'include',
+            // Switching to another surface would leave the panel narrowing behind.
+            surfaceSwitching: 'exclude',
+        };
+    }
+    return {
+        video: options.PreferredSurface ? { displaySurface: PICKER_SURFACE[options.PreferredSurface] } : true,
+        audio,
+        // Sharing this page would show the agent its own call UI, mirrored without end.
+        selfBrowserSurface: 'exclude',
+        surfaceSwitching: 'include',
+        monitorTypeSurfaces: 'include',
+        systemAudio: audio ? 'include' : 'exclude',
+    };
+}
+/**
+ * Narrows a self-capture to the panel: with Element Capture where the browser has it, else with Region Capture.
+ *
+ * @returns How it was narrowed, or a failure when the user shared something other than this tab.
+ */
+async function narrowToPanel(track, panel, support) {
+    if (track.getSettings().displaySurface !== PICKER_SURFACE.tab) {
+        return failure('panel-wrong-surface', `${WRONG_SURFACE_MESSAGE}.`);
+    }
+    const scope = globalThis;
+    const narrowable = track;
+    if (support.ElementCapture && scope.RestrictionTarget && narrowable.restrictTo) {
+        // Isolated first, so the restricted track has frames to emit from the start.
+        const release = isolatePanel(panel);
+        try {
+            await narrowable.restrictTo(await scope.RestrictionTarget.fromElement(panel));
+            return { Method: 'element', Release: release };
+        }
+        catch (err) {
+            release();
+            return wrongSurface(err);
+        }
+    }
+    if (scope.CropTarget && narrowable.cropTo) {
+        try {
+            await narrowable.cropTo(await scope.CropTarget.fromElement(panel));
+            return { Method: 'region', Release: () => undefined };
+        }
+        catch (err) {
+            return wrongSurface(err);
+        }
+    }
+    return failure('panel-unsupported', 'This browser cannot narrow a share to a single panel.');
+}
+/** The browser refuses to narrow a capture of any tab but this one. */
+function wrongSurface(err) {
+    return failure('panel-wrong-surface', `${WRONG_SURFACE_MESSAGE} (${errorMessage(err)}).`);
+}
+/**
+ * Makes the panel its own stacking context, which Element Capture needs to emit frames, unless it already is
+ * one through `isolation`. Set inline and marked important, so no style sheet undoes it during the share.
+ *
+ * @returns A function that puts the panel's own inline `isolation` back; the share calls it once, as it ends.
+ */
+function isolatePanel(panel) {
+    const style = panel.style;
+    if (!style || (typeof getComputedStyle === 'function' && getComputedStyle(panel).isolation === 'isolate')) {
+        return () => undefined;
+    }
+    const value = style.getPropertyValue('isolation');
+    const priority = style.getPropertyPriority('isolation');
+    style.setProperty('isolation', 'isolate', 'important');
+    return () => {
+        if (value) {
+            style.setProperty('isolation', value, priority);
+        }
+        else {
+            style.removeProperty('isolation');
+        }
+    };
+}
+/**
+ * Turns a picker rejection into a result. Closing the picker rejects with `NotAllowedError`, and so does an
+ * operating-system refusal, which Chromium words "Permission denied by system".
+ */
+function pickerFailure(err) {
+    const name = err instanceof Error ? err.name : '';
+    const message = errorMessage(err);
+    if (name === 'NotAllowedError') {
+        return /system|policy/i.test(message) ? failure('denied', message) : { Status: 'cancelled' };
+    }
+    return failure('error', message);
+}
+function failure(reason, message) {
+    return { Status: 'failed', Reason: reason, Message: message };
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function stopTracks(stream) {
+    stream.getTracks().forEach((track) => track.stop());
+}
+/**
+ * The kind of surface a captured track shows, from the browser's `displaySurface` setting. It reads any display
+ * track, including one captured elsewhere, such as a LiveKit screen share.
+ */
+export function CapturedSurfaceOf(track) {
+    switch (track.getSettings().displaySurface) {
+        case 'monitor':
+            return 'screen';
+        case 'window':
+            return 'window';
+        case 'browser':
+            return 'tab';
+        default:
+            return 'unknown';
+    }
+}
+/** A share in progress. */
+class ActiveDisplayCapture {
+    constructor(Stream, Track, panel) {
+        this.Stream = Stream;
+        this.Track = Track;
+        this.panel = panel;
+        this.endedHandlers = new Set();
+        this.ended = false;
+        // `ended` fires when the user stops sharing from the browser's bar or the surface closes.
+        Track.addEventListener('ended', () => this.end());
+    }
+    get Surface() {
+        return CapturedSurfaceOf(this.Track);
+    }
+    get Label() {
+        return this.Track.label;
+    }
+    get PanelMethod() {
+        return this.panel?.Method;
+    }
+    get PanelLabel() {
+        return this.panel?.Label;
+    }
+    OnEnded(handler) {
+        if (this.ended) {
+            handler();
+            return () => undefined;
+        }
+        this.endedHandlers.add(handler);
+        return () => this.endedHandlers.delete(handler);
+    }
+    Stop() {
+        stopTracks(this.Stream);
+        // A track stopped by the page fires no `ended` event, so report the end here.
+        this.end();
+    }
+    end() {
+        if (this.ended) {
+            return;
+        }
+        this.ended = true;
+        // The panel's own style comes back before anyone hears that the share ended.
+        this.panel?.Release();
+        for (const handler of [...this.endedHandlers]) {
+            handler();
+        }
+    }
+}
+//# sourceMappingURL=displayCapture.js.map

package/dist/media/displayCapture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"displayCapture.js","sourceRoot":"","sources":["../../src/media/displayCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AA8HH,4DAA4D;AAC5D,MAAM,cAAc,GAA0C,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAEtH,+GAA+G;AAC/G,MAAM,qBAAqB,GAAG,kEAAkE,CAAC;AAEjG,qCAAqC;AACrC,MAAM,UAAU,wBAAwB;IACpC,MAAM,KAAK,GAAG,UAA0B,CAAC;IACzC,OAAO;QACH,OAAO,EAAE,OAAO,SAAS,KAAK,WAAW,IAAI,OAAO,SAAS,CAAC,YAAY,EAAE,eAAe,KAAK,UAAU;QAC1G,cAAc,EAAE,OAAO,KAAK,CAAC,iBAAiB,EAAE,WAAW,KAAK,UAAU;QAC1E,aAAa,EAAE,OAAO,KAAK,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU;KACrE,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,UAAiC,EAAE;IAC3E,MAAM,OAAO,GAAG,wBAAwB,EAAE,CAAC;IAC3C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,aAAa,EAAE,qCAAqC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACrE,OAAO,OAAO,CAAC,mBAAmB,EAAE,gFAAgF,CAAC,CAAC;IAC1H,CAAC;IACD,IAAI,MAAmB,CAAC;IACxB,IAAI,CAAC;QACD,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,OAAO,CAAC,OAAO,EAAE,uCAAuC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpE,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACvB,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,KAAK,GAAe,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,mHAAmH;AACnH,SAAS,aAAa,CAAC,OAA8B;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC;IAC5C,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO;YACH,KAAK,EAAE,EAAE,cAAc,EAAE,cAAc,CAAC,GAAG,EAAE;YAC7C,KAAK;YACL,gBAAgB,EAAE,IAAI;YACtB,kBAAkB,EAAE,SAAS;YAC7B,uEAAuE;YACvE,gBAAgB,EAAE,SAAS;SAC9B,CAAC;IACN,CAAC;IACD,OAAO;QACH,KAAK,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;QACrG,KAAK;QACL,gFAAgF;QAChF,kBAAkB,EAAE,SAAS;QAC7B,gBAAgB,EAAE,SAAS;QAC3B,mBAAmB,EAAE,SAAS;QAC9B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CACxB,KAAuB,EACvB,KAAc,EACd,OAA8B;IAE9B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,cAAc,KAAK,cAAc,CAAC,GAAG,EAAE,CAAC;QAC5D,OAAO,OAAO,CAAC,qBAAqB,EAAE,GAAG,qBAAqB,GAAG,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,KAAK,GAAG,UAA0B,CAAC;IACzC,MAAM,UAAU,GAAG,KAAwB,CAAC;IAC5C,IAAI,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7E,6EAA6E;QAC7E,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC;YACD,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9E,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACxC,IAAI,CAAC;YACD,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC,mBAAmB,EAAE,uDAAuD,CAAC,CAAC;AACjG,CAAC;AAED,uEAAuE;AACvE,SAAS,YAAY,CAAC,GAAY;IAC9B,OAAO,OAAO,CAAC,qBAAqB,EAAE,GAAG,qBAAqB,KAAK,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAc;IAChC,MAAM,KAAK,GAAI,KAAmD,CAAC,KAAK,CAAC;IACzE,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,gBAAgB,KAAK,UAAU,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;QACxG,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;IAC3B,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACxD,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IACvD,OAAO,GAAG,EAAE;QACR,IAAI,KAAK,EAAE,CAAC;YACR,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,GAAY;IAC/B,MAAM,IAAI,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QAC7B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,OAAO,CAAC,MAA6B,EAAE,OAAe;IAC3D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAC9B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,MAAmB;IACnC,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAuB;IACrD,QAAQ,KAAK,CAAC,WAAW,EAAE,CAAC,cAAc,EAAE,CAAC;QACzC,KAAK,SAAS;YACV,OAAO,QAAQ,CAAC;QACpB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB,KAAK,SAAS;YACV,OAAO,KAAK,CAAC;QACjB;YACI,OAAO,SAAS,CAAC;IACzB,CAAC;AACL,CAAC;AAED,2BAA2B;AAC3B,MAAM,oBAAoB;IAItB,YACoB,MAAmB,EACnB,KAAuB,EACtB,KAAkB;QAFnB,WAAM,GAAN,MAAM,CAAa;QACnB,UAAK,GAAL,KAAK,CAAkB;QACtB,UAAK,GAAL,KAAK,CAAa;QANtB,kBAAa,GAAG,IAAI,GAAG,EAAc,CAAC;QAC/C,UAAK,GAAG,KAAK,CAAC;QAOlB,0FAA0F;QAC1F,KAAK,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,IAAW,OAAO;QACd,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC;IAC9B,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;IAC7B,CAAC;IAEM,OAAO,CAAC,OAAmB;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAEM,IAAI;QACP,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxB,8EAA8E;QAC9E,IAAI,CAAC,GAAG,EAAE,CAAC;IACf,CAAC;IAEO,GAAG;QACP,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,6EAA6E;QAC7E,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACtB,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/dist/media/frameCapture.d.ts

@@ -1,12 +1,22 @@
 /**
  * @fileoverview FRAME CAPTURE for realtime video tracks — camera and screen capture pipelines.
  *
- * Patterned beside `micCapture.ts`:
- * - `CreateCameraCapture`: prompts via `getUserMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateScreenCapture`: prompts via `getDisplayMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateStreamFrameCapture`: extracts periodic JPEG frames from an already-acquired `MediaStream`.
+ * Patterned beside `micCapture.ts`. Every exported function here is deprecated; new code uses the `/media` modules:
+ * - `CreateCameraCapture` (deprecated): prompts via `getUserMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code starts the camera with `LocalMediaController` and samples
+ *   its stream with {@link FrameSampler}.
+ * - `CreateScreenCapture` (deprecated): prompts via `getDisplayMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code shares with `RequestDisplayCapture` and samples with
+ *   {@link FrameSampler}.
+ * - `CreateStreamFrameCapture` (deprecated): extracts periodic JPEG frames from an already-acquired
+ *   `MediaStream`. It is now a thin wrapper over {@link FrameSampler}, which new code uses directly.
  *
- * Cadence is throttled to at most 1 frame per second (1 fps ceiling per Live API capabilities guide).
+ * The two capture helpers keep their own `getUserMedia` and `getDisplayMedia` calls rather than wrapping the new
+ * modules: those differ in what they offer (the screen picker leaves out the current tab, for one) and in how they
+ * report a failure, so a caller of a helper keeps the behaviour it was written against.
+ *
+ * Capture runs at the device's native frame rate; only the sampling is paced, at the rate the caller passes
+ * (the negotiated track rate). There is no 1 fps ceiling here.
  *
  * @module @memberjunction/ai-realtime-client
  * @author MemberJunction.com
@@ -33,8 +43,8 @@ export interface FrameCaptureOptions {
      */
     ConsentGranted?: boolean;
     /**
-     * Capture cadence in frames per second (fps).
-     * Capped at 1 fps maximum (Gemini Live ceiling). Defaults to `Descriptor?.Rate ?? 1`.
+     * Frames per second to sample. No ceiling: pass the negotiated track rate. Defaults to
+     * `Descriptor?.Rate ?? 1`.
      */
     Rate?: number;
     /** Output image format ('image/jpeg' or 'image/png'). Defaults to 'image/jpeg'. */
@@ -50,6 +60,8 @@ export interface FrameCaptureOptions {
 /**
  * Creates a frame capture pump over an already-acquired {@link MediaStream}.
  *
+ * @deprecated Use {@link FrameSampler}, which reports whether it started and returns frame sizes and
+ *   timestamps. This wrapper passes `Rate` through unchanged; it no longer caps it at 1 fps.
  * @param stream The media stream containing one or more video tracks.
  * @param options Frame capture options (cadence, format, frame callback).
  */
@@ -58,6 +70,9 @@ export declare function CreateStreamFrameCapture(stream: MediaStream, options: F
  * Requests user camera video capture via `navigator.mediaDevices.getUserMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Start the camera with `LocalMediaController` (in `localMediaController.ts`), which lists and switches
+ *   devices and reports failures as a state, then sample its stream (`GetStream('camera')`) with a
+ *   {@link FrameSampler}.
  * @param options Frame capture options.
  * @param constraints Video track constraints.
  */
@@ -66,11 +81,13 @@ export declare function CreateCameraCapture(options: FrameCaptureOptions, constr
  * Requests screen share video capture via `navigator.mediaDevices.getDisplayMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Use `RequestDisplayCapture` (in `displayCapture.ts`), which offers surface hints and panel
+ *   capture and reports what was shared, then a {@link FrameSampler} on its stream.
  * @param options Frame capture options.
  * @param displayMediaOptions Display media options.
  */
 export declare function CreateScreenCapture(options: FrameCaptureOptions, displayMediaOptions?: DisplayMediaStreamOptions): Promise<IFrameCapture>;
-/** @deprecated Use {@link CreateStreamFrameCapture}. */
+/** @deprecated Use {@link FrameSampler}. */
 export declare function createStreamFrameCapture(stream: MediaStream, options: FrameCaptureOptions): IFrameCapture;
 /** @deprecated Use {@link CreateCameraCapture}. */
 export declare function createCameraCapture(options: FrameCaptureOptions, constraints?: MediaTrackConstraints): Promise<IFrameCapture>;

package/dist/media/frameCapture.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"frameCapture.d.ts","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,yFAAyF;IACzF,IAAI,IAAI,IAAI,CAAC;IACb,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,+CAA+C;IAC/C,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;IACtC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAChE;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAyEf;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAqBxB;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAqBxB;AAED,wDAAwD;AACxD,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAExB;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAExB"}
+{"version":3,"file":"frameCapture.d.ts","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAGlE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,yFAAyF;IACzF,IAAI,IAAI,IAAI,CAAC;IACb,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,+CAA+C;IAC/C,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;IACtC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAChE;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAmBD;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAsBxB;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAoBxB;AAED,4CAA4C;AAC5C,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAExB;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAExB"}

package/dist/media/frameCapture.js

@@ -1,86 +1,49 @@
 /**
  * @fileoverview FRAME CAPTURE for realtime video tracks — camera and screen capture pipelines.
  *
- * Patterned beside `micCapture.ts`:
- * - `CreateCameraCapture`: prompts via `getUserMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateScreenCapture`: prompts via `getDisplayMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateStreamFrameCapture`: extracts periodic JPEG frames from an already-acquired `MediaStream`.
+ * Patterned beside `micCapture.ts`. Every exported function here is deprecated; new code uses the `/media` modules:
+ * - `CreateCameraCapture` (deprecated): prompts via `getUserMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code starts the camera with `LocalMediaController` and samples
+ *   its stream with {@link FrameSampler}.
+ * - `CreateScreenCapture` (deprecated): prompts via `getDisplayMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code shares with `RequestDisplayCapture` and samples with
+ *   {@link FrameSampler}.
+ * - `CreateStreamFrameCapture` (deprecated): extracts periodic JPEG frames from an already-acquired
+ *   `MediaStream`. It is now a thin wrapper over {@link FrameSampler}, which new code uses directly.
  *
- * Cadence is throttled to at most 1 frame per second (1 fps ceiling per Live API capabilities guide).
+ * The two capture helpers keep their own `getUserMedia` and `getDisplayMedia` calls rather than wrapping the new
+ * modules: those differ in what they offer (the screen picker leaves out the current tab, for one) and in how they
+ * report a failure, so a caller of a helper keeps the behaviour it was written against.
+ *
+ * Capture runs at the device's native frame rate; only the sampling is paced, at the rate the caller passes
+ * (the negotiated track rate). There is no 1 fps ceiling here.
  *
  * @module @memberjunction/ai-realtime-client
  * @author MemberJunction.com
  */
+import { FrameSampler } from './frameSampler.js';
 /**
  * Creates a frame capture pump over an already-acquired {@link MediaStream}.
  *
+ * @deprecated Use {@link FrameSampler}, which reports whether it started and returns frame sizes and
+ *   timestamps. This wrapper passes `Rate` through unchanged; it no longer caps it at 1 fps.
  * @param stream The media stream containing one or more video tracks.
  * @param options Frame capture options (cadence, format, frame callback).
  */
 export function CreateStreamFrameCapture(stream, options) {
-    const rawRate = options.Rate ?? options.Descriptor?.Rate ?? 1;
-    // Cadence ceiling: max 1 fps per Gemini Live API specs; minimum 0.1 fps
-    const rate = Math.min(Math.max(rawRate, 0.1), 1);
-    const intervalMs = Math.floor(1000 / rate);
-    const mimeType = options.MimeType ?? 'image/jpeg';
-    const quality = options.Quality ?? 0.8;
-    let stopped = false;
-    let timer = null;
-    let videoEl = null;
-    let canvasEl = null;
-    if (typeof document !== 'undefined' && typeof document.createElement === 'function') {
-        try {
-            videoEl = document.createElement('video');
-            videoEl.muted = true;
-            videoEl.playsInline = true;
-            videoEl.srcObject = stream;
-            void videoEl.play().catch(() => {
-                // Autoplay may be restricted without user interaction; non-fatal for capture
-            });
-            canvasEl = document.createElement('canvas');
-        }
-        catch (err) {
-            console.error('[FrameCapture] Failed to initialize DOM elements for video capture:', err);
-        }
-    }
-    const captureTick = () => {
-        if (stopped) {
-            return;
-        }
-        if (videoEl && canvasEl && videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
-            canvasEl.width = videoEl.videoWidth;
-            canvasEl.height = videoEl.videoHeight;
-            const ctx = canvasEl.getContext('2d');
-            if (ctx) {
-                ctx.drawImage(videoEl, 0, 0);
-                const dataUrl = canvasEl.toDataURL(mimeType, quality);
-                const base64 = dataUrl.split(',')[1] ?? '';
-                if (base64.length > 0) {
-                    options.OnFrame({ data: base64, mimeType });
-                }
-            }
-        }
-    };
-    if (videoEl && canvasEl) {
-        timer = setInterval(captureTick, intervalMs);
-    }
-    else {
-        console.warn('[FrameCapture] No video/canvas element available (no DOM, or element construction failed) — ' +
-            'this capture will emit no frames.');
-    }
+    return startSampler(stream, options);
+}
+/** Samples a stream with a {@link FrameSampler}, behind the older {@link IFrameCapture} handle. */
+function startSampler(stream, options) {
+    const sampler = new FrameSampler(stream, {
+        Rate: options.Rate ?? options.Descriptor?.Rate ?? 1,
+        MimeType: options.MimeType,
+        Quality: options.Quality,
+        OnFrame: (frame) => options.OnFrame({ data: frame.Data, mimeType: frame.MimeType }),
+    });
+    sampler.Start();
     return {
-        Stop: () => {
-            stopped = true;
-            if (timer) {
-                clearInterval(timer);
-                timer = null;
-            }
-            if (videoEl) {
-                videoEl.srcObject = null;
-                videoEl = null;
-            }
-            canvasEl = null;
-        },
+        Stop: () => sampler.Stop(),
         get Stream() {
             return stream;
         },
@@ -90,6 +53,9 @@ export function CreateStreamFrameCapture(stream, options) {
  * Requests user camera video capture via `navigator.mediaDevices.getUserMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Start the camera with `LocalMediaController` (in `localMediaController.ts`), which lists and switches
+ *   devices and reports failures as a state, then sample its stream (`GetStream('camera')`) with a
+ *   {@link FrameSampler}.
  * @param options Frame capture options.
  * @param constraints Video track constraints.
  */
@@ -101,10 +67,11 @@ export async function CreateCameraCapture(options, constraints) {
     if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
         throw new Error('Camera capture is unavailable: navigator.mediaDevices.getUserMedia not supported in this environment.');
     }
+    // No frameRate cap: the camera runs at its native rate (a smooth self-view); only sampling is paced.
     const stream = await navigator.mediaDevices.getUserMedia({
-        video: constraints ?? { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { max: 1 } },
+        video: constraints ?? { width: { ideal: 1280 }, height: { ideal: 720 } },
     });
-    const capture = CreateStreamFrameCapture(stream, options);
+    const capture = startSampler(stream, options);
     return {
         Stop: () => {
             capture.Stop();
@@ -119,6 +86,8 @@ export async function CreateCameraCapture(options, constraints) {
  * Requests screen share video capture via `navigator.mediaDevices.getDisplayMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Use `RequestDisplayCapture` (in `displayCapture.ts`), which offers surface hints and panel
+ *   capture and reports what was shared, then a {@link FrameSampler} on its stream.
  * @param options Frame capture options.
  * @param displayMediaOptions Display media options.
  */
@@ -130,8 +99,9 @@ export async function CreateScreenCapture(options, displayMediaOptions) {
     if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getDisplayMedia) {
         throw new Error('Screen capture is unavailable: navigator.mediaDevices.getDisplayMedia not supported in this environment.');
     }
-    const stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions ?? { video: { frameRate: { max: 1 } } });
-    const capture = CreateStreamFrameCapture(stream, options);
+    // No frameRate cap: the share runs at its native rate; only sampling is paced.
+    const stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions ?? { video: true });
+    const capture = startSampler(stream, options);
     return {
         Stop: () => {
             capture.Stop();
@@ -142,7 +112,7 @@ export async function CreateScreenCapture(options, displayMediaOptions) {
         },
     };
 }
-/** @deprecated Use {@link CreateStreamFrameCapture}. */
+/** @deprecated Use {@link FrameSampler}. */
 export function createStreamFrameCapture(stream, options) {
     return CreateStreamFrameCapture(stream, options);
 }

package/dist/media/frameCapture.js.map

@@ -1 +1 @@
-{"version":3,"file":"frameCapture.js","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAsCH;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CACpC,MAAmB,EACnB,OAA4B;IAE5B,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,IAAI,IAAI,CAAC,CAAC;IAC9D,wEAAwE;IACxE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC;IAClD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC;IAEvC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,KAAK,GAA0C,IAAI,CAAC;IACxD,IAAI,OAAO,GAA4B,IAAI,CAAC;IAC5C,IAAI,QAAQ,GAA6B,IAAI,CAAC;IAE9C,IAAI,OAAO,QAAQ,KAAK,WAAW,IAAI,OAAO,QAAQ,CAAC,aAAa,KAAK,UAAU,EAAE,CAAC;QAClF,IAAI,CAAC;YACD,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;YAC1C,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC;YACrB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC;YAC3B,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC;YAC3B,KAAK,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE;gBAC3B,6EAA6E;YACjF,CAAC,CAAC,CAAC;YACH,QAAQ,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAChD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,CAAC,KAAK,CAAC,qEAAqE,EAAE,GAAG,CAAC,CAAC;QAC9F,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,GAAG,EAAE;QACrB,IAAI,OAAO,EAAE,CAAC;YACV,OAAO;QACX,CAAC;QACD,IAAI,OAAO,IAAI,QAAQ,IAAI,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YAC3E,QAAQ,CAAC,KAAK,GAAG,OAAO,CAAC,UAAU,CAAC;YACpC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC;YACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,GAAG,EAAE,CAAC;gBACN,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBACtD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC3C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACpB,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAChD,CAAC;YACL,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IAEF,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;QACtB,KAAK,GAAG,WAAW,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,IAAI,CACR,8FAA8F;YAC9F,mCAAmC,CACtC,CAAC;IACN,CAAC;IAED,OAAO;QACH,IAAI,EAAE,GAAG,EAAE;YACP,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,KAAK,EAAE,CAAC;gBACR,aAAa,CAAC,KAAK,CAAC,CAAC;gBACrB,KAAK,GAAG,IAAI,CAAC;YACjB,CAAC;YACD,IAAI,OAAO,EAAE,CAAC;gBACV,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;gBACzB,OAAO,GAAG,IAAI,CAAC;YACnB,CAAC;YACD,QAAQ,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,WAAmC;IAEnC,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,eAAe,IAAI,IAAI,CAAC;IACpE,IAAI,eAAe,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACvF,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,uGAAuG,CAAC,CAAC;IAC7H,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC;QACrD,KAAK,EAAE,WAAW,IAAI,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;KAClG,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1D,OAAO;QACH,IAAI,EAAE,GAAG,EAAE;YACP,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,mBAA+C;IAE/C,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,eAAe,IAAI,IAAI,CAAC;IACpE,IAAI,eAAe,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACvF,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,eAAe,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,0GAA0G,CAAC,CAAC;IAChI,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,eAAe,CACvD,mBAAmB,IAAI,EAAE,KAAK,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,CAC9D,CAAC;IACF,MAAM,OAAO,GAAG,wBAAwB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1D,OAAO;QACH,IAAI,EAAE,GAAG,EAAE;YACP,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,wBAAwB,CACpC,MAAmB,EACnB,OAA4B;IAE5B,OAAO,wBAAwB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,WAAmC;IAEnC,OAAO,mBAAmB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;AACrD,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,mBAA+C;IAE/C,OAAO,mBAAmB,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;AAC7D,CAAC"}
+{"version":3,"file":"frameCapture.js","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAoC9C;;;;;;;GAOG;AACH,MAAM,UAAU,wBAAwB,CACpC,MAAmB,EACnB,OAA4B;IAE5B,OAAO,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACzC,CAAC;AAED,mGAAmG;AACnG,SAAS,YAAY,CAAC,MAAmB,EAAE,OAA4B;IACnE,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,MAAM,EAAE;QACrC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,UAAU,EAAE,IAAI,IAAI,CAAC;QACnD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;KACtF,CAAC,CAAC;IACH,OAAO,CAAC,KAAK,EAAE,CAAC;IAChB,OAAO;QACH,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE;QAC1B,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,WAAmC;IAEnC,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,eAAe,IAAI,IAAI,CAAC;IACpE,IAAI,eAAe,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACvF,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,YAAY,EAAE,CAAC;QAC5E,MAAM,IAAI,KAAK,CAAC,uGAAuG,CAAC,CAAC;IAC7H,CAAC;IACD,qGAAqG;IACrG,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC;QACrD,KAAK,EAAE,WAAW,IAAI,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;KAC3E,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,OAAO;QACH,IAAI,EAAE,GAAG,EAAE;YACP,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,mBAA+C;IAE/C,MAAM,eAAe,GAAG,OAAO,CAAC,UAAU,EAAE,eAAe,IAAI,IAAI,CAAC;IACpE,IAAI,eAAe,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACvF,CAAC;IACD,IAAI,OAAO,SAAS,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,eAAe,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,0GAA0G,CAAC,CAAC;IAChI,CAAC;IACD,+EAA+E;IAC/E,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,eAAe,CAAC,mBAAmB,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACpG,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,OAAO;QACH,IAAI,EAAE,GAAG,EAAE;YACP,OAAO,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,IAAI,MAAM;YACN,OAAO,MAAM,CAAC;QAClB,CAAC;KACJ,CAAC;AACN,CAAC;AAED,4CAA4C;AAC5C,MAAM,UAAU,wBAAwB,CACpC,MAAmB,EACnB,OAA4B;IAE5B,OAAO,wBAAwB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,WAAmC;IAEnC,OAAO,mBAAmB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;AACrD,CAAC;AAED,mDAAmD;AACnD,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACrC,OAA4B,EAC5B,mBAA+C;IAE/C,OAAO,mBAAmB,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;AAC7D,CAAC"}

package/dist/media/frameSampler.d.ts

@@ -0,0 +1,94 @@
+/**
+ * @fileoverview FRAME SAMPLER: takes still frames from any `MediaStream` at the rate the caller asks
+ * for, and returns each as a base64 JPEG or PNG.
+ *
+ * The rate is the caller's: pass the rate the inbound video track negotiated
+ * (`BaseRealtimeClient.InboundVideoRate`). The sampler applies no ceiling of its own, so the stream itself
+ * can run at its native frame rate (a smooth self-view) while the model receives only the frames it can
+ * take. An absent or invalid rate means one frame per second, like every pacer in this package
+ * ({@link NominalVideoFrameIntervalMs}).
+ *
+ * Frames are drawn from a hidden `<video>` onto a `<canvas>`. Without a DOM (Node, SSR) the sampler emits
+ * nothing, and {@link FrameSampler.Start} says so by returning `false`.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** Image formats a frame can be encoded as. */
+export type SampledFrameMimeType = 'image/jpeg' | 'image/png';
+/** A width and height in pixels. */
+export interface FrameSize {
+    Width: number;
+    Height: number;
+}
+/** One frame taken from the stream. */
+export interface SampledFrame {
+    /** The encoded image, base64 (no `data:` prefix). */
+    Data: string;
+    MimeType: SampledFrameMimeType;
+    /** The encoded image's width in pixels, after any {@link FrameSamplerOptions.MaxDimension} scaling. */
+    Width: number;
+    /** The encoded image's height in pixels, after any scaling. */
+    Height: number;
+    /** Epoch milliseconds when the frame was taken. */
+    TimestampMs: number;
+}
+export interface FrameSamplerOptions {
+    /**
+     * Frames per second. Pass the rate the inbound video track negotiated; there is no ceiling. An
+     * absent, zero, negative or non-finite rate means one frame per second.
+     */
+    Rate: number;
+    /** Defaults to `'image/jpeg'`. */
+    MimeType?: SampledFrameMimeType;
+    /** JPEG quality from 0 to 1. Defaults to 0.8. Ignored for PNG. */
+    Quality?: number;
+    /**
+     * Caps the longer side of each frame, in pixels, keeping the aspect ratio. A smaller frame is never
+     * enlarged. Defaults to no cap: frames are encoded at the stream's own size.
+     */
+    MaxDimension?: number;
+    /** Receives each frame. */
+    OnFrame: (frame: SampledFrame) => void;
+}
+/**
+ * The size a frame is encoded at: its own size, or scaled down so the longer side is at most
+ * `maxDimension` pixels. Never enlarges, and never returns a side smaller than 1. The math is
+ * `@memberjunction/ai`'s `ScaleRealtimeVideoFrame`, shared with the meeting bot.
+ *
+ * @param size The frame's own size.
+ * @param maxDimension The cap on the longer side; `undefined` or a non-positive value means no cap.
+ */
+export declare function ScaleToMaxDimension(size: FrameSize, maxDimension?: number): FrameSize;
+/**
+ * Samples still frames from a `MediaStream`. The caller owns the stream: {@link FrameSampler.Stop}
+ * releases the sampler's elements and timer but never stops the stream's tracks.
+ */
+export declare class FrameSampler {
+    private readonly stream;
+    private readonly options;
+    private readonly intervalMs;
+    private timer;
+    private video;
+    private canvas;
+    /**
+     * @param stream The stream to sample. Its first video track is drawn.
+     * @param options Rate, format, size cap and the frame callback.
+     */
+    constructor(stream: MediaStream, options: FrameSamplerOptions);
+    /** The stream being sampled. */
+    get Stream(): MediaStream;
+    /** Whether the sampler is taking frames. */
+    get IsRunning(): boolean;
+    /**
+     * Starts taking frames. Calling it while running does nothing.
+     *
+     * @returns `false` when there is no DOM to draw frames in, so no frames will come.
+     */
+    Start(): boolean;
+    /** Stops taking frames and releases the elements. The stream's tracks keep running. */
+    Stop(): void;
+    private createElements;
+    /** Draws and encodes the current frame. Skips the tick until the video has a frame to draw. */
+    private sampleFrame;
+}
+//# sourceMappingURL=frameSampler.d.ts.map

package/dist/media/frameSampler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"frameSampler.d.ts","sourceRoot":"","sources":["../../src/media/frameSampler.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,+CAA+C;AAC/C,MAAM,MAAM,oBAAoB,GAAG,YAAY,GAAG,WAAW,CAAC;AAE9D,oCAAoC;AACpC,MAAM,WAAW,SAAS;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAClB;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IACzB,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,uGAAuG;IACvG,KAAK,EAAE,MAAM,CAAC;IACd,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAChC;;;OAGG;IACH,IAAI,EAAE,MAAM,CAAC;IACb,kCAAkC;IAClC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IAChC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,2BAA2B;IAC3B,OAAO,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;CAC1C;AAKD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,SAAS,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAErF;AAED;;;GAGG;AACH,qBAAa,YAAY;IAWjB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAX5B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,KAAK,CAAiC;IAC9C,OAAO,CAAC,MAAM,CAAkC;IAEhD;;;OAGG;gBAEkB,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB;IAKjD,gCAAgC;IAChC,IAAW,MAAM,IAAI,WAAW,CAE/B;IAED,4CAA4C;IAC5C,IAAW,SAAS,IAAI,OAAO,CAE9B;IAED;;;;OAIG;IACI,KAAK,IAAI,OAAO;IAYvB,uFAAuF;IAChF,IAAI,IAAI,IAAI;IAkBnB,OAAO,CAAC,cAAc;IAsBtB,+FAA+F;IAC/F,OAAO,CAAC,WAAW;CAoBtB"}