@memberjunction/ai-realtime-client 6.2.0-edge.3 → 6.2.0-edge.5

package/dist/media/displayCapture.d.ts

@@ -0,0 +1,128 @@
+/**
+ * @fileoverview DISPLAY CAPTURE: asks the browser to share a screen, a window, a tab or one panel of this
+ * page, and reports what the user actually shared.
+ *
+ * Sharing works wherever `getDisplayMedia` exists: desktop Chrome, Edge, Firefox and Safari (no mobile browser
+ * has it). The picker hints sent with it (`displaySurface`, `selfBrowserSurface`, `surfaceSwitching`,
+ * `preferCurrentTab`, `monitorTypeSurfaces`, `systemAudio`) are read only by Chromium browsers; others ignore
+ * them.
+ *
+ * A PANEL capture shares one element of this page. The user picks this tab in the picker, and the stream is
+ * then narrowed to the element with Element Capture (`restrictTo`, Chromium 132+: the element alone, without
+ * anything drawn over it) or, failing that, Region Capture (`cropTo`, Chromium 104+: the element's rectangle,
+ * including anything over it). Element Capture emits frames only while the element is its own stacking
+ * context, so the element gets `isolation: isolate` for the share and its own value back when the share ends.
+ * Firefox and Safari have neither; {@link GetDisplayCaptureSupport} says so, and the caller shows the panel to
+ * the agent another way (its channel frames). If the user shares something other than this tab, the capture is
+ * stopped rather than sharing more than the panel.
+ *
+ * The browser's picker is the user's consent: it opens on every request, with no remembered grant. Like the
+ * picker itself, {@link RequestDisplayCapture} must be called from a user gesture (a click handler).
+ *
+ * Capture runs at the surface's native frame rate; pace what the model receives with a `FrameSampler`.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** What the user is asked to share. */
+export type DisplayCaptureSurface = 'screen' | 'window' | 'tab';
+/** What a capture turned out to be; `'unknown'` when the browser doesn't say. */
+export type CapturedDisplaySurface = DisplayCaptureSurface | 'unknown';
+/** How a panel capture was narrowed to its element. */
+export type PanelCaptureMethod = 'element' | 'region';
+/** What this browser can capture. */
+export interface DisplayCaptureSupport {
+    /** A screen, window or tab can be shared (`getDisplayMedia`). */
+    Display: boolean;
+    /** Element Capture (`RestrictionTarget`, Chromium 132+). */
+    ElementCapture: boolean;
+    /** Region Capture (`CropTarget`, Chromium 104+). */
+    RegionCapture: boolean;
+}
+export interface DisplayCaptureOptions {
+    /**
+     * The kind of surface the picker offers first. A hint: the user can still pick another kind, and only
+     * Chromium browsers read it. Ignored for a panel capture, which asks for this tab.
+     */
+    PreferredSurface?: DisplayCaptureSurface;
+    /**
+     * Share only this element of the page. Needs Element or Region Capture (see
+     * {@link GetDisplayCaptureSupport}); Element Capture is used where the browser has it. Element Capture emits
+     * frames only while the element is its own stacking context, so unless it already is one, the element's
+     * inline style gets `isolation: isolate` while it is shared and its own inline value back when the share
+     * ends. Meanwhile its descendants stack inside it: one with a high `z-index` no longer paints over content
+     * outside the panel.
+     */
+    Panel?: Element;
+    /**
+     * The panel's name, such as "Whiteboard", carried on the capture ({@link DisplayCapture.PanelLabel}) so the
+     * UI and the agent can name what is shared. Read only with {@link Panel}.
+     */
+    PanelLabel?: string;
+    /**
+     * Also capture the shared surface's audio, where the browser offers it (Chromium: tab or system audio).
+     * Defaults to `false`: the agent already hears the microphone.
+     */
+    IncludeAudio?: boolean;
+}
+/** A live share. */
+export interface DisplayCapture {
+    readonly Stream: MediaStream;
+    /** The shared video track. */
+    readonly Track: MediaStreamTrack;
+    /** What is being shared now. It can change if the user switches surfaces mid-share. */
+    readonly Surface: CapturedDisplaySurface;
+    /** The browser's label for the shared surface, such as a window or tab title. */
+    readonly Label: string;
+    /** How the stream was narrowed, for a panel capture. */
+    readonly PanelMethod?: PanelCaptureMethod;
+    /** The shared panel's name, for a panel capture that was given one ({@link DisplayCaptureOptions.PanelLabel}). */
+    readonly PanelLabel?: string;
+    /**
+     * Calls `handler` once when the share ends: the user pressed the browser's "Stop sharing", the shared
+     * surface closed, or {@link DisplayCapture.Stop} was called. If the share has already ended, `handler` is
+     * called at once.
+     *
+     * @returns A function that removes the handler.
+     */
+    OnEnded(handler: () => void): () => void;
+    /** Stops sharing. Calling it again does nothing. */
+    Stop(): void;
+}
+/** Why a capture failed. */
+export type DisplayCaptureFailure = 
+/** The browser cannot share a screen at all (mobile browsers, or no DOM). */
+'unsupported'
+/** A panel was asked for, but the browser has neither Element nor Region Capture (Firefox, Safari). */
+ | 'panel-unsupported'
+/** A panel was asked for, but the user shared a screen, a window or another tab. The capture was stopped. */
+ | 'panel-wrong-surface'
+/** The operating system or the page's permissions policy blocked sharing. */
+ | 'denied'
+/** Anything else; see the message. */
+ | 'error';
+/** The outcome of {@link RequestDisplayCapture}. It always resolves; it never rejects. */
+export type DisplayCaptureResult = {
+    Status: 'started';
+    Capture: DisplayCapture;
+} | {
+    Status: 'cancelled';
+} | {
+    Status: 'failed';
+    Reason: DisplayCaptureFailure;
+    Message: string;
+};
+/** What this browser can capture. */
+export declare function GetDisplayCaptureSupport(): DisplayCaptureSupport;
+/**
+ * Opens the browser's share picker and starts sharing what the user picks. Call it from a user gesture.
+ *
+ * @param options What to offer, whether to share only a panel, and whether to include audio.
+ * @returns `started` with the live capture, `cancelled` when the user closed the picker, or `failed` with a reason.
+ */
+export declare function RequestDisplayCapture(options?: DisplayCaptureOptions): Promise<DisplayCaptureResult>;
+/**
+ * The kind of surface a captured track shows, from the browser's `displaySurface` setting. It reads any display
+ * track, including one captured elsewhere, such as a LiveKit screen share.
+ */
+export declare function CapturedSurfaceOf(track: MediaStreamTrack): CapturedDisplaySurface;
+//# sourceMappingURL=displayCapture.d.ts.map

package/dist/media/displayCapture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"displayCapture.d.ts","sourceRoot":"","sources":["../../src/media/displayCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,uCAAuC;AACvC,MAAM,MAAM,qBAAqB,GAAG,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAC;AAEhE,iFAAiF;AACjF,MAAM,MAAM,sBAAsB,GAAG,qBAAqB,GAAG,SAAS,CAAC;AAEvE,uDAAuD;AACvD,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEtD,qCAAqC;AACrC,MAAM,WAAW,qBAAqB;IAClC,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAC;IACjB,4DAA4D;IAC5D,cAAc,EAAE,OAAO,CAAC;IACxB,oDAAoD;IACpD,aAAa,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,qBAAqB,CAAC;IACzC;;;;;;;OAOG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;OAGG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED,oBAAoB;AACpB,MAAM,WAAW,cAAc;IAC3B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,8BAA8B;IAC9B,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,uFAAuF;IACvF,QAAQ,CAAC,OAAO,EAAE,sBAAsB,CAAC;IACzC,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,QAAQ,CAAC,WAAW,CAAC,EAAE,kBAAkB,CAAC;IAC1C,kHAAkH;IAClH,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;OAMG;IACH,OAAO,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC;IACzC,oDAAoD;IACpD,IAAI,IAAI,IAAI,CAAC;CAChB;AAED,4BAA4B;AAC5B,MAAM,MAAM,qBAAqB;AAC7B,6EAA6E;AAC3E,aAAa;AACf,uGAAuG;GACrG,mBAAmB;AACrB,6GAA6G;GAC3G,qBAAqB;AACvB,6EAA6E;GAC3E,QAAQ;AACV,sCAAsC;GACpC,OAAO,CAAC;AAEd,0FAA0F;AAC1F,MAAM,MAAM,oBAAoB,GAC1B;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,cAAc,CAAA;CAAE,GAC9C;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,qBAAqB,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA0C3E,qCAAqC;AACrC,wBAAgB,wBAAwB,IAAI,qBAAqB,CAOhE;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,GAAE,qBAA0B,GAAG,OAAO,CAAC,oBAAoB,CAAC,CA6B9G;AAoHD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,gBAAgB,GAAG,sBAAsB,CAWjF"}

package/dist/media/displayCapture.js

@@ -0,0 +1,245 @@
+/**
+ * @fileoverview DISPLAY CAPTURE: asks the browser to share a screen, a window, a tab or one panel of this
+ * page, and reports what the user actually shared.
+ *
+ * Sharing works wherever `getDisplayMedia` exists: desktop Chrome, Edge, Firefox and Safari (no mobile browser
+ * has it). The picker hints sent with it (`displaySurface`, `selfBrowserSurface`, `surfaceSwitching`,
+ * `preferCurrentTab`, `monitorTypeSurfaces`, `systemAudio`) are read only by Chromium browsers; others ignore
+ * them.
+ *
+ * A PANEL capture shares one element of this page. The user picks this tab in the picker, and the stream is
+ * then narrowed to the element with Element Capture (`restrictTo`, Chromium 132+: the element alone, without
+ * anything drawn over it) or, failing that, Region Capture (`cropTo`, Chromium 104+: the element's rectangle,
+ * including anything over it). Element Capture emits frames only while the element is its own stacking
+ * context, so the element gets `isolation: isolate` for the share and its own value back when the share ends.
+ * Firefox and Safari have neither; {@link GetDisplayCaptureSupport} says so, and the caller shows the panel to
+ * the agent another way (its channel frames). If the user shares something other than this tab, the capture is
+ * stopped rather than sharing more than the panel.
+ *
+ * The browser's picker is the user's consent: it opens on every request, with no remembered grant. Like the
+ * picker itself, {@link RequestDisplayCapture} must be called from a user gesture (a click handler).
+ *
+ * Capture runs at the surface's native frame rate; pace what the model receives with a `FrameSampler`.
+ *
+ * @module @memberjunction/ai-realtime-client/media
+ */
+/** The picker's `displaySurface` value for each surface. */
+const PICKER_SURFACE = { screen: 'monitor', window: 'window', tab: 'browser' };
+/** What the user is told when a panel share got something other than this tab (without the closing period). */
+const WRONG_SURFACE_MESSAGE = "To share only the panel, choose this tab in the browser's picker";
+/** What this browser can capture. */
+export function GetDisplayCaptureSupport() {
+    const scope = globalThis;
+    return {
+        Display: typeof navigator !== 'undefined' && typeof navigator.mediaDevices?.getDisplayMedia === 'function',
+        ElementCapture: typeof scope.RestrictionTarget?.fromElement === 'function',
+        RegionCapture: typeof scope.CropTarget?.fromElement === 'function',
+    };
+}
+/**
+ * Opens the browser's share picker and starts sharing what the user picks. Call it from a user gesture.
+ *
+ * @param options What to offer, whether to share only a panel, and whether to include audio.
+ * @returns `started` with the live capture, `cancelled` when the user closed the picker, or `failed` with a reason.
+ */
+export async function RequestDisplayCapture(options = {}) {
+    const support = GetDisplayCaptureSupport();
+    if (!support.Display) {
+        return failure('unsupported', 'This browser cannot share a screen.');
+    }
+    if (options.Panel && !support.ElementCapture && !support.RegionCapture) {
+        return failure('panel-unsupported', 'This browser cannot share a single panel; show it through its channel instead.');
+    }
+    let stream;
+    try {
+        stream = await navigator.mediaDevices.getDisplayMedia(pickerOptions(options));
+    }
+    catch (err) {
+        return pickerFailure(err);
+    }
+    const track = stream.getVideoTracks()[0];
+    if (!track) {
+        stopTracks(stream);
+        return failure('error', 'The shared stream has no video track.');
+    }
+    if (!options.Panel) {
+        return { Status: 'started', Capture: new ActiveDisplayCapture(stream, track) };
+    }
+    const narrowed = await narrowToPanel(track, options.Panel, support);
+    if ('Status' in narrowed) {
+        stopTracks(stream);
+        return narrowed;
+    }
+    const panel = options.PanelLabel ? { ...narrowed, Label: options.PanelLabel } : narrowed;
+    return { Status: 'started', Capture: new ActiveDisplayCapture(stream, track, panel) };
+}
+/** The picker request: this tab for a panel; otherwise any surface but this tab, with the preferred kind first. */
+function pickerOptions(options) {
+    const audio = options.IncludeAudio ?? false;
+    if (options.Panel) {
+        return {
+            video: { displaySurface: PICKER_SURFACE.tab },
+            audio,
+            preferCurrentTab: true,
+            selfBrowserSurface: 'include',
+            // Switching to another surface would leave the panel narrowing behind.
+            surfaceSwitching: 'exclude',
+        };
+    }
+    return {
+        video: options.PreferredSurface ? { displaySurface: PICKER_SURFACE[options.PreferredSurface] } : true,
+        audio,
+        // Sharing this page would show the agent its own call UI, mirrored without end.
+        selfBrowserSurface: 'exclude',
+        surfaceSwitching: 'include',
+        monitorTypeSurfaces: 'include',
+        systemAudio: audio ? 'include' : 'exclude',
+    };
+}
+/**
+ * Narrows a self-capture to the panel: with Element Capture where the browser has it, else with Region Capture.
+ *
+ * @returns How it was narrowed, or a failure when the user shared something other than this tab.
+ */
+async function narrowToPanel(track, panel, support) {
+    if (track.getSettings().displaySurface !== PICKER_SURFACE.tab) {
+        return failure('panel-wrong-surface', `${WRONG_SURFACE_MESSAGE}.`);
+    }
+    const scope = globalThis;
+    const narrowable = track;
+    if (support.ElementCapture && scope.RestrictionTarget && narrowable.restrictTo) {
+        // Isolated first, so the restricted track has frames to emit from the start.
+        const release = isolatePanel(panel);
+        try {
+            await narrowable.restrictTo(await scope.RestrictionTarget.fromElement(panel));
+            return { Method: 'element', Release: release };
+        }
+        catch (err) {
+            release();
+            return wrongSurface(err);
+        }
+    }
+    if (scope.CropTarget && narrowable.cropTo) {
+        try {
+            await narrowable.cropTo(await scope.CropTarget.fromElement(panel));
+            return { Method: 'region', Release: () => undefined };
+        }
+        catch (err) {
+            return wrongSurface(err);
+        }
+    }
+    return failure('panel-unsupported', 'This browser cannot narrow a share to a single panel.');
+}
+/** The browser refuses to narrow a capture of any tab but this one. */
+function wrongSurface(err) {
+    return failure('panel-wrong-surface', `${WRONG_SURFACE_MESSAGE} (${errorMessage(err)}).`);
+}
+/**
+ * Makes the panel its own stacking context, which Element Capture needs to emit frames, unless it already is
+ * one through `isolation`. Set inline and marked important, so no style sheet undoes it during the share.
+ *
+ * @returns A function that puts the panel's own inline `isolation` back; the share calls it once, as it ends.
+ */
+function isolatePanel(panel) {
+    const style = panel.style;
+    if (!style || (typeof getComputedStyle === 'function' && getComputedStyle(panel).isolation === 'isolate')) {
+        return () => undefined;
+    }
+    const value = style.getPropertyValue('isolation');
+    const priority = style.getPropertyPriority('isolation');
+    style.setProperty('isolation', 'isolate', 'important');
+    return () => {
+        if (value) {
+            style.setProperty('isolation', value, priority);
+        }
+        else {
+            style.removeProperty('isolation');
+        }
+    };
+}
+/**
+ * Turns a picker rejection into a result. Closing the picker rejects with `NotAllowedError`, and so does an
+ * operating-system refusal, which Chromium words "Permission denied by system".
+ */
+function pickerFailure(err) {
+    const name = err instanceof Error ? err.name : '';
+    const message = errorMessage(err);
+    if (name === 'NotAllowedError') {
+        return /system|policy/i.test(message) ? failure('denied', message) : { Status: 'cancelled' };
+    }
+    return failure('error', message);
+}
+function failure(reason, message) {
+    return { Status: 'failed', Reason: reason, Message: message };
+}
+function errorMessage(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function stopTracks(stream) {
+    stream.getTracks().forEach((track) => track.stop());
+}
+/**
+ * The kind of surface a captured track shows, from the browser's `displaySurface` setting. It reads any display
+ * track, including one captured elsewhere, such as a LiveKit screen share.
+ */
+export function CapturedSurfaceOf(track) {
+    switch (track.getSettings().displaySurface) {
+        case 'monitor':
+            return 'screen';
+        case 'window':
+            return 'window';
+        case 'browser':
+            return 'tab';
+        default:
+            return 'unknown';
+    }
+}
+/** A share in progress. */
+class ActiveDisplayCapture {
+    constructor(Stream, Track, panel) {
+        this.Stream = Stream;
+        this.Track = Track;
+        this.panel = panel;
+        this.endedHandlers = new Set();
+        this.ended = false;
+        // `ended` fires when the user stops sharing from the browser's bar or the surface closes.
+        Track.addEventListener('ended', () => this.end());
+    }
+    get Surface() {
+        return CapturedSurfaceOf(this.Track);
+    }
+    get Label() {
+        return this.Track.label;
+    }
+    get PanelMethod() {
+        return this.panel?.Method;
+    }
+    get PanelLabel() {
+        return this.panel?.Label;
+    }
+    OnEnded(handler) {
+        if (this.ended) {
+            handler();
+            return () => undefined;
+        }
+        this.endedHandlers.add(handler);
+        return () => this.endedHandlers.delete(handler);
+    }
+    Stop() {
+        stopTracks(this.Stream);
+        // A track stopped by the page fires no `ended` event, so report the end here.
+        this.end();
+    }
+    end() {
+        if (this.ended) {
+            return;
+        }
+        this.ended = true;
+        // The panel's own style comes back before anyone hears that the share ended.
+        this.panel?.Release();
+        for (const handler of [...this.endedHandlers]) {
+            handler();
+        }
+    }
+}
+//# sourceMappingURL=displayCapture.js.map

package/dist/media/displayCapture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"displayCapture.js","sourceRoot":"","sources":["../../src/media/displayCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AA8HH,4DAA4D;AAC5D,MAAM,cAAc,GAA0C,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC;AAEtH,+GAA+G;AAC/G,MAAM,qBAAqB,GAAG,kEAAkE,CAAC;AAEjG,qCAAqC;AACrC,MAAM,UAAU,wBAAwB;IACpC,MAAM,KAAK,GAAG,UAA0B,CAAC;IACzC,OAAO;QACH,OAAO,EAAE,OAAO,SAAS,KAAK,WAAW,IAAI,OAAO,SAAS,CAAC,YAAY,EAAE,eAAe,KAAK,UAAU;QAC1G,cAAc,EAAE,OAAO,KAAK,CAAC,iBAAiB,EAAE,WAAW,KAAK,UAAU;QAC1E,aAAa,EAAE,OAAO,KAAK,CAAC,UAAU,EAAE,WAAW,KAAK,UAAU;KACrE,CAAC;AACN,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,UAAiC,EAAE;IAC3E,MAAM,OAAO,GAAG,wBAAwB,EAAE,CAAC;IAC3C,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC,aAAa,EAAE,qCAAqC,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;QACrE,OAAO,OAAO,CAAC,mBAAmB,EAAE,gFAAgF,CAAC,CAAC;IAC1H,CAAC;IACD,IAAI,MAAmB,CAAC;IACxB,IAAI,CAAC;QACD,MAAM,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,eAAe,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;IAClF,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACX,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,CAAC,KAAK,EAAE,CAAC;QACT,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,OAAO,CAAC,OAAO,EAAE,uCAAuC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;IACnF,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACpE,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QACvB,UAAU,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,MAAM,KAAK,GAAe,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;IACrG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,oBAAoB,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,CAAC;AAC1F,CAAC;AAED,mHAAmH;AACnH,SAAS,aAAa,CAAC,OAA8B;IACjD,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,IAAI,KAAK,CAAC;IAC5C,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,OAAO;YACH,KAAK,EAAE,EAAE,cAAc,EAAE,cAAc,CAAC,GAAG,EAAE;YAC7C,KAAK;YACL,gBAAgB,EAAE,IAAI;YACtB,kBAAkB,EAAE,SAAS;YAC7B,uEAAuE;YACvE,gBAAgB,EAAE,SAAS;SAC9B,CAAC;IACN,CAAC;IACD,OAAO;QACH,KAAK,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,cAAc,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI;QACrG,KAAK;QACL,gFAAgF;QAChF,kBAAkB,EAAE,SAAS;QAC7B,gBAAgB,EAAE,SAAS;QAC3B,mBAAmB,EAAE,SAAS;QAC9B,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;KAC7C,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,aAAa,CACxB,KAAuB,EACvB,KAAc,EACd,OAA8B;IAE9B,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,cAAc,KAAK,cAAc,CAAC,GAAG,EAAE,CAAC;QAC5D,OAAO,OAAO,CAAC,qBAAqB,EAAE,GAAG,qBAAqB,GAAG,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,KAAK,GAAG,UAA0B,CAAC;IACzC,MAAM,UAAU,GAAG,KAAwB,CAAC;IAC5C,IAAI,OAAO,CAAC,cAAc,IAAI,KAAK,CAAC,iBAAiB,IAAI,UAAU,CAAC,UAAU,EAAE,CAAC;QAC7E,6EAA6E;QAC7E,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;QACpC,IAAI,CAAC;YACD,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YAC9E,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;QACnD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACxC,IAAI,CAAC;YACD,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACnE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;QAC1D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC,mBAAmB,EAAE,uDAAuD,CAAC,CAAC;AACjG,CAAC;AAED,uEAAuE;AACvE,SAAS,YAAY,CAAC,GAAY;IAC9B,OAAO,OAAO,CAAC,qBAAqB,EAAE,GAAG,qBAAqB,KAAK,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CAAC,KAAc;IAChC,MAAM,KAAK,GAAI,KAAmD,CAAC,KAAK,CAAC;IACzE,IAAI,CAAC,KAAK,IAAI,CAAC,OAAO,gBAAgB,KAAK,UAAU,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,EAAE,CAAC;QACxG,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;IAC3B,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,KAAK,CAAC,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACxD,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;IACvD,OAAO,GAAG,EAAE;QACR,IAAI,KAAK,EAAE,CAAC;YACR,KAAK,CAAC,WAAW,CAAC,WAAW,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QACpD,CAAC;aAAM,CAAC;YACJ,KAAK,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;QACtC,CAAC;IACL,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAS,aAAa,CAAC,GAAY;IAC/B,MAAM,IAAI,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAClD,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QAC7B,OAAO,gBAAgB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,OAAO,CAAC,MAA6B,EAAE,OAAe;IAC3D,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAC9B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,UAAU,CAAC,MAAmB;IACnC,MAAM,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;AACxD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAuB;IACrD,QAAQ,KAAK,CAAC,WAAW,EAAE,CAAC,cAAc,EAAE,CAAC;QACzC,KAAK,SAAS;YACV,OAAO,QAAQ,CAAC;QACpB,KAAK,QAAQ;YACT,OAAO,QAAQ,CAAC;QACpB,KAAK,SAAS;YACV,OAAO,KAAK,CAAC;QACjB;YACI,OAAO,SAAS,CAAC;IACzB,CAAC;AACL,CAAC;AAED,2BAA2B;AAC3B,MAAM,oBAAoB;IAItB,YACoB,MAAmB,EACnB,KAAuB,EACtB,KAAkB;QAFnB,WAAM,GAAN,MAAM,CAAa;QACnB,UAAK,GAAL,KAAK,CAAkB;QACtB,UAAK,GAAL,KAAK,CAAa;QANtB,kBAAa,GAAG,IAAI,GAAG,EAAc,CAAC;QAC/C,UAAK,GAAG,KAAK,CAAC;QAOlB,0FAA0F;QAC1F,KAAK,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,IAAW,OAAO;QACd,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC;IAC5B,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC;IAC9B,CAAC;IAED,IAAW,UAAU;QACjB,OAAO,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC;IAC7B,CAAC;IAEM,OAAO,CAAC,OAAmB;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,OAAO,GAAG,EAAE,CAAC,SAAS,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC;IAEM,IAAI;QACP,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxB,8EAA8E;QAC9E,IAAI,CAAC,GAAG,EAAE,CAAC;IACf,CAAC;IAEO,GAAG;QACP,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO;QACX,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,6EAA6E;QAC7E,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;QACtB,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,CAAC;QACd,CAAC;IACL,CAAC;CACJ"}

package/dist/media/frameCapture.d.ts

@@ -1,12 +1,22 @@
 /**
  * @fileoverview FRAME CAPTURE for realtime video tracks — camera and screen capture pipelines.
  *
- * Patterned beside `micCapture.ts`:
- * - `CreateCameraCapture`: prompts via `getUserMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateScreenCapture`: prompts via `getDisplayMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateStreamFrameCapture`: extracts periodic JPEG frames from an already-acquired `MediaStream`.
+ * Patterned beside `micCapture.ts`. Every exported function here is deprecated; new code uses the `/media` modules:
+ * - `CreateCameraCapture` (deprecated): prompts via `getUserMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code starts the camera with `LocalMediaController` and samples
+ *   its stream with {@link FrameSampler}.
+ * - `CreateScreenCapture` (deprecated): prompts via `getDisplayMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code shares with `RequestDisplayCapture` and samples with
+ *   {@link FrameSampler}.
+ * - `CreateStreamFrameCapture` (deprecated): extracts periodic JPEG frames from an already-acquired
+ *   `MediaStream`. It is now a thin wrapper over {@link FrameSampler}, which new code uses directly.
  *
- * Cadence is throttled to at most 1 frame per second (1 fps ceiling per Live API capabilities guide).
+ * The two capture helpers keep their own `getUserMedia` and `getDisplayMedia` calls rather than wrapping the new
+ * modules: those differ in what they offer (the screen picker leaves out the current tab, for one) and in how they
+ * report a failure, so a caller of a helper keeps the behaviour it was written against.
+ *
+ * Capture runs at the device's native frame rate; only the sampling is paced, at the rate the caller passes
+ * (the negotiated track rate). There is no 1 fps ceiling here.
  *
  * @module @memberjunction/ai-realtime-client
  * @author MemberJunction.com
@@ -33,8 +43,8 @@ export interface FrameCaptureOptions {
      */
     ConsentGranted?: boolean;
     /**
-     * Capture cadence in frames per second (fps).
-     * Capped at 1 fps maximum (Gemini Live ceiling). Defaults to `Descriptor?.Rate ?? 1`.
+     * Frames per second to sample. No ceiling: pass the negotiated track rate. Defaults to
+     * `Descriptor?.Rate ?? 1`.
      */
     Rate?: number;
     /** Output image format ('image/jpeg' or 'image/png'). Defaults to 'image/jpeg'. */
@@ -50,6 +60,8 @@ export interface FrameCaptureOptions {
 /**
  * Creates a frame capture pump over an already-acquired {@link MediaStream}.
  *
+ * @deprecated Use {@link FrameSampler}, which reports whether it started and returns frame sizes and
+ *   timestamps. This wrapper passes `Rate` through unchanged; it no longer caps it at 1 fps.
  * @param stream The media stream containing one or more video tracks.
  * @param options Frame capture options (cadence, format, frame callback).
  */
@@ -58,6 +70,9 @@ export declare function CreateStreamFrameCapture(stream: MediaStream, options: F
  * Requests user camera video capture via `navigator.mediaDevices.getUserMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Start the camera with `LocalMediaController` (in `localMediaController.ts`), which lists and switches
+ *   devices and reports failures as a state, then sample its stream (`GetStream('camera')`) with a
+ *   {@link FrameSampler}.
  * @param options Frame capture options.
  * @param constraints Video track constraints.
  */
@@ -66,11 +81,13 @@ export declare function CreateCameraCapture(options: FrameCaptureOptions, constr
  * Requests screen share video capture via `navigator.mediaDevices.getDisplayMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Use `RequestDisplayCapture` (in `displayCapture.ts`), which offers surface hints and panel
+ *   capture and reports what was shared, then a {@link FrameSampler} on its stream.
  * @param options Frame capture options.
  * @param displayMediaOptions Display media options.
  */
 export declare function CreateScreenCapture(options: FrameCaptureOptions, displayMediaOptions?: DisplayMediaStreamOptions): Promise<IFrameCapture>;
-/** @deprecated Use {@link CreateStreamFrameCapture}. */
+/** @deprecated Use {@link FrameSampler}. */
 export declare function createStreamFrameCapture(stream: MediaStream, options: FrameCaptureOptions): IFrameCapture;
 /** @deprecated Use {@link CreateCameraCapture}. */
 export declare function createCameraCapture(options: FrameCaptureOptions, constraints?: MediaTrackConstraints): Promise<IFrameCapture>;

package/dist/media/frameCapture.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"frameCapture.d.ts","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,yFAAyF;IACzF,IAAI,IAAI,IAAI,CAAC;IACb,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,+CAA+C;IAC/C,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;IACtC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAChE;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAyEf;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAqBxB;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAqBxB;AAED,wDAAwD;AACxD,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAExB;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAExB"}
+{"version":3,"file":"frameCapture.d.ts","sourceRoot":"","sources":["../../src/media/frameCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAGlE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC1B,yFAAyF;IACzF,IAAI,IAAI,IAAI,CAAC;IACb,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAChC,+CAA+C;IAC/C,UAAU,CAAC,EAAE,uBAAuB,CAAC;IACrC;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,mFAAmF;IACnF,QAAQ,CAAC,EAAE,YAAY,GAAG,WAAW,CAAC;IACtC,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qEAAqE;IACrE,OAAO,EAAE,CAAC,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CAChE;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAmBD;;;;;;;;;GASG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAsBxB;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAoBxB;AAED,4CAA4C;AAC5C,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,WAAW,EACnB,OAAO,EAAE,mBAAmB,GAC7B,aAAa,CAEf;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,WAAW,CAAC,EAAE,qBAAqB,GACpC,OAAO,CAAC,aAAa,CAAC,CAExB;AAED,mDAAmD;AACnD,wBAAsB,mBAAmB,CACrC,OAAO,EAAE,mBAAmB,EAC5B,mBAAmB,CAAC,EAAE,yBAAyB,GAChD,OAAO,CAAC,aAAa,CAAC,CAExB"}

package/dist/media/frameCapture.js

@@ -1,86 +1,49 @@
 /**
  * @fileoverview FRAME CAPTURE for realtime video tracks — camera and screen capture pipelines.
  *
- * Patterned beside `micCapture.ts`:
- * - `CreateCameraCapture`: prompts via `getUserMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateScreenCapture`: prompts via `getDisplayMedia` (consent-gated via {@link RealtimeTrackDescriptor.RequiresConsent}).
- * - `CreateStreamFrameCapture`: extracts periodic JPEG frames from an already-acquired `MediaStream`.
+ * Patterned beside `micCapture.ts`. Every exported function here is deprecated; new code uses the `/media` modules:
+ * - `CreateCameraCapture` (deprecated): prompts via `getUserMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code starts the camera with `LocalMediaController` and samples
+ *   its stream with {@link FrameSampler}.
+ * - `CreateScreenCapture` (deprecated): prompts via `getDisplayMedia` (consent-gated via
+ *   {@link RealtimeTrackDescriptor.RequiresConsent}). New code shares with `RequestDisplayCapture` and samples with
+ *   {@link FrameSampler}.
+ * - `CreateStreamFrameCapture` (deprecated): extracts periodic JPEG frames from an already-acquired
+ *   `MediaStream`. It is now a thin wrapper over {@link FrameSampler}, which new code uses directly.
  *
- * Cadence is throttled to at most 1 frame per second (1 fps ceiling per Live API capabilities guide).
+ * The two capture helpers keep their own `getUserMedia` and `getDisplayMedia` calls rather than wrapping the new
+ * modules: those differ in what they offer (the screen picker leaves out the current tab, for one) and in how they
+ * report a failure, so a caller of a helper keeps the behaviour it was written against.
+ *
+ * Capture runs at the device's native frame rate; only the sampling is paced, at the rate the caller passes
+ * (the negotiated track rate). There is no 1 fps ceiling here.
  *
  * @module @memberjunction/ai-realtime-client
  * @author MemberJunction.com
  */
+import { FrameSampler } from './frameSampler.js';
 /**
  * Creates a frame capture pump over an already-acquired {@link MediaStream}.
  *
+ * @deprecated Use {@link FrameSampler}, which reports whether it started and returns frame sizes and
+ *   timestamps. This wrapper passes `Rate` through unchanged; it no longer caps it at 1 fps.
  * @param stream The media stream containing one or more video tracks.
  * @param options Frame capture options (cadence, format, frame callback).
  */
 export function CreateStreamFrameCapture(stream, options) {
-    const rawRate = options.Rate ?? options.Descriptor?.Rate ?? 1;
-    // Cadence ceiling: max 1 fps per Gemini Live API specs; minimum 0.1 fps
-    const rate = Math.min(Math.max(rawRate, 0.1), 1);
-    const intervalMs = Math.floor(1000 / rate);
-    const mimeType = options.MimeType ?? 'image/jpeg';
-    const quality = options.Quality ?? 0.8;
-    let stopped = false;
-    let timer = null;
-    let videoEl = null;
-    let canvasEl = null;
-    if (typeof document !== 'undefined' && typeof document.createElement === 'function') {
-        try {
-            videoEl = document.createElement('video');
-            videoEl.muted = true;
-            videoEl.playsInline = true;
-            videoEl.srcObject = stream;
-            void videoEl.play().catch(() => {
-                // Autoplay may be restricted without user interaction; non-fatal for capture
-            });
-            canvasEl = document.createElement('canvas');
-        }
-        catch (err) {
-            console.error('[FrameCapture] Failed to initialize DOM elements for video capture:', err);
-        }
-    }
-    const captureTick = () => {
-        if (stopped) {
-            return;
-        }
-        if (videoEl && canvasEl && videoEl.videoWidth > 0 && videoEl.videoHeight > 0) {
-            canvasEl.width = videoEl.videoWidth;
-            canvasEl.height = videoEl.videoHeight;
-            const ctx = canvasEl.getContext('2d');
-            if (ctx) {
-                ctx.drawImage(videoEl, 0, 0);
-                const dataUrl = canvasEl.toDataURL(mimeType, quality);
-                const base64 = dataUrl.split(',')[1] ?? '';
-                if (base64.length > 0) {
-                    options.OnFrame({ data: base64, mimeType });
-                }
-            }
-        }
-    };
-    if (videoEl && canvasEl) {
-        timer = setInterval(captureTick, intervalMs);
-    }
-    else {
-        console.warn('[FrameCapture] No video/canvas element available (no DOM, or element construction failed) — ' +
-            'this capture will emit no frames.');
-    }
+    return startSampler(stream, options);
+}
+/** Samples a stream with a {@link FrameSampler}, behind the older {@link IFrameCapture} handle. */
+function startSampler(stream, options) {
+    const sampler = new FrameSampler(stream, {
+        Rate: options.Rate ?? options.Descriptor?.Rate ?? 1,
+        MimeType: options.MimeType,
+        Quality: options.Quality,
+        OnFrame: (frame) => options.OnFrame({ data: frame.Data, mimeType: frame.MimeType }),
+    });
+    sampler.Start();
     return {
-        Stop: () => {
-            stopped = true;
-            if (timer) {
-                clearInterval(timer);
-                timer = null;
-            }
-            if (videoEl) {
-                videoEl.srcObject = null;
-                videoEl = null;
-            }
-            canvasEl = null;
-        },
+        Stop: () => sampler.Stop(),
         get Stream() {
             return stream;
         },
@@ -90,6 +53,9 @@ export function CreateStreamFrameCapture(stream, options) {
  * Requests user camera video capture via `navigator.mediaDevices.getUserMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Start the camera with `LocalMediaController` (in `localMediaController.ts`), which lists and switches
+ *   devices and reports failures as a state, then sample its stream (`GetStream('camera')`) with a
+ *   {@link FrameSampler}.
  * @param options Frame capture options.
  * @param constraints Video track constraints.
  */
@@ -101,10 +67,11 @@ export async function CreateCameraCapture(options, constraints) {
     if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getUserMedia) {
         throw new Error('Camera capture is unavailable: navigator.mediaDevices.getUserMedia not supported in this environment.');
     }
+    // No frameRate cap: the camera runs at its native rate (a smooth self-view); only sampling is paced.
     const stream = await navigator.mediaDevices.getUserMedia({
-        video: constraints ?? { width: { ideal: 1280 }, height: { ideal: 720 }, frameRate: { max: 1 } },
+        video: constraints ?? { width: { ideal: 1280 }, height: { ideal: 720 } },
     });
-    const capture = CreateStreamFrameCapture(stream, options);
+    const capture = startSampler(stream, options);
     return {
         Stop: () => {
             capture.Stop();
@@ -119,6 +86,8 @@ export async function CreateCameraCapture(options, constraints) {
  * Requests screen share video capture via `navigator.mediaDevices.getDisplayMedia`.
  * Gated on {@link FrameCaptureOptions.ConsentGranted} when consent is required.
  *
+ * @deprecated Use `RequestDisplayCapture` (in `displayCapture.ts`), which offers surface hints and panel
+ *   capture and reports what was shared, then a {@link FrameSampler} on its stream.
  * @param options Frame capture options.
  * @param displayMediaOptions Display media options.
  */
@@ -130,8 +99,9 @@ export async function CreateScreenCapture(options, displayMediaOptions) {
     if (typeof navigator === 'undefined' || !navigator.mediaDevices?.getDisplayMedia) {
         throw new Error('Screen capture is unavailable: navigator.mediaDevices.getDisplayMedia not supported in this environment.');
     }
-    const stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions ?? { video: { frameRate: { max: 1 } } });
-    const capture = CreateStreamFrameCapture(stream, options);
+    // No frameRate cap: the share runs at its native rate; only sampling is paced.
+    const stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions ?? { video: true });
+    const capture = startSampler(stream, options);
     return {
         Stop: () => {
             capture.Stop();
@@ -142,7 +112,7 @@ export async function CreateScreenCapture(options, displayMediaOptions) {
         },
     };
 }
-/** @deprecated Use {@link CreateStreamFrameCapture}. */
+/** @deprecated Use {@link FrameSampler}. */
 export function createStreamFrameCapture(stream, options) {
     return CreateStreamFrameCapture(stream, options);
 }