@memberjunction/ng-conversations 5.40.1 → 5.41.0

package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js

@@ -0,0 +1,309 @@
+/**
+ * The CLIENT-SIDE player for the Remote Browser tab-audio stream — the audio sibling of the surface's
+ * screencast canvas. The channel feeds it each pushed {@link RemoteBrowserAudioChunkInput} (a base64
+ * `audio/webm;codecs=opus` slice); the player appends them to a `MediaSource` `SourceBuffer` feeding a
+ * hidden `<audio>` element so the user HEARS what the co-agent is playing in the browser.
+ *
+ * The append/queue/seq logic is split from the DOM so it's unit-testable without a browser:
+ *  - {@link IAudioSink} is the thin DOM seam (create/append/error/muted) — faked in tests, wired to a real
+ *    `MediaSource` + `<audio>` in {@link MediaSourceAudioSink}.
+ *  - {@link RemoteBrowserAudioPlayer} owns the queue, sequential append (a `SourceBuffer` can only take one
+ *    append at a time), overflow drop, and sequence-gap resync — all framework- and DOM-free.
+ *
+ * Codec note: only `'webm-opus'` chunks are appendable to the `audio/webm;codecs=opus` SourceBuffer; other
+ * codecs (reserved for future backend capture paths) are dropped with a one-time warning.
+ */
+/** Max chunks buffered ahead of the sink before the OLDEST are dropped (keeps latency + memory bounded). */
+const MAX_QUEUE_DEPTH = 64;
+/**
+ * Sequential, overflow-bounded player for a continuous webm-opus chunk stream. Construct it with an
+ * {@link IAudioSink}; call {@link Enqueue} for each pushed chunk and {@link SetMuted} for the speaker
+ * toggle; call {@link Dispose} on teardown.
+ */
+export class RemoteBrowserAudioPlayer {
+    /** The DOM (or fake) sink chunks are appended to. */
+    sink;
+    /** Pending decoded chunks awaiting append, oldest first. Bounded by {@link MAX_QUEUE_DEPTH}. */
+    queue = [];
+    /** True while an append is in flight (a SourceBuffer accepts one append at a time). */
+    appending = false;
+    /** The sequence number of the last chunk we ACCEPTED, or `null` before the first. Drives gap detection. */
+    lastSeq = null;
+    /** True once {@link Open} has been kicked off, so we don't re-open on every chunk. */
+    opening = false;
+    /** True after {@link Dispose} so a late append / open resolution becomes a no-op. */
+    disposed = false;
+    /** Pending muted state to apply once the sink opens (the toggle can fire before the first chunk). */
+    muted = false;
+    /** Set once we've warned about an unplayable codec, so the log isn't spammed per chunk. */
+    warnedUnplayableCodec = false;
+    /**
+     * Constructs a player over the given sink.
+     *
+     * @param sink The audio sink (real {@link MediaSourceAudioSink} in the app, a fake in tests).
+     */
+    constructor(sink) {
+        this.sink = sink;
+    }
+    /** The current queue depth — exposed for tests / diagnostics. */
+    get QueueDepth() {
+        return this.queue.length;
+    }
+    /**
+     * Enqueues one pushed audio chunk for playback. Drops chunks whose codec isn't playable, opens the sink
+     * lazily on the first playable chunk, detects sequence gaps (logged; the webm stream self-recovers on the
+     * next keyframe), and bounds the queue by dropping the oldest on overflow. Never throws.
+     *
+     * @param chunk The pushed chunk to play.
+     */
+    Enqueue(chunk) {
+        if (this.disposed || !chunk || typeof chunk.dataBase64 !== 'string' || chunk.dataBase64.length === 0) {
+            return;
+        }
+        if (chunk.codec !== 'webm-opus') {
+            if (!this.warnedUnplayableCodec) {
+                this.warnedUnplayableCodec = true;
+                console.warn(`[RemoteBrowserAudio] dropping chunk with unplayable codec '${chunk.codec}' (only webm-opus is supported).`);
+            }
+            return;
+        }
+        this.noteSequence(chunk.seq);
+        const bytes = this.decodeBase64(chunk.dataBase64);
+        if (!bytes) {
+            return;
+        }
+        this.pushBounded(bytes);
+        void this.ensureOpenThenDrain();
+    }
+    /**
+     * Mutes / unmutes playback (the speaker toggle). Safe to call before the sink opens — the state is
+     * applied once it does.
+     *
+     * @param muted `true` to mute, `false` to unmute.
+     */
+    SetMuted(muted) {
+        this.muted = muted;
+        if (this.sink.IsOpen) {
+            this.sink.SetMuted(muted);
+        }
+    }
+    /** Tears the player down: clears the queue and closes the sink. Idempotent. */
+    Dispose() {
+        this.disposed = true;
+        this.queue.length = 0;
+        this.sink.Close();
+    }
+    /**
+     * Records a chunk's sequence number and logs a one-line note when a gap is detected (a non-consecutive
+     * seq). We don't try to reorder — webm-opus self-recovers at the next cluster — but the note aids
+     * diagnosis of a lossy transport.
+     *
+     * @param seq The incoming chunk's sequence number.
+     */
+    noteSequence(seq) {
+        if (this.lastSeq !== null && seq > this.lastSeq + 1) {
+            console.warn(`[RemoteBrowserAudio] audio sequence gap: expected ${this.lastSeq + 1}, got ${seq} (dropped ${seq - this.lastSeq - 1}).`);
+        }
+        // Track the highest seq seen so an out-of-order late chunk doesn't reset the expectation backwards.
+        this.lastSeq = this.lastSeq === null ? seq : Math.max(this.lastSeq, seq);
+    }
+    /** Pushes bytes onto the queue, dropping the OLDEST when the queue is at {@link MAX_QUEUE_DEPTH}. */
+    pushBounded(bytes) {
+        if (this.queue.length >= MAX_QUEUE_DEPTH) {
+            this.queue.shift(); // drop-oldest: bound latency + memory under a slow sink
+        }
+        this.queue.push(bytes);
+    }
+    /**
+     * Ensures the sink is open (opening it lazily on first use) and then drains the queue. The open is
+     * kicked off once; subsequent calls just drain. Best-effort — an open/append failure is logged and the
+     * player keeps trying on the next chunk.
+     */
+    async ensureOpenThenDrain() {
+        if (!this.sink.IsOpen && !this.opening) {
+            this.opening = true;
+            try {
+                await this.sink.Open();
+                if (this.disposed) {
+                    this.sink.Close();
+                    return;
+                }
+                this.sink.SetMuted(this.muted);
+            }
+            catch (err) {
+                this.opening = false;
+                console.warn(`[RemoteBrowserAudio] failed to open audio sink: ${err instanceof Error ? err.message : String(err)}`);
+                return;
+            }
+        }
+        void this.drain();
+    }
+    /**
+     * Appends queued chunks to the sink one at a time (a SourceBuffer rejects concurrent appends). Re-entrant
+     * safe via the {@link appending} guard; stops when the queue empties, the sink closes, or on disposal.
+     */
+    async drain() {
+        if (this.appending || this.disposed || !this.sink.IsOpen) {
+            return;
+        }
+        this.appending = true;
+        try {
+            while (this.queue.length > 0 && this.sink.IsOpen && !this.disposed) {
+                const bytes = this.queue.shift();
+                try {
+                    await this.sink.AppendChunk(bytes);
+                }
+                catch (err) {
+                    // A single rejected append (e.g. a quota hiccup) shouldn't kill playback — drop it and continue.
+                    console.warn(`[RemoteBrowserAudio] dropped an audio chunk on append: ${err instanceof Error ? err.message : String(err)}`);
+                }
+            }
+        }
+        finally {
+            this.appending = false;
+        }
+    }
+    /**
+     * Decodes a base64 string to bytes WITHOUT the DOM `atob` so the queueing logic is testable in Node too.
+     * Returns `null` on malformed input (the chunk is then skipped).
+     *
+     * @param base64 The base64 string to decode.
+     * @returns The decoded bytes, or `null` when the input can't be decoded.
+     */
+    decodeBase64(base64) {
+        try {
+            // Prefer the platform decoder when present (browser `atob`), else Node's Buffer.
+            const globalAtob = globalThis.atob;
+            if (typeof globalAtob === 'function') {
+                const binary = globalAtob(base64);
+                const bytes = new Uint8Array(binary.length);
+                for (let i = 0; i < binary.length; i++) {
+                    bytes[i] = binary.charCodeAt(i);
+                }
+                return bytes;
+            }
+            const nodeBuffer = globalThis.Buffer;
+            if (nodeBuffer) {
+                return Uint8Array.from(nodeBuffer.from(base64, 'base64'));
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+}
+/**
+ * The real, DOM-backed {@link IAudioSink}: a `MediaSource` whose single `audio/webm;codecs=opus`
+ * `SourceBuffer` is fed appended chunks, attached to a hidden `<audio>` element that auto-plays. Autoplay
+ * is permitted because the realtime call the user already started is the activating gesture.
+ *
+ * Kept out of the player's hot path so the queueing logic stays unit-testable; this class is exercised in
+ * the live app (it can't be meaningfully unit-tested without a browser media stack).
+ */
+export class MediaSourceAudioSink {
+    /** MIME the SourceBuffer is created with — the in-page recorder's output container/codec. */
+    static MIME = 'audio/webm;codecs=opus';
+    /** The hidden audio element playback is routed through; `null` before {@link Open} / after {@link Close}. */
+    audio = null;
+    /** The MediaSource backing the audio element; `null` before {@link Open} / after {@link Close}. */
+    mediaSource = null;
+    /** The single source buffer chunks are appended to; `null` until `sourceopen` wires it. */
+    sourceBuffer = null;
+    /** Object URL bound to the audio element's `src`, revoked on {@link Close}. */
+    objectUrl = null;
+    /** Latched muted state applied to the audio element as it (re)opens. */
+    mutedState = false;
+    /** @inheritdoc */
+    get IsOpen() {
+        return this.sourceBuffer !== null && this.mediaSource?.readyState === 'open';
+    }
+    /** @inheritdoc */
+    async Open() {
+        if (this.audio) {
+            return; // already open / opening
+        }
+        if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(MediaSourceAudioSink.MIME)) {
+            throw new Error(`MediaSource webm-opus playback is not supported in this browser.`);
+        }
+        const audio = new Audio();
+        audio.autoplay = true;
+        audio.muted = this.mutedState;
+        const mediaSource = new MediaSource();
+        const objectUrl = URL.createObjectURL(mediaSource);
+        audio.src = objectUrl;
+        this.audio = audio;
+        this.mediaSource = mediaSource;
+        this.objectUrl = objectUrl;
+        await new Promise((resolve, reject) => {
+            mediaSource.addEventListener('sourceopen', () => {
+                try {
+                    this.sourceBuffer = mediaSource.addSourceBuffer(MediaSourceAudioSink.MIME);
+                    this.sourceBuffer.mode = 'sequence';
+                    resolve();
+                }
+                catch (err) {
+                    reject(err instanceof Error ? err : new Error(String(err)));
+                }
+            }, { once: true });
+        });
+        // Best-effort start — autoplay is allowed (the call is the user gesture); ignore a rejected play().
+        void audio.play().catch(() => undefined);
+    }
+    /** @inheritdoc */
+    async AppendChunk(bytes) {
+        const buffer = this.sourceBuffer;
+        if (!buffer || !this.IsOpen) {
+            throw new Error('Audio sink is not open.');
+        }
+        await new Promise((resolve, reject) => {
+            const onUpdateEnd = () => {
+                buffer.removeEventListener('updateend', onUpdateEnd);
+                buffer.removeEventListener('error', onError);
+                resolve();
+            };
+            const onError = () => {
+                buffer.removeEventListener('updateend', onUpdateEnd);
+                buffer.removeEventListener('error', onError);
+                reject(new Error('SourceBuffer append error.'));
+            };
+            buffer.addEventListener('updateend', onUpdateEnd);
+            buffer.addEventListener('error', onError);
+            try {
+                // Copy into a fresh, definitely-non-shared ArrayBuffer so the typed `BufferSource` signature is
+                // satisfied across TS lib versions (a plain Uint8Array may be typed as ArrayBufferLike).
+                const copy = new Uint8Array(bytes.length);
+                copy.set(bytes);
+                buffer.appendBuffer(copy);
+            }
+            catch (err) {
+                buffer.removeEventListener('updateend', onUpdateEnd);
+                buffer.removeEventListener('error', onError);
+                reject(err instanceof Error ? err : new Error(String(err)));
+            }
+        });
+    }
+    /** @inheritdoc */
+    SetMuted(muted) {
+        this.mutedState = muted;
+        if (this.audio) {
+            this.audio.muted = muted;
+        }
+    }
+    /** @inheritdoc */
+    Close() {
+        if (this.audio) {
+            this.audio.pause();
+            this.audio.removeAttribute('src');
+            this.audio.load();
+            this.audio = null;
+        }
+        if (this.objectUrl) {
+            URL.revokeObjectURL(this.objectUrl);
+            this.objectUrl = null;
+        }
+        this.sourceBuffer = null;
+        this.mediaSource = null;
+    }
+}
+//# sourceMappingURL=remote-browser-audio-player.js.map

package/dist/lib/components/realtime/remote-browser/remote-browser-audio-player.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-browser-audio-player.js","sourceRoot":"","sources":["../../../../../src/lib/components/realtime/remote-browser/remote-browser-audio-player.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AA8CH,4GAA4G;AAC5G,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B;;;;GAIG;AACH,MAAM,OAAO,wBAAwB;IACnC,qDAAqD;IACpC,IAAI,CAAa;IAElC,gGAAgG;IAC/E,KAAK,GAAiB,EAAE,CAAC;IAE1C,uFAAuF;IAC/E,SAAS,GAAG,KAAK,CAAC;IAE1B,2GAA2G;IACnG,OAAO,GAAkB,IAAI,CAAC;IAEtC,sFAAsF;IAC9E,OAAO,GAAG,KAAK,CAAC;IAExB,qFAAqF;IAC7E,QAAQ,GAAG,KAAK,CAAC;IAEzB,qGAAqG;IAC7F,KAAK,GAAG,KAAK,CAAC;IAEtB,2FAA2F;IACnF,qBAAqB,GAAG,KAAK,CAAC;IAEtC;;;;OAIG;IACH,YAAY,IAAgB;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,iEAAiE;IACjE,IAAW,UAAU;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;;;;;OAMG;IACI,OAAO,CAAC,KAAmC;QAChD,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrG,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAChC,IAAI,CAAC,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAChC,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,8DAA8D,KAAK,CAAC,KAAK,kCAAkC,CAAC,CAAC;YAC5H,CAAC;YACD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;QACT,CAAC;QACD,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxB,KAAK,IAAI,CAAC,mBAAmB,EAAE,CAAC;IAClC,CAAC;IAED;;;;;OAKG;IACI,QAAQ,CAAC,KAAc;QAC5B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,+EAA+E;IACxE,OAAO;QACZ,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED;;;;;;OAMG;IACK,YAAY,CAAC,GAAW;QAC9B,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,qDAAqD,IAAI,CAAC,OAAO,GAAG,CAAC,SAAS,GAAG,aAAa,GAAG,GAAG,IAAI,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC;QACzI,CAAC;QACD,oGAAoG;QACpG,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC3E,CAAC;IAED,qGAAqG;IAC7F,WAAW,CAAC,KAAiB;QACnC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,eAAe,EAAE,CAAC;YACzC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,wDAAwD;QAC9E,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,mBAAmB;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;oBAClB,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,mDAAmD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpH,OAAO;YACT,CAAC;QACH,CAAC;QACD,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACnE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAG,CAAC;gBAClC,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;gBACrC,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,iGAAiG;oBACjG,OAAO,CAAC,IAAI,CAAC,0DAA0D,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC7H,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACK,YAAY,CAAC,MAAc;QACjC,IAAI,CAAC;YACH,iFAAiF;YACjF,MAAM,UAAU,GAAI,UAA+C,CAAC,IAAI,CAAC;YACzE,IAAI,OAAO,UAAU,KAAK,UAAU,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;gBAClC,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACvC,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;gBAClC,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,MAAM,UAAU,GAAI,UAAwE,CAAC,MAAM,CAAC;YACpG,IAAI,UAAU,EAAE,CAAC;gBACf,OAAO,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,OAAO,oBAAoB;IAC/B,6FAA6F;IACrF,MAAM,CAAU,IAAI,GAAG,wBAAwB,CAAC;IAExD,6GAA6G;IACrG,KAAK,GAA4B,IAAI,CAAC;IAC9C,mGAAmG;IAC3F,WAAW,GAAuB,IAAI,CAAC;IAC/C,2FAA2F;IACnF,YAAY,GAAwB,IAAI,CAAC;IACjD,+EAA+E;IACvE,SAAS,GAAkB,IAAI,CAAC;IACxC,wEAAwE;IAChE,UAAU,GAAG,KAAK,CAAC;IAE3B,kBAAkB;IAClB,IAAW,MAAM;QACf,OAAO,IAAI,CAAC,YAAY,KAAK,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,UAAU,KAAK,MAAM,CAAC;IAC/E,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,IAAI;QACf,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,yBAAyB;QACnC,CAAC;QACD,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,EAAE,CAAC;YAClG,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC;QAC1B,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;QACtB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9B,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QACnD,KAAK,CAAC,GAAG,GAAG,SAAS,CAAC;QAEtB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,WAAW,CAAC,gBAAgB,CAC1B,YAAY,EACZ,GAAG,EAAE;gBACH,IAAI,CAAC;oBACH,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;oBAC3E,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,UAAU,CAAC;oBACpC,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC9D,CAAC;YACH,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;QACJ,CAAC,CAAC,CAAC;QACH,oGAAoG;QACpG,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,kBAAkB;IACX,KAAK,CAAC,WAAW,CAAC,KAAiB;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC;QACjC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,WAAW,GAAG,GAAS,EAAE;gBAC7B,MAAM,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;gBACrD,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,MAAM,OAAO,GAAG,GAAS,EAAE;gBACzB,MAAM,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;gBACrD,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,CAAC,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC,CAAC;YAClD,CAAC,CAAC;YACF,MAAM,CAAC,gBAAgB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YAClD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC;gBACH,gGAAgG;gBAChG,yFAAyF;gBACzF,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC1C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAChB,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,CAAC,mBAAmB,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;gBACrD,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC7C,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,kBAAkB;IACX,QAAQ,CAAC,KAAc;QAC5B,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;QAC3B,CAAC;IACH,CAAC;IAED,kBAAkB;IACX,KAAK;QACV,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QACpB,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC","sourcesContent":["/**\n * The CLIENT-SIDE player for the Remote Browser tab-audio stream — the audio sibling of the surface's\n * screencast canvas. The channel feeds it each pushed {@link RemoteBrowserAudioChunkInput} (a base64\n * `audio/webm;codecs=opus` slice); the player appends them to a `MediaSource` `SourceBuffer` feeding a\n * hidden `<audio>` element so the user HEARS what the co-agent is playing in the browser.\n *\n * The append/queue/seq logic is split from the DOM so it's unit-testable without a browser:\n *  - {@link IAudioSink} is the thin DOM seam (create/append/error/muted) — faked in tests, wired to a real\n *    `MediaSource` + `<audio>` in {@link MediaSourceAudioSink}.\n *  - {@link RemoteBrowserAudioPlayer} owns the queue, sequential append (a `SourceBuffer` can only take one\n *    append at a time), overflow drop, and sequence-gap resync — all framework- and DOM-free.\n *\n * Codec note: only `'webm-opus'` chunks are appendable to the `audio/webm;codecs=opus` SourceBuffer; other\n * codecs (reserved for future backend capture paths) are dropped with a one-time warning.\n */\n\n/** The narrow shape of a pushed audio chunk the player consumes (mirrors the server envelope fields). */\nexport interface RemoteBrowserAudioChunkInput {\n  /** The encoded audio slice as raw base64 (no `data:` prefix). */\n  dataBase64: string;\n  /** The codec / container. Only `'webm-opus'` is currently playable. */\n  codec: string;\n  /** Sample rate in Hz (informational; the webm container is self-describing). */\n  sampleRate: number;\n  /** Channel count (informational). */\n  channels: number;\n  /** Monotonic sequence number for ordering / gap detection. */\n  seq: number;\n}\n\n/**\n * The DOM seam the {@link RemoteBrowserAudioPlayer} drives. A real implementation wires a `MediaSource` +\n * `SourceBuffer` + hidden `<audio>`; tests provide a fake that records appends. Kept deliberately tiny so\n * the player's queueing logic is the part under test, not the browser media stack.\n */\nexport interface IAudioSink {\n  /**\n   * Prepares the sink for playback: creates the media source / source buffer / audio element and resolves\n   * once it is ready to accept {@link AppendChunk}. Idempotent — a second call while ready is a no-op.\n   */\n  Open(): Promise<void>;\n\n  /**\n   * Appends one decoded-container byte slice to the underlying buffer. Rejects if the sink isn't open or\n   * the underlying buffer rejects the append (the player then drops the chunk and continues).\n   *\n   * @param bytes The raw container bytes (e.g. a webm-opus slice) to append.\n   */\n  AppendChunk(bytes: Uint8Array): Promise<void>;\n\n  /** Whether the sink can currently accept an append (open and not mid-error / closed). */\n  readonly IsOpen: boolean;\n\n  /** Mutes or unmutes playback (the speaker toggle). */\n  SetMuted(muted: boolean): void;\n\n  /** Tears the sink down: stops playback and releases the media source / audio element. Idempotent. */\n  Close(): void;\n}\n\n/** Max chunks buffered ahead of the sink before the OLDEST are dropped (keeps latency + memory bounded). */\nconst MAX_QUEUE_DEPTH = 64;\n\n/**\n * Sequential, overflow-bounded player for a continuous webm-opus chunk stream. Construct it with an\n * {@link IAudioSink}; call {@link Enqueue} for each pushed chunk and {@link SetMuted} for the speaker\n * toggle; call {@link Dispose} on teardown.\n */\nexport class RemoteBrowserAudioPlayer {\n  /** The DOM (or fake) sink chunks are appended to. */\n  private readonly sink: IAudioSink;\n\n  /** Pending decoded chunks awaiting append, oldest first. Bounded by {@link MAX_QUEUE_DEPTH}. */\n  private readonly queue: Uint8Array[] = [];\n\n  /** True while an append is in flight (a SourceBuffer accepts one append at a time). */\n  private appending = false;\n\n  /** The sequence number of the last chunk we ACCEPTED, or `null` before the first. Drives gap detection. */\n  private lastSeq: number | null = null;\n\n  /** True once {@link Open} has been kicked off, so we don't re-open on every chunk. */\n  private opening = false;\n\n  /** True after {@link Dispose} so a late append / open resolution becomes a no-op. */\n  private disposed = false;\n\n  /** Pending muted state to apply once the sink opens (the toggle can fire before the first chunk). */\n  private muted = false;\n\n  /** Set once we've warned about an unplayable codec, so the log isn't spammed per chunk. */\n  private warnedUnplayableCodec = false;\n\n  /**\n   * Constructs a player over the given sink.\n   *\n   * @param sink The audio sink (real {@link MediaSourceAudioSink} in the app, a fake in tests).\n   */\n  constructor(sink: IAudioSink) {\n    this.sink = sink;\n  }\n\n  /** The current queue depth — exposed for tests / diagnostics. */\n  public get QueueDepth(): number {\n    return this.queue.length;\n  }\n\n  /**\n   * Enqueues one pushed audio chunk for playback. Drops chunks whose codec isn't playable, opens the sink\n   * lazily on the first playable chunk, detects sequence gaps (logged; the webm stream self-recovers on the\n   * next keyframe), and bounds the queue by dropping the oldest on overflow. Never throws.\n   *\n   * @param chunk The pushed chunk to play.\n   */\n  public Enqueue(chunk: RemoteBrowserAudioChunkInput): void {\n    if (this.disposed || !chunk || typeof chunk.dataBase64 !== 'string' || chunk.dataBase64.length === 0) {\n      return;\n    }\n    if (chunk.codec !== 'webm-opus') {\n      if (!this.warnedUnplayableCodec) {\n        this.warnedUnplayableCodec = true;\n        console.warn(`[RemoteBrowserAudio] dropping chunk with unplayable codec '${chunk.codec}' (only webm-opus is supported).`);\n      }\n      return;\n    }\n    this.noteSequence(chunk.seq);\n\n    const bytes = this.decodeBase64(chunk.dataBase64);\n    if (!bytes) {\n      return;\n    }\n    this.pushBounded(bytes);\n    void this.ensureOpenThenDrain();\n  }\n\n  /**\n   * Mutes / unmutes playback (the speaker toggle). Safe to call before the sink opens — the state is\n   * applied once it does.\n   *\n   * @param muted `true` to mute, `false` to unmute.\n   */\n  public SetMuted(muted: boolean): void {\n    this.muted = muted;\n    if (this.sink.IsOpen) {\n      this.sink.SetMuted(muted);\n    }\n  }\n\n  /** Tears the player down: clears the queue and closes the sink. Idempotent. */\n  public Dispose(): void {\n    this.disposed = true;\n    this.queue.length = 0;\n    this.sink.Close();\n  }\n\n  /**\n   * Records a chunk's sequence number and logs a one-line note when a gap is detected (a non-consecutive\n   * seq). We don't try to reorder — webm-opus self-recovers at the next cluster — but the note aids\n   * diagnosis of a lossy transport.\n   *\n   * @param seq The incoming chunk's sequence number.\n   */\n  private noteSequence(seq: number): void {\n    if (this.lastSeq !== null && seq > this.lastSeq + 1) {\n      console.warn(`[RemoteBrowserAudio] audio sequence gap: expected ${this.lastSeq + 1}, got ${seq} (dropped ${seq - this.lastSeq - 1}).`);\n    }\n    // Track the highest seq seen so an out-of-order late chunk doesn't reset the expectation backwards.\n    this.lastSeq = this.lastSeq === null ? seq : Math.max(this.lastSeq, seq);\n  }\n\n  /** Pushes bytes onto the queue, dropping the OLDEST when the queue is at {@link MAX_QUEUE_DEPTH}. */\n  private pushBounded(bytes: Uint8Array): void {\n    if (this.queue.length >= MAX_QUEUE_DEPTH) {\n      this.queue.shift(); // drop-oldest: bound latency + memory under a slow sink\n    }\n    this.queue.push(bytes);\n  }\n\n  /**\n   * Ensures the sink is open (opening it lazily on first use) and then drains the queue. The open is\n   * kicked off once; subsequent calls just drain. Best-effort — an open/append failure is logged and the\n   * player keeps trying on the next chunk.\n   */\n  private async ensureOpenThenDrain(): Promise<void> {\n    if (!this.sink.IsOpen && !this.opening) {\n      this.opening = true;\n      try {\n        await this.sink.Open();\n        if (this.disposed) {\n          this.sink.Close();\n          return;\n        }\n        this.sink.SetMuted(this.muted);\n      } catch (err) {\n        this.opening = false;\n        console.warn(`[RemoteBrowserAudio] failed to open audio sink: ${err instanceof Error ? err.message : String(err)}`);\n        return;\n      }\n    }\n    void this.drain();\n  }\n\n  /**\n   * Appends queued chunks to the sink one at a time (a SourceBuffer rejects concurrent appends). Re-entrant\n   * safe via the {@link appending} guard; stops when the queue empties, the sink closes, or on disposal.\n   */\n  private async drain(): Promise<void> {\n    if (this.appending || this.disposed || !this.sink.IsOpen) {\n      return;\n    }\n    this.appending = true;\n    try {\n      while (this.queue.length > 0 && this.sink.IsOpen && !this.disposed) {\n        const bytes = this.queue.shift()!;\n        try {\n          await this.sink.AppendChunk(bytes);\n        } catch (err) {\n          // A single rejected append (e.g. a quota hiccup) shouldn't kill playback — drop it and continue.\n          console.warn(`[RemoteBrowserAudio] dropped an audio chunk on append: ${err instanceof Error ? err.message : String(err)}`);\n        }\n      }\n    } finally {\n      this.appending = false;\n    }\n  }\n\n  /**\n   * Decodes a base64 string to bytes WITHOUT the DOM `atob` so the queueing logic is testable in Node too.\n   * Returns `null` on malformed input (the chunk is then skipped).\n   *\n   * @param base64 The base64 string to decode.\n   * @returns The decoded bytes, or `null` when the input can't be decoded.\n   */\n  private decodeBase64(base64: string): Uint8Array | null {\n    try {\n      // Prefer the platform decoder when present (browser `atob`), else Node's Buffer.\n      const globalAtob = (globalThis as { atob?: (s: string) => string }).atob;\n      if (typeof globalAtob === 'function') {\n        const binary = globalAtob(base64);\n        const bytes = new Uint8Array(binary.length);\n        for (let i = 0; i < binary.length; i++) {\n          bytes[i] = binary.charCodeAt(i);\n        }\n        return bytes;\n      }\n      const nodeBuffer = (globalThis as { Buffer?: { from(s: string, enc: string): Uint8Array } }).Buffer;\n      if (nodeBuffer) {\n        return Uint8Array.from(nodeBuffer.from(base64, 'base64'));\n      }\n      return null;\n    } catch {\n      return null;\n    }\n  }\n}\n\n/**\n * The real, DOM-backed {@link IAudioSink}: a `MediaSource` whose single `audio/webm;codecs=opus`\n * `SourceBuffer` is fed appended chunks, attached to a hidden `<audio>` element that auto-plays. Autoplay\n * is permitted because the realtime call the user already started is the activating gesture.\n *\n * Kept out of the player's hot path so the queueing logic stays unit-testable; this class is exercised in\n * the live app (it can't be meaningfully unit-tested without a browser media stack).\n */\nexport class MediaSourceAudioSink implements IAudioSink {\n  /** MIME the SourceBuffer is created with — the in-page recorder's output container/codec. */\n  private static readonly MIME = 'audio/webm;codecs=opus';\n\n  /** The hidden audio element playback is routed through; `null` before {@link Open} / after {@link Close}. */\n  private audio: HTMLAudioElement | null = null;\n  /** The MediaSource backing the audio element; `null` before {@link Open} / after {@link Close}. */\n  private mediaSource: MediaSource | null = null;\n  /** The single source buffer chunks are appended to; `null` until `sourceopen` wires it. */\n  private sourceBuffer: SourceBuffer | null = null;\n  /** Object URL bound to the audio element's `src`, revoked on {@link Close}. */\n  private objectUrl: string | null = null;\n  /** Latched muted state applied to the audio element as it (re)opens. */\n  private mutedState = false;\n\n  /** @inheritdoc */\n  public get IsOpen(): boolean {\n    return this.sourceBuffer !== null && this.mediaSource?.readyState === 'open';\n  }\n\n  /** @inheritdoc */\n  public async Open(): Promise<void> {\n    if (this.audio) {\n      return; // already open / opening\n    }\n    if (typeof MediaSource === 'undefined' || !MediaSource.isTypeSupported(MediaSourceAudioSink.MIME)) {\n      throw new Error(`MediaSource webm-opus playback is not supported in this browser.`);\n    }\n    const audio = new Audio();\n    audio.autoplay = true;\n    audio.muted = this.mutedState;\n    const mediaSource = new MediaSource();\n    const objectUrl = URL.createObjectURL(mediaSource);\n    audio.src = objectUrl;\n\n    this.audio = audio;\n    this.mediaSource = mediaSource;\n    this.objectUrl = objectUrl;\n\n    await new Promise<void>((resolve, reject) => {\n      mediaSource.addEventListener(\n        'sourceopen',\n        () => {\n          try {\n            this.sourceBuffer = mediaSource.addSourceBuffer(MediaSourceAudioSink.MIME);\n            this.sourceBuffer.mode = 'sequence';\n            resolve();\n          } catch (err) {\n            reject(err instanceof Error ? err : new Error(String(err)));\n          }\n        },\n        { once: true },\n      );\n    });\n    // Best-effort start — autoplay is allowed (the call is the user gesture); ignore a rejected play().\n    void audio.play().catch(() => undefined);\n  }\n\n  /** @inheritdoc */\n  public async AppendChunk(bytes: Uint8Array): Promise<void> {\n    const buffer = this.sourceBuffer;\n    if (!buffer || !this.IsOpen) {\n      throw new Error('Audio sink is not open.');\n    }\n    await new Promise<void>((resolve, reject) => {\n      const onUpdateEnd = (): void => {\n        buffer.removeEventListener('updateend', onUpdateEnd);\n        buffer.removeEventListener('error', onError);\n        resolve();\n      };\n      const onError = (): void => {\n        buffer.removeEventListener('updateend', onUpdateEnd);\n        buffer.removeEventListener('error', onError);\n        reject(new Error('SourceBuffer append error.'));\n      };\n      buffer.addEventListener('updateend', onUpdateEnd);\n      buffer.addEventListener('error', onError);\n      try {\n        // Copy into a fresh, definitely-non-shared ArrayBuffer so the typed `BufferSource` signature is\n        // satisfied across TS lib versions (a plain Uint8Array may be typed as ArrayBufferLike).\n        const copy = new Uint8Array(bytes.length);\n        copy.set(bytes);\n        buffer.appendBuffer(copy);\n      } catch (err) {\n        buffer.removeEventListener('updateend', onUpdateEnd);\n        buffer.removeEventListener('error', onError);\n        reject(err instanceof Error ? err : new Error(String(err)));\n      }\n    });\n  }\n\n  /** @inheritdoc */\n  public SetMuted(muted: boolean): void {\n    this.mutedState = muted;\n    if (this.audio) {\n      this.audio.muted = muted;\n    }\n  }\n\n  /** @inheritdoc */\n  public Close(): void {\n    if (this.audio) {\n      this.audio.pause();\n      this.audio.removeAttribute('src');\n      this.audio.load();\n      this.audio = null;\n    }\n    if (this.objectUrl) {\n      URL.revokeObjectURL(this.objectUrl);\n      this.objectUrl = null;\n    }\n    this.sourceBuffer = null;\n    this.mediaSource = null;\n  }\n}\n"]}

package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts

@@ -0,0 +1,168 @@
+import type { Type } from '@angular/core';
+import { RealtimeToolDefinition } from '@memberjunction/ai';
+import { BaseRealtimeChannelClient, ChannelOnboardingDetails } from '../channels/base-realtime-channel-client';
+import { RemoteBrowserSurfaceComponent } from './remote-browser-surface.component';
+import { RemoteBrowserAudioChunkInput } from './remote-browser-audio-player';
+/**
+ * The REMOTE BROWSER as a pluggable interactive channel — a {@link BaseRealtimeChannelClient}
+ * resolved from the `MJ: AI Agent Channels` registry row whose `ClientPluginClass` is
+ * `'RealtimeRemoteBrowserChannel'`. One instance per session (ClassFactory-created at session
+ * start).
+ *
+ * Topology (CLIENT-DIRECT, like the whiteboard — the browser just happens to live on the
+ * server):
+ *  - **Action**: the `browser_*` client-executed tool set ({@link REMOTE_BROWSER_TOOL_DEFINITIONS}).
+ *    {@link ApplyAgentTool} maps a tool call → a normalized {@link RemoteBrowserAction}
+ *    ({@link MapToolToAction}) and awaits the `ExecuteRemoteBrowserAction` GraphQL mutation
+ *    (via {@link RealtimeChannelContext.ExecuteServerAction}) to drive the server browser,
+ *    returning a concise JSON result for the model. Never throws — argument and transport
+ *    failures map to a `{ success: false, error }` payload.
+ *  - **Perception**: after a successful action, a `[browser]` context note carries the new
+ *    page URL so the agent perceives where the page went; the surface independently polls a
+ *    live screenshot.
+ *  - **Surface**: {@link RemoteBrowserSurfaceComponent}, created dynamically in the channel
+ *    tab; the plugin hands it the session's provider + id in {@link BindSurface} so it can
+ *    poll `RemoteBrowserSnapshot`. HUMAN TAKEOVER is enabled by default (Collaborative): the
+ *    user can click/type into the live canvas and those events relay to the server browser via
+ *    {@link RELAY_HUMAN_INPUT_MUTATION} (server-gated on the backend's `HumanTakeover` capability).
+ *
+ * The channel keeps NO client-side state of record — the browser's state lives entirely on
+ * the server — so {@link SerializeState} / {@link RestoreState} use the base no-op behavior.
+ */
+export declare class RemoteBrowserChannel extends BaseRealtimeChannelClient<RemoteBrowserSurfaceComponent> {
+    /** The live bound surface, when the channel tab's pane is instantiated. */
+    private surface;
+    /** Subscription to the bound surface's `HumanInput` output, torn down on unbind/dispose. */
+    private humanInputSub;
+    /**
+     * Whether the server confirmed it is PUSHING screencast frames for this session (`ScreenStreaming`
+     * supported). When `true` the surface renders pushed frames on its canvas and skips its poll, and
+     * {@link OnScreencastFrame} forwards each pushed frame to it. When `false` the surface keeps polling.
+     */
+    private streaming;
+    /**
+     * Whether the server confirmed it is PUSHING tab-audio chunks for this session (the backend can capture
+     * audio). When `true` {@link OnAudioChunk} feeds each pushed chunk to {@link audioPlayer}; when `false`
+     * no audio plays.
+     */
+    private audioStreaming;
+    /** The client-side audio player fed pushed chunks while {@link audioStreaming}; `null` until started. */
+    private audioPlayer;
+    /** Subscription to the bound surface's `AudioMutedChange` output (the speaker toggle), torn down on unbind. */
+    private audioMutedSub;
+    get ChannelName(): string;
+    get ToolNamePrefix(): string;
+    get TabTitle(): string;
+    get TabIcon(): string;
+    GetToolDefinitions(): RealtimeToolDefinition[];
+    GetSurfaceComponent(): Type<RemoteBrowserSurfaceComponent>;
+    /** First-run intro shown the first time the user opens the Browser tab (once per user). */
+    GetOnboardingDetails(): ChannelOnboardingDetails;
+    /**
+     * Wires the dynamically-created surface: hands it a snapshot fetcher closing over the
+     * channel context (session id + provider live there), so the surface stays transport-
+     * agnostic. Set BEFORE the surface's first change detection (the pane binds synchronously),
+     * so its `ngOnInit` poll has the fetcher.
+     *
+     * Also asks the server to start a live screencast. When the backend supports `ScreenStreaming` the
+     * server starts PUSHING frames and reports `Streaming: true`; we flip the surface to canvas mode (its
+     * poll is then skipped) and {@link OnScreencastFrame} paints each pushed frame. When the backend lacks
+     * the capability (`Streaming: false`) the surface keeps the snapshot poll already running — graceful
+     * fallback, no further action.
+     *
+     * HUMAN TAKEOVER is enabled BY DEFAULT (Collaborative): the surface is flipped `Interactive = true` and
+     * its `HumanInput` output subscribed, forwarding each pointer/keyboard event to the server via
+     * {@link RELAY_HUMAN_INPUT_MUTATION}. Takeover only takes effect on the canvas (screencast) path; the
+     * `<img>` poll fallback stays view-only. The server gates the actual routing on the backend's
+     * `HumanTakeover` capability, so this is safe to enable unconditionally.
+     */
+    BindSurface(instance: RemoteBrowserSurfaceComponent): void;
+    UnbindSurface(): void;
+    Dispose(): void;
+    /**
+     * Forwards one PUSHED screencast frame to the bound surface's canvas. Called by the session service
+     * when a `RemoteBrowserScreencastFrame` arrives on the push-status stream for THIS session. No-op when
+     * the channel isn't streaming or has no bound surface (e.g. the tab pane is collapsed).
+     *
+     * @param dataBase64 The frame image as raw base64 JPEG (no `data:` prefix).
+     */
+    OnScreencastFrame(dataBase64: string): void;
+    /**
+     * Feeds one PUSHED tab-audio chunk to the client-side audio player. Called by the session service when a
+     * `RemoteBrowserAudioChunk` arrives on the push-status stream for THIS session. No-op when the channel
+     * isn't audio-streaming or the player has been torn down.
+     *
+     * @param chunk The pushed audio chunk (base64 webm-opus + codec/seq metadata).
+     */
+    OnAudioChunk(chunk: RemoteBrowserAudioChunkInput): void;
+    /**
+     * Asks the server to start the live screencast and, on success, flips the surface to canvas mode so it
+     * stops polling and starts painting pushed frames. Best-effort: a `null`/`Streaming: false` result
+     * (capability absent or transport failure) leaves the surface on its poll fallback.
+     */
+    private startScreencast;
+    /**
+     * Asks the server to start the live tab-audio stream and, on success, spins up the client-side
+     * {@link RemoteBrowserAudioPlayer} (speaker ON by default — the call is the activating user gesture) so
+     * pushed chunks play. Best-effort: a `null`/`Streaming: false` result (no audio capability or transport
+     * failure) leaves the channel silent with no player.
+     *
+     * @param instance The bound surface, flipped to show the speaker toggle when audio starts.
+     */
+    private startAudioStream;
+    /** Asks the server to stop the tab-audio stream (best-effort), disposes the player, and clears state. */
+    private stopAudioStream;
+    /**
+     * Relays ONE human-takeover input from the surface to the server's {@link RELAY_HUMAN_INPUT_MUTATION}.
+     * Best-effort and fire-and-forget: never throws, and the boolean result is ignored (the server already
+     * gates routing on the backend's `HumanTakeover` capability). No-op when no live session id exists.
+     *
+     * @param input The flattened pointer/keyboard input the user performed on the live canvas.
+     */
+    private relayHumanInput;
+    /** Asks the server to stop the screencast (best-effort) and clears the streaming flag. */
+    private stopScreencast;
+    /**
+     * Fetches one live snapshot of the server browser for the surface — the PERCEPTION poll.
+     * Best-effort by contract ({@link RemoteBrowserSnapshotFetcher}): returns `null` when no
+     * session is live or the query fails (the surface keeps its last good frame).
+     */
+    private fetchSnapshot;
+    /**
+     * Executes ONE `browser_*` tool call: maps it to a server action and awaits the
+     * `ExecuteRemoteBrowserAction` mutation, then returns a concise JSON result the model reads.
+     * On success it also feeds the new page URL to the agent as a `[browser]` context note. Never
+     * throws — bad arguments, no live session, and server failures all become a failed-result
+     * payload so the model can narrate the problem.
+     */
+    ApplyAgentTool(toolName: string, argsJson: string): Promise<string>;
+    /** Builds the flat mutation variables from a normalized action (omitted fields ride as null). */
+    private buildVariables;
+    /**
+     * Runs the VISION-QUERY path for the `browser_DescribePage` / `browser_LocateElement` tools: awaits the
+     * `InterpretRemoteBrowserPage` mutation (server captures a screenshot + runs a fast vision model) and returns
+     * a concise JSON string the model reads. For a describe (no `query`) it returns `{ description }`; for a locate
+     * it returns `{ found, element, all }` so the agent can `browser_Click` the centroid. Never throws — a null
+     * session, no response, or a server detail-only result all map to a result string the model can narrate.
+     *
+     * @param query The visual target to locate, or `undefined` for a plain page description.
+     * @returns A JSON string describing the interpretation for the model.
+     */
+    private interpretPage;
+    /** Coerces a parsed tool-arg to a non-empty string, or `undefined` when absent / wrong-typed. */
+    private asArgString;
+    /** Parses the tool-args JSON into a plain object; returns `{}` for empty/malformed input. */
+    private parseArgs;
+    /** Builds a successful tool-result JSON string for the model. */
+    private ok;
+    /** Builds a failed tool-result JSON string for the model. */
+    private fail;
+}
+/**
+ * Tree-shaking prevention: the Remote Browser channel is resolved dynamically through the
+ * ClassFactory (by the registry row's `ClientPluginClass` key), so this static call is what
+ * keeps its `@RegisterClass` side effect from being eliminated by the bundler. Called from
+ * `conversations.module.ts` alongside {@link LoadRealtimeWhiteboardChannel}.
+ */
+export declare function LoadRealtimeRemoteBrowserChannel(): void;
+//# sourceMappingURL=remote-browser-channel.d.ts.map

package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"remote-browser-channel.d.ts","sourceRoot":"","sources":["../../../../../src/lib/components/realtime/remote-browser/remote-browser-channel.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,eAAe,CAAC;AAG1C,OAAO,EAAa,sBAAsB,EAAE,MAAM,oBAAoB,CAAC;AACvE,OAAO,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,0CAA0C,CAAC;AAC/G,OAAO,EAA2D,6BAA6B,EAAE,MAAM,oCAAoC,CAAC;AAS5I,OAAO,EAAwB,4BAA4B,EAA4B,MAAM,+BAA+B,CAAC;AA0K7H;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,qBACa,oBAAqB,SAAQ,yBAAyB,CAAC,6BAA6B,CAAC;IAChG,2EAA2E;IAC3E,OAAO,CAAC,OAAO,CAA8C;IAE7D,4FAA4F;IAC5F,OAAO,CAAC,aAAa,CAA6B;IAElD;;;;OAIG;IACH,OAAO,CAAC,SAAS,CAAS;IAE1B;;;;OAIG;IACH,OAAO,CAAC,cAAc,CAAS;IAE/B,yGAAyG;IACzG,OAAO,CAAC,WAAW,CAAyC;IAE5D,+GAA+G;IAC/G,OAAO,CAAC,aAAa,CAA6B;IAElD,IAAW,WAAW,IAAI,MAAM,CAE/B;IAED,IAAW,cAAc,IAAI,MAAM,CAElC;IAED,IAAW,QAAQ,IAAI,MAAM,CAE5B;IAED,IAAW,OAAO,IAAI,MAAM,CAE3B;IAEM,kBAAkB,IAAI,sBAAsB,EAAE;IAIrC,mBAAmB,IAAI,IAAI,CAAC,6BAA6B,CAAC;IAI1E,2FAA2F;IAC3E,oBAAoB,IAAI,wBAAwB;IAgBhE;;;;;;;;;;;;;;;;;OAiBG;IACI,WAAW,CAAC,QAAQ,EAAE,6BAA6B,GAAG,IAAI;IAUjD,aAAa,IAAI,IAAI;IAUrB,OAAO,IAAI,IAAI;IAU/B;;;;;;OAMG;IACI,iBAAiB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAMlD;;;;;;OAMG;IACI,YAAY,CAAC,KAAK,EAAE,4BAA4B,GAAG,IAAI;IAM9D;;;;OAIG;YACW,eAAe;IAY7B;;;;;;;OAOG;YACW,gBAAgB;IAe9B,yGAAyG;YAC3F,eAAe;IAY7B;;;;;;OAMG;IACH,OAAO,CAAC,eAAe;IAmBvB,0FAA0F;YAC5E,cAAc;IAU5B;;;;OAIG;YACW,aAAa;IAS3B;;;;;;OAMG;IACU,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAwDhF,iGAAiG;IACjG,OAAO,CAAC,cAAc;IAgBtB;;;;;;;;;OASG;YACW,aAAa;IAmC3B,iGAAiG;IACjG,OAAO,CAAC,WAAW;IAInB,6FAA6F;IAC7F,OAAO,CAAC,SAAS;IAajB,iEAAiE;IACjE,OAAO,CAAC,EAAE;IAWV,6DAA6D;IAC7D,OAAO,CAAC,IAAI;CAIb;AAED;;;;;GAKG;AACH,wBAAgB,gCAAgC,IAAI,IAAI,CAEvD"}