@vanira/sdk-react-native 0.0.2

package/src/adapters/browser/BrowserAudioAdapter.ts

@@ -0,0 +1,43 @@
+import type { AudioAdapter, AudioPlayerHandle } from '../interfaces';
+
+/**
+ * Browser implementation of AudioPlayerHandle.
+ * Wraps HTMLAudioElement to play the incoming remote MediaStream.
+ */
+class BrowserAudioPlayerHandle implements AudioPlayerHandle {
+    private el: HTMLAudioElement;
+
+    constructor(stream: MediaStream) {
+        this.el = new Audio();
+        this.el.srcObject = stream;
+        this.el.play().catch(e =>
+            console.warn('[BrowserAudio] Autoplay blocked — user gesture may be required:', e)
+        );
+    }
+
+    pause(): void {
+        // NOTE: Do NOT pause a live WebRTC audio track element.
+        // Pausing a MediaStream-backed audio element mutes all future packets permanently.
+        // clearAudio from the server means the server stops transmitting — no client action needed.
+        // This is kept here as a no-op to satisfy the interface without breaking the stream.
+    }
+
+    cleanup(): void {
+        this.el.pause();
+        this.el.srcObject = null;
+    }
+
+    onEnded(callback: () => void): void {
+        this.el.onended = callback;
+    }
+}
+
+/**
+ * Browser AudioAdapter.
+ * Creates an HTMLAudioElement-backed player for the incoming remote audio track.
+ */
+export class BrowserAudioAdapter implements AudioAdapter {
+    createRemotePlayer(stream: MediaStream): AudioPlayerHandle {
+        return new BrowserAudioPlayerHandle(stream);
+    }
+}

package/src/adapters/browser/BrowserDataChannelAdapter.ts

@@ -0,0 +1,69 @@
+import type {
+    DataChannelAdapter,
+    DataChannelController,
+    DataChannelHandlers,
+} from '../PeerConnectionAdapter';
+
+/**
+ * Browser DataChannelAdapter.
+ *
+ * Normalises browser's three wire formats (string / ArrayBuffer / Blob) to
+ * decoded UTF-8 text before invoking onMessage.
+ *
+ * Binary frame policy — preserved from original WebRTCClient behaviour:
+ *   Only binary frames that decode to JSON with event === 'client_tool_call'
+ *   are forwarded.  All other binary frames are logged but dropped to prevent
+ *   false clearAudio events from audio packet payloads.
+ */
+export class BrowserDataChannelAdapter implements DataChannelAdapter {
+    bind(channel: RTCDataChannel, handlers: DataChannelHandlers): DataChannelController {
+        channel.onopen = () => handlers.onOpen();
+        channel.onerror = (e) => handlers.onError(e);
+
+        channel.onmessage = (e: MessageEvent) => {
+            if (typeof e.data === 'string') {
+                handlers.onMessage({ text: e.data });
+                return;
+            }
+
+            if (e.data instanceof ArrayBuffer) {
+                try {
+                    const text = new TextDecoder().decode(e.data);
+                    const parsed = JSON.parse(text) as { event?: string };
+                    if (parsed?.event === 'client_tool_call') {
+                        console.log('[DataChannel] Binary client_tool_call decoded and forwarded');
+                        handlers.onMessage({ text });
+                    } else {
+                        console.log('[DataChannel] Binary frame dropped (not client_tool_call):', parsed?.event);
+                    }
+                } catch {
+                    console.log('[DataChannel] Non-decodable binary frame:',
+                        (e.data as ArrayBuffer).byteLength, 'bytes — dropped');
+                }
+                return;
+            }
+
+            if (e.data instanceof Blob) {
+                e.data.text().then(text => {
+                    handlers.onMessage({ text });
+                }).catch(err => {
+                    console.warn('[DataChannel] Failed to read Blob message:', err);
+                });
+            }
+        };
+
+        return {
+            send(text: string): void {
+                if (channel.readyState === 'open') {
+                    channel.send(text);
+                }
+            },
+            isOpen(): boolean {
+                return channel.readyState === 'open';
+            },
+            close(): void {
+                channel.close();
+            },
+        };
+    }
+}

package/src/adapters/browser/BrowserMediaAdapter.ts

@@ -0,0 +1,15 @@
+import type { MediaAdapter, MediaAudioConstraints } from '../interfaces';
+
+/**
+ * Browser MediaAdapter.
+ * Delegates directly to navigator.mediaDevices — available in all modern browsers.
+ */
+export class BrowserMediaAdapter implements MediaAdapter {
+    async getUserAudio(constraints: MediaAudioConstraints): Promise<MediaStream> {
+        return navigator.mediaDevices.getUserMedia({ audio: constraints as MediaTrackConstraints });
+    }
+
+    stopStream(stream: MediaStream): void {
+        stream.getTracks().forEach(track => track.stop());
+    }
+}

package/src/adapters/browser/BrowserPeerAdapter.ts

@@ -0,0 +1,14 @@
+import type { PeerConnectionAdapter, PeerConnectionConfig } from '../PeerConnectionAdapter';
+
+/**
+ * Browser PeerConnectionAdapter.
+ * Wraps `new RTCPeerConnection(config)` so the core never instantiates it directly.
+ */
+export class BrowserPeerAdapter implements PeerConnectionAdapter {
+    create(config: PeerConnectionConfig): RTCPeerConnection {
+        return new RTCPeerConnection({
+            iceServers: config.iceServers as RTCIceServer[],
+            iceTransportPolicy: config.iceTransportPolicy ?? 'all',
+        });
+    }
+}

package/src/adapters/browser/index.ts

@@ -0,0 +1,4 @@
+export { BrowserAudioAdapter } from './BrowserAudioAdapter';
+export { BrowserMediaAdapter } from './BrowserMediaAdapter';
+export { BrowserPeerAdapter } from './BrowserPeerAdapter';
+export { BrowserDataChannelAdapter } from './BrowserDataChannelAdapter';

package/src/adapters/interfaces.ts

@@ -0,0 +1,84 @@
+/**
+ * Cross-platform adapter interfaces for the Vanira SDK.
+ *
+ * The core session (WebRTCClient / VaniraAI) depends only on these interfaces.
+ * Platform-specific implementations live in:
+ *   adapters/browser/   — Web / browser
+ *   adapters/react-native/ — React Native (react-native-webrtc)
+ *
+ * Future:
+ *   flutter-sdk/lib/src/adapters/ — Dart implementations for Flutter
+ */
+
+// ─── Audio constraints passed to the media adapter ───────────────────────────
+
+export interface MediaAudioConstraints {
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    autoGainControl?: boolean;
+    /** Accepts a number or an ideal-hint object */
+    sampleRate?: number | { ideal: number };
+    channelCount?: number | { ideal: number };
+}
+
+// ─── AudioPlayerHandle ────────────────────────────────────────────────────────
+//
+// Returned by AudioAdapter.createRemotePlayer().  Core session code holds a
+// reference and calls these methods to control playback.
+
+export interface AudioPlayerHandle {
+    /**
+     * Stop current playback and reset position.
+     * Called when the server sends a `clearAudio` event or on action interrupt.
+     */
+    pause(): void;
+
+    /**
+     * Release all resources tied to this player.
+     * Called during disconnect() cleanup.
+     */
+    cleanup(): void;
+
+    /**
+     * Register a callback to fire when natural playback ends.
+     * The core session uses this to send the `playedStream` event back to the server.
+     *
+     * NOTE — React Native: this callback is never invoked because react-native-webrtc
+     * routes audio automatically through the native stack.  The RNAudioAdapter stubs
+     * this out and logs a warning.  The server must not rely on receiving `playedStream`
+     * from RN clients; use a server-side estimated-duration fallback instead.
+     */
+    onEnded(callback: () => void): void;
+}
+
+// ─── AudioAdapter ─────────────────────────────────────────────────────────────
+
+export interface AudioAdapter {
+    /**
+     * Attach the incoming remote MediaStream to an audio output and start playing.
+     * Returns a handle the session holds for later pause / cleanup calls.
+     *
+     * Browser: creates an HTMLAudioElement, sets srcObject, calls play().
+     * React Native: no-op — react-native-webrtc routes audio to the native speaker
+     *               automatically via the peer connection.  Returns a stub handle.
+     */
+    createRemotePlayer(stream: MediaStream): AudioPlayerHandle;
+}
+
+// ─── MediaAdapter ─────────────────────────────────────────────────────────────
+
+export interface MediaAdapter {
+    /**
+     * Request the user's microphone.
+     *
+     * Browser: navigator.mediaDevices.getUserMedia({ audio: constraints })
+     * React Native: react-native-webrtc mediaDevices.getUserMedia (same API, polyfilled)
+     * Flutter: flutter_webrtc navigator.mediaDevices.getUserMedia (Dart, same shape)
+     */
+    getUserAudio(constraints: MediaAudioConstraints): Promise<MediaStream>;
+
+    /**
+     * Stop all tracks on a stream, releasing the underlying hardware device.
+     */
+    stopStream(stream: MediaStream): void;
+}

package/src/adapters/react-native/RNAudioAdapter.ts

@@ -0,0 +1,42 @@
+import type { AudioAdapter, AudioPlayerHandle } from '../interfaces';
+import { activateCallAudioPlayback } from './callAudioRouting';
+
+/**
+ * React Native AudioPlayerHandle — no HTMLAudioElement equivalent.
+ * Remote audio is routed by react-native-webrtc + InCallManager.
+ */
+class RNAudioPlayerHandle implements AudioPlayerHandle {
+    pause(): void {
+        // clearAudio means the server stops transmitting — do not pause the track.
+    }
+
+    cleanup(): void {
+        // react-native-webrtc cleans up when RTCPeerConnection closes.
+    }
+
+    onEnded(_callback: () => void): void {
+        // RN has no audio onended — WebRTCClient sends playedStream after final transcription.
+    }
+}
+
+/**
+ * React Native AudioAdapter.
+ * Starts in-call audio routing and enables remote audio tracks for playback.
+ */
+export class RNAudioAdapter implements AudioAdapter {
+    createRemotePlayer(stream: MediaStream): AudioPlayerHandle {
+        activateCallAudioPlayback();
+
+        const tracks = stream.getTracks?.() ?? [];
+        for (const track of tracks) {
+            if (track.kind === 'audio') {
+                track.enabled = true;
+                console.log(
+                    `[RNAudio] Remote audio track enabled (id=${track.id}, state=${track.readyState})`,
+                );
+            }
+        }
+
+        return new RNAudioPlayerHandle();
+    }
+}

package/src/adapters/react-native/RNDataChannelAdapter.ts

@@ -0,0 +1,79 @@
+import type {
+    DataChannelAdapter,
+    DataChannelController,
+    DataChannelHandlers,
+} from '../PeerConnectionAdapter';
+import { decodeUtf8ArrayBuffer, toArrayBuffer } from './decodeUtf8';
+
+/**
+ * React Native DataChannelAdapter.
+ *
+ * react-native-webrtc usually delivers string frames; on Android, client_tool_call
+ * may arrive as ArrayBuffer. Must not use TextDecoder directly — Hermes may lack it.
+ */
+export class RNDataChannelAdapter implements DataChannelAdapter {
+    bind(channel: RTCDataChannel, handlers: DataChannelHandlers): DataChannelController {
+        channel.onopen = () => handlers.onOpen();
+        channel.onerror = (e) => handlers.onError(e);
+
+        const forwardText = (text: string, source: string) => {
+            try {
+                const parsed = JSON.parse(text) as { event?: string };
+                if (parsed?.event) {
+                    if (parsed.event === 'client_tool_call') {
+                        console.log(`[RN DataChannel] ${source} client_tool_call decoded`);
+                    }
+                    handlers.onMessage({ text });
+                } else {
+                    console.log(`[RN DataChannel] ${source} JSON without event — dropped`);
+                }
+            } catch (err) {
+                console.warn(`[RN DataChannel] ${source} non-JSON frame — dropped`, err);
+            }
+        };
+
+        channel.onmessage = (e: MessageEvent) => {
+            if (typeof e.data === 'string') {
+                handlers.onMessage({ text: e.data });
+                return;
+            }
+
+            const buffer = toArrayBuffer(e.data);
+            if (buffer) {
+                try {
+                    forwardText(decodeUtf8ArrayBuffer(buffer), 'Binary');
+                } catch (err) {
+                    console.error('[RN DataChannel] Binary decode failed:', err);
+                }
+                return;
+            }
+
+            const blob = e.data as Blob | undefined;
+            if (blob && typeof blob.text === 'function') {
+                blob
+                    .text()
+                    .then((text) => forwardText(text, 'Blob'))
+                    .catch((err) => {
+                        console.warn('[RN DataChannel] Failed to read Blob message:', err);
+                    });
+                return;
+            }
+
+            console.warn('[RN DataChannel] Unexpected message type:', typeof e.data);
+        };
+
+        return {
+            send(text: string): void {
+                if (channel.readyState === 'open') {
+                    channel.send(text);
+                }
+            },
+            isOpen(): boolean {
+                return channel.readyState === 'open';
+            },
+            close(): void {
+                channel.close();
+            },
+        };
+    }
+}

package/src/adapters/react-native/RNMediaAdapter.ts

@@ -0,0 +1,46 @@
+import { Platform } from 'react-native';
+import type { MediaAdapter, MediaAudioConstraints } from '../interfaces';
+
+/**
+ * React Native MediaAdapter.
+ *
+ * Android: disable heavy DSP (AEC/NS/AGC) — on emulators and some devices they
+ * strip speech or pass comfort-noise/beep-like uplink. Shallow constraints
+ * match flutter_webrtc VaniraSession on Android.
+ */
+export class RNMediaAdapter implements MediaAdapter {
+    async getUserAudio(constraints: MediaAudioConstraints): Promise<MediaStream> {
+        const audioConstraints: boolean | MediaTrackConstraints =
+            Platform.OS === 'android'
+                ? {
+                      echoCancellation: false,
+                      noiseSuppression: false,
+                      autoGainControl: false,
+                  }
+                : (constraints as MediaTrackConstraints);
+
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const stream = (await (navigator as any).mediaDevices.getUserMedia({
+            audio: audioConstraints,
+            video: false,
+        })) as MediaStream;
+
+        for (const track of stream.getTracks()) {
+            if (track.kind === 'audio') {
+                track.enabled = true;
+                const muted = (track as MediaStreamTrack & { muted?: boolean }).muted;
+                console.log(
+                    `[RNMedia] Local mic track: enabled=${track.enabled}` +
+                        `${muted !== undefined ? ` muted=${muted}` : ''}` +
+                        ` readyState=${track.readyState} id=${track.id}`,
+                );
+            }
+        }
+
+        return stream;
+    }
+
+    stopStream(stream: MediaStream): void {
+        stream.getTracks().forEach((track: MediaStreamTrack) => track.stop());
+    }
+}

package/src/adapters/react-native/RNPeerAdapter.ts

@@ -0,0 +1,28 @@
+import type { PeerConnectionAdapter, PeerConnectionConfig } from '../PeerConnectionAdapter';
+
+/**
+ * React Native PeerConnectionAdapter.
+ *
+ * react-native-webrtc polyfills RTCPeerConnection into the global scope with
+ * the same API surface as the browser.  This adapter wraps the construction
+ * identically to BrowserPeerAdapter — the value is isolation, not difference.
+ *
+ * Usage note: call `registerGlobals()` from react-native-webrtc once at
+ * app entry before constructing this adapter.  Without that call,
+ * RTCPeerConnection will be undefined at runtime.
+ *
+ *   import { registerGlobals } from 'react-native-webrtc';
+ *   registerGlobals(); // call once in app entry (e.g. index.js)
+ */
+export class RNPeerAdapter implements PeerConnectionAdapter {
+    create(config: PeerConnectionConfig): RTCPeerConnection {
+        // react-native-webrtc polyfills RTCPeerConnection as a global.
+        // The cast is required because TypeScript's lib.dom.d.ts types differ
+        // from the RN polyfill's type declarations for iceServers.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        return new (globalThis as any).RTCPeerConnection({
+            iceServers: config.iceServers,
+            iceTransportPolicy: config.iceTransportPolicy ?? 'all',
+        }) as RTCPeerConnection;
+    }
+}

package/src/adapters/react-native/callAudioRouting.ts

@@ -0,0 +1,115 @@
+import InCallManager from 'react-native-incall-manager';
+
+export type CallAudioSnapshot = {
+  speakerOn: boolean;
+  active: boolean;
+};
+
+let speakerOn = true;
+let sessionActive = false;
+let cachedSnapshot: CallAudioSnapshot = {speakerOn: true, active: false};
+const listeners = new Set<() => void>();
+
+const SERVER_SNAPSHOT: CallAudioSnapshot = {speakerOn: true, active: false};
+
+function refreshSnapshot(): CallAudioSnapshot {
+  if (
+    cachedSnapshot.speakerOn !== speakerOn ||
+    cachedSnapshot.active !== sessionActive
+  ) {
+    cachedSnapshot = {speakerOn, active: sessionActive};
+  }
+  return cachedSnapshot;
+}
+
+/** Stable snapshot for useSyncExternalStore — must reuse same object when values unchanged. */
+export function getCallAudioSnapshot(): CallAudioSnapshot {
+  return refreshSnapshot();
+}
+
+export function getCallAudioServerSnapshot(): CallAudioSnapshot {
+  return SERVER_SNAPSHOT;
+}
+
+function notify(): void {
+  refreshSnapshot();
+  listeners.forEach(listener => listener());
+}
+
+export function subscribeCallAudio(listener: () => void): () => void {
+  listeners.add(listener);
+  return () => listeners.delete(listener);
+}
+
+export function isSpeakerOn(): boolean {
+  return speakerOn;
+}
+
+export function isCallAudioSessionActive(): boolean {
+  return sessionActive;
+}
+
+/**
+ * Open VoIP audio mode before getUserMedia — earpiece, no ringback.
+ * Speaker during capture causes emulator loopback (agent TTS → mic → uplink beep).
+ */
+export function prepareCallAudioForCapture(): void {
+  if (sessionActive) {
+    return;
+  }
+  InCallManager.start({media: 'audio', auto: false, ringback: ''});
+  InCallManager.setKeepScreenOn(true);
+  InCallManager.setSpeakerphoneOn(false);
+  InCallManager.setForceSpeakerphoneOn(false);
+  sessionActive = true;
+  notify();
+  console.log(
+    '[CallAudio] Prepared for mic capture (MODE_IN_COMMUNICATION, earpiece, no ringback)',
+  );
+}
+
+/** Route remote WebRTC audio to speaker/earpiece after connect or first remote track. */
+export function activateCallAudioPlayback(): void {
+  if (!sessionActive) {
+    prepareCallAudioForCapture();
+  }
+  applySpeakerRoute(speakerOn);
+  notify();
+  console.log(`[CallAudio] Playback routing active (speaker=${speakerOn})`);
+}
+
+/** Full session — capture prep + speaker playback (backward compatible). */
+export function startCallAudioSession(): void {
+  prepareCallAudioForCapture();
+  activateCallAudioPlayback();
+}
+
+/** Tear down in-call audio session — call on disconnect. */
+export function stopCallAudioSession(): void {
+  if (!sessionActive) {
+    return;
+  }
+  InCallManager.setKeepScreenOn(false);
+  InCallManager.stop();
+  sessionActive = false;
+  notify();
+  console.log('[CallAudio] In-call session stopped');
+}
+
+export function setSpeakerphoneOn(enabled: boolean): void {
+  speakerOn = enabled;
+  if (sessionActive) {
+    applySpeakerRoute(enabled);
+  }
+  notify();
+}
+
+export function toggleSpeakerphone(): boolean {
+  setSpeakerphoneOn(!speakerOn);
+  return speakerOn;
+}
+
+function applySpeakerRoute(enabled: boolean): void {
+  InCallManager.setSpeakerphoneOn(enabled);
+  InCallManager.setForceSpeakerphoneOn(enabled);
+}

package/src/adapters/react-native/decodeUtf8.ts

@@ -0,0 +1,72 @@
+/**
+ * Decode UTF-8 bytes from a DataChannel ArrayBuffer.
+ * React Native / Hermes may not provide global TextDecoder.
+ */
+export function decodeUtf8ArrayBuffer(buffer: ArrayBuffer): string {
+    const Decoder = globalThis.TextDecoder;
+    if (typeof Decoder === 'function') {
+        return new Decoder('utf-8').decode(buffer);
+    }
+
+    const bytes = new Uint8Array(buffer);
+    let out = '';
+    let i = 0;
+
+    while (i < bytes.length) {
+        const byte1 = bytes[i++];
+
+        if (byte1 < 0x80) {
+            out += String.fromCharCode(byte1);
+            continue;
+        }
+
+        if ((byte1 & 0xe0) === 0xc0) {
+            const byte2 = bytes[i++];
+            out += String.fromCharCode(((byte1 & 0x1f) << 6) | (byte2 & 0x3f));
+            continue;
+        }
+
+        if ((byte1 & 0xf0) === 0xe0) {
+            const byte2 = bytes[i++];
+            const byte3 = bytes[i++];
+            out += String.fromCharCode(
+                ((byte1 & 0x0f) << 12) | ((byte2 & 0x3f) << 6) | (byte3 & 0x3f),
+            );
+            continue;
+        }
+
+        if ((byte1 & 0xf8) === 0xf0) {
+            const byte2 = bytes[i++];
+            const byte3 = bytes[i++];
+            const byte4 = bytes[i++];
+            let codePoint =
+                ((byte1 & 0x07) << 18) |
+                ((byte2 & 0x3f) << 12) |
+                ((byte3 & 0x3f) << 6) |
+                (byte4 & 0x3f);
+            codePoint -= 0x10000;
+            out += String.fromCharCode(
+                0xd800 + (codePoint >> 10),
+                0xdc00 + (codePoint & 0x3ff),
+            );
+        }
+    }
+
+    return out;
+}
+
+/** Normalize react-native-webrtc message payloads to ArrayBuffer. */
+export function toArrayBuffer(data: unknown): ArrayBuffer | null {
+    if (data instanceof ArrayBuffer) {
+        return data;
+    }
+    if (ArrayBuffer.isView(data)) {
+        const view = data as ArrayBufferView;
+        const slice = view.buffer.slice(
+            view.byteOffset,
+            view.byteOffset + view.byteLength,
+        );
+        return slice as ArrayBuffer;
+    }
+    return null;
+}

package/src/adapters/react-native/index.ts

@@ -0,0 +1,4 @@
+export { RNAudioAdapter } from './RNAudioAdapter';
+export { RNMediaAdapter } from './RNMediaAdapter';
+export { RNPeerAdapter } from './RNPeerAdapter';
+export { RNDataChannelAdapter } from './RNDataChannelAdapter';