@akhilpulse/samadhaan-session-replay 0.4.3 → 0.5.0

package/README.md

@@ -122,15 +122,34 @@ if a module is missing that capability is simply skipped (no crash):
 | Capability                | Package                            | If missing |
 |---------------------------|------------------------------------|------------|
 | Shake detection           | `react-native-shake`               | Use `triggerAssist()` manually |
-| Microphone (voice input)  | `react-native-live-audio-stream`   | Assistant can still speak + draw arrows; just can't hear the user |
-| Agent voice playback      | provide a `PcmAudioPlayer` (see below) | Agent audio is dropped (arrows + transcript still work) |
+| Microphone (voice input)  | `react-native-live-audio-stream`   | Voice bar shows "Mic unavailable" — assistant can still speak + draw arrows |
+| Agent voice playback      | **built-in on Android (v0.5.0+)**; custom `PcmAudioPlayer` optional | On iOS without a custom player, agent audio is dropped (arrows + transcript still work) |
 
 ```bash
 npm install react-native-shake react-native-live-audio-stream
 cd ios && pod install
 ```
 
-The agent streams **int16 mono PCM at 16 kHz**. Plug in your own player to hear it:
+#### Microphone permission (required for voice input)
+
+- **Android** — declare in your app's `AndroidManifest.xml`:
+  ```xml
+  <uses-permission android:name="android.permission.RECORD_AUDIO" />
+  ```
+  Starting in **0.5.0 the SDK requests the runtime permission automatically**
+  the first time the assistant opens. If the user denies it, the voice bar
+  shows "Mic blocked — allow microphone access in Settings" instead of
+  pretending to listen.
+- **iOS** — add `NSMicrophoneUsageDescription` to `Info.plist` or the OS will
+  kill the app when recording starts.
+
+#### Agent voice playback
+
+The agent streams **int16 mono PCM at 16 kHz**. Starting in **0.5.0 the SDK
+ships a built-in Android AudioTrack player** (part of the bundled native
+module — rebuild the Android app after upgrading). It's used automatically
+when you don't pass `assistantAudioPlayer`. On iOS (or to override), plug in
+your own player:
 
 ```tsx
 import type { PcmAudioPlayer } from "@akhilpulse/samadhaan-session-replay";

package/android/src/main/java/com/akhilpulse/sessionreplay/SessionReplayPcmPlayerModule.kt

@@ -0,0 +1,119 @@
+package com.akhilpulse.sessionreplay
+
+import android.media.AudioAttributes
+import android.media.AudioFormat
+import android.media.AudioTrack
+import android.os.Handler
+import android.os.HandlerThread
+import android.util.Base64
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.bridge.ReactContextBaseJavaModule
+import com.facebook.react.bridge.ReactMethod
+
+/**
+ * Built-in streaming PCM player for the Shake-to-Assist agent voice
+ * (v0.5.0+). Plays int16 mono PCM chunks (base64 over the bridge) through
+ * a single reused AudioTrack in streaming mode, so host apps get agent
+ * audio out of the box without wiring their own PcmAudioPlayer.
+ *
+ * All AudioTrack work happens on a dedicated HandlerThread — never on the
+ * React native-modules thread — because AudioTrack.write() in blocking
+ * mode can stall for the duration of the buffered audio.
+ */
+class SessionReplayPcmPlayerModule(reactContext: ReactApplicationContext) :
+    ReactContextBaseJavaModule(reactContext) {
+
+  override fun getName() = "SessionReplayPcmPlayer"
+
+  private var track: AudioTrack? = null
+  private var thread: HandlerThread? = null
+  private var handler: Handler? = null
+
+  private fun ensureHandler(): Handler {
+    val alive = thread
+    val h = handler
+    if (alive != null && alive.isAlive && h != null) return h
+    val nt = HandlerThread("SessionReplayPcmPlayer")
+    nt.start()
+    thread = nt
+    val nh = Handler(nt.looper)
+    handler = nh
+    return nh
+  }
+
+  private fun releaseTrack() {
+    try { track?.stop() } catch (_: Throwable) {}
+    try { track?.release() } catch (_: Throwable) {}
+    track = null
+  }
+
+  @ReactMethod
+  fun start(sampleRateIn: Int) {
+    ensureHandler().post {
+      try {
+        releaseTrack()
+        val sampleRate = if (sampleRateIn > 0) sampleRateIn else 16000
+        val minBuf = AudioTrack.getMinBufferSize(
+            sampleRate, AudioFormat.CHANNEL_OUT_MONO, AudioFormat.ENCODING_PCM_16BIT)
+        // ~0.5 s of 16-bit mono headroom, never below the platform minimum.
+        val bufBytes = maxOf(minBuf, sampleRate)
+        val t = AudioTrack.Builder()
+            .setAudioAttributes(
+                AudioAttributes.Builder()
+                    .setUsage(AudioAttributes.USAGE_MEDIA)
+                    .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
+                    .build())
+            .setAudioFormat(
+                AudioFormat.Builder()
+                    .setEncoding(AudioFormat.ENCODING_PCM_16BIT)
+                    .setSampleRate(sampleRate)
+                    .setChannelMask(AudioFormat.CHANNEL_OUT_MONO)
+                    .build())
+            .setBufferSizeInBytes(bufBytes)
+            .setTransferMode(AudioTrack.MODE_STREAM)
+            .build()
+        t.play()
+        track = t
+      } catch (_: Throwable) {
+        track = null
+      }
+    }
+  }
+
+  @ReactMethod
+  fun play(base64Pcm: String) {
+    ensureHandler().post {
+      val t = track ?: return@post
+      try {
+        val bytes = Base64.decode(base64Pcm, Base64.DEFAULT)
+        if (bytes.isNotEmpty()) t.write(bytes, 0, bytes.size)
+      } catch (_: Throwable) {}
+    }
+  }
+
+  /** Barge-in: drop any queued agent audio immediately. */
+  @ReactMethod
+  fun clear() {
+    ensureHandler().post {
+      val t = track ?: return@post
+      try {
+        t.pause()
+        t.flush()
+        t.play()
+      } catch (_: Throwable) {}
+    }
+  }
+
+  @ReactMethod
+  fun stop() {
+    ensureHandler().post { releaseTrack() }
+  }
+
+  override fun invalidate() {
+    super.invalidate()
+    try { handler?.post { releaseTrack() } } catch (_: Throwable) {}
+    try { thread?.quitSafely() } catch (_: Throwable) {}
+    thread = null
+    handler = null
+  }
+}

package/android/src/main/java/com/akhilpulse/sessionreplay/SessionReplayPixelCopyPackage.kt

@@ -7,7 +7,10 @@ import com.facebook.react.uimanager.ViewManager
 
 class SessionReplayPixelCopyPackage : ReactPackage {
   override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> =
-      listOf(SessionReplayPixelCopyModule(reactContext))
+      listOf(
+          SessionReplayPixelCopyModule(reactContext),
+          SessionReplayPcmPlayerModule(reactContext),
+      )
 
   override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> =
       emptyList()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@akhilpulse/samadhaan-session-replay",
-  "version": "0.4.3",
+  "version": "0.5.0",
   "description": "React Native SDK for Samadhaan Guided Support — session recording, live remote screen-share, take-control, guided-tour overlays, and Shake-to-Assist AI voice guide.",
   "main": "src/index.ts",
   "module": "src/index.ts",

package/src/SessionReplayProvider.tsx

@@ -38,6 +38,8 @@ import { captureRef, captureScreen } from "react-native-view-shot";
 import {
   subscribeShake,
   UltravoxVoiceSession,
+  createDefaultPcmPlayer,
+  type MicStatus,
   type PcmAudioPlayer,
   type VoiceState,
 } from "./voice";
@@ -54,7 +56,7 @@ let pixelCopyDisabled = false; // turned true after repeated runtime failures to
 let pixelCopyFailStreak = 0; // consecutive non-busy failures; resets on success
 
 const STORAGE_KEY = "@akhilpulse/samadhaan-session-replay/sessionId";
-const SDK_VERSION = "0.4.3";
+const SDK_VERSION = "0.5.0";
 
 // ---- Module-level session singleton --------------------------------------
 // Multiple SessionReplayProvider mounts (intentional or via React StrictMode's
@@ -158,6 +160,7 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
 
   // ---- Shake-to-Assist (AI voice guide) state ----
   const [assistantPhase, setAssistantPhase] = useState<AssistantPhase>("idle");
+  const [assistMicStatus, setAssistMicStatus] = useState<MicStatus | null>(null);
   const [assistantArrow, setAssistantArrow] = useState<
     { nx: number; ny: number; label?: string; instruction?: string } | null
   >(null);
@@ -567,6 +570,7 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     try { voice?.stop(); } catch {}
     setAssistantArrow(null);
     setAssistantPhase("idle");
+    setAssistMicStatus(null);
     if (asid && sessionId && ingestToken) {
       fetch(`${serverUrl}/api/guided-support/assistant/end`, {
         method: "POST",
@@ -610,15 +614,20 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
         return;
       }
       assistantSessionIdRef.current = j.assistantSessionId;
+      setAssistMicStatus(null);
       const voice = new UltravoxVoiceSession({
         joinUrl: j.joinUrl,
         sampleRate: 16000,
-        audioPlayer: config.assistantAudioPlayer,
+        // Prefer the host-supplied player; otherwise fall back to the SDK's
+        // built-in Android AudioTrack player (v0.5.0+) so the agent's voice
+        // is audible out of the box.
+        audioPlayer: config.assistantAudioPlayer ?? createDefaultPcmPlayer() ?? undefined,
         onState: (s) => {
           // Natural socket close ("ended") tears the whole assistant down.
           if (s === "ended") { endAssist(); return; }
           setAssistantPhase(s);
         },
+        onMicStatus: (status) => setAssistMicStatus(status),
         onError: () => setAssistantPhase("error"),
       });
       voiceRef.current = voice;
@@ -903,7 +912,9 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
           <View style={styles.assistDock} pointerEvents="box-none">
             <View style={styles.assistDockCard}>
               <VoiceWave phase={assistantPhase} />
-              <Text style={styles.assistStatusText}>{assistantStatusLabel(assistantPhase)}</Text>
+              <Text style={styles.assistStatusText}>
+                {assistantStatusLabel(assistantPhase, assistMicStatus)}
+              </Text>
               <Pressable
                 onPress={endAssist}
                 style={({ pressed }) => [styles.assistClose, pressed && styles.btnPressed]}
@@ -920,7 +931,15 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
 }
 
 // ---- Shake-to-Assist UI bits ----
-function assistantStatusLabel(phase: AssistantPhase): string {
+function assistantStatusLabel(phase: AssistantPhase, micStatus?: MicStatus | null): string {
+  // Be honest when we can't actually hear the user — a fake "Listening…"
+  // with a dead mic is the worst possible UX (bar stuck forever).
+  if (phase === "listening" && micStatus === "permission_denied") {
+    return "Mic blocked — allow microphone access in Settings";
+  }
+  if (phase === "listening" && micStatus === "unavailable") {
+    return "Mic unavailable — voice input is off";
+  }
   switch (phase) {
     case "connecting": return "Connecting…";
     case "listening": return "Listening… ask me anything";

package/src/index.ts

@@ -9,9 +9,11 @@ export {
 export {
   subscribeShake,
   UltravoxVoiceSession,
+  createDefaultPcmPlayer,
 } from "./voice";
 export type {
   PcmAudioPlayer,
   VoiceState,
+  MicStatus,
   StartVoiceOpts,
 } from "./voice";

package/src/voice.ts

@@ -21,6 +21,8 @@
  * matching server/ultravox.ts (serverWebSocket inputSampleRate/outputSampleRate).
  */
 
+import { NativeModules, PermissionsAndroid, Platform } from "react-native";
+
 // Loads an optional native module without crashing Metro when it's absent.
 function loadOptional<T = any>(name: string): T | null {
   try {
@@ -31,6 +33,34 @@ function loadOptional<T = any>(name: string): T | null {
   }
 }
 
+// ---- Mic permission --------------------------------------------------------
+/**
+ * Android requires a runtime RECORD_AUDIO grant — react-native-live-audio-stream
+ * does NOT request it itself; without it the recorder silently captures nothing
+ * and the assistant can never hear the user. iOS shows its own prompt when the
+ * recorder starts (host app must have NSMicrophoneUsageDescription in Info.plist).
+ */
+async function ensureMicPermission(): Promise<boolean> {
+  if (Platform.OS !== "android") return true;
+  try {
+    const perm = PermissionsAndroid.PERMISSIONS.RECORD_AUDIO;
+    const already = await PermissionsAndroid.check(perm);
+    if (already) return true;
+    const res = await PermissionsAndroid.request(perm, {
+      title: "Microphone access",
+      message: "Needed so the voice assistant can hear you.",
+      buttonPositive: "Allow",
+      buttonNegative: "Not now",
+    });
+    return res === PermissionsAndroid.RESULTS.GRANTED;
+  } catch {
+    return false;
+  }
+}
+
+/** Why voice input isn't working (surfaced so the UI can tell the user). */
+export type MicStatus = "ok" | "permission_denied" | "unavailable";
+
 // ---- Shake detection -----------------------------------------------------
 /**
  * Subscribe to phone-shake events. Returns an unsubscribe function.
@@ -106,6 +136,8 @@ export type StartVoiceOpts = {
   /** Live transcript text (both user + agent), with role + final flag. */
   onTranscript?: (t: { role: string; text: string; final: boolean }) => void;
   onError?: (e: Error) => void;
+  /** Reports whether the microphone actually started (module + permission). */
+  onMicStatus?: (status: MicStatus) => void;
 };
 
 /**
@@ -140,7 +172,7 @@ export class UltravoxVoiceSession {
     ws.onopen = () => {
       if (this.stopped) return;
       this.opts.audioPlayer?.start?.(this.sampleRate);
-      this.startMic();
+      void this.startMic(); // async: requests mic permission, reports onMicStatus
       this.opts.onState?.("listening");
     };
     ws.onmessage = (ev: any) => {
@@ -184,9 +216,21 @@ export class UltravoxVoiceSession {
     }
   }
 
-  private startMic() {
+  private async startMic() {
     const audioMod: any = loadOptional("react-native-live-audio-stream");
-    if (!audioMod) return; // No mic module — assistant can still speak/guide.
+    if (!audioMod) {
+      // No mic module — assistant can still speak/guide, but tell the UI.
+      if (__DEV__) console.warn("[session-replay] react-native-live-audio-stream not installed — voice input disabled");
+      this.opts.onMicStatus?.("unavailable");
+      return;
+    }
+    const granted = await ensureMicPermission();
+    if (this.stopped) return;
+    if (!granted) {
+      if (__DEV__) console.warn("[session-replay] RECORD_AUDIO permission denied — voice input disabled");
+      this.opts.onMicStatus?.("permission_denied");
+      return;
+    }
     const LiveAudioStream = audioMod.default || audioMod;
     try {
       LiveAudioStream.init({
@@ -209,9 +253,12 @@ export class UltravoxVoiceSession {
       LiveAudioStream.start();
       this.audioStream = LiveAudioStream;
       this.micActive = true;
+      this.opts.onMicStatus?.("ok");
     } catch (e) {
       // Mic failed — keep the session alive for agent audio + arrows.
+      if (__DEV__) console.warn("[session-replay] mic capture failed to start:", e);
       this.audioStream = null;
+      this.opts.onMicStatus?.("unavailable");
     }
   }
 
@@ -241,4 +288,23 @@ export class UltravoxVoiceSession {
   }
 }
 
+// ---- Built-in agent-voice player (Android) ---------------------------------
+/**
+ * Returns a PcmAudioPlayer backed by the SDK's bundled Android AudioTrack
+ * module (SessionReplayPcmPlayer, v0.5.0+), or null when the native module
+ * isn't available (iOS, Expo Go, or host app not rebuilt after upgrading).
+ * Used automatically when the integrator doesn't pass `assistantAudioPlayer`.
+ */
+export function createDefaultPcmPlayer(): PcmAudioPlayer | null {
+  if (Platform.OS !== "android") return null;
+  const native: any = (NativeModules as any).SessionReplayPcmPlayer;
+  if (!native || typeof native.play !== "function") return null;
+  return {
+    start: (sampleRate: number) => { try { native.start(sampleRate); } catch {} },
+    play: (pcm: ArrayBuffer) => { try { native.play(arrayBufferToBase64(pcm)); } catch {} },
+    clear: () => { try { native.clear(); } catch {} },
+    stop: () => { try { native.stop(); } catch {} },
+  };
+}
+
 export { arrayBufferToBase64 };