@akhilpulse/samadhaan-session-replay 0.3.6 → 0.4.0

package/README.md

@@ -27,7 +27,11 @@ export default function App() {
       config={{
         // serverUrl is optional — defaults to https://samadhaan.pulseenergy.io
         appId: "app_xxx",
-        user: { userId: "user_123", email: "user@example.com" },
+        // `user` can be omitted at first (anonymous) and supplied later once the
+        // user logs in. Update the value passed to `config.user` after OTP login
+        // and the SDK automatically backfills the session with the phone/name/
+        // email — so the dashboard stops showing "Anonymous".
+        user: { userId: "user_123", phone: "+91xxxxxxxxxx", email: "user@example.com" },
         appVersion: "1.4.2",
       }}
     >
@@ -68,6 +72,70 @@ import { useSessionReplay } from "@akhilpulse/samadhaan-session-replay";
 const { pause, resume, sessionId, status } = useSessionReplay();
 ```
 
+### Shake-to-Assist (AI voice guide)
+
+**New in 0.4.0.** The user shakes their phone, a voice assistant connects, they ask
+a question out loud ("how do I pay my bill?"), the AI looks at their current screen
+and draws an **on-screen arrow** pointing at the next element to tap — step by step.
+Works in **English and Hindi**. Every assistant session (questions + AI steps +
+screenshots) is recorded and visible in the dashboard.
+
+```tsx
+<SessionReplayProvider
+  config={{
+    appId: "app_xxx",
+    appName: "Pulse Energy",   // spoken by the assistant in its greeting
+    assistantLang: "hi",        // "en" (default) or "hi"
+    // disableAssistant: true,  // turn the whole feature off
+  }}
+>
+  <YourApp />
+</SessionReplayProvider>
+```
+
+Manual trigger (e.g. a "Need help?" button) — useful if you don't install the shake
+dependency, or want a tappable entry point too:
+
+```tsx
+const { triggerAssist, assistantPhase } = useSessionReplay();
+// assistantPhase: "idle" | "connecting" | "listening" | "thinking" | "speaking" | "error"
+<Button title="Need help?" onPress={triggerAssist} />
+```
+
+#### Optional native peer deps
+
+Voice and shake rely on **optional** native modules. The SDK degrades gracefully —
+if a module is missing that capability is simply skipped (no crash):
+
+| Capability                | Package                            | If missing |
+|---------------------------|------------------------------------|------------|
+| Shake detection           | `react-native-shake`               | Use `triggerAssist()` manually |
+| Microphone (voice input)  | `react-native-live-audio-stream`   | Assistant can still speak + draw arrows; just can't hear the user |
+| Agent voice playback      | provide a `PcmAudioPlayer` (see below) | Agent audio is dropped (arrows + transcript still work) |
+
+```bash
+npm install react-native-shake react-native-live-audio-stream
+cd ios && pod install
+```
+
+The agent streams **int16 mono PCM at 16 kHz**. Plug in your own player to hear it:
+
+```tsx
+import type { PcmAudioPlayer } from "@akhilpulse/samadhaan-session-replay";
+
+const myPlayer: PcmAudioPlayer = {
+  start: (sampleRate) => {/* init native player */},
+  play:  (pcm, sampleRate) => {/* feed int16 mono PCM chunk */},
+  clear: () => {/* barge-in: drop buffered audio */},
+  stop:  () => {/* tear down */},
+};
+
+<SessionReplayProvider config={{ appId: "app_xxx", assistantAudioPlayer: myPlayer }}>
+```
+
+After installing native deps on Android, **rebuild the app**
+(`cd android && ./gradlew clean && cd .. && npx react-native run-android`).
+
 ## What it does
 
 - Creates a session against `POST /api/guided-support/sessions` and persists `sessionId` in AsyncStorage so it can be resumed after relaunch.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@akhilpulse/samadhaan-session-replay",
-  "version": "0.3.6",
-  "description": "React Native SDK for Samadhaan Guided Support — session recording, live remote screen-share, take-control, and guided-tour overlays.",
+  "version": "0.4.0",
+  "description": "React Native SDK for Samadhaan Guided Support — session recording, live remote screen-share, take-control, guided-tour overlays, and Shake-to-Assist AI voice guide.",
   "main": "src/index.ts",
   "module": "src/index.ts",
   "types": "src/index.ts",
@@ -40,6 +40,16 @@
     "react": ">=17.0.0",
     "react-native": ">=0.68.0",
     "react-native-view-shot": ">=3.0.0",
-    "@react-native-async-storage/async-storage": ">=1.17.0"
+    "@react-native-async-storage/async-storage": ">=1.17.0",
+    "react-native-shake": ">=5.0.0",
+    "react-native-live-audio-stream": ">=1.1.0"
+  },
+  "peerDependenciesMeta": {
+    "react-native-shake": {
+      "optional": true
+    },
+    "react-native-live-audio-stream": {
+      "optional": true
+    }
   }
 }

package/src/SessionReplayProvider.tsx

@@ -35,6 +35,12 @@ import {
 } from "react-native";
 import AsyncStorage from "@react-native-async-storage/async-storage";
 import { captureRef, captureScreen } from "react-native-view-shot";
+import {
+  subscribeShake,
+  UltravoxVoiceSession,
+  type PcmAudioPlayer,
+  type VoiceState,
+} from "./voice";
 
 // Bundled native module (Android only) that uses PixelCopy to capture the
 // real on-screen pixels — including hardware-accelerated layers (views with
@@ -47,7 +53,7 @@ const SessionReplayPixelCopy: PixelCopyNative | undefined =
 let pixelCopyDisabled = false; // turned true after a runtime failure to avoid retrying every frame
 
 const STORAGE_KEY = "@akhilpulse/samadhaan-session-replay/sessionId";
-const SDK_VERSION = "0.3.6";
+const SDK_VERSION = "0.4.0";
 
 // ---- Module-level session singleton --------------------------------------
 // Multiple SessionReplayProvider mounts (intentional or via React StrictMode's
@@ -80,13 +86,32 @@ type Config = {
   appVersion?: string;
   /** Disable screenshot capture entirely */
   disableScreenshots?: boolean;
+  // ---- Shake-to-Assist (AI voice guide) ----
+  /** Friendly app name spoken by the assistant (defaults to "this app"). */
+  appName?: string;
+  /** Primary language for the voice guide. Default "en". */
+  assistantLang?: "en" | "hi";
+  /** Disable the shake-to-assist voice guide entirely. */
+  disableAssistant?: boolean;
+  /**
+   * Optional streaming-PCM player for the agent's voice. Without it the
+   * assistant still listens + shows the on-screen arrow, but won't speak aloud.
+   * See mobile-sdk/README.md → "Shake-to-Assist".
+   */
+  assistantAudioPlayer?: PcmAudioPlayer;
 };
 
+type AssistantPhase = "idle" | VoiceState;
+
 type Ctx = {
   sessionId: string | null;
   status: "idle" | "active" | "live" | "ended";
   pause: () => void;
   resume: () => void;
+  /** Manually open the AI voice guide (fallback when shake isn't available). */
+  triggerAssist: () => void;
+  /** Current voice-guide phase, or "idle" when closed. */
+  assistantPhase: AssistantPhase;
 };
 
 const SessionReplayContext = createContext<Ctx>({
@@ -94,6 +119,8 @@ const SessionReplayContext = createContext<Ctx>({
   status: "idle",
   pause: () => {},
   resume: () => {},
+  triggerAssist: () => {},
+  assistantPhase: "idle",
 });
 
 export function useSessionReplay() {
@@ -122,6 +149,20 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
   const eventBufRef = useRef<any[]>([]);
   const lastTapRef = useRef<{ x: number; y: number; t: number } | null>(null);
 
+  // ---- Shake-to-Assist (AI voice guide) state ----
+  const [assistantPhase, setAssistantPhase] = useState<AssistantPhase>("idle");
+  const [assistantArrow, setAssistantArrow] = useState<
+    { nx: number; ny: number; label?: string; instruction?: string } | null
+  >(null);
+  const assistantSessionIdRef = useRef<string | null>(null);
+  const voiceRef = useRef<UltravoxVoiceSession | null>(null);
+  const assistantStartingRef = useRef(false);
+  // Incremented on every endAssist so an in-flight startAssist can detect it was
+  // cancelled mid-connect and abort instead of resurrecting the session.
+  const assistEpochRef = useRef(0);
+  // Assistant is "active" while not idle — drives WS connect + frame streaming.
+  const assistantActive = assistantPhase !== "idle";
+
   const flushEvents = useCallback(async () => {
     if (!sessionId || !ingestToken || eventBufRef.current.length === 0) return;
     const events = eventBufRef.current.splice(0, eventBufRef.current.length);
@@ -240,6 +281,43 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     return () => { cancelled = true; };
   }, [config.appId, serverUrl, config.appVersion, config.user, initTick]);
 
+  // ---- Identify: backfill user identity after login ----
+  // A session is usually created while the user is still anonymous (e.g. before
+  // OTP login). Once the host app supplies user info via config.user, we PATCH
+  // the existing session so the dashboard shows the real phone/name/email
+  // instead of "Anonymous". We dedupe on the serialized identity so this only
+  // fires when the identity actually changes, and retry on failure.
+  const lastIdentitySentRef = useRef<string | null>(null);
+  const lastIdentitySessionRef = useRef<string | null>(null);
+  useEffect(() => {
+    if (!sessionId || !ingestToken) return;
+    // A recycled session (server idle-end -> new session) must re-send identity,
+    // so reset the dedupe key whenever the session id changes.
+    if (lastIdentitySessionRef.current !== sessionId) {
+      lastIdentitySessionRef.current = sessionId;
+      lastIdentitySentRef.current = null;
+    }
+    const u = config.user;
+    if (!u) return;
+    const hasIdentity = !!(u.userId || u.userName || u.email || u.phone);
+    if (!hasIdentity) return;
+    const key = JSON.stringify({ userId: u.userId, userName: u.userName, email: u.email, phone: u.phone, metadata: u.metadata });
+    if (lastIdentitySentRef.current === key) return;
+    lastIdentitySentRef.current = key;
+    fetch(`${serverUrl}/api/guided-support/sessions/${sessionId}`, {
+      method: "PATCH",
+      headers: { "Content-Type": "application/json", "x-ingest-token": ingestToken },
+      body: JSON.stringify({ user: u }),
+    }).then((res) => {
+      // Treat non-2xx (e.g. transient 401/5xx) as failure so we retry instead
+      // of permanently deduping on an identity the server never stored.
+      if (!res.ok) lastIdentitySentRef.current = null;
+    }).catch(() => {
+      // allow retry on the next render/identity change
+      lastIdentitySentRef.current = null;
+    });
+  }, [sessionId, ingestToken, serverUrl, config.user]);
+
   // ---- Keyboard tracking (iOS auto-pause) ----
   useEffect(() => {
     const showSub = Keyboard.addListener(Platform.OS === "ios" ? "keyboardWillShow" : "keyboardDidShow", () => setKeyboardVisible(true));
@@ -273,11 +351,14 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
 
   // ---- Screenshot capture ----
   const liveMode = status === "live";
+  // Frames must also stream while the AI voice guide is active so its vision
+  // tool can "see" the current screen.
+  const streamActive = liveMode || assistantActive;
   useEffect(() => {
-    // Only capture screenshots in live mode. Idle sessions emit interaction
-    // events (taps, navigation, etc.) but do not waste CPU/battery/network on
-    // periodic screenshots — frames stream only when an agent is actively viewing.
-    if (config.disableScreenshots || !sessionId || !liveMode) return;
+    // Capture frames in live mode OR while the assistant is active. Otherwise
+    // idle sessions emit interaction events (taps, navigation, etc.) but do not
+    // waste CPU/battery/network on periodic screenshots.
+    if (config.disableScreenshots || !sessionId || !streamActive) return;
     let stopped = false;
     // ~4 fps target — low device load while still feeling responsive. Capture
     // takes time, so the loop self-paces: next tick is scheduled AFTER the previous capture completes.
@@ -340,11 +421,11 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     };
     setTimeout(tick, liveTargetMs);
     return () => { stopped = true; };
-  }, [sessionId, ingestToken, liveMode, paused, keyboardVisible, config.disableScreenshots, serverUrl, queueEvent]);
+  }, [sessionId, ingestToken, streamActive, paused, keyboardVisible, config.disableScreenshots, serverUrl, queueEvent]);
 
-  // ---- WebSocket (live mode) ----
+  // ---- WebSocket (live mode OR assistant active) ----
   useEffect(() => {
-    if (!sessionId || !ingestToken || !liveMode) {
+    if (!sessionId || !ingestToken || !streamActive) {
       if (wsRef.current) { try { wsRef.current.close(); } catch {} wsRef.current = null; }
       return;
     }
@@ -362,7 +443,7 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     };
     ws.onclose = () => { if (wsRef.current === ws) wsRef.current = null; };
     return () => { try { ws.close(); } catch {} };
-  }, [sessionId, ingestToken, liveMode, serverUrl]);
+  }, [sessionId, ingestToken, streamActive, serverUrl]);
 
   // ---- Gesture capture (passive PanResponder) ----
   const panResponder = useMemo(() => PanResponder.create({
@@ -422,6 +503,16 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
         setTourState({ phase: "idle" });
         setPointer(null);
         break;
+      case "assistant_arrow": {
+        // AI voice guide pointing at the next element. Coords are normalized 0..1.
+        if (typeof msg.nx === "number" && typeof msg.ny === "number") {
+          setAssistantArrow({ nx: msg.nx, ny: msg.ny, label: msg.label, instruction: msg.instruction });
+        }
+        break;
+      }
+      case "assistant_arrow_clear":
+        setAssistantArrow(null);
+        break;
     }
   }
 
@@ -433,6 +524,106 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     setTimeout(() => setRipples((r) => r.filter((p) => p.id !== id)), 600);
   }
 
+  // ---- Shake-to-Assist: start / end the AI voice guide ----
+  const endAssist = useCallback(() => {
+    // Null the refs BEFORE stopping the voice session: stop() fires onState
+    // ("ended") which routes back here, and clearing first makes the re-entrant
+    // call a no-op (no double-stop, no duplicate /assistant/end).
+    const asid = assistantSessionIdRef.current;
+    const voice = voiceRef.current;
+    assistEpochRef.current += 1; // cancel any in-flight startAssist
+    assistantSessionIdRef.current = null;
+    voiceRef.current = null;
+    try { voice?.stop(); } catch {}
+    setAssistantArrow(null);
+    setAssistantPhase("idle");
+    if (asid && sessionId && ingestToken) {
+      fetch(`${serverUrl}/api/guided-support/assistant/end`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "x-ingest-token": ingestToken },
+        body: JSON.stringify({ sessionId, assistantSessionId: asid }),
+      }).catch(() => {});
+    }
+  }, [sessionId, ingestToken, serverUrl]);
+
+  const startAssist = useCallback(async () => {
+    if (config.disableAssistant) return;
+    if (!sessionId || !ingestToken) return;
+    if (assistantStartingRef.current || assistantSessionIdRef.current) return;
+    assistantStartingRef.current = true;
+    const epoch = assistEpochRef.current;
+    setAssistantPhase("connecting");
+    try {
+      const res = await fetch(`${serverUrl}/api/guided-support/assistant/start`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "x-ingest-token": ingestToken },
+        body: JSON.stringify({
+          sessionId,
+          lang: config.assistantLang || "en",
+          appName: config.appName,
+        }),
+      });
+      const j = await res.json();
+      if (!res.ok || !j.joinUrl || !j.assistantSessionId) {
+        throw new Error(j?.error || "assistant start failed");
+      }
+      // User closed the assistant while we were connecting — abort and tell the
+      // server to end the just-created call instead of resurrecting it.
+      if (epoch !== assistEpochRef.current) {
+        if (sessionId && ingestToken) {
+          fetch(`${serverUrl}/api/guided-support/assistant/end`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json", "x-ingest-token": ingestToken },
+            body: JSON.stringify({ sessionId, assistantSessionId: j.assistantSessionId }),
+          }).catch(() => {});
+        }
+        return;
+      }
+      assistantSessionIdRef.current = j.assistantSessionId;
+      const voice = new UltravoxVoiceSession({
+        joinUrl: j.joinUrl,
+        sampleRate: 16000,
+        audioPlayer: config.assistantAudioPlayer,
+        onState: (s) => {
+          // Natural socket close ("ended") tears the whole assistant down.
+          if (s === "ended") { endAssist(); return; }
+          setAssistantPhase(s);
+        },
+        onError: () => setAssistantPhase("error"),
+      });
+      voiceRef.current = voice;
+      voice.start();
+    } catch (e) {
+      if (__DEV__) console.warn("[session-replay] shake-to-assist failed to start:", e);
+      assistantSessionIdRef.current = null;
+      setAssistantPhase("error");
+      // Auto-dismiss the error state after a moment.
+      setTimeout(() => setAssistantPhase((p) => (p === "error" ? "idle" : p)), 2500);
+    } finally {
+      assistantStartingRef.current = false;
+    }
+  }, [
+    sessionId, ingestToken, serverUrl, endAssist,
+    config.disableAssistant, config.assistantLang, config.appName, config.assistantAudioPlayer,
+  ]);
+
+  // Shake detection (optional react-native-shake dep). The manual trigger
+  // exposed via context is the fallback when the dep isn't installed.
+  useEffect(() => {
+    if (config.disableAssistant) return;
+    const unsub = subscribeShake(() => {
+      if (!assistantSessionIdRef.current && !assistantStartingRef.current) startAssist();
+    });
+    return unsub;
+  }, [config.disableAssistant, startAssist]);
+
+  // Keep a ref to the latest endAssist so the unmount cleanup can run the full
+  // teardown (stop voice + best-effort /assistant/end) without a stale closure.
+  const endAssistRef = useRef(endAssist);
+  endAssistRef.current = endAssist;
+  // Tear down the assistant (voice + server session) when the provider unmounts.
+  useEffect(() => () => { try { endAssistRef.current(); } catch {} }, []);
+
   // ---- Guided tour state ----
   const [tourState, setTourState] = useState<{ phase: "idle" | "consent" | "active"; agentName?: string }>({ phase: "idle" });
   const [pointer, setPointer] = useState<{ x: number; y: number; t: number } | null>(null);
@@ -511,6 +702,8 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
     status,
     pause: () => setPaused(true),
     resume: () => setPaused(false),
+    triggerAssist: () => { startAssist(); },
+    assistantPhase,
   };
 
   return (
@@ -587,11 +780,105 @@ export function SessionReplayProvider({ config, children }: { config: Config; ch
             )}
           </>
         )}
+
+        {/* Shake-to-Assist: on-screen arrow pointing at the next element to tap. */}
+        {assistantActive && assistantArrow && (() => {
+          const { w, h } = viewSizeRef.current;
+          const px = assistantArrow.nx * (w || 0);
+          const py = assistantArrow.ny * (h || 0);
+          // Keep the label bubble on-screen: flip below the target near the top edge.
+          const below = py < 96;
+          return (
+            <View pointerEvents="none" style={StyleSheet.absoluteFill}>
+              <View style={[styles.assistHalo, { left: px - 26, top: py - 26 }]} />
+              <View style={[styles.assistDot, { left: px - 9, top: py - 9 }]} />
+              {/* Arrow shaft pointing down at the target (or up when flipped). */}
+              <View
+                style={[
+                  styles.assistArrow,
+                  below
+                    ? { left: px - 9, top: py + 12 }
+                    : { left: px - 9, top: py - 30, transform: [{ rotate: "180deg" }] },
+                ]}
+              />
+              {!!(assistantArrow.label || assistantArrow.instruction) && (
+                <View style={[styles.assistBubble, { left: 16, right: 16, [below ? "top" : "bottom"]: below ? py + 36 : (h || 0) - py + 36 } as any]}>
+                  {!!assistantArrow.label && <Text style={styles.assistBubbleLabel}>{assistantArrow.label}</Text>}
+                  {!!assistantArrow.instruction && <Text style={styles.assistBubbleText}>{assistantArrow.instruction}</Text>}
+                </View>
+              )}
+            </View>
+          );
+        })()}
+
+        {/* Shake-to-Assist: bottom voice "wave" control + status. */}
+        {assistantActive && (
+          <View style={styles.assistDock} pointerEvents="box-none">
+            <View style={styles.assistDockCard}>
+              <VoiceWave phase={assistantPhase} />
+              <Text style={styles.assistStatusText}>{assistantStatusLabel(assistantPhase)}</Text>
+              <Pressable
+                onPress={endAssist}
+                style={({ pressed }) => [styles.assistClose, pressed && styles.btnPressed]}
+                hitSlop={10}
+              >
+                <Text style={styles.assistCloseText}>Done</Text>
+              </Pressable>
+            </View>
+          </View>
+        )}
       </View>
     </SessionReplayContext.Provider>
   );
 }
 
+// ---- Shake-to-Assist UI bits ----
+function assistantStatusLabel(phase: AssistantPhase): string {
+  switch (phase) {
+    case "connecting": return "Connecting…";
+    case "listening": return "Listening… ask me anything";
+    case "thinking": return "Looking at your screen…";
+    case "speaking": return "Speaking…";
+    case "error": return "Couldn't start — try again";
+    default: return "";
+  }
+}
+
+/** Animated 5-bar voice wave. Bars idle when not listening/speaking. */
+function VoiceWave({ phase }: { phase: AssistantPhase }) {
+  const active = phase === "listening" || phase === "speaking";
+  const bars = useRef([0, 1, 2, 3, 4].map(() => new Animated.Value(0.3))).current;
+  useEffect(() => {
+    if (!active) {
+      bars.forEach((b) => Animated.timing(b, { toValue: 0.3, duration: 200, useNativeDriver: false }).start());
+      return;
+    }
+    const loops = bars.map((b, i) =>
+      Animated.loop(
+        Animated.sequence([
+          Animated.timing(b, { toValue: 1, duration: 300 + i * 60, useNativeDriver: false }),
+          Animated.timing(b, { toValue: 0.3, duration: 300 + i * 60, useNativeDriver: false }),
+        ])
+      )
+    );
+    loops.forEach((l) => l.start());
+    return () => loops.forEach((l) => l.stop());
+  }, [active, bars]);
+  return (
+    <View style={styles.waveRow}>
+      {bars.map((b, i) => (
+        <Animated.View
+          key={i}
+          style={[
+            styles.waveBar,
+            { transform: [{ scaleY: b }], opacity: active ? 1 : 0.5 },
+          ]}
+        />
+      ))}
+    </View>
+  );
+}
+
 // ---- helpers ----
 async function uploadFrameAsync(
   serverUrl: string,
@@ -702,4 +989,48 @@ const styles = StyleSheet.create({
     position: "absolute", width: 48, height: 48, borderRadius: 24,
     backgroundColor: "rgba(239,68,68,0.25)",
   },
+  // ---- Shake-to-Assist arrow + bubble ----
+  assistHalo: {
+    position: "absolute", width: 52, height: 52, borderRadius: 26,
+    backgroundColor: "rgba(59,130,246,0.22)",
+  },
+  assistDot: {
+    position: "absolute", width: 18, height: 18, borderRadius: 9,
+    backgroundColor: "#3b82f6", borderWidth: 3, borderColor: "#fff",
+    shadowColor: "#3b82f6", shadowOpacity: 0.9, shadowRadius: 8,
+  },
+  // A simple chevron-like arrow built from a rotated bordered box.
+  assistArrow: {
+    position: "absolute", width: 18, height: 18,
+    borderRightWidth: 4, borderBottomWidth: 4, borderColor: "#3b82f6",
+    transform: [{ rotate: "45deg" }],
+  },
+  assistBubble: {
+    position: "absolute", backgroundColor: "rgba(17,24,39,0.94)",
+    borderRadius: 12, paddingVertical: 10, paddingHorizontal: 14,
+    shadowColor: "#000", shadowOpacity: 0.3, shadowRadius: 10, shadowOffset: { width: 0, height: 4 },
+  },
+  assistBubbleLabel: { color: "#93c5fd", fontSize: 12, fontWeight: "700", marginBottom: 2 },
+  assistBubbleText: { color: "#fff", fontSize: 14, lineHeight: 19 },
+  // ---- Shake-to-Assist bottom dock ----
+  assistDock: {
+    position: "absolute", left: 0, right: 0, bottom: 0,
+    alignItems: "center", paddingBottom: 28, paddingHorizontal: 16,
+  },
+  assistDockCard: {
+    flexDirection: "row", alignItems: "center",
+    backgroundColor: "rgba(17,24,39,0.96)", borderRadius: 999,
+    paddingVertical: 10, paddingHorizontal: 16, maxWidth: "100%",
+    shadowColor: "#000", shadowOpacity: 0.3, shadowRadius: 12, shadowOffset: { width: 0, height: 6 },
+  },
+  assistStatusText: { color: "#e5e7eb", fontSize: 14, fontWeight: "600", marginHorizontal: 12, flexShrink: 1 },
+  assistClose: {
+    backgroundColor: "#3b82f6", borderRadius: 999,
+    paddingVertical: 8, paddingHorizontal: 16, minHeight: 36, justifyContent: "center",
+  },
+  assistCloseText: { color: "#fff", fontSize: 14, fontWeight: "700" },
+  waveRow: { flexDirection: "row", alignItems: "center", height: 28, width: 44 },
+  waveBar: {
+    width: 4, height: 22, marginHorizontal: 2, borderRadius: 2, backgroundColor: "#60a5fa",
+  },
 });

package/src/index.ts

@@ -4,3 +4,14 @@ export {
   useSessionReplay,
   DEFAULT_SERVER_URL,
 } from "./SessionReplayProvider";
+
+// Shake-to-Assist voice transport (optional advanced usage / custom audio).
+export {
+  subscribeShake,
+  UltravoxVoiceSession,
+} from "./voice";
+export type {
+  PcmAudioPlayer,
+  VoiceState,
+  StartVoiceOpts,
+} from "./voice";

package/src/voice.ts

@@ -0,0 +1,244 @@
+/**
+ * Shake-to-Assist voice transport for @akhilpulse/samadhaan-session-replay.
+ *
+ * Two concerns live here:
+ *   1. subscribeShake()        — optional accelerometer shake detection.
+ *   2. UltravoxVoiceSession    — duplex raw-PCM voice over the Ultravox
+ *                                serverWebSocket medium (mic up / agent down).
+ *
+ * Everything degrades gracefully: the native audio modules are OPTIONAL peer
+ * deps. If they aren't installed the rest of the SDK keeps working — the
+ * assistant just can't capture/play audio, and the manual trigger + on-screen
+ * arrow still function.
+ *
+ * OPTIONAL peer deps (install in the host app for full voice):
+ *   - react-native-shake            (shake detection; otherwise manual button only)
+ *   - react-native-live-audio-stream(microphone PCM16 capture)
+ *   - an audio player for streamed PCM16 (pass `audioPlayer` to startVoiceSession
+ *     if you have one; otherwise the agent's voice is silently dropped).
+ *
+ * Server audio format is int16 mono PCM at `sampleRate` (default 16 kHz),
+ * matching server/ultravox.ts (serverWebSocket inputSampleRate/outputSampleRate).
+ */
+
+// Loads an optional native module without crashing Metro when it's absent.
+function loadOptional<T = any>(name: string): T | null {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires, global-require
+    return require(name) as T;
+  } catch {
+    return null;
+  }
+}
+
+// ---- Shake detection -----------------------------------------------------
+/**
+ * Subscribe to phone-shake events. Returns an unsubscribe function.
+ * Uses `react-native-shake` if installed; otherwise it's a no-op and the host
+ * app should rely on the manual trigger exposed via useSessionReplay().
+ */
+export function subscribeShake(onShake: () => void): () => void {
+  const mod: any = loadOptional("react-native-shake");
+  if (!mod) return () => {};
+  try {
+    // react-native-shake v5+: default export is { addListener }
+    const api = mod.default || mod;
+    const sub = api.addListener(onShake);
+    return () => { try { sub?.remove?.(); } catch {} };
+  } catch {
+    return () => {};
+  }
+}
+
+// ---- Audio helpers -------------------------------------------------------
+function base64ToArrayBuffer(b64: string): ArrayBuffer {
+  const bin =
+    typeof globalThis.atob === "function"
+      ? globalThis.atob(b64)
+      : // RN always ships atob in modern versions; Buffer is the fallback.
+        (globalThis as any).Buffer
+        ? (globalThis as any).Buffer.from(b64, "base64").toString("binary")
+        : "";
+  const len = bin.length;
+  const bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) bytes[i] = bin.charCodeAt(i);
+  return bytes.buffer;
+}
+
+function arrayBufferToBase64(buf: ArrayBuffer): string {
+  const bytes = new Uint8Array(buf);
+  let bin = "";
+  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);
+  if (typeof globalThis.btoa === "function") return globalThis.btoa(bin);
+  if ((globalThis as any).Buffer) return (globalThis as any).Buffer.from(bin, "binary").toString("base64");
+  return "";
+}
+
+/** Pluggable streaming-PCM player so integrators can wire their own native module. */
+export type PcmAudioPlayer = {
+  /** Called once before the first chunk. */
+  start?: (sampleRate: number) => void;
+  /** Feed one int16 mono PCM chunk (ArrayBuffer). */
+  play: (pcm: ArrayBuffer, sampleRate: number) => void;
+  /** Drop any buffered audio (server asked us to interrupt). */
+  clear?: () => void;
+  /** Tear down. */
+  stop?: () => void;
+};
+
+export type VoiceState = "connecting" | "listening" | "thinking" | "speaking" | "ended" | "error";
+
+// react-native-live-audio-stream is a singleton with no per-listener removal,
+// so we install the "data" handler exactly once and route frames to whichever
+// session is currently active. This prevents listener accumulation (and
+// duplicate frame sends) across repeated assist sessions.
+let _micListenerInstalled = false;
+let _activeMicSession: UltravoxVoiceSession | null = null;
+
+export type StartVoiceOpts = {
+  /** wss joinUrl returned by /assistant/start. */
+  joinUrl: string;
+  /** Must match the server medium sample rate (server/ultravox.ts → 16000). */
+  sampleRate?: number;
+  /** Optional player for the agent's streamed voice. */
+  audioPlayer?: PcmAudioPlayer;
+  onState?: (s: VoiceState) => void;
+  /** Live transcript text (both user + agent), with role + final flag. */
+  onTranscript?: (t: { role: string; text: string; final: boolean }) => void;
+  onError?: (e: Error) => void;
+};
+
+/**
+ * A single realtime voice session. Connects the WS, streams mic PCM up, plays
+ * agent PCM down (if a player is provided), and surfaces state + transcripts.
+ */
+export class UltravoxVoiceSession {
+  private ws: WebSocket | null = null;
+  private opts: StartVoiceOpts;
+  private sampleRate: number;
+  private audioStream: any = null;
+  private micActive = false;
+  private stopped = false;
+
+  constructor(opts: StartVoiceOpts) {
+    this.opts = opts;
+    this.sampleRate = opts.sampleRate || 16000;
+  }
+
+  start() {
+    this.opts.onState?.("connecting");
+    let ws: WebSocket;
+    try {
+      ws = new WebSocket(this.opts.joinUrl);
+    } catch (e: any) {
+      this.fail(e);
+      return;
+    }
+    ws.binaryType = "arraybuffer" as any;
+    this.ws = ws;
+
+    ws.onopen = () => {
+      if (this.stopped) return;
+      this.opts.audioPlayer?.start?.(this.sampleRate);
+      this.startMic();
+      this.opts.onState?.("listening");
+    };
+    ws.onmessage = (ev: any) => {
+      const data = ev.data;
+      if (data instanceof ArrayBuffer) {
+        // Agent voice — raw int16 PCM at outputSampleRate.
+        try { this.opts.audioPlayer?.play(data, this.sampleRate); } catch {}
+        return;
+      }
+      if (typeof data === "string") {
+        this.handleControl(data);
+      }
+    };
+    ws.onerror = (e: any) => { this.fail(new Error(e?.message || "voice websocket error")); };
+    ws.onclose = () => { if (!this.stopped) this.opts.onState?.("ended"); };
+  }
+
+  private handleControl(raw: string) {
+    let msg: any;
+    try { msg = JSON.parse(raw); } catch { return; }
+    switch (msg.type) {
+      case "state":
+        // Ultravox emits agent state: "listening" | "thinking" | "speaking".
+        if (msg.state === "listening" || msg.state === "thinking" || msg.state === "speaking") {
+          this.opts.onState?.(msg.state);
+        }
+        break;
+      case "transcript":
+        this.opts.onTranscript?.({
+          role: msg.role || "agent",
+          text: msg.text ?? msg.delta ?? "",
+          final: !!msg.final,
+        });
+        break;
+      case "playback_clear_buffer":
+        // Barge-in: user started talking, drop queued agent audio.
+        try { this.opts.audioPlayer?.clear?.(); } catch {}
+        break;
+      default:
+        break;
+    }
+  }
+
+  private startMic() {
+    const audioMod: any = loadOptional("react-native-live-audio-stream");
+    if (!audioMod) return; // No mic module — assistant can still speak/guide.
+    const LiveAudioStream = audioMod.default || audioMod;
+    try {
+      LiveAudioStream.init({
+        sampleRate: this.sampleRate,
+        channels: 1,
+        bitsPerSample: 16,
+        audioSource: 6, // Android VOICE_RECOGNITION
+        bufferSize: 4096,
+      });
+      // Install the data handler once, route to the active session.
+      if (!_micListenerInstalled) {
+        _micListenerInstalled = true;
+        LiveAudioStream.on("data", (b64: string) => {
+          const s = _activeMicSession;
+          if (!s || s.stopped || !s.ws || s.ws.readyState !== WebSocket.OPEN) return;
+          try { s.ws.send(base64ToArrayBuffer(b64)); } catch {}
+        });
+      }
+      _activeMicSession = this;
+      LiveAudioStream.start();
+      this.audioStream = LiveAudioStream;
+      this.micActive = true;
+    } catch (e) {
+      // Mic failed — keep the session alive for agent audio + arrows.
+      this.audioStream = null;
+    }
+  }
+
+  /** Send a text message to the agent (optional helper for typed input). */
+  sendText(text: string) {
+    if (this.ws?.readyState === WebSocket.OPEN) {
+      try { this.ws.send(JSON.stringify({ type: "input_text_message", text })); } catch {}
+    }
+  }
+
+  stop() {
+    this.stopped = true;
+    if (_activeMicSession === this) _activeMicSession = null;
+    if (this.micActive && this.audioStream) {
+      try { this.audioStream.stop(); } catch {}
+    }
+    this.micActive = false;
+    try { this.opts.audioPlayer?.stop?.(); } catch {}
+    if (this.ws) { try { this.ws.close(); } catch {} this.ws = null; }
+    this.opts.onState?.("ended");
+  }
+
+  private fail(e: Error) {
+    this.opts.onError?.(e);
+    this.opts.onState?.("error");
+    this.stop();
+  }
+}
+
+export { arrayBufferToBase64 };