@facepe/widget-mobile-app 1.0.0 → 1.1.0

package/README.md

@@ -1,32 +1,35 @@
 # @facepe/widget-mobile-app
 
-The official **FacePe AI 3D Avatar Live Streaming SDK** for **React Native** (supporting both **iOS** and **Android**).
+The official **FacePe AI Digital Human Live Streaming SDK** for **React Native** (supporting both **iOS** and **Android**).
 
-Embed interactive, photorealistic, lip-synced 3D digital avatars directly inside your native mobile apps with real-time speech recognition, natural audio dialog, and event callbacks.
+Embed interactive, photorealistic, lip-synced 3D digital avatars directly inside your native mobile apps with pure WebRTC media streaming, native microphone capture, bidirectional voice dialog, and live data events — **without webviews**.
 
 ---
 
 ## 📱 Features
 
-- **⚡ Low-Latency Live Video & Audio**: Real-time WebRTC media streaming for avatar rendering.
-- **🗣️ Two-Way Audio Conversation**: Bidirectional voice talk with interrupt handling & VAD.
-- **🍎 Full iOS Support**: Compatible with iOS 13.0+ (iPhone & iPad).
-- **🤖 Full Android Support**: Compatible with Android 7.0+ (API 24+).
-- **💬 Real-Time Transcripts**: Captions and user transcripts via `onTranscript`.
-- **🛒 Conversational Ordering**: Extract order items and payment intents directly in your app.
+- **⚡ Native WebRTC Streaming**: Pure native media streaming powered by `@livekit/react-native` (no WebViews or embedded iframes).
+- **🗣️ Natural Two-Way Voice Dialog**: Real-time microphone capture with hardware echo cancellation, noise suppression, and server VAD.
+- **🍎 Full iOS Support**: Hardware-accelerated video decoding on iOS 13.0+ (iPhone & iPad).
+- **🤖 Full Android Support**: Hardware-accelerated video decoding on Android 7.0+ (API 24+).
+- **💬 Real-Time Transcriptions**: Bidirectional captions for avatar speech and user speech.
+- **🛒 Conversational Commerce**: Automatic detection of cart items, orders, and payment intents.
+- **🎛️ Imperative Controls**: Ref methods to start/stop stream, mute/unmute mic, speak text, or interrupt speech.
 
 ---
 
 ## 📦 Installation
 
-In your React Native / Expo project:
+In your React Native project:
 
 ```bash
-npm install @facepe/widget-mobile-app react-native-webview
+npm install @facepe/widget-mobile-app @livekit/react-native @livekit/react-native-webrtc
 ```
+
 *(or using yarn / pnpm)*:
+
 ```bash
-yarn add @facepe/widget-mobile-app react-native-webview
+yarn add @facepe/widget-mobile-app @livekit/react-native @livekit/react-native-webrtc
 ```
 
 ---
@@ -35,13 +38,13 @@ yarn add @facepe/widget-mobile-app react-native-webview
 
 ### iOS Setup (`ios/Podfile` & `Info.plist`)
 
-1. Open `ios/YourAppName/Info.plist` and add permission strings for microphone and camera:
+1. Open `ios/YourAppName/Info.plist` and add permission strings for microphone and audio:
 
 ```xml
 <key>NSMicrophoneUsageDescription</key>
-<string>We need access to your microphone to converse with the AI avatar.</string>
+<string>We need access to your microphone to converse with the FacePe AI avatar.</string>
 <key>NSCameraUsageDescription</key>
-<string>We need access to your camera for interactive features.</string>
+<string>Camera access for interactive features.</string>
 ```
 
 2. Run CocoaPods:
@@ -60,6 +63,7 @@ Open `android/app/src/main/AndroidManifest.xml` and include permissions:
 <uses-permission android:name="android.permission.RECORD_AUDIO" />
 <uses-permission android:name="android.permission.CAMERA" />
 <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
 ```
 
 ---
@@ -68,10 +72,10 @@ Open `android/app/src/main/AndroidManifest.xml` and include permissions:
 
 ### 1. Server-side Token Endpoint
 
-Exchange your `accessKey` and `secretKey` on your secure backend (Express / Next.js / Python / Go) so secrets are never exposed on mobile:
+Exchange your `accessKey` and `secretKey` on your secure backend (Node.js / Express / Next.js / Python / Go) so credentials are never exposed inside mobile apps:
 
 ```javascript
-// POST /api/facepe-token
+// Express server route: GET /api/facepe-token
 app.get('/api/facepe-token', async (req, res) => {
   const response = await fetch('https://api.facepe.ai/api/v1/sdk/token', {
     method: 'POST',
@@ -92,17 +96,21 @@ app.get('/api/facepe-token', async (req, res) => {
 ### 2. React Native Component
 
 ```tsx
-import React from 'react';
-import { StyleSheet, View, SafeAreaView, Text } from 'react-native';
+import React, { useRef } from 'react';
+import { StyleSheet, View, SafeAreaView, Text, TouchableOpacity } from 'react-native';
 import {
   FacePeMobileProvider,
   FacePeMobileAvatar,
+  FacePeMobileAvatarRef,
+  useFacePeAvatar,
 } from '@facepe/widget-mobile-app';
 
 export default function App() {
+  const avatarRef = useRef<FacePeMobileAvatarRef>(null);
+
   return (
     <SafeAreaView style={styles.container}>
-      <Text style={styles.title}>FacePe AI Digital Human</Text>
+      <Text style={styles.title}>FacePe AI Native Digital Human</Text>
 
       <FacePeMobileProvider
         apiBaseUrl="https://api.facepe.ai/api/v1/sdk"
@@ -111,41 +119,117 @@ export default function App() {
           return await res.text();
         }}
       >
-        <View style={styles.avatarContainer}>
-          <FacePeMobileAvatar
-            language="en"
-            onTranscript={({ payload }) => {
-              console.log(`[${payload.role}]:`, payload.text);
-            }}
-            onSpeakStart={() => console.log('Avatar speaking')}
-            onSpeakEnd={() => console.log('Avatar stopped speaking')}
-            onOrderUpdate={({ items }) => console.log('Order update:', items)}
-            onCheckout={() => console.log('Checkout requested!')}
-          />
-        </View>
+        <AvatarScreen avatarRef={avatarRef} />
       </FacePeMobileProvider>
     </SafeAreaView>
   );
 }
 
+function AvatarScreen({ avatarRef }: { avatarRef: React.RefObject<FacePeMobileAvatarRef> }) {
+  const { transcripts, isAvatarSpeaking, avatarProps } = useFacePeAvatar();
+
+  return (
+    <View style={styles.content}>
+      <View style={styles.avatarContainer}>
+        <FacePeMobileAvatar
+          ref={avatarRef}
+          language="en"
+          objectFit="cover"
+          autoStart={true}
+          enableMic={true}
+          {...avatarProps}
+        />
+      </View>
+
+      {/* Control bar */}
+      <View style={styles.controls}>
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => avatarRef.current?.toggleMic()}
+        >
+          <Text style={styles.buttonText}>Toggle Mic</Text>
+        </TouchableOpacity>
+
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => avatarRef.current?.interrupt()}
+        >
+          <Text style={styles.buttonText}>Interrupt</Text>
+        </TouchableOpacity>
+      </View>
+
+      {/* Real-time speech transcript log */}
+      <View style={styles.transcriptBox}>
+        {transcripts.slice(-3).map((item, idx) => (
+          <Text key={idx} style={styles.transcriptLine}>
+            <Text style={item.role === 'assistant' ? styles.avatarTag : styles.userTag}>
+              {item.role === 'assistant' ? 'Avatar: ' : 'You: '}
+            </Text>
+            {item.text}
+          </Text>
+        ))}
+      </View>
+    </View>
+  );
+}
+
 const styles = StyleSheet.create({
   container: {
     flex: 1,
     backgroundColor: '#0F172A',
   },
+  content: {
+    flex: 1,
+    padding: 16,
+  },
   title: {
     color: '#FFFFFF',
     fontSize: 20,
     fontWeight: 'bold',
     textAlign: 'center',
-    marginVertical: 16,
+    marginVertical: 12,
   },
   avatarContainer: {
     flex: 1,
-    marginHorizontal: 16,
-    marginBottom: 24,
     borderRadius: 24,
     overflow: 'hidden',
+    backgroundColor: '#1E293B',
+  },
+  controls: {
+    flexDirection: 'row',
+    justifyContent: 'center',
+    gap: 12,
+    marginVertical: 14,
+  },
+  button: {
+    backgroundColor: '#7C3AED',
+    paddingHorizontal: 20,
+    paddingVertical: 10,
+    borderRadius: 12,
+  },
+  buttonText: {
+    color: '#FFFFFF',
+    fontWeight: '600',
+    fontSize: 14,
+  },
+  transcriptBox: {
+    backgroundColor: '#1E293B',
+    padding: 12,
+    borderRadius: 12,
+    minHeight: 80,
+  },
+  transcriptLine: {
+    color: '#E2E8F0',
+    fontSize: 13,
+    marginBottom: 4,
+  },
+  avatarTag: {
+    color: '#A78BFA',
+    fontWeight: '700',
+  },
+  userTag: {
+    color: '#60A5FA',
+    fontWeight: '700',
   },
 });
 ```
@@ -156,19 +240,48 @@ const styles = StyleSheet.create({
 
 ### `<FacePeMobileAvatar />`
 
-| Prop | Type | Description |
-| :--- | :--- | :--- |
-| `avatarKey` | `string` | Optional specific avatar key assigned to your company. |
-| `language` | `string` | Spoken language (`'en'`, `'hi'`, `'es'`, `'ta'`, `'te'`, etc.). |
-| `style` | `ViewStyle` | Style applied to container element. |
-| `onTranscript` | `function` | Fires whenever the user or avatar utters a phrase. |
-| `onSpeakStart` | `function` | Fires when the avatar begins vocal response. |
-| `onSpeakEnd` | `function` | Fires when the avatar stops speaking. |
-| `onUserSpeakStart` | `function` | Fires when the user starts speaking (voice activity). |
-| `onUserSpeakEnd` | `function` | Fires when the user pauses / stops speaking. |
-| `onOrderUpdate` | `function` | Fires when items are detected in conversation. |
-| `onCheckout` | `function` | Fires when customer signals intent to pay. |
-| `onError` | `function` | Error callback `{ code, message }`. |
+| Prop | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `avatarKey` | `string` | *(optional)* | Specific avatar key or ID assigned to your company. |
+| `language` | `string` | `'en'` | Spoken language code (`'en'`, `'hi'`, `'es'`, `'ta'`, `'te'`, etc.). |
+| `style` | `ViewStyle` | `undefined` | Style applied to avatar container. |
+| `videoStyle` | `ViewStyle` | `undefined` | Style applied to native video view. |
+| `objectFit` | `'cover' \| 'contain'` | `'cover'` | Video aspect scaling mode. |
+| `mirror` | `boolean` | `false` | Mirror video orientation. |
+| `autoStart` | `boolean` | `true` | Automatically connect and start stream on mount. |
+| `enableMic` | `boolean` | `true` | Automatically enable microphone on connect. |
+| `onTranscript` | `function` | `undefined` | Fires when speech transcription arrives. |
+| `onSpeakStart` | `function` | `undefined` | Fires when avatar starts speaking. |
+| `onSpeakEnd` | `function` | `undefined` | Fires when avatar finishes speaking. |
+| `onUserSpeakStart` | `function` | `undefined` | Fires when user voice activity begins. |
+| `onUserSpeakEnd` | `function` | `undefined` | Fires when user voice activity ends. |
+| `onOrderUpdate` | `function` | `undefined` | Fires when cart items or intents are recognized. |
+| `onCheckout` | `function` | `undefined` | Fires when customer requests checkout / payment. |
+| `onStatusChange` | `function` | `undefined` | Stream lifecycle status: `'idle' \| 'connecting' \| 'connected' \| 'reconnecting' \| 'disconnected' \| 'error'`. |
+| `onError` | `function` | `undefined` | Stream error callback `{ code, message }`. |
+
+---
+
+### Imperative Ref (`FacePeMobileAvatarRef`)
+
+```tsx
+const avatarRef = useRef<FacePeMobileAvatarRef>(null);
+
+// Start streaming session
+await avatarRef.current?.start();
+
+// Stop streaming session & finalize billing usage
+await avatarRef.current?.stop();
+
+// Toggle mic on/off (returns new state boolean)
+const isNowMuted = await avatarRef.current?.toggleMic();
+
+// Make avatar speak programmatically
+await avatarRef.current?.speak("Hello! How can I assist you today?");
+
+// Interrupt avatar speech immediately
+await avatarRef.current?.interrupt();
+```
 
 ---
 

package/dist/FacePeMobileAvatar.d.ts

@@ -1,8 +1,3 @@
 import React from 'react';
-import type { FacePeMobileAvatarProps } from './types';
-export interface FacePeMobileAvatarRef {
-    start: () => Promise<void>;
-    stop: () => Promise<void>;
-    toggleMic: () => Promise<void>;
-}
+import type { FacePeMobileAvatarProps, FacePeMobileAvatarRef } from './types';
 export declare const FacePeMobileAvatar: React.ForwardRefExoticComponent<FacePeMobileAvatarProps & React.RefAttributes<FacePeMobileAvatarRef>>;

package/dist/FacePeMobileAvatar.js

@@ -1,100 +1,445 @@
-import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
-import { useRef, useState, useImperativeHandle, forwardRef } from 'react';
-import { View, StyleSheet, ActivityIndicator, Text } from 'react-native';
+import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
+import { useEffect, useRef, useState, useImperativeHandle, forwardRef, useCallback, } from 'react';
+import { View, StyleSheet, ActivityIndicator, Text, TouchableOpacity } from 'react-native';
 import { useFacePeMobileContext } from './FacePeMobileProvider';
-export const FacePeMobileAvatar = forwardRef(({ avatarKey, language = 'en', style, onTranscript, onSpeakStart, onSpeakEnd, onUserSpeakStart, onUserSpeakEnd, onOrderUpdate, onCheckout, onError, }, ref) => {
-    const { token, loading, error, apiBaseUrl } = useFacePeMobileContext();
-    const [streamReady, setStreamReady] = useState(false);
-    const webViewRef = useRef(null);
-    // Build the mobile streaming URL
-    const streamUrl = (() => {
-        if (!token)
-            return '';
-        const base = apiBaseUrl.replace(/\/sdk$/, '').replace(/\/api\/v1$/, '');
-        const search = new URLSearchParams({
-            token,
-            embedded: '1',
-            mobile: '1',
-            lang: language,
-        });
-        if (avatarKey)
-            search.set('avatar', avatarKey);
-        return `${base}/?${search.toString()}`;
-    })();
+// Resolve @livekit/react-native dynamically at runtime so bare/Expo projects can run smoothly
+function getLiveKitRN() {
+    try {
+        return require('@livekit/react-native');
+    }
+    catch {
+        return null;
+    }
+}
+let globalsRegistered = false;
+function ensureLiveKitGlobals(LiveKitRN) {
+    if (globalsRegistered || !LiveKitRN)
+        return;
+    try {
+        if (typeof LiveKitRN.registerGlobals === 'function') {
+            LiveKitRN.registerGlobals();
+            globalsRegistered = true;
+        }
+    }
+    catch (err) {
+        console.warn('[FacePeMobileAvatar] registerGlobals warning:', err);
+    }
+}
+export const FacePeMobileAvatar = forwardRef(({ avatarKey, language = 'en', style, videoStyle, objectFit = 'cover', mirror = false, autoStart = true, enableMic = true, onTranscript, onSpeakStart, onSpeakEnd, onUserSpeakStart, onUserSpeakEnd, onOrderUpdate, onCheckout, onStatusChange, onError, }, ref) => {
+    const { token, loading: tokenLoading, error: tokenError, apiBaseUrl, refreshToken } = useFacePeMobileContext();
+    const [status, setStatus] = useState('idle');
+    const [videoTrack, setVideoTrack] = useState(null);
+    const [isMicEnabled, setIsMicEnabled] = useState(enableMic);
+    const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
+    const [isUserSpeaking, setIsUserSpeaking] = useState(false);
+    const [errorMessage, setErrorMessage] = useState(null);
+    const LiveKitRN = getLiveKitRN();
+    const hasLiveKit = !!LiveKitRN;
+    const roomRef = useRef(null);
+    const usageIdRef = useRef(null);
+    const mountedRef = useRef(true);
+    const isConnectingRef = useRef(false);
+    const tokenRef = useRef(token);
+    tokenRef.current = token;
+    // Helper: Safely emit status changes
+    const updateStatus = useCallback((newStatus) => {
+        if (!mountedRef.current)
+            return;
+        setStatus(newStatus);
+        onStatusChange?.(newStatus);
+    }, [onStatusChange]);
+    // Helper: Safely publish data channel packet
+    const sendData = (msg) => {
+        const room = roomRef.current;
+        if (!room || room.state !== 'connected')
+            return;
+        try {
+            const jsonStr = JSON.stringify(msg);
+            let payload;
+            if (typeof TextEncoder !== 'undefined') {
+                payload = new TextEncoder().encode(jsonStr);
+            }
+            else {
+                const arr = new Uint8Array(jsonStr.length);
+                for (let i = 0; i < jsonStr.length; i++) {
+                    arr[i] = jsonStr.charCodeAt(i);
+                }
+                payload = arr;
+            }
+            room.localParticipant.publishData(payload, { reliable: true });
+        }
+        catch (err) {
+            console.warn('[FacePeMobileAvatar] Error publishing data packet:', err);
+        }
+    };
+    // Helper: Notify FacePe backend that session ended (for billing/usage)
+    const endUsage = async () => {
+        const uId = usageIdRef.current;
+        usageIdRef.current = null;
+        const currentToken = tokenRef.current;
+        if (!uId || !currentToken)
+            return;
+        try {
+            const cleanBase = apiBaseUrl.replace(/\/+$/, '');
+            await fetch(`${cleanBase}/avatar-sessions/${uId}/end`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${currentToken}`,
+                },
+            });
+        }
+        catch {
+            // ignore background teardown error
+        }
+    };
+    // Start streaming session
+    const startStream = useCallback(async () => {
+        if (isConnectingRef.current || roomRef.current)
+            return;
+        if (!hasLiveKit) {
+            const msg = 'Please install `@livekit/react-native` and `@livekit/react-native-webrtc` to enable native AI avatar streaming.';
+            setErrorMessage(msg);
+            updateStatus('error');
+            onError?.({ code: 'MISSING_DEPENDENCY', message: msg });
+            return;
+        }
+        isConnectingRef.current = true;
+        setErrorMessage(null);
+        updateStatus('connecting');
+        try {
+            ensureLiveKitGlobals(LiveKitRN);
+            // Ensure active SDK auth token
+            let currentToken = tokenRef.current;
+            if (!currentToken) {
+                currentToken = await refreshToken();
+            }
+            if (!currentToken) {
+                throw new Error('Authentication token not available. Please verify your token provider.');
+            }
+            const cleanBase = apiBaseUrl.replace(/\/+$/, '');
+            // Resolve target avatar ID
+            let targetAvatarId = avatarKey;
+            if (!targetAvatarId) {
+                // If no avatar key passed, retrieve assigned avatars from /sdk/session
+                const sessionRes = await fetch(`${cleanBase}/session`, {
+                    method: 'GET',
+                    headers: {
+                        Authorization: `Bearer ${currentToken}`,
+                    },
+                });
+                if (sessionRes.ok) {
+                    const sessionData = await sessionRes.json();
+                    const avatars = sessionData?.data?.avatars || sessionData?.avatars || [];
+                    if (avatars.length > 0) {
+                        targetAvatarId = avatars[0].id || avatars[0].key;
+                    }
+                }
+            }
+            if (!targetAvatarId) {
+                throw new Error('No Avatar configured or assigned to this token.');
+            }
+            // Request live streaming session from FacePe SDK backend
+            const sessionResp = await fetch(`${cleanBase}/avatars/${targetAvatarId}/sessions`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${currentToken}`,
+                },
+                body: JSON.stringify({ language }),
+            });
+            if (!sessionResp.ok) {
+                const errBody = await sessionResp.json().catch(() => ({}));
+                const errMsg = errBody?.error?.message || errBody?.message || 'Failed to initialize avatar session';
+                throw new Error(errMsg);
+            }
+            const sessionPayload = await sessionResp.json();
+            const sessionInfo = sessionPayload?.data || sessionPayload;
+            usageIdRef.current = sessionInfo?.usageId || null;
+            const connection = sessionInfo?.connection;
+            if (!connection || connection.type !== 'room' || !connection.url || !connection.token) {
+                throw new Error('Avatar server did not return a valid LiveKit room connection.');
+            }
+            // Initialize LiveKit Room natively
+            const room = new LiveKitRN.Room({
+                adaptiveStream: true,
+                dynacast: true,
+            });
+            roomRef.current = room;
+            // Start Audio Session on iOS/Android
+            if (LiveKitRN.AudioSession?.startAudioSession) {
+                try {
+                    await LiveKitRN.AudioSession.startAudioSession();
+                }
+                catch {
+                    // non-fatal
+                }
+            }
+            // 1. Remote video track subscription
+            room.on(LiveKitRN.RoomEvent.TrackSubscribed, (track) => {
+                if (!mountedRef.current)
+                    return;
+                const isVideo = track.kind === 'video' ||
+                    (LiveKitRN.Track?.Kind?.Video && track.kind === LiveKitRN.Track.Kind.Video);
+                if (isVideo) {
+                    setVideoTrack(track);
+                    updateStatus('connected');
+                }
+            });
+            room.on(LiveKitRN.RoomEvent.TrackUnsubscribed, (track) => {
+                if (!mountedRef.current)
+                    return;
+                const isVideo = track.kind === 'video' ||
+                    (LiveKitRN.Track?.Kind?.Video && track.kind === LiveKitRN.Track.Kind.Video);
+                if (isVideo) {
+                    setVideoTrack(null);
+                }
+            });
+            // 2. Active speaker detection
+            room.on(LiveKitRN.RoomEvent.ActiveSpeakersChanged, (speakers) => {
+                if (!mountedRef.current)
+                    return;
+                const hasAssistant = speakers.some((p) => !p.isLocal);
+                const hasUser = speakers.some((p) => p.isLocal);
+                setIsAvatarSpeaking(hasAssistant);
+                if (hasAssistant)
+                    onSpeakStart?.();
+                else
+                    onSpeakEnd?.();
+                setIsUserSpeaking(hasUser);
+                if (hasUser)
+                    onUserSpeakStart?.();
+                else
+                    onUserSpeakEnd?.();
+            });
+            // 3. Bidirectional data-channel events (transcriptions, orders, interrupts)
+            room.on(LiveKitRN.RoomEvent.DataReceived, (payload) => {
+                if (!mountedRef.current)
+                    return;
+                try {
+                    let text = '';
+                    if (typeof TextDecoder !== 'undefined') {
+                        text = new TextDecoder().decode(payload);
+                    }
+                    else {
+                        text = String.fromCharCode.apply(null, Array.from(payload));
+                    }
+                    const msg = JSON.parse(text);
+                    const type = String(msg.type ?? msg.event_type ?? '');
+                    // Speaking indicators
+                    if (type === 'avatar.speak_started' || type === 'response.created') {
+                        setIsAvatarSpeaking(true);
+                        onSpeakStart?.();
+                    }
+                    else if (type === 'avatar.speak_ended' || type === 'response.done') {
+                        setIsAvatarSpeaking(false);
+                        onSpeakEnd?.();
+                    }
+                    if (type === 'user.speak_started' || type === 'input_audio_buffer.speech_started') {
+                        setIsUserSpeaking(true);
+                        onUserSpeakStart?.();
+                    }
+                    else if (type === 'user.speak_ended' || type === 'input_audio_buffer.speech_stopped') {
+                        setIsUserSpeaking(false);
+                        onUserSpeakEnd?.();
+                    }
+                    // Real-time transcriptions
+                    if (type === 'avatar.transcription' || type === 'response.audio_transcript.done') {
+                        const str = msg.text || msg.transcript || msg.payload?.text;
+                        if (str)
+                            onTranscript?.({ payload: { role: 'assistant', text: String(str).trim() } });
+                    }
+                    else if (type === 'user.transcription' ||
+                        type === 'conversation.item.input_audio_transcription.completed') {
+                        const str = msg.text || msg.transcript || msg.payload?.text;
+                        if (str)
+                            onTranscript?.({ payload: { role: 'user', text: String(str).trim() } });
+                    }
+                    // Order & checkout intents
+                    if (type === 'order.update') {
+                        const items = Array.isArray(msg.items) ? msg.items : msg.payload?.items || [];
+                        onOrderUpdate?.({ items });
+                    }
+                    else if (type === 'order.checkout') {
+                        onCheckout?.();
+                    }
+                }
+                catch {
+                    // ignore non-json data frames
+                }
+            });
+            // 4. Connection state transitions
+            room.on(LiveKitRN.RoomEvent.Disconnected, () => {
+                if (!mountedRef.current)
+                    return;
+                setVideoTrack(null);
+                setIsAvatarSpeaking(false);
+                setIsUserSpeaking(false);
+                updateStatus('disconnected');
+            });
+            room.on(LiveKitRN.RoomEvent.Reconnecting, () => {
+                updateStatus('reconnecting');
+            });
+            room.on(LiveKitRN.RoomEvent.Reconnected, () => {
+                updateStatus('connected');
+            });
+            // Connect room over WebRTC
+            await room.connect(connection.url, connection.token, { autoSubscribe: true });
+            // Enable microphone capture if requested
+            if (enableMic) {
+                try {
+                    await room.localParticipant.setMicrophoneEnabled(true);
+                    setIsMicEnabled(true);
+                }
+                catch (micErr) {
+                    console.warn('[FacePeMobileAvatar] Microphone setup warning:', micErr);
+                }
+            }
+            // Configure server VAD and Whisper transcription
+            const sessionUpdate = {
+                type: 'session.update',
+                session: {
+                    input_audio_transcription: { model: 'whisper-1' },
+                    turn_detection: { type: 'server_vad' },
+                },
+            };
+            sendData(sessionUpdate);
+            // Fallback status if video arrives quickly
+            if (mountedRef.current && status !== 'connected') {
+                updateStatus('connected');
+            }
+        }
+        catch (err) {
+            if (!mountedRef.current)
+                return;
+            const msg = err?.message || 'Connection failed';
+            setErrorMessage(msg);
+            updateStatus('error');
+            onError?.({ code: 'CONNECT_ERROR', message: msg });
+            await endUsage();
+        }
+        finally {
+            isConnectingRef.current = false;
+        }
+    }, [
+        hasLiveKit,
+        avatarKey,
+        language,
+        enableMic,
+        apiBaseUrl,
+        refreshToken,
+        updateStatus,
+        onSpeakStart,
+        onSpeakEnd,
+        onUserSpeakStart,
+        onUserSpeakEnd,
+        onTranscript,
+        onOrderUpdate,
+        onCheckout,
+        onError,
+    ]);
+    // Stop streaming session
+    const stopStream = useCallback(async () => {
+        const room = roomRef.current;
+        roomRef.current = null;
+        if (room) {
+            try {
+                if (room.localParticipant) {
+                    await room.localParticipant.setMicrophoneEnabled(false);
+                }
+                await room.disconnect();
+            }
+            catch {
+                // ignore
+            }
+        }
+        setVideoTrack(null);
+        setIsAvatarSpeaking(false);
+        setIsUserSpeaking(false);
+        updateStatus('idle');
+        await endUsage();
+    }, [updateStatus]);
+    // Imperative methods exposed via ref
     useImperativeHandle(ref, () => ({
         start: async () => {
-            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.start()');
+            await startStream();
         },
         stop: async () => {
-            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.stop()');
+            await stopStream();
         },
         toggleMic: async () => {
-            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.toggleMic()');
+            const room = roomRef.current;
+            if (!room || !room.localParticipant)
+                return isMicEnabled;
+            const nextState = !isMicEnabled;
+            try {
+                await room.localParticipant.setMicrophoneEnabled(nextState);
+                setIsMicEnabled(nextState);
+                return nextState;
+            }
+            catch (err) {
+                console.warn('[FacePeMobileAvatar] toggleMic error:', err);
+                return isMicEnabled;
+            }
         },
-    }));
-    // Handle incoming messages from bridge (transcripts, order updates, speaking states)
-    const handleBridgeMessage = (event) => {
-        try {
-            const raw = event?.nativeEvent?.data;
-            if (!raw)
+        setMicEnabled: async (enabled) => {
+            const room = roomRef.current;
+            if (!room || !room.localParticipant)
                 return;
-            const msg = JSON.parse(raw);
-            switch (msg.type) {
-                case 'avatar:ready':
-                    setStreamReady(true);
-                    break;
-                case 'avatar:transcript':
-                    onTranscript?.({ payload: msg.payload });
-                    break;
-                case 'avatar:speak:start':
-                    onSpeakStart?.();
-                    break;
-                case 'avatar:speak:end':
-                    onSpeakEnd?.();
-                    break;
-                case 'user:speak:start':
-                    onUserSpeakStart?.();
-                    break;
-                case 'user:speak:end':
-                    onUserSpeakEnd?.();
-                    break;
-                case 'order:update':
-                    onOrderUpdate?.({ items: msg.payload?.items || [] });
-                    break;
-                case 'order:checkout':
-                    onCheckout?.();
-                    break;
-                case 'avatar:error':
-                    onError?.({ code: msg.code || 'UNKNOWN', message: msg.message || 'Stream error' });
-                    break;
-                default:
-                    break;
+            try {
+                await room.localParticipant.setMicrophoneEnabled(enabled);
+                setIsMicEnabled(enabled);
             }
+            catch (err) {
+                console.warn('[FacePeMobileAvatar] setMicEnabled error:', err);
+            }
+        },
+        speak: async (text) => {
+            sendData({ event_type: 'avatar.speak_text', text });
+        },
+        interrupt: async () => {
+            sendData({ event_type: 'avatar.interrupt' });
+            setIsAvatarSpeaking(false);
+            onSpeakEnd?.();
+        },
+        get status() {
+            return status;
+        },
+        get isMicEnabled() {
+            return isMicEnabled;
+        },
+    }), [startStream, stopStream, isMicEnabled, status, onSpeakEnd]);
+    // Auto-start on mount once token is resolved
+    useEffect(() => {
+        mountedRef.current = true;
+        if (autoStart && token && !tokenLoading && !tokenError && status === 'idle') {
+            void startStream();
         }
-        catch {
-            // ignore parse error
-        }
-    };
-    if (loading) {
-        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(ActivityIndicator, { size: "large", color: "#7c3aed" }), _jsx(Text, { style: styles.loadingText, children: "Connecting to FacePe AI Avatar\u2026" })] }));
+        return () => {
+            mountedRef.current = false;
+            void stopStream();
+        };
+    }, [autoStart, token, tokenLoading, tokenError]);
+    // Handle token loading
+    if (tokenLoading) {
+        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(ActivityIndicator, { size: "large", color: "#8B5CF6" }), _jsx(Text, { style: styles.loadingText, children: "Authenticating with FacePe AI\u2026" })] }));
     }
-    if (error) {
-        return (_jsx(View, { style: [styles.container, styles.center, style], children: _jsxs(Text, { style: styles.errorText, children: ["Unable to connect to Avatar: ", error.message] }) }));
+    // Handle token error
+    if (tokenError) {
+        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(Text, { style: styles.errorTitle, children: "Authentication Failed" }), _jsx(Text, { style: styles.errorText, children: tokenError.message }), _jsx(TouchableOpacity, { style: styles.retryButton, onPress: () => void refreshToken(true), children: _jsx(Text, { style: styles.retryText, children: "Retry" }) })] }));
     }
-    // Try dynamically requiring react-native-webview
-    let WebViewComponent = null;
-    try {
-        WebViewComponent = require('react-native-webview').WebView;
-    }
-    catch {
-        // WebView not installed in project
+    // If @livekit/react-native is not installed in the host app, render native guidance banner
+    if (!hasLiveKit) {
+        return (_jsxs(View, { style: [styles.container, styles.center, styles.fallbackBox, style], children: [_jsx(View, { style: styles.badge, children: _jsx(Text, { style: styles.badgeText, children: "Native Streaming Required" }) }), _jsx(Text, { style: styles.noticeTitle, children: "FacePe Mobile SDK" }), _jsx(Text, { style: styles.noticeBody, children: "To stream interactive 3D digital avatars natively without webviews, please install LiveKit WebRTC peer dependencies:" }), _jsx(Text, { style: styles.codeSnippet, children: "npm install @livekit/react-native @livekit/react-native-webrtc" })] }));
     }
-    if (!WebViewComponent) {
-        return (_jsxs(View, { style: [styles.container, styles.center, styles.fallbackBox, style], children: [_jsx(Text, { style: styles.noticeTitle, children: "FacePe Mobile SDK" }), _jsx(Text, { style: styles.noticeBody, children: "Please install `react-native-webview` or `@livekit/react-native` in your project to render the live stream:" }), _jsx(Text, { style: styles.codeSnippet, children: "npm install react-native-webview" })] }));
-    }
-    return (_jsx(View, { style: [styles.container, style], children: _jsx(WebViewComponent, { ref: webViewRef, source: { uri: streamUrl }, style: styles.webview, onMessage: handleBridgeMessage, allowsInlineMediaPlayback: true, mediaPlaybackRequiresUserAction: false, javaScriptEnabled: true, domStorageEnabled: true, cacheEnabled: true, 
-            // iOS WebRTC permissions
-            originWhitelist: ['*'], allowsAirPlayForMediaPlayback: false, bounces: false, scrollEnabled: false }) }));
+    // Native LiveKit VideoView component
+    const VideoViewComponent = LiveKitRN.VideoView;
+    return (_jsxs(View, { style: [styles.container, style], children: [videoTrack && VideoViewComponent ? (_jsx(VideoViewComponent, { videoTrack: videoTrack, style: [styles.video, videoStyle], objectFit: objectFit, mirror: mirror })) : (_jsx(View, { style: [styles.container, styles.center], children: status === 'connecting' || status === 'reconnecting' ? (_jsxs(_Fragment, { children: [_jsx(ActivityIndicator, { size: "large", color: "#8B5CF6" }), _jsx(Text, { style: styles.loadingText, children: status === 'connecting'
+                                ? 'Connecting to FacePe AI Avatar…'
+                                : 'Reconnecting stream…' })] })) : status === 'error' ? (_jsxs(_Fragment, { children: [_jsx(Text, { style: styles.errorTitle, children: "Connection Failed" }), _jsx(Text, { style: styles.errorText, children: errorMessage || 'Unable to connect to stream' }), _jsx(TouchableOpacity, { style: styles.retryButton, onPress: () => void startStream(), children: _jsx(Text, { style: styles.retryText, children: "Reconnect" }) })] })) : (_jsx(TouchableOpacity, { style: styles.startButton, onPress: () => void startStream(), children: _jsx(Text, { style: styles.startText, children: "Start Avatar Session" }) })) })), status === 'connected' && (isAvatarSpeaking || isUserSpeaking) && (_jsxs(View, { style: styles.speakerPill, children: [_jsx(View, { style: [
+                            styles.speakerDot,
+                            isAvatarSpeaking ? styles.dotAvatar : styles.dotUser,
+                        ] }), _jsx(Text, { style: styles.speakerText, children: isAvatarSpeaking ? 'Avatar speaking' : 'Listening to you…' })] }))] }));
 });
 FacePeMobileAvatar.displayName = 'FacePeMobileAvatar';
 const styles = StyleSheet.create({
@@ -102,12 +447,13 @@ const styles = StyleSheet.create({
         width: '100%',
         height: '100%',
         backgroundColor: '#0F172A',
-        borderRadius: 16,
+        borderRadius: 20,
         overflow: 'hidden',
+        position: 'relative',
     },
-    webview: {
-        flex: 1,
-        backgroundColor: 'transparent',
+    video: {
+        width: '100%',
+        height: '100%',
     },
     center: {
         justifyContent: 'center',
@@ -115,41 +461,116 @@ const styles = StyleSheet.create({
         padding: 24,
     },
     loadingText: {
-        marginTop: 12,
+        marginTop: 14,
         color: '#94A3B8',
-        fontSize: 13,
+        fontSize: 14,
         fontWeight: '500',
     },
-    errorText: {
+    errorTitle: {
         color: '#EF4444',
+        fontSize: 16,
+        fontWeight: '700',
+        marginBottom: 6,
+    },
+    errorText: {
+        color: '#F87171',
         fontSize: 13,
         textAlign: 'center',
+        lineHeight: 18,
+        marginBottom: 16,
+    },
+    retryButton: {
+        backgroundColor: '#7C3AED',
+        paddingHorizontal: 20,
+        paddingVertical: 10,
+        borderRadius: 12,
+    },
+    retryText: {
+        color: '#FFFFFF',
+        fontWeight: '600',
+        fontSize: 13,
+    },
+    startButton: {
+        backgroundColor: '#7C3AED',
+        paddingHorizontal: 24,
+        paddingVertical: 14,
+        borderRadius: 14,
+    },
+    startText: {
+        color: '#FFFFFF',
+        fontWeight: '700',
+        fontSize: 15,
     },
     fallbackBox: {
         backgroundColor: '#1E293B',
         borderColor: '#334155',
         borderWidth: 1,
     },
+    badge: {
+        backgroundColor: 'rgba(124, 58, 237, 0.15)',
+        borderColor: '#7C3AED',
+        borderWidth: 1,
+        paddingHorizontal: 12,
+        paddingVertical: 4,
+        borderRadius: 20,
+        marginBottom: 12,
+    },
+    badgeText: {
+        color: '#A78BFA',
+        fontSize: 11,
+        fontWeight: '600',
+    },
     noticeTitle: {
         color: '#FFFFFF',
         fontWeight: '700',
-        fontSize: 16,
+        fontSize: 17,
         marginBottom: 8,
     },
     noticeBody: {
         color: '#94A3B8',
         fontSize: 13,
         textAlign: 'center',
-        lineHeight: 18,
-        marginBottom: 12,
+        lineHeight: 19,
+        marginBottom: 16,
     },
     codeSnippet: {
-        color: '#A78BFA',
+        color: '#C4B5FD',
         backgroundColor: '#0F172A',
+        paddingHorizontal: 14,
+        paddingVertical: 10,
+        borderRadius: 10,
+        fontFamily: 'monospace',
+        fontSize: 12,
+        textAlign: 'center',
+    },
+    speakerPill: {
+        position: 'absolute',
+        top: 16,
+        alignSelf: 'center',
+        backgroundColor: 'rgba(15, 23, 42, 0.85)',
+        borderColor: 'rgba(255, 255, 255, 0.1)',
+        borderWidth: 1,
+        flexDirection: 'row',
+        alignItems: 'center',
         paddingHorizontal: 12,
         paddingVertical: 6,
-        borderRadius: 8,
-        fontFamily: 'monospace',
+        borderRadius: 20,
+    },
+    speakerDot: {
+        width: 8,
+        height: 8,
+        borderRadius: 4,
+        marginRight: 8,
+    },
+    dotAvatar: {
+        backgroundColor: '#10B981',
+    },
+    dotUser: {
+        backgroundColor: '#3B82F6',
+    },
+    speakerText: {
+        color: '#F8FAFC',
         fontSize: 12,
+        fontWeight: '500',
     },
 });

package/dist/types.d.ts

@@ -1,5 +1,5 @@
 export type AvatarGender = 'female' | 'male';
-export type StreamMode = 'native' | 'webview' | 'auto';
+export type FacePeConnectionStatus = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'disconnected' | 'error';
 export interface FacePeMobileTheme {
     primaryColor?: string;
     backgroundColor?: string;
@@ -23,11 +23,9 @@ export interface FacePeMobileConfig {
     avatarKey?: string;
     /** Default language code (e.g., 'en', 'es', 'hi') */
     language?: string;
-    /** Integration mode: 'auto', 'native', or 'webview' */
-    mode?: StreamMode;
     /** Theme styling overrides */
     theme?: FacePeMobileTheme;
-    /** Optional audio autoplay unblocking flag */
+    /** Auto start streaming when component mounts (default: true) */
     autoStart?: boolean;
 }
 export interface TokenProviderParams {
@@ -35,12 +33,22 @@ export interface TokenProviderParams {
 }
 export type TokenProvider = (params: TokenProviderParams) => Promise<string>;
 export interface FacePeMobileAvatarProps {
-    /** Optional avatar key override */
+    /** Specific avatar identifier or key override */
     avatarKey?: string;
     /** Language to converse in ('en', 'es', 'hi', etc.) */
     language?: string;
-    /** Custom style object */
+    /** Custom container style */
     style?: any;
+    /** Custom video style */
+    videoStyle?: any;
+    /** Video scaling: 'cover' or 'contain' (default: 'cover') */
+    objectFit?: 'cover' | 'contain';
+    /** Whether to mirror the video stream (default: false) */
+    mirror?: boolean;
+    /** Auto start streaming when component mounts (default: true) */
+    autoStart?: boolean;
+    /** Whether microphone should be automatically enabled on stream start (default: true) */
+    enableMic?: boolean;
     /** Called when real-time speech transcription arrives */
     onTranscript?: (event: {
         payload: TranscriptPayload;
@@ -49,7 +57,7 @@ export interface FacePeMobileAvatarProps {
     onSpeakStart?: () => void;
     /** Called when avatar finishes speaking */
     onSpeakEnd?: () => void;
-    /** Called when user starts speaking (detected by VAD) */
+    /** Called when user starts speaking (detected by server VAD) */
     onUserSpeakStart?: () => void;
     /** Called when user finishes speaking */
     onUserSpeakEnd?: () => void;
@@ -59,9 +67,29 @@ export interface FacePeMobileAvatarProps {
     }) => void;
     /** Called when customer says checkout / pay */
     onCheckout?: () => void;
+    /** Called when stream connection status changes */
+    onStatusChange?: (status: FacePeConnectionStatus) => void;
     /** Called when an error occurs */
     onError?: (event: {
         code: string;
         message: string;
     }) => void;
 }
+export interface FacePeMobileAvatarRef {
+    /** Start avatar streaming session */
+    start: () => Promise<void>;
+    /** Stop streaming and close session */
+    stop: () => Promise<void>;
+    /** Toggle user microphone on/off, returning new enabled state */
+    toggleMic: () => Promise<boolean>;
+    /** Explicitly set microphone state */
+    setMicEnabled: (enabled: boolean) => Promise<void>;
+    /** Programmatically make avatar speak a phrase */
+    speak: (text: string) => Promise<void>;
+    /** Interrupt avatar speech immediately */
+    interrupt: () => Promise<void>;
+    /** Current connection status */
+    readonly status: FacePeConnectionStatus;
+    /** Whether microphone is currently enabled */
+    readonly isMicEnabled: boolean;
+}

package/dist/useFacePeAvatar.d.ts

@@ -1,13 +1,15 @@
-import type { TranscriptPayload, OrderItem } from './types';
+import type { TranscriptPayload, OrderItem, FacePeConnectionStatus, FacePeMobileAvatarProps } from './types';
 export interface UseFacePeAvatarResult {
     token: string | null;
     loading: boolean;
     error: Error | null;
+    status: FacePeConnectionStatus;
     transcripts: TranscriptPayload[];
     isAvatarSpeaking: boolean;
     isUserSpeaking: boolean;
     orderItems: OrderItem[];
     refreshToken: (force?: boolean) => Promise<string | null>;
     clearTranscripts: () => void;
+    avatarProps: Pick<FacePeMobileAvatarProps, 'onTranscript' | 'onSpeakStart' | 'onSpeakEnd' | 'onUserSpeakStart' | 'onUserSpeakEnd' | 'onOrderUpdate' | 'onStatusChange'>;
 }
 export declare function useFacePeAvatar(): UseFacePeAvatarResult;

package/dist/useFacePeAvatar.js

@@ -1,7 +1,8 @@
-import { useState, useCallback } from 'react';
+import { useState, useCallback, useMemo } from 'react';
 import { useFacePeMobileContext } from './FacePeMobileProvider';
 export function useFacePeAvatar() {
     const { token, loading, error, refreshToken } = useFacePeMobileContext();
+    const [status, setStatus] = useState('idle');
     const [transcripts, setTranscripts] = useState([]);
     const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
     const [isUserSpeaking, setIsUserSpeaking] = useState(false);
@@ -9,15 +10,55 @@ export function useFacePeAvatar() {
     const clearTranscripts = useCallback(() => {
         setTranscripts([]);
     }, []);
+    const handleTranscript = useCallback((event) => {
+        setTranscripts((prev) => [...prev, event.payload]);
+    }, []);
+    const handleSpeakStart = useCallback(() => {
+        setIsAvatarSpeaking(true);
+    }, []);
+    const handleSpeakEnd = useCallback(() => {
+        setIsAvatarSpeaking(false);
+    }, []);
+    const handleUserSpeakStart = useCallback(() => {
+        setIsUserSpeaking(true);
+    }, []);
+    const handleUserSpeakEnd = useCallback(() => {
+        setIsUserSpeaking(false);
+    }, []);
+    const handleOrderUpdate = useCallback((event) => {
+        setOrderItems(event.items);
+    }, []);
+    const handleStatusChange = useCallback((newStatus) => {
+        setStatus(newStatus);
+    }, []);
+    const avatarProps = useMemo(() => ({
+        onTranscript: handleTranscript,
+        onSpeakStart: handleSpeakStart,
+        onSpeakEnd: handleSpeakEnd,
+        onUserSpeakStart: handleUserSpeakStart,
+        onUserSpeakEnd: handleUserSpeakEnd,
+        onOrderUpdate: handleOrderUpdate,
+        onStatusChange: handleStatusChange,
+    }), [
+        handleTranscript,
+        handleSpeakStart,
+        handleSpeakEnd,
+        handleUserSpeakStart,
+        handleUserSpeakEnd,
+        handleOrderUpdate,
+        handleStatusChange,
+    ]);
     return {
         token,
         loading,
         error,
+        status,
         transcripts,
         isAvatarSpeaking,
         isUserSpeaking,
         orderItems,
         refreshToken,
         clearTranscripts,
+        avatarProps,
     };
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@facepe/widget-mobile-app",
-  "version": "1.0.0",
-  "description": "FacePe AI 3D Video Avatar Live Streaming SDK for React Native (iOS & Android)",
+  "version": "1.1.0",
+  "description": "FacePe AI 3D Video Avatar Native Live Streaming SDK for React Native (iOS & Android)",
   "main": "./dist/index.js",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -21,7 +21,8 @@
     "ios",
     "android",
     "webrtc",
-    "streaming"
+    "streaming",
+    "digital-human"
   ],
   "author": "FacePe AI",
   "license": "MIT",
@@ -30,6 +31,8 @@
     "build": "tsc"
   },
   "peerDependencies": {
+    "@livekit/react-native": ">=2.0.0",
+    "@livekit/react-native-webrtc": ">=124.0.0",
     "react": ">=18.0.0",
     "react-native": ">=0.70.0"
   },
@@ -39,9 +42,6 @@
     },
     "@livekit/react-native-webrtc": {
       "optional": true
-    },
-    "react-native-webview": {
-      "optional": true
     }
   },
   "devDependencies": {