@facepe/widget-mobile-app 0.0.0-stage → 1.1.0

package/README.md

@@ -1,3 +1,289 @@
-# Temporary Holding Version
+# @facepe/widget-mobile-app
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The official **FacePe AI Digital Human Live Streaming SDK** for **React Native** (supporting both **iOS** and **Android**).
+
+Embed interactive, photorealistic, lip-synced 3D digital avatars directly inside your native mobile apps with pure WebRTC media streaming, native microphone capture, bidirectional voice dialog, and live data events — **without webviews**.
+
+---
+
+## 📱 Features
+
+- **⚡ Native WebRTC Streaming**: Pure native media streaming powered by `@livekit/react-native` (no WebViews or embedded iframes).
+- **🗣️ Natural Two-Way Voice Dialog**: Real-time microphone capture with hardware echo cancellation, noise suppression, and server VAD.
+- **🍎 Full iOS Support**: Hardware-accelerated video decoding on iOS 13.0+ (iPhone & iPad).
+- **🤖 Full Android Support**: Hardware-accelerated video decoding on Android 7.0+ (API 24+).
+- **💬 Real-Time Transcriptions**: Bidirectional captions for avatar speech and user speech.
+- **🛒 Conversational Commerce**: Automatic detection of cart items, orders, and payment intents.
+- **🎛️ Imperative Controls**: Ref methods to start/stop stream, mute/unmute mic, speak text, or interrupt speech.
+
+---
+
+## 📦 Installation
+
+In your React Native project:
+
+```bash
+npm install @facepe/widget-mobile-app @livekit/react-native @livekit/react-native-webrtc
+```
+
+*(or using yarn / pnpm)*:
+
+```bash
+yarn add @facepe/widget-mobile-app @livekit/react-native @livekit/react-native-webrtc
+```
+
+---
+
+## 🛠️ Platform Permissions Setup
+
+### iOS Setup (`ios/Podfile` & `Info.plist`)
+
+1. Open `ios/YourAppName/Info.plist` and add permission strings for microphone and audio:
+
+```xml
+<key>NSMicrophoneUsageDescription</key>
+<string>We need access to your microphone to converse with the FacePe AI avatar.</string>
+<key>NSCameraUsageDescription</key>
+<string>Camera access for interactive features.</string>
+```
+
+2. Run CocoaPods:
+```bash
+cd ios && pod install && cd ..
+```
+
+---
+
+### Android Setup (`android/app/src/main/AndroidManifest.xml`)
+
+Open `android/app/src/main/AndroidManifest.xml` and include permissions:
+
+```xml
+<uses-permission android:name="android.permission.INTERNET" />
+<uses-permission android:name="android.permission.RECORD_AUDIO" />
+<uses-permission android:name="android.permission.CAMERA" />
+<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
+```
+
+---
+
+## 🚀 Quick Start Example
+
+### 1. Server-side Token Endpoint
+
+Exchange your `accessKey` and `secretKey` on your secure backend (Node.js / Express / Next.js / Python / Go) so credentials are never exposed inside mobile apps:
+
+```javascript
+// Express server route: GET /api/facepe-token
+app.get('/api/facepe-token', async (req, res) => {
+  const response = await fetch('https://api.facepe.ai/api/v1/sdk/token', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      accessKey: process.env.FACEPE_SDK_ACCESS_KEY,
+      secretKey: process.env.FACEPE_SDK_SECRET_KEY,
+      channel: 'sdk',
+    }),
+  });
+  const data = await response.json();
+  res.send(data.data.token);
+});
+```
+
+---
+
+### 2. React Native Component
+
+```tsx
+import React, { useRef } from 'react';
+import { StyleSheet, View, SafeAreaView, Text, TouchableOpacity } from 'react-native';
+import {
+  FacePeMobileProvider,
+  FacePeMobileAvatar,
+  FacePeMobileAvatarRef,
+  useFacePeAvatar,
+} from '@facepe/widget-mobile-app';
+
+export default function App() {
+  const avatarRef = useRef<FacePeMobileAvatarRef>(null);
+
+  return (
+    <SafeAreaView style={styles.container}>
+      <Text style={styles.title}>FacePe AI Native Digital Human</Text>
+
+      <FacePeMobileProvider
+        apiBaseUrl="https://api.facepe.ai/api/v1/sdk"
+        getToken={async () => {
+          const res = await fetch('https://your-backend.com/api/facepe-token');
+          return await res.text();
+        }}
+      >
+        <AvatarScreen avatarRef={avatarRef} />
+      </FacePeMobileProvider>
+    </SafeAreaView>
+  );
+}
+
+function AvatarScreen({ avatarRef }: { avatarRef: React.RefObject<FacePeMobileAvatarRef> }) {
+  const { transcripts, isAvatarSpeaking, avatarProps } = useFacePeAvatar();
+
+  return (
+    <View style={styles.content}>
+      <View style={styles.avatarContainer}>
+        <FacePeMobileAvatar
+          ref={avatarRef}
+          language="en"
+          objectFit="cover"
+          autoStart={true}
+          enableMic={true}
+          {...avatarProps}
+        />
+      </View>
+
+      {/* Control bar */}
+      <View style={styles.controls}>
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => avatarRef.current?.toggleMic()}
+        >
+          <Text style={styles.buttonText}>Toggle Mic</Text>
+        </TouchableOpacity>
+
+        <TouchableOpacity
+          style={styles.button}
+          onPress={() => avatarRef.current?.interrupt()}
+        >
+          <Text style={styles.buttonText}>Interrupt</Text>
+        </TouchableOpacity>
+      </View>
+
+      {/* Real-time speech transcript log */}
+      <View style={styles.transcriptBox}>
+        {transcripts.slice(-3).map((item, idx) => (
+          <Text key={idx} style={styles.transcriptLine}>
+            <Text style={item.role === 'assistant' ? styles.avatarTag : styles.userTag}>
+              {item.role === 'assistant' ? 'Avatar: ' : 'You: '}
+            </Text>
+            {item.text}
+          </Text>
+        ))}
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    backgroundColor: '#0F172A',
+  },
+  content: {
+    flex: 1,
+    padding: 16,
+  },
+  title: {
+    color: '#FFFFFF',
+    fontSize: 20,
+    fontWeight: 'bold',
+    textAlign: 'center',
+    marginVertical: 12,
+  },
+  avatarContainer: {
+    flex: 1,
+    borderRadius: 24,
+    overflow: 'hidden',
+    backgroundColor: '#1E293B',
+  },
+  controls: {
+    flexDirection: 'row',
+    justifyContent: 'center',
+    gap: 12,
+    marginVertical: 14,
+  },
+  button: {
+    backgroundColor: '#7C3AED',
+    paddingHorizontal: 20,
+    paddingVertical: 10,
+    borderRadius: 12,
+  },
+  buttonText: {
+    color: '#FFFFFF',
+    fontWeight: '600',
+    fontSize: 14,
+  },
+  transcriptBox: {
+    backgroundColor: '#1E293B',
+    padding: 12,
+    borderRadius: 12,
+    minHeight: 80,
+  },
+  transcriptLine: {
+    color: '#E2E8F0',
+    fontSize: 13,
+    marginBottom: 4,
+  },
+  avatarTag: {
+    color: '#A78BFA',
+    fontWeight: '700',
+  },
+  userTag: {
+    color: '#60A5FA',
+    fontWeight: '700',
+  },
+});
+```
+
+---
+
+## ⚙️ Props & Configuration
+
+### `<FacePeMobileAvatar />`
+
+| Prop | Type | Default | Description |
+| :--- | :--- | :--- | :--- |
+| `avatarKey` | `string` | *(optional)* | Specific avatar key or ID assigned to your company. |
+| `language` | `string` | `'en'` | Spoken language code (`'en'`, `'hi'`, `'es'`, `'ta'`, `'te'`, etc.). |
+| `style` | `ViewStyle` | `undefined` | Style applied to avatar container. |
+| `videoStyle` | `ViewStyle` | `undefined` | Style applied to native video view. |
+| `objectFit` | `'cover' \| 'contain'` | `'cover'` | Video aspect scaling mode. |
+| `mirror` | `boolean` | `false` | Mirror video orientation. |
+| `autoStart` | `boolean` | `true` | Automatically connect and start stream on mount. |
+| `enableMic` | `boolean` | `true` | Automatically enable microphone on connect. |
+| `onTranscript` | `function` | `undefined` | Fires when speech transcription arrives. |
+| `onSpeakStart` | `function` | `undefined` | Fires when avatar starts speaking. |
+| `onSpeakEnd` | `function` | `undefined` | Fires when avatar finishes speaking. |
+| `onUserSpeakStart` | `function` | `undefined` | Fires when user voice activity begins. |
+| `onUserSpeakEnd` | `function` | `undefined` | Fires when user voice activity ends. |
+| `onOrderUpdate` | `function` | `undefined` | Fires when cart items or intents are recognized. |
+| `onCheckout` | `function` | `undefined` | Fires when customer requests checkout / payment. |
+| `onStatusChange` | `function` | `undefined` | Stream lifecycle status: `'idle' \| 'connecting' \| 'connected' \| 'reconnecting' \| 'disconnected' \| 'error'`. |
+| `onError` | `function` | `undefined` | Stream error callback `{ code, message }`. |
+
+---
+
+### Imperative Ref (`FacePeMobileAvatarRef`)
+
+```tsx
+const avatarRef = useRef<FacePeMobileAvatarRef>(null);
+
+// Start streaming session
+await avatarRef.current?.start();
+
+// Stop streaming session & finalize billing usage
+await avatarRef.current?.stop();
+
+// Toggle mic on/off (returns new state boolean)
+const isNowMuted = await avatarRef.current?.toggleMic();
+
+// Make avatar speak programmatically
+await avatarRef.current?.speak("Hello! How can I assist you today?");
+
+// Interrupt avatar speech immediately
+await avatarRef.current?.interrupt();
+```
+
+---
+
+## 📄 License
+MIT © FacePe AI

package/dist/FacePeMobileAvatar.d.ts

@@ -0,0 +1,3 @@
+import React from 'react';
+import type { FacePeMobileAvatarProps, FacePeMobileAvatarRef } from './types';
+export declare const FacePeMobileAvatar: React.ForwardRefExoticComponent<FacePeMobileAvatarProps & React.RefAttributes<FacePeMobileAvatarRef>>;

package/dist/FacePeMobileAvatar.js

@@ -0,0 +1,576 @@
+import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
+import { useEffect, useRef, useState, useImperativeHandle, forwardRef, useCallback, } from 'react';
+import { View, StyleSheet, ActivityIndicator, Text, TouchableOpacity } from 'react-native';
+import { useFacePeMobileContext } from './FacePeMobileProvider';
+// Resolve @livekit/react-native dynamically at runtime so bare/Expo projects can run smoothly
+function getLiveKitRN() {
+    try {
+        return require('@livekit/react-native');
+    }
+    catch {
+        return null;
+    }
+}
+let globalsRegistered = false;
+function ensureLiveKitGlobals(LiveKitRN) {
+    if (globalsRegistered || !LiveKitRN)
+        return;
+    try {
+        if (typeof LiveKitRN.registerGlobals === 'function') {
+            LiveKitRN.registerGlobals();
+            globalsRegistered = true;
+        }
+    }
+    catch (err) {
+        console.warn('[FacePeMobileAvatar] registerGlobals warning:', err);
+    }
+}
+export const FacePeMobileAvatar = forwardRef(({ avatarKey, language = 'en', style, videoStyle, objectFit = 'cover', mirror = false, autoStart = true, enableMic = true, onTranscript, onSpeakStart, onSpeakEnd, onUserSpeakStart, onUserSpeakEnd, onOrderUpdate, onCheckout, onStatusChange, onError, }, ref) => {
+    const { token, loading: tokenLoading, error: tokenError, apiBaseUrl, refreshToken } = useFacePeMobileContext();
+    const [status, setStatus] = useState('idle');
+    const [videoTrack, setVideoTrack] = useState(null);
+    const [isMicEnabled, setIsMicEnabled] = useState(enableMic);
+    const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
+    const [isUserSpeaking, setIsUserSpeaking] = useState(false);
+    const [errorMessage, setErrorMessage] = useState(null);
+    const LiveKitRN = getLiveKitRN();
+    const hasLiveKit = !!LiveKitRN;
+    const roomRef = useRef(null);
+    const usageIdRef = useRef(null);
+    const mountedRef = useRef(true);
+    const isConnectingRef = useRef(false);
+    const tokenRef = useRef(token);
+    tokenRef.current = token;
+    // Helper: Safely emit status changes
+    const updateStatus = useCallback((newStatus) => {
+        if (!mountedRef.current)
+            return;
+        setStatus(newStatus);
+        onStatusChange?.(newStatus);
+    }, [onStatusChange]);
+    // Helper: Safely publish data channel packet
+    const sendData = (msg) => {
+        const room = roomRef.current;
+        if (!room || room.state !== 'connected')
+            return;
+        try {
+            const jsonStr = JSON.stringify(msg);
+            let payload;
+            if (typeof TextEncoder !== 'undefined') {
+                payload = new TextEncoder().encode(jsonStr);
+            }
+            else {
+                const arr = new Uint8Array(jsonStr.length);
+                for (let i = 0; i < jsonStr.length; i++) {
+                    arr[i] = jsonStr.charCodeAt(i);
+                }
+                payload = arr;
+            }
+            room.localParticipant.publishData(payload, { reliable: true });
+        }
+        catch (err) {
+            console.warn('[FacePeMobileAvatar] Error publishing data packet:', err);
+        }
+    };
+    // Helper: Notify FacePe backend that session ended (for billing/usage)
+    const endUsage = async () => {
+        const uId = usageIdRef.current;
+        usageIdRef.current = null;
+        const currentToken = tokenRef.current;
+        if (!uId || !currentToken)
+            return;
+        try {
+            const cleanBase = apiBaseUrl.replace(/\/+$/, '');
+            await fetch(`${cleanBase}/avatar-sessions/${uId}/end`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${currentToken}`,
+                },
+            });
+        }
+        catch {
+            // ignore background teardown error
+        }
+    };
+    // Start streaming session
+    const startStream = useCallback(async () => {
+        if (isConnectingRef.current || roomRef.current)
+            return;
+        if (!hasLiveKit) {
+            const msg = 'Please install `@livekit/react-native` and `@livekit/react-native-webrtc` to enable native AI avatar streaming.';
+            setErrorMessage(msg);
+            updateStatus('error');
+            onError?.({ code: 'MISSING_DEPENDENCY', message: msg });
+            return;
+        }
+        isConnectingRef.current = true;
+        setErrorMessage(null);
+        updateStatus('connecting');
+        try {
+            ensureLiveKitGlobals(LiveKitRN);
+            // Ensure active SDK auth token
+            let currentToken = tokenRef.current;
+            if (!currentToken) {
+                currentToken = await refreshToken();
+            }
+            if (!currentToken) {
+                throw new Error('Authentication token not available. Please verify your token provider.');
+            }
+            const cleanBase = apiBaseUrl.replace(/\/+$/, '');
+            // Resolve target avatar ID
+            let targetAvatarId = avatarKey;
+            if (!targetAvatarId) {
+                // If no avatar key passed, retrieve assigned avatars from /sdk/session
+                const sessionRes = await fetch(`${cleanBase}/session`, {
+                    method: 'GET',
+                    headers: {
+                        Authorization: `Bearer ${currentToken}`,
+                    },
+                });
+                if (sessionRes.ok) {
+                    const sessionData = await sessionRes.json();
+                    const avatars = sessionData?.data?.avatars || sessionData?.avatars || [];
+                    if (avatars.length > 0) {
+                        targetAvatarId = avatars[0].id || avatars[0].key;
+                    }
+                }
+            }
+            if (!targetAvatarId) {
+                throw new Error('No Avatar configured or assigned to this token.');
+            }
+            // Request live streaming session from FacePe SDK backend
+            const sessionResp = await fetch(`${cleanBase}/avatars/${targetAvatarId}/sessions`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${currentToken}`,
+                },
+                body: JSON.stringify({ language }),
+            });
+            if (!sessionResp.ok) {
+                const errBody = await sessionResp.json().catch(() => ({}));
+                const errMsg = errBody?.error?.message || errBody?.message || 'Failed to initialize avatar session';
+                throw new Error(errMsg);
+            }
+            const sessionPayload = await sessionResp.json();
+            const sessionInfo = sessionPayload?.data || sessionPayload;
+            usageIdRef.current = sessionInfo?.usageId || null;
+            const connection = sessionInfo?.connection;
+            if (!connection || connection.type !== 'room' || !connection.url || !connection.token) {
+                throw new Error('Avatar server did not return a valid LiveKit room connection.');
+            }
+            // Initialize LiveKit Room natively
+            const room = new LiveKitRN.Room({
+                adaptiveStream: true,
+                dynacast: true,
+            });
+            roomRef.current = room;
+            // Start Audio Session on iOS/Android
+            if (LiveKitRN.AudioSession?.startAudioSession) {
+                try {
+                    await LiveKitRN.AudioSession.startAudioSession();
+                }
+                catch {
+                    // non-fatal
+                }
+            }
+            // 1. Remote video track subscription
+            room.on(LiveKitRN.RoomEvent.TrackSubscribed, (track) => {
+                if (!mountedRef.current)
+                    return;
+                const isVideo = track.kind === 'video' ||
+                    (LiveKitRN.Track?.Kind?.Video && track.kind === LiveKitRN.Track.Kind.Video);
+                if (isVideo) {
+                    setVideoTrack(track);
+                    updateStatus('connected');
+                }
+            });
+            room.on(LiveKitRN.RoomEvent.TrackUnsubscribed, (track) => {
+                if (!mountedRef.current)
+                    return;
+                const isVideo = track.kind === 'video' ||
+                    (LiveKitRN.Track?.Kind?.Video && track.kind === LiveKitRN.Track.Kind.Video);
+                if (isVideo) {
+                    setVideoTrack(null);
+                }
+            });
+            // 2. Active speaker detection
+            room.on(LiveKitRN.RoomEvent.ActiveSpeakersChanged, (speakers) => {
+                if (!mountedRef.current)
+                    return;
+                const hasAssistant = speakers.some((p) => !p.isLocal);
+                const hasUser = speakers.some((p) => p.isLocal);
+                setIsAvatarSpeaking(hasAssistant);
+                if (hasAssistant)
+                    onSpeakStart?.();
+                else
+                    onSpeakEnd?.();
+                setIsUserSpeaking(hasUser);
+                if (hasUser)
+                    onUserSpeakStart?.();
+                else
+                    onUserSpeakEnd?.();
+            });
+            // 3. Bidirectional data-channel events (transcriptions, orders, interrupts)
+            room.on(LiveKitRN.RoomEvent.DataReceived, (payload) => {
+                if (!mountedRef.current)
+                    return;
+                try {
+                    let text = '';
+                    if (typeof TextDecoder !== 'undefined') {
+                        text = new TextDecoder().decode(payload);
+                    }
+                    else {
+                        text = String.fromCharCode.apply(null, Array.from(payload));
+                    }
+                    const msg = JSON.parse(text);
+                    const type = String(msg.type ?? msg.event_type ?? '');
+                    // Speaking indicators
+                    if (type === 'avatar.speak_started' || type === 'response.created') {
+                        setIsAvatarSpeaking(true);
+                        onSpeakStart?.();
+                    }
+                    else if (type === 'avatar.speak_ended' || type === 'response.done') {
+                        setIsAvatarSpeaking(false);
+                        onSpeakEnd?.();
+                    }
+                    if (type === 'user.speak_started' || type === 'input_audio_buffer.speech_started') {
+                        setIsUserSpeaking(true);
+                        onUserSpeakStart?.();
+                    }
+                    else if (type === 'user.speak_ended' || type === 'input_audio_buffer.speech_stopped') {
+                        setIsUserSpeaking(false);
+                        onUserSpeakEnd?.();
+                    }
+                    // Real-time transcriptions
+                    if (type === 'avatar.transcription' || type === 'response.audio_transcript.done') {
+                        const str = msg.text || msg.transcript || msg.payload?.text;
+                        if (str)
+                            onTranscript?.({ payload: { role: 'assistant', text: String(str).trim() } });
+                    }
+                    else if (type === 'user.transcription' ||
+                        type === 'conversation.item.input_audio_transcription.completed') {
+                        const str = msg.text || msg.transcript || msg.payload?.text;
+                        if (str)
+                            onTranscript?.({ payload: { role: 'user', text: String(str).trim() } });
+                    }
+                    // Order & checkout intents
+                    if (type === 'order.update') {
+                        const items = Array.isArray(msg.items) ? msg.items : msg.payload?.items || [];
+                        onOrderUpdate?.({ items });
+                    }
+                    else if (type === 'order.checkout') {
+                        onCheckout?.();
+                    }
+                }
+                catch {
+                    // ignore non-json data frames
+                }
+            });
+            // 4. Connection state transitions
+            room.on(LiveKitRN.RoomEvent.Disconnected, () => {
+                if (!mountedRef.current)
+                    return;
+                setVideoTrack(null);
+                setIsAvatarSpeaking(false);
+                setIsUserSpeaking(false);
+                updateStatus('disconnected');
+            });
+            room.on(LiveKitRN.RoomEvent.Reconnecting, () => {
+                updateStatus('reconnecting');
+            });
+            room.on(LiveKitRN.RoomEvent.Reconnected, () => {
+                updateStatus('connected');
+            });
+            // Connect room over WebRTC
+            await room.connect(connection.url, connection.token, { autoSubscribe: true });
+            // Enable microphone capture if requested
+            if (enableMic) {
+                try {
+                    await room.localParticipant.setMicrophoneEnabled(true);
+                    setIsMicEnabled(true);
+                }
+                catch (micErr) {
+                    console.warn('[FacePeMobileAvatar] Microphone setup warning:', micErr);
+                }
+            }
+            // Configure server VAD and Whisper transcription
+            const sessionUpdate = {
+                type: 'session.update',
+                session: {
+                    input_audio_transcription: { model: 'whisper-1' },
+                    turn_detection: { type: 'server_vad' },
+                },
+            };
+            sendData(sessionUpdate);
+            // Fallback status if video arrives quickly
+            if (mountedRef.current && status !== 'connected') {
+                updateStatus('connected');
+            }
+        }
+        catch (err) {
+            if (!mountedRef.current)
+                return;
+            const msg = err?.message || 'Connection failed';
+            setErrorMessage(msg);
+            updateStatus('error');
+            onError?.({ code: 'CONNECT_ERROR', message: msg });
+            await endUsage();
+        }
+        finally {
+            isConnectingRef.current = false;
+        }
+    }, [
+        hasLiveKit,
+        avatarKey,
+        language,
+        enableMic,
+        apiBaseUrl,
+        refreshToken,
+        updateStatus,
+        onSpeakStart,
+        onSpeakEnd,
+        onUserSpeakStart,
+        onUserSpeakEnd,
+        onTranscript,
+        onOrderUpdate,
+        onCheckout,
+        onError,
+    ]);
+    // Stop streaming session
+    const stopStream = useCallback(async () => {
+        const room = roomRef.current;
+        roomRef.current = null;
+        if (room) {
+            try {
+                if (room.localParticipant) {
+                    await room.localParticipant.setMicrophoneEnabled(false);
+                }
+                await room.disconnect();
+            }
+            catch {
+                // ignore
+            }
+        }
+        setVideoTrack(null);
+        setIsAvatarSpeaking(false);
+        setIsUserSpeaking(false);
+        updateStatus('idle');
+        await endUsage();
+    }, [updateStatus]);
+    // Imperative methods exposed via ref
+    useImperativeHandle(ref, () => ({
+        start: async () => {
+            await startStream();
+        },
+        stop: async () => {
+            await stopStream();
+        },
+        toggleMic: async () => {
+            const room = roomRef.current;
+            if (!room || !room.localParticipant)
+                return isMicEnabled;
+            const nextState = !isMicEnabled;
+            try {
+                await room.localParticipant.setMicrophoneEnabled(nextState);
+                setIsMicEnabled(nextState);
+                return nextState;
+            }
+            catch (err) {
+                console.warn('[FacePeMobileAvatar] toggleMic error:', err);
+                return isMicEnabled;
+            }
+        },
+        setMicEnabled: async (enabled) => {
+            const room = roomRef.current;
+            if (!room || !room.localParticipant)
+                return;
+            try {
+                await room.localParticipant.setMicrophoneEnabled(enabled);
+                setIsMicEnabled(enabled);
+            }
+            catch (err) {
+                console.warn('[FacePeMobileAvatar] setMicEnabled error:', err);
+            }
+        },
+        speak: async (text) => {
+            sendData({ event_type: 'avatar.speak_text', text });
+        },
+        interrupt: async () => {
+            sendData({ event_type: 'avatar.interrupt' });
+            setIsAvatarSpeaking(false);
+            onSpeakEnd?.();
+        },
+        get status() {
+            return status;
+        },
+        get isMicEnabled() {
+            return isMicEnabled;
+        },
+    }), [startStream, stopStream, isMicEnabled, status, onSpeakEnd]);
+    // Auto-start on mount once token is resolved
+    useEffect(() => {
+        mountedRef.current = true;
+        if (autoStart && token && !tokenLoading && !tokenError && status === 'idle') {
+            void startStream();
+        }
+        return () => {
+            mountedRef.current = false;
+            void stopStream();
+        };
+    }, [autoStart, token, tokenLoading, tokenError]);
+    // Handle token loading
+    if (tokenLoading) {
+        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(ActivityIndicator, { size: "large", color: "#8B5CF6" }), _jsx(Text, { style: styles.loadingText, children: "Authenticating with FacePe AI\u2026" })] }));
+    }
+    // Handle token error
+    if (tokenError) {
+        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(Text, { style: styles.errorTitle, children: "Authentication Failed" }), _jsx(Text, { style: styles.errorText, children: tokenError.message }), _jsx(TouchableOpacity, { style: styles.retryButton, onPress: () => void refreshToken(true), children: _jsx(Text, { style: styles.retryText, children: "Retry" }) })] }));
+    }
+    // If @livekit/react-native is not installed in the host app, render native guidance banner
+    if (!hasLiveKit) {
+        return (_jsxs(View, { style: [styles.container, styles.center, styles.fallbackBox, style], children: [_jsx(View, { style: styles.badge, children: _jsx(Text, { style: styles.badgeText, children: "Native Streaming Required" }) }), _jsx(Text, { style: styles.noticeTitle, children: "FacePe Mobile SDK" }), _jsx(Text, { style: styles.noticeBody, children: "To stream interactive 3D digital avatars natively without webviews, please install LiveKit WebRTC peer dependencies:" }), _jsx(Text, { style: styles.codeSnippet, children: "npm install @livekit/react-native @livekit/react-native-webrtc" })] }));
+    }
+    // Native LiveKit VideoView component
+    const VideoViewComponent = LiveKitRN.VideoView;
+    return (_jsxs(View, { style: [styles.container, style], children: [videoTrack && VideoViewComponent ? (_jsx(VideoViewComponent, { videoTrack: videoTrack, style: [styles.video, videoStyle], objectFit: objectFit, mirror: mirror })) : (_jsx(View, { style: [styles.container, styles.center], children: status === 'connecting' || status === 'reconnecting' ? (_jsxs(_Fragment, { children: [_jsx(ActivityIndicator, { size: "large", color: "#8B5CF6" }), _jsx(Text, { style: styles.loadingText, children: status === 'connecting'
+                                ? 'Connecting to FacePe AI Avatar…'
+                                : 'Reconnecting stream…' })] })) : status === 'error' ? (_jsxs(_Fragment, { children: [_jsx(Text, { style: styles.errorTitle, children: "Connection Failed" }), _jsx(Text, { style: styles.errorText, children: errorMessage || 'Unable to connect to stream' }), _jsx(TouchableOpacity, { style: styles.retryButton, onPress: () => void startStream(), children: _jsx(Text, { style: styles.retryText, children: "Reconnect" }) })] })) : (_jsx(TouchableOpacity, { style: styles.startButton, onPress: () => void startStream(), children: _jsx(Text, { style: styles.startText, children: "Start Avatar Session" }) })) })), status === 'connected' && (isAvatarSpeaking || isUserSpeaking) && (_jsxs(View, { style: styles.speakerPill, children: [_jsx(View, { style: [
+                            styles.speakerDot,
+                            isAvatarSpeaking ? styles.dotAvatar : styles.dotUser,
+                        ] }), _jsx(Text, { style: styles.speakerText, children: isAvatarSpeaking ? 'Avatar speaking' : 'Listening to you…' })] }))] }));
+});
+FacePeMobileAvatar.displayName = 'FacePeMobileAvatar';
+const styles = StyleSheet.create({
+    container: {
+        width: '100%',
+        height: '100%',
+        backgroundColor: '#0F172A',
+        borderRadius: 20,
+        overflow: 'hidden',
+        position: 'relative',
+    },
+    video: {
+        width: '100%',
+        height: '100%',
+    },
+    center: {
+        justifyContent: 'center',
+        alignItems: 'center',
+        padding: 24,
+    },
+    loadingText: {
+        marginTop: 14,
+        color: '#94A3B8',
+        fontSize: 14,
+        fontWeight: '500',
+    },
+    errorTitle: {
+        color: '#EF4444',
+        fontSize: 16,
+        fontWeight: '700',
+        marginBottom: 6,
+    },
+    errorText: {
+        color: '#F87171',
+        fontSize: 13,
+        textAlign: 'center',
+        lineHeight: 18,
+        marginBottom: 16,
+    },
+    retryButton: {
+        backgroundColor: '#7C3AED',
+        paddingHorizontal: 20,
+        paddingVertical: 10,
+        borderRadius: 12,
+    },
+    retryText: {
+        color: '#FFFFFF',
+        fontWeight: '600',
+        fontSize: 13,
+    },
+    startButton: {
+        backgroundColor: '#7C3AED',
+        paddingHorizontal: 24,
+        paddingVertical: 14,
+        borderRadius: 14,
+    },
+    startText: {
+        color: '#FFFFFF',
+        fontWeight: '700',
+        fontSize: 15,
+    },
+    fallbackBox: {
+        backgroundColor: '#1E293B',
+        borderColor: '#334155',
+        borderWidth: 1,
+    },
+    badge: {
+        backgroundColor: 'rgba(124, 58, 237, 0.15)',
+        borderColor: '#7C3AED',
+        borderWidth: 1,
+        paddingHorizontal: 12,
+        paddingVertical: 4,
+        borderRadius: 20,
+        marginBottom: 12,
+    },
+    badgeText: {
+        color: '#A78BFA',
+        fontSize: 11,
+        fontWeight: '600',
+    },
+    noticeTitle: {
+        color: '#FFFFFF',
+        fontWeight: '700',
+        fontSize: 17,
+        marginBottom: 8,
+    },
+    noticeBody: {
+        color: '#94A3B8',
+        fontSize: 13,
+        textAlign: 'center',
+        lineHeight: 19,
+        marginBottom: 16,
+    },
+    codeSnippet: {
+        color: '#C4B5FD',
+        backgroundColor: '#0F172A',
+        paddingHorizontal: 14,
+        paddingVertical: 10,
+        borderRadius: 10,
+        fontFamily: 'monospace',
+        fontSize: 12,
+        textAlign: 'center',
+    },
+    speakerPill: {
+        position: 'absolute',
+        top: 16,
+        alignSelf: 'center',
+        backgroundColor: 'rgba(15, 23, 42, 0.85)',
+        borderColor: 'rgba(255, 255, 255, 0.1)',
+        borderWidth: 1,
+        flexDirection: 'row',
+        alignItems: 'center',
+        paddingHorizontal: 12,
+        paddingVertical: 6,
+        borderRadius: 20,
+    },
+    speakerDot: {
+        width: 8,
+        height: 8,
+        borderRadius: 4,
+        marginRight: 8,
+    },
+    dotAvatar: {
+        backgroundColor: '#10B981',
+    },
+    dotUser: {
+        backgroundColor: '#3B82F6',
+    },
+    speakerText: {
+        color: '#F8FAFC',
+        fontSize: 12,
+        fontWeight: '500',
+    },
+});

package/dist/FacePeMobileProvider.d.ts

@@ -0,0 +1,19 @@
+import React, { type ReactNode } from 'react';
+import type { FacePeMobileConfig, TokenProvider } from './types';
+interface FacePeContextValue {
+    config: FacePeMobileConfig;
+    token: string | null;
+    loading: boolean;
+    error: Error | null;
+    refreshToken: (force?: boolean) => Promise<string | null>;
+    apiBaseUrl: string;
+}
+export interface FacePeMobileProviderProps {
+    children: ReactNode;
+    getToken: TokenProvider;
+    apiBaseUrl?: string;
+    config?: FacePeMobileConfig;
+}
+export declare function FacePeMobileProvider({ children, getToken, apiBaseUrl, config, }: FacePeMobileProviderProps): React.JSX.Element;
+export declare function useFacePeMobileContext(): FacePeContextValue;
+export {};

package/dist/FacePeMobileProvider.js

@@ -0,0 +1,54 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { createContext, useContext, useState, useEffect, useRef, useMemo } from 'react';
+const FacePeContext = createContext(null);
+export function FacePeMobileProvider({ children, getToken, apiBaseUrl = 'https://api.facepe.ai/api/v1/sdk', config = {}, }) {
+    const [token, setToken] = useState(null);
+    const [loading, setLoading] = useState(true);
+    const [error, setError] = useState(null);
+    const tokenPromiseRef = useRef(null);
+    const refreshToken = async (force = false) => {
+        if (tokenPromiseRef.current && !force) {
+            return tokenPromiseRef.current;
+        }
+        setLoading(true);
+        setError(null);
+        const promise = (async () => {
+            try {
+                const fetched = await getToken({ forceRefresh: force });
+                setToken(fetched);
+                setLoading(false);
+                return fetched;
+            }
+            catch (err) {
+                const normalized = err instanceof Error ? err : new Error(String(err));
+                setError(normalized);
+                setLoading(false);
+                return null;
+            }
+            finally {
+                tokenPromiseRef.current = null;
+            }
+        })();
+        tokenPromiseRef.current = promise;
+        return promise;
+    };
+    useEffect(() => {
+        void refreshToken();
+    }, []);
+    const value = useMemo(() => ({
+        config,
+        token,
+        loading,
+        error,
+        refreshToken,
+        apiBaseUrl: config.apiBaseUrl || apiBaseUrl,
+    }), [config, token, loading, error, apiBaseUrl]);
+    return _jsx(FacePeContext.Provider, { value: value, children: children });
+}
+export function useFacePeMobileContext() {
+    const ctx = useContext(FacePeContext);
+    if (!ctx) {
+        throw new Error('useFacePeMobileContext must be used within a <FacePeMobileProvider>');
+    }
+    return ctx;
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from './types';
+export * from './FacePeMobileProvider';
+export * from './FacePeMobileAvatar';
+export * from './useFacePeAvatar';

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from './types';
+export * from './FacePeMobileProvider';
+export * from './FacePeMobileAvatar';
+export * from './useFacePeAvatar';

package/dist/types.d.ts

@@ -0,0 +1,95 @@
+export type AvatarGender = 'female' | 'male';
+export type FacePeConnectionStatus = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'disconnected' | 'error';
+export interface FacePeMobileTheme {
+    primaryColor?: string;
+    backgroundColor?: string;
+    borderRadius?: number;
+    textColor?: string;
+}
+export interface TranscriptPayload {
+    role: 'user' | 'assistant';
+    text: string;
+    isFinal?: boolean;
+}
+export interface OrderItem {
+    name: string;
+    qty: number;
+    price?: number;
+}
+export interface FacePeMobileConfig {
+    /** FacePe Gateway URL (e.g. https://api.facepe.ai/api/v1/sdk) */
+    apiBaseUrl?: string;
+    /** Avatar identifier or key */
+    avatarKey?: string;
+    /** Default language code (e.g., 'en', 'es', 'hi') */
+    language?: string;
+    /** Theme styling overrides */
+    theme?: FacePeMobileTheme;
+    /** Auto start streaming when component mounts (default: true) */
+    autoStart?: boolean;
+}
+export interface TokenProviderParams {
+    forceRefresh?: boolean;
+}
+export type TokenProvider = (params: TokenProviderParams) => Promise<string>;
+export interface FacePeMobileAvatarProps {
+    /** Specific avatar identifier or key override */
+    avatarKey?: string;
+    /** Language to converse in ('en', 'es', 'hi', etc.) */
+    language?: string;
+    /** Custom container style */
+    style?: any;
+    /** Custom video style */
+    videoStyle?: any;
+    /** Video scaling: 'cover' or 'contain' (default: 'cover') */
+    objectFit?: 'cover' | 'contain';
+    /** Whether to mirror the video stream (default: false) */
+    mirror?: boolean;
+    /** Auto start streaming when component mounts (default: true) */
+    autoStart?: boolean;
+    /** Whether microphone should be automatically enabled on stream start (default: true) */
+    enableMic?: boolean;
+    /** Called when real-time speech transcription arrives */
+    onTranscript?: (event: {
+        payload: TranscriptPayload;
+    }) => void;
+    /** Called when avatar starts speaking */
+    onSpeakStart?: () => void;
+    /** Called when avatar finishes speaking */
+    onSpeakEnd?: () => void;
+    /** Called when user starts speaking (detected by server VAD) */
+    onUserSpeakStart?: () => void;
+    /** Called when user finishes speaking */
+    onUserSpeakEnd?: () => void;
+    /** Called when an order or intent snapshot is emitted */
+    onOrderUpdate?: (event: {
+        items: OrderItem[];
+    }) => void;
+    /** Called when customer says checkout / pay */
+    onCheckout?: () => void;
+    /** Called when stream connection status changes */
+    onStatusChange?: (status: FacePeConnectionStatus) => void;
+    /** Called when an error occurs */
+    onError?: (event: {
+        code: string;
+        message: string;
+    }) => void;
+}
+export interface FacePeMobileAvatarRef {
+    /** Start avatar streaming session */
+    start: () => Promise<void>;
+    /** Stop streaming and close session */
+    stop: () => Promise<void>;
+    /** Toggle user microphone on/off, returning new enabled state */
+    toggleMic: () => Promise<boolean>;
+    /** Explicitly set microphone state */
+    setMicEnabled: (enabled: boolean) => Promise<void>;
+    /** Programmatically make avatar speak a phrase */
+    speak: (text: string) => Promise<void>;
+    /** Interrupt avatar speech immediately */
+    interrupt: () => Promise<void>;
+    /** Current connection status */
+    readonly status: FacePeConnectionStatus;
+    /** Whether microphone is currently enabled */
+    readonly isMicEnabled: boolean;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/useFacePeAvatar.d.ts

@@ -0,0 +1,15 @@
+import type { TranscriptPayload, OrderItem, FacePeConnectionStatus, FacePeMobileAvatarProps } from './types';
+export interface UseFacePeAvatarResult {
+    token: string | null;
+    loading: boolean;
+    error: Error | null;
+    status: FacePeConnectionStatus;
+    transcripts: TranscriptPayload[];
+    isAvatarSpeaking: boolean;
+    isUserSpeaking: boolean;
+    orderItems: OrderItem[];
+    refreshToken: (force?: boolean) => Promise<string | null>;
+    clearTranscripts: () => void;
+    avatarProps: Pick<FacePeMobileAvatarProps, 'onTranscript' | 'onSpeakStart' | 'onSpeakEnd' | 'onUserSpeakStart' | 'onUserSpeakEnd' | 'onOrderUpdate' | 'onStatusChange'>;
+}
+export declare function useFacePeAvatar(): UseFacePeAvatarResult;

package/dist/useFacePeAvatar.js

@@ -0,0 +1,64 @@
+import { useState, useCallback, useMemo } from 'react';
+import { useFacePeMobileContext } from './FacePeMobileProvider';
+export function useFacePeAvatar() {
+    const { token, loading, error, refreshToken } = useFacePeMobileContext();
+    const [status, setStatus] = useState('idle');
+    const [transcripts, setTranscripts] = useState([]);
+    const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
+    const [isUserSpeaking, setIsUserSpeaking] = useState(false);
+    const [orderItems, setOrderItems] = useState([]);
+    const clearTranscripts = useCallback(() => {
+        setTranscripts([]);
+    }, []);
+    const handleTranscript = useCallback((event) => {
+        setTranscripts((prev) => [...prev, event.payload]);
+    }, []);
+    const handleSpeakStart = useCallback(() => {
+        setIsAvatarSpeaking(true);
+    }, []);
+    const handleSpeakEnd = useCallback(() => {
+        setIsAvatarSpeaking(false);
+    }, []);
+    const handleUserSpeakStart = useCallback(() => {
+        setIsUserSpeaking(true);
+    }, []);
+    const handleUserSpeakEnd = useCallback(() => {
+        setIsUserSpeaking(false);
+    }, []);
+    const handleOrderUpdate = useCallback((event) => {
+        setOrderItems(event.items);
+    }, []);
+    const handleStatusChange = useCallback((newStatus) => {
+        setStatus(newStatus);
+    }, []);
+    const avatarProps = useMemo(() => ({
+        onTranscript: handleTranscript,
+        onSpeakStart: handleSpeakStart,
+        onSpeakEnd: handleSpeakEnd,
+        onUserSpeakStart: handleUserSpeakStart,
+        onUserSpeakEnd: handleUserSpeakEnd,
+        onOrderUpdate: handleOrderUpdate,
+        onStatusChange: handleStatusChange,
+    }), [
+        handleTranscript,
+        handleSpeakStart,
+        handleSpeakEnd,
+        handleUserSpeakStart,
+        handleUserSpeakEnd,
+        handleOrderUpdate,
+        handleStatusChange,
+    ]);
+    return {
+        token,
+        loading,
+        error,
+        status,
+        transcripts,
+        isAvatarSpeaking,
+        isUserSpeaking,
+        orderItems,
+        refreshToken,
+        clearTranscripts,
+        avatarProps,
+    };
+}

package/package.json

@@ -1,6 +1,53 @@
 {
   "name": "@facepe/widget-mobile-app",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.1.0",
+  "description": "FacePe AI 3D Video Avatar Native Live Streaming SDK for React Native (iOS & Android)",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "facepe",
+    "avatar",
+    "ai-avatar",
+    "livekit",
+    "react-native",
+    "ios",
+    "android",
+    "webrtc",
+    "streaming",
+    "digital-human"
+  ],
+  "author": "FacePe AI",
+  "license": "MIT",
+  "scripts": {
+    "lint": "tsc --noEmit",
+    "build": "tsc"
+  },
+  "peerDependencies": {
+    "@livekit/react-native": ">=2.0.0",
+    "@livekit/react-native-webrtc": ">=124.0.0",
+    "react": ">=18.0.0",
+    "react-native": ">=0.70.0"
+  },
+  "peerDependenciesMeta": {
+    "@livekit/react-native": {
+      "optional": true
+    },
+    "@livekit/react-native-webrtc": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "react": "18.3.1",
+    "react-native": "0.74.0",
+    "typescript": "^5.4.5"
+  }
+}