@facepe/widget-mobile-app 0.0.0-stage → 1.0.0

package/README.md

@@ -1,3 +1,176 @@
-# Temporary Holding Version
+# @facepe/widget-mobile-app
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The official **FacePe AI 3D Avatar Live Streaming SDK** for **React Native** (supporting both **iOS** and **Android**).
+
+Embed interactive, photorealistic, lip-synced 3D digital avatars directly inside your native mobile apps with real-time speech recognition, natural audio dialog, and event callbacks.
+
+---
+
+## 📱 Features
+
+- **⚡ Low-Latency Live Video & Audio**: Real-time WebRTC media streaming for avatar rendering.
+- **🗣️ Two-Way Audio Conversation**: Bidirectional voice talk with interrupt handling & VAD.
+- **🍎 Full iOS Support**: Compatible with iOS 13.0+ (iPhone & iPad).
+- **🤖 Full Android Support**: Compatible with Android 7.0+ (API 24+).
+- **💬 Real-Time Transcripts**: Captions and user transcripts via `onTranscript`.
+- **🛒 Conversational Ordering**: Extract order items and payment intents directly in your app.
+
+---
+
+## 📦 Installation
+
+In your React Native / Expo project:
+
+```bash
+npm install @facepe/widget-mobile-app react-native-webview
+```
+*(or using yarn / pnpm)*:
+```bash
+yarn add @facepe/widget-mobile-app react-native-webview
+```
+
+---
+
+## 🛠️ Platform Permissions Setup
+
+### iOS Setup (`ios/Podfile` & `Info.plist`)
+
+1. Open `ios/YourAppName/Info.plist` and add permission strings for microphone and camera:
+
+```xml
+<key>NSMicrophoneUsageDescription</key>
+<string>We need access to your microphone to converse with the AI avatar.</string>
+<key>NSCameraUsageDescription</key>
+<string>We need access to your camera for interactive features.</string>
+```
+
+2. Run CocoaPods:
+```bash
+cd ios && pod install && cd ..
+```
+
+---
+
+### Android Setup (`android/app/src/main/AndroidManifest.xml`)
+
+Open `android/app/src/main/AndroidManifest.xml` and include permissions:
+
+```xml
+<uses-permission android:name="android.permission.INTERNET" />
+<uses-permission android:name="android.permission.RECORD_AUDIO" />
+<uses-permission android:name="android.permission.CAMERA" />
+<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
+```
+
+---
+
+## 🚀 Quick Start Example
+
+### 1. Server-side Token Endpoint
+
+Exchange your `accessKey` and `secretKey` on your secure backend (Express / Next.js / Python / Go) so secrets are never exposed on mobile:
+
+```javascript
+// POST /api/facepe-token
+app.get('/api/facepe-token', async (req, res) => {
+  const response = await fetch('https://api.facepe.ai/api/v1/sdk/token', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      accessKey: process.env.FACEPE_SDK_ACCESS_KEY,
+      secretKey: process.env.FACEPE_SDK_SECRET_KEY,
+      channel: 'sdk',
+    }),
+  });
+  const data = await response.json();
+  res.send(data.data.token);
+});
+```
+
+---
+
+### 2. React Native Component
+
+```tsx
+import React from 'react';
+import { StyleSheet, View, SafeAreaView, Text } from 'react-native';
+import {
+  FacePeMobileProvider,
+  FacePeMobileAvatar,
+} from '@facepe/widget-mobile-app';
+
+export default function App() {
+  return (
+    <SafeAreaView style={styles.container}>
+      <Text style={styles.title}>FacePe AI Digital Human</Text>
+
+      <FacePeMobileProvider
+        apiBaseUrl="https://api.facepe.ai/api/v1/sdk"
+        getToken={async () => {
+          const res = await fetch('https://your-backend.com/api/facepe-token');
+          return await res.text();
+        }}
+      >
+        <View style={styles.avatarContainer}>
+          <FacePeMobileAvatar
+            language="en"
+            onTranscript={({ payload }) => {
+              console.log(`[${payload.role}]:`, payload.text);
+            }}
+            onSpeakStart={() => console.log('Avatar speaking')}
+            onSpeakEnd={() => console.log('Avatar stopped speaking')}
+            onOrderUpdate={({ items }) => console.log('Order update:', items)}
+            onCheckout={() => console.log('Checkout requested!')}
+          />
+        </View>
+      </FacePeMobileProvider>
+    </SafeAreaView>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    backgroundColor: '#0F172A',
+  },
+  title: {
+    color: '#FFFFFF',
+    fontSize: 20,
+    fontWeight: 'bold',
+    textAlign: 'center',
+    marginVertical: 16,
+  },
+  avatarContainer: {
+    flex: 1,
+    marginHorizontal: 16,
+    marginBottom: 24,
+    borderRadius: 24,
+    overflow: 'hidden',
+  },
+});
+```
+
+---
+
+## ⚙️ Props & Configuration
+
+### `<FacePeMobileAvatar />`
+
+| Prop | Type | Description |
+| :--- | :--- | :--- |
+| `avatarKey` | `string` | Optional specific avatar key assigned to your company. |
+| `language` | `string` | Spoken language (`'en'`, `'hi'`, `'es'`, `'ta'`, `'te'`, etc.). |
+| `style` | `ViewStyle` | Style applied to container element. |
+| `onTranscript` | `function` | Fires whenever the user or avatar utters a phrase. |
+| `onSpeakStart` | `function` | Fires when the avatar begins vocal response. |
+| `onSpeakEnd` | `function` | Fires when the avatar stops speaking. |
+| `onUserSpeakStart` | `function` | Fires when the user starts speaking (voice activity). |
+| `onUserSpeakEnd` | `function` | Fires when the user pauses / stops speaking. |
+| `onOrderUpdate` | `function` | Fires when items are detected in conversation. |
+| `onCheckout` | `function` | Fires when customer signals intent to pay. |
+| `onError` | `function` | Error callback `{ code, message }`. |
+
+---
+
+## 📄 License
+MIT © FacePe AI

package/dist/FacePeMobileAvatar.d.ts

@@ -0,0 +1,8 @@
+import React from 'react';
+import type { FacePeMobileAvatarProps } from './types';
+export interface FacePeMobileAvatarRef {
+    start: () => Promise<void>;
+    stop: () => Promise<void>;
+    toggleMic: () => Promise<void>;
+}
+export declare const FacePeMobileAvatar: React.ForwardRefExoticComponent<FacePeMobileAvatarProps & React.RefAttributes<FacePeMobileAvatarRef>>;

package/dist/FacePeMobileAvatar.js

@@ -0,0 +1,155 @@
+import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
+import { useRef, useState, useImperativeHandle, forwardRef } from 'react';
+import { View, StyleSheet, ActivityIndicator, Text } from 'react-native';
+import { useFacePeMobileContext } from './FacePeMobileProvider';
+export const FacePeMobileAvatar = forwardRef(({ avatarKey, language = 'en', style, onTranscript, onSpeakStart, onSpeakEnd, onUserSpeakStart, onUserSpeakEnd, onOrderUpdate, onCheckout, onError, }, ref) => {
+    const { token, loading, error, apiBaseUrl } = useFacePeMobileContext();
+    const [streamReady, setStreamReady] = useState(false);
+    const webViewRef = useRef(null);
+    // Build the mobile streaming URL
+    const streamUrl = (() => {
+        if (!token)
+            return '';
+        const base = apiBaseUrl.replace(/\/sdk$/, '').replace(/\/api\/v1$/, '');
+        const search = new URLSearchParams({
+            token,
+            embedded: '1',
+            mobile: '1',
+            lang: language,
+        });
+        if (avatarKey)
+            search.set('avatar', avatarKey);
+        return `${base}/?${search.toString()}`;
+    })();
+    useImperativeHandle(ref, () => ({
+        start: async () => {
+            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.start()');
+        },
+        stop: async () => {
+            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.stop()');
+        },
+        toggleMic: async () => {
+            webViewRef.current?.injectJavaScript?.('window.FacePeBridge && window.FacePeBridge.toggleMic()');
+        },
+    }));
+    // Handle incoming messages from bridge (transcripts, order updates, speaking states)
+    const handleBridgeMessage = (event) => {
+        try {
+            const raw = event?.nativeEvent?.data;
+            if (!raw)
+                return;
+            const msg = JSON.parse(raw);
+            switch (msg.type) {
+                case 'avatar:ready':
+                    setStreamReady(true);
+                    break;
+                case 'avatar:transcript':
+                    onTranscript?.({ payload: msg.payload });
+                    break;
+                case 'avatar:speak:start':
+                    onSpeakStart?.();
+                    break;
+                case 'avatar:speak:end':
+                    onSpeakEnd?.();
+                    break;
+                case 'user:speak:start':
+                    onUserSpeakStart?.();
+                    break;
+                case 'user:speak:end':
+                    onUserSpeakEnd?.();
+                    break;
+                case 'order:update':
+                    onOrderUpdate?.({ items: msg.payload?.items || [] });
+                    break;
+                case 'order:checkout':
+                    onCheckout?.();
+                    break;
+                case 'avatar:error':
+                    onError?.({ code: msg.code || 'UNKNOWN', message: msg.message || 'Stream error' });
+                    break;
+                default:
+                    break;
+            }
+        }
+        catch {
+            // ignore parse error
+        }
+    };
+    if (loading) {
+        return (_jsxs(View, { style: [styles.container, styles.center, style], children: [_jsx(ActivityIndicator, { size: "large", color: "#7c3aed" }), _jsx(Text, { style: styles.loadingText, children: "Connecting to FacePe AI Avatar\u2026" })] }));
+    }
+    if (error) {
+        return (_jsx(View, { style: [styles.container, styles.center, style], children: _jsxs(Text, { style: styles.errorText, children: ["Unable to connect to Avatar: ", error.message] }) }));
+    }
+    // Try dynamically requiring react-native-webview
+    let WebViewComponent = null;
+    try {
+        WebViewComponent = require('react-native-webview').WebView;
+    }
+    catch {
+        // WebView not installed in project
+    }
+    if (!WebViewComponent) {
+        return (_jsxs(View, { style: [styles.container, styles.center, styles.fallbackBox, style], children: [_jsx(Text, { style: styles.noticeTitle, children: "FacePe Mobile SDK" }), _jsx(Text, { style: styles.noticeBody, children: "Please install `react-native-webview` or `@livekit/react-native` in your project to render the live stream:" }), _jsx(Text, { style: styles.codeSnippet, children: "npm install react-native-webview" })] }));
+    }
+    return (_jsx(View, { style: [styles.container, style], children: _jsx(WebViewComponent, { ref: webViewRef, source: { uri: streamUrl }, style: styles.webview, onMessage: handleBridgeMessage, allowsInlineMediaPlayback: true, mediaPlaybackRequiresUserAction: false, javaScriptEnabled: true, domStorageEnabled: true, cacheEnabled: true, 
+            // iOS WebRTC permissions
+            originWhitelist: ['*'], allowsAirPlayForMediaPlayback: false, bounces: false, scrollEnabled: false }) }));
+});
+FacePeMobileAvatar.displayName = 'FacePeMobileAvatar';
+const styles = StyleSheet.create({
+    container: {
+        width: '100%',
+        height: '100%',
+        backgroundColor: '#0F172A',
+        borderRadius: 16,
+        overflow: 'hidden',
+    },
+    webview: {
+        flex: 1,
+        backgroundColor: 'transparent',
+    },
+    center: {
+        justifyContent: 'center',
+        alignItems: 'center',
+        padding: 24,
+    },
+    loadingText: {
+        marginTop: 12,
+        color: '#94A3B8',
+        fontSize: 13,
+        fontWeight: '500',
+    },
+    errorText: {
+        color: '#EF4444',
+        fontSize: 13,
+        textAlign: 'center',
+    },
+    fallbackBox: {
+        backgroundColor: '#1E293B',
+        borderColor: '#334155',
+        borderWidth: 1,
+    },
+    noticeTitle: {
+        color: '#FFFFFF',
+        fontWeight: '700',
+        fontSize: 16,
+        marginBottom: 8,
+    },
+    noticeBody: {
+        color: '#94A3B8',
+        fontSize: 13,
+        textAlign: 'center',
+        lineHeight: 18,
+        marginBottom: 12,
+    },
+    codeSnippet: {
+        color: '#A78BFA',
+        backgroundColor: '#0F172A',
+        paddingHorizontal: 12,
+        paddingVertical: 6,
+        borderRadius: 8,
+        fontFamily: 'monospace',
+        fontSize: 12,
+    },
+});

package/dist/FacePeMobileProvider.d.ts

@@ -0,0 +1,19 @@
+import React, { type ReactNode } from 'react';
+import type { FacePeMobileConfig, TokenProvider } from './types';
+interface FacePeContextValue {
+    config: FacePeMobileConfig;
+    token: string | null;
+    loading: boolean;
+    error: Error | null;
+    refreshToken: (force?: boolean) => Promise<string | null>;
+    apiBaseUrl: string;
+}
+export interface FacePeMobileProviderProps {
+    children: ReactNode;
+    getToken: TokenProvider;
+    apiBaseUrl?: string;
+    config?: FacePeMobileConfig;
+}
+export declare function FacePeMobileProvider({ children, getToken, apiBaseUrl, config, }: FacePeMobileProviderProps): React.JSX.Element;
+export declare function useFacePeMobileContext(): FacePeContextValue;
+export {};

package/dist/FacePeMobileProvider.js

@@ -0,0 +1,54 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { createContext, useContext, useState, useEffect, useRef, useMemo } from 'react';
+const FacePeContext = createContext(null);
+export function FacePeMobileProvider({ children, getToken, apiBaseUrl = 'https://api.facepe.ai/api/v1/sdk', config = {}, }) {
+    const [token, setToken] = useState(null);
+    const [loading, setLoading] = useState(true);
+    const [error, setError] = useState(null);
+    const tokenPromiseRef = useRef(null);
+    const refreshToken = async (force = false) => {
+        if (tokenPromiseRef.current && !force) {
+            return tokenPromiseRef.current;
+        }
+        setLoading(true);
+        setError(null);
+        const promise = (async () => {
+            try {
+                const fetched = await getToken({ forceRefresh: force });
+                setToken(fetched);
+                setLoading(false);
+                return fetched;
+            }
+            catch (err) {
+                const normalized = err instanceof Error ? err : new Error(String(err));
+                setError(normalized);
+                setLoading(false);
+                return null;
+            }
+            finally {
+                tokenPromiseRef.current = null;
+            }
+        })();
+        tokenPromiseRef.current = promise;
+        return promise;
+    };
+    useEffect(() => {
+        void refreshToken();
+    }, []);
+    const value = useMemo(() => ({
+        config,
+        token,
+        loading,
+        error,
+        refreshToken,
+        apiBaseUrl: config.apiBaseUrl || apiBaseUrl,
+    }), [config, token, loading, error, apiBaseUrl]);
+    return _jsx(FacePeContext.Provider, { value: value, children: children });
+}
+export function useFacePeMobileContext() {
+    const ctx = useContext(FacePeContext);
+    if (!ctx) {
+        throw new Error('useFacePeMobileContext must be used within a <FacePeMobileProvider>');
+    }
+    return ctx;
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export * from './types';
+export * from './FacePeMobileProvider';
+export * from './FacePeMobileAvatar';
+export * from './useFacePeAvatar';

package/dist/index.js

@@ -0,0 +1,4 @@
+export * from './types';
+export * from './FacePeMobileProvider';
+export * from './FacePeMobileAvatar';
+export * from './useFacePeAvatar';

package/dist/types.d.ts

@@ -0,0 +1,67 @@
+export type AvatarGender = 'female' | 'male';
+export type StreamMode = 'native' | 'webview' | 'auto';
+export interface FacePeMobileTheme {
+    primaryColor?: string;
+    backgroundColor?: string;
+    borderRadius?: number;
+    textColor?: string;
+}
+export interface TranscriptPayload {
+    role: 'user' | 'assistant';
+    text: string;
+    isFinal?: boolean;
+}
+export interface OrderItem {
+    name: string;
+    qty: number;
+    price?: number;
+}
+export interface FacePeMobileConfig {
+    /** FacePe Gateway URL (e.g. https://api.facepe.ai/api/v1/sdk) */
+    apiBaseUrl?: string;
+    /** Avatar identifier or key */
+    avatarKey?: string;
+    /** Default language code (e.g., 'en', 'es', 'hi') */
+    language?: string;
+    /** Integration mode: 'auto', 'native', or 'webview' */
+    mode?: StreamMode;
+    /** Theme styling overrides */
+    theme?: FacePeMobileTheme;
+    /** Optional audio autoplay unblocking flag */
+    autoStart?: boolean;
+}
+export interface TokenProviderParams {
+    forceRefresh?: boolean;
+}
+export type TokenProvider = (params: TokenProviderParams) => Promise<string>;
+export interface FacePeMobileAvatarProps {
+    /** Optional avatar key override */
+    avatarKey?: string;
+    /** Language to converse in ('en', 'es', 'hi', etc.) */
+    language?: string;
+    /** Custom style object */
+    style?: any;
+    /** Called when real-time speech transcription arrives */
+    onTranscript?: (event: {
+        payload: TranscriptPayload;
+    }) => void;
+    /** Called when avatar starts speaking */
+    onSpeakStart?: () => void;
+    /** Called when avatar finishes speaking */
+    onSpeakEnd?: () => void;
+    /** Called when user starts speaking (detected by VAD) */
+    onUserSpeakStart?: () => void;
+    /** Called when user finishes speaking */
+    onUserSpeakEnd?: () => void;
+    /** Called when an order or intent snapshot is emitted */
+    onOrderUpdate?: (event: {
+        items: OrderItem[];
+    }) => void;
+    /** Called when customer says checkout / pay */
+    onCheckout?: () => void;
+    /** Called when an error occurs */
+    onError?: (event: {
+        code: string;
+        message: string;
+    }) => void;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/useFacePeAvatar.d.ts

@@ -0,0 +1,13 @@
+import type { TranscriptPayload, OrderItem } from './types';
+export interface UseFacePeAvatarResult {
+    token: string | null;
+    loading: boolean;
+    error: Error | null;
+    transcripts: TranscriptPayload[];
+    isAvatarSpeaking: boolean;
+    isUserSpeaking: boolean;
+    orderItems: OrderItem[];
+    refreshToken: (force?: boolean) => Promise<string | null>;
+    clearTranscripts: () => void;
+}
+export declare function useFacePeAvatar(): UseFacePeAvatarResult;

package/dist/useFacePeAvatar.js

@@ -0,0 +1,23 @@
+import { useState, useCallback } from 'react';
+import { useFacePeMobileContext } from './FacePeMobileProvider';
+export function useFacePeAvatar() {
+    const { token, loading, error, refreshToken } = useFacePeMobileContext();
+    const [transcripts, setTranscripts] = useState([]);
+    const [isAvatarSpeaking, setIsAvatarSpeaking] = useState(false);
+    const [isUserSpeaking, setIsUserSpeaking] = useState(false);
+    const [orderItems, setOrderItems] = useState([]);
+    const clearTranscripts = useCallback(() => {
+        setTranscripts([]);
+    }, []);
+    return {
+        token,
+        loading,
+        error,
+        transcripts,
+        isAvatarSpeaking,
+        isUserSpeaking,
+        orderItems,
+        refreshToken,
+        clearTranscripts,
+    };
+}

package/package.json

@@ -1,6 +1,53 @@
 {
   "name": "@facepe/widget-mobile-app",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "1.0.0",
+  "description": "FacePe AI 3D Video Avatar Live Streaming SDK for React Native (iOS & Android)",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "facepe",
+    "avatar",
+    "ai-avatar",
+    "livekit",
+    "react-native",
+    "ios",
+    "android",
+    "webrtc",
+    "streaming"
+  ],
+  "author": "FacePe AI",
+  "license": "MIT",
+  "scripts": {
+    "lint": "tsc --noEmit",
+    "build": "tsc"
+  },
+  "peerDependencies": {
+    "react": ">=18.0.0",
+    "react-native": ">=0.70.0"
+  },
+  "peerDependenciesMeta": {
+    "@livekit/react-native": {
+      "optional": true
+    },
+    "@livekit/react-native-webrtc": {
+      "optional": true
+    },
+    "react-native-webview": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@types/react": "^18.3.3",
+    "react": "18.3.1",
+    "react-native": "0.74.0",
+    "typescript": "^5.4.5"
+  }
+}