@elevenlabs/react-native 0.1.0

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@elevenlabs/react-native",
+  "version": "0.1.0",
+  "description": "ElevenLabs React Native SDK for Conversational AI",
+  "main": "./dist/lib.js",
+  "module": "./dist/lib.module.js",
+  "source": "src/index.ts",
+  "unpkg": "./dist/lib.umd.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/lib.modern.js",
+      "require": "./dist/lib.js"
+    }
+  },
+  "keywords": [
+    "elevenlabs",
+    "react-native",
+    "conversational-ai",
+    "webrtc",
+    "voice",
+    "ai"
+  ],
+  "author": "ElevenLabs",
+  "license": "MIT",
+  "peerDependencies": {
+    "@livekit/react-native": "^2.7.0",
+    "@livekit/react-native-webrtc": "^125.0.0",
+    "livekit-client": "^2.11.0",
+    "react": ">=17.0.0",
+    "react-native": ">=0.70.0"
+  },
+  "devDependencies": {
+    "@testing-library/jest-native": "^5.4.3",
+    "@testing-library/react-native": "^13.2.0",
+    "@types/jest": "^29.5.12",
+    "@types/react": "^19.0.0",
+    "@types/react-native": "^0.73.0",
+    "eslint": "^9.8.0",
+    "jest": "^29.7.0",
+    "microbundle": "^0.15.1",
+    "react-test-renderer": "19.0.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.5.4"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/elevenlabs/packages.git",
+    "directory": "packages/react-native"
+  },
+  "scripts": {
+    "generate-version": "printf \"// This file is auto-generated during build\\nexport const PACKAGE_VERSION = \\\"%s\\\";\\n\" \"$npm_package_version\" > src/version.ts",
+    "prebuild": "npm run generate-version",
+    "build": "BROWSERSLIST_ENV=modern microbundle --jsx React.createElement --jsxFragment React.Fragment --jsxImportSource react src/index.ts",
+    "clean": "rm -rf ./dist",
+    "dev": "npm run clean && npm run generate-version && BROWSERSLIST_ENV=modern microbundle --jsx React.createElement --jsxFragment React.Fragment --jsxImportSource react src/index.ts -w --no-compress",
+    "lint": "npm run lint:ts && npm run lint:es && npm run lint:prettier",
+    "lint:ts": "tsc --noEmit --skipLibCheck",
+    "lint:es": "npx eslint .",
+    "lint:prettier": "prettier 'src/**/*.ts' --check",
+    "test": "jest"
+  }
+}

package/src/ElevenLabsProvider.test.tsx

@@ -0,0 +1,187 @@
+import React from 'react';
+import { render } from '@testing-library/react-native';
+import { ElevenLabsProvider, useConversation } from './ElevenLabsProvider';
+import type { ReactNode } from 'react';
+
+// Suppress react-test-renderer deprecation warnings during tests
+const originalWarn = console.warn;
+const originalError = console.error;
+
+beforeAll(() => {
+  console.warn = (...args) => {
+    if (args[0]?.includes?.('react-test-renderer is deprecated')) return;
+    originalWarn(...args);
+  };
+  console.error = (...args) => {
+    if (args[0]?.includes?.('react-test-renderer is deprecated')) return;
+    originalError(...args);
+  };
+});
+
+afterAll(() => {
+  console.warn = originalWarn;
+  console.error = originalError;
+});
+
+// Simple test component to avoid React Native import issues
+const TestText = ({ children }: { children: ReactNode }) => (
+  <div>{children}</div>
+);
+
+// Mock LiveKit
+jest.mock('@livekit/react-native', () => ({
+  registerGlobals: jest.fn(),
+}));
+
+// Mock hooks with inline functions to avoid hoisting issues
+jest.mock('./hooks/useConversationSession', () => ({
+  useConversationSession: () => ({
+    startSession: jest.fn(),
+    endSession: jest.fn(),
+    overrides: {},
+    customLlmExtraBody: undefined,
+    defaultServerUrl: 'https://api.elevenlabs.io/v1/convai',
+  }),
+}));
+
+jest.mock('./hooks/useConversationCallbacks', () => ({
+  useConversationCallbacks: () => ({
+    callbacksRef: { current: {} },
+    setCallbacks: jest.fn(),
+  }),
+}));
+
+jest.mock('./hooks/useLiveKitRoom', () => ({
+  useLiveKitRoom: () => ({
+    room: null,
+    localParticipant: null,
+    roomConnected: false,
+    handleParticipantReady: jest.fn(),
+    handleConnected: jest.fn(),
+    handleDisconnected: jest.fn(),
+    handleError: jest.fn(),
+  }),
+}));
+
+jest.mock('./hooks/useMessageSending', () => ({
+  useMessageSending: () => ({
+    sendMessage: jest.fn(),
+    sendFeedback: jest.fn(),
+    sendContextualUpdate: jest.fn(),
+    sendUserMessage: jest.fn(),
+    sendUserActivity: jest.fn(),
+  }),
+}));
+
+jest.mock('./components/MessageHandler', () => ({
+  MessageHandler: () => null,
+}));
+
+jest.mock('./components/LiveKitRoomWrapper', () => ({
+  LiveKitRoomWrapper: ({ children }: { children: ReactNode }) => <>{children}</>,
+}));
+
+describe('ElevenLabsProvider', () => {
+  describe('Core Functionality', () => {
+    it('should throw error when useConversation is used outside provider', () => {
+      const BadComponent = () => {
+        useConversation();
+        return <TestText>Should not render</TestText>;
+      };
+
+      const originalError = console.error;
+      console.error = jest.fn();
+
+      expect(() => {
+        render(<BadComponent />);
+      }).toThrow('useConversation must be used within ElevenLabsProvider');
+
+      console.error = originalError;
+    });
+
+    it('should provide conversation context and all methods when used within provider', () => {
+      const TestComponent = () => {
+        const conversation = useConversation();
+
+        // Verify all methods exist and are functions
+        expect(typeof conversation.startSession).toBe('function');
+        expect(typeof conversation.endSession).toBe('function');
+        expect(typeof conversation.sendFeedback).toBe('function');
+        expect(typeof conversation.sendContextualUpdate).toBe('function');
+        expect(typeof conversation.sendUserMessage).toBe('function');
+        expect(typeof conversation.sendUserActivity).toBe('function');
+
+        // Verify all properties exist with correct types
+        expect(typeof conversation.status).toBe('string');
+        expect(typeof conversation.isSpeaking).toBe('boolean');
+        expect(typeof conversation.canSendFeedback).toBe('boolean');
+
+        return <TestText>Test passed</TestText>;
+      };
+
+      expect(() => {
+        render(
+          <ElevenLabsProvider>
+            <TestComponent />
+          </ElevenLabsProvider>
+        );
+      }).not.toThrow();
+    });
+
+    it('should allow methods to be called without throwing errors', () => {
+      const TestComponent = () => {
+        const conversation = useConversation();
+
+        // Test that all methods can be called safely
+        expect(() => conversation.startSession({ agentId: 'test' })).not.toThrow();
+        expect(() => conversation.endSession()).not.toThrow();
+        expect(() => conversation.sendFeedback(true)).not.toThrow();
+        expect(() => conversation.sendContextualUpdate('test context')).not.toThrow();
+        expect(() => conversation.sendUserMessage('test message')).not.toThrow();
+        expect(() => conversation.sendUserActivity()).not.toThrow();
+
+        return <TestText>Method tests passed</TestText>;
+      };
+
+      expect(() => {
+        render(
+          <ElevenLabsProvider>
+            <TestComponent />
+          </ElevenLabsProvider>
+        );
+      }).not.toThrow();
+    });
+
+    it('should render children components successfully', () => {
+      // Simple test - if rendering doesn't throw, children are successfully rendered
+      expect(() => {
+        render(
+          <ElevenLabsProvider>
+            <TestText>Child component rendered</TestText>
+          </ElevenLabsProvider>
+        );
+      }).not.toThrow();
+    });
+
+    it('should provide conversation context with options', () => {
+      const TestComponent = () => {
+        const conversation = useConversation({
+          onConnect: jest.fn(),
+          onDisconnect: jest.fn(),
+          onError: jest.fn(),
+        });
+
+        expect(conversation).toBeDefined();
+        return <TestText>Options test passed</TestText>;
+      };
+
+      expect(() => {
+        render(
+          <ElevenLabsProvider>
+            <TestComponent />
+          </ElevenLabsProvider>
+        );
+      }).not.toThrow();
+    });
+  });
+});

package/src/ElevenLabsProvider.tsx

@@ -0,0 +1,259 @@
+import React from 'react';
+import { createContext, useContext, useState } from 'react';
+import { registerGlobals } from '@livekit/react-native';
+import type { LocalParticipant } from 'livekit-client';
+import type { Callbacks, ConversationConfig, ConversationStatus, ClientToolsConfig } from './types';
+import { constructOverrides } from './utils/overrides';
+import { DEFAULT_SERVER_URL } from './utils/constants';
+import { useConversationCallbacks } from './hooks/useConversationCallbacks';
+import { useConversationSession } from './hooks/useConversationSession';
+import { useLiveKitRoom } from './hooks/useLiveKitRoom';
+import { useMessageSending } from './hooks/useMessageSending';
+import { LiveKitRoomWrapper } from './components/LiveKitRoomWrapper';
+
+interface ConversationOptions extends Callbacks, Partial<ClientToolsConfig> {
+  serverUrl?: string;
+}
+
+interface Conversation {
+  startSession: (config: ConversationConfig) => Promise<void>;
+  endSession: () => Promise<void>;
+  status: ConversationStatus;
+  isSpeaking: boolean;
+  // TODO: Implement setVolume when LiveKit React Native supports it
+  // setVolume: (volume: number) => void;
+  canSendFeedback: boolean;
+  sendFeedback: (like: boolean) => void;
+  sendContextualUpdate: (text: string) => void;
+  sendUserMessage: (text: string) => void;
+  sendUserActivity: () => void;
+}
+
+interface ElevenLabsContextType {
+  conversation: Conversation;
+  callbacksRef: { current: Callbacks };
+  serverUrl: string;
+  clientTools: ClientToolsConfig['clientTools'];
+  setCallbacks: (callbacks: Callbacks) => void;
+  setServerUrl: (url: string) => void;
+  setClientTools: (tools: ClientToolsConfig['clientTools']) => void;
+}
+
+const ElevenLabsContext = createContext<ElevenLabsContextType | null>(null);
+
+export const useConversation = (options: ConversationOptions = {}): Conversation => {
+  const context = useContext(ElevenLabsContext);
+  if (!context) {
+    throw new Error('useConversation must be used within ElevenLabsProvider');
+  }
+
+  const { serverUrl, clientTools, ...callbacks } = options;
+
+  React.useEffect(() => {
+    if (serverUrl) {
+      context.setServerUrl(serverUrl);
+    }
+  }, [context, serverUrl]);
+
+  if (clientTools) {
+    context.setClientTools(clientTools);
+  }
+
+  context.setCallbacks(callbacks);
+
+  return context.conversation;
+};
+
+interface ElevenLabsProviderProps {
+  children: React.ReactNode;
+}
+
+export const ElevenLabsProvider: React.FC<ElevenLabsProviderProps> = ({ children }) => {
+  // Initialize globals on mount
+  registerGlobals();
+
+  // State management
+  const [token, setToken] = useState('');
+  const [connect, setConnect] = useState(false);
+  const [status, setStatus] = useState<ConversationStatus>('disconnected');
+  const [serverUrl, setServerUrl] = useState(DEFAULT_SERVER_URL);
+  const [roomId, setRoomId] = useState<string>('');
+  const [isSpeaking, setIsSpeaking] = useState(false);
+  const [canSendFeedback, setCanSendFeedback] = useState(false);
+
+  // Feedback state tracking
+  const currentEventIdRef = React.useRef(1);
+  const lastFeedbackEventIdRef = React.useRef(1);
+
+  // Use ref for clientTools to avoid re-renders (like callbacks)
+  const clientToolsRef = React.useRef<ClientToolsConfig['clientTools']>({});
+
+  // Custom hooks
+  const { callbacksRef, setCallbacks: setCallbacksBase } = useConversationCallbacks();
+
+  // Enhanced setCallbacks that wraps onModeChange to update isSpeaking state
+  const setCallbacks = React.useCallback((callbacks: Callbacks) => {
+    const wrappedCallbacks = {
+      ...callbacks,
+      onModeChange: (event: { mode: 'speaking' | 'listening' }) => {
+        setIsSpeaking(event.mode === 'speaking');
+        callbacks.onModeChange?.(event);
+      }
+    };
+    setCallbacksBase(wrappedCallbacks);
+  }, [setCallbacksBase]);
+
+  const {
+    startSession,
+    endSession,
+    overrides,
+    customLlmExtraBody,
+    dynamicVariables,
+  } = useConversationSession(callbacksRef, setStatus, setConnect, setToken, setRoomId);
+
+  const {
+    roomConnected,
+    localParticipant,
+    handleParticipantReady,
+    handleConnected,
+    handleDisconnected,
+    handleError,
+  } = useLiveKitRoom(callbacksRef, setStatus, roomId);
+
+  // Enhanced connection handler to initialize feedback state
+  const handleConnectedWithFeedback = React.useCallback(() => {
+    // Reset feedback state when connecting
+    currentEventIdRef.current = 1;
+    lastFeedbackEventIdRef.current = 1;
+    setCanSendFeedback(false);
+    callbacksRef.current.onCanSendFeedbackChange?.({ canSendFeedback: false });
+
+    handleConnected();
+  }, [handleConnected, callbacksRef]);
+
+  // Enhanced disconnection handler to reset feedback state
+  const handleDisconnectedWithFeedback = React.useCallback(() => {
+    setCanSendFeedback(false);
+    handleDisconnected();
+  }, [handleDisconnected]);
+
+  const { sendMessage } = useMessageSending(status, localParticipant, callbacksRef);
+
+  const updateCanSendFeedback = React.useCallback(() => {
+    const newCanSendFeedback = currentEventIdRef.current !== lastFeedbackEventIdRef.current;
+
+    if (canSendFeedback !== newCanSendFeedback) {
+      setCanSendFeedback(newCanSendFeedback);
+      callbacksRef.current.onCanSendFeedbackChange?.({ canSendFeedback: newCanSendFeedback });
+    }
+  }, [canSendFeedback, callbacksRef]);
+
+  const sendFeedback = React.useCallback((like: boolean) => {
+    if (!canSendFeedback) {
+      console.warn(
+        lastFeedbackEventIdRef.current === 0
+          ? "Cannot send feedback: the conversation has not started yet."
+          : "Cannot send feedback: feedback has already been sent for the current response."
+      );
+      return;
+    }
+
+    const feedbackMessage = {
+      type: "feedback",
+      score: like ? "like" : "dislike",
+      event_id: currentEventIdRef.current,
+    };
+
+    sendMessage(feedbackMessage);
+    lastFeedbackEventIdRef.current = currentEventIdRef.current;
+    updateCanSendFeedback();
+  }, [canSendFeedback, sendMessage, updateCanSendFeedback]);
+
+  // setVolume placeholder (to be implemented when LiveKit supports it)
+  const setVolume = React.useCallback((volume: number) => {
+    console.warn('setVolume is not yet implemented in React Native SDK');
+  }, []);
+
+  // Update current event ID for feedback tracking
+  const updateCurrentEventId = React.useCallback((eventId: number) => {
+    currentEventIdRef.current = eventId;
+    updateCanSendFeedback();
+  }, [updateCanSendFeedback]);
+
+  // Handle participant ready with overrides
+  const handleParticipantReadyWithOverrides = React.useCallback((participant: LocalParticipant) => {
+    handleParticipantReady(participant);
+
+    if (localParticipant) {
+      const overridesEvent = constructOverrides({
+        overrides,
+        customLlmExtraBody,
+        dynamicVariables,
+      });
+      sendMessage(overridesEvent);
+    }
+  }, [handleParticipantReady, localParticipant, overrides, customLlmExtraBody, dynamicVariables, sendMessage]);
+
+  const conversation: Conversation = {
+    startSession,
+    endSession,
+    status,
+    isSpeaking,
+    // setVolume,
+    canSendFeedback,
+    sendFeedback,
+    sendContextualUpdate: (text: string) => {
+      sendMessage({
+        type: "contextual_update",
+        text,
+      });
+    },
+    sendUserMessage: (text: string) => {
+      sendMessage({
+        type: "user_message",
+        text,
+      });
+    },
+    sendUserActivity: () => {
+      sendMessage({
+        type: "user_activity",
+      });
+    },
+  };
+
+  // Create setClientTools function that only updates ref
+  const setClientTools = React.useCallback((tools: ClientToolsConfig['clientTools']) => {
+    clientToolsRef.current = tools;
+  }, []);
+
+  const contextValue: ElevenLabsContextType = {
+    conversation,
+    callbacksRef,
+    serverUrl,
+    clientTools: clientToolsRef.current,
+    setCallbacks,
+    setServerUrl,
+    setClientTools,
+  };
+
+  return (
+    <ElevenLabsContext.Provider value={contextValue}>
+      <LiveKitRoomWrapper
+        serverUrl={serverUrl}
+        token={token}
+        connect={connect}
+        onConnected={handleConnectedWithFeedback}
+        onDisconnected={handleDisconnectedWithFeedback}
+        onError={handleError}
+        roomConnected={roomConnected}
+        callbacks={callbacksRef.current}
+        onParticipantReady={handleParticipantReadyWithOverrides}
+        sendMessage={sendMessage}
+        clientTools={clientToolsRef.current}
+        updateCurrentEventId={updateCurrentEventId}
+      >
+        {children}
+      </LiveKitRoomWrapper>
+    </ElevenLabsContext.Provider>
+  );
+};

package/src/components/LiveKitRoomWrapper.tsx

@@ -0,0 +1,63 @@
+import React from 'react';
+import { LiveKitRoom } from '@livekit/react-native';
+import type { LocalParticipant } from 'livekit-client';
+import type { Callbacks, ClientToolsConfig } from '../types';
+import { MessageHandler } from './MessageHandler';
+
+interface LiveKitRoomWrapperProps {
+  children: React.ReactNode;
+  serverUrl: string;
+  token: string;
+  connect: boolean;
+  onConnected: () => void;
+  onDisconnected: () => void;
+  onError: (error: Error) => void;
+  roomConnected: boolean;
+  callbacks: Callbacks;
+  onParticipantReady: (participant: LocalParticipant) => void;
+  sendMessage: (message: unknown) => void;
+  clientTools: ClientToolsConfig['clientTools'];
+  updateCurrentEventId?: (eventId: number) => void;
+}
+
+export const LiveKitRoomWrapper = ({
+  children,
+  serverUrl,
+  token,
+  connect,
+  onConnected,
+  onDisconnected,
+  onError,
+  roomConnected,
+  callbacks,
+  onParticipantReady,
+  sendMessage,
+  clientTools,
+  updateCurrentEventId,
+}: LiveKitRoomWrapperProps) => {
+  return (
+    <LiveKitRoom
+      serverUrl={serverUrl}
+      token={token}
+      connect={connect}
+      audio={true}
+      video={false}
+      options={{
+        adaptiveStream: { pixelDensity: 'screen' },
+      }}
+      onConnected={onConnected}
+      onDisconnected={onDisconnected}
+      onError={onError}
+    >
+      <MessageHandler
+        onReady={onParticipantReady}
+        isConnected={roomConnected}
+        callbacks={callbacks}
+        sendMessage={sendMessage}
+        clientTools={clientTools}
+        updateCurrentEventId={updateCurrentEventId}
+      />
+      {children}
+    </LiveKitRoom>
+  );
+};

package/src/components/MessageHandler.tsx

@@ -0,0 +1,127 @@
+import { useEffect } from 'react';
+import { useLocalParticipant, useDataChannel } from '@livekit/react-native';
+import type { LocalParticipant } from 'livekit-client';
+import type { Callbacks, ClientToolsConfig, ClientToolCallEvent } from '../types';
+import React from 'react';
+
+interface MessageHandlerProps {
+  onReady: (participant: LocalParticipant) => void;
+  isConnected: boolean;
+  callbacks: Callbacks;
+  sendMessage: (message: unknown) => void;
+  clientTools?: ClientToolsConfig['clientTools'];
+  updateCurrentEventId?: (eventId: number) => void;
+}
+
+export const MessageHandler = ({
+  onReady,
+  isConnected,
+  callbacks,
+  sendMessage,
+  clientTools = {},
+  updateCurrentEventId
+}: MessageHandlerProps) => {
+  const { localParticipant } = useLocalParticipant();
+
+  // Track agent response count for synthetic event IDs (WebRTC mode)
+  const agentResponseCountRef = React.useRef(1);
+
+  useEffect(() => {
+    if (isConnected && localParticipant) {
+      onReady(localParticipant);
+    }
+  }, [isConnected, localParticipant, onReady]);
+
+  const handleClientToolCall = async (clientToolCall: ClientToolCallEvent) => {
+    if (
+      Object.prototype.hasOwnProperty.call(
+        clientTools,
+        clientToolCall.tool_name
+      )
+    ) {
+      try {
+        const result =
+          (await clientTools[clientToolCall.tool_name](
+            clientToolCall.parameters
+          )) ?? "Client tool execution successful."; // default client-tool call response
+
+        // The API expects result to be a string, so we need to convert it if it's not already a string
+        const formattedResult =
+          typeof result === "object" ? JSON.stringify(result) : String(result);
+
+        sendMessage({
+          type: "client_tool_result",
+          tool_call_id: clientToolCall.tool_call_id,
+          result: formattedResult,
+          is_error: false,
+        });
+      } catch (e) {
+        const errorMessage = `Client tool execution failed with following error: ${(e as Error)?.message}`;
+        callbacks.onError?.(errorMessage, {
+          clientToolName: clientToolCall.tool_name,
+        });
+        sendMessage({
+          type: "client_tool_result",
+          tool_call_id: clientToolCall.tool_call_id,
+          result: `Client tool execution failed: ${(e as Error)?.message}`,
+          is_error: true,
+        });
+      }
+    } else {
+      if (callbacks.onUnhandledClientToolCall) {
+        callbacks.onUnhandledClientToolCall(clientToolCall);
+        return;
+      }
+
+      const errorMessage = `Client tool with name ${clientToolCall.tool_name} is not defined on client`;
+      callbacks.onError?.(errorMessage, {
+        clientToolName: clientToolCall.tool_name,
+      });
+      sendMessage({
+        type: "client_tool_result",
+        tool_call_id: clientToolCall.tool_call_id,
+        result: errorMessage,
+        is_error: true,
+      });
+    }
+  };
+
+  const _ = useDataChannel((msg) => {
+    const decoder = new TextDecoder();
+    const message = JSON.parse(decoder.decode(msg.payload));
+
+    callbacks.onMessage?.({
+      message,
+      source: msg.from?.isAgent ? 'ai' : 'user',
+    });
+
+    if (msg.from?.isAgent) {
+      callbacks.onModeChange?.({
+        mode: msg.from?.isSpeaking ? 'speaking' : 'listening'
+      });
+
+      // Track agent responses for feedback (WebRTC mode needs synthetic event IDs)
+      if (message.type === "agent_response" && updateCurrentEventId) {
+        const eventId = agentResponseCountRef.current++;
+        updateCurrentEventId(eventId);
+      }
+    }
+
+    switch (message.type) {
+      case "ping":
+        sendMessage({
+          type: "pong",
+          event_id: message.ping_event.event_id,
+        });
+        break;
+      case "client_tool_call":
+        handleClientToolCall(message.client_tool_call);
+        break;
+      default:
+        callbacks.onDebug?.(message);
+        break;
+    }
+  });
+
+  return null;
+};