@elevenlabs/react-native 0.5.12 → 1.0.0-rc.0

package/src/ElevenLabsProvider.tsx

@@ -1,365 +0,0 @@
-import React from 'react';
-import { createContext, useContext, useState } from 'react';
-import { registerGlobals } from '@livekit/react-native';
-import type { LocalParticipant } from 'livekit-client';
-import type { Callbacks, ConversationConfig, ConversationStatus, ClientToolsConfig, AudioSessionConfig } from './types';
-import { constructOverrides } from './utils/overrides';
-import { DEFAULT_SERVER_URL } from './utils/constants';
-import { useConversationCallbacks } from './hooks/useConversationCallbacks';
-import { useConversationSession } from './hooks/useConversationSession';
-import { useLiveKitRoom } from './hooks/useLiveKitRoom';
-import { useMessageSending } from './hooks/useMessageSending';
-import { LiveKitRoomWrapper } from './components/LiveKitRoomWrapper';
-
-interface ConversationOptions extends Callbacks, Partial<ClientToolsConfig> {
-  serverUrl?: string;
-  tokenFetchUrl?: string;
-}
-
-export interface Conversation {
-  startSession: (config: ConversationConfig) => Promise<void>;
-  endSession: (reason?: "user" | "agent") => Promise<void>;
-  status: ConversationStatus;
-  isSpeaking: boolean;
-  // TODO: Implement setVolume when LiveKit React Native supports it
-  // setVolume: (volume: number) => void;
-  canSendFeedback: boolean;
-  getId: () => string;
-  sendFeedback: (like: boolean) => void;
-  sendContextualUpdate: (text: string) => void;
-  sendUserMessage: (text: string) => void;
-  sendUserActivity: () => void;
-  sendMultimodalMessage: (options: { text?: string; fileId?: string }) => void;
-  setMicMuted: (muted: boolean) => void;
-}
-
-interface ElevenLabsContextType {
-  conversation: Conversation;
-  callbacksRef: { current: Callbacks };
-  serverUrl: string;
-  tokenFetchUrl?: string;
-  clientTools: ClientToolsConfig['clientTools'];
-  setCallbacks: (callbacks: Callbacks) => void;
-  setServerUrl: (url: string) => void;
-  setTokenFetchUrl: (url: string) => void;
-  setClientTools: (tools: ClientToolsConfig['clientTools']) => void;
-}
-
-const ElevenLabsContext = createContext<ElevenLabsContextType | null>(null);
-
-export const useConversation = (options: ConversationOptions = {}): Conversation => {
-  const context = useContext(ElevenLabsContext);
-  if (!context) {
-    throw new Error('useConversation must be used within ElevenLabsProvider');
-  }
-
-  const { serverUrl, tokenFetchUrl, clientTools, ...callbacks } = options;
-
-  React.useEffect(() => {
-    if (serverUrl) {
-      context.setServerUrl(serverUrl);
-    }
-  }, [context, serverUrl]);
-
-  React.useEffect(() => {
-    if (tokenFetchUrl) {
-      context.setTokenFetchUrl(tokenFetchUrl);
-    }
-  }, [context, tokenFetchUrl]);
-
-  React.useEffect(() => {
-    if (clientTools) {
-      context.setClientTools(clientTools);
-    }
-  }, [context, clientTools]);
-
-  // Update callbacks - since this only updates a ref, it's safe to call on every render
-  // and doesn't cause re-renders of the provider or its consumers
-  context.setCallbacks(callbacks);
-
-  return context.conversation;
-};
-
-interface ElevenLabsProviderProps {
-  children: React.ReactNode;
-  audioSessionConfig?: AudioSessionConfig;
-}
-
-export const ElevenLabsProvider: React.FC<ElevenLabsProviderProps> = ({ children, audioSessionConfig }) => {
-  // Initialize globals on mount
-  registerGlobals();
-
-  // State management
-  const [token, setToken] = useState('');
-  const [connect, setConnect] = useState(false);
-  const [status, setStatus] = useState<ConversationStatus>('disconnected');
-  const [serverUrl, setServerUrl] = useState(DEFAULT_SERVER_URL);
-  const [tokenFetchUrl, setTokenFetchUrl] = useState<string | undefined>(undefined);
-  const [conversationId, setConversationId] = useState<string>('');
-  const [isSpeaking, setIsSpeaking] = useState(false);
-  const [canSendFeedback, setCanSendFeedback] = useState(false);
-
-  // Feedback state tracking
-  const currentEventIdRef = React.useRef(1);
-  const lastFeedbackEventIdRef = React.useRef(1);
-
-  // Use ref for clientTools to avoid re-renders (like callbacks)
-  const clientToolsRef = React.useRef<ClientToolsConfig['clientTools']>({});
-
-  // Custom hooks
-  const { callbacksRef, setCallbacks: setCallbacksBase } = useConversationCallbacks();
-
-  // Enhanced setCallbacks that wraps onModeChange to update isSpeaking state
-  const setCallbacks = React.useCallback((callbacks: Callbacks) => {
-    const wrappedCallbacks = {
-      ...callbacks,
-      onModeChange: (event: { mode: 'speaking' | 'listening' }) => {
-        setIsSpeaking(event.mode === 'speaking');
-        callbacks.onModeChange?.(event);
-      }
-    };
-    setCallbacksBase(wrappedCallbacks);
-  }, [setCallbacksBase]);
-
-  const {
-    startSession,
-    endSession,
-    overrides,
-    customLlmExtraBody,
-    dynamicVariables,
-    userId,
-    textOnly,
-  } = useConversationSession(callbacksRef, setStatus, setConnect, setToken, setConversationId, tokenFetchUrl);
-
-  const {
-    roomConnected,
-    localParticipant,
-    handleParticipantReady,
-    handleConnected,
-    handleDisconnected,
-    handleError,
-  } = useLiveKitRoom(callbacksRef, setStatus, conversationId, status, textOnly);
-
-  // Enhanced connection handler to initialize feedback state
-  const handleConnectedWithFeedback = React.useCallback(() => {
-    // Reset feedback state when connecting
-    currentEventIdRef.current = 1;
-    lastFeedbackEventIdRef.current = 1;
-    setCanSendFeedback(false);
-    callbacksRef.current.onCanSendFeedbackChange?.({ canSendFeedback: false });
-
-    handleConnected();
-  }, [handleConnected, callbacksRef]);
-
-  // Enhanced disconnection handler to reset feedback state
-  const handleDisconnectedWithFeedback = React.useCallback(() => {
-    setCanSendFeedback(false);
-    setIsSpeaking(false);
-    handleDisconnected();
-  }, [handleDisconnected]);
-
-  const { sendMessage } = useMessageSending(status, localParticipant, callbacksRef);
-
-  const updateCanSendFeedback = React.useCallback(() => {
-    const newCanSendFeedback = currentEventIdRef.current !== lastFeedbackEventIdRef.current;
-
-    if (canSendFeedback !== newCanSendFeedback) {
-      setCanSendFeedback(newCanSendFeedback);
-      callbacksRef.current.onCanSendFeedbackChange?.({ canSendFeedback: newCanSendFeedback });
-    }
-  }, [canSendFeedback, callbacksRef]);
-
-  const sendFeedback = React.useCallback((like: boolean) => {
-    if (!canSendFeedback) {
-      console.warn(
-        lastFeedbackEventIdRef.current === 0
-          ? "Cannot send feedback: the conversation has not started yet."
-          : "Cannot send feedback: feedback has already been sent for the current response."
-      );
-      return;
-    }
-
-    const feedbackMessage = {
-      type: "feedback",
-      score: like ? "like" : "dislike",
-      event_id: currentEventIdRef.current,
-    };
-
-    sendMessage(feedbackMessage);
-    lastFeedbackEventIdRef.current = currentEventIdRef.current;
-    updateCanSendFeedback();
-  }, [canSendFeedback, sendMessage, updateCanSendFeedback]);
-
-  // setVolume placeholder (to be implemented when LiveKit supports it)
-  const setVolume = React.useCallback((volume: number) => {
-    console.warn('setVolume is not yet implemented in React Native SDK');
-  }, []);
-
-  const getId = React.useCallback(() => conversationId, [conversationId]);
-
-  const setMicMuted = React.useCallback((muted: boolean) => {
-    if (localParticipant) {
-      localParticipant.setMicrophoneEnabled(!muted);
-    }
-  }, [localParticipant]);
-
-  // Update current event ID for feedback tracking
-  const updateCurrentEventId = React.useCallback((eventId: number) => {
-    currentEventIdRef.current = eventId;
-    updateCanSendFeedback();
-  }, [updateCanSendFeedback]);
-
-  // Handle participant ready with overrides
-  const handleParticipantReadyWithOverrides = React.useCallback((participant: LocalParticipant) => {
-    handleParticipantReady(participant);
-
-    const overridesEvent = constructOverrides({
-      overrides,
-      customLlmExtraBody,
-      dynamicVariables,
-      userId,
-    });
-
-    if (overridesEvent) {
-      try {
-        const encoder = new TextEncoder();
-        const data = encoder.encode(JSON.stringify(overridesEvent));
-        participant.publishData(data, { reliable: true });
-      } catch (error) {
-        console.error("Failed to send overrides:", error);
-        callbacksRef.current.onError?.(error as string);
-      }
-    }
-  }, [handleParticipantReady, overrides, customLlmExtraBody, dynamicVariables, userId, callbacksRef]);
-
-  // Create setClientTools function that only updates ref
-  const setClientTools = React.useCallback((tools: ClientToolsConfig['clientTools']) => {
-    clientToolsRef.current = tools;
-  }, []);
-
-  // Memoize inline callback functions
-  const sendContextualUpdate = React.useCallback((text: string) => {
-    sendMessage({
-      type: "contextual_update",
-      text,
-    });
-  }, [sendMessage]);
-
-  const sendUserMessage = React.useCallback((text: string) => {
-    sendMessage({
-      type: "user_message",
-      text,
-    });
-  }, [sendMessage]);
-
-  const sendUserActivity = React.useCallback(() => {
-    sendMessage({
-      type: "user_activity",
-    });
-  }, [sendMessage]);
-
-  const sendMultimodalMessage = React.useCallback((options: { text?: string; fileId?: string }) => {
-    sendMessage({
-      type: "multimodal_message",
-      text: options.text
-        ? { type: "user_message" as const, text: options.text }
-        : undefined,
-      file: options.fileId
-        ? { type: "file_input" as const, file_id: options.fileId }
-        : undefined,
-    });
-  }, [sendMessage]);
-
-  // Store all conversation values/functions in refs for stable access
-  const conversationValuesRef = React.useRef({
-    startSession,
-    endSession,
-    status,
-    isSpeaking,
-    canSendFeedback,
-    getId,
-    setMicMuted,
-    sendFeedback,
-    sendContextualUpdate,
-    sendUserMessage,
-    sendUserActivity,
-    sendMultimodalMessage,
-  });
-
-  // Update ref on every render
-  conversationValuesRef.current = {
-    startSession,
-    endSession,
-    status,
-    isSpeaking,
-    canSendFeedback,
-    getId,
-    setMicMuted,
-    sendFeedback,
-    sendContextualUpdate,
-    sendUserMessage,
-    sendUserActivity,
-    sendMultimodalMessage,
-  };
-
-  // Create a stable conversation object that never changes reference
-  // This prevents infinite loops when conversation is used in useEffect dependencies
-  const conversation = React.useMemo<Conversation>(() => ({
-    get startSession() { return conversationValuesRef.current.startSession; },
-    get endSession() { return conversationValuesRef.current.endSession; },
-    get status() { return conversationValuesRef.current.status; },
-    get isSpeaking() { return conversationValuesRef.current.isSpeaking; },
-    get canSendFeedback() { return conversationValuesRef.current.canSendFeedback; },
-    get getId() { return conversationValuesRef.current.getId; },
-    get setMicMuted() { return conversationValuesRef.current.setMicMuted; },
-    get sendFeedback() { return conversationValuesRef.current.sendFeedback; },
-    get sendContextualUpdate() { return conversationValuesRef.current.sendContextualUpdate; },
-    get sendUserMessage() { return conversationValuesRef.current.sendUserMessage; },
-    get sendUserActivity() { return conversationValuesRef.current.sendUserActivity; },
-    get sendMultimodalMessage() { return conversationValuesRef.current.sendMultimodalMessage; },
-  }), []); // Empty deps - object is created once and never recreated
-
-  // Memoize the context value to prevent unnecessary re-renders of consumers
-  const contextValue = React.useMemo<ElevenLabsContextType>(() => ({
-    conversation,
-    callbacksRef,
-    serverUrl,
-    tokenFetchUrl,
-    clientTools: clientToolsRef.current,
-    setCallbacks,
-    setServerUrl,
-    setTokenFetchUrl,
-    setClientTools,
-  }), [
-    conversation,
-    callbacksRef,
-    serverUrl,
-    tokenFetchUrl,
-    setCallbacks,
-    setClientTools,
-  ]);
-
-  return (
-    <ElevenLabsContext.Provider value={contextValue}>
-      <LiveKitRoomWrapper
-        serverUrl={serverUrl}
-        token={token}
-        connect={connect}
-        onConnected={handleConnectedWithFeedback}
-        onDisconnected={handleDisconnectedWithFeedback}
-        onError={handleError}
-        roomConnected={roomConnected}
-        callbacks={callbacksRef.current}
-        onParticipantReady={handleParticipantReadyWithOverrides}
-        sendMessage={sendMessage}
-        clientTools={clientToolsRef.current}
-        updateCurrentEventId={updateCurrentEventId}
-        onEndSession={endSession}
-        audioSessionConfig={audioSessionConfig}
-        textOnly={textOnly}
-      >
-        {children}
-      </LiveKitRoomWrapper>
-    </ElevenLabsContext.Provider>
-  );
-};

package/src/components/LiveKitRoomWrapper.tsx

@@ -1,94 +0,0 @@
-// @ts-nocheck - pnpm hoisting causes duplicate React type definitions
-import React from 'react';
-import { LiveKitRoom } from '@livekit/react-native';
-import type { LocalParticipant } from 'livekit-client';
-import type { Callbacks, ClientToolsConfig, AudioSessionConfig } from '../types';
-import { MessageHandler } from './MessageHandler';
-
-interface LiveKitRoomWrapperProps {
-  children: React.ReactNode;
-  serverUrl: string;
-  token: string;
-  connect: boolean;
-  onConnected: () => void;
-  onDisconnected: () => void;
-  onError: (error: Error) => void;
-  roomConnected: boolean;
-  callbacks: Callbacks;
-  onParticipantReady: (participant: LocalParticipant) => void;
-  sendMessage: (message: unknown) => void;
-  clientTools: ClientToolsConfig['clientTools'];
-  onEndSession: (reason?: "user" | "agent") => void;
-  updateCurrentEventId?: (eventId: number) => void;
-  audioSessionConfig?: AudioSessionConfig;
-  textOnly?: boolean;
-}
-
-export const LiveKitRoomWrapper = ({
-  children,
-  serverUrl,
-  token,
-  connect,
-  onConnected,
-  onDisconnected,
-  onError,
-  roomConnected,
-  callbacks,
-  onParticipantReady,
-  sendMessage,
-  clientTools,
-  updateCurrentEventId,
-  onEndSession,
-  audioSessionConfig,
-  textOnly = false,
-}: LiveKitRoomWrapperProps) => {
-  // Configure audio options based on audioSessionConfig
-  const audioOptions = React.useMemo(() => {
-    if (textOnly) {
-      // For now, we have to enable an audio session (even for text only conversations) to be able to send messages.
-      return true;
-    }
-
-    if (!audioSessionConfig?.allowMixingWithOthers) {
-      return true;
-    }
-
-    // When mixing is enabled, configure audio to allow concurrent playback
-    return {
-      audio: {
-        noiseSuppression: true,
-        echoCancellation: true,
-      },
-      audioSessionConfiguration: {
-        allowMixingWithOthers: true,
-      },
-    };
-  }, [audioSessionConfig, textOnly]);
-
-  return (
-    <LiveKitRoom
-      serverUrl={serverUrl}
-      token={token}
-      connect={connect}
-      audio={audioOptions}
-      video={false}
-      options={{
-        adaptiveStream: { pixelDensity: 'screen' },
-      }}
-      onConnected={onConnected}
-      onDisconnected={onDisconnected}
-      onError={onError}
-    >
-      <MessageHandler
-        onReady={onParticipantReady}
-        isConnected={roomConnected}
-        callbacks={callbacks}
-        sendMessage={sendMessage}
-        clientTools={clientTools}
-        updateCurrentEventId={updateCurrentEventId}
-        onEndSession={onEndSession}
-      />
-      {children as any}
-    </LiveKitRoom>
-  );
-};

package/src/components/MessageHandler.tsx

@@ -1,237 +0,0 @@
-import { useEffect } from "react";
-import { useLocalParticipant, useDataChannel, useRoomContext } from "@livekit/react-native";
-import { RoomEvent } from "livekit-client";
-import type { LocalParticipant, RemoteParticipant } from "livekit-client";
-import type {
-  Callbacks,
-  ClientToolsConfig,
-  ClientToolCallEvent,
-  ConversationEvent,
-  AudioEventWithAlignment,
-} from "../types";
-import React from "react";
-
-interface MessageHandlerProps {
-  onReady: (participant: LocalParticipant) => void;
-  isConnected: boolean;
-  callbacks: Callbacks;
-  sendMessage: (message: unknown) => void;
-  onEndSession: (reason: "user" | "agent") => void;
-  clientTools?: ClientToolsConfig["clientTools"];
-  updateCurrentEventId?: (eventId: number) => void;
-}
-
-export function isValidEvent(event: unknown): event is ConversationEvent {
-  return typeof event === "object" && event !== null && "type" in event;
-}
-
-function extractMessageText(event: ConversationEvent): string | null {
-  switch (event.type) {
-    case "user_transcript":
-      return event.user_transcription_event.user_transcript;
-    case "agent_response":
-      return event.agent_response_event.agent_response;
-    default:
-      return null;
-  }
-}
-
-export const MessageHandler = ({
-  onReady,
-  isConnected,
-  callbacks,
-  sendMessage,
-  clientTools = {},
-  updateCurrentEventId,
-  onEndSession,
-}: MessageHandlerProps) => {
-  const { localParticipant } = useLocalParticipant();
-  const room = useRoomContext();
-
-  // Track agent response count for synthetic event IDs (WebRTC mode)
-  const agentResponseCountRef = React.useRef(1);
-
-  // Refs for callbacks to avoid unnecessary effect re-runs
-  const onEndSessionRef = React.useRef(onEndSession);
-  onEndSessionRef.current = onEndSession;
-  const onReadyRef = React.useRef(onReady);
-  onReadyRef.current = onReady;
-  const callbacksRef = React.useRef(callbacks);
-  callbacksRef.current = callbacks;
-
-  // Detect agent disconnection
-  useEffect(() => {
-    const handleParticipantDisconnected = (participant: RemoteParticipant) => {
-      if (participant.identity?.startsWith("agent")) {
-        onEndSessionRef.current("agent");
-      }
-    };
-
-    room.on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected);
-
-    return () => {
-      room.off(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected);
-    };
-  }, [room]);
-
-  // Reset agent response count when connection status changes
-  useEffect(() => {
-    if (!isConnected) {
-      agentResponseCountRef.current = 1;
-    }
-  }, [isConnected]);
-
-  useEffect(() => {
-    if (isConnected && localParticipant) {
-      onReadyRef.current(localParticipant);
-    }
-  }, [isConnected, localParticipant]);
-
-  const handleClientToolCall = async (clientToolCall: ClientToolCallEvent) => {
-    if (clientToolCall.client_tool_call.tool_name in clientTools) {
-      try {
-        const result =
-          (await clientTools[clientToolCall.client_tool_call.tool_name](
-            clientToolCall.client_tool_call.parameters
-          )) ?? "Client tool execution successful."; // default client-tool call response
-
-        // The API expects result to be a string, so we need to convert it if it's not already a string
-        const formattedResult =
-          typeof result === "object" ? JSON.stringify(result) : String(result);
-
-        sendMessage({
-          type: "client_tool_result",
-          tool_call_id: clientToolCall.client_tool_call.tool_call_id,
-          result: formattedResult,
-          is_error: false,
-        });
-      } catch (e) {
-        const errorMessage = `Client tool execution failed with following error: ${(e as Error)?.message}`;
-        callbacksRef.current.onError?.(errorMessage, {
-          clientToolName: clientToolCall.client_tool_call.tool_name,
-        });
-        sendMessage({
-          type: "client_tool_result",
-          tool_call_id: clientToolCall.client_tool_call.tool_call_id,
-          result: `Client tool execution failed: ${(e as Error)?.message}`,
-          is_error: true,
-        });
-      }
-    } else {
-      if (callbacksRef.current.onUnhandledClientToolCall) {
-        callbacksRef.current.onUnhandledClientToolCall(clientToolCall.client_tool_call);
-        return;
-      }
-
-      const errorMessage = `Client tool with name ${clientToolCall.client_tool_call.tool_name} is not defined on client`;
-      callbacksRef.current.onError?.(errorMessage, {
-        clientToolName: clientToolCall.client_tool_call.tool_name,
-      });
-      sendMessage({
-        type: "client_tool_result",
-        tool_call_id: clientToolCall.client_tool_call.tool_call_id,
-        result: errorMessage,
-        is_error: true,
-      });
-    }
-  };
-
-  const _ = useDataChannel(msg => {
-    const decoder = new TextDecoder();
-    const message = JSON.parse(decoder.decode(msg.payload));
-
-    if (!isValidEvent(message)) {
-      callbacksRef.current.onDebug?.({
-        type: "invalid_event",
-        message,
-      });
-      return;
-    }
-
-    const messageText = extractMessageText(message);
-    if (messageText !== null) {
-      callbacksRef.current.onMessage?.({
-        message: messageText,
-        source: message.type === "user_transcript" ? "user" : "ai",
-        role: message.type === "user_transcript" ? "user" : "agent",
-      });
-    }
-
-    if (msg.from?.isAgent) {
-      callbacksRef.current.onModeChange?.({
-        mode: msg.from?.isSpeaking ? "speaking" : "listening",
-      });
-
-      // Track agent responses for feedback (WebRTC mode needs synthetic event IDs)
-      if (message.type === "agent_response" && updateCurrentEventId) {
-        const eventId = agentResponseCountRef.current++;
-        updateCurrentEventId(eventId);
-      }
-    }
-
-    switch (message.type) {
-      case "ping":
-        sendMessage({
-          type: "pong",
-          event_id: message.ping_event.event_id,
-        });
-        break;
-      case "client_tool_call":
-        handleClientToolCall(message);
-        break;
-      case "audio": {
-        const audioEvent = message.audio_event as AudioEventWithAlignment;
-        if (audioEvent.audio_base_64) {
-          callbacksRef.current.onAudio?.(audioEvent.audio_base_64);
-        }
-        if (audioEvent.alignment) {
-          callbacksRef.current.onAudioAlignment?.(audioEvent.alignment);
-        }
-        break;
-      }
-      case "vad_score":
-        callbacksRef.current.onVadScore?.({
-          vadScore: message.vad_score_event.vad_score,
-        });
-        break;
-      case "interruption":
-        callbacksRef.current.onInterruption?.(message.interruption_event);
-        break;
-      case "mcp_tool_call":
-        callbacksRef.current.onMCPToolCall?.(message.mcp_tool_call);
-        break;
-      case "mcp_connection_status":
-        callbacksRef.current.onMCPConnectionStatus?.(message.mcp_connection_status);
-        break;
-      case "agent_tool_request":
-        callbacksRef.current.onAgentToolRequest?.(message.agent_tool_request);
-        break;
-      case "agent_tool_response":
-        callbacksRef.current.onAgentToolResponse?.(message.agent_tool_response);
-
-        if (message.agent_tool_response.tool_name === "end_call") {
-          // End the call
-          onEndSessionRef.current("agent");
-        }
-        break;
-      case "conversation_initiation_metadata":
-        callbacksRef.current.onConversationMetadata?.(
-          message.conversation_initiation_metadata_event
-        );
-        break;
-      case "asr_initiation_metadata":
-        callbacksRef.current.onAsrInitiationMetadata?.(
-          message.asr_initiation_metadata_event
-        );
-        break;
-      case "agent_chat_response_part":
-        callbacksRef.current.onAgentChatResponsePart?.(message.text_response_part);
-        break;
-      default:
-        callbacksRef.current.onDebug?.(message);
-        break;
-    }
-  });
-
-  return null;
-};