@mobileai/react-native 0.4.1 → 0.4.3

package/src/components/AIAgent.tsx

@@ -22,11 +22,15 @@ import { AgentContext } from '../hooks/useAction';
 import { AgentChatBar } from './AgentChatBar';
 import { AgentOverlay } from './AgentOverlay';
 import { logger } from '../utils/logger';
+import { buildVoiceSystemPrompt } from '../core/systemPrompt';
 import { MCPBridge } from '../core/MCPBridge';
-import type { AgentConfig, ExecutionResult, ToolDefinition, AgentStep } from '../core/types';
+import { VoiceService } from '../services/VoiceService';
+import { AudioInputService } from '../services/AudioInputService';
+import { AudioOutputService } from '../services/AudioOutputService';
+import type { AgentConfig, AgentMode, ExecutionResult, ToolDefinition, AgentStep, TokenUsage } from '../core/types';
 
 // ─── Context ───────────────────────────────────────────────────
-
+console.log('🚀 AIAgent.tsx MODULE LOADED');
 
 // ─── Props ─────────────────────────────────────────────────────
 
@@ -83,6 +87,12 @@ interface AIAgentProps {
   };
   /** Expo Router pathname (from usePathname()) */
   pathname?: string;
+  /** Enable voice mode (requires expo-av) */
+  enableVoice?: boolean;
+  /** Called after each step with token usage data */
+  onTokenUsage?: (usage: TokenUsage) => void;
+  /** Enable SDK debug logging (disabled by default) */
+  debug?: boolean;
 }
 
 // ─── Component ─────────────────────────────────────────────────
@@ -110,12 +120,43 @@ export function AIAgent({
   mcpServerUrl,
   router,
   pathname,
+  enableVoice = false,
+  onTokenUsage,
+  debug = false,
 }: AIAgentProps) {
+  // Configure logger based on debug prop
+  React.useEffect(() => {
+    console.log('[AIAgent] DEBUG PROP =', debug, '— enabling logger');
+    logger.setEnabled(debug);
+    if (debug) {
+      logger.info('AIAgent', '🔧 Debug logging enabled');
+    }
+  }, [debug]);
+
   const rootViewRef = useRef<any>(null);
   const [isThinking, setIsThinking] = useState(false);
   const [statusText, setStatusText] = useState('');
   const [lastResult, setLastResult] = useState<ExecutionResult | null>(null);
 
+  // ─── Voice/Live Mode State ──────────────────────────────────
+  const [mode, setMode] = useState<AgentMode>('text');
+  const [isMicActive, setIsMicActive] = useState(false);
+  const [isSpeakerMuted, setIsSpeakerMuted] = useState(false);
+  const [isAISpeaking, setIsAISpeaking] = useState(false);
+  const [isVoiceConnected, setIsVoiceConnected] = useState(false);
+
+  const voiceServiceRef = useRef<VoiceService | null>(null);
+  const audioInputRef = useRef<AudioInputService | null>(null);
+  const audioOutputRef = useRef<AudioOutputService | null>(null);
+
+  // Compute available modes from props
+  const availableModes: AgentMode[] = useMemo(() => {
+    const modes: AgentMode[] = ['text'];
+    if (enableVoice) modes.push('voice');
+    logger.info('AIAgent', `Available modes: ${modes.join(', ')}`);
+    return modes;
+  }, [enableVoice]);
+
   // Ref-based resolver for ask_user — stays alive across renders
   const askUserResolverRef = useRef<((answer: string) => void) | null>(null);
 
@@ -140,6 +181,7 @@ export function AIAgent({
     router,
     pathname,
     onStatusUpdate: setStatusText,
+    onTokenUsage,
     // Page-agent pattern: block the agent loop until user responds
     onAskUser: (question: string) => {
       return new Promise<string>((resolve) => {
@@ -155,7 +197,7 @@ export function AIAgent({
     interactiveBlacklist, interactiveWhitelist,
     onBeforeStep, onAfterStep, onBeforeTask, onAfterTask,
     transformScreenContent, customTools, instructions, stepDelay,
-    mcpServerUrl, router, pathname,
+    mcpServerUrl, router, pathname, onTokenUsage,
   ]);
 
   const provider = useMemo(() => new GeminiProvider(apiKey, model), [apiKey, model]);
@@ -184,6 +226,144 @@ export function AIAgent({
     };
   }, [mcpServerUrl, runtime]);
 
+  // ─── Voice/Live Service Initialization ──────────────────────
+
+  // Initialize voice services when mode changes to voice or live
+  useEffect(() => {
+    if (mode === 'text') {
+      logger.info('AIAgent', 'Text mode — skipping voice service init');
+      return;
+    }
+
+    logger.info('AIAgent', `Mode changed to "${mode}" — initializing voice services...`);
+
+    // Create VoiceService with runtime's built-in tools (navigate, tap, type, done, etc.)
+    if (!voiceServiceRef.current) {
+      logger.info('AIAgent', 'Creating VoiceService...');
+      const runtimeTools = runtime.getTools();
+      logger.info('AIAgent', `Registering ${runtimeTools.length} tools with VoiceService: ${runtimeTools.map(t => t.name).join(', ')}`);
+      // Build the full voice system prompt (screen format + tool descriptions + guardrails)
+      // This gives voice mode the same screen understanding as text mode
+      const voicePrompt = buildVoiceSystemPrompt(language, instructions?.system);
+      voiceServiceRef.current = new VoiceService({
+        apiKey,
+        systemPrompt: voicePrompt,
+        tools: runtimeTools,
+        language,
+      });
+      logger.info('AIAgent', 'VoiceService created with full voice system prompt and tools');
+    }
+
+    // Create AudioOutputService if not exists
+    if (!audioOutputRef.current) {
+      logger.info('AIAgent', 'Creating AudioOutputService...');
+      audioOutputRef.current = new AudioOutputService({
+        onError: (err) => logger.error('AIAgent', `AudioOutput error: ${err}`),
+      });
+      audioOutputRef.current.initialize().then((ok) => {
+        logger.info('AIAgent', `AudioOutputService initialized: ${ok}`);
+      });
+    }
+
+    // Create AudioInputService if not exists
+    if (!audioInputRef.current) {
+      logger.info('AIAgent', 'Creating AudioInputService...');
+      audioInputRef.current = new AudioInputService({
+        // Default 16kHz — Gemini Live API input standard
+        onAudioChunk: (chunk) => {
+          logger.debug('AIAgent', `Mic chunk: ${chunk.length} chars`);
+          voiceServiceRef.current?.sendAudio(chunk);
+        },
+        onError: (err) => logger.error('AIAgent', `AudioInput error: ${err}`),
+        onPermissionDenied: () => logger.warn('AIAgent', 'Mic permission denied by user'),
+      });
+    }
+
+    // Connect VoiceService
+    logger.info('AIAgent', 'Connecting VoiceService...');
+    voiceServiceRef.current.connect({
+      onAudioResponse: (audio) => {
+        logger.info('AIAgent', `Received audio response (${audio.length} chars)`);
+        setIsAISpeaking(true);
+        audioOutputRef.current?.enqueue(audio);
+      },
+      onStatusChange: (status) => {
+        logger.info('AIAgent', `Voice status: ${status}`);
+        const connected = status === 'connected';
+        setIsVoiceConnected(connected);
+        if (connected) {
+          logger.info('AIAgent', '✅ VoiceService connected — auto-starting mic...');
+          // Auto-start mic streaming once WebSocket is ready
+          audioInputRef.current?.start().then((ok) => {
+            if (ok) {
+              setIsMicActive(true);
+              logger.info('AIAgent', '🎙️ Mic auto-started after connection');
+            }
+          });
+          // Send initial screen context (tree) so the model knows what's on screen
+          const initialContext = runtime.getScreenContext();
+          voiceServiceRef.current?.sendScreenContext(initialContext);
+          logger.info('AIAgent', '📡 Initial screen context sent to voice model');
+        }
+      },
+      onTranscript: (text, isFinal, role) => {
+        logger.info('AIAgent', `Transcript [${role}] (final=${isFinal}): "${text}"`);
+      },
+      onToolCall: async (toolCall) => {
+        logger.info('AIAgent', `Voice tool call: ${toolCall.name}(${JSON.stringify(toolCall.args)})`);
+        // Execute the tool via AgentRuntime and send result back to Gemini
+        const result = await runtime.executeTool(toolCall.name, toolCall.args);
+        logger.info('AIAgent', `Voice tool result: ${result}`);
+        voiceServiceRef.current?.sendFunctionResponse(toolCall.name, toolCall.id, { result });
+
+        // After tool execution, push updated screen context
+        // (the screen may have changed from tap/type/navigate)
+        const updatedContext = runtime.getScreenContext();
+        voiceServiceRef.current?.sendScreenContext(updatedContext);
+        logger.info('AIAgent', '📡 Updated screen context sent after tool call');
+      },
+      onError: (err) => {
+        logger.error('AIAgent', `VoiceService error: ${err}`);
+      },
+      onTurnComplete: () => {
+        logger.info('AIAgent', 'AI turn complete');
+        setIsAISpeaking(false);
+      },
+    });
+
+    // Cleanup on mode change back to text
+    return () => {
+      logger.info('AIAgent', `Cleaning up voice services (leaving "${mode}" mode)`);
+      voiceServiceRef.current?.disconnect();
+      voiceServiceRef.current = null; // Ensure fresh instance on next connect
+      audioInputRef.current?.stop();
+      setIsMicActive(false);
+      setIsAISpeaking(false);
+      setIsVoiceConnected(false);
+    };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [mode, apiKey, runtime, language, instructions]);
+
+  // ─── Stop Voice Session (full cleanup) ─────────────────────
+
+  const stopVoiceSession = useCallback(() => {
+    logger.info('AIAgent', '🛑 Stopping voice session (full cleanup)...');
+    // 1. Stop mic input
+    audioInputRef.current?.stop();
+    // 2. Stop audio output (clear queued chunks)
+    audioOutputRef.current?.stop();
+    // 3. Disconnect WebSocket
+    voiceServiceRef.current?.disconnect();
+    voiceServiceRef.current = null;
+    // 4. Reset state
+    setIsMicActive(false);
+    setIsAISpeaking(false);
+    setIsVoiceConnected(false);
+    // 5. Switch back to text mode (triggers cleanup effect naturally)
+    setMode('text');
+    logger.info('AIAgent', '🛑 Voice session fully stopped');
+  }, [runtime]);
+
   // ─── Execute ──────────────────────────────────────────────────
 
   const handleSend = useCallback(async (message: string) => {
@@ -249,12 +429,51 @@ export function AIAgent({
           lastResult={lastResult}
           language={language}
           onDismiss={() => setLastResult(null)}
+          availableModes={availableModes}
+          mode={mode}
+          onModeChange={(newMode) => {
+            logger.info('AIAgent', `Mode change: ${mode} → ${newMode}`);
+            setMode(newMode);
+          }}
+          isMicActive={isMicActive}
+          isSpeakerMuted={isSpeakerMuted}
+          isAISpeaking={isAISpeaking}
+          onStopSession={stopVoiceSession}
+          isVoiceConnected={isVoiceConnected}
+          onMicToggle={(active) => {
+            if (active && !isVoiceConnected) {
+              logger.warn('AIAgent', 'Cannot toggle mic — VoiceService not connected yet');
+              return;
+            }
+            logger.info('AIAgent', `Mic toggle: ${active ? 'ON' : 'OFF'}`);
+            setIsMicActive(active);
+            if (active) {
+              logger.info('AIAgent', 'Starting AudioInput...');
+              audioInputRef.current?.start().then((ok) => {
+                logger.info('AIAgent', `AudioInput start result: ${ok}`);
+              });
+            } else {
+              logger.info('AIAgent', 'Stopping AudioInput...');
+              audioInputRef.current?.stop();
+            }
+          }}
+          onSpeakerToggle={(muted) => {
+            logger.info('AIAgent', `Speaker toggle: ${muted ? 'MUTED' : 'UNMUTED'}`);
+            setIsSpeakerMuted(muted);
+            if (muted) {
+              audioOutputRef.current?.mute();
+            } else {
+              audioOutputRef.current?.unmute();
+            }
+          }}
+
         />
       )}
     </AgentContext.Provider>
   );
 }
 
+
 const styles = StyleSheet.create({
   root: {
     flex: 1,