@mobileai/react-native 0.4.5 → 0.5.0

package/lib/module/services/VoiceService.js

@@ -1,81 +1,151 @@
 "use strict";
 
 /**
- * VoiceService — WebSocket connection to Gemini Live API.
+ * VoiceService — @google/genai SDK Live API connection.
+ *
+ * Uses the official `ai.live.connect()` method instead of raw WebSocket.
+ * This fixes function calling reliability: the SDK handles protocol details
+ * (binary framing, message transforms, model name prefixes) that our
+ * previous raw WebSocket implementation missed.
  *
  * Handles bidirectional audio streaming between the app and Gemini:
  * - Sends PCM 16kHz 16-bit audio chunks (mic input)
  * - Receives PCM 24kHz 16-bit audio chunks (AI responses)
  * - Receives function calls (tap, navigate, etc.) for agentic actions
- * - Sends screen context (DOM text + optional screenshot) for live mode
- *
- * Protocol: wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent
+ * - Sends screen context (DOM text) for live mode
  */
 
+// Platform-specific import: Metro can't resolve '@google/genai/web' sub-path
+// export, so we use the full path to the web bundle. This is what the SDK
+// recommends ('use a platform specific import') — RN's WebSocket API is
+// browser-compatible so the web bundle works correctly.
+// @ts-ignore — TS can't find declarations for the deep path
+import { GoogleGenAI, Modality } from '@google/genai/dist/web/index.mjs';
+// @ts-ignore
+
 import { logger } from "../utils/logger.js";
 
 // ─── Types ─────────────────────────────────────────────────────
 
 // ─── Constants ─────────────────────────────────────────────────
 
-const WS_HOST = 'generativelanguage.googleapis.com';
-const WS_PATH = '/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
 const DEFAULT_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025';
 const DEFAULT_INPUT_SAMPLE_RATE = 16000;
 
 // ─── Service ───────────────────────────────────────────────────
 
 export class VoiceService {
-  ws = null;
+  session = null;
   callbacks = {};
-  setupComplete = false;
+  lastCallbacks = null;
   _status = 'disconnected';
+  intentionalDisconnect = false;
   constructor(config) {
     this.config = config;
   }
 
   // ─── Connection ────────────────────────────────────────────
 
-  connect(callbacks) {
-    if (this.ws?.readyState === WebSocket.OPEN) {
+  /**
+   * Connect to Gemini Live API via the official SDK.
+   * Now async because `ai.live.connect()` returns a Promise.
+   */
+  async connect(callbacks) {
+    if (this.session) {
       logger.info('VoiceService', 'Already connected');
       return;
     }
     this.callbacks = callbacks;
+    this.lastCallbacks = callbacks;
     this.setStatus('connecting');
+    this.intentionalDisconnect = false;
     const model = this.config.model || DEFAULT_MODEL;
-    const url = `wss://${WS_HOST}${WS_PATH}?key=${this.config.apiKey}`;
-    logger.info('VoiceService', `Connecting to Gemini Live API (model: ${model})`);
-    this.ws = new WebSocket(url);
-    this.ws.onopen = () => {
-      logger.info('VoiceService', 'WebSocket connected, sending setup...');
-      this.sendSetup();
-    };
-    this.ws.onclose = event => {
-      logger.info('VoiceService', `WebSocket closed: ${event.code} ${event.reason}`);
-      this.setStatus('disconnected');
-      this.setupComplete = false;
-    };
-    this.ws.onerror = error => {
-      logger.error('VoiceService', `WebSocket error: ${error.message || 'Unknown'}`);
+    logger.info('VoiceService', `Connecting via SDK (model: ${model})`);
+    try {
+      const ai = new GoogleGenAI({
+        apiKey: this.config.apiKey
+      });
+      const toolDeclarations = this.buildToolDeclarations();
+
+      // Build SDK config matching the official docs pattern
+      const sdkConfig = {
+        responseModalities: [Modality.AUDIO]
+      };
+
+      // Enable transcription for debugging and UX
+      sdkConfig.inputAudioTranscription = {};
+      sdkConfig.outputAudioTranscription = {};
+      logger.info('VoiceService', 'Transcription enabled');
+      if (this.config.systemPrompt) {
+        sdkConfig.systemInstruction = {
+          parts: [{
+            text: this.config.systemPrompt
+          }]
+        };
+      }
+      if (toolDeclarations.length > 0) {
+        sdkConfig.tools = [{
+          functionDeclarations: toolDeclarations
+        }];
+      }
+
+      // FULL CONFIG DUMP — see exactly what we send to SDK
+      const configDump = JSON.stringify({
+        ...sdkConfig,
+        systemInstruction: sdkConfig.systemInstruction ? '(present)' : '(none)',
+        tools: sdkConfig.tools ? `${toolDeclarations.length} declarations` : '(none)'
+      });
+      logger.info('VoiceService', `📋 SDK config: ${configDump}`);
+      logger.info('VoiceService', `📋 Tool names: ${toolDeclarations.map(t => t.name).join(', ')}`);
+      const session = await ai.live.connect({
+        model: model,
+        config: sdkConfig,
+        callbacks: {
+          onopen: () => {
+            logger.info('VoiceService', '✅ SDK session connected');
+            this.setStatus('connected');
+          },
+          onmessage: message => {
+            this.handleSDKMessage(message);
+          },
+          onerror: error => {
+            const errDetail = error ? JSON.stringify(error, Object.getOwnPropertyNames(error)).substring(0, 500) : 'null';
+            logger.error('VoiceService', `SDK error: ${errDetail}`);
+            this.setStatus('error');
+            this.callbacks.onError?.(error?.message || 'SDK connection error');
+          },
+          onclose: event => {
+            const closeDetail = event ? JSON.stringify(event, Object.getOwnPropertyNames(event)).substring(0, 500) : 'null';
+            if (this.intentionalDisconnect) {
+              logger.info('VoiceService', `SDK session closed (intentional)`);
+            } else {
+              logger.error('VoiceService', `SDK session closed UNEXPECTEDLY — code: ${event?.code}, reason: ${event?.reason}, detail: ${closeDetail}`);
+              this.callbacks.onError?.(`Connection lost (code: ${event?.code || 'unknown'})`);
+            }
+            this.session = null;
+            this.setStatus('disconnected');
+          }
+        }
+      });
+      this.session = session;
+      logger.info('VoiceService', 'SDK session established');
+    } catch (error) {
+      logger.error('VoiceService', `Connection failed: ${error.message}`);
       this.setStatus('error');
-      this.callbacks.onError?.(error.message || 'WebSocket connection error');
-    };
-    this.ws.onmessage = event => {
-      this.handleMessage(event);
-    };
+      this.callbacks.onError?.(error.message || 'Failed to connect');
+    }
   }
   disconnect() {
-    if (this.ws) {
-      logger.info('VoiceService', 'Disconnecting...');
-      this.ws.close();
-      this.ws = null;
-      this.setupComplete = false;
+    if (this.session) {
+      logger.info('VoiceService', 'Disconnecting (intentional)...');
+      this.intentionalDisconnect = true;
+      this.session.close();
+      this.session = null;
       this.setStatus('disconnected');
     }
   }
   get isConnected() {
-    return this.ws?.readyState === WebSocket.OPEN && this.setupComplete;
+    return this.session !== null && this._status === 'connected';
   }
   get currentStatus() {
     return this._status;
@@ -83,263 +153,246 @@ export class VoiceService {
 
   // ─── Send Audio ────────────────────────────────────────────
 
-  /** Send PCM audio chunk (base64 encoded) to Gemini */
+  /** Send PCM audio chunk (base64 encoded) via SDK's sendRealtimeInput */
   sendCount = 0;
   sendAudio(base64Audio) {
     this.sendCount++;
-    if (!this.isConnected) {
-      logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected (ws=${this.ws?.readyState}, setup=${this.setupComplete})`);
+    if (!this.isConnected || !this.session) {
+      if (this.sendCount % 20 === 0) {
+        logger.warn('VoiceService', `sendAudio #${this.sendCount} DROPPED — not connected`);
+      }
       return;
     }
-    const message = {
-      realtimeInput: {
+    const mimeType = `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`;
+
+    // DEBUG: log every send call
+    if (this.sendCount <= 5 || this.sendCount % 10 === 0) {
+      logger.info('VoiceService', `📡 sendAudio #${this.sendCount}: len=${base64Audio.length}, mime=${mimeType}, preview=${base64Audio.substring(0, 30)}...`);
+    }
+    try {
+      this.session.sendRealtimeInput({
         audio: {
-          mimeType: `audio/pcm;rate=${this.config.inputSampleRate || DEFAULT_INPUT_SAMPLE_RATE}`,
-          data: base64Audio
+          data: base64Audio,
+          mimeType
         }
+      });
+      // Log every 50th successful send to confirm data is reaching WebSocket
+      if (this.sendCount % 50 === 0) {
+        logger.info('VoiceService', `✅ sendAudio #${this.sendCount} OK — session.isOpen=${!!this.session}`);
       }
-    };
-    logger.info('VoiceService', `📤 #${this.sendCount} sending ${base64Audio.length} chars (ws=${this.ws?.readyState})`);
-    this.ws.send(JSON.stringify(message));
+    } catch (error) {
+      logger.error('VoiceService', `❌ sendAudio EXCEPTION: ${error.message}\n${error.stack?.substring(0, 300)}`);
+      this.session = null;
+      this.setStatus('disconnected');
+    }
   }
 
   // ─── Send Text ─────────────────────────────────────────────
 
-  /** Send text message via realtimeInput (same channel as audio) */
+  /** Send text message via SDK's sendClientContent */
   sendText(text) {
-    if (!this.isConnected) return;
-    const message = {
-      realtimeInput: {
-        text
-      }
-    };
-    this.ws.send(JSON.stringify(message));
+    if (!this.isConnected || !this.session) return;
+    logger.info('VoiceService', `🗣️ USER (text): "${text}"`);
+    try {
+      this.session.sendClientContent({
+        turns: [{
+          role: 'user',
+          parts: [{
+            text
+          }]
+        }],
+        turnComplete: true
+      });
+    } catch (error) {
+      logger.error('VoiceService', `sendText failed: ${error.message}`);
+    }
   }
 
-  /** Send DOM tree as passive context during live conversation.
-   *
-   * Uses `clientContent` with `turnComplete: false` to inject context
-   * WITHOUT triggering a model response. This is the "incremental content
-   * updates" pattern from the Gemini docs for establishing session context.
-   *
-   * Called once at connect + after each tool call (not on a timer).
-   * Screenshots are handled separately via the capture_screenshot tool.
+  /**
+   * Send DOM tree as passive context during live conversation.
+   * Uses turnComplete: false — the model receives context without responding.
    */
   sendScreenContext(domText) {
-    if (!this.isConnected) return;
-    const message = {
-      clientContent: {
+    if (!this.isConnected || !this.session) return;
+    try {
+      this.session.sendClientContent({
         turns: [{
           role: 'user',
           parts: [{
             text: domText
           }]
         }],
-        turnComplete: false // Passive context — don't trigger a response
-      }
-    };
-    this.ws.send(JSON.stringify(message));
-    logger.debug('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
+        turnComplete: true
+      });
+      logger.info('VoiceService', `📤 Screen context sent (${domText.length} chars)`);
+    } catch (error) {
+      logger.error('VoiceService', `sendScreenContext failed: ${error.message}`);
+    }
   }
 
   // ─── Send Function Response ────────────────────────────────
 
-  /** Send function call result back to Gemini */
+  /** Send function call result back via SDK's sendToolResponse */
   sendFunctionResponse(name, id, result) {
-    if (!this.isConnected) return;
-    const message = {
-      toolResponse: {
+    if (!this.isConnected || !this.session) return;
+    logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
+    try {
+      this.session.sendToolResponse({
         functionResponses: [{
           name,
           id,
           response: result
         }]
-      }
-    };
-    logger.info('VoiceService', `📤 Sending tool response for ${name} (id=${id})`);
-    this.ws.send(JSON.stringify(message));
+      });
+    } catch (error) {
+      logger.error('VoiceService', `sendFunctionResponse failed: ${error.message}`);
+    }
   }
 
-  // ─── Internal: Setup ───────────────────────────────────────
+  // ─── Internal: Tool Declarations ───────────────────────────
 
-  sendSetup() {
-    if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
-    const model = this.config.model || DEFAULT_MODEL;
-    const setup = {
-      model: `models/${model}`,
-      generationConfig: {
-        responseModalities: ['AUDIO']
-      }
-    };
-
-    // For native audio models, language is enforced via system instructions.
-    // Explicitly setting speechConfig.languageCode causes silent API failures 
-    // without a prebuiltVoiceConfig.
-
-    // Add system instruction if provided
-    if (this.config.systemPrompt) {
-      setup.systemInstruction = {
-        parts: [{
-          text: this.config.systemPrompt
-        }]
+  /**
+   * Builds function declarations from configured tools.
+   * Converts BOOLEAN params to STRING (native audio model limitation).
+   */
+  buildToolDeclarations() {
+    if (!this.config.tools?.length) return [];
+    const validTools = this.config.tools.filter(t => t.name !== 'capture_screenshot');
+    if (validTools.length === 0) return [];
+    return validTools.map(tool => {
+      const hasParams = Object.keys(tool.parameters || {}).length > 0;
+      const functionDecl = {
+        name: tool.name,
+        description: tool.description
       };
-    }
-
-    // Add tool declarations for function calling
-    if (this.config.tools?.length) {
-      setup.tools = [{
-        functionDeclarations: this.config.tools.map(tool => ({
-          name: tool.name,
-          description: tool.description,
-          parameters: {
-            type: 'OBJECT',
-            properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => [key, {
-              type: param.type.toUpperCase(),
-              description: param.description
-            }])),
-            required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
-          }
-        }))
-      }];
-    }
-
-    // Enable transcription
-    setup.inputAudioTranscription = {};
-    setup.outputAudioTranscription = {};
-    const setupMessage = {
-      setup
-    };
-    logger.info('VoiceService', `Sending setup (model: ${model}, tools: ${this.config.tools?.length || 0})`);
-    this.ws.send(JSON.stringify(setupMessage));
+      if (hasParams) {
+        functionDecl.parameters = {
+          type: 'OBJECT',
+          properties: Object.fromEntries(Object.entries(tool.parameters).map(([key, param]) => {
+            let paramType = param.type.toUpperCase();
+            let desc = param.description;
+            if (paramType === 'BOOLEAN') {
+              paramType = 'STRING';
+              desc = `${desc} (use "true" or "false")`;
+            }
+            return [key, {
+              type: paramType,
+              description: desc
+            }];
+          })),
+          required: Object.entries(tool.parameters).filter(([, param]) => param.required).map(([key]) => key)
+        };
+      }
+      return functionDecl;
+    });
   }
 
   // ─── Internal: Message Handling ────────────────────────────
 
-  handleMessage(event) {
+  /**
+   * Handle messages from the SDK's onmessage callback.
+   * The SDK parses binary/JSON automatically — we get clean objects.
+   *
+   * Per official docs, tool calls come at the top level as
+   * `response.toolCall.functionCalls`.
+   */
+  handleSDKMessage(message) {
     try {
-      const dataType = typeof event.data;
-      const dataLen = typeof event.data === 'string' ? event.data.length : event.data?.byteLength || 'unknown';
-      logger.info('VoiceService', `📥 WS message received: type=${dataType}, length=${dataLen}`);
-
-      // Handle binary data (could be JSON or raw PCM)
-      if (typeof event.data !== 'string') {
-        logger.info('VoiceService', '📥 Binary message — processing...');
-        this.handleBinaryMessage(event.data);
-        return;
+      // RAW MESSAGE DUMP — full session visibility
+      const msgKeys = Object.keys(message || {}).join(', ');
+      logger.info('VoiceService', `📨 SDK message keys: [${msgKeys}]`);
+
+      // Full raw dump for non-audio messages (audio is too large)
+      if (!message.serverContent?.modelTurn?.parts?.some(p => p.inlineData)) {
+        const rawDump = JSON.stringify(message).substring(0, 1000);
+        logger.info('VoiceService', `📨 RAW: ${rawDump}`);
       }
 
-      // Handle JSON text messages
-      const message = JSON.parse(event.data);
-      logger.info('VoiceService', `📥 JSON message keys: ${Object.keys(message).join(', ')}`);
-      this.processMessage(message);
-    } catch (error) {
-      logger.error('VoiceService', `Error handling message: ${error.message}`);
-    }
-  }
-  handleBinaryMessage(data) {
-    try {
-      // Try to decode as JSON first
-      let bytes;
-      if (data instanceof ArrayBuffer) {
-        bytes = new Uint8Array(data);
-      } else if (data instanceof Blob) {
-        // Blob handling — read as ArrayBuffer
-        const reader = new FileReader();
-        reader.onload = () => {
-          if (reader.result instanceof ArrayBuffer) {
-            this.processBinaryBytes(new Uint8Array(reader.result));
-          }
-        };
-        reader.readAsArrayBuffer(data);
-        return;
-      } else {
+      // Tool calls — top-level (per official docs)
+      if (message.toolCall?.functionCalls) {
+        this.handleToolCalls(message.toolCall.functionCalls);
         return;
       }
-      this.processBinaryBytes(bytes);
+
+      // Server content (audio, text, transcripts, turn events)
+      if (message.serverContent) {
+        this.handleServerContent(message.serverContent);
+      }
+
+      // Setup complete acknowledgment
+      if (message.setupComplete !== undefined) {
+        logger.info('VoiceService', '✅ Setup complete — ready for audio');
+        this.callbacks.onSetupComplete?.();
+      }
+
+      // Error messages
+      if (message.error) {
+        logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
+        this.callbacks.onError?.(message.error.message || 'Server error');
+      }
     } catch (error) {
-      logger.error('VoiceService', `Error handling binary message: ${error.message}`);
+      logger.error('VoiceService', `Error handling SDK message: ${error.message}`);
     }
   }
-  processBinaryBytes(bytes) {
-    // Check if it looks like JSON (starts with '{' or '[')
-    const looksLikeJson = bytes.length > 0 && (bytes[0] === 123 || bytes[0] === 91);
-    if (looksLikeJson) {
-      try {
-        const text = new TextDecoder('utf-8').decode(bytes);
-        const message = JSON.parse(text);
-        this.processMessage(message);
-      } catch {
-        // Not JSON — treat as raw PCM audio
-        this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
-      }
-    } else {
-      // Raw PCM audio data
-      this.callbacks.onAudioResponse?.(this.arrayBufferToBase64(bytes.buffer));
+
+  /** Process tool calls from the model */
+  handleToolCalls(functionCalls) {
+    for (const fn of functionCalls) {
+      logger.info('VoiceService', `🎯 Tool call: ${fn.name}(${JSON.stringify(fn.args)}) [id=${fn.id}]`);
+      this.callbacks.onToolCall?.({
+        name: fn.name,
+        args: fn.args || {},
+        id: fn.id
+      });
     }
   }
-  processMessage(message) {
-    // Setup complete acknowledgment
-    if (message.setupComplete !== undefined) {
-      logger.info('VoiceService', '✅ Setup complete — ready for audio exchange');
-      this.setupComplete = true;
-      this.setStatus('connected');
-      return;
+  audioResponseCount = 0;
+
+  /** Process server content (audio responses, transcripts, turn events) */
+  handleServerContent(content) {
+    // Log all keys for full visibility
+    const contentKeys = Object.keys(content || {}).join(', ');
+    logger.debug('VoiceService', `📦 serverContent keys: [${contentKeys}]`);
+
+    // Turn complete
+    if (content.turnComplete) {
+      logger.info('VoiceService', `🏁 Turn complete (audioChunks sent: ${this.audioResponseCount})`);
+      this.audioResponseCount = 0;
+      this.callbacks.onTurnComplete?.();
     }
 
-    // Server content (audio response + transcripts)
-    if (message.serverContent) {
-      const content = message.serverContent;
-      logger.info('VoiceService', `📥 serverContent received — turnComplete=${content.turnComplete}, hasParts=${!!content.modelTurn?.parts}, inputTranscription=${!!content.inputTranscription}, outputTranscription=${!!content.outputTranscription}`);
-
-      // Check for turn complete
-      if (content.turnComplete) {
-        this.callbacks.onTurnComplete?.();
-      }
-
-      // Process model output parts
-      if (content.modelTurn?.parts) {
-        for (const part of content.modelTurn.parts) {
-          // Audio response
-          if (part.inlineData?.data) {
-            logger.info('VoiceService', `🔊 Audio response: ${part.inlineData.data.length} chars`);
-            this.callbacks.onAudioResponse?.(part.inlineData.data);
-          }
-
-          // Text response (transcript)
-          if (part.text) {
-            logger.info('VoiceService', `💬 Text response: "${part.text}"`);
-            this.callbacks.onTranscript?.(part.text, true, 'model');
+    // Model output parts (audio + optional thinking text)
+    if (content.modelTurn?.parts) {
+      for (const part of content.modelTurn.parts) {
+        if (part.inlineData?.data) {
+          this.audioResponseCount++;
+          if (this.audioResponseCount <= 3 || this.audioResponseCount % 20 === 0) {
+            logger.info('VoiceService', `🔊 Audio chunk #${this.audioResponseCount}: ${part.inlineData.data.length} b64 chars, mime=${part.inlineData.mimeType || 'unknown'}`);
           }
+          this.callbacks.onAudioResponse?.(part.inlineData.data);
+        }
+        if (part.text) {
+          logger.info('VoiceService', `🤖 MODEL: "${part.text}"`);
+          this.callbacks.onTranscript?.(part.text, true, 'model');
         }
       }
+    }
 
-      // Input transcription (user's speech)
-      if (content.inputTranscription?.text) {
-        this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
-      }
-
-      // Output transcription (model's speech-to-text)
-      if (content.outputTranscription?.text) {
-        this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
-      }
+    // Input transcription (user's speech-to-text)
+    if (content.inputTranscription?.text) {
+      logger.info('VoiceService', `🗣️ USER (voice): "${content.inputTranscription.text}"`);
+      this.callbacks.onTranscript?.(content.inputTranscription.text, true, 'user');
     }
 
-    // Tool calls from the model
-    if (message.toolCall?.functionCalls) {
-      for (const fn of message.toolCall.functionCalls) {
-        logger.info('VoiceService', `Tool call: ${fn.name}(${JSON.stringify(fn.args)})`);
-        this.callbacks.onToolCall?.({
-          name: fn.name,
-          args: fn.args || {},
-          id: fn.id
-        });
-      }
+    // Output transcription (model's speech-to-text)
+    if (content.outputTranscription?.text) {
+      logger.info('VoiceService', `🤖 MODEL (voice): "${content.outputTranscription.text}"`);
+      this.callbacks.onTranscript?.(content.outputTranscription.text, true, 'model');
     }
 
-    // Error messages
-    if (message.error) {
-      logger.error('VoiceService', `Server error: ${JSON.stringify(message.error)}`);
-      this.callbacks.onError?.(message.error.message || 'Server error');
+    // Tool calls inside serverContent (some SDK versions deliver here)
+    if (content.toolCall?.functionCalls) {
+      this.handleToolCalls(content.toolCall.functionCalls);
     }
   }
 
@@ -349,13 +402,5 @@ export class VoiceService {
     this._status = newStatus;
     this.callbacks.onStatusChange?.(newStatus);
   }
-  arrayBufferToBase64(buffer) {
-    const bytes = new Uint8Array(buffer);
-    let binary = '';
-    for (let i = 0; i < bytes.byteLength; i++) {
-      binary += String.fromCharCode(bytes[i]);
-    }
-    return btoa(binary);
-  }
 }
 //# sourceMappingURL=VoiceService.js.map