@mobileai/react-native 0.4.2 → 0.4.3

package/lib/module/providers/GeminiProvider.js

@@ -1,83 +1,81 @@
 "use strict";
 
 /**
- * GeminiProvider — Gemini API integration with structured action pattern.
+ * GeminiProvider — Gemini API integration via @google/genai SDK.
  *
- * Uses a single forced function call (`agent_step`) that bundles
- * structured reasoning (evaluation, memory, plan) alongside the action.
- * This replaces free-form text + separate tool calls for stability.
+ * Uses the official Google GenAI SDK for:
+ * - generateContent with structured function calling (agent_step)
+ * - inlineData for vision (base64 screenshots)
+ * - System instructions
+ *
+ * Implements the AIProvider interface so it can be swapped
+ * with OpenAIProvider, AnthropicProvider, etc.
  */
 
+import { GoogleGenAI, FunctionCallingConfigMode, Type } from '@google/genai';
 import { logger } from "../utils/logger.js";
 // ─── Constants ─────────────────────────────────────────────────
 
 const AGENT_STEP_FN = 'agent_step';
 
-// Reasoning fields that are always present in the agent_step schema
+// Reasoning fields always present in the agent_step schema
 const REASONING_FIELDS = ['previous_goal_eval', 'memory', 'plan'];
 
-// ─── Gemini API Types ──────────────────────────────────────────
-
 // ─── Provider ──────────────────────────────────────────────────
 
 export class GeminiProvider {
   constructor(apiKey, model = 'gemini-2.5-flash') {
-    this.apiKey = apiKey;
+    this.ai = new GoogleGenAI({
+      apiKey
+    });
     this.model = model;
   }
-  async generateContent(systemPrompt, userMessage, tools, history) {
-    logger.info('GeminiProvider', `Sending request. Model: ${this.model}, Tools: ${tools.length}`);
+  async generateContent(systemPrompt, userMessage, tools, history, screenshot) {
+    logger.info('GeminiProvider', `Sending request. Model: ${this.model}, Tools: ${tools.length}${screenshot ? ', with screenshot' : ''}`);
 
     // Build single agent_step function declaration
     const agentStepDeclaration = this.buildAgentStepDeclaration(tools);
 
-    // Build conversation history with proper function call/response pairs
-    const contents = this.buildContents(userMessage, history);
-
-    // Make API request
-    const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`;
-    const body = {
-      contents,
-      tools: [{
-        functionDeclarations: [agentStepDeclaration]
-      }],
-      systemInstruction: {
-        parts: [{
-          text: systemPrompt
-        }]
-      },
-      // Force the model to always call agent_step
-      tool_config: {
-        function_calling_config: {
-          mode: 'ANY',
-          allowed_function_names: [AGENT_STEP_FN]
-        }
-      },
-      generationConfig: {
-        temperature: 0.2,
-        maxOutputTokens: 2048
-      }
-    };
+    // Build contents (user message + optional screenshot)
+    const contents = this.buildContents(userMessage, history, screenshot);
     const startTime = Date.now();
     try {
-      const response = await fetch(url, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json'
-        },
-        body: JSON.stringify(body)
+      const response = await this.ai.models.generateContent({
+        model: this.model,
+        contents,
+        config: {
+          systemInstruction: systemPrompt,
+          tools: [{
+            functionDeclarations: [agentStepDeclaration]
+          }],
+          toolConfig: {
+            functionCallingConfig: {
+              mode: FunctionCallingConfigMode.ANY,
+              allowedFunctionNames: [AGENT_STEP_FN]
+            }
+          },
+          temperature: 0.2,
+          maxOutputTokens: 2048
+        }
       });
       const elapsed = Date.now() - startTime;
       logger.info('GeminiProvider', `Response received in ${elapsed}ms`);
-      if (!response.ok) {
-        const errorText = await response.text();
-        logger.error('GeminiProvider', `API error ${response.status}: ${errorText}`);
-        throw new Error(`Gemini API error ${response.status}: ${errorText}`);
+
+      // Extract token usage from SDK response
+      const tokenUsage = this.extractTokenUsage(response);
+      if (tokenUsage) {
+        logger.info('GeminiProvider', `Tokens: ${tokenUsage.promptTokens} in / ${tokenUsage.completionTokens} out / $${tokenUsage.estimatedCostUSD.toFixed(6)}`);
       }
-      const data = await response.json();
-      return this.parseAgentStepResponse(data, tools);
+      const result = this.parseAgentStepResponse(response, tools);
+      result.tokenUsage = tokenUsage;
+      return result;
     } catch (error) {
       logger.error('GeminiProvider', 'Request failed:', error.message);
+
+      // Preserve HTTP error format for backward compatibility with tests
+      if (error.status) {
+        throw new Error(`Gemini API error ${error.status}: ${error.message}`);
+      }
       throw error;
     }
   }
@@ -99,7 +97,6 @@ export class GeminiProvider {
     const actionProperties = {};
     for (const tool of tools) {
       for (const [paramName, param] of Object.entries(tool.parameters)) {
-        // Skip if already added (shared field names like 'text', 'index')
         if (actionProperties[paramName]) continue;
         actionProperties[paramName] = {
           type: this.mapParamType(param.type),
@@ -120,28 +117,25 @@ export class GeminiProvider {
       name: AGENT_STEP_FN,
       description: `Execute one agent step. Choose an action and provide reasoning.\n\nAvailable actions:\n${toolDescriptions}`,
       parameters: {
-        type: 'OBJECT',
+        type: Type.OBJECT,
         properties: {
-          // ── Reasoning fields ──
           previous_goal_eval: {
-            type: 'STRING',
+            type: Type.STRING,
             description: 'One-sentence assessment of your last action. State success, failure, or uncertain. Skip on first step.'
           },
           memory: {
-            type: 'STRING',
+            type: Type.STRING,
             description: 'Key facts to remember for future steps: progress made, items found, counters, field values already collected.'
           },
           plan: {
-            type: 'STRING',
+            type: Type.STRING,
             description: 'Your immediate next goal — what action you will take and why.'
           },
-          // ── Action selection ──
           action_name: {
-            type: 'STRING',
+            type: Type.STRING,
             description: 'Which action to execute.',
             enum: toolNames
           },
-          // ── Action parameters (flat) ──
           ...actionProperties
         },
         required: ['plan', 'action_name']
@@ -151,48 +145,52 @@ export class GeminiProvider {
   mapParamType(type) {
     switch (type) {
       case 'number':
-        return 'NUMBER';
+        return Type.NUMBER;
       case 'integer':
-        return 'INTEGER';
+        return Type.INTEGER;
       case 'boolean':
-        return 'BOOLEAN';
+        return Type.BOOLEAN;
       case 'string':
       default:
-        return 'STRING';
+        return Type.STRING;
     }
   }
 
   // ─── Build Contents ────────────────────────────────────────
 
   /**
-   * Builds Gemini conversation contents.
-   * 
-   * Each step is a STATELESS single-turn request (matching page-agent's approach):
-   * - System prompt has general instructions
-   * - User message contains full context: task, history, screen state
-   * - Model responds with agent_step function call
-   * 
-   * History is embedded as text in assembleUserPrompt (via <agent_history>),
-   * NOT as functionCall/functionResponse pairs. This avoids Gemini's
-   * conversation format requirements and thought_signature complexity.
+   * Builds contents for the generateContent call.
+   * Single-turn: user message + optional screenshot as inlineData.
    */
-  buildContents(userMessage, _history) {
+  buildContents(userMessage, _history, screenshot) {
+    const parts = [{
+      text: userMessage
+    }];
+
+    // Append screenshot as inlineData for Gemini vision
+    if (screenshot) {
+      parts.push({
+        inlineData: {
+          mimeType: 'image/jpeg',
+          data: screenshot
+        }
+      });
+    }
     return [{
       role: 'user',
-      parts: [{
-        text: userMessage
-      }]
+      parts
     }];
   }
 
   // ─── Parse Response ────────────────────────────────────────
 
   /**
-   * Parses the Gemini response expecting a single agent_step function call.
-   * Extracts structured reasoning + action, and determines which tool to execute.
+   * Parses the SDK response expecting a single agent_step function call.
+   * Extracts structured reasoning + action.
    */
-  parseAgentStepResponse(data, tools) {
-    if (!data.candidates || data.candidates.length === 0) {
+  parseAgentStepResponse(response, tools) {
+    const candidates = response.candidates || [];
+    if (candidates.length === 0) {
       logger.warn('GeminiProvider', 'No candidates in response');
       return {
         toolCalls: [{
@@ -210,7 +208,7 @@ export class GeminiProvider {
         text: 'No response generated.'
       };
     }
-    const candidate = data.candidates[0];
+    const candidate = candidates[0];
     const parts = candidate.content?.parts || [];
 
     // Find the function call part
@@ -260,11 +258,9 @@ export class GeminiProvider {
       };
     }
 
-    // Build action args: everything except reasoning fields and action_name
+    // Build action args: extract only the params that belong to the matched tool
     const actionArgs = {};
     const reservedKeys = new Set([...REASONING_FIELDS, 'action_name']);
-
-    // Find the matching tool to know which params belong to it
     const matchedTool = tools.find(t => t.name === actionName);
     if (matchedTool) {
       for (const paramName of Object.keys(matchedTool.parameters)) {
@@ -273,7 +269,6 @@ export class GeminiProvider {
         }
       }
     } else {
-      // Custom/registered tool — grab all non-reserved fields
       for (const [key, value] of Object.entries(args)) {
         if (!reservedKeys.has(key)) {
           actionArgs[key] = value;
@@ -290,5 +285,33 @@ export class GeminiProvider {
       text: textPart?.text
     };
   }
+
+  // ─── Token Usage Extraction ─────────────────────────────────
+
+  /**
+   * Extracts token usage from SDK response and calculates estimated cost.
+   *
+   * Pricing (Gemini 2.5 Flash):
+   * - Input:  $0.30 / 1M tokens
+   * - Output: $2.50 / 1M tokens
+   */
+  extractTokenUsage(response) {
+    const meta = response?.usageMetadata;
+    if (!meta) return undefined;
+    const promptTokens = meta.promptTokenCount ?? 0;
+    const completionTokens = meta.candidatesTokenCount ?? 0;
+    const totalTokens = meta.totalTokenCount ?? promptTokens + completionTokens;
+
+    // Cost estimation based on Gemini 2.5 Flash pricing
+    const INPUT_COST_PER_M = 0.30;
+    const OUTPUT_COST_PER_M = 2.50;
+    const estimatedCostUSD = promptTokens / 1_000_000 * INPUT_COST_PER_M + completionTokens / 1_000_000 * OUTPUT_COST_PER_M;
+    return {
+      promptTokens,
+      completionTokens,
+      totalTokens,
+      estimatedCostUSD
+    };
+  }
 }
 //# sourceMappingURL=GeminiProvider.js.map

package/lib/module/providers/GeminiProvider.js.map

@@ -1 +1 @@
-{"version":3,"names":["logger","AGENT_STEP_FN","REASONING_FIELDS","GeminiProvider","constructor","apiKey","model","generateContent","systemPrompt","userMessage","tools","history","info","length","agentStepDeclaration","buildAgentStepDeclaration","contents","buildContents","url","body","functionDeclarations","systemInstruction","parts","text","tool_config","function_calling_config","mode","allowed_function_names","generationConfig","temperature","maxOutputTokens","startTime","Date","now","response","fetch","method","headers","JSON","stringify","elapsed","ok","errorText","error","status","Error","data","json","parseAgentStepResponse","message","toolNames","map","t","name","actionProperties","tool","paramName","param","Object","entries","parameters","type","mapParamType","description","enum","toolDescriptions","params","keys","join","properties","previous_goal_eval","memory","plan","action_name","required","_history","role","candidates","warn","toolCalls","args","success","reasoning","previousGoalEval","candidate","content","fnCallPart","find","p","functionCall","textPart","actionName","actionArgs","reservedKeys","Set","matchedTool","undefined","key","value","has"],"sourceRoot":"../../../src","sources":["providers/GeminiProvider.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;AAGxC;;AAEA,MAAMC,aAAa,GAAG,YAAY;;AAElC;AACA,MAAMC,gBAAgB,GAAG,CAAC,oBAAoB,EAAE,QAAQ,EAAE,MAAM,CAAU;;AAE1E;;AAWA;;AAEA,OAAO,MAAMC,cAAc,CAAuB;EAIhDC,WAAWA,CAACC,MAAc,EAAEC,KAAa,GAAG,kBAAkB,EAAE;IAC9D,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,KAAK,GAAGA,KAAK;EACpB;EAEA,MAAMC,eAAeA,CACnBC,YAAoB,EACpBC,WAAmB,EACnBC,KAAuB,EACvBC,OAAoB,EACK;IAEzBX,MAAM,CAACY,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,IAAI,CAACN,KAAK,YAAYI,KAAK,CAACG,MAAM,EAAE,CAAC;;IAE9F;IACA,MAAMC,oBAAoB,GAAG,IAAI,CAACC,yBAAyB,CAACL,KAAK,CAAC;;IAElE;IACA,MAAMM,QAAQ,GAAG,IAAI,CAACC,aAAa,CAACR,WAAW,EAAEE,OAAO,CAAC;;IAEzD;IACA,MAAMO,GAAG,GAAG,2DAA2D,IAAI,CAACZ,KAAK,wBAAwB,IAAI,CAACD,MAAM,EAAE;IAEtH,MAAMc,IAAS,GAAG;MAChBH,QAAQ;MACRN,KAAK,EAAE,CAAC;QAAEU,oBAAoB,EAAE,CAACN,oBAAoB;MAAE,CAAC,CAAC;MACzDO,iBAAiB,EAAE;QAAEC,KAAK,EAAE,CAAC;UAAEC,IAAI,EAAEf;QAAa,CAAC;MAAE,CAAC;MACtD;MACAgB,WAAW,EAAE;QACXC,uBAAuB,EAAE;UACvBC,IAAI,EAAE,KAAK;UACXC,sBAAsB,EAAE,CAAC1B,aAAa;QACxC;MACF,CAAC;MACD2B,gBAAgB,EAAE;QAChBC,WAAW,EAAE,GAAG;QAChBC,eAAe,EAAE;MACnB;IACF,CAAC;IAED,MAAMC,SAAS,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;IAE5B,IAAI;MACF,MAAMC,QAAQ,GAAG,MAAMC,KAAK,CAACjB,GAAG,EAAE;QAChCkB,MAAM,EAAE,MAAM;QACdC,OAAO,EAAE;UAAE,cAAc,EAAE;QAAmB,CAAC;QAC/ClB,IAAI,EAAEmB,IAAI,CAACC,SAAS,CAACpB,IAAI;MAC3B,CAAC,CAAC;MAEF,MAAMqB,OAAO,GAAGR,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,SAAS;MACtC/B,MAAM,CAACY,IAAI,CAAC,gBAAgB,EAAE,wBAAwB4B,OAAO,IAAI,CAAC;MAElE,IAAI,CAACN,QAAQ,CAACO,EAAE,EAAE;QAChB,MAAMC,SAAS,GAAG,MAAMR,QAAQ,CAACX,IAAI,CAAC,CAAC;QACvCvB,MAAM,CAAC2C,KAAK,CAAC,gBAAgB,EAAE,aAAaT,QAAQ,CAACU,MAAM,KAAKF,SAAS,EAAE,CAAC;QAC5E,MAAM,IAAIG,KAAK,CAAC,oBAAoBX,QAAQ,CAACU,MAAM,KAAKF,SAAS,EAAE,CAAC;MACtE;MAEA,MAAMI,IAAI,GAAG,MAAMZ,QAAQ,CAACa,IAAI,CAAC,CAAC;MAElC,OAAO,IAAI,CAACC,sBAAsB,CAACF,IAAI,EAAEpC,KAAK,CAAC;IACjD,CAAC,CAAC,OAAOiC,KAAU,EAAE;MACnB3C,MAAM,CAAC2C,KAAK,CAAC,gBAAgB,EAAE,iBAAiB,EAAEA,KAAK,CAACM,OAAO,CAAC;MAChE,MAAMN,KAAK;IACb;EACF;;EAEA;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACU5B,yBAAyBA,CAACL,KAAuB,EAAO;IAC9D,MAAMwC,SAAS,GAAGxC,KAAK,CAACyC,GAAG,CAACC,CAAC,IAAIA,CAAC,CAACC,IAAI,CAAC;;IAExC;IACA,MAAMC,gBAAqC,GAAG,CAAC,CAAC;IAChD,KAAK,MAAMC,IAAI,IAAI7C,KAAK,EAAE;MACxB,KAAK,MAAM,CAAC8C,SAAS,EAAEC,KAAK,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACJ,IAAI,CAACK,UAAU,CAAC,EAAE;QAChE;QACA,IAAIN,gBAAgB,CAACE,SAAS,CAAC,EAAE;QACjCF,gBAAgB,CAACE,SAAS,CAAC,GAAG;UAC5BK,IAAI,EAAE,IAAI,CAACC,YAAY,CAACL,KAAK,CAACI,IAAI,CAAC;UACnCE,WAAW,EAAEN,KAAK,CAACM,WAAW;UAC9B,IAAIN,KAAK,CAACO,IAAI,GAAG;YAAEA,IAAI,EAAEP,KAAK,CAACO;UAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;MACH;IACF;;IAEA;IACA,MAAMC,gBAAgB,GAAGvD,KAAK,CAC3ByC,GAAG,CAACC,CAAC,IAAI;MACR,MAAMc,MAAM,GAAGR,MAAM,CAACS,IAAI,CAACf,CAAC,CAACQ,UAAU,CAAC,CAACQ,IAAI,CAAC,IAAI,CAAC;MACnD,OAAO,KAAKhB,CAAC,CAACC,IAAI,IAAIa,MAAM,MAAMd,CAAC,CAACW,WAAW,EAAE;IACnD,CAAC,CAAC,CACDK,IAAI,CAAC,IAAI,CAAC;IAEb,OAAO;MACLf,IAAI,EAAEpD,aAAa;MACnB8D,WAAW,EAAE,0FAA0FE,gBAAgB,EAAE;MACzHL,UAAU,EAAE;QACVC,IAAI,EAAE,QAAQ;QACdQ,UAAU,EAAE;UACV;UACAC,kBAAkB,EAAE;YAClBT,IAAI,EAAE,QAAQ;YACdE,WAAW,EAAE;UACf,CAAC;UACDQ,MAAM,EAAE;YACNV,IAAI,EAAE,QAAQ;YACdE,WAAW,EAAE;UACf,CAAC;UACDS,IAAI,EAAE;YACJX,IAAI,EAAE,QAAQ;YACdE,WAAW,EAAE;UACf,CAAC;UACD;UACAU,WAAW,EAAE;YACXZ,IAAI,EAAE,QAAQ;YACdE,WAAW,EAAE,0BAA0B;YACvCC,IAAI,EAAEd;UACR,CAAC;UACD;UACA,GAAGI;QACL,CAAC;QACDoB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa;MAClC;IACF,CAAC;EACH;EAEQZ,YAAYA,CAACD,IAAY,EAAU;IACzC,QAAQA,IAAI;MACV,KAAK,QAAQ;QAAE,OAAO,QAAQ;MAC9B,KAAK,SAAS;QAAE,OAAO,SAAS;MAChC,KAAK,SAAS;QAAE,OAAO,SAAS;MAChC,KAAK,QAAQ;MACb;QAAS,OAAO,QAAQ;IAC1B;EACF;;EAEA;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;EACU5C,aAAaA,CAACR,WAAmB,EAAEkE,QAAqB,EAAmB;IACjF,OAAO,CAAC;MACNC,IAAI,EAAE,MAAM;MACZtD,KAAK,EAAE,CAAC;QAAEC,IAAI,EAAEd;MAAY,CAAC;IAC/B,CAAC,CAAC;EACJ;;EAEA;;EAEA;AACF;AACA;AACA;EACUuC,sBAAsBA,CAACF,IAAS,EAAEpC,KAAuB,EAAkB;IACjF,IAAI,CAACoC,IAAI,CAAC+B,UAAU,IAAI/B,IAAI,CAAC+B,UAAU,CAAChE,MAAM,KAAK,CAAC,EAAE;MACpDb,MAAM,CAAC8E,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC;MAC1D,OAAO;QACLC,SAAS,EAAE,CAAC;UAAE1B,IAAI,EAAE,MAAM;UAAE2B,IAAI,EAAE;YAAEzD,IAAI,EAAE,wBAAwB;YAAE0D,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QACvFC,SAAS,EAAE;UAAEC,gBAAgB,EAAE,EAAE;UAAEZ,MAAM,EAAE,EAAE;UAAEC,IAAI,EAAE;QAAG,CAAC;QACzDjD,IAAI,EAAE;MACR,CAAC;IACH;IAEA,MAAM6D,SAAS,GAAGtC,IAAI,CAAC+B,UAAU,CAAC,CAAC,CAAC;IACpC,MAAMvD,KAAK,GAAG8D,SAAS,CAACC,OAAO,EAAE/D,KAAK,IAAI,EAAE;;IAE5C;IACA,MAAMgE,UAAU,GAAGhE,KAAK,CAACiE,IAAI,CAAEC,CAAM,IAAKA,CAAC,CAACC,YAAY,CAAC;IACzD,MAAMC,QAAQ,GAAGpE,KAAK,CAACiE,IAAI,CAAEC,CAAM,IAAKA,CAAC,CAACjE,IAAI,CAAC;IAE/C,IAAI,CAAC+D,UAAU,EAAEG,YAAY,EAAE;MAC7BzF,MAAM,CAAC8E,IAAI,CAAC,gBAAgB,EAAE,qCAAqC,EAAEY,QAAQ,EAAEnE,IAAI,CAAC;MACpF,OAAO;QACLwD,SAAS,EAAE,CAAC;UAAE1B,IAAI,EAAE,MAAM;UAAE2B,IAAI,EAAE;YAAEzD,IAAI,EAAEmE,QAAQ,EAAEnE,IAAI,IAAI,kBAAkB;YAAE0D,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QACnGC,SAAS,EAAE;UAAEC,gBAAgB,EAAE,EAAE;UAAEZ,MAAM,EAAE,EAAE;UAAEC,IAAI,EAAE;QAAG,CAAC;QACzDjD,IAAI,EAAEmE,QAAQ,EAAEnE;MAClB,CAAC;IACH;IAEA,MAAMyD,IAAI,GAAGM,UAAU,CAACG,YAAY,CAACT,IAAI,IAAI,CAAC,CAAC;;IAE/C;IACA,MAAME,SAAyB,GAAG;MAChCC,gBAAgB,EAAEH,IAAI,CAACV,kBAAkB,IAAI,EAAE;MAC/CC,MAAM,EAAES,IAAI,CAACT,MAAM,IAAI,EAAE;MACzBC,IAAI,EAAEQ,IAAI,CAACR,IAAI,IAAI;IACrB,CAAC;;IAED;IACA,MAAMmB,UAAU,GAAGX,IAAI,CAACP,WAAW;IACnC,IAAI,CAACkB,UAAU,EAAE;MACf3F,MAAM,CAAC8E,IAAI,CAAC,gBAAgB,EAAE,qDAAqD,CAAC;MACpF,OAAO;QACLC,SAAS,EAAE,CAAC;UAAE1B,IAAI,EAAE,MAAM;UAAE2B,IAAI,EAAE;YAAEzD,IAAI,EAAE,iCAAiC;YAAE0D,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QAChGC,SAAS;QACT3D,IAAI,EAAEmE,QAAQ,EAAEnE;MAClB,CAAC;IACH;;IAEA;IACA,MAAMqE,UAA+B,GAAG,CAAC,CAAC;IAC1C,MAAMC,YAAY,GAAG,IAAIC,GAAG,CAAC,CAAC,GAAG5F,gBAAgB,EAAE,aAAa,CAAC,CAAC;;IAElE;IACA,MAAM6F,WAAW,GAAGrF,KAAK,CAAC6E,IAAI,CAACnC,CAAC,IAAIA,CAAC,CAACC,IAAI,KAAKsC,UAAU,CAAC;IAC1D,IAAII,WAAW,EAAE;MACf,KAAK,MAAMvC,SAAS,IAAIE,MAAM,CAACS,IAAI,CAAC4B,WAAW,CAACnC,UAAU,CAAC,EAAE;QAC3D,IAAIoB,IAAI,CAACxB,SAAS,CAAC,KAAKwC,SAAS,EAAE;UACjCJ,UAAU,CAACpC,SAAS,CAAC,GAAGwB,IAAI,CAACxB,SAAS,CAAC;QACzC;MACF;IACF,CAAC,MAAM;MACL;MACA,KAAK,MAAM,CAACyC,GAAG,EAAEC,KAAK,CAAC,IAAIxC,MAAM,CAACC,OAAO,CAACqB,IAAI,CAAC,EAAE;QAC/C,IAAI,CAACa,YAAY,CAACM,GAAG,CAACF,GAAG,CAAC,EAAE;UAC1BL,UAAU,CAACK,GAAG,CAAC,GAAGC,KAAK;QACzB;MACF;IACF;IAEAlG,MAAM,CAACY,IAAI,CAAC,gBAAgB,EAAE,kBAAkB+E,UAAU,WAAWT,SAAS,CAACV,IAAI,GAAG,CAAC;IAEvF,OAAO;MACLO,SAAS,EAAE,CAAC;QAAE1B,IAAI,EAAEsC,UAAU;QAAEX,IAAI,EAAEY;MAAW,CAAC,CAAC;MACnDV,SAAS;MACT3D,IAAI,EAAEmE,QAAQ,EAAEnE;IAClB,CAAC;EACH;AACF","ignoreList":[]}
+{"version":3,"names":["GoogleGenAI","FunctionCallingConfigMode","Type","logger","AGENT_STEP_FN","REASONING_FIELDS","GeminiProvider","constructor","apiKey","model","ai","generateContent","systemPrompt","userMessage","tools","history","screenshot","info","length","agentStepDeclaration","buildAgentStepDeclaration","contents","buildContents","startTime","Date","now","response","models","config","systemInstruction","functionDeclarations","toolConfig","functionCallingConfig","mode","ANY","allowedFunctionNames","temperature","maxOutputTokens","elapsed","tokenUsage","extractTokenUsage","promptTokens","completionTokens","estimatedCostUSD","toFixed","result","parseAgentStepResponse","error","message","status","Error","toolNames","map","t","name","actionProperties","tool","paramName","param","Object","entries","parameters","type","mapParamType","description","enum","toolDescriptions","params","keys","join","OBJECT","properties","previous_goal_eval","STRING","memory","plan","action_name","required","NUMBER","INTEGER","BOOLEAN","_history","parts","text","push","inlineData","mimeType","data","role","candidates","warn","toolCalls","args","success","reasoning","previousGoalEval","candidate","content","fnCallPart","find","p","functionCall","textPart","actionName","actionArgs","reservedKeys","Set","matchedTool","undefined","key","value","has","meta","usageMetadata","promptTokenCount","candidatesTokenCount","totalTokens","totalTokenCount","INPUT_COST_PER_M","OUTPUT_COST_PER_M"],"sourceRoot":"../../../src","sources":["providers/GeminiProvider.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,WAAW,EAAEC,yBAAyB,EAAEC,IAAI,QAAQ,eAAe;AAC5E,SAASC,MAAM,QAAQ,oBAAiB;AAGxC;;AAEA,MAAMC,aAAa,GAAG,YAAY;;AAElC;AACA,MAAMC,gBAAgB,GAAG,CAAC,oBAAoB,EAAE,QAAQ,EAAE,MAAM,CAAU;;AAE1E;;AAEA,OAAO,MAAMC,cAAc,CAAuB;EAIhDC,WAAWA,CAACC,MAAc,EAAEC,KAAa,GAAG,kBAAkB,EAAE;IAC9D,IAAI,CAACC,EAAE,GAAG,IAAIV,WAAW,CAAC;MAAEQ;IAAO,CAAC,CAAC;IACrC,IAAI,CAACC,KAAK,GAAGA,KAAK;EACpB;EAEA,MAAME,eAAeA,CACnBC,YAAoB,EACpBC,WAAmB,EACnBC,KAAuB,EACvBC,OAAoB,EACpBC,UAAmB,EACM;IAEzBb,MAAM,CAACc,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,IAAI,CAACR,KAAK,YAAYK,KAAK,CAACI,MAAM,GAAGF,UAAU,GAAG,mBAAmB,GAAG,EAAE,EAAE,CAAC;;IAEtI;IACA,MAAMG,oBAAoB,GAAG,IAAI,CAACC,yBAAyB,CAACN,KAAK,CAAC;;IAElE;IACA,MAAMO,QAAQ,GAAG,IAAI,CAACC,aAAa,CAACT,WAAW,EAAEE,OAAO,EAAEC,UAAU,CAAC;IAErE,MAAMO,SAAS,GAAGC,IAAI,CAACC,GAAG,CAAC,CAAC;IAE5B,IAAI;MACF,MAAMC,QAAQ,GAAG,MAAM,IAAI,CAAChB,EAAE,CAACiB,MAAM,CAAChB,eAAe,CAAC;QACpDF,KAAK,EAAE,IAAI,CAACA,KAAK;QACjBY,QAAQ;QACRO,MAAM,EAAE;UACNC,iBAAiB,EAAEjB,YAAY;UAC/BE,KAAK,EAAE,CAAC;YAAEgB,oBAAoB,EAAE,CAACX,oBAAoB;UAAE,CAAC,CAAC;UACzDY,UAAU,EAAE;YACVC,qBAAqB,EAAE;cACrBC,IAAI,EAAEhC,yBAAyB,CAACiC,GAAG;cACnCC,oBAAoB,EAAE,CAAC/B,aAAa;YACtC;UACF,CAAC;UACDgC,WAAW,EAAE,GAAG;UAChBC,eAAe,EAAE;QACnB;MACF,CAAC,CAAC;MAEF,MAAMC,OAAO,GAAGd,IAAI,CAACC,GAAG,CAAC,CAAC,GAAGF,SAAS;MACtCpB,MAAM,CAACc,IAAI,CAAC,gBAAgB,EAAE,wBAAwBqB,OAAO,IAAI,CAAC;;MAElE;MACA,MAAMC,UAAU,GAAG,IAAI,CAACC,iBAAiB,CAACd,QAAQ,CAAC;MACnD,IAAIa,UAAU,EAAE;QACdpC,MAAM,CAACc,IAAI,CAAC,gBAAgB,EAAE,WAAWsB,UAAU,CAACE,YAAY,SAASF,UAAU,CAACG,gBAAgB,WAAWH,UAAU,CAACI,gBAAgB,CAACC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;MAC1J;MAEA,MAAMC,MAAM,GAAG,IAAI,CAACC,sBAAsB,CAACpB,QAAQ,EAAEZ,KAAK,CAAC;MAC3D+B,MAAM,CAACN,UAAU,GAAGA,UAAU;MAC9B,OAAOM,MAAM;IACf,CAAC,CAAC,OAAOE,KAAU,EAAE;MACnB5C,MAAM,CAAC4C,KAAK,CAAC,gBAAgB,EAAE,iBAAiB,EAAEA,KAAK,CAACC,OAAO,CAAC;;MAEhE;MACA,IAAID,KAAK,CAACE,MAAM,EAAE;QAChB,MAAM,IAAIC,KAAK,CAAC,oBAAoBH,KAAK,CAACE,MAAM,KAAKF,KAAK,CAACC,OAAO,EAAE,CAAC;MACvE;MACA,MAAMD,KAAK;IACb;EACF;;EAEA;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;AACA;EACU3B,yBAAyBA,CAACN,KAAuB,EAAO;IAC9D,MAAMqC,SAAS,GAAGrC,KAAK,CAACsC,GAAG,CAACC,CAAC,IAAIA,CAAC,CAACC,IAAI,CAAC;;IAExC;IACA,MAAMC,gBAAqC,GAAG,CAAC,CAAC;IAChD,KAAK,MAAMC,IAAI,IAAI1C,KAAK,EAAE;MACxB,KAAK,MAAM,CAAC2C,SAAS,EAAEC,KAAK,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACJ,IAAI,CAACK,UAAU,CAAC,EAAE;QAChE,IAAIN,gBAAgB,CAACE,SAAS,CAAC,EAAE;QACjCF,gBAAgB,CAACE,SAAS,CAAC,GAAG;UAC5BK,IAAI,EAAE,IAAI,CAACC,YAAY,CAACL,KAAK,CAACI,IAAI,CAAC;UACnCE,WAAW,EAAEN,KAAK,CAACM,WAAW;UAC9B,IAAIN,KAAK,CAACO,IAAI,GAAG;YAAEA,IAAI,EAAEP,KAAK,CAACO;UAAK,CAAC,GAAG,CAAC,CAAC;QAC5C,CAAC;MACH;IACF;;IAEA;IACA,MAAMC,gBAAgB,GAAGpD,KAAK,CAC3BsC,GAAG,CAACC,CAAC,IAAI;MACR,MAAMc,MAAM,GAAGR,MAAM,CAACS,IAAI,CAACf,CAAC,CAACQ,UAAU,CAAC,CAACQ,IAAI,CAAC,IAAI,CAAC;MACnD,OAAO,KAAKhB,CAAC,CAACC,IAAI,IAAIa,MAAM,MAAMd,CAAC,CAACW,WAAW,EAAE;IACnD,CAAC,CAAC,CACDK,IAAI,CAAC,IAAI,CAAC;IAEb,OAAO;MACLf,IAAI,EAAElD,aAAa;MACnB4D,WAAW,EAAE,0FAA0FE,gBAAgB,EAAE;MACzHL,UAAU,EAAE;QACVC,IAAI,EAAE5D,IAAI,CAACoE,MAAM;QACjBC,UAAU,EAAE;UACVC,kBAAkB,EAAE;YAClBV,IAAI,EAAE5D,IAAI,CAACuE,MAAM;YACjBT,WAAW,EAAE;UACf,CAAC;UACDU,MAAM,EAAE;YACNZ,IAAI,EAAE5D,IAAI,CAACuE,MAAM;YACjBT,WAAW,EAAE;UACf,CAAC;UACDW,IAAI,EAAE;YACJb,IAAI,EAAE5D,IAAI,CAACuE,MAAM;YACjBT,WAAW,EAAE;UACf,CAAC;UACDY,WAAW,EAAE;YACXd,IAAI,EAAE5D,IAAI,CAACuE,MAAM;YACjBT,WAAW,EAAE,0BAA0B;YACvCC,IAAI,EAAEd;UACR,CAAC;UACD,GAAGI;QACL,CAAC;QACDsB,QAAQ,EAAE,CAAC,MAAM,EAAE,aAAa;MAClC;IACF,CAAC;EACH;EAEQd,YAAYA,CAACD,IAAY,EAAU;IACzC,QAAQA,IAAI;MACV,KAAK,QAAQ;QAAE,OAAO5D,IAAI,CAAC4E,MAAM;MACjC,KAAK,SAAS;QAAE,OAAO5E,IAAI,CAAC6E,OAAO;MACnC,KAAK,SAAS;QAAE,OAAO7E,IAAI,CAAC8E,OAAO;MACnC,KAAK,QAAQ;MACb;QAAS,OAAO9E,IAAI,CAACuE,MAAM;IAC7B;EACF;;EAEA;;EAEA;AACF;AACA;AACA;EACUnD,aAAaA,CAACT,WAAmB,EAAEoE,QAAqB,EAAEjE,UAAmB,EAAS;IAC5F,MAAMkE,KAAY,GAAG,CAAC;MAAEC,IAAI,EAAEtE;IAAY,CAAC,CAAC;;IAE5C;IACA,IAAIG,UAAU,EAAE;MACdkE,KAAK,CAACE,IAAI,CAAC;QACTC,UAAU,EAAE;UACVC,QAAQ,EAAE,YAAY;UACtBC,IAAI,EAAEvE;QACR;MACF,CAAC,CAAC;IACJ;IAEA,OAAO,CAAC;MAAEwE,IAAI,EAAE,MAAM;MAAEN;IAAM,CAAC,CAAC;EAClC;;EAEA;;EAEA;AACF;AACA;AACA;EACUpC,sBAAsBA,CAACpB,QAAa,EAAEZ,KAAuB,EAAkB;IACrF,MAAM2E,UAAU,GAAG/D,QAAQ,CAAC+D,UAAU,IAAI,EAAE;IAE5C,IAAIA,UAAU,CAACvE,MAAM,KAAK,CAAC,EAAE;MAC3Bf,MAAM,CAACuF,IAAI,CAAC,gBAAgB,EAAE,2BAA2B,CAAC;MAC1D,OAAO;QACLC,SAAS,EAAE,CAAC;UAAErC,IAAI,EAAE,MAAM;UAAEsC,IAAI,EAAE;YAAET,IAAI,EAAE,wBAAwB;YAAEU,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QACvFC,SAAS,EAAE;UAAEC,gBAAgB,EAAE,EAAE;UAAErB,MAAM,EAAE,EAAE;UAAEC,IAAI,EAAE;QAAG,CAAC;QACzDQ,IAAI,EAAE;MACR,CAAC;IACH;IAEA,MAAMa,SAAS,GAAGP,UAAU,CAAC,CAAC,CAAC;IAC/B,MAAMP,KAAK,GAAGc,SAAS,CAACC,OAAO,EAAEf,KAAK,IAAI,EAAE;;IAE5C;IACA,MAAMgB,UAAU,GAAGhB,KAAK,CAACiB,IAAI,CAAEC,CAAM,IAAKA,CAAC,CAACC,YAAY,CAAC;IACzD,MAAMC,QAAQ,GAAGpB,KAAK,CAACiB,IAAI,CAAEC,CAAM,IAAKA,CAAC,CAACjB,IAAI,CAAC;IAE/C,IAAI,CAACe,UAAU,EAAEG,YAAY,EAAE;MAC7BlG,MAAM,CAACuF,IAAI,CAAC,gBAAgB,EAAE,qCAAqC,EAAEY,QAAQ,EAAEnB,IAAI,CAAC;MACpF,OAAO;QACLQ,SAAS,EAAE,CAAC;UAAErC,IAAI,EAAE,MAAM;UAAEsC,IAAI,EAAE;YAAET,IAAI,EAAEmB,QAAQ,EAAEnB,IAAI,IAAI,kBAAkB;YAAEU,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QACnGC,SAAS,EAAE;UAAEC,gBAAgB,EAAE,EAAE;UAAErB,MAAM,EAAE,EAAE;UAAEC,IAAI,EAAE;QAAG,CAAC;QACzDQ,IAAI,EAAEmB,QAAQ,EAAEnB;MAClB,CAAC;IACH;IAEA,MAAMS,IAAI,GAAGM,UAAU,CAACG,YAAY,CAACT,IAAI,IAAI,CAAC,CAAC;;IAE/C;IACA,MAAME,SAAyB,GAAG;MAChCC,gBAAgB,EAAEH,IAAI,CAACpB,kBAAkB,IAAI,EAAE;MAC/CE,MAAM,EAAEkB,IAAI,CAAClB,MAAM,IAAI,EAAE;MACzBC,IAAI,EAAEiB,IAAI,CAACjB,IAAI,IAAI;IACrB,CAAC;;IAED;IACA,MAAM4B,UAAU,GAAGX,IAAI,CAAChB,WAAW;IACnC,IAAI,CAAC2B,UAAU,EAAE;MACfpG,MAAM,CAACuF,IAAI,CAAC,gBAAgB,EAAE,qDAAqD,CAAC;MACpF,OAAO;QACLC,SAAS,EAAE,CAAC;UAAErC,IAAI,EAAE,MAAM;UAAEsC,IAAI,EAAE;YAAET,IAAI,EAAE,iCAAiC;YAAEU,OAAO,EAAE;UAAM;QAAE,CAAC,CAAC;QAChGC,SAAS;QACTX,IAAI,EAAEmB,QAAQ,EAAEnB;MAClB,CAAC;IACH;;IAEA;IACA,MAAMqB,UAA+B,GAAG,CAAC,CAAC;IAC1C,MAAMC,YAAY,GAAG,IAAIC,GAAG,CAAC,CAAC,GAAGrG,gBAAgB,EAAE,aAAa,CAAC,CAAC;IAElE,MAAMsG,WAAW,GAAG7F,KAAK,CAACqF,IAAI,CAAC9C,CAAC,IAAIA,CAAC,CAACC,IAAI,KAAKiD,UAAU,CAAC;IAC1D,IAAII,WAAW,EAAE;MACf,KAAK,MAAMlD,SAAS,IAAIE,MAAM,CAACS,IAAI,CAACuC,WAAW,CAAC9C,UAAU,CAAC,EAAE;QAC3D,IAAI+B,IAAI,CAACnC,SAAS,CAAC,KAAKmD,SAAS,EAAE;UACjCJ,UAAU,CAAC/C,SAAS,CAAC,GAAGmC,IAAI,CAACnC,SAAS,CAAC;QACzC;MACF;IACF,CAAC,MAAM;MACL,KAAK,MAAM,CAACoD,GAAG,EAAEC,KAAK,CAAC,IAAInD,MAAM,CAACC,OAAO,CAACgC,IAAI,CAAC,EAAE;QAC/C,IAAI,CAACa,YAAY,CAACM,GAAG,CAACF,GAAG,CAAC,EAAE;UAC1BL,UAAU,CAACK,GAAG,CAAC,GAAGC,KAAK;QACzB;MACF;IACF;IAEA3G,MAAM,CAACc,IAAI,CAAC,gBAAgB,EAAE,kBAAkBsF,UAAU,WAAWT,SAAS,CAACnB,IAAI,GAAG,CAAC;IAEvF,OAAO;MACLgB,SAAS,EAAE,CAAC;QAAErC,IAAI,EAAEiD,UAAU;QAAEX,IAAI,EAAEY;MAAW,CAAC,CAAC;MACnDV,SAAS;MACTX,IAAI,EAAEmB,QAAQ,EAAEnB;IAClB,CAAC;EACH;;EAEA;;EAEA;AACF;AACA;AACA;AACA;AACA;AACA;EACU3C,iBAAiBA,CAACd,QAAa,EAA0B;IAC/D,MAAMsF,IAAI,GAAGtF,QAAQ,EAAEuF,aAAa;IACpC,IAAI,CAACD,IAAI,EAAE,OAAOJ,SAAS;IAE3B,MAAMnE,YAAY,GAAGuE,IAAI,CAACE,gBAAgB,IAAI,CAAC;IAC/C,MAAMxE,gBAAgB,GAAGsE,IAAI,CAACG,oBAAoB,IAAI,CAAC;IACvD,MAAMC,WAAW,GAAGJ,IAAI,CAACK,eAAe,IAAK5E,YAAY,GAAGC,gBAAiB;;IAE7E;IACA,MAAM4E,gBAAgB,GAAG,IAAI;IAC7B,MAAMC,iBAAiB,GAAG,IAAI;IAE9B,MAAM5E,gBAAgB,GACnBF,YAAY,GAAG,SAAS,GAAI6E,gBAAgB,GAC5C5E,gBAAgB,GAAG,SAAS,GAAI6E,iBAAiB;IAEpD,OAAO;MAAE9E,YAAY;MAAEC,gBAAgB;MAAE0E,WAAW;MAAEzE;IAAiB,CAAC;EAC1E;AACF","ignoreList":[]}

package/lib/module/services/AudioInputService.js

@@ -0,0 +1,128 @@
+"use strict";
+
+/**
+ * AudioInputService — Real-time microphone capture for voice mode.
+ *
+ * Uses react-native-audio-api (Software Mansion) AudioRecorder for native
+ * PCM streaming from the microphone. Each chunk is converted from Float32
+ * to Int16 PCM and base64-encoded for the Gemini Live API.
+ *
+ * Requires: react-native-audio-api (development build only, not Expo Go)
+ */
+
+import { logger } from "../utils/logger.js";
+import { float32ToInt16Base64 } from "../utils/audioUtils.js";
+
+// ─── Types ─────────────────────────────────────────────────────
+
+// ─── Service ───────────────────────────────────────────────────
+
+export class AudioInputService {
+  status = 'idle';
+  recorder = null;
+  constructor(config) {
+    this.config = config;
+  }
+
+  // ─── Lifecycle ─────────────────────────────────────────────
+
+  async start() {
+    try {
+      // Lazy-load react-native-audio-api (optional peer dependency)
+      let audioApi;
+      try {
+        audioApi = require('react-native-audio-api');
+      } catch {
+        const msg = 'Voice mode requires react-native-audio-api. Install with: npm install react-native-audio-api';
+        logger.error('AudioInput', msg);
+        this.config.onError?.(msg);
+        return false;
+      }
+
+      // Request mic permission (Android)
+      try {
+        const {
+          PermissionsAndroid,
+          Platform
+        } = require('react-native');
+        if (Platform.OS === 'android') {
+          const result = await PermissionsAndroid.request(PermissionsAndroid.PERMISSIONS.RECORD_AUDIO);
+          if (result !== PermissionsAndroid.RESULTS.GRANTED) {
+            logger.warn('AudioInput', 'Microphone permission denied');
+            this.config.onPermissionDenied?.();
+            return false;
+          }
+        }
+      } catch {
+        // Permission check failed — continue and let native layer handle it
+      }
+
+      // Create AudioRecorder
+      this.recorder = new audioApi.AudioRecorder();
+      const sampleRate = this.config.sampleRate || 16000;
+      const bufferLength = this.config.bufferLength || 4096;
+
+      // Register audio data callback
+      let frameCount = 0;
+      this.recorder.onAudioReady({
+        sampleRate,
+        bufferLength,
+        channelCount: 1
+      }, event => {
+        frameCount++;
+        try {
+          // event.buffer is an AudioBuffer — get Float32 channel data
+          const float32Data = event.buffer.getChannelData(0);
+          // Convert Float32 → Int16 → base64 for Gemini
+          const base64Chunk = float32ToInt16Base64(float32Data);
+          logger.debug('AudioInput', `🎤 Frame #${frameCount}: size=${base64Chunk.length}`);
+          this.config.onAudioChunk(base64Chunk);
+        } catch (err) {
+          logger.error('AudioInput', `Frame processing error: ${err.message}`);
+        }
+      });
+
+      // Register error callback
+      this.recorder.onError(error => {
+        logger.error('AudioInput', `Recorder error: ${error.message || error}`);
+        this.config.onError?.(error.message || String(error));
+      });
+
+      // Start recording
+      this.recorder.start();
+      this.status = 'recording';
+      logger.info('AudioInput', `Streaming started (${sampleRate}Hz, bufLen=${bufferLength})`);
+      return true;
+    } catch (error) {
+      logger.error('AudioInput', `Failed to start: ${error.message}`);
+      this.config.onError?.(error.message);
+      return false;
+    }
+  }
+  async stop() {
+    try {
+      if (this.recorder && this.status !== 'idle') {
+        this.recorder.clearOnAudioReady();
+        this.recorder.clearOnError();
+        this.recorder.stop();
+      }
+      this.recorder = null;
+      this.status = 'idle';
+      logger.info('AudioInput', 'Streaming stopped');
+    } catch (error) {
+      logger.error('AudioInput', `Failed to stop: ${error.message}`);
+      this.recorder = null;
+      this.status = 'idle';
+    }
+  }
+
+  // ─── Status ───────────────────────────────────────────────
+
+  get isRecording() {
+    return this.status === 'recording';
+  }
+  get currentStatus() {
+    return this.status;
+  }
+}
+//# sourceMappingURL=AudioInputService.js.map

package/lib/module/services/AudioInputService.js.map

@@ -0,0 +1 @@
+{"version":3,"names":["logger","float32ToInt16Base64","AudioInputService","status","recorder","constructor","config","start","audioApi","require","msg","error","onError","PermissionsAndroid","Platform","OS","result","request","PERMISSIONS","RECORD_AUDIO","RESULTS","GRANTED","warn","onPermissionDenied","AudioRecorder","sampleRate","bufferLength","frameCount","onAudioReady","channelCount","event","float32Data","buffer","getChannelData","base64Chunk","debug","length","onAudioChunk","err","message","String","info","stop","clearOnAudioReady","clearOnError","isRecording","currentStatus"],"sourceRoot":"../../../src","sources":["services/AudioInputService.ts"],"mappings":";;AAAA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;AACA;;AAEA,SAASA,MAAM,QAAQ,oBAAiB;AACxC,SAASC,oBAAoB,QAAQ,wBAAqB;;AAE1D;;AAcA;;AAEA,OAAO,MAAMC,iBAAiB,CAAC;EAErBC,MAAM,GAAoB,MAAM;EAChCC,QAAQ,GAAQ,IAAI;EAE5BC,WAAWA,CAACC,MAAwB,EAAE;IACpC,IAAI,CAACA,MAAM,GAAGA,MAAM;EACtB;;EAEA;;EAEA,MAAMC,KAAKA,CAAA,EAAqB;IAC9B,IAAI;MACF;MACA,IAAIC,QAAa;MACjB,IAAI;QACFA,QAAQ,GAAGC,OAAO,CAAC,wBAAwB,CAAC;MAC9C,CAAC,CAAC,MAAM;QACN,MAAMC,GAAG,GACP,8FAA8F;QAChGV,MAAM,CAACW,KAAK,CAAC,YAAY,EAAED,GAAG,CAAC;QAC/B,IAAI,CAACJ,MAAM,CAACM,OAAO,GAAGF,GAAG,CAAC;QAC1B,OAAO,KAAK;MACd;;MAEA;MACA,IAAI;QACF,MAAM;UAAEG,kBAAkB;UAAEC;QAAS,CAAC,GAAGL,OAAO,CAAC,cAAc,CAAC;QAChE,IAAIK,QAAQ,CAACC,EAAE,KAAK,SAAS,EAAE;UAC7B,MAAMC,MAAM,GAAG,MAAMH,kBAAkB,CAACI,OAAO,CAC7CJ,kBAAkB,CAACK,WAAW,CAACC,YACjC,CAAC;UACD,IAAIH,MAAM,KAAKH,kBAAkB,CAACO,OAAO,CAACC,OAAO,EAAE;YACjDrB,MAAM,CAACsB,IAAI,CAAC,YAAY,EAAE,8BAA8B,CAAC;YACzD,IAAI,CAAChB,MAAM,CAACiB,kBAAkB,GAAG,CAAC;YAClC,OAAO,KAAK;UACd;QACF;MACF,CAAC,CAAC,MAAM;QACN;MAAA;;MAGF;MACA,IAAI,CAACnB,QAAQ,GAAG,IAAII,QAAQ,CAACgB,aAAa,CAAC,CAAC;MAE5C,MAAMC,UAAU,GAAG,IAAI,CAACnB,MAAM,CAACmB,UAAU,IAAI,KAAK;MAClD,MAAMC,YAAY,GAAG,IAAI,CAACpB,MAAM,CAACoB,YAAY,IAAI,IAAI;;MAErD;MACA,IAAIC,UAAU,GAAG,CAAC;MAClB,IAAI,CAACvB,QAAQ,CAACwB,YAAY,CACxB;QAAEH,UAAU;QAAEC,YAAY;QAAEG,YAAY,EAAE;MAAE,CAAC,EAC5CC,KAAU,IAAK;QACdH,UAAU,EAAE;QACZ,IAAI;UACF;UACA,MAAMI,WAAW,GAAGD,KAAK,CAACE,MAAM,CAACC,cAAc,CAAC,CAAC,CAAC;UAClD;UACA,MAAMC,WAAW,GAAGjC,oBAAoB,CAAC8B,WAAW,CAAC;UACrD/B,MAAM,CAACmC,KAAK,CAAC,YAAY,EAAE,aAAaR,UAAU,UAAUO,WAAW,CAACE,MAAM,EAAE,CAAC;UACjF,IAAI,CAAC9B,MAAM,CAAC+B,YAAY,CAACH,WAAW,CAAC;QACvC,CAAC,CAAC,OAAOI,GAAQ,EAAE;UACjBtC,MAAM,CAACW,KAAK,CAAC,YAAY,EAAE,2BAA2B2B,GAAG,CAACC,OAAO,EAAE,CAAC;QACtE;MACF,CACF,CAAC;;MAED;MACA,IAAI,CAACnC,QAAQ,CAACQ,OAAO,CAAED,KAAU,IAAK;QACpCX,MAAM,CAACW,KAAK,CAAC,YAAY,EAAE,mBAAmBA,KAAK,CAAC4B,OAAO,IAAI5B,KAAK,EAAE,CAAC;QACvE,IAAI,CAACL,MAAM,CAACM,OAAO,GAAGD,KAAK,CAAC4B,OAAO,IAAIC,MAAM,CAAC7B,KAAK,CAAC,CAAC;MACvD,CAAC,CAAC;;MAEF;MACA,IAAI,CAACP,QAAQ,CAACG,KAAK,CAAC,CAAC;MACrB,IAAI,CAACJ,MAAM,GAAG,WAAW;MACzBH,MAAM,CAACyC,IAAI,CAAC,YAAY,EAAE,sBAAsBhB,UAAU,cAAcC,YAAY,GAAG,CAAC;MACxF,OAAO,IAAI;IACb,CAAC,CAAC,OAAOf,KAAU,EAAE;MACnBX,MAAM,CAACW,KAAK,CAAC,YAAY,EAAE,oBAAoBA,KAAK,CAAC4B,OAAO,EAAE,CAAC;MAC/D,IAAI,CAACjC,MAAM,CAACM,OAAO,GAAGD,KAAK,CAAC4B,OAAO,CAAC;MACpC,OAAO,KAAK;IACd;EACF;EAEA,MAAMG,IAAIA,CAAA,EAAkB;IAC1B,IAAI;MACF,IAAI,IAAI,CAACtC,QAAQ,IAAI,IAAI,CAACD,MAAM,KAAK,MAAM,EAAE;QAC3C,IAAI,CAACC,QAAQ,CAACuC,iBAAiB,CAAC,CAAC;QACjC,IAAI,CAACvC,QAAQ,CAACwC,YAAY,CAAC,CAAC;QAC5B,IAAI,CAACxC,QAAQ,CAACsC,IAAI,CAAC,CAAC;MACtB;MACA,IAAI,CAACtC,QAAQ,GAAG,IAAI;MACpB,IAAI,CAACD,MAAM,GAAG,MAAM;MACpBH,MAAM,CAACyC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC;IAChD,CAAC,CAAC,OAAO9B,KAAU,EAAE;MACnBX,MAAM,CAACW,KAAK,CAAC,YAAY,EAAE,mBAAmBA,KAAK,CAAC4B,OAAO,EAAE,CAAC;MAC9D,IAAI,CAACnC,QAAQ,GAAG,IAAI;MACpB,IAAI,CAACD,MAAM,GAAG,MAAM;IACtB;EACF;;EAEA;;EAEA,IAAI0C,WAAWA,CAAA,EAAY;IACzB,OAAO,IAAI,CAAC1C,MAAM,KAAK,WAAW;EACpC;EAEA,IAAI2C,aAAaA,CAAA,EAAoB;IACnC,OAAO,IAAI,CAAC3C,MAAM;EACpB;AACF","ignoreList":[]}

package/lib/module/services/AudioOutputService.js

@@ -0,0 +1,154 @@
+"use strict";
+
+/**
+ * AudioOutputService — AI speech playback for voice mode.
+ *
+ * Uses react-native-audio-api (Software Mansion) for gapless, low-latency
+ * PCM playback. Decodes base64 PCM from Gemini Live API and queues it via
+ * AudioBufferQueueSourceNode for seamless streaming.
+ *
+ * Requires: react-native-audio-api (development build only, not Expo Go)
+ */
+
+import { logger } from "../utils/logger.js";
+import { base64ToFloat32 } from "../utils/audioUtils.js";
+
+// ─── Types ─────────────────────────────────────────────────────
+
+/** Gemini Live API outputs 24kHz 16-bit mono PCM */
+const GEMINI_OUTPUT_SAMPLE_RATE = 24000;
+// ─── Service ───────────────────────────────────────────────────
+
+export class AudioOutputService {
+  audioContext = null;
+  queueSourceNode = null;
+  gainNode = null;
+  muted = false;
+  isStarted = false;
+  chunkCount = 0;
+  constructor(config = {}) {
+    this.config = config;
+  }
+
+  // ─── Lifecycle ─────────────────────────────────────────────
+
+  async initialize() {
+    try {
+      let audioApi;
+      try {
+        audioApi = require('react-native-audio-api');
+      } catch {
+        const msg = 'react-native-audio-api is required for audio output. Install with: npm install react-native-audio-api';
+        logger.error('AudioOutput', msg);
+        this.config.onError?.(msg);
+        return false;
+      }
+      const sampleRate = this.config.sampleRate || GEMINI_OUTPUT_SAMPLE_RATE;
+
+      // Create AudioContext at Gemini's output sample rate
+      this.audioContext = new audioApi.AudioContext({
+        sampleRate
+      });
+
+      // Create GainNode for mute control
+      this.gainNode = this.audioContext.createGain();
+      this.gainNode.gain.value = 1.0;
+      this.gainNode.connect(this.audioContext.destination);
+
+      // Create AudioBufferQueueSourceNode for gapless streaming
+      this.queueSourceNode = this.audioContext.createBufferQueueSource();
+      this.queueSourceNode.connect(this.gainNode);
+      logger.info('AudioOutput', `Initialized (${sampleRate}Hz, AudioBufferQueueSourceNode)`);
+      return true;
+    } catch (error) {
+      logger.error('AudioOutput', `Failed to initialize: ${error.message}`);
+      this.config.onError?.(error.message);
+      return false;
+    }
+  }
+
+  // ─── Enqueue Audio ─────────────────────────────────────────
+
+  /** Add a base64-encoded PCM chunk from Gemini to the playback queue */
+  enqueue(base64Audio) {
+    if (this.muted || !this.audioContext || !this.queueSourceNode) return;
+    try {
+      this.chunkCount++;
+
+      // Decode base64 Int16 PCM → Float32
+      const float32Data = base64ToFloat32(base64Audio);
+      const sampleRate = this.config.sampleRate || GEMINI_OUTPUT_SAMPLE_RATE;
+
+      // Create an AudioBuffer and fill it with PCM data
+      const audioBuffer = this.audioContext.createBuffer(1, float32Data.length, sampleRate);
+      audioBuffer.copyToChannel(float32Data, 0);
+
+      // Enqueue the buffer for gapless playback
+      this.queueSourceNode.enqueueBuffer(audioBuffer);
+
+      // Start playback on first enqueue
+      if (!this.isStarted) {
+        this.queueSourceNode.start();
+        this.isStarted = true;
+        this.config.onPlaybackStart?.();
+        logger.info('AudioOutput', '▶️ Playback started');
+      }
+      if (this.chunkCount % 20 === 0) {
+        logger.debug('AudioOutput', `Queued chunk #${this.chunkCount}`);
+      }
+    } catch (error) {
+      logger.error('AudioOutput', `Enqueue error: ${error.message}`);
+    }
+  }
+
+  // ─── Mute/Unmute ──────────────────────────────────────────
+
+  mute() {
+    this.muted = true;
+    if (this.gainNode) {
+      this.gainNode.gain.value = 0;
+    }
+    logger.info('AudioOutput', 'Speaker muted');
+  }
+  unmute() {
+    this.muted = false;
+    if (this.gainNode) {
+      this.gainNode.gain.value = 1.0;
+    }
+    logger.info('AudioOutput', 'Speaker unmuted');
+  }
+  get isMuted() {
+    return this.muted;
+  }
+
+  // ─── Stop & Cleanup ───────────────────────────────────────
+
+  async stop() {
+    try {
+      if (this.queueSourceNode && this.isStarted) {
+        this.queueSourceNode.stop();
+        this.queueSourceNode.clearBuffers();
+      }
+      this.isStarted = false;
+      this.chunkCount = 0;
+      this.config.onPlaybackEnd?.();
+      logger.info('AudioOutput', 'Playback stopped');
+    } catch (error) {
+      logger.error('AudioOutput', `Stop error: ${error.message}`);
+    }
+  }
+  async cleanup() {
+    await this.stop();
+    try {
+      if (this.audioContext) {
+        await this.audioContext.close();
+      }
+    } catch {
+      // Non-critical
+    }
+    this.audioContext = null;
+    this.queueSourceNode = null;
+    this.gainNode = null;
+  }
+}
+//# sourceMappingURL=AudioOutputService.js.map