@mobileai/react-native 0.9.16 → 0.9.18

package/src/providers/OpenAIProvider.ts

@@ -1,379 +0,0 @@
-/**
- * OpenAIProvider — OpenAI Chat Completions API via raw fetch.
- *
- * Uses the same flat `agent_step` function pattern as GeminiProvider:
- * - Reasoning fields (previous_goal_eval, memory, plan) + action in one tool call
- * - `tool_choice: "required"` forces a tool call every step
- * - `strict: true` guarantees schema adherence
- *
- * No SDK dependency — raw fetch for full React Native compatibility.
- * Implements the AIProvider interface so it can be swapped with GeminiProvider.
- */
-
-import { logger } from '../utils/logger';
-import type {
-  AIProvider,
-  ToolDefinition,
-  AgentStep,
-  ProviderResult,
-  AgentReasoning,
-  TokenUsage,
-} from '../core/types';
-
-// ─── Constants ─────────────────────────────────────────────────
-
-const AGENT_STEP_FN = 'agent_step';
-const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
-const REASONING_FIELDS = ['previous_goal_eval', 'memory', 'plan'] as const;
-
-// ─── Provider ──────────────────────────────────────────────────
-
-export class OpenAIProvider implements AIProvider {
-  private model: string;
-  private baseUrl: string;
-  private headers: Record<string, string>;
-
-  constructor(
-    apiKey?: string,
-    model: string = 'gpt-4.1-mini',
-    proxyUrl?: string,
-    proxyHeaders?: Record<string, string>,
-  ) {
-    if (proxyUrl) {
-      this.baseUrl = proxyUrl.endsWith('/')
-        ? `${proxyUrl}v1/chat/completions`
-        : proxyUrl;
-      this.headers = {
-        'Content-Type': 'application/json',
-        ...(proxyHeaders || {}),
-      };
-    } else if (apiKey) {
-      this.baseUrl = OPENAI_API_URL;
-      this.headers = {
-        'Content-Type': 'application/json',
-        Authorization: `Bearer ${apiKey}`,
-      };
-    } else {
-      throw new Error(
-        '[mobileai] You must provide either "apiKey" or "proxyUrl" to use OpenAI provider.',
-      );
-    }
-
-    this.model = model;
-  }
-
-  async generateContent(
-    systemPrompt: string,
-    userMessage: string,
-    tools: ToolDefinition[],
-    _history: AgentStep[],
-    screenshot?: string,
-  ): Promise<ProviderResult> {
-    logger.info(
-      'OpenAIProvider',
-      `Sending request. Model: ${this.model}, Tools: ${tools.length}${screenshot ? ', with screenshot' : ''}`,
-    );
-
-    const agentStepTool = this.buildAgentStepTool(tools);
-    const messages = this.buildMessages(systemPrompt, userMessage, screenshot);
-
-    const startTime = Date.now();
-
-    try {
-      const response = await fetch(this.baseUrl, {
-        method: 'POST',
-        headers: this.headers,
-        body: JSON.stringify({
-          model: this.model,
-          messages,
-          tools: [agentStepTool],
-          tool_choice: 'required',
-          temperature: 0.2,
-          max_tokens: 2048,
-        }),
-      });
-
-      if (!response.ok) {
-        const errorBody = await response.text();
-        throw new Error(`OpenAI API error ${response.status}: ${errorBody}`);
-      }
-
-      const data = await response.json();
-      const elapsed = Date.now() - startTime;
-      logger.info('OpenAIProvider', `Response received in ${elapsed}ms`);
-
-      const tokenUsage = this.extractTokenUsage(data);
-      if (tokenUsage) {
-        logger.info(
-          'OpenAIProvider',
-          `Tokens: ${tokenUsage.promptTokens} in / ${tokenUsage.completionTokens} out / $${tokenUsage.estimatedCostUSD.toFixed(6)}`,
-        );
-      }
-
-      const result = this.parseAgentStepResponse(data, tools);
-      result.tokenUsage = tokenUsage;
-      return result;
-    } catch (error: any) {
-      logger.error('OpenAIProvider', 'Request failed:', error.message);
-      throw error;
-    }
-  }
-
-  // ─── Build agent_step Tool ──────────────────────────────────
-
-  /**
-   * Builds the OpenAI tool definition for `agent_step`.
-   * Same flat pattern as Gemini — reasoning fields + action in one function.
-   * Uses `strict: true` for guaranteed schema adherence.
-   */
-  private buildAgentStepTool(tools: ToolDefinition[]): any {
-    const toolNames = tools.map((t) => t.name);
-
-    // Collect all unique parameter fields across all tools
-    const actionProperties: Record<string, any> = {};
-    for (const tool of tools) {
-      for (const [paramName, param] of Object.entries(tool.parameters)) {
-        if (actionProperties[paramName]) continue;
-        actionProperties[paramName] = {
-          type: param.type,
-          description: param.description,
-          ...(param.enum ? { enum: param.enum } : {}),
-        };
-      }
-    }
-
-    // Build tool descriptions for enum
-    const toolDescriptions = tools
-      .map((t) => {
-        const params = Object.keys(t.parameters).join(', ');
-        return `- ${t.name}(${params}): ${t.description}`;
-      })
-      .join('\n');
-
-    // OpenAI strict mode requires additionalProperties: false
-    // and ALL properties in `required` array
-    const allProperties: Record<string, any> = {
-      previous_goal_eval: {
-        type: 'string',
-        description:
-          'One-sentence assessment of your last action. State success, failure, or uncertain. Skip on first step.',
-      },
-      memory: {
-        type: 'string',
-        description:
-          'Key facts to remember for future steps: progress made, items found, counters, field values already collected.',
-      },
-      plan: {
-        type: 'string',
-        description:
-          'Your immediate next goal — what action you will take and why.',
-      },
-      action_name: {
-        type: 'string',
-        description: 'Which action to execute.',
-        enum: toolNames,
-      },
-      ...actionProperties,
-    };
-
-    return {
-      type: 'function',
-      function: {
-        name: AGENT_STEP_FN,
-        description: `Execute one agent step. Choose an action and provide reasoning.\n\nAvailable actions:\n${toolDescriptions}`,
-        parameters: {
-          type: 'object',
-          properties: allProperties,
-          required: Object.keys(allProperties),
-          additionalProperties: false,
-        },
-        strict: true,
-      },
-    };
-  }
-
-  // ─── Build Messages ────────────────────────────────────────
-
-  private buildMessages(
-    systemPrompt: string,
-    userMessage: string,
-    screenshot?: string,
-  ): any[] {
-    const messages: any[] = [
-      { role: 'system', content: systemPrompt },
-    ];
-
-    // User message — text or multimodal with screenshot
-    if (screenshot) {
-      messages.push({
-        role: 'user',
-        content: [
-          { type: 'text', text: userMessage },
-          {
-            type: 'image_url',
-            image_url: {
-              url: `data:image/jpeg;base64,${screenshot}`,
-              detail: 'low',
-            },
-          },
-        ],
-      });
-    } else {
-      messages.push({ role: 'user', content: userMessage });
-    }
-
-    return messages;
-  }
-
-  // ─── Parse Response ────────────────────────────────────────
-
-  private parseAgentStepResponse(
-    data: any,
-    tools: ToolDefinition[],
-  ): ProviderResult {
-    const choice = data.choices?.[0];
-
-    if (!choice) {
-      logger.warn('OpenAIProvider', 'No choices in response');
-      return {
-        toolCalls: [
-          {
-            name: 'done',
-            args: { text: 'No response generated.', success: false },
-          },
-        ],
-        reasoning: { previousGoalEval: '', memory: '', plan: '' },
-        text: 'No response generated.',
-      };
-    }
-
-    const message = choice.message;
-    const toolCall = message?.tool_calls?.[0];
-
-    if (!toolCall?.function) {
-      logger.warn(
-        'OpenAIProvider',
-        'No tool call in response. Text:',
-        message?.content,
-      );
-      return {
-        toolCalls: [
-          {
-            name: 'done',
-            args: {
-              text: message?.content || 'No action taken.',
-              success: false,
-            },
-          },
-        ],
-        reasoning: { previousGoalEval: '', memory: '', plan: '' },
-        text: message?.content,
-      };
-    }
-
-    // OpenAI returns arguments as a JSON STRING — must parse
-    let args: Record<string, any>;
-    try {
-      args = JSON.parse(toolCall.function.arguments);
-    } catch (err) {
-      logger.error(
-        'OpenAIProvider',
-        'Failed to parse tool arguments:',
-        toolCall.function.arguments,
-      );
-      return {
-        toolCalls: [
-          {
-            name: 'done',
-            args: { text: 'Failed to parse AI response.', success: false },
-          },
-        ],
-        reasoning: { previousGoalEval: '', memory: '', plan: '' },
-      };
-    }
-
-    // Extract reasoning fields
-    const reasoning: AgentReasoning = {
-      previousGoalEval: args.previous_goal_eval || '',
-      memory: args.memory || '',
-      plan: args.plan || '',
-    };
-
-    // Extract action
-    const actionName = args.action_name;
-    if (!actionName) {
-      logger.warn(
-        'OpenAIProvider',
-        'No action_name in agent_step. Falling back to done.',
-      );
-      return {
-        toolCalls: [
-          {
-            name: 'done',
-            args: { text: 'Agent did not choose an action.', success: false },
-          },
-        ],
-        reasoning,
-        text: message?.content,
-      };
-    }
-
-    // Build action args: extract only the params that belong to the matched tool
-    const actionArgs: Record<string, any> = {};
-    const reservedKeys = new Set([...REASONING_FIELDS, 'action_name']);
-
-    const matchedTool = tools.find((t) => t.name === actionName);
-    if (matchedTool) {
-      for (const paramName of Object.keys(matchedTool.parameters)) {
-        if (args[paramName] !== undefined) {
-          actionArgs[paramName] = args[paramName];
-        }
-      }
-    } else {
-      for (const [key, value] of Object.entries(args)) {
-        if (!reservedKeys.has(key)) {
-          actionArgs[key] = value;
-        }
-      }
-    }
-
-    logger.info(
-      'OpenAIProvider',
-      `Parsed: action=${actionName}, plan="${reasoning.plan}"`,
-    );
-
-    return {
-      toolCalls: [{ name: actionName, args: actionArgs }],
-      reasoning,
-      text: message?.content,
-    };
-  }
-
-  // ─── Token Usage Extraction ─────────────────────────────────
-
-  /**
-   * Extracts token usage from OpenAI response and calculates estimated cost.
-   *
-   * Pricing (GPT-4.1-mini):
-   * - Input:  $0.40 / 1M tokens
-   * - Output: $1.60 / 1M tokens
-   */
-  private extractTokenUsage(data: any): TokenUsage | undefined {
-    const usage = data?.usage;
-    if (!usage) return undefined;
-
-    const promptTokens = usage.prompt_tokens ?? 0;
-    const completionTokens = usage.completion_tokens ?? 0;
-    const totalTokens = usage.total_tokens ?? promptTokens + completionTokens;
-
-    // Cost estimation based on GPT-4.1-mini pricing
-    const INPUT_COST_PER_M = 0.4;
-    const OUTPUT_COST_PER_M = 1.6;
-
-    const estimatedCostUSD =
-      (promptTokens / 1_000_000) * INPUT_COST_PER_M +
-      (completionTokens / 1_000_000) * OUTPUT_COST_PER_M;
-
-    return { promptTokens, completionTokens, totalTokens, estimatedCostUSD };
-  }
-}

package/src/providers/ProviderFactory.ts

@@ -1,36 +0,0 @@
-/**
- * ProviderFactory — Creates the appropriate AI provider based on config.
- *
- * Centralizes provider instantiation so AIAgent.tsx doesn't need to
- * know about individual provider implementations.
- */
-
-import type { AIProvider, AIProviderName } from '../core/types';
-import { GeminiProvider } from './GeminiProvider';
-import { OpenAIProvider } from './OpenAIProvider';
-
-export function createProvider(
-  provider: AIProviderName = 'gemini',
-  apiKey?: string,
-  model?: string,
-  proxyUrl?: string,
-  proxyHeaders?: Record<string, string>,
-): AIProvider {
-  switch (provider) {
-    case 'openai':
-      return new OpenAIProvider(
-        apiKey,
-        model || 'gpt-4.1-mini',
-        proxyUrl,
-        proxyHeaders,
-      );
-    case 'gemini':
-    default:
-      return new GeminiProvider(
-        apiKey,
-        model || 'gemini-2.5-flash',
-        proxyUrl,
-        proxyHeaders,
-      );
-  }
-}

package/src/services/AudioInputService.ts

@@ -1,226 +0,0 @@
-/**
- * AudioInputService — Real-time microphone capture for voice mode.
- *
- * Uses react-native-audio-api (Software Mansion) AudioRecorder for native
- * PCM streaming from the microphone. Each chunk is converted from Float32
- * to Int16 PCM and base64-encoded for the Gemini Live API.
- *
- * Echo cancellation is handled at the OS/hardware level via
- * react-native-incall-manager (VOICE_COMMUNICATION mode) — not in JS.
- *
- * Requires: react-native-audio-api (development build only, not Expo Go)
- */
-
-import { logger } from '../utils/logger';
-import { float32ToInt16Base64 } from '../utils/audioUtils';
-
-// ─── Types ─────────────────────────────────────────────────────
-
-export interface AudioInputConfig {
-  sampleRate?: number;
-  /** Number of samples per callback buffer (default: 4096) */
-  bufferLength?: number;
-  /** Callback with base64 PCM audio chunk */
-  onAudioChunk: (base64Audio: string) => void;
-  onError?: (error: string) => void;
-  onPermissionDenied?: () => void;
-}
-
-type RecordingStatus = 'idle' | 'recording' | 'paused';
-
-// ─── Service ───────────────────────────────────────────────────
-
-export class AudioInputService {
-  private config: AudioInputConfig;
-  private status: RecordingStatus = 'idle';
-  private recorder: any = null;
-
-  // Auto-recovery: detect when mic session dies after audio playback.
-  // This is a react-native-audio-api bug where AudioRecorder loses mic access
-  // after AudioBufferQueueSourceNode plays audio (audio session conflict).
-  private consecutiveSilentFrames = 0;
-  private isRecovering = false;
-  private static readonly SILENT_THRESHOLD = 0.01;
-  private static readonly SILENT_FRAMES_BEFORE_RESTART = 15;
-
-  constructor(config: AudioInputConfig) {
-    this.config = config;
-  }
-
-  // ─── Lifecycle ─────────────────────────────────────────────
-
-  async start(): Promise<boolean> {
-    try {
-      // Lazy-load react-native-audio-api (optional peer dependency)
-      let audioApi: any;
-      try {
-        const { NativeModules } = require('react-native');
-        if (!NativeModules.AudioApiModule) {
-          const msg =
-            '[mobileai] react-native-audio-api native module not found. '
-            + 'Voice mode requires a development build (not Expo Go).';
-          logger.warn('AudioInput', msg);
-          this.config.onError?.(msg);
-          return false;
-        }
-        // Static require — Metro needs a literal string for bundling.
-        audioApi = require('react-native-audio-api');
-      } catch {
-        const msg =
-          'Voice mode requires react-native-audio-api. Install with: npm install react-native-audio-api';
-        logger.error('AudioInput', msg);
-        this.config.onError?.(msg);
-        return false;
-      }
-
-      // Request mic permission (Android)
-      try {
-        const { PermissionsAndroid, Platform } = require('react-native');
-        if (Platform.OS === 'android') {
-          const result = await PermissionsAndroid.request(
-            PermissionsAndroid.PERMISSIONS.RECORD_AUDIO
-          );
-          if (result !== PermissionsAndroid.RESULTS.GRANTED) {
-            logger.warn('AudioInput', 'Microphone permission denied');
-            this.config.onPermissionDenied?.();
-            return false;
-          }
-        }
-      } catch {
-        // Permission check failed — continue and let native layer handle it
-      }
-
-      // Create AudioRecorder
-      this.recorder = new audioApi.AudioRecorder();
-      this.consecutiveSilentFrames = 0;
-
-      const sampleRate = this.config.sampleRate || 16000;
-      const bufferLength = this.config.bufferLength || 4096;
-
-      // Register audio data callback
-      let frameCount = 0;
-      this.recorder.onAudioReady(
-        { sampleRate, bufferLength, channelCount: 1 },
-        (event: any) => {
-          frameCount++;
-          try {
-            // event.buffer is an AudioBuffer — get Float32 channel data
-            const float32Data = event.buffer.getChannelData(0);
-
-            // Measure peak amplitude for diagnostics + silent detection
-            let maxAmp = 0;
-            for (let i = 0; i < float32Data.length; i++) {
-              const abs = Math.abs(float32Data[i] || 0);
-              if (abs > maxAmp) maxAmp = abs;
-            }
-
-            // Diagnostic: log amplitude on first 5 frames, then every 10th
-            if (frameCount <= 5 || frameCount % 10 === 0) {
-              logger.info('AudioInput', `🔬 Frame #${frameCount}: maxAmp=${maxAmp.toFixed(6)}, samples=${float32Data.length}`);
-            }
-
-            // ─── Auto-Recovery: Silent mic detection ─────────────
-            // After audio playback, react-native-audio-api's AudioRecorder
-            // can lose its mic session (all-zero frames). Detect this and
-            // restart the recorder to re-acquire the audio session.
-            if (maxAmp < AudioInputService.SILENT_THRESHOLD) {
-              this.consecutiveSilentFrames++;
-              if (
-                this.consecutiveSilentFrames >= AudioInputService.SILENT_FRAMES_BEFORE_RESTART &&
-                !this.isRecovering
-              ) {
-                this.isRecovering = true;
-                logger.warn('AudioInput', `⚠️ ${this.consecutiveSilentFrames} silent frames — restarting recorder...`);
-                this.restartRecorder().then(() => {
-                  this.isRecovering = false;
-                  this.consecutiveSilentFrames = 0;
-                  logger.info('AudioInput', '✅ Recorder restarted — mic session re-acquired');
-                }).catch((err: any) => {
-                  this.isRecovering = false;
-                  logger.error('AudioInput', `❌ Recorder restart failed: ${err?.message || err}`);
-                });
-                return; // Skip this frame
-              }
-            } else {
-              // Got real audio — reset counter
-              if (this.consecutiveSilentFrames > 5) {
-                logger.info('AudioInput', `🎤 Mic recovered after ${this.consecutiveSilentFrames} silent frames`);
-              }
-              this.consecutiveSilentFrames = 0;
-            }
-
-            const base64Chunk = float32ToInt16Base64(float32Data);
-            if (frameCount <= 5 || frameCount % 10 === 0) {
-              logger.info('AudioInput', `🎤 Frame #${frameCount}: chunk=${base64Chunk.length} chars, calling onAudioChunk...`);
-            }
-            this.config.onAudioChunk(base64Chunk);
-          } catch (err: any) {
-            logger.error('AudioInput', `Frame processing error: ${err.message}`);
-          }
-        }
-      );
-
-      // Register error callback
-      this.recorder.onError((error: any) => {
-        logger.error('AudioInput', `Recorder error: ${error.message || error}`);
-        this.config.onError?.(error.message || String(error));
-      });
-
-      // Start recording
-      this.recorder.start();
-      this.status = 'recording';
-      logger.info('AudioInput', `Streaming started (${sampleRate}Hz, bufLen=${bufferLength})`);
-      return true;
-    } catch (error: any) {
-      logger.error('AudioInput', `Failed to start: ${error.message}`);
-      this.config.onError?.(error.message);
-      return false;
-    }
-  }
-
-  async stop(): Promise<void> {
-    try {
-      if (this.recorder && this.status !== 'idle') {
-        this.recorder.clearOnAudioReady();
-        this.recorder.clearOnError();
-        this.recorder.stop();
-      }
-      this.recorder = null;
-      this.status = 'idle';
-      this.consecutiveSilentFrames = 0;
-      logger.info('AudioInput', 'Streaming stopped');
-    } catch (error: any) {
-      logger.error('AudioInput', `Failed to stop: ${error.message}`);
-      this.recorder = null;
-      this.status = 'idle';
-    }
-  }
-
-  // ─── Auto-Recovery ─────────────────────────────────────────
-
-  /**
-   * Restart the recorder to re-acquire the audio session.
-   * Fixes react-native-audio-api bug where AudioRecorder loses mic access
-   * after AudioBufferQueueSourceNode plays audio.
-   */
-  private async restartRecorder(): Promise<void> {
-    logger.info('AudioInput', '🔄 Restarting recorder for mic recovery...');
-    await this.stop();
-    // Brief pause to let the audio system release resources
-    await new Promise(resolve => setTimeout(resolve, 300));
-    const ok = await this.start();
-    if (!ok) {
-      throw new Error('Recorder restart failed');
-    }
-  }
-
-  // ─── Status ───────────────────────────────────────────────
-
-  get isRecording(): boolean {
-    return this.status === 'recording';
-  }
-
-  get currentStatus(): RecordingStatus {
-    return this.status;
-  }
-}