@mobileai/react-native 0.4.2 → 0.4.4

package/src/core/AgentRuntime.ts

@@ -21,6 +21,7 @@ import type {
   ExecutionResult,
   ToolDefinition,
   ActionDefinition,
+  TokenUsage,
 } from './types';
 
 const DEFAULT_MAX_STEPS = 10;
@@ -185,9 +186,14 @@ export class AgentRuntime {
         }
         try {
           const params = args.params ? (typeof args.params === 'string' ? JSON.parse(args.params) : args.params) : undefined;
-          this.navRef.navigate(args.screen, params);
+          // Case-insensitive screen name matching
+          const availableRoutes = this.getRouteNames();
+          const matchedScreen = availableRoutes.find(
+            r => r.toLowerCase() === args.screen.toLowerCase()
+          ) || args.screen;
+          this.navRef.navigate(matchedScreen, params);
           await new Promise(resolve => setTimeout(resolve, 500));
-          return `✅ Navigated to "${args.screen}"${params ? ` with params: ${JSON.stringify(params)}` : ''}`;
+          return `✅ Navigated to "${matchedScreen}"${params ? ` with params: ${JSON.stringify(params)}` : ''}`;
         } catch (error: any) {
           return `❌ Navigation error: ${error.message}. Available screens: ${this.getRouteNames().join(', ')}`;
         }
@@ -225,6 +231,20 @@ export class AgentRuntime {
         return `❓ ${args.question}`;
       },
     });
+
+    // capture_screenshot — on-demand visual capture (for image/video content questions)
+    this.tools.set('capture_screenshot', {
+      name: 'capture_screenshot',
+      description: 'Capture a screenshot of the current screen. Use when the user asks about visual content (images, videos, colors, layout appearance) that cannot be determined from the element tree alone.',
+      parameters: {},
+      execute: async () => {
+        const screenshot = await this.captureScreenshot();
+        if (screenshot) {
+          return `✅ Screenshot captured (${Math.round(screenshot.length / 1024)}KB). Visual content is now available for analysis.`;
+        }
+        return '❌ Screenshot capture failed. react-native-view-shot may not be installed.';
+      },
+    });
   }
 
   // ─── Action Registration (useAction hook) ──────────────────
@@ -319,6 +339,69 @@ export class AgentRuntime {
     }
   }
 
+  // ─── Screenshot Capture (optional react-native-view-shot) ─────
+
+  /**
+   * Captures the current screen as a base64 JPEG for Gemini vision.
+   * Uses react-native-view-shot as an optional peer dependency.
+   * Returns null if the library is not installed (graceful fallback).
+   */
+  private async captureScreenshot(): Promise<string | undefined> {
+    try {
+      const viewShot = require('react-native-view-shot');
+      const captureRef = viewShot.captureRef || viewShot.default?.captureRef;
+      if (!captureRef || !this.rootRef) return undefined;
+
+      const uri = await captureRef(this.rootRef, {
+        format: 'jpg',
+        quality: 0.4,
+        width: 720,
+        result: 'base64',
+      });
+
+      logger.info('AgentRuntime', `Screenshot captured (${Math.round((uri?.length || 0) / 1024)}KB base64)`);
+      return uri || undefined;
+    } catch (error: any) {
+      // Detect missing dependency vs runtime failure
+      if (error.message?.includes('Cannot find module') || error.code === 'MODULE_NOT_FOUND') {
+        logger.warn('AgentRuntime', 'Screenshot requires react-native-view-shot. Install with: npx expo install react-native-view-shot');
+      } else {
+        logger.debug('AgentRuntime', `Screenshot skipped: ${error.message}`);
+      }
+      return undefined;
+    }
+  }
+
+  // ─── Screen Context for Voice Mode ──────────────────────
+
+  /**
+   * Get current screen context as formatted text.
+   * Used by voice mode: sent once at connect + after each tool call.
+   * Follows page-agent pattern: tree in user prompt, not system instructions.
+   */
+  public getScreenContext(): string {
+    try {
+      const walkResult = walkFiberTree(this.rootRef, this.getWalkConfig());
+      const screenName = this.getCurrentScreenName();
+      const screen = dehydrateScreen(
+        screenName,
+        this.getRouteNames(),
+        walkResult.elementsText,
+        walkResult.interactives,
+      );
+
+      return `<screen_update>
+Current Screen: ${screenName}
+Available Screens: ${this.getRouteNames().join(', ')}
+
+${screen.elementsText}
+</screen_update>`;
+    } catch (error: any) {
+      logger.error('AgentRuntime', `getScreenContext failed: ${error.message}`);
+      return '<screen_update>Error reading screen</screen_update>';
+    }
+  }
+
   // ─── Build Tools Array for Provider ────────────────────────
 
   private buildToolsForProvider(): ToolDefinition[] {
@@ -349,6 +432,28 @@ export class AgentRuntime {
     return allTools;
   }
 
+  /** Public accessor for voice mode — returns all registered tool definitions. */
+  public getTools(): ToolDefinition[] {
+    return this.buildToolsForProvider();
+  }
+
+  /** Execute a tool by name (for voice mode tool calls from WebSocket). */
+  public async executeTool(name: string, args: Record<string, any>): Promise<string> {
+    const tool = this.tools.get(name) ||
+      this.buildToolsForProvider().find(t => t.name === name);
+    if (!tool) {
+      return `❌ Unknown tool: ${name}`;
+    }
+    try {
+      const result = await tool.execute(args);
+      logger.info('AgentRuntime', `Voice tool executed: ${name} → ${result}`);
+      return result;
+    } catch (error: any) {
+      logger.error('AgentRuntime', `Voice tool error: ${name} — ${error.message}`);
+      return `❌ Tool "${name}" failed: ${error.message}`;
+    }
+  }
+
   // ─── Walk Config (passes security settings to FiberTreeWalker) ─
 
   private getWalkConfig(): WalkConfig {
@@ -477,6 +582,14 @@ export class AgentRuntime {
     const maxSteps = this.config.maxSteps || DEFAULT_MAX_STEPS;
     const stepDelay = this.config.stepDelay ?? 300;
 
+    // Token usage accumulator for the entire task
+    const sessionUsage: TokenUsage = {
+      promptTokens: 0,
+      completionTokens: 0,
+      totalTokens: 0,
+      estimatedCostUSD: 0,
+    };
+
     // Inject conversational context if we are answering the AI's question
     let contextualMessage = userMessage;
     if (this.lastAskUserQuestion) {
@@ -523,6 +636,9 @@ export class AgentRuntime {
           step, maxSteps, contextualMessage, screenName, screenContent,
         );
 
+        // 4.5. Capture screenshot for Gemini vision (optional)
+        const screenshot = await this.captureScreenshot();
+
         // 5. Send to AI provider
         this.config.onStatusUpdate?.('Analyzing screen...');
         const systemPrompt = buildSystemPrompt(this.config.language || 'en');
@@ -535,8 +651,18 @@ export class AgentRuntime {
           contextMessage,
           tools,
           this.history,
+          screenshot,
         );
 
+        // Accumulate token usage
+        if (response.tokenUsage) {
+          sessionUsage.promptTokens += response.tokenUsage.promptTokens;
+          sessionUsage.completionTokens += response.tokenUsage.completionTokens;
+          sessionUsage.totalTokens += response.tokenUsage.totalTokens;
+          sessionUsage.estimatedCostUSD += response.tokenUsage.estimatedCostUSD;
+          this.config.onTokenUsage?.(response.tokenUsage);
+        }
+
         // 6. Process tool calls
         if (!response.toolCalls || response.toolCalls.length === 0) {
           logger.warn('AgentRuntime', 'No tool calls in response. Text:', response.text);
@@ -544,6 +670,7 @@ export class AgentRuntime {
             success: true,
             message: response.text || 'Task completed.',
             steps: this.history,
+            tokenUsage: sessionUsage,
           };
           await this.config.onAfterTask?.(result);
           return result;
@@ -631,6 +758,7 @@ export class AgentRuntime {
         success: false,
         message: `Reached maximum steps (${maxSteps}) without completing the task.`,
         steps: this.history,
+        tokenUsage: sessionUsage,
       };
       await this.config.onAfterTask?.(result);
       return result;
@@ -640,6 +768,7 @@ export class AgentRuntime {
         success: false,
         message: `Error: ${error.message}`,
         steps: this.history,
+        tokenUsage: sessionUsage,
       };
       await this.config.onAfterTask?.(result);
       return result;

package/src/core/systemPrompt.ts

@@ -160,3 +160,65 @@ memory: "Added 2x Margherita pizza. Cart total visible."
 plan: "Call done to report the cart contents to the user."
 </output>`;
 }
+
+/**
+ * Voice-optimized system prompt for the Gemini Live API.
+ *
+ * Includes the same screen format and tool semantics as text mode,
+ * but condensed for voice context and with guardrails against
+ * unprompted actions.
+ */
+export function buildVoiceSystemPrompt(
+  language: string,
+  userInstructions?: string,
+): string {
+  const isArabic = language === 'ar';
+
+  let prompt = `You are a voice-controlled AI agent operating a React Native mobile app. You can see the screen content and interact with UI elements using tools.
+
+<language>
+${isArabic ? 'Respond in Arabic.' : 'Respond in English.'}
+Use the same language as the user.
+</language>
+
+<screen_format>
+You receive periodic screen updates showing the current UI. Interactive elements appear as:
+[index]<type attrs>label</type>
+
+- index: numeric ID for interaction (use with tap/type tools)
+- type: element type (pressable, text-input, switch)
+- attrs: state like value="true", checked="false", role="switch"
+- label: visible text content
+
+Only elements with [index] are interactive. Text without [] is display-only.
+Example: [5]<switch value="true">Order Updates</switch> means element 5 is a switch currently ON.
+</screen_format>
+
+<tools>
+Available tools:
+- tap(index): Tap an element. For switches, this toggles their value.
+- type(index, text): Type text into a text-input.
+- navigate(screen): Navigate to a named screen.
+- done(text, success): Complete the task with a spoken response.
+- ask_user(question): Ask the user for clarification.
+</tools>
+
+<rules>
+CRITICAL ACTION RULES:
+- ONLY perform actions (tap, type, navigate) when the user explicitly asks you to do something.
+- NEVER tap or navigate on your own initiative — wait for the user's voice command.
+- When the user asks a question about what's on screen, answer verbally via done(). Do NOT tap anything.
+- When the user asks to toggle/enable/disable something, find the matching element by its label and use tap(index).
+- When a screen update arrives, do NOT interact with elements unless the user asked you to.
+- Use element indexes from the most recent screen update — they refresh every few seconds.
+- For switches: tap(index) toggles the value. You do NOT need to find a separate button.
+- Keep spoken responses concise — the user is listening, not reading.
+</rules>`;
+
+  // Append user-provided instructions if any
+  if (userInstructions?.trim()) {
+    prompt += `\n\n<app_instructions>\n${userInstructions.trim()}\n</app_instructions>`;
+  }
+
+  return prompt;
+}

package/src/core/types.ts

@@ -2,6 +2,10 @@
  * Core types for the page-agent-style React Native AI SDK.
  */
 
+// ─── Agent Modes ──────────────────────────────────────────────
+
+export type AgentMode = 'text' | 'voice';
+
 // ─── Interactive Element (discovered from Fiber tree) ─────────
 
 export type ElementType = 'pressable' | 'text-input' | 'switch' | 'scrollable';
@@ -134,6 +138,12 @@ export interface AgentConfig {
    */
   onStatusUpdate?: (status: string) => void;
 
+  /**
+   * Called after each step with token usage data.
+   * Use to track cost, enforce budgets, or display usage to the user.
+   */
+  onTokenUsage?: (usage: TokenUsage) => void;
+
   /**
    * Callback for when agent needs user input (ask_user tool).
    * Mirrors page-agent: the agent loop blocks until the user responds.
@@ -174,6 +184,8 @@ export interface ExecutionResult {
   success: boolean;
   message: string;
   steps: AgentStep[];
+  /** Accumulated token usage for the entire task */
+  tokenUsage?: TokenUsage;
 }
 
 // ─── Tool Types ───────────────────────────────────────────────
@@ -213,6 +225,20 @@ export interface AgentReasoning {
   plan: string;
 }
 
+// ─── Token Usage ──────────────────────────────────────────────
+
+/** Token usage metrics for cost tracking. */
+export interface TokenUsage {
+  /** Tokens in the input prompt */
+  promptTokens: number;
+  /** Tokens generated by the model */
+  completionTokens: number;
+  /** Total tokens (prompt + completion) */
+  totalTokens: number;
+  /** Estimated cost in USD (based on model pricing) */
+  estimatedCostUSD: number;
+}
+
 /** Result from the AI provider's generateContent call. */
 export interface ProviderResult {
   /** Extracted action tool call (action_name + params). */
@@ -221,6 +247,8 @@ export interface ProviderResult {
   reasoning: AgentReasoning;
   /** Raw text response (if any). */
   text?: string;
+  /** Token usage for this specific call */
+  tokenUsage?: TokenUsage;
 }
 
 export interface AIProvider {
@@ -229,5 +257,7 @@ export interface AIProvider {
     userMessage: string,
     tools: ToolDefinition[],
     history: AgentStep[],
+    /** Optional base64-encoded JPEG screenshot for vision */
+    screenshot?: string,
   ): Promise<ProviderResult>;
 }

package/src/index.ts

@@ -11,12 +11,28 @@ export { AIAgent } from './components/AIAgent';
 // ─── Hooks ───────────────────────────────────────────────────
 export { useAction } from './hooks/useAction';
 
+// ─── Services ────────────────────────────────────────────────
+export { VoiceService } from './services/VoiceService';
+export { AudioInputService } from './services/AudioInputService';
+export { AudioOutputService } from './services/AudioOutputService';
+
+// ─── Utilities ───────────────────────────────────────────────
+export { logger } from './utils/logger';
+
 // ─── Types ───────────────────────────────────────────────────
 export type {
   AgentConfig,
+  AgentMode,
   ExecutionResult,
   InteractiveElement,
   DehydratedScreen,
   ToolDefinition,
   ActionDefinition,
+  TokenUsage,
 } from './core/types';
+
+export type {
+  VoiceServiceConfig,
+  VoiceServiceCallbacks,
+  VoiceStatus,
+} from './services/VoiceService';