@mobileai/react-native 0.9.5 → 0.9.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mobileai/react-native",
-  "version": "0.9.5",
+  "version": "0.9.10",
   "description": "Build autonomous AI agents for React Native and Expo apps. Provides AI-native UI traversal, tool calling, and structured reasoning.",
   "main": "./lib/module/index.js",
   "source": "./src/index.ts",
@@ -169,6 +169,9 @@
   },
   "jest": {
     "preset": "react-native",
+    "setupFilesAfterEnv": [
+      "<rootDir>/src/__tests__/setup.ts"
+    ],
     "testMatch": [
       "**/?(*.)+(spec|test).[jt]s?(x)"
     ],

package/src/__cli_tmp__.tsx

@@ -0,0 +1,9 @@
+import { useAction } from './hooks/useAction';
+
+export function CheckoutScreen() {
+  useAction('checkout_cart', 'Process checkout', {
+    amount: { type: 'number', description: 'Total amount' },
+    currency: { type: 'string', description: 'Currency code', enum: ['USD', 'EUR'] },
+    isExpress: { type: 'boolean', description: 'Express checkout' }
+  }, async () => {});
+}

package/src/cli/generate-intents.ts

@@ -0,0 +1,140 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import * as path from 'path';
+import { parse } from '@babel/parser';
+import traverse from '@babel/traverse';
+// @ts-ignore - no types installed
+import glob from 'glob';
+
+// Define the schema format for our extracted intents
+export interface ExtractedIntent {
+  name: string;
+  description: string;
+  parameters: Record<string, any>;
+  sourceFile: string;
+}
+
+/**
+ * Validates and statically extracts `useAction` and `registerAction` definitions
+ * from a target directory by parsing the AST of all TS/JS files.
+ */
+export function extractIntentsFromAST(sourceDir: string): ExtractedIntent[] {
+  const files = glob.sync(`${sourceDir}/**/*.{ts,tsx,js,jsx}`, {
+    ignore: ['**/node_modules/**', '**/*.d.ts', '**/__tests__/**']
+  });
+
+  const intents: ExtractedIntent[] = [];
+
+  for (const file of files) {
+    const code = fs.readFileSync(file, 'utf-8');
+    
+    // Quick heuristic: ignore files that don't even talk about useAction
+    if (!code.includes('useAction') && !code.includes('registerAction')) {
+      continue;
+    }
+
+    try {
+      const ast = parse(code, {
+        sourceType: 'module',
+        plugins: ['jsx', 'typescript'],
+      });
+
+      traverse(ast, {
+        CallExpression(pathNode: any) {
+          const callee = pathNode.node.callee;
+          if (
+            callee.type === 'Identifier' &&
+            (callee.name === 'useAction' || callee.name === 'registerAction')
+          ) {
+            const args = pathNode.node.arguments;
+            if (args.length >= 2) {
+              const nameArg = args[0];
+              const descArg = args[1];
+              const schemaArg = args[2];
+
+              // We only process if name and desc are static string literals
+              if (nameArg.type === 'StringLiteral' && descArg.type === 'StringLiteral') {
+                const name = nameArg.value;
+                const description = descArg.value;
+                let parameters: Record<string, any> = {};
+
+                // Parse schema object if provided
+                if (schemaArg && schemaArg.type === 'ObjectExpression') {
+                  parameters = parseObjectExpression(schemaArg);
+                }
+
+                intents.push({
+                  name,
+                  description,
+                  parameters,
+                  sourceFile: path.relative(process.cwd(), file)
+                });
+              }
+            }
+          }
+        }
+      });
+    } catch (error: any) {
+      console.warn(`[WARN] Skipping file ${file} due to parse error: ${error.message}`);
+    }
+  }
+
+  return intents;
+}
+
+/**
+ * Naively converts an AST ObjectExpression back to a JS Runtime object.
+ * Assumes the schema is statically defined (no variables/computed keys).
+ */
+function parseObjectExpression(node: any): any {
+  const result: any = {};
+  for (const prop of node.properties) {
+    if (prop.type === 'ObjectProperty' && prop.key.type === 'Identifier') {
+      const key = prop.key.name;
+      
+      if (prop.value.type === 'StringLiteral') {
+        result[key] = prop.value.value;
+      } else if (prop.value.type === 'NumericLiteral') {
+        result[key] = prop.value.value;
+      } else if (prop.value.type === 'BooleanLiteral') {
+        result[key] = prop.value.value;
+      } else if (prop.value.type === 'ObjectExpression') {
+        result[key] = parseObjectExpression(prop.value);
+      } else if (prop.value.type === 'ArrayExpression') {
+        result[key] = prop.value.elements.map((el: any) => {
+          if (el.type === 'StringLiteral') return el.value;
+          if (el.type === 'NumericLiteral') return el.value;
+          return null;
+        }).filter(Boolean);
+      }
+    }
+  }
+  return result;
+}
+
+async function main() {
+  const sourceDir = process.argv[2] || 'src';
+  const outPath = process.argv[3] || 'intent-manifest.json';
+  
+  const absoluteSource = path.resolve(process.cwd(), sourceDir);
+  const absoluteOut = path.resolve(process.cwd(), outPath);
+
+  console.log(`Scanning ${absoluteSource} for AI Actions...`);
+  
+  const intents = extractIntentsFromAST(absoluteSource);
+  
+  console.log(`Found ${intents.length} actions.`);
+  intents.forEach(i => console.log(`  - ${i.name} (from ${i.sourceFile})`));
+  
+  fs.writeFileSync(absoluteOut, JSON.stringify(intents, null, 2));
+  console.log(`\n✅ Wrote intent manifest to ${outPath}`);
+}
+
+// Run if called directly
+if (require.main === module) {
+  main().catch(err => {
+    console.error(err);
+    process.exit(1);
+  });
+}

package/src/cli/generate-swift.ts

@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+export function generateSwiftCode(manifestPath: string, appScheme: string): string {
+  const manifests = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+
+  let swiftCode = `import AppIntents
+import UIKit
+
+// Auto-generated by react-native-ai-agent
+// Do not edit manually.
+
+`;
+
+  for (const intent of manifests) {
+    // Convert snake_case to CamelCaseIntent
+    const structName = intent.name
+      .split('_')
+      .map((w: string) => w.charAt(0).toUpperCase() + w.slice(1))
+      .join('') + 'Intent';
+
+    swiftCode += `@available(iOS 16.0, *)\n`;
+    swiftCode += `struct ${structName}: AppIntent {\n`;
+    swiftCode += `    static var title: LocalizedStringResource = "${intent.name}"\n`;
+    swiftCode += `    static var description = IntentDescription("${intent.description.replace(/"/g, '\\"')}")\n`;
+    swiftCode += `    static var openAppWhenRun: Bool = true\n\n`;
+
+    // Parameters
+    const params = intent.parameters || {};
+    for (const [key, param] of Object.entries<any>(params)) {
+      let swiftType = 'String';
+      if (param.type === 'number') swiftType = 'Double';
+      if (param.type === 'boolean') swiftType = 'Bool';
+      
+      swiftCode += `    @Parameter(title: "${param.description || key}")\n`;
+      swiftCode += `    var ${key}: ${swiftType}?\n\n`;
+    }
+
+    // Perform function
+    swiftCode += `    @MainActor\n`;
+    swiftCode += `    func perform() async throws -> some IntentResult {\n`;
+    swiftCode += `        var components = URLComponents()\n`;
+    swiftCode += `        components.scheme = "${appScheme}"\n`;
+    swiftCode += `        components.host = "ai-action"\n`;
+    swiftCode += `        components.path = "/${intent.name}"\n`;
+    
+    if (Object.keys(params).length > 0) {
+      swiftCode += `        var queryItems: [URLQueryItem] = []\n`;
+      for (const [key, _param] of Object.entries<any>(params)) {
+        swiftCode += `        if let val = ${key} {\n`;
+        swiftCode += `            queryItems.append(URLQueryItem(name: "${key}", value: String(describing: val)))\n`;
+        swiftCode += `        }\n`;
+      }
+      swiftCode += `        components.queryItems = queryItems\n`;
+    }
+
+    swiftCode += `        if let url = components.url {\n`;
+    swiftCode += `            await UIApplication.shared.open(url)\n`;
+    swiftCode += `        }\n`;
+    swiftCode += `        return .result()\n`;
+    swiftCode += `    }\n`;
+    swiftCode += `}\n\n`;
+  }
+
+  // AppShortcuts provider (exposes intents to Siri automatically)
+  if (manifests.length > 0) {
+    swiftCode += `@available(iOS 16.0, *)\n`;
+    swiftCode += `struct MobileAIAppShortcuts: AppShortcutsProvider {\n`;
+    swiftCode += `    static var appShortcuts: [AppShortcut] {\n`;
+    for (const intent of manifests) {
+       const structName = intent.name.split('_').map((w: string) => w.charAt(0).toUpperCase() + w.slice(1)).join('') + 'Intent';
+       // Generate a basic short phrase mapping
+       swiftCode += `        AppShortcut(intent: ${structName}(), phrases: ["\\\\(.applicationName) ${intent.name.replace(/_/g, ' ')}"])\n`;
+    }
+    swiftCode += `    }\n`;
+    swiftCode += `}\n`;
+  }
+
+  return swiftCode;
+}
+
+async function main() {
+  const manifestPath = process.argv[2] || 'intent-manifest.json';
+  const appScheme = process.argv[3] || 'mobileai';
+  const outPath = process.argv[4] || 'ios/MobileAIPilotIntents.swift';
+  
+  const absoluteManifest = path.resolve(process.cwd(), manifestPath);
+  const absoluteOut = path.resolve(process.cwd(), outPath);
+
+  if (!fs.existsSync(absoluteManifest)) {
+    console.error(`❌ Manifest file not found at ${absoluteManifest}`);
+    process.exit(1);
+  }
+
+  console.log(`Generating Swift code for schema ${appScheme}...`);
+  const swiftCode = generateSwiftCode(absoluteManifest, appScheme);
+  
+  // Ensure dir exists
+  const targetDir = path.dirname(absoluteOut);
+  if (!fs.existsSync(targetDir)) {
+    fs.mkdirSync(targetDir, { recursive: true });
+  }
+
+  fs.writeFileSync(absoluteOut, swiftCode);
+  console.log(`✅ Wrote Swift AppIntents to ${outPath}`);
+}
+
+// Run if called directly
+if (require.main === module) {
+  main().catch(err => {
+    console.error(err);
+    process.exit(1);
+  });
+}

package/src/components/AIAgent.tsx

@@ -171,20 +171,17 @@ interface AIAgentProps {
    */
   useScreenMap?: boolean;
 
-  // ── Analytics (opt-in) ── @internal — requires api.mobileai.dev ──
+  // ── Analytics (opt-in) ──
 
   /**
-   * @internal Requires api.mobileai.dev — not yet available.
    * Publishable analytics key (mobileai_pub_xxx).
    */
   analyticsKey?: string;
   /**
-   * @internal Requires api.mobileai.dev — not yet available.
    * Proxy URL for enterprise customers — routes events through your backend.
    */
   analyticsProxyUrl?: string;
   /**
-   * @internal Requires api.mobileai.dev — not yet available.
    * Custom headers for analyticsProxyUrl (e.g., auth tokens).
    */
   analyticsProxyHeaders?: Record<string, string>;

package/src/components/AgentChatBar.tsx

@@ -150,9 +150,8 @@ function AudioControlButton({
  */
 let SpeechModule: any = null;
 try {
-  // Dynamic require prevents Metro from statically resolving this optional dep
-  const speechModuleName = ['expo', 'speech', 'recognition'].join('-');
-  SpeechModule = require(speechModuleName);
+  // Static require — Metro needs a literal string for bundling.
+  SpeechModule = require('expo-speech-recognition');
 } catch {
   // Not installed — dictation button won't appear
 }

package/src/components/HighlightOverlay.tsx

@@ -53,7 +53,7 @@ export function HighlightOverlay() {
   return (
     <View style={StyleSheet.absoluteFill} pointerEvents="box-none">
       {/* Invisible pressable to dismiss on tap anywhere */}
-      <Pressable style={StyleSheet.absoluteFill} onPress={() => setHighlight(null)} />
+      <Pressable testID="highlight-close-zone" style={StyleSheet.absoluteFill} onPress={() => setHighlight(null)} />
 
       {/* The Animated Ring */}
       <Animated.View

package/src/core/ActionRegistry.ts

@@ -0,0 +1,105 @@
+import type { ActionDefinition, ActionParameterDef } from './types';
+
+export interface MCPToolDeclaration {
+  name: string;
+  description: string;
+  inputSchema: {
+    type: string;
+    properties: Record<string, any>;
+    required: string[];
+  };
+}
+
+/**
+ * A central registry for all actions registered via `useAction`.
+ * This acts as the single source of truth for:
+ * 1. The in-app AI Agent (AgentRuntime)
+ * 2. The MCP Server (external agents)
+ * 3. iOS App Intents (Siri)
+ * 4. Android AppFunctions (Gemini)
+ */
+export class ActionRegistry {
+  private actions = new Map<string, ActionDefinition>();
+  private listeners = new Set<() => void>();
+
+  /** Register a new action definition */
+  register(action: ActionDefinition): void {
+    this.actions.set(action.name, action);
+    this.notify();
+  }
+
+  /** Unregister an action by name */
+  unregister(name: string): void {
+    this.actions.delete(name);
+    this.notify();
+  }
+
+  /** Get a specific action by name */
+  get(name: string): ActionDefinition | undefined {
+    return this.actions.get(name);
+  }
+
+  /** Get all registered actions */
+  getAll(): ActionDefinition[] {
+    return Array.from(this.actions.values());
+  }
+
+  /** Clear all registered actions (useful for testing) */
+  clear(): void {
+    this.actions.clear();
+    this.notify();
+  }
+
+  /**
+   * Subscribe to changes (e.g. when a new screen mounts and registers actions).
+   * Useful for the MCP server to re-announce tools.
+   */
+  onChange(listener: () => void): () => void {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  }
+
+  /** Serialize all actions as strictly-typed MCP tool declarations */
+  toMCPTools(): MCPToolDeclaration[] {
+    return this.getAll().map((a) => ({
+      name: a.name,
+      description: a.description,
+      inputSchema: this.buildInputSchema(a.parameters),
+    }));
+  }
+
+  private buildInputSchema(params: Record<string, string | ActionParameterDef>) {
+    const properties: Record<string, any> = {};
+    const required: string[] = [];
+
+    for (const [key, val] of Object.entries(params)) {
+      if (typeof val === 'string') {
+        // Backward compatibility: passing a string means it's a required string param.
+        properties[key] = { type: 'string', description: val };
+        required.push(key);
+      } else {
+        // New strict parameter definition
+        properties[key] = { type: val.type, description: val.description };
+        if (val.enum) {
+          properties[key].enum = val.enum;
+        }
+        if (val.required !== false) {
+          required.push(key);
+        }
+      }
+    }
+
+    return { type: 'object', properties, required };
+  }
+
+  private notify() {
+    this.listeners.forEach((l) => l());
+  }
+}
+
+// Export a singleton instance. 
+// This allows background channels (like App Intents bridging) to access actions 
+// even if the React tree hasn't accessed the AIAgent context yet.
+export const actionRegistry = new ActionRegistry();

package/src/core/AgentRuntime.ts

@@ -35,9 +35,9 @@ import type {
   AgentStep,
   ExecutionResult,
   ToolDefinition,
-  ActionDefinition,
   TokenUsage,
 } from './types';
+import { actionRegistry } from './ActionRegistry';
 
 const DEFAULT_MAX_STEPS = 25;
 
@@ -49,7 +49,6 @@ export class AgentRuntime {
   private rootRef: any;
   private navRef: any;
   private tools: Map<string, ToolDefinition> = new Map();
-  private actions: Map<string, ActionDefinition> = new Map();
   private history: AgentStep[] = [];
   private isRunning = false;
   private isCancelRequested = false;
@@ -67,6 +66,10 @@ export class AgentRuntime {
   private graceTimer: ReturnType<typeof setTimeout> | null = null;
   private originalReportErrorsAsExceptions: boolean | undefined = undefined;
 
+  public getConfig(): AgentConfig {
+    return this.config;
+  }
+
   constructor(
     provider: AIProvider,
     config: AgentConfig,
@@ -340,17 +343,6 @@ export class AgentRuntime {
     }
   }
 
-  // ─── Action Registration (useAction hook) ──────────────────
-
-  registerAction(action: ActionDefinition): void {
-    this.actions.set(action.name, action);
-    logger.info('AgentRuntime', `Registered action: ${action.name}`);
-  }
-
-  unregisterAction(name: string): void {
-    this.actions.delete(name);
-  }
-
   // ─── Navigation Helpers ────────────────────────────────────
 
   /**
@@ -518,11 +510,8 @@ export class AgentRuntime {
    */
   private async captureScreenshot(): Promise<string | undefined> {
     try {
-      // Use variable to prevent Metro from statically resolving this optional dep.
-      // A bare `require('react-native-view-shot')` is traced at bundle time
-      // and crashes with "unknown module" when the package isn't installed.
-      const moduleName = ['react-native', 'view-shot'].join('-');
-      const viewShot = require(moduleName);
+      // Static require — Metro needs a literal string; the try/catch handles MODULE_NOT_FOUND.
+      const viewShot = require('react-native-view-shot');
       const captureRef = viewShot.captureRef || viewShot.default?.captureRef;
       if (!captureRef || !this.rootRef) return undefined;
 
@@ -645,16 +634,25 @@ ${screen.elementsText}
     const allTools = [...this.tools.values()];
 
     // Add registered actions as tools
-    for (const action of this.actions.values()) {
+    for (const action of actionRegistry.getAll()) {
+      const toolParams: Record<string, any> = {};
+      for (const [key, val] of Object.entries(action.parameters)) {
+        if (typeof val === 'string') {
+          toolParams[key] = { type: 'string', description: val, required: true };
+        } else {
+          toolParams[key] = { 
+            type: val.type, 
+            description: val.description, 
+            required: val.required !== false, 
+            enum: val.enum 
+          };
+        }
+      }
+
       allTools.push({
         name: action.name,
         description: action.description,
-        parameters: Object.fromEntries(
-          Object.entries(action.parameters).map(([key, typeStr]) => [
-            key,
-            { type: typeStr as any, description: key, required: true },
-          ]),
-        ),
+        parameters: toolParams,
         execute: async (args) => {
           try {
             const result = await action.handler(args);

package/src/core/MCPBridge.ts

@@ -41,23 +41,76 @@ export class MCPBridge {
     this.ws.onmessage = async (event) => {
       try {
         const data = JSON.parse(event.data);
-        if (data.type === 'request' && data.command && data.requestId) {
-          logger.info('MCPBridge', `Received task from MCP: "${data.command}"`);
-          
-          if (this.runtime.getIsRunning()) {
-            this.sendResponse(data.requestId, {
-              success: false,
-              message: 'Agent is already running a task. Please wait.',
-              steps: [],
-            });
-            return;
+        const serverMode = this.runtime.getConfig().mcpServerMode ?? 'auto';
+        const serverEnabled = serverMode === 'enabled' || (serverMode !== 'disabled' && __DEV__);
+
+        switch (data.type) {
+          case 'request': {
+            if (!data.command || !data.requestId) return;
+            logger.info('MCPBridge', `Received task from MCP: "${data.command}"`);
+            
+            if (this.runtime.getIsRunning()) {
+              this.sendResponse(data.requestId, {
+                success: false,
+                message: 'Agent is already running a task. Please wait.',
+                steps: [],
+              });
+              return;
+            }
+
+            // Execute the task using the SDK's existing runtime loop
+            const result = await this.runtime.execute(data.command);
+            
+            // Send result back to MCP server
+            this.sendResponse(data.requestId, result);
+            break;
+          }
+
+          case 'tools/list': {
+            if (!serverEnabled) {
+              this.sendResponse(data.requestId, { error: 'MCP server mode is disabled.' });
+              break;
+            }
+            
+            const tools = this.runtime.getTools().map(t => ({
+              name: t.name,
+              description: t.description,
+              inputSchema: {
+                type: 'object',
+                properties: t.parameters || {},
+                required: Object.entries(t.parameters || {})
+                  .filter(([_, p]: [string, any]) => p.required !== false)
+                  .map(([k]) => k),
+              }
+            }));
+            
+            this.sendResponse(data.requestId, { tools });
+            break;
           }
 
-          // Execute the task using the SDK's existing runtime loop
-          const result = await this.runtime.execute(data.command);
-          
-          // Send result back to MCP server
-          this.sendResponse(data.requestId, result);
+          case 'tools/call': {
+            if (!serverEnabled) {
+              this.sendResponse(data.requestId, { error: 'MCP server mode is disabled.' });
+              break;
+            }
+            try {
+              const result = await this.runtime.executeTool(data.name, data.arguments || {});
+              this.sendResponse(data.requestId, { result });
+            } catch (err: any) {
+              this.sendResponse(data.requestId, { error: err.message });
+            }
+            break;
+          }
+
+          case 'screen/state': {
+            if (!serverEnabled) {
+              this.sendResponse(data.requestId, { error: 'MCP server mode is disabled.' });
+              break;
+            }
+            const screen = this.runtime.getScreenContext();
+            this.sendResponse(data.requestId, { screen });
+            break;
+          }
         }
       } catch (err) {
         logger.error('MCPBridge', 'Error handling message:', err);

package/src/core/types.ts

@@ -114,9 +114,19 @@ export interface AgentConfig {
    */
   voiceProxyHeaders?: Record<string, string>;
 
-  model?: string;  /** Maximum steps per task */
+  model?: string;  
+  
+  /** Maximum steps per task */
   maxSteps?: number;
 
+  /**
+   * MCP server mode — controls whether external agents can discover and invoke actions.
+   * 'auto' (default): enabled in __DEV__, disabled in production
+   * 'enabled': always on (opt-in for production)
+   * 'disabled': always off
+   */
+  mcpServerMode?: 'auto' | 'enabled' | 'disabled';
+
   // ─── Element Gating ──
 
   /**
@@ -309,10 +319,21 @@ export interface ToolParam {
 
 // ─── Action (optional useAction hook) ─────────────────────────
 
+export interface ActionParameterDef {
+  /** The primitive type of the parameter. Maps to MCP schemas and native iOS/Android types. */
+  type: 'string' | 'number' | 'boolean';
+  /** A clear description of what the parameter is for (read by the AI). */
+  description: string;
+  /** Whether the AI must provide this parameter. Defaults to true. */
+  required?: boolean;
+  /** If provided, the AI is restricted to these specific string values. */
+  enum?: string[];
+}
+
 export interface ActionDefinition {
   name: string;
   description: string;
-  parameters: Record<string, string>;
+  parameters: Record<string, string | ActionParameterDef>;
   handler: (args: Record<string, any>) => any;
 }