@mobileai/react-native 0.9.12 → 0.9.14

package/src/components/DiscoveryTooltip.tsx

@@ -0,0 +1,148 @@
+/**
+ * DiscoveryTooltip — One-time tooltip shown above the FAB on first use.
+ *
+ * Tells users the AI can navigate the app and do things for them.
+ * Shows once, then persists dismissal via AsyncStorage.
+ * Bilingual: EN/AR.
+ */
+
+import { useEffect, useRef } from 'react';
+import {
+  View,
+  Text,
+  Pressable,
+  StyleSheet,
+  Animated,
+} from 'react-native';
+
+interface DiscoveryTooltipProps {
+  language: 'en' | 'ar';
+  primaryColor?: string;
+  onDismiss: () => void;
+}
+
+const LABELS = {
+  en: '✨ I can help you navigate the app and do things for you!',
+  ar: '✨ أقدر أساعدك تتنقل في التطبيق وأعمل حاجات بدالك!',
+};
+
+const AUTO_DISMISS_MS = 6000;
+
+export function DiscoveryTooltip({
+  language,
+  primaryColor,
+  onDismiss,
+}: DiscoveryTooltipProps) {
+  const scaleAnim = useRef(new Animated.Value(0)).current;
+  const opacityAnim = useRef(new Animated.Value(0)).current;
+
+  useEffect(() => {
+    // Spring-in entry
+    Animated.parallel([
+      Animated.spring(scaleAnim, {
+        toValue: 1,
+        friction: 6,
+        tension: 80,
+        useNativeDriver: true,
+      }),
+      Animated.timing(opacityAnim, {
+        toValue: 1,
+        duration: 200,
+        useNativeDriver: true,
+      }),
+    ]).start();
+
+    // Auto-dismiss after timeout
+    const timer = setTimeout(() => {
+      dismissWithAnimation();
+    }, AUTO_DISMISS_MS);
+
+    return () => clearTimeout(timer);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, []);
+
+  const dismissWithAnimation = () => {
+    Animated.parallel([
+      Animated.timing(scaleAnim, {
+        toValue: 0,
+        duration: 200,
+        useNativeDriver: true,
+      }),
+      Animated.timing(opacityAnim, {
+        toValue: 0,
+        duration: 200,
+        useNativeDriver: true,
+      }),
+    ]).start(() => onDismiss());
+  };
+
+  const isArabic = language === 'ar';
+  const bgColor = primaryColor || '#1a1a2e';
+
+  return (
+    <Animated.View
+      style={[
+        styles.container,
+        {
+          backgroundColor: bgColor,
+          transform: [{ scale: scaleAnim }],
+          opacity: opacityAnim,
+        },
+      ]}
+    >
+      <Pressable onPress={dismissWithAnimation} style={styles.contentArea}>
+        <Text style={[styles.text, isArabic && styles.textRTL]}>
+          {LABELS[language]}
+        </Text>
+      </Pressable>
+
+      {/* Triangle pointer toward FAB */}
+      <View style={[styles.pointer, { borderTopColor: bgColor }]} />
+    </Animated.View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    position: 'absolute',
+    bottom: 70,
+    right: -4,
+    minWidth: 200,
+    maxWidth: 260,
+    borderRadius: 16,
+    paddingHorizontal: 14,
+    paddingVertical: 10,
+    shadowColor: '#000',
+    shadowOffset: { width: 0, height: 4 },
+    shadowOpacity: 0.3,
+    shadowRadius: 8,
+    elevation: 6,
+  },
+  contentArea: {
+    flexDirection: 'row',
+    alignItems: 'center',
+  },
+  text: {
+    color: '#ffffff',
+    fontSize: 13,
+    lineHeight: 19,
+    fontWeight: '500',
+  },
+  textRTL: {
+    textAlign: 'right',
+    writingDirection: 'rtl',
+  },
+  pointer: {
+    position: 'absolute',
+    bottom: -8,
+    right: 22,
+    width: 0,
+    height: 0,
+    borderLeftWidth: 8,
+    borderRightWidth: 8,
+    borderTopWidth: 8,
+    borderLeftColor: 'transparent',
+    borderRightColor: 'transparent',
+    borderTopColor: '#1a1a2e',
+  },
+});

package/src/core/AgentRuntime.ts

@@ -786,6 +786,15 @@ ${screen.elementsText}
         return validationError;
       }
 
+      // ── Copilot aiConfirm gate ──────────────────────────────────
+      // In copilot mode, elements marked with aiConfirm={true} require
+      // user confirmation before execution. This is the code-level safety net
+      // complementing the prompt-level copilot instructions.
+      if (this.config.interactionMode !== 'autopilot') {
+        const confirmResult = await this.checkCopilotConfirmation(toolName, args);
+        if (confirmResult) return confirmResult;
+      }
+
       const result = await tool.execute(args);
 
       // Settle window for async side-effects (useEffect, native callbacks)
@@ -838,6 +847,74 @@ ${screen.elementsText}
     return null;
   }
 
+  // ─── Copilot Confirmation ─────────────────────────────────────
+
+  /** Write tools that can mutate state — only these are checked for aiConfirm */
+  private static readonly WRITE_TOOLS = new Set([
+    'tap', 'type', 'long_press', 'adjust_slider', 'select_picker', 'set_date',
+  ]);
+
+  /**
+   * Check if a tool call targets an aiConfirm element and request user confirmation.
+   * Returns null if the action should proceed, or an error string if rejected.
+   */
+  private async checkCopilotConfirmation(
+    toolName: string,
+    args: Record<string, any>,
+  ): Promise<string | null> {
+    // Only gate write tools
+    if (!AgentRuntime.WRITE_TOOLS.has(toolName)) return null;
+
+    // Look up the target element by index
+    const index = args.index;
+    if (typeof index !== 'number') return null;
+
+    const screen = this.lastDehydratedRoot as import('./types').DehydratedScreen | null;
+    if (!screen?.elements) return null;
+
+    const element = screen.elements.find(e => e.index === index);
+    if (!element?.requiresConfirmation) return null;
+
+    // Element has aiConfirm — request user confirmation
+    const label = element.label || `[${element.type}]`;
+    const description = this.getToolStatusLabel(toolName, args);
+    const question = `I'm about to ${description} on "${label}". Should I proceed?`;
+
+    logger.info('AgentRuntime', `🛡️ Copilot: aiConfirm gate triggered for "${toolName}" on "${label}"`);
+
+    // Use onAskUser if available (integrated into chat UI), otherwise Alert.alert
+    if (this.config.onAskUser) {
+      const response = await this.config.onAskUser(question);
+      const approved = /^(yes|ok|sure|go|proceed|confirm|y)/i.test(response.trim());
+      if (!approved) {
+        logger.info('AgentRuntime', `🛑 User rejected "${toolName}" on "${label}"`);
+        return `❌ User rejected "${toolName}" action on "${label}". Ask what they would like to do instead.`;
+      }
+      return null;
+    }
+
+    // Fallback: React Native Alert
+    const { Alert } = require('react-native');
+    const approved = await new Promise<boolean>(resolve => {
+      Alert.alert(
+        'Confirm Action',
+        question,
+        [
+          { text: 'Cancel', style: 'cancel', onPress: () => resolve(false) },
+          { text: 'Continue', onPress: () => resolve(true) },
+        ],
+        { cancelable: false },
+      );
+    });
+
+    if (!approved) {
+      logger.info('AgentRuntime', `🛑 User rejected "${toolName}" on "${label}"`);
+      return `❌ User rejected "${toolName}" action on "${label}". Ask what they would like to do instead.`;
+    }
+
+    return null;
+  }
+
   // ─── Walk Config (passes security settings to FiberTreeWalker) ─
 
   private getWalkConfig(): WalkConfig {
@@ -1190,7 +1267,8 @@ ${screen.elementsText}
         // 5. Send to AI provider
         this.config.onStatusUpdate?.('Analyzing screen...');
         const hasKnowledge = !!this.knowledgeService;
-        const systemPrompt = buildSystemPrompt('en', hasKnowledge);
+        const isCopilot = this.config.interactionMode !== 'autopilot';
+        const systemPrompt = buildSystemPrompt('en', hasKnowledge, isCopilot);
         const tools = this.buildToolsForProvider();
 
         logger.info('AgentRuntime', `Sending to AI with ${tools.length} tools...`);
@@ -1339,6 +1417,14 @@ ${screen.elementsText}
         steps: this.history,
         tokenUsage: sessionUsage,
       };
+
+      // Dev warning: remind developers to add aiConfirm for extra safety
+      if (__DEV__ && this.config.interactionMode !== 'autopilot') {
+        logger.info('AgentRuntime',
+          'ℹ️ Copilot mode active. Tip: Add aiConfirm={true} to critical buttons (e.g. "Place Order", "Delete") for extra safety.'
+        );
+      }
+
       await this.config.onAfterTask?.(result);
       return result;
     } catch (error: any) {

package/src/core/FiberTreeWalker.ts

@@ -617,6 +617,7 @@ export function walkFiberTree(rootRef: any, config?: WalkConfig): WalkResult {
         zoneId: currentZoneId,
         fiberNode: node,
         props: { ...props },
+        requiresConfirmation: props.aiConfirm === true,
       });
 
       // Build output tag with state attributes
@@ -627,6 +628,10 @@ export function walkFiberTree(rootRef: any, config?: WalkConfig): WalkResult {
         if (currentZoneId) attrStr += ` zoneId="${currentZoneId}"`;
       }
       
+      if (props.aiConfirm === true) {
+        attrStr += ' aiConfirm';
+      }
+      
       const textContent = label || '';
       const elementOutput = `${indent}[${currentIndex}]<${resolvedType}${attrStr}>${textContent} />${childrenText.trim() ? '\n' + childrenText : ''}\n`;
       currentIndex++;

package/src/core/systemPrompt.ts

@@ -89,9 +89,43 @@ const SHARED_CAPABILITY = `- It is ok to fail the task. User would rather you re
 - The app can have bugs. If something is not working as expected, report it to the user.
 - Trying too hard can be harmful. If stuck, report partial progress rather than repeating failed actions.`;
 
+/**
+ * Copilot mode rules — AI pauses once before final irreversible commit.
+ * Injected when interactionMode is 'copilot' (the default).
+ */
+const COPILOT_RULES = `<copilot_mode>
+You are in COPILOT mode. This means:
+
+Execute ALL intermediate actions SILENTLY — no confirmation needed:
+- Navigating between screens, tabs, menus
+- Scrolling to find content
+- Typing into form fields
+- Selecting options, filters, categories
+- Adding items to cart (user can remove later)
+- Opening/closing dialogs or details
+- Toggling form controls while filling a form
+
+PAUSE only when you reach the FINAL action that IRREVERSIBLY commits
+the user's change. Use ask_user to summarize what you have done so far
+and ask permission BEFORE tapping the commit button. Examples of commit actions:
+- Placing an order / completing a purchase
+- Submitting a form that sends data
+- Deleting something (account, item, message)
+- Confirming a payment or transaction
+- Sending a message or email
+- Saving account/profile changes
+
+Elements marked with aiConfirm in the element tree are developer-flagged
+as requiring confirmation. Treat them as commit actions regardless of context.
+
+Call ask_user EXACTLY ONCE per task — at the final commit moment, not at
+every step. If the task has no irreversible commit (e.g., "show me my orders",
+"find the cheapest item"), complete the task without pausing.
+</copilot_mode>`;
+
 // ─── Text Agent Prompt ──────────────────────────────────────────────────────
 
-export function buildSystemPrompt(language: string, hasKnowledge = false): string {
+export function buildSystemPrompt(language: string, hasKnowledge = false, isCopilot = true): string {
   const isArabic = language === 'ar';
 
   return `${CONFIDENTIALITY("I'm your app assistant — I can help you navigate and use this app. What would you like to do?")}
@@ -170,6 +204,8 @@ ${NAVIGATION_RULE}
 ${UI_SIMPLIFICATION_RULE}
 </rules>
 
+${isCopilot ? COPILOT_RULES : ''}
+
 <task_completion_rules>
 You must call the done action in one of these cases:
 - When you have fully completed the USER REQUEST.
@@ -188,8 +224,10 @@ The done action is your opportunity to communicate findings and provide a cohere
 - Use the text field to answer questions, summarize what you found, or explain what you did.
 - You are ONLY ALLOWED to call done as a single action. Do not call it together with other actions.
 
-The ask_user action should ONLY be used when the user gave an action request but you lack specific information to execute it (e.g., user says "order a pizza" but there are multiple options and you don't know which one).
-- Do NOT use ask_user to confirm actions the user explicitly requested. If they said "place my order", just do it.
+The ask_user action should ONLY be used when:
+- The user gave an action request but you lack specific information to execute it (e.g., user says "order a pizza" but there are multiple options and you don't know which one).
+- You are in copilot mode and about to perform an irreversible commit action (see copilot_mode rules above).
+- Do NOT use ask_user for routine confirmations the user already gave. If they said "place my order", proceed to the commit step and confirm there.
 - NEVER ask for the same confirmation twice. If the user already answered, proceed with their answer.
 - For destructive/purchase actions (place order, delete, pay), tap the button exactly ONCE. Do not repeat the same action — the user could be charged multiple times.
 </task_completion_rules>

package/src/core/types.ts

@@ -6,6 +6,15 @@
 
 export type AgentMode = 'text' | 'voice' | 'human';
 
+/**
+ * Controls how the agent handles irreversible actions.
+ * 'copilot' (default): AI pauses before final commit actions (place order, delete, submit).
+ *   The prompt instructs the AI to ask_user before the final irreversible step.
+ *   Elements with aiConfirm={true} also trigger a code-level confirmation gate.
+ * 'autopilot': Full autonomy — all actions execute without confirmation.
+ */
+export type InteractionMode = 'copilot' | 'autopilot';
+
 // ─── Provider Names ──────────────────────────────────────────
 
 export type AIProviderName = 'gemini' | 'openai';
@@ -38,6 +47,11 @@ export interface InteractiveElement {
    * - Switch: onValueChange, value
    */
   props: Record<string, any>;
+  /**
+   * If true, AI interaction with this element requires user confirmation (copilot safety net).
+   * Set automatically by the FiberTreeWalker when the element has aiConfirm={true} prop.
+   */
+  requiresConfirmation?: boolean;
 }
 
 // ─── Dehydrated Screen State ──────────────────────────────────
@@ -119,6 +133,13 @@ export interface AgentConfig {
   /** Maximum steps per task */
   maxSteps?: number;
 
+  /**
+   * Controls how the agent handles irreversible actions.
+   * 'copilot' (default): AI pauses before final commit actions.
+   * 'autopilot': Full autonomy, no pauses.
+   */
+  interactionMode?: InteractionMode;
+
   /**
    * MCP server mode — controls whether external agents can discover and invoke actions.
    * 'auto' (default): enabled in __DEV__, disabled in production

package/src/index.ts

@@ -51,6 +51,7 @@ export type {
   AIProviderName,
   ScreenMap,
   ScreenMapEntry,
+  InteractionMode,
 } from './core/types';
 
 export type {