@mobileai/react-native 0.1.0 → 0.3.0

package/lib/typescript/src/providers/GeminiProvider.d.ts

@@ -1,23 +1,43 @@
 /**
- * GeminiProvider — Simplified Gemini API integration.
- * Sends dehydrated screen state + tools to Gemini and parses tool call responses.
+ * GeminiProvider — Gemini API integration with structured action pattern.
+ *
+ * Uses a single forced function call (`agent_step`) that bundles
+ * structured reasoning (evaluation, memory, plan) alongside the action.
+ * This replaces free-form text + separate tool calls for stability.
  */
-import type { AIProvider, ToolDefinition, AgentStep } from '../core/types';
+import type { AIProvider, ToolDefinition, AgentStep, ProviderResult } from '../core/types';
 export declare class GeminiProvider implements AIProvider {
     private apiKey;
     private model;
     constructor(apiKey: string, model?: string);
-    generateContent(systemPrompt: string, userMessage: string, tools: ToolDefinition[], history: AgentStep[]): Promise<{
-        toolCalls: Array<{
-            name: string;
-            args: Record<string, any>;
-        }>;
-        text?: string;
-    }>;
-    private buildGeminiTools;
+    generateContent(systemPrompt: string, userMessage: string, tools: ToolDefinition[], history: AgentStep[]): Promise<ProviderResult>;
+    /**
+     * Builds a single `agent_step` function declaration that combines:
+     * - Structured reasoning fields (previous_goal_eval, memory, plan)
+     * - action_name (enum of all available tool names)
+     * - All tool parameter fields as flat top-level properties
+     *
+     * Flat schema avoids Gemini's "deeply nested schema" rejection in ANY mode.
+     */
+    private buildAgentStepDeclaration;
     private mapParamType;
+    /**
+     * Builds Gemini conversation contents.
+     *
+     * Each step is a STATELESS single-turn request (matching page-agent's approach):
+     * - System prompt has general instructions
+     * - User message contains full context: task, history, screen state
+     * - Model responds with agent_step function call
+     *
+     * History is embedded as text in assembleUserPrompt (via <agent_history>),
+     * NOT as functionCall/functionResponse pairs. This avoids Gemini's
+     * conversation format requirements and thought_signature complexity.
+     */
     private buildContents;
-    private ensureAlternatingRoles;
-    private parseResponse;
+    /**
+     * Parses the Gemini response expecting a single agent_step function call.
+     * Extracts structured reasoning + action, and determines which tool to execute.
+     */
+    private parseAgentStepResponse;
 }
 //# sourceMappingURL=GeminiProvider.d.ts.map

package/lib/typescript/src/providers/GeminiProvider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"GeminiProvider.d.ts","sourceRoot":"","sources":["../../../../src/providers/GeminiProvider.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAyB3E,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAGV,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAA2B;IAKxD,eAAe,CACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC;QAAE,SAAS,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,CAAC,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAqD5F,OAAO,CAAC,gBAAgB;IAyBxB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,aAAa;IAsBrB,OAAO,CAAC,sBAAsB;IAsB9B,OAAO,CAAC,aAAa;CA8BtB"}
+{"version":3,"file":"GeminiProvider.d.ts","sourceRoot":"","sources":["../../../../src/providers/GeminiProvider.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,SAAS,EAAE,cAAc,EAAkB,MAAM,eAAe,CAAC;AAsB3G,qBAAa,cAAe,YAAW,UAAU;IAC/C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,MAAM,EAAE,KAAK,GAAE,MAA2B;IAKxD,eAAe,CACnB,YAAY,EAAE,MAAM,EACpB,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,cAAc,EAAE,EACvB,OAAO,EAAE,SAAS,EAAE,GACnB,OAAO,CAAC,cAAc,CAAC;IA2D1B;;;;;;;OAOG;IACH,OAAO,CAAC,yBAAyB;IA0DjC,OAAO,CAAC,YAAY;IAYpB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,aAAa;IASrB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;CA2E/B"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mobileai/react-native",
-  "version": "0.1.0",
-  "description": "Autonomous AI agent for React Native — reads your UI via Fiber tree, understands layout context, and interacts with your app. Zero wrappers.",
+  "version": "0.3.0",
+  "description": "Build autonomous AI agents for React Native and Expo apps. Provides AI-native UI traversal, tool calling, and structured reasoning.",
   "main": "./lib/module/index.js",
   "source": "./src/index.ts",
   "types": "./lib/typescript/src/index.d.ts",
@@ -40,32 +40,34 @@
     "prepare": "bob build",
     "typecheck": "tsc",
     "lint": "eslint \"**/*.{js,ts,tsx}\"",
-    "test": "jest"
+    "test": "jest",
+    "publish:dual": "bash scripts/publish-dual.sh",
+    "publish:dual:dry": "bash scripts/publish-dual.sh --dry-run"
   },
   "keywords": [
     "react-native",
+    "expo",
     "ai",
+    "llm",
     "agent",
-    "agentic",
+    "autonomous-agent",
     "gemini",
-    "llm",
-    "autonomous",
-    "fiber-tree",
-    "page-agent",
-    "mcp",
-    "ios",
-    "android"
+    "openai",
+    "artificial-intelligence",
+    "mobile-ai",
+    "ui-automation",
+    "react native ai agent"
   ],
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/mohamed2m2018/react-native-ai-agent.git"
+    "url": "git+https://github.com/mohamed2m2018/mobileai-react-native.git"
   },
   "author": "Mohamed Salah <mohamedsalah@example.com> (https://example.com)",
   "license": "MIT",
   "bugs": {
-    "url": "https://github.com/mohamed2m2018/react-native-ai-agent/issues"
+    "url": "https://github.com/mohamed2m2018/mobileai-react-native/issues"
   },
-  "homepage": "https://github.com/mohamed2m2018/react-native-ai-agent#readme",
+  "homepage": "https://github.com/mohamed2m2018/mobileai-react-native#readme",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/"
   },

package/src/components/AIAgent.tsx

@@ -75,6 +75,14 @@ interface AIAgentProps {
   stepDelay?: number;
   /** WebSocket URL to companion MCP server bridge (e.g., ws://localhost:3101) */
   mcpServerUrl?: string;
+  /** Expo Router instance (from useRouter()) */
+  router?: {
+    push: (href: string) => void;
+    replace: (href: string) => void;
+    back: () => void;
+  };
+  /** Expo Router pathname (from usePathname()) */
+  pathname?: string;
 }
 
 // ─── Component ─────────────────────────────────────────────────
@@ -100,12 +108,17 @@ export function AIAgent({
   instructions,
   stepDelay,
   mcpServerUrl,
+  router,
+  pathname,
 }: AIAgentProps) {
   const rootViewRef = useRef<any>(null);
   const [isThinking, setIsThinking] = useState(false);
   const [statusText, setStatusText] = useState('');
   const [lastResult, setLastResult] = useState<ExecutionResult | null>(null);
 
+  // Ref-based resolver for ask_user — stays alive across renders
+  const askUserResolverRef = useRef<((answer: string) => void) | null>(null);
+
   // ─── Create Runtime ──────────────────────────────────────────
 
   const config: AgentConfig = useMemo(() => ({
@@ -124,12 +137,25 @@ export function AIAgent({
     instructions,
     stepDelay,
     mcpServerUrl,
+    router,
+    pathname,
+    onStatusUpdate: setStatusText,
+    // Page-agent pattern: block the agent loop until user responds
+    onAskUser: (question: string) => {
+      return new Promise<string>((resolve) => {
+        askUserResolverRef.current = resolve;
+        // Show question in chat bar, allow user input
+        setLastResult({ success: true, message: `❓ ${question}`, steps: [] });
+        setIsThinking(false);
+        setStatusText('');
+      });
+    },
   }), [
     apiKey, model, language, maxSteps,
     interactiveBlacklist, interactiveWhitelist,
     onBeforeStep, onAfterStep, onBeforeTask, onAfterTask,
     transformScreenContent, customTools, instructions, stepDelay,
-    mcpServerUrl,
+    mcpServerUrl, router, pathname,
   ]);
 
   const provider = useMemo(() => new GeminiProvider(apiKey, model), [apiKey, model]);
@@ -164,6 +190,19 @@ export function AIAgent({
     if (!message.trim()) return;
 
     logger.info('AIAgent', `User message: "${message}"`);
+
+    // If there's a pending ask_user, resolve it instead of starting a new execution
+    if (askUserResolverRef.current) {
+      const resolver = askUserResolverRef.current;
+      askUserResolverRef.current = null;
+      setIsThinking(true);
+      setStatusText('Processing your answer...');
+      setLastResult(null);
+      resolver(message);
+      return;
+    }
+
+    // Normal execution — new task
     setIsThinking(true);
     setStatusText('Thinking...');
     setLastResult(null);
@@ -209,6 +248,7 @@ export function AIAgent({
           isThinking={isThinking}
           lastResult={lastResult}
           language={language}
+          onDismiss={() => setLastResult(null)}
         />
       )}
     </AgentContext.Provider>

package/src/components/AgentChatBar.tsx

@@ -1,17 +1,18 @@
 /**
- * AgentChatBar — Floating chat input at the bottom of the screen.
- * User sends messages here, and results are shown inline.
+ * AgentChatBar — Floating, draggable, compressible chat widget.
+ * Does not block underlying UI natively.
  */
 
-import { useState } from 'react';
+import { useState, useRef } from 'react';
 import {
   View,
   TextInput,
   Pressable,
   Text,
   StyleSheet,
-  KeyboardAvoidingView,
-  Platform,
+  Animated,
+  PanResponder,
+  useWindowDimensions,
 } from 'react-native';
 import type { ExecutionResult } from '../core/types';
 
@@ -20,12 +21,43 @@ interface AgentChatBarProps {
   isThinking: boolean;
   lastResult: ExecutionResult | null;
   language: 'en' | 'ar';
+  onDismiss?: () => void;
 }
 
-export function AgentChatBar({ onSend, isThinking, lastResult, language }: AgentChatBarProps) {
+export function AgentChatBar({ onSend, isThinking, lastResult, language, onDismiss }: AgentChatBarProps) {
   const [text, setText] = useState('');
+  const [isExpanded, setIsExpanded] = useState(false);
+  const { height } = useWindowDimensions();
   const isArabic = language === 'ar';
 
+  // Initial position: Bottom right for FAB, Bottom center for Expanded
+  // For simplicity, we just initialize to a safe generic spot on screen.
+  const pan = useRef(new Animated.ValueXY({ x: 10, y: height - 200 })).current;
+
+  // PanResponder for dragging the widget
+  const panResponder = useRef(
+    PanResponder.create({
+      onMoveShouldSetPanResponder: (_, gestureState) => {
+        // Only trigger drag if moving more than 5px (allows taps to register inside)
+        return Math.abs(gestureState.dx) > 5 || Math.abs(gestureState.dy) > 5;
+      },
+      onPanResponderGrant: () => {
+        pan.setOffset({
+          x: (pan.x as any)._value,
+          y: (pan.y as any)._value,
+        });
+        pan.setValue({ x: 0, y: 0 });
+      },
+      onPanResponderMove: Animated.event(
+        [null, { dx: pan.x, dy: pan.y }],
+        { useNativeDriver: false }
+      ),
+      onPanResponderRelease: () => {
+        pan.flattenOffset();
+      },
+    })
+  ).current;
+
   const handleSend = () => {
     if (text.trim() && !isThinking) {
       onSend(text.trim());
@@ -33,15 +65,41 @@ export function AgentChatBar({ onSend, isThinking, lastResult, language }: Agent
     }
   };
 
+  // ─── Compressed State (FAB) ───
+  if (!isExpanded) {
+    return (
+      <Animated.View style={[styles.fabContainer, pan.getLayout()]} {...panResponder.panHandlers}>
+        <Pressable 
+          style={styles.fab} 
+          onPress={() => setIsExpanded(true)}
+          accessibilityLabel="Open AI Agent Chat"
+        >
+          <Text style={styles.fabIcon}>{isThinking ? '⏳' : '🤖'}</Text>
+        </Pressable>
+      </Animated.View>
+    );
+  }
+
+  // ─── Expanded State (Widget) ───
   return (
-    <KeyboardAvoidingView
-      behavior={Platform.OS === 'ios' ? 'padding' : undefined}
-      style={styles.container}
-    >
+    <Animated.View style={[styles.expandedContainer, pan.getLayout()]}>
+      {/* Drag Handle Area */}
+      <View {...panResponder.panHandlers} style={styles.dragHandleArea} accessibilityLabel="Drag AI Agent">
+        <View style={styles.dragGrip} />
+        <Pressable onPress={() => setIsExpanded(false)} style={styles.minimizeBtn} accessibilityLabel="Minimize AI Agent">
+          <Text style={styles.minimizeText}>—</Text>
+        </Pressable>
+      </View>
+
       {/* Result message */}
       {lastResult && (
         <View style={[styles.resultBubble, lastResult.success ? styles.resultSuccess : styles.resultError]}>
           <Text style={styles.resultText}>{lastResult.message}</Text>
+          {onDismiss && (
+            <Pressable style={styles.dismissButton} onPress={onDismiss} hitSlop={12}>
+              <Text style={styles.dismissText}>✕</Text>
+            </Pressable>
+          )}
         </View>
       )}
 
@@ -49,7 +107,7 @@ export function AgentChatBar({ onSend, isThinking, lastResult, language }: Agent
       <View style={styles.inputRow}>
         <TextInput
           style={[styles.input, isArabic && styles.inputRTL]}
-          placeholder={isArabic ? 'اكتب طلبك...' : 'Ask the AI agent...'}
+          placeholder={isArabic ? 'اكتب طلبك...' : 'Ask AI...'}
           placeholderTextColor="#999"
           value={text}
           onChangeText={setText}
@@ -62,33 +120,87 @@ export function AgentChatBar({ onSend, isThinking, lastResult, language }: Agent
           style={[styles.sendButton, isThinking && styles.sendButtonDisabled]}
           onPress={handleSend}
           disabled={isThinking || !text.trim()}
+          accessibilityLabel="Send request to AI Agent"
         >
           <Text style={styles.sendButtonText}>
             {isThinking ? '⏳' : '🚀'}
           </Text>
         </Pressable>
       </View>
-    </KeyboardAvoidingView>
+    </Animated.View>
   );
 }
 
 const styles = StyleSheet.create({
-  container: {
+  // FAB Styles
+  fabContainer: {
     position: 'absolute',
-    bottom: 0,
-    left: 0,
-    right: 0,
-    paddingHorizontal: 12,
-    paddingBottom: Platform.OS === 'ios' ? 34 : 12,
-    paddingTop: 8,
+    zIndex: 9999,
+  },
+  fab: {
+    width: 60,
+    height: 60,
+    borderRadius: 30,
+    backgroundColor: '#1a1a2e',
+    justifyContent: 'center',
+    alignItems: 'center',
+    elevation: 5,
+    shadowColor: '#000',
+    shadowOffset: { width: 0, height: 4 },
+    shadowOpacity: 0.3,
+    shadowRadius: 6,
+  },
+  fabIcon: {
+    fontSize: 28,
+  },
+
+  // Expanded Styles
+  expandedContainer: {
+    position: 'absolute',
+    zIndex: 9999,
+    width: 340,
     backgroundColor: 'rgba(26, 26, 46, 0.95)',
-    borderTopLeftRadius: 20,
-    borderTopRightRadius: 20,
+    borderRadius: 24,
+    padding: 16,
+    paddingTop: 8,
+    elevation: 8,
+    shadowColor: '#000',
+    shadowOffset: { width: 0, height: 8 },
+    shadowOpacity: 0.4,
+    shadowRadius: 10,
+  },
+  dragHandleArea: {
+    width: '100%',
+    height: 30,
+    justifyContent: 'center',
+    alignItems: 'center',
+    marginBottom: 8,
+  },
+  dragGrip: {
+    width: 40,
+    height: 5,
+    backgroundColor: 'rgba(255, 255, 255, 0.3)',
+    borderRadius: 4,
+  },
+  minimizeBtn: {
+    position: 'absolute',
+    right: 0,
+    top: 0,
+    padding: 8,
   },
+  minimizeText: {
+    color: '#fff',
+    fontSize: 18,
+    fontWeight: 'bold',
+  },
+  
+  // Results & Input
   resultBubble: {
     borderRadius: 12,
     padding: 12,
-    marginBottom: 8,
+    marginBottom: 12,
+    flexDirection: 'row',
+    alignItems: 'flex-start',
   },
   resultSuccess: {
     backgroundColor: 'rgba(40, 167, 69, 0.2)',
@@ -100,6 +212,16 @@ const styles = StyleSheet.create({
     color: '#fff',
     fontSize: 14,
     lineHeight: 20,
+    flex: 1,
+  },
+  dismissButton: {
+    marginLeft: 8,
+    padding: 2,
+  },
+  dismissText: {
+    color: 'rgba(255, 255, 255, 0.6)',
+    fontSize: 14,
+    fontWeight: 'bold',
   },
   inputRow: {
     flexDirection: 'row',
@@ -109,9 +231,9 @@ const styles = StyleSheet.create({
   input: {
     flex: 1,
     backgroundColor: 'rgba(255, 255, 255, 0.1)',
-    borderRadius: 24,
+    borderRadius: 20,
     paddingHorizontal: 16,
-    paddingVertical: 12,
+    paddingVertical: 10,
     color: '#fff',
     fontSize: 16,
   },
@@ -120,9 +242,9 @@ const styles = StyleSheet.create({
     writingDirection: 'rtl',
   },
   sendButton: {
-    width: 44,
-    height: 44,
-    borderRadius: 22,
+    width: 40,
+    height: 40,
+    borderRadius: 20,
     backgroundColor: 'rgba(255, 255, 255, 0.15)',
     justifyContent: 'center',
     alignItems: 'center',
@@ -131,6 +253,6 @@ const styles = StyleSheet.create({
     opacity: 0.5,
   },
   sendButtonText: {
-    fontSize: 20,
+    fontSize: 18,
   },
 });