@mobileai/react-native 0.9.27 → 0.9.29

package/README.md

@@ -307,14 +307,13 @@ npm install react-native-agentic-ai
 
 No native modules required by default. Works with **Expo managed workflow** out of the box — no eject needed.
 
-### Optional Dependencies
+### Screenshot Capture
 
 <details>
 <summary><b>📸 Screenshots</b> — for image/video content understanding</summary>
 
-```bash
-npx expo install react-native-view-shot
-```
+`react-native-view-shot` is a required dependency for screenshot capture and is included with
+`@mobileai/react-native`, so you do **not** need to add it separately.
 
 </details>
 
@@ -391,6 +390,8 @@ npx @mobileai/react-native generate-map
 
 ### 2. Wrap Your App
 
+If you use a MobileAI publishable key, the SDK now defaults to the hosted MobileAI text and voice proxies automatically. You only need to pass `proxyUrl` and `voiceProxyUrl` when you want to override them with your own backend.
+
 #### React Navigation
 
 ```tsx
@@ -403,13 +404,10 @@ export default function App() {
 
   return (
     <AIAgent
-      // Your MobileAI Dashboard ID — instantly enables cloud intelligence
+      // Your MobileAI Dashboard ID
+      // This now auto-configures the hosted MobileAI text + voice proxies too.
       analyticsKey="mobileai_pub_xxxxxxxx"
 
-      // Route all traffic through the secure MobileAI Cloud proxies
-      proxyUrl="https://mobileai.cloud/api/v1/hosted-proxy/text"
-      voiceProxyUrl="wss://mobileai.cloud/ws/hosted-proxy/voice"
-
       navRef={navRef}
       screenMap={screenMap} // optional but recommended
     >
@@ -435,9 +433,8 @@ export default function RootLayout() {
 
   return (
     <AIAgent
+      // Hosted MobileAI proxies are inferred automatically from analyticsKey
       analyticsKey="mobileai_pub_xxxxxxxx"
-      proxyUrl="https://mobileai.cloud/api/v1/hosted-proxy/text"
-      voiceProxyUrl="wss://mobileai.cloud/ws/hosted-proxy/voice"
       navRef={navRef}
       screenMap={screenMap}
     >
@@ -464,6 +461,19 @@ The examples above use **Gemini** (default). To use **OpenAI** for text mode, ad
 
 A floating chat bar appears automatically. Ask the AI to navigate, tap buttons, fill forms, answer questions.
 
+### Hosted MobileAI Defaults
+
+For the standard MobileAI Cloud setup, this is enough:
+
+```tsx
+<AIAgent analyticsKey="mobileai_pub_xxxxxxxx" navRef={navRef} />
+```
+
+Only pass explicit proxy props when:
+- you want to use your own backend proxy
+- you want a dedicated voice proxy
+- you are self-hosting the MobileAI backend
+
 ### Knowledge-Only Mode — AI Assistant Without UI Automation
 
 Set `enableUIControl={false}` for a lightweight FAQ / support assistant. Single LLM call, ~70% fewer tokens:
@@ -871,9 +881,9 @@ Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
 |------|------|---------|-------------|
 | `apiKey` | `string` | — | API key for your provider (prototyping only — use `proxyUrl` in production). |
 | `provider` | `'gemini' \| 'openai'` | `'gemini'` | LLM provider for text mode. |
-| `proxyUrl` | `string` | — | Backend proxy URL (production). Routes all LLM traffic through your server. |
+| `proxyUrl` | `string` | Hosted MobileAI text proxy when `analyticsKey` is set | Backend proxy URL (production). Routes all LLM traffic through your server. |
 | `proxyHeaders` | `Record<string, string>` | — | Auth headers for proxy (e.g., `Authorization: Bearer ${token}`). |
-| `voiceProxyUrl` | `string` | — | Dedicated proxy for Voice Mode WebSockets. Falls back to `proxyUrl`. |
+| `voiceProxyUrl` | `string` | Hosted MobileAI voice proxy when `analyticsKey` is set; otherwise falls back to `proxyUrl` | Dedicated proxy for Voice Mode WebSockets. |
 | `voiceProxyHeaders` | `Record<string, string>` | — | Auth headers for voice proxy. |
 | `model` | `string` | Provider default | Model name (e.g. `gemini-2.5-flash`, `gpt-4.1-mini`). |
 | `navRef` | `NavigationContainerRef` | — | Navigation ref for auto-navigation. |
@@ -953,7 +963,7 @@ Add to `~/Library/Application Support/Claude/claude_desktop_config.json`:
 
 | Prop | Type | Default | Description |
 |------|------|---------|-------------|
-| `analyticsKey` | `string` | — | Publishable key (`mobileai_pub_xxx`) — enables auto-analytics. |
+| `analyticsKey` | `string` | — | Publishable key (`mobileai_pub_xxx`) — enables auto-analytics and, by default, the hosted MobileAI text/voice proxies. |
 | `analyticsProxyUrl` | `string` | — | Enterprise: route events through your backend. |
 | `analyticsProxyHeaders` | `Record<string, string>` | — | Auth headers for analytics proxy. |
 
@@ -1065,7 +1075,7 @@ const { send } = useAI({
 
 ## 📊 Zero-Config Analytics — Auto-Capture Every Tap
 
-Just add `analyticsKey` — every button tap, screen navigation, and session is tracked automatically. **Zero code changes** to your app components.
+Just add `analyticsKey` — every button tap, screen navigation, and session is tracked automatically. It also enables the default hosted MobileAI AI proxies unless you override them. **Zero code changes** to your app components.
 
 ```tsx
 <AIAgent
@@ -1116,6 +1126,8 @@ MobileAI.identify('user_123', { plan: 'pro' });
 
 ### Backend Proxy — Keep API Keys Secure
 
+Use this only when you want to override the default hosted MobileAI proxy behavior or route traffic through your own backend.
+
 ```tsx
 <AIAgent
   proxyUrl="https://myapp.vercel.app/api/gemini"
@@ -1262,7 +1274,7 @@ Tag any element with `aiPriority` to control AI visibility:
 | `date_picker(index, date)` | Set a date on a date picker |
 | `navigate(screen)` | Navigate to any screen |
 | `wait(seconds)` | Wait for loading states before acting |
-| `capture_screenshot(reason)` | Capture the screen as an image (requires `react-native-view-shot`) |
+| `capture_screenshot(reason)` | Capture the SDK root component as an image (requires `react-native-view-shot`) |
 | `done(text)` | Finish the task with a response |
 | `ask_user(question)` | Ask the user for clarification |
 | `query_knowledge(question)` | Search the knowledge base |

package/android/build.gradle

@@ -10,6 +10,15 @@ buildscript {
   }
 }
 
+def isNewArchitectureEnabled() {
+  // Libraries opt into Fabric/TurboModule codegen only when the consuming app enables it.
+  return project.hasProperty("newArchEnabled") && project.newArchEnabled == "true"
+}
+
+if (isNewArchitectureEnabled()) {
+  apply plugin: "com.facebook.react"
+}
+
 apply plugin: "com.android.library"
 apply plugin: "kotlin-android"
 
@@ -59,3 +68,11 @@ repositories {
 dependencies {
   implementation "com.facebook.react:react-android"
 }
+
+if (isNewArchitectureEnabled()) {
+  react {
+    jsRootDir = file("../")
+    libraryName = "RNMobileAIOverlaySpec"
+    codegenJavaPackageName = "com.mobileai.overlay"
+  }
+}

package/android/src/main/java/com/mobileai/overlay/FloatingOverlayDialogRootViewGroup.kt

@@ -0,0 +1,243 @@
+package com.mobileai.overlay
+
+import android.annotation.SuppressLint
+import android.content.Context
+import android.view.MotionEvent
+import android.view.View
+import android.view.ViewConfiguration
+import com.facebook.react.bridge.GuardedRunnable
+import com.facebook.react.bridge.WritableMap
+import com.facebook.react.bridge.WritableNativeMap
+import com.facebook.react.common.annotations.UnstableReactNativeAPI
+import com.facebook.react.common.build.ReactBuildConfig
+import com.facebook.react.config.ReactFeatureFlags
+import com.facebook.react.uimanager.JSPointerDispatcher
+import com.facebook.react.uimanager.JSTouchDispatcher
+import com.facebook.react.uimanager.PixelUtil.pxToDp
+import com.facebook.react.uimanager.RootView
+import com.facebook.react.uimanager.StateWrapper
+import com.facebook.react.uimanager.ThemedReactContext
+import com.facebook.react.uimanager.UIManagerModule
+import com.facebook.react.uimanager.events.EventDispatcher
+import com.facebook.react.views.view.ReactViewGroup
+import kotlin.math.abs
+import kotlin.math.roundToInt
+
+/**
+ * Root view rendered inside the Android floating panel dialog.
+ *
+ * This mirrors the touch and layout behavior of React Native's modal dialog
+ * host so JS children remain fully interactive even though they render inside
+ * a separate native window.
+ */
+internal class FloatingOverlayDialogRootViewGroup(context: Context) :
+  ReactViewGroup(context),
+  RootView {
+
+  var stateWrapper: StateWrapper? = null
+  var eventDispatcher: EventDispatcher? = null
+  var overlayHost: FloatingOverlayView? = null
+
+  private var viewWidth: Int = 0
+  private var viewHeight: Int = 0
+  private val jsTouchDispatcher = JSTouchDispatcher(this)
+  private var jsPointerDispatcher: JSPointerDispatcher? = null
+  private val touchSlop: Float = ViewConfiguration.get(context).scaledTouchSlop.toFloat()
+  private var dragCandidate: Boolean = false
+  private var isDraggingWindow: Boolean = false
+  private var dragStartRawX: Float = 0f
+  private var dragStartRawY: Float = 0f
+  private var dragStartWindowX: Int = 0
+  private var dragStartWindowY: Int = 0
+
+  private val reactContext: ThemedReactContext
+    get() = context as ThemedReactContext
+
+  init {
+    if (ReactFeatureFlags.dispatchPointerEvents) {
+      jsPointerDispatcher = JSPointerDispatcher(this)
+    }
+  }
+
+  override fun onSizeChanged(w: Int, h: Int, oldw: Int, oldh: Int) {
+    super.onSizeChanged(w, h, oldw, oldh)
+    viewWidth = w
+    viewHeight = h
+    updateState(viewWidth, viewHeight)
+  }
+
+  fun updateState(width: Int, height: Int) {
+    val realWidth = width.toFloat().pxToDp()
+    val realHeight = height.toFloat().pxToDp()
+
+    val wrapper = stateWrapper
+    if (wrapper != null) {
+      val newStateData: WritableMap = WritableNativeMap()
+      newStateData.putDouble("screenWidth", realWidth.toDouble())
+      newStateData.putDouble("screenHeight", realHeight.toDouble())
+      wrapper.updateState(newStateData)
+    } else if (
+      !ReactBuildConfig.UNSTABLE_ENABLE_MINIFY_LEGACY_ARCHITECTURE &&
+      !reactContext.isBridgeless
+    ) {
+      @Suppress("DEPRECATION")
+      reactContext.runOnNativeModulesQueueThread(
+        object : GuardedRunnable(reactContext) {
+          override fun runGuarded() {
+            reactContext.reactApplicationContext
+              .getNativeModule(UIManagerModule::class.java)
+              ?.updateNodeSize(id, viewWidth, viewHeight)
+          }
+        }
+      )
+    }
+  }
+
+  override fun handleException(t: Throwable) {
+    reactContext.reactApplicationContext.handleException(RuntimeException(t))
+  }
+
+  override fun onInterceptTouchEvent(event: MotionEvent): Boolean {
+    when (event.actionMasked) {
+      MotionEvent.ACTION_DOWN -> {
+        dragCandidate = isInNativeDragRegion(event.x, event.y)
+        isDraggingWindow = false
+        if (dragCandidate) {
+          dragStartRawX = event.rawX
+          dragStartRawY = event.rawY
+          dragStartWindowX = overlayHost?.getWindowXPx() ?: 0
+          dragStartWindowY = overlayHost?.getWindowYPx() ?: 0
+        }
+      }
+
+      MotionEvent.ACTION_MOVE -> {
+        if (dragCandidate && !isDraggingWindow && hasExceededTouchSlop(event)) {
+          isDraggingWindow = true
+          parent?.requestDisallowInterceptTouchEvent(true)
+          updateNativeWindowDrag(event)
+          return true
+        }
+      }
+
+      MotionEvent.ACTION_UP,
+      MotionEvent.ACTION_CANCEL -> resetNativeDrag()
+    }
+
+    if (isDraggingWindow) return true
+
+    eventDispatcher?.let { dispatcher ->
+      jsTouchDispatcher.handleTouchEvent(event, dispatcher, reactContext)
+      jsPointerDispatcher?.handleMotionEvent(event, dispatcher, true)
+    }
+    return super.onInterceptTouchEvent(event)
+  }
+
+  @SuppressLint("ClickableViewAccessibility")
+  override fun onTouchEvent(event: MotionEvent): Boolean {
+    if (isDraggingWindow || dragCandidate) {
+      when (event.actionMasked) {
+        MotionEvent.ACTION_MOVE -> {
+          if (!isDraggingWindow && hasExceededTouchSlop(event)) {
+            isDraggingWindow = true
+            parent?.requestDisallowInterceptTouchEvent(true)
+          }
+
+          if (isDraggingWindow) {
+            updateNativeWindowDrag(event)
+            return true
+          }
+        }
+
+        MotionEvent.ACTION_UP -> {
+          if (isDraggingWindow) {
+            updateNativeWindowDrag(event)
+            overlayHost?.emitWindowDragEnd()
+            resetNativeDrag()
+            return true
+          }
+          resetNativeDrag()
+        }
+
+        MotionEvent.ACTION_CANCEL -> {
+          if (isDraggingWindow) {
+            overlayHost?.emitWindowDragEnd()
+            resetNativeDrag()
+            return true
+          }
+          resetNativeDrag()
+        }
+      }
+    }
+
+    eventDispatcher?.let { dispatcher ->
+      jsTouchDispatcher.handleTouchEvent(event, dispatcher, reactContext)
+      jsPointerDispatcher?.handleMotionEvent(event, dispatcher, false)
+    }
+    super.onTouchEvent(event)
+    return true
+  }
+
+  override fun onInterceptHoverEvent(event: MotionEvent): Boolean {
+    eventDispatcher?.let { dispatcher ->
+      jsPointerDispatcher?.handleMotionEvent(event, dispatcher, true)
+    }
+    return super.onInterceptHoverEvent(event)
+  }
+
+  override fun onHoverEvent(event: MotionEvent): Boolean {
+    eventDispatcher?.let { dispatcher ->
+      jsPointerDispatcher?.handleMotionEvent(event, dispatcher, false)
+    }
+    return super.onHoverEvent(event)
+  }
+
+  @OptIn(UnstableReactNativeAPI::class)
+  override fun onChildStartedNativeGesture(childView: View?, ev: MotionEvent) {
+    eventDispatcher?.let { dispatcher ->
+      jsTouchDispatcher.onChildStartedNativeGesture(ev, dispatcher, reactContext)
+      jsPointerDispatcher?.onChildStartedNativeGesture(childView, ev, dispatcher)
+    }
+  }
+
+  override fun onChildEndedNativeGesture(childView: View, ev: MotionEvent) {
+    eventDispatcher?.let { dispatcher ->
+      jsTouchDispatcher.onChildEndedNativeGesture(ev, dispatcher)
+    }
+    jsPointerDispatcher?.onChildEndedNativeGesture()
+  }
+
+  override fun requestDisallowInterceptTouchEvent(disallowIntercept: Boolean) {
+    // No-op so the root can continue receiving events for JS dispatch.
+  }
+
+  private fun hasExceededTouchSlop(event: MotionEvent): Boolean {
+    return abs(event.rawX - dragStartRawX) > touchSlop || abs(event.rawY - dragStartRawY) > touchSlop
+  }
+
+  private fun updateNativeWindowDrag(event: MotionEvent) {
+    val targetX = dragStartWindowX + (event.rawX - dragStartRawX).roundToInt()
+    val targetY = dragStartWindowY + (event.rawY - dragStartRawY).roundToInt()
+    overlayHost?.updateWindowPositionFromNative(targetX, targetY)
+  }
+
+  private fun resetNativeDrag() {
+    dragCandidate = false
+    isDraggingWindow = false
+  }
+
+  private fun isInNativeDragRegion(x: Float, y: Float): Boolean {
+    if (viewWidth <= dpToPx(80) && viewHeight <= dpToPx(80)) {
+      return true
+    }
+
+    val handleTop = dpToPx(44)
+    val handleHalfWidth = dpToPx(72)
+    val centerX = viewWidth / 2f
+
+    return y <= handleTop && x in (centerX - handleHalfWidth)..(centerX + handleHalfWidth)
+  }
+
+  private fun dpToPx(value: Int): Float {
+    return value * resources.displayMetrics.density
+  }
+}