expo-realtime-ivs-broadcast 0.1.10 → 0.2.1

package/android/src/main/java/expo/modules/realtimeivsbroadcast/ExpoIVSRemoteStreamView.kt

@@ -5,18 +5,22 @@ import android.os.Build
 import android.os.Handler
 import android.os.Looper
 import android.util.Log
+import android.view.View
 import android.widget.FrameLayout
 import androidx.annotation.RequiresApi
 import com.amazonaws.ivs.broadcast.BroadcastConfiguration.AspectMode
 import com.amazonaws.ivs.broadcast.Device
 import com.amazonaws.ivs.broadcast.ImageDevice
-import com.amazonaws.ivs.broadcast.ImagePreviewView
+import com.amazonaws.ivs.broadcast.ImageDeviceFrame
+import com.amazonaws.ivs.broadcast.ImagePreviewSurfaceView
 import expo.modules.kotlin.AppContext
 import expo.modules.kotlin.views.ExpoView
 
 @RequiresApi(Build.VERSION_CODES.P)
-class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoView(context, appContext) {
-    private var ivsImagePreviewView: ImagePreviewView? = null
+class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoView(context, appContext), PiPStateListener {
+    // Use SurfaceView instead of TextureView for better PiP performance
+    private var ivsSurfaceView: ImagePreviewSurfaceView? = null
+    private var currentImageDevice: ImageDevice? = null
     private var stageManager: IVSStageManager? = null
     private val mainHandler = Handler(Looper.getMainLooper())
     
@@ -31,12 +35,117 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
 
     // Props
     private var scaleMode: String = "fill"
+    
+    // PiP registration flag
+    private var isRegisteredForPiP = false
+    
+    // Frame monitoring for PiP
+    private var lastFrameTime: Long = 0
+    private var frameCount: Long = 0
+    private var isInPiPMode = false
 
     init {
         Log.i("ExpoIVSRemoteStreamView", "Initializing Remote Stream View...")
         resolveStageManagerWithRetry()
     }
     
+    // MARK: - PiPStateListener
+    
+    override fun onEnterPiP() {
+        isInPiPMode = true
+        Log.i("ExpoIVSRemoteStreamView", "📺 Entered PiP mode - keeping surface active")
+        
+        // Ensure view stays visible and rendering
+        ivsSurfaceView?.let { surface ->
+            surface.visibility = View.VISIBLE
+            surface.keepScreenOn = true
+        }
+    }
+    
+    override fun onExitPiP() {
+        isInPiPMode = false
+        Log.i("ExpoIVSRemoteStreamView", "📺 Exited PiP mode - refreshing stream")
+        
+        ivsSurfaceView?.keepScreenOn = false
+        
+        // Re-render the stream to fix potential surface issues after PiP
+        mainHandler.postDelayed({
+            refreshStream()
+        }, 100)
+    }
+    
+    /**
+     * Refresh the stream by re-attaching it
+     * Called after exiting PiP to fix potential surface rendering issues
+     */
+    private fun refreshStream() {
+        val device = currentImageDevice as? Device
+        val urn = currentRenderedDeviceUrn
+        
+        if (device != null && urn != null) {
+            Log.i("ExpoIVSRemoteStreamView", "🔄 Refreshing stream for device: $urn")
+            
+            // Clear current URN to force re-render
+            currentRenderedDeviceUrn = null
+            
+            // Re-render the stream
+            renderStream(device)
+        } else {
+            Log.w("ExpoIVSRemoteStreamView", "🔄 Cannot refresh - no device or URN available")
+        }
+    }
+    
+    /**
+     * Register this view as the PiP source view
+     * Called when this view starts rendering a stream
+     */
+    private fun registerForPiP() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && !isRegisteredForPiP) {
+            val pipManager = PictureInPictureManager.getInstance()
+            pipManager.setSourceView(this)
+            pipManager.addStateListener(this)
+            isRegisteredForPiP = true
+            Log.i("ExpoIVSRemoteStreamView", "📺 Registered as PiP source view")
+        }
+    }
+    
+    /**
+     * Unregister this view from PiP
+     */
+    private fun unregisterFromPiP() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isRegisteredForPiP) {
+            val pipManager = PictureInPictureManager.getInstance()
+            pipManager.removeStateListener(this)
+            pipManager.setSourceView(null)
+            isRegisteredForPiP = false
+            Log.i("ExpoIVSRemoteStreamView", "📺 Unregistered from PiP source view")
+        }
+    }
+    
+    /**
+     * Set up frame callback to monitor frame flow
+     */
+    private fun setupFrameCallback(imageDevice: ImageDevice) {
+        imageDevice.setOnFrameCallback { frame: ImageDeviceFrame ->
+            frameCount++
+            lastFrameTime = System.currentTimeMillis()
+            
+            // Log frame info periodically (every 60 frames ~2 seconds at 30fps)
+            if (frameCount % 60 == 0L) {
+                Log.d("ExpoIVSRemoteStreamView", "📹 Frame #$frameCount received, size: ${frame.size.x}x${frame.size.y}, inPiP: $isInPiPMode")
+            }
+        }
+        Log.i("ExpoIVSRemoteStreamView", "📹 Frame callback set up for device: ${imageDevice.descriptor.urn}")
+    }
+    
+    /**
+     * Clear frame callback
+     */
+    private fun clearFrameCallback() {
+        currentImageDevice?.setOnFrameCallback(null)
+        Log.d("ExpoIVSRemoteStreamView", "📹 Frame callback cleared")
+    }
+    
     private fun resolveStageManagerWithRetry() {
         Log.d("ExpoIVSRemoteStreamView", "Attempting to resolve StageManager singleton (attempt ${retryCount + 1})...")
         
@@ -82,23 +191,47 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
                 return
             }
             
-            val newPreview = imageDevice.previewView
-            if (newPreview == null) {
-                Log.e("ExpoIVSRemoteStreamView", "Failed to get preview view from ImageDevice")
-                return
+            // Use SurfaceView instead of TextureView for better PiP performance
+            // SurfaceView renders to its own window which works better when Activity enters PiP
+            val aspectMode = when (scaleMode.lowercase()) {
+                "fill" -> AspectMode.FILL
+                "fit" -> AspectMode.FIT
+                else -> AspectMode.FILL
             }
             
-            newPreview.layoutParams = FrameLayout.LayoutParams(
-                FrameLayout.LayoutParams.MATCH_PARENT,
-                FrameLayout.LayoutParams.MATCH_PARENT
-            )
-
-            this.ivsImagePreviewView = newPreview
-            this.currentRenderedDeviceUrn = device.descriptor.urn
-            addView(newPreview)
+            val newSurfaceView = try {
+                imageDevice.getPreviewSurfaceView(aspectMode)
+            } catch (e: Exception) {
+                Log.w("ExpoIVSRemoteStreamView", "getPreviewSurfaceView failed, trying previewView: ${e.message}")
+                // Fall back to TextureView if SurfaceView fails
+                null
+            }
             
-            applyProps()
-            Log.i("ExpoIVSRemoteStreamView", "✅ Successfully rendering stream for device: ${device.descriptor.urn}")
+            if (newSurfaceView != null) {
+                newSurfaceView.layoutParams = FrameLayout.LayoutParams(
+                    FrameLayout.LayoutParams.MATCH_PARENT,
+                    FrameLayout.LayoutParams.MATCH_PARENT
+                )
+                
+                // SurfaceView needs to be on top for PiP
+                newSurfaceView.setZOrderOnTop(false)
+                newSurfaceView.setZOrderMediaOverlay(true)
+                
+                this.ivsSurfaceView = newSurfaceView
+                this.currentImageDevice = imageDevice
+                this.currentRenderedDeviceUrn = device.descriptor.urn
+                addView(newSurfaceView)
+                
+                // Set up frame callback for monitoring
+                setupFrameCallback(imageDevice)
+                
+                // Register as PiP source view
+                registerForPiP()
+                
+                Log.i("ExpoIVSRemoteStreamView", "✅ Successfully rendering stream with SurfaceView for device: ${device.descriptor.urn}")
+            } else {
+                Log.e("ExpoIVSRemoteStreamView", "Failed to get preview surface view from ImageDevice")
+            }
         } catch (e: Exception) {
             Log.e("ExpoIVSRemoteStreamView", "❌ Failed to render stream: ${e.message}", e)
             cleanupStreamView()
@@ -114,12 +247,18 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
     }
 
     private fun cleanupStreamView() {
-        if (ivsImagePreviewView != null) {
+        // Clear frame callback first
+        clearFrameCallback()
+        
+        if (ivsSurfaceView != null) {
             Log.d("ExpoIVSRemoteStreamView", "Cleaning up stream view")
-            removeView(ivsImagePreviewView)
-            ivsImagePreviewView = null
+            removeView(ivsSurfaceView)
+            ivsSurfaceView = null
         }
+        currentImageDevice = null
         currentRenderedDeviceUrn = null
+        frameCount = 0
+        lastFrameTime = 0
     }
 
     private fun applyProps() {
@@ -134,8 +273,11 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
             else -> AspectMode.FIT
         }
         try {
-            val method = ivsImagePreviewView?.javaClass?.getMethod("setPreviewAspectMode", AspectMode::class.java)
-            method?.invoke(ivsImagePreviewView, aspectMode)
+            // Try to set aspect mode on SurfaceView
+            ivsSurfaceView?.let { surface ->
+                val method = surface.javaClass.getMethod("setPreviewAspectMode", AspectMode::class.java)
+                method.invoke(surface, aspectMode)
+            }
         } catch (e: Exception) {
             Log.w("ExpoIVSRemoteStreamView", "Could not set aspect mode: ${e.message}")
         }
@@ -143,11 +285,17 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
 
     override fun onAttachedToWindow() {
         super.onAttachedToWindow()
-        Log.d("ExpoIVSRemoteStreamView", "View attached to window")
+        Log.d("ExpoIVSRemoteStreamView", "View attached to window, currentURN: $currentRenderedDeviceUrn")
+        
         // Re-register if we lost the manager reference
         if (stageManager == null) {
             retryCount = 0
             resolveStageManagerWithRetry()
+        } else {
+            // Already have manager - re-register to get stream assigned
+            // This handles the case when returning from PiP mode
+            Log.d("ExpoIVSRemoteStreamView", "Re-registering with existing manager")
+            stageManager?.registerRemoteView(this)
         }
     }
 
@@ -157,6 +305,7 @@ class ExpoIVSRemoteStreamView(context: Context, appContext: AppContext) : ExpoVi
         // Remove any pending retries
         mainHandler.removeCallbacksAndMessages(null)
         stageManager?.unregisterRemoteView(this)
+        unregisterFromPiP()
         cleanupStreamView()
     }
 }

package/android/src/main/java/expo/modules/realtimeivsbroadcast/ExpoIVSStagePreviewView.kt

@@ -33,6 +33,9 @@ class ExpoIVSStagePreviewView(context: Context, appContext: AppContext) : ExpoVi
     // Track view attachment state
     @Volatile
     private var isViewAttached = false
+    
+    // PiP registration flag
+    private var isRegisteredForPiP = false
 
     init {
         Log.i("ExpoIVSStagePreviewView", "Initializing Stage Preview View...")
@@ -40,6 +43,30 @@ class ExpoIVSStagePreviewView(context: Context, appContext: AppContext) : ExpoVi
         // The initialization will happen in onAttachedToWindow when the view is ready
     }
     
+    /**
+     * Register this view as the PiP source view (for local/broadcaster mode)
+     */
+    private fun registerForPiP() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && !isRegisteredForPiP) {
+            val pipManager = PictureInPictureManager.getInstance()
+            pipManager.setSourceView(this)
+            isRegisteredForPiP = true
+            Log.i("ExpoIVSStagePreviewView", "📺 Registered as PiP source view (local)")
+        }
+    }
+    
+    /**
+     * Unregister this view from PiP
+     */
+    private fun unregisterFromPiP() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isRegisteredForPiP) {
+            val pipManager = PictureInPictureManager.getInstance()
+            pipManager.setSourceView(null)
+            isRegisteredForPiP = false
+            Log.i("ExpoIVSStagePreviewView", "📺 Unregistered from PiP source view")
+        }
+    }
+    
     // Override onLayout to ensure child views are properly sized
     override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) {
         super.onLayout(changed, left, top, right, bottom)
@@ -278,6 +305,9 @@ class ExpoIVSStagePreviewView(context: Context, appContext: AppContext) : ExpoVi
             Log.i("ExpoIVSStagePreviewView", "✅ Successfully attached camera preview for: ${cameraDevice.descriptor.friendlyName}")
             Log.i("ExpoIVSStagePreviewView", "📷 Preview dimensions: ${newPreview.width}x${newPreview.height}, visibility: ${newPreview.visibility}")
             Log.i("ExpoIVSStagePreviewView", "📷 Parent dimensions: ${this.width}x${this.height}, childCount: ${this.childCount}")
+            
+            // Register as PiP source view for broadcaster mode
+            registerForPiP()
         } catch (e: Exception) {
             Log.e("ExpoIVSStagePreviewView", "❌ Failed to attach camera preview: ${e.message}", e)
             this.ivsImagePreviewView = null
@@ -426,6 +456,9 @@ class ExpoIVSStagePreviewView(context: Context, appContext: AppContext) : ExpoVi
         // Unregister from manager BEFORE touching views
         stageManager?.unregisterPreviewView(this)
         
+        // Unregister from PiP
+        unregisterFromPiP()
+        
         // Capture reference before clearing
         val preview = ivsImagePreviewView
         

package/android/src/main/java/expo/modules/realtimeivsbroadcast/ExpoRealtimeIvsBroadcastModule.kt

@@ -1,8 +1,10 @@
 package expo.modules.realtimeivsbroadcast
 
 import android.Manifest
+import android.app.Activity
 import android.content.pm.PackageManager
 import android.os.Build
+import android.util.Rational
 import androidx.annotation.RequiresApi
 import androidx.core.content.ContextCompat
 import android.util.Log
@@ -10,7 +12,17 @@ import expo.modules.kotlin.modules.Module
 import expo.modules.kotlin.modules.ModuleDefinition
 import expo.modules.kotlin.exception.Exceptions
 
-class ExpoRealtimeIvsBroadcastModule : Module(), IVSStageManagerDelegate {
+class ExpoRealtimeIvsBroadcastModule : Module(), IVSStageManagerDelegate, PictureInPictureDelegate {
+    
+    // PiP Manager reference (lazy initialized)
+    private val pipManager: PictureInPictureManager? by lazy {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            PictureInPictureManager.getInstance().also {
+                it.delegate = this
+            }
+        } else null
+    }
+    
     @RequiresApi(Build.VERSION_CODES.P)
     override fun definition() = ModuleDefinition {
         Name("ExpoRealtimeIvsBroadcast")
@@ -22,7 +34,9 @@ class ExpoRealtimeIvsBroadcastModule : Module(), IVSStageManagerDelegate {
             "onParticipantStreamsAdded",
             "onParticipantStreamsRemoved",
             "onPublishStateChanged",
-            "onStageError"
+            "onStageError",
+            "onPiPStateChanged",
+            "onPiPError"
         )
 
         OnCreate {
@@ -82,6 +96,72 @@ class ExpoRealtimeIvsBroadcastModule : Module(), IVSStageManagerDelegate {
             IVSStageManager.instance?.setMicrophoneMuted(muted)
         }
 
+        // --- Picture-in-Picture Methods ---
+        
+        AsyncFunction("enablePictureInPicture") { options: Map<String, Any>? ->
+            if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
+                Log.w("ExpoRealtimeIvsBroadcast", "PiP requires Android O (API 26) or higher")
+                return@AsyncFunction false
+            }
+            
+            val activity = appContext.currentActivity
+            if (activity == null) {
+                Log.e("ExpoRealtimeIvsBroadcast", "No activity available for PiP")
+                return@AsyncFunction false
+            }
+            
+            val pipOptions = PiPOptions().apply {
+                options?.let { opts ->
+                    (opts["autoEnterOnBackground"] as? Boolean)?.let { autoEnterOnBackground = it }
+                    (opts["sourceView"] as? String)?.let { 
+                        sourceView = if (it == "local") PiPOptions.PiPSourceView.LOCAL else PiPOptions.PiPSourceView.REMOTE
+                    }
+                    (opts["preferredAspectRatio"] as? Map<*, *>)?.let { ratio ->
+                        val width = (ratio["width"] as? Number)?.toInt() ?: 9
+                        val height = (ratio["height"] as? Number)?.toInt() ?: 16
+                        preferredAspectRatio = Rational(width, height)
+                    }
+                }
+            }
+            
+            return@AsyncFunction pipManager?.enable(activity, pipOptions) ?: false
+        }
+        
+        AsyncFunction("disablePictureInPicture") {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                pipManager?.disable()
+            }
+        }
+        
+        AsyncFunction("startPictureInPicture") {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                pipManager?.start()
+            }
+        }
+        
+        AsyncFunction("stopPictureInPicture") {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                pipManager?.stop()
+            }
+        }
+        
+        AsyncFunction("isPictureInPictureActive") {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                return@AsyncFunction pipManager?.isActive() ?: false
+            }
+            return@AsyncFunction false
+        }
+        
+        AsyncFunction("isPictureInPictureSupported") {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+                val activity = appContext.currentActivity
+                return@AsyncFunction if (activity != null) {
+                    pipManager?.isPiPSupported(activity) ?: false
+                } else false
+            }
+            return@AsyncFunction false
+        }
+
         // --- View Definitions ---
         
         View(ExpoIVSStagePreviewView::class) {
@@ -107,4 +187,14 @@ class ExpoRealtimeIvsBroadcastModule : Module(), IVSStageManagerDelegate {
     override fun stageManagerDidEmitEvent(eventName: String, body: Map<String, Any?>) {
         sendEvent(eventName, body)
     }
+    
+    // --- PictureInPictureDelegate Implementation ---
+    
+    override fun onPiPStateChanged(state: String) {
+        sendEvent("onPiPStateChanged", mapOf("state" to state))
+    }
+    
+    override fun onPiPError(error: String) {
+        sendEvent("onPiPError", mapOf("error" to error))
+    }
 }

package/android/src/main/java/expo/modules/realtimeivsbroadcast/IVSStageManager.kt

@@ -245,8 +245,11 @@ class IVSStageManager(private val context: Context) : Stage.Strategy, StageRende
         remoteViews.removeAll { it.get() == null }
         
         // Check if this view is already registered
-        if (remoteViews.any { it.get() === view }) {
-            Log.w("ExpoIVSStageManager", "🧠 [MANAGER] View already registered, skipping...")
+        val existingView = remoteViews.find { it.get() === view }
+        if (existingView != null) {
+            Log.i("ExpoIVSStageManager", "🧠 [MANAGER] View already registered, triggering stream assignment anyway...")
+            // Still call assignStreamsToAvailableViews in case the view needs a stream
+            assignStreamsToAvailableViews()
             return
         }