@elizaos/capacitor-screencapture 2.0.3-beta.2 → 2.0.3-beta.4

package/android/src/main/AndroidManifest.xml

@@ -3,4 +3,13 @@
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" android:minSdkVersion="34" />
+
+    <application>
+        <!-- Android 14+ requires a running mediaProjection FGS before
+             getMediaProjection(); the plugin starts/stops this around a capture. -->
+        <service
+            android:name="ai.eliza.plugins.screencapture.ScreenCaptureFgService"
+            android:exported="false"
+            android:foregroundServiceType="mediaProjection" />
+    </application>
 </manifest>

package/android/src/main/java/ai/eliza/plugins/screencapture/ScreenCaptureFgService.kt

@@ -0,0 +1,59 @@
+package ai.eliza.plugins.screencapture
+
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
+import android.app.Service
+import android.content.Intent
+import android.content.pm.ServiceInfo
+import android.os.Build
+import android.os.IBinder
+
+/**
+ * Foreground service of type `mediaProjection`. Android 14+ (API 34) throws
+ *   SecurityException: Media projections require a foreground service of type
+ *   ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION
+ * from `MediaProjectionManager.getMediaProjection()` unless such a service is
+ * already running. The screencapture plugin starts this service immediately
+ * after the user grants the projection consent and stops it when the projection
+ * is released, so a single screenshot / continuous capture works on Android 14+.
+ */
+class ScreenCaptureFgService : Service() {
+    override fun onBind(intent: Intent?): IBinder? = null
+
+    override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
+        val channelId = "eliza_screen_capture"
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
+            val nm = getSystemService(NotificationManager::class.java)
+            if (nm != null && nm.getNotificationChannel(channelId) == null) {
+                nm.createNotificationChannel(
+                    NotificationChannel(
+                        channelId,
+                        "Screen capture",
+                        NotificationManager.IMPORTANCE_LOW
+                    )
+                )
+            }
+        }
+        val notification: Notification = Notification.Builder(this, channelId)
+            .setContentTitle("Screen sharing active")
+            .setContentText("This app is reading the screen.")
+            .setSmallIcon(android.R.drawable.ic_menu_camera)
+            .setOngoing(true)
+            .build()
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            startForeground(
+                NOTIF_ID,
+                notification,
+                ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION
+            )
+        } else {
+            startForeground(NOTIF_ID, notification)
+        }
+        return START_NOT_STICKY
+    }
+
+    companion object {
+        const val NOTIF_ID = 8451
+    }
+}

package/android/src/main/java/ai/eliza/plugins/screencapture/ScreenCapturePlugin.kt

@@ -7,6 +7,7 @@ import android.app.NotificationChannel
 import android.app.NotificationManager
 import android.content.Context
 import android.content.Intent
+import androidx.core.content.ContextCompat
 import android.graphics.Bitmap
 import android.graphics.PixelFormat
 import android.hardware.display.DisplayManager
@@ -57,6 +58,10 @@ class ScreenCapturePlugin : Plugin() {
     private var virtualDisplay: VirtualDisplay? = null
     private var mediaRecorder: MediaRecorder? = null
     private var imageReader: ImageReader? = null
+    // Dimensions of the warm screenshot VirtualDisplay, so repeated captures at
+    // the same scale reuse it instead of re-creating it each frame.
+    private var screenshotVdWidth = 0
+    private var screenshotVdHeight = 0
 
     // Recording state
     private var isRecording = false
@@ -153,6 +158,17 @@ class ScreenCapturePlugin : Plugin() {
         pendingCall = call
         pendingAction = "screenshot"
 
+        // Reuse an already-granted session projection instead of re-prompting
+        // the system consent dialog on every capture. Continuous screen
+        // understanding (EPIC #9105) captures repeatedly; tearing the projection
+        // down + re-consenting per frame is a battery/latency/UX killer. The
+        // projection is kept warm across captures and only released on
+        // background/destroy or when the system stops it.
+        if (mediaProjection != null) {
+            captureScreenshotInternal(call)
+            return
+        }
+
         val intent = mediaProjectionManager?.createScreenCaptureIntent()
         if (intent != null) {
             startActivityForResult(call, intent, "handleProjectionResult")
@@ -339,27 +355,69 @@ class ScreenCapturePlugin : Plugin() {
             return
         }
 
-        mediaProjection = mediaProjectionManager?.getMediaProjection(result.resultCode, result.data!!)
-
-        if (mediaProjection == null) {
-            call.reject("Failed to get media projection")
-            return
+        // Android 14+ (API 34) throws SecurityException from getMediaProjection()
+        // unless a foreground service of type mediaProjection is ALREADY running.
+        // Start it now, then acquire the projection once it has gone foreground.
+        try {
+            ContextCompat.startForegroundService(
+                context,
+                Intent(context, ScreenCaptureFgService::class.java)
+            )
+        } catch (e: Exception) {
+            Log.w(TAG, "Could not start mediaProjection FGS", e)
         }
 
-        // Register stop callback for cleanup
-        mediaProjection?.registerCallback(object : MediaProjection.Callback() {
-            override fun onStop() {
-                Log.d(TAG, "MediaProjection stopped by system")
-                if (isRecording) {
-                    scope.launch { stopRecordingInternal() }
+        Handler(Looper.getMainLooper()).postDelayed({
+            try {
+                mediaProjection =
+                    mediaProjectionManager?.getMediaProjection(result.resultCode, result.data!!)
+            } catch (e: Exception) {
+                Log.e(TAG, "getMediaProjection failed", e)
+                stopFgService()
+                call.reject("Media projection failed: ${e.message}")
+                return@postDelayed
+            }
+            if (mediaProjection == null) {
+                stopFgService()
+                call.reject("Failed to get media projection")
+                return@postDelayed
+            }
+
+            // Register stop callback for cleanup
+            mediaProjection?.registerCallback(object : MediaProjection.Callback() {
+                override fun onStop() {
+                    Log.d(TAG, "MediaProjection stopped by system")
+                    if (isRecording) {
+                        scope.launch { stopRecordingInternal() }
+                    }
+                    // The session projection is gone — drop our warm references so
+                    // the next captureScreenshot re-acquires consent cleanly instead
+                    // of using a dead projection.
+                    virtualDisplay?.release()
+                    virtualDisplay = null
+                    imageReader?.close()
+                    imageReader = null
+                    screenshotVdWidth = 0
+                    screenshotVdHeight = 0
+                    mediaProjection = null
+                    stopFgService()
                 }
+            }, Handler(Looper.getMainLooper()))
+
+            when (pendingAction) {
+                "screenshot" -> captureScreenshotInternal(call)
+                "recording" -> startRecordingInternal(call)
+                else -> call.reject("Unknown action")
             }
-        }, Handler(Looper.getMainLooper()))
+        }, 600)
+    }
 
-        when (pendingAction) {
-            "screenshot" -> captureScreenshotInternal(call)
-            "recording" -> startRecordingInternal(call)
-            else -> call.reject("Unknown action")
+    /** Stop the mediaProjection foreground service (idempotent). */
+    private fun stopFgService() {
+        try {
+            context.stopService(Intent(context, ScreenCaptureFgService::class.java))
+        } catch (e: Exception) {
+            Log.w(TAG, "stopFgService failed", e)
         }
     }
 
@@ -370,23 +428,38 @@ class ScreenCapturePlugin : Plugin() {
         val quality = call.getInt("quality") ?: 100
         val scale = call.getFloat("scale") ?: 1f
 
-        val width = (screenWidth * scale).toInt()
-        val height = (screenHeight * scale).toInt()
-
-        imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2)
-
-        virtualDisplay = mediaProjection?.createVirtualDisplay(
-            "ScreenCapture",
-            width,
-            height,
-            screenDensity,
-            DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
-            imageReader?.surface,
-            null,
-            null
-        )
+        val width = Math.max(1, (screenWidth * scale).toInt())
+        val height = Math.max(1, (screenHeight * scale).toInt())
+
+        // Reuse the warm VirtualDisplay + ImageReader when the requested size
+        // matches; only (re)create them when missing or the scale changed. The
+        // resize itself is native — the VirtualDisplay renders directly at the
+        // target resolution, so the agent never resizes pixels in JS.
+        val warm = virtualDisplay != null &&
+            imageReader != null &&
+            screenshotVdWidth == width &&
+            screenshotVdHeight == height
+        if (!warm) {
+            virtualDisplay?.release()
+            imageReader?.close()
+            imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2)
+            virtualDisplay = mediaProjection?.createVirtualDisplay(
+                "ScreenCapture",
+                width,
+                height,
+                screenDensity,
+                DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
+                imageReader?.surface,
+                null,
+                null
+            )
+            screenshotVdWidth = width
+            screenshotVdHeight = height
+        }
 
-        // Brief delay to allow the VirtualDisplay to render a frame
+        // A freshly-created mirror needs ~250ms to render its first frame; a warm
+        // display is already mirroring continuously, so a short settle suffices.
+        val settleMs = if (warm) 60L else 250L
         Handler(Looper.getMainLooper()).postDelayed({
             try {
                 val image = imageReader?.acquireLatestImage()
@@ -412,7 +485,9 @@ class ScreenCapturePlugin : Plugin() {
 
                     val base64 = Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP)
 
-                    cleanup()
+                    // Keep the projection + VirtualDisplay warm for the next
+                    // capture — do NOT cleanup() here (that stopped the
+                    // projection and forced a re-consent every frame).
 
                     call.resolve(JSObject().apply {
                         put("base64", base64)
@@ -422,16 +497,36 @@ class ScreenCapturePlugin : Plugin() {
                         put("timestamp", System.currentTimeMillis())
                     })
                 } else {
-                    cleanup()
+                    // Transient empty frame — keep the session warm, just fail
+                    // this one capture so the caller can retry on the next tick.
                     call.reject("Failed to capture screenshot")
                 }
             } catch (e: Exception) {
                 Log.e(TAG, "Screenshot capture failed", e)
-                cleanup()
+                // A real error may have invalidated the projection — tear it all
+                // down so the next capture re-acquires cleanly.
+                releaseProjection()
                 notifyError("screenshot_failed", "Screenshot failed: ${e.message}")
                 call.reject("Screenshot failed: ${e.message}")
             }
-        }, 250) // 250ms delay to ensure frame is rendered
+        }, settleMs)
+    }
+
+    /**
+     * Full teardown of the warm screenshot projection + its mirror. Use on
+     * background/destroy or after an error invalidates the session. Recording
+     * has its own lifecycle via [cleanup].
+     */
+    private fun releaseProjection() {
+        virtualDisplay?.release()
+        virtualDisplay = null
+        imageReader?.close()
+        imageReader = null
+        screenshotVdWidth = 0
+        screenshotVdHeight = 0
+        mediaProjection?.stop()
+        mediaProjection = null
+        stopFgService()
     }
 
     private fun imageToBitmap(image: Image, width: Int, height: Int): Bitmap {
@@ -755,11 +850,28 @@ class ScreenCapturePlugin : Plugin() {
         virtualDisplay = null
         imageReader?.close()
         imageReader = null
+        screenshotVdWidth = 0
+        screenshotVdHeight = 0
         mediaProjection?.stop()
         mediaProjection = null
+        stopFgService()
     }
 
-    // ── Destroy ─────────────────────────────────────────────────────────
+    // ── Lifecycle ───────────────────────────────────────────────────────
+
+    override fun handleOnStop() {
+        super.handleOnStop()
+        // Release the warm screenshot MediaProjection when the app is no longer
+        // visible. A backgrounded app must not hold a live screen-capture
+        // session — it drains battery, keeps the system cast/record indicator
+        // up, and is a privacy concern. (handleOnStop fires on full background,
+        // not the transient pause the consent dialog causes, so it won't tear
+        // down a projection mid-acquisition.) Recording owns its own projection
+        // lifecycle, so leave it alone while a recording is active.
+        if (!isRecording) {
+            releaseProjection()
+        }
+    }
 
     override fun handleOnDestroy() {
         super.handleOnDestroy()

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@elizaos/capacitor-screencapture",
-  "version": "2.0.3-beta.2",
+  "version": "2.0.3-beta.4",
   "description": "Captures screenshots and records the screen across web, mobile, and desktop.",
   "keywords": [
     "screen-capture",
@@ -81,5 +81,5 @@
       "android": true
     }
   },
-  "gitHead": "82fe0f44215954c2417328203f5bd6510985c1fc"
+  "gitHead": "f76f55793a0fb8d6b869dd8ddce03970de061e34"
 }