@elizaos/capacitor-screencapture 2.0.0-beta.1 → 2.0.3-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +120 -0
- package/android/build.gradle +16 -2
- package/android/src/main/AndroidManifest.xml +9 -0
- package/android/src/main/java/ai/eliza/plugins/screencapture/ScreenCaptureFgService.kt +59 -0
- package/android/src/main/java/ai/eliza/plugins/screencapture/ScreenCapturePlugin.kt +149 -37
- package/dist/esm/web.d.ts.map +1 -1
- package/dist/esm/web.js +48 -9
- package/dist/esm/web.test.d.ts +2 -0
- package/dist/esm/web.test.d.ts.map +1 -0
- package/dist/esm/web.test.js +313 -0
- package/dist/plugin.cjs.js +48 -9
- package/dist/plugin.cjs.js.map +1 -1
- package/dist/plugin.js +48 -9
- package/dist/plugin.js.map +1 -1
- package/package.json +13 -12
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shaw Walters and elizaOS Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# @elizaos/capacitor-screencapture
|
|
2
|
+
|
|
3
|
+
Cross-platform screen capture Capacitor plugin for elizaOS. Captures screenshots and records the screen on browser (Web API), iOS (ReplayKit + AVFoundation), and Android (MediaProjection).
|
|
4
|
+
|
|
5
|
+
## What it does
|
|
6
|
+
|
|
7
|
+
- **Screenshot:** Captures a single frame of the screen as a base64-encoded PNG, JPEG, or WebP image.
|
|
8
|
+
- **Screen recording:** Records the screen to a video file (WebM on browser, MP4 on native), with optional audio from the system, microphone, or both.
|
|
9
|
+
- **Pause / resume:** Pause and resume an active recording without creating a new file (Android requires API 24+).
|
|
10
|
+
- **Permission checks:** Query and request screen-capture and microphone permissions in a unified cross-platform API.
|
|
11
|
+
- **Live events:** Subscribe to `recordingState` and `error` events emitted during recording.
|
|
12
|
+
|
|
13
|
+
## Platform support
|
|
14
|
+
|
|
15
|
+
| Feature | Browser | iOS | Android |
|
|
16
|
+
|---------|---------|-----|---------|
|
|
17
|
+
| Screenshot | Yes (getDisplayMedia) | Yes (UIKit) | Yes (MediaProjection) |
|
|
18
|
+
| Screen recording | Yes (MediaRecorder) | Yes (ReplayKit) | Yes (MediaRecorder) |
|
|
19
|
+
| Pause/resume | Yes | Yes | API 24+ only |
|
|
20
|
+
| System audio | Browser-dependent | Yes (RPSampleBufferType.audioApp) | No (microphone only) |
|
|
21
|
+
| Microphone audio | Yes | Yes | Yes |
|
|
22
|
+
|
|
23
|
+
## Installation
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
npm install @elizaos/capacitor-screencapture
|
|
27
|
+
npx cap sync
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
For iOS, add to your app's `Info.plist`:
|
|
31
|
+
```xml
|
|
32
|
+
<key>NSMicrophoneUsageDescription</key>
|
|
33
|
+
<string>Microphone is used to capture audio during screen recording.</string>
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
For Android 14+ (API 34), declare in `AndroidManifest.xml`:
|
|
37
|
+
```xml
|
|
38
|
+
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
|
|
39
|
+
```
|
|
40
|
+
This is in addition to the `FOREGROUND_SERVICE` permission that this plugin's own `AndroidManifest.xml` already declares.
|
|
41
|
+
|
|
42
|
+
## Usage
|
|
43
|
+
|
|
44
|
+
```typescript
|
|
45
|
+
import { ScreenCapture } from '@elizaos/capacitor-screencapture';
|
|
46
|
+
|
|
47
|
+
// Check support
|
|
48
|
+
const { supported, features } = await ScreenCapture.isSupported();
|
|
49
|
+
|
|
50
|
+
// Take a screenshot
|
|
51
|
+
const shot = await ScreenCapture.captureScreenshot({ format: 'png', quality: 100 });
|
|
52
|
+
// shot.base64 contains the image data
|
|
53
|
+
|
|
54
|
+
// Record the screen
|
|
55
|
+
await ScreenCapture.startRecording({
|
|
56
|
+
quality: 'high', // 'low' | 'medium' | 'high' | 'highest'
|
|
57
|
+
fps: 30,
|
|
58
|
+
captureSystemAudio: true,
|
|
59
|
+
captureMicrophone: false,
|
|
60
|
+
maxDuration: 300, // seconds; undefined = unlimited
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// Listen for state updates
|
|
64
|
+
const handle = await ScreenCapture.addListener('recordingState', (state) => {
|
|
65
|
+
console.log(`Recording: ${state.isRecording}, duration: ${state.duration}s`);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// Stop and get the result
|
|
69
|
+
const result = await ScreenCapture.stopRecording();
|
|
70
|
+
// result.path — blob: URL (browser) or filesystem path (native)
|
|
71
|
+
// result.mimeType — video/webm (browser) or video/mp4 (native)
|
|
72
|
+
|
|
73
|
+
await handle.remove();
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
## Permissions
|
|
77
|
+
|
|
78
|
+
Screen capture permission works differently per platform:
|
|
79
|
+
|
|
80
|
+
- **Browser:** `getDisplayMedia` always shows an OS-level picker dialog. There is no way to pre-grant or query this permission. `checkPermissions()` returns `"prompt"` when the API is available.
|
|
81
|
+
- **iOS:** Screenshots use UIKit only — no permission required. `startRecording` uses ReplayKit, which shows a system broadcast picker on first use.
|
|
82
|
+
- **Android:** `captureScreenshot` and `startRecording` both trigger a `MediaProjection` consent dialog. Microphone permission (`RECORD_AUDIO`) is requested at runtime when `captureMicrophone: true`.
|
|
83
|
+
|
|
84
|
+
```typescript
|
|
85
|
+
// Check permissions
|
|
86
|
+
const status = await ScreenCapture.checkPermissions();
|
|
87
|
+
// status.screenCapture: 'granted' | 'denied' | 'prompt' | 'not_supported'
|
|
88
|
+
// status.microphone: 'granted' | 'denied' | 'prompt'
|
|
89
|
+
|
|
90
|
+
// Request microphone permission before recording with audio
|
|
91
|
+
await ScreenCapture.requestPermissions();
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Output formats
|
|
95
|
+
|
|
96
|
+
| Platform | Screenshot | Recording |
|
|
97
|
+
|----------|-----------|-----------|
|
|
98
|
+
| Browser | PNG / JPEG / WebP | video/webm (VP9 preferred), blob: URL |
|
|
99
|
+
| iOS | PNG / JPEG / WebP (WebP requires iOS 14+) | video/mp4 (H.264 + AAC), file:// path |
|
|
100
|
+
| Android | PNG / JPEG / WebP (WebP lossy requires API 30+) | video/mp4 (H.264), file path |
|
|
101
|
+
|
|
102
|
+
## Recording options
|
|
103
|
+
|
|
104
|
+
| Option | Type | Default | Description |
|
|
105
|
+
|--------|------|---------|-------------|
|
|
106
|
+
| `quality` | `'low' \| 'medium' \| 'high' \| 'highest'` | `'high'` | Quality preset (sets bitrate on iOS; sets fps + bitrate on Android) |
|
|
107
|
+
| `fps` | `number` | 30 | Frames per second (1–60; overrides quality preset) |
|
|
108
|
+
| `bitrate` | `number` | Estimated | Video bitrate in bits/s (overrides quality preset) |
|
|
109
|
+
| `maxDuration` | `number` | unlimited | Stop automatically after N seconds |
|
|
110
|
+
| `maxFileSize` | `number` | unlimited | Stop automatically after N bytes |
|
|
111
|
+
| `captureSystemAudio` | `boolean` | `true` | Include app/system audio (browser + iOS; Android records microphone only) |
|
|
112
|
+
| `captureMicrophone` | `boolean` | `false` | Include microphone audio in recording |
|
|
113
|
+
|
|
114
|
+
## Building from source
|
|
115
|
+
|
|
116
|
+
```bash
|
|
117
|
+
bun run --cwd plugins/plugin-native-screencapture build
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
This runs `tsc` then `rollup` and outputs `dist/esm/`, `dist/plugin.js` (IIFE), and `dist/plugin.cjs.js`.
|
package/android/build.gradle
CHANGED
|
@@ -7,6 +7,16 @@ ext {
|
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
apply plugin: 'com.android.library'
|
|
10
|
+
// Explicitly apply the Kotlin Android plugin. The kotlin-gradle-plugin is on
|
|
11
|
+
// the root buildscript classpath, but without applying it here AGP 8.13 falls
|
|
12
|
+
// back to its "built-in Kotlin" compile path (build/intermediates/
|
|
13
|
+
// built_in_kotlinc), which compiles the .kt sources but does NOT bundle the
|
|
14
|
+
// resulting .class files into the *release* library jar. The app's
|
|
15
|
+
// :app:assembleRelease then links a library AAR with zero plugin classes, so
|
|
16
|
+
// the Capacitor plugin (and any manifest-declared component) is absent from
|
|
17
|
+
// the release dex. Applying the standard Kotlin plugin wires Kotlin
|
|
18
|
+
// compilation into both the debug and release jar-bundling tasks.
|
|
19
|
+
apply plugin: 'org.jetbrains.kotlin.android'
|
|
10
20
|
android {
|
|
11
21
|
namespace = "ai.eliza.plugins.screencapture"
|
|
12
22
|
compileSdk project.hasProperty('compileSdkVersion') ? rootProject.ext.compileSdkVersion : 34
|
|
@@ -25,8 +35,12 @@ android {
|
|
|
25
35
|
}
|
|
26
36
|
|
|
27
37
|
compileOptions {
|
|
28
|
-
sourceCompatibility JavaVersion.
|
|
29
|
-
targetCompatibility JavaVersion.
|
|
38
|
+
sourceCompatibility JavaVersion.VERSION_21
|
|
39
|
+
targetCompatibility JavaVersion.VERSION_21
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
kotlinOptions {
|
|
43
|
+
jvmTarget = "21"
|
|
30
44
|
}
|
|
31
45
|
|
|
32
46
|
}
|
|
@@ -3,4 +3,13 @@
|
|
|
3
3
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
|
4
4
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
|
5
5
|
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" android:minSdkVersion="34" />
|
|
6
|
+
|
|
7
|
+
<application>
|
|
8
|
+
<!-- Android 14+ requires a running mediaProjection FGS before
|
|
9
|
+
getMediaProjection(); the plugin starts/stops this around a capture. -->
|
|
10
|
+
<service
|
|
11
|
+
android:name="ai.eliza.plugins.screencapture.ScreenCaptureFgService"
|
|
12
|
+
android:exported="false"
|
|
13
|
+
android:foregroundServiceType="mediaProjection" />
|
|
14
|
+
</application>
|
|
6
15
|
</manifest>
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
package ai.eliza.plugins.screencapture
|
|
2
|
+
|
|
3
|
+
import android.app.Notification
|
|
4
|
+
import android.app.NotificationChannel
|
|
5
|
+
import android.app.NotificationManager
|
|
6
|
+
import android.app.Service
|
|
7
|
+
import android.content.Intent
|
|
8
|
+
import android.content.pm.ServiceInfo
|
|
9
|
+
import android.os.Build
|
|
10
|
+
import android.os.IBinder
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Foreground service of type `mediaProjection`. Android 14+ (API 34) throws
|
|
14
|
+
* SecurityException: Media projections require a foreground service of type
|
|
15
|
+
* ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION
|
|
16
|
+
* from `MediaProjectionManager.getMediaProjection()` unless such a service is
|
|
17
|
+
* already running. The screencapture plugin starts this service immediately
|
|
18
|
+
* after the user grants the projection consent and stops it when the projection
|
|
19
|
+
* is released, so a single screenshot / continuous capture works on Android 14+.
|
|
20
|
+
*/
|
|
21
|
+
class ScreenCaptureFgService : Service() {
|
|
22
|
+
override fun onBind(intent: Intent?): IBinder? = null
|
|
23
|
+
|
|
24
|
+
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
|
|
25
|
+
val channelId = "eliza_screen_capture"
|
|
26
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
|
|
27
|
+
val nm = getSystemService(NotificationManager::class.java)
|
|
28
|
+
if (nm != null && nm.getNotificationChannel(channelId) == null) {
|
|
29
|
+
nm.createNotificationChannel(
|
|
30
|
+
NotificationChannel(
|
|
31
|
+
channelId,
|
|
32
|
+
"Screen capture",
|
|
33
|
+
NotificationManager.IMPORTANCE_LOW
|
|
34
|
+
)
|
|
35
|
+
)
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
val notification: Notification = Notification.Builder(this, channelId)
|
|
39
|
+
.setContentTitle("Screen sharing active")
|
|
40
|
+
.setContentText("This app is reading the screen.")
|
|
41
|
+
.setSmallIcon(android.R.drawable.ic_menu_camera)
|
|
42
|
+
.setOngoing(true)
|
|
43
|
+
.build()
|
|
44
|
+
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
|
45
|
+
startForeground(
|
|
46
|
+
NOTIF_ID,
|
|
47
|
+
notification,
|
|
48
|
+
ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION
|
|
49
|
+
)
|
|
50
|
+
} else {
|
|
51
|
+
startForeground(NOTIF_ID, notification)
|
|
52
|
+
}
|
|
53
|
+
return START_NOT_STICKY
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
companion object {
|
|
57
|
+
const val NOTIF_ID = 8451
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -7,6 +7,7 @@ import android.app.NotificationChannel
|
|
|
7
7
|
import android.app.NotificationManager
|
|
8
8
|
import android.content.Context
|
|
9
9
|
import android.content.Intent
|
|
10
|
+
import androidx.core.content.ContextCompat
|
|
10
11
|
import android.graphics.Bitmap
|
|
11
12
|
import android.graphics.PixelFormat
|
|
12
13
|
import android.hardware.display.DisplayManager
|
|
@@ -57,6 +58,10 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
57
58
|
private var virtualDisplay: VirtualDisplay? = null
|
|
58
59
|
private var mediaRecorder: MediaRecorder? = null
|
|
59
60
|
private var imageReader: ImageReader? = null
|
|
61
|
+
// Dimensions of the warm screenshot VirtualDisplay, so repeated captures at
|
|
62
|
+
// the same scale reuse it instead of re-creating it each frame.
|
|
63
|
+
private var screenshotVdWidth = 0
|
|
64
|
+
private var screenshotVdHeight = 0
|
|
60
65
|
|
|
61
66
|
// Recording state
|
|
62
67
|
private var isRecording = false
|
|
@@ -153,6 +158,17 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
153
158
|
pendingCall = call
|
|
154
159
|
pendingAction = "screenshot"
|
|
155
160
|
|
|
161
|
+
// Reuse an already-granted session projection instead of re-prompting
|
|
162
|
+
// the system consent dialog on every capture. Continuous screen
|
|
163
|
+
// understanding (EPIC #9105) captures repeatedly; tearing the projection
|
|
164
|
+
// down + re-consenting per frame is a battery/latency/UX killer. The
|
|
165
|
+
// projection is kept warm across captures and only released on
|
|
166
|
+
// background/destroy or when the system stops it.
|
|
167
|
+
if (mediaProjection != null) {
|
|
168
|
+
captureScreenshotInternal(call)
|
|
169
|
+
return
|
|
170
|
+
}
|
|
171
|
+
|
|
156
172
|
val intent = mediaProjectionManager?.createScreenCaptureIntent()
|
|
157
173
|
if (intent != null) {
|
|
158
174
|
startActivityForResult(call, intent, "handleProjectionResult")
|
|
@@ -339,27 +355,69 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
339
355
|
return
|
|
340
356
|
}
|
|
341
357
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
358
|
+
// Android 14+ (API 34) throws SecurityException from getMediaProjection()
|
|
359
|
+
// unless a foreground service of type mediaProjection is ALREADY running.
|
|
360
|
+
// Start it now, then acquire the projection once it has gone foreground.
|
|
361
|
+
try {
|
|
362
|
+
ContextCompat.startForegroundService(
|
|
363
|
+
context,
|
|
364
|
+
Intent(context, ScreenCaptureFgService::class.java)
|
|
365
|
+
)
|
|
366
|
+
} catch (e: Exception) {
|
|
367
|
+
Log.w(TAG, "Could not start mediaProjection FGS", e)
|
|
347
368
|
}
|
|
348
369
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
370
|
+
Handler(Looper.getMainLooper()).postDelayed({
|
|
371
|
+
try {
|
|
372
|
+
mediaProjection =
|
|
373
|
+
mediaProjectionManager?.getMediaProjection(result.resultCode, result.data!!)
|
|
374
|
+
} catch (e: Exception) {
|
|
375
|
+
Log.e(TAG, "getMediaProjection failed", e)
|
|
376
|
+
stopFgService()
|
|
377
|
+
call.reject("Media projection failed: ${e.message}")
|
|
378
|
+
return@postDelayed
|
|
379
|
+
}
|
|
380
|
+
if (mediaProjection == null) {
|
|
381
|
+
stopFgService()
|
|
382
|
+
call.reject("Failed to get media projection")
|
|
383
|
+
return@postDelayed
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
// Register stop callback for cleanup
|
|
387
|
+
mediaProjection?.registerCallback(object : MediaProjection.Callback() {
|
|
388
|
+
override fun onStop() {
|
|
389
|
+
Log.d(TAG, "MediaProjection stopped by system")
|
|
390
|
+
if (isRecording) {
|
|
391
|
+
scope.launch { stopRecordingInternal() }
|
|
392
|
+
}
|
|
393
|
+
// The session projection is gone — drop our warm references so
|
|
394
|
+
// the next captureScreenshot re-acquires consent cleanly instead
|
|
395
|
+
// of using a dead projection.
|
|
396
|
+
virtualDisplay?.release()
|
|
397
|
+
virtualDisplay = null
|
|
398
|
+
imageReader?.close()
|
|
399
|
+
imageReader = null
|
|
400
|
+
screenshotVdWidth = 0
|
|
401
|
+
screenshotVdHeight = 0
|
|
402
|
+
mediaProjection = null
|
|
403
|
+
stopFgService()
|
|
355
404
|
}
|
|
405
|
+
}, Handler(Looper.getMainLooper()))
|
|
406
|
+
|
|
407
|
+
when (pendingAction) {
|
|
408
|
+
"screenshot" -> captureScreenshotInternal(call)
|
|
409
|
+
"recording" -> startRecordingInternal(call)
|
|
410
|
+
else -> call.reject("Unknown action")
|
|
356
411
|
}
|
|
357
|
-
},
|
|
412
|
+
}, 600)
|
|
413
|
+
}
|
|
358
414
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
415
|
+
/** Stop the mediaProjection foreground service (idempotent). */
|
|
416
|
+
private fun stopFgService() {
|
|
417
|
+
try {
|
|
418
|
+
context.stopService(Intent(context, ScreenCaptureFgService::class.java))
|
|
419
|
+
} catch (e: Exception) {
|
|
420
|
+
Log.w(TAG, "stopFgService failed", e)
|
|
363
421
|
}
|
|
364
422
|
}
|
|
365
423
|
|
|
@@ -370,23 +428,38 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
370
428
|
val quality = call.getInt("quality") ?: 100
|
|
371
429
|
val scale = call.getFloat("scale") ?: 1f
|
|
372
430
|
|
|
373
|
-
val width = (screenWidth * scale).toInt()
|
|
374
|
-
val height = (screenHeight * scale).toInt()
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
431
|
+
val width = Math.max(1, (screenWidth * scale).toInt())
|
|
432
|
+
val height = Math.max(1, (screenHeight * scale).toInt())
|
|
433
|
+
|
|
434
|
+
// Reuse the warm VirtualDisplay + ImageReader when the requested size
|
|
435
|
+
// matches; only (re)create them when missing or the scale changed. The
|
|
436
|
+
// resize itself is native — the VirtualDisplay renders directly at the
|
|
437
|
+
// target resolution, so the agent never resizes pixels in JS.
|
|
438
|
+
val warm = virtualDisplay != null &&
|
|
439
|
+
imageReader != null &&
|
|
440
|
+
screenshotVdWidth == width &&
|
|
441
|
+
screenshotVdHeight == height
|
|
442
|
+
if (!warm) {
|
|
443
|
+
virtualDisplay?.release()
|
|
444
|
+
imageReader?.close()
|
|
445
|
+
imageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, 2)
|
|
446
|
+
virtualDisplay = mediaProjection?.createVirtualDisplay(
|
|
447
|
+
"ScreenCapture",
|
|
448
|
+
width,
|
|
449
|
+
height,
|
|
450
|
+
screenDensity,
|
|
451
|
+
DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
|
|
452
|
+
imageReader?.surface,
|
|
453
|
+
null,
|
|
454
|
+
null
|
|
455
|
+
)
|
|
456
|
+
screenshotVdWidth = width
|
|
457
|
+
screenshotVdHeight = height
|
|
458
|
+
}
|
|
388
459
|
|
|
389
|
-
//
|
|
460
|
+
// A freshly-created mirror needs ~250ms to render its first frame; a warm
|
|
461
|
+
// display is already mirroring continuously, so a short settle suffices.
|
|
462
|
+
val settleMs = if (warm) 60L else 250L
|
|
390
463
|
Handler(Looper.getMainLooper()).postDelayed({
|
|
391
464
|
try {
|
|
392
465
|
val image = imageReader?.acquireLatestImage()
|
|
@@ -412,7 +485,9 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
412
485
|
|
|
413
486
|
val base64 = Base64.encodeToString(outputStream.toByteArray(), Base64.NO_WRAP)
|
|
414
487
|
|
|
415
|
-
|
|
488
|
+
// Keep the projection + VirtualDisplay warm for the next
|
|
489
|
+
// capture — do NOT cleanup() here (that stopped the
|
|
490
|
+
// projection and forced a re-consent every frame).
|
|
416
491
|
|
|
417
492
|
call.resolve(JSObject().apply {
|
|
418
493
|
put("base64", base64)
|
|
@@ -422,16 +497,36 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
422
497
|
put("timestamp", System.currentTimeMillis())
|
|
423
498
|
})
|
|
424
499
|
} else {
|
|
425
|
-
|
|
500
|
+
// Transient empty frame — keep the session warm, just fail
|
|
501
|
+
// this one capture so the caller can retry on the next tick.
|
|
426
502
|
call.reject("Failed to capture screenshot")
|
|
427
503
|
}
|
|
428
504
|
} catch (e: Exception) {
|
|
429
505
|
Log.e(TAG, "Screenshot capture failed", e)
|
|
430
|
-
|
|
506
|
+
// A real error may have invalidated the projection — tear it all
|
|
507
|
+
// down so the next capture re-acquires cleanly.
|
|
508
|
+
releaseProjection()
|
|
431
509
|
notifyError("screenshot_failed", "Screenshot failed: ${e.message}")
|
|
432
510
|
call.reject("Screenshot failed: ${e.message}")
|
|
433
511
|
}
|
|
434
|
-
},
|
|
512
|
+
}, settleMs)
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
/**
|
|
516
|
+
* Full teardown of the warm screenshot projection + its mirror. Use on
|
|
517
|
+
* background/destroy or after an error invalidates the session. Recording
|
|
518
|
+
* has its own lifecycle via [cleanup].
|
|
519
|
+
*/
|
|
520
|
+
private fun releaseProjection() {
|
|
521
|
+
virtualDisplay?.release()
|
|
522
|
+
virtualDisplay = null
|
|
523
|
+
imageReader?.close()
|
|
524
|
+
imageReader = null
|
|
525
|
+
screenshotVdWidth = 0
|
|
526
|
+
screenshotVdHeight = 0
|
|
527
|
+
mediaProjection?.stop()
|
|
528
|
+
mediaProjection = null
|
|
529
|
+
stopFgService()
|
|
435
530
|
}
|
|
436
531
|
|
|
437
532
|
private fun imageToBitmap(image: Image, width: Int, height: Int): Bitmap {
|
|
@@ -755,11 +850,28 @@ class ScreenCapturePlugin : Plugin() {
|
|
|
755
850
|
virtualDisplay = null
|
|
756
851
|
imageReader?.close()
|
|
757
852
|
imageReader = null
|
|
853
|
+
screenshotVdWidth = 0
|
|
854
|
+
screenshotVdHeight = 0
|
|
758
855
|
mediaProjection?.stop()
|
|
759
856
|
mediaProjection = null
|
|
857
|
+
stopFgService()
|
|
760
858
|
}
|
|
761
859
|
|
|
762
|
-
// ──
|
|
860
|
+
// ── Lifecycle ───────────────────────────────────────────────────────
|
|
861
|
+
|
|
862
|
+
override fun handleOnStop() {
|
|
863
|
+
super.handleOnStop()
|
|
864
|
+
// Release the warm screenshot MediaProjection when the app is no longer
|
|
865
|
+
// visible. A backgrounded app must not hold a live screen-capture
|
|
866
|
+
// session — it drains battery, keeps the system cast/record indicator
|
|
867
|
+
// up, and is a privacy concern. (handleOnStop fires on full background,
|
|
868
|
+
// not the transient pause the consent dialog causes, so it won't tear
|
|
869
|
+
// down a projection mid-acquisition.) Recording owns its own projection
|
|
870
|
+
// lifecycle, so leave it alone while a recording is active.
|
|
871
|
+
if (!isRecording) {
|
|
872
|
+
releaseProjection()
|
|
873
|
+
}
|
|
874
|
+
}
|
|
763
875
|
|
|
764
876
|
override fun handleOnDestroy() {
|
|
765
877
|
super.handleOnDestroy()
|
package/dist/esm/web.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,KAAK,EACV,uBAAuB,EACvB,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,eAAe,CAAC;AAEvB,KAAK,sBAAsB,GAAG,oBAAoB,GAAG,uBAAuB,CAAC;
|
|
1
|
+
{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,KAAK,EACV,uBAAuB,EACvB,6BAA6B,EAC7B,sBAAsB,EACtB,qBAAqB,EACrB,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,EACjB,MAAM,eAAe,CAAC;AAEvB,KAAK,sBAAsB,GAAG,oBAAoB,GAAG,uBAAuB,CAAC;AAiD7E,qBAAa,gBAAiB,SAAQ,SAAS;IAC7C,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,aAAa,CAA8B;IACnD,OAAO,CAAC,cAAc,CAAc;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,kBAAkB,CAAK;IAC/B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,sBAAsB,CAA+C;IAC7E,OAAO,CAAC,eAAe,CAGf;IAEF,WAAW,IAAI,OAAO,CAAC;QAAE,SAAS,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IASlE,iBAAiB,CACrB,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,gBAAgB,CAAC;IAgEtB,cAAc,CAAC,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC;IA+G/D,aAAa,IAAI,OAAO,CAAC,qBAAqB,CAAC;IAuE/C,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC;IA2B/B,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAchC,iBAAiB,IAAI,OAAO,CAAC,oBAAoB,CAAC;IAgBxD;;;;;;;;;;OAUG;IACG,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAiBhE;;;;;;;;;;OAUG;IACG,kBAAkB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAkB5D,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,YAAY,EAAE,CAAC,KAAK,EAAE,sBAAsB,KAAK,IAAI,GACpD,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,CAAC;IAWrC,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC;IAIzC,SAAS,CAAC,eAAe,CACvB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,sBAAsB,GAC3B,IAAI;CAOR"}
|
package/dist/esm/web.js
CHANGED
|
@@ -5,8 +5,28 @@ const VIDEO_MIME_TYPES = [
|
|
|
5
5
|
"video/webm",
|
|
6
6
|
"video/mp4",
|
|
7
7
|
];
|
|
8
|
-
const getSupportedMimeType = () =>
|
|
9
|
-
|
|
8
|
+
const getSupportedMimeType = () => typeof MediaRecorder === "undefined"
|
|
9
|
+
? null
|
|
10
|
+
: (VIDEO_MIME_TYPES.find((m) => MediaRecorder.isTypeSupported(m)) ?? null);
|
|
11
|
+
const hasDisplayMedia = () => !!navigator.mediaDevices
|
|
12
|
+
?.getDisplayMedia;
|
|
13
|
+
function assertPositiveFiniteNumber(value, label) {
|
|
14
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
15
|
+
throw new Error(`${label} must be a positive finite number`);
|
|
16
|
+
}
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
function assertQuality(value) {
|
|
20
|
+
if (value === undefined)
|
|
21
|
+
return 1;
|
|
22
|
+
if (typeof value !== "number" || !Number.isFinite(value)) {
|
|
23
|
+
throw new Error("quality must be a finite number between 0 and 100");
|
|
24
|
+
}
|
|
25
|
+
if (value < 0 || value > 100) {
|
|
26
|
+
throw new Error("quality must be between 0 and 100");
|
|
27
|
+
}
|
|
28
|
+
return value / 100;
|
|
29
|
+
}
|
|
10
30
|
const getDisplayMedia = (opts) => navigator.mediaDevices.getDisplayMedia(opts);
|
|
11
31
|
export class ScreenCaptureWeb extends WebPlugin {
|
|
12
32
|
constructor() {
|
|
@@ -35,8 +55,10 @@ export class ScreenCaptureWeb extends WebPlugin {
|
|
|
35
55
|
}
|
|
36
56
|
async captureScreenshot(options) {
|
|
37
57
|
const format = options?.format || "png";
|
|
38
|
-
const quality = (options?.quality
|
|
39
|
-
const scale = options?.scale
|
|
58
|
+
const quality = assertQuality(options?.quality);
|
|
59
|
+
const scale = options?.scale === undefined
|
|
60
|
+
? 1
|
|
61
|
+
: assertPositiveFiniteNumber(options.scale, "scale");
|
|
40
62
|
// PERMISSIONS_MIGRATION: getDisplayMedia() triggers the OS screen
|
|
41
63
|
// recording / picker dialog implicitly. New flow probes via
|
|
42
64
|
// `screenRecordingProber` in
|
|
@@ -51,11 +73,16 @@ export class ScreenCaptureWeb extends WebPlugin {
|
|
|
51
73
|
const settings = track.getSettings();
|
|
52
74
|
const width = (settings.width || 1920) * scale;
|
|
53
75
|
const height = (settings.height || 1080) * scale;
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
}
|
|
76
|
+
let bitmap = null;
|
|
77
|
+
try {
|
|
78
|
+
const imageCapture = new ImageCapture(track);
|
|
79
|
+
bitmap = await imageCapture.grabFrame();
|
|
80
|
+
}
|
|
81
|
+
finally {
|
|
82
|
+
stream.getTracks().forEach((t) => {
|
|
83
|
+
t.stop();
|
|
84
|
+
});
|
|
85
|
+
}
|
|
59
86
|
const canvas = document.createElement("canvas");
|
|
60
87
|
canvas.width = width;
|
|
61
88
|
canvas.height = height;
|
|
@@ -83,6 +110,18 @@ export class ScreenCaptureWeb extends WebPlugin {
|
|
|
83
110
|
async startRecording(options) {
|
|
84
111
|
if (this.isRecording)
|
|
85
112
|
throw new Error("Recording already in progress");
|
|
113
|
+
if (options?.fps !== undefined) {
|
|
114
|
+
assertPositiveFiniteNumber(options.fps, "fps");
|
|
115
|
+
}
|
|
116
|
+
if (options?.bitrate !== undefined) {
|
|
117
|
+
assertPositiveFiniteNumber(options.bitrate, "bitrate");
|
|
118
|
+
}
|
|
119
|
+
if (options?.maxDuration !== undefined) {
|
|
120
|
+
assertPositiveFiniteNumber(options.maxDuration, "maxDuration");
|
|
121
|
+
}
|
|
122
|
+
if (options?.maxFileSize !== undefined) {
|
|
123
|
+
assertPositiveFiniteNumber(options.maxFileSize, "maxFileSize");
|
|
124
|
+
}
|
|
86
125
|
const videoConstraints = {
|
|
87
126
|
displaySurface: "monitor",
|
|
88
127
|
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web.test.d.ts","sourceRoot":"","sources":["../../src/web.test.ts"],"names":[],"mappings":""}
|