@apirtc/expo-apirtc-options-plugin 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -38
- package/build/static/kotlin/BackgroundBlurModule.kt +420 -0
- package/build/static/kotlin/BackgroundBlurPackage.kt +16 -0
- package/build/static/kotlin/BackgroundImageProcessor.kt +433 -0
- package/build/static/kotlin/BlurVideoProcessor.kt +448 -0
- package/build/withAndroidPlugin.d.ts +1 -0
- package/build/withAndroidPlugin.js +73 -17
- package/build/withPlugin.d.ts +1 -0
- package/build/withPlugin.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
package com.reactnativeapirtc
|
|
2
|
+
|
|
3
|
+
import android.content.Context
|
|
4
|
+
import android.graphics.Bitmap
|
|
5
|
+
import android.graphics.Canvas
|
|
6
|
+
import android.graphics.Paint
|
|
7
|
+
import android.graphics.PorterDuff
|
|
8
|
+
import android.graphics.PorterDuffXfermode
|
|
9
|
+
import android.renderscript.Allocation
|
|
10
|
+
import android.renderscript.Element
|
|
11
|
+
import android.renderscript.RenderScript
|
|
12
|
+
import android.renderscript.ScriptIntrinsicBlur
|
|
13
|
+
import android.util.Log
|
|
14
|
+
import com.google.android.gms.tasks.Tasks
|
|
15
|
+
import com.google.mlkit.vision.common.InputImage
|
|
16
|
+
import com.google.mlkit.vision.segmentation.Segmentation
|
|
17
|
+
import com.google.mlkit.vision.segmentation.Segmenter
|
|
18
|
+
import com.google.mlkit.vision.segmentation.selfie.SelfieSegmenterOptions
|
|
19
|
+
import org.webrtc.JavaI420Buffer
|
|
20
|
+
import org.webrtc.VideoFrame
|
|
21
|
+
import org.webrtc.VideoProcessor
|
|
22
|
+
import org.webrtc.VideoSink
|
|
23
|
+
import java.nio.ByteBuffer
|
|
24
|
+
import java.nio.ByteOrder
|
|
25
|
+
import java.util.concurrent.TimeUnit
|
|
26
|
+
import java.util.concurrent.atomic.AtomicBoolean
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* BlurVideoProcessor implements WebRTC's VideoProcessor interface.
|
|
30
|
+
*
|
|
31
|
+
* When set on a VideoSource via videoSource.setVideoProcessor(processor),
|
|
32
|
+
* it intercepts every camera frame BEFORE it reaches the VideoTrack.
|
|
33
|
+
*
|
|
34
|
+
* Flow:
|
|
35
|
+
* Camera → VideoProcessor.onFrameCaptured(frame) → process → downstreamSink.onFrame(processedFrame)
|
|
36
|
+
* ↓
|
|
37
|
+
* VideoTrack → RTCView + Encoder
|
|
38
|
+
*/
|
|
39
|
+
class BlurVideoProcessor(
|
|
40
|
+
private val context: Context,
|
|
41
|
+
private val blurRadius: Float = 20f,
|
|
42
|
+
private val blurPasses: Int = 1,
|
|
43
|
+
) : VideoProcessor {
|
|
44
|
+
|
|
45
|
+
companion object {
|
|
46
|
+
private const val TAG = "BlurVideoProcessor"
|
|
47
|
+
private const val SEGMENTATION_INTERVAL = 2
|
|
48
|
+
private const val SEG_WIDTH = 256
|
|
49
|
+
private const val SEG_HEIGHT = 192
|
|
50
|
+
private const val SEGMENTATION_TIMEOUT_MS = 200L
|
|
51
|
+
private const val SEGMENTATION_TIMEOUT_FIRST_MS = 600L
|
|
52
|
+
private const val MASK_DECAY = 0.5f
|
|
53
|
+
private const val MASK_THRESHOLD_LOW = 0.25f
|
|
54
|
+
private const val MASK_THRESHOLD_HIGH = 0.50f
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
@Volatile
|
|
58
|
+
private var downstreamSink: VideoSink? = null
|
|
59
|
+
|
|
60
|
+
private val enabled = AtomicBoolean(false)
|
|
61
|
+
private var segmenter: Segmenter? = null
|
|
62
|
+
private var renderScript: RenderScript? = null
|
|
63
|
+
private var blurScript: ScriptIntrinsicBlur? = null
|
|
64
|
+
private var frameCount = 0L
|
|
65
|
+
|
|
66
|
+
@Volatile private var cachedMaskBuffer: ByteBuffer? = null
|
|
67
|
+
@Volatile private var cachedMaskWidth = 0
|
|
68
|
+
@Volatile private var cachedMaskHeight = 0
|
|
69
|
+
@Volatile private var previousMaskFloats: FloatArray? = null
|
|
70
|
+
|
|
71
|
+
private var blurredBitmap: Bitmap? = null
|
|
72
|
+
private var compositeBitmap: Bitmap? = null
|
|
73
|
+
|
|
74
|
+
// =====================================================================
|
|
75
|
+
// VideoProcessor interface
|
|
76
|
+
// =====================================================================
|
|
77
|
+
|
|
78
|
+
override fun setSink(sink: VideoSink?) {
|
|
79
|
+
Log.d(TAG, "setSink called, sink=${if (sink != null) "non-null" else "null"}")
|
|
80
|
+
downstreamSink = sink
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
override fun onCapturerStarted(success: Boolean) {
|
|
84
|
+
Log.d(TAG, "onCapturerStarted: success=$success")
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
override fun onCapturerStopped() {
|
|
88
|
+
Log.d(TAG, "onCapturerStopped")
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
override fun onFrameCaptured(frame: VideoFrame) {
|
|
92
|
+
val sink = downstreamSink
|
|
93
|
+
if (sink == null) {
|
|
94
|
+
Log.w(TAG, "onFrameCaptured: downstreamSink is null")
|
|
95
|
+
return
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
if (!enabled.get()) {
|
|
99
|
+
sink.onFrame(frame)
|
|
100
|
+
return
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
try {
|
|
104
|
+
val processedFrame = processFrame(frame)
|
|
105
|
+
if (processedFrame != null) {
|
|
106
|
+
sink.onFrame(processedFrame)
|
|
107
|
+
processedFrame.release()
|
|
108
|
+
} else {
|
|
109
|
+
sink.onFrame(frame)
|
|
110
|
+
}
|
|
111
|
+
} catch (e: Exception) {
|
|
112
|
+
Log.e(TAG, "Error in onFrameCaptured, forwarding original", e)
|
|
113
|
+
sink.onFrame(frame)
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
fun onFrame(frame: VideoFrame) {
|
|
118
|
+
onFrameCaptured(frame)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// =====================================================================
|
|
122
|
+
// Enable / Disable
|
|
123
|
+
// =====================================================================
|
|
124
|
+
|
|
125
|
+
fun enable() {
|
|
126
|
+
if (enabled.getAndSet(true)) return
|
|
127
|
+
Log.d(TAG, "Enabling blur processor")
|
|
128
|
+
|
|
129
|
+
try {
|
|
130
|
+
val options = SelfieSegmenterOptions.Builder()
|
|
131
|
+
.setDetectorMode(SelfieSegmenterOptions.STREAM_MODE)
|
|
132
|
+
.enableRawSizeMask()
|
|
133
|
+
.build()
|
|
134
|
+
segmenter = Segmentation.getClient(options)
|
|
135
|
+
Log.d(TAG, "ML Kit segmenter initialized successfully")
|
|
136
|
+
} catch (e: Exception) {
|
|
137
|
+
Log.e(TAG, "Failed to initialize ML Kit segmenter", e)
|
|
138
|
+
enabled.set(false)
|
|
139
|
+
throw e
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
renderScript = RenderScript.create(context)
|
|
144
|
+
blurScript = ScriptIntrinsicBlur.create(renderScript, Element.U8_4(renderScript))
|
|
145
|
+
blurScript?.setRadius(blurRadius.coerceIn(1f, 25f))
|
|
146
|
+
Log.d(TAG, "RenderScript initialized successfully")
|
|
147
|
+
} catch (e: Exception) {
|
|
148
|
+
Log.e(TAG, "Failed to initialize RenderScript", e)
|
|
149
|
+
segmenter?.close()
|
|
150
|
+
segmenter = null
|
|
151
|
+
enabled.set(false)
|
|
152
|
+
throw e
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
frameCount = 0
|
|
156
|
+
cachedMaskBuffer = null
|
|
157
|
+
Log.d(TAG, "Blur processor enabled")
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
fun disable() {
|
|
161
|
+
if (!enabled.getAndSet(false)) return
|
|
162
|
+
Log.d(TAG, "Disabling blur processor")
|
|
163
|
+
|
|
164
|
+
segmenter?.close()
|
|
165
|
+
segmenter = null
|
|
166
|
+
blurScript?.destroy()
|
|
167
|
+
blurScript = null
|
|
168
|
+
renderScript?.destroy()
|
|
169
|
+
renderScript = null
|
|
170
|
+
|
|
171
|
+
blurredBitmap?.recycle()
|
|
172
|
+
blurredBitmap = null
|
|
173
|
+
compositeBitmap?.recycle()
|
|
174
|
+
compositeBitmap = null
|
|
175
|
+
cachedMaskBuffer = null
|
|
176
|
+
previousMaskFloats = null
|
|
177
|
+
|
|
178
|
+
Log.d(TAG, "Blur processor disabled")
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// =====================================================================
|
|
182
|
+
// Frame Processing Pipeline
|
|
183
|
+
// =====================================================================
|
|
184
|
+
|
|
185
|
+
private fun processFrame(frame: VideoFrame): VideoFrame? {
|
|
186
|
+
val buffer = frame.buffer
|
|
187
|
+
buffer.retain()
|
|
188
|
+
|
|
189
|
+
try {
|
|
190
|
+
val width = buffer.width
|
|
191
|
+
val height = buffer.height
|
|
192
|
+
|
|
193
|
+
val bitmap = i420ToBitmap(buffer, width, height)
|
|
194
|
+
if (bitmap == null) {
|
|
195
|
+
Log.w(TAG, "Failed to convert frame to bitmap")
|
|
196
|
+
return null
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
frameCount++
|
|
200
|
+
|
|
201
|
+
if (frameCount % SEGMENTATION_INTERVAL == 0L || cachedMaskBuffer == null) {
|
|
202
|
+
updateSegmentationMask(bitmap)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
val mask = cachedMaskBuffer
|
|
206
|
+
if (mask == null) {
|
|
207
|
+
if (frameCount == 1L || frameCount % 30L == 1L) {
|
|
208
|
+
Log.w(TAG, "No segmentation mask available yet (frame #$frameCount)")
|
|
209
|
+
}
|
|
210
|
+
bitmap.recycle()
|
|
211
|
+
return null
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
applyBlurComposite(bitmap, width, height, mask, cachedMaskWidth, cachedMaskHeight)
|
|
215
|
+
bitmap.recycle()
|
|
216
|
+
|
|
217
|
+
val comp = compositeBitmap ?: run {
|
|
218
|
+
Log.e(TAG, "compositeBitmap is null after applyBlurComposite")
|
|
219
|
+
return null
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
val i420Buffer = bitmapToI420Buffer(comp, width, height)
|
|
223
|
+
return VideoFrame(i420Buffer, frame.rotation, frame.timestampNs)
|
|
224
|
+
|
|
225
|
+
} catch (e: Exception) {
|
|
226
|
+
Log.e(TAG, "processFrame error", e)
|
|
227
|
+
return null
|
|
228
|
+
} finally {
|
|
229
|
+
buffer.release()
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
private fun updateSegmentationMask(bitmap: Bitmap) {
|
|
234
|
+
val seg = segmenter ?: run {
|
|
235
|
+
Log.e(TAG, "updateSegmentationMask: segmenter is null")
|
|
236
|
+
return
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
val isFirstMask = cachedMaskBuffer == null
|
|
240
|
+
val timeout = if (isFirstMask) SEGMENTATION_TIMEOUT_FIRST_MS else SEGMENTATION_TIMEOUT_MS
|
|
241
|
+
|
|
242
|
+
try {
|
|
243
|
+
val scaled = Bitmap.createScaledBitmap(bitmap, SEG_WIDTH, SEG_HEIGHT, true)
|
|
244
|
+
val input = InputImage.fromBitmap(scaled, 0)
|
|
245
|
+
|
|
246
|
+
val result = Tasks.await(
|
|
247
|
+
seg.process(input),
|
|
248
|
+
timeout,
|
|
249
|
+
TimeUnit.MILLISECONDS
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
val maskBuf = result.buffer
|
|
253
|
+
cachedMaskWidth = result.width
|
|
254
|
+
cachedMaskHeight = result.height
|
|
255
|
+
val size = cachedMaskWidth * cachedMaskHeight
|
|
256
|
+
|
|
257
|
+
maskBuf.rewind()
|
|
258
|
+
val rawFloats = FloatArray(size) { if (maskBuf.remaining() >= 4) maskBuf.float else 0f }
|
|
259
|
+
|
|
260
|
+
// Max-blend with decayed previous: fills transient holes, ghost fades in 2 frames
|
|
261
|
+
val prev = previousMaskFloats
|
|
262
|
+
val blended = if (prev != null && prev.size == size) {
|
|
263
|
+
FloatArray(size) { i -> maxOf(rawFloats[i], MASK_DECAY * prev[i]) }
|
|
264
|
+
} else rawFloats
|
|
265
|
+
previousMaskFloats = blended
|
|
266
|
+
|
|
267
|
+
val newMask = ByteBuffer.allocateDirect(size * 4)
|
|
268
|
+
newMask.order(ByteOrder.nativeOrder())
|
|
269
|
+
for (f in blended) newMask.putFloat(f)
|
|
270
|
+
newMask.rewind()
|
|
271
|
+
cachedMaskBuffer = newMask
|
|
272
|
+
|
|
273
|
+
if (scaled !== bitmap) scaled.recycle()
|
|
274
|
+
|
|
275
|
+
} catch (e: java.util.concurrent.TimeoutException) {
|
|
276
|
+
if (isFirstMask) {
|
|
277
|
+
Log.e(TAG, "Segmentation FIRST-CALL timeout (${timeout}ms) — ML Kit model may not be ready")
|
|
278
|
+
} else {
|
|
279
|
+
Log.d(TAG, "Segmentation timeout (${timeout}ms), using cached mask")
|
|
280
|
+
}
|
|
281
|
+
} catch (e: Exception) {
|
|
282
|
+
if (isFirstMask) {
|
|
283
|
+
Log.e(TAG, "Segmentation FIRST-CALL error", e)
|
|
284
|
+
} else {
|
|
285
|
+
Log.w(TAG, "Segmentation error (frame #$frameCount)", e)
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
private fun applyBlurComposite(
|
|
291
|
+
original: Bitmap, width: Int, height: Int,
|
|
292
|
+
maskBuffer: ByteBuffer, maskWidth: Int, maskHeight: Int
|
|
293
|
+
) {
|
|
294
|
+
val rs = renderScript ?: run {
|
|
295
|
+
Log.e(TAG, "applyBlurComposite: renderScript is null")
|
|
296
|
+
return
|
|
297
|
+
}
|
|
298
|
+
val blur = blurScript ?: run {
|
|
299
|
+
Log.e(TAG, "applyBlurComposite: blurScript is null")
|
|
300
|
+
return
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
if (blurredBitmap == null || blurredBitmap!!.width != width || blurredBitmap!!.height != height) {
|
|
304
|
+
blurredBitmap?.recycle()
|
|
305
|
+
blurredBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
306
|
+
}
|
|
307
|
+
if (compositeBitmap == null || compositeBitmap!!.width != width || compositeBitmap!!.height != height) {
|
|
308
|
+
compositeBitmap?.recycle()
|
|
309
|
+
compositeBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
// Multi-pass blur: each pass takes the output of the previous as input.
|
|
313
|
+
// Intermediate bitmaps are recycled; blurredBitmap receives the final result.
|
|
314
|
+
var src = original.copy(Bitmap.Config.ARGB_8888, true)
|
|
315
|
+
repeat(blurPasses) { pass ->
|
|
316
|
+
val isLast = pass == blurPasses - 1
|
|
317
|
+
val dst = if (isLast) blurredBitmap!! else Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
318
|
+
val inAlloc = Allocation.createFromBitmap(rs, src)
|
|
319
|
+
val outAlloc = Allocation.createFromBitmap(rs, dst)
|
|
320
|
+
blur.setInput(inAlloc)
|
|
321
|
+
blur.forEach(outAlloc)
|
|
322
|
+
outAlloc.copyTo(dst)
|
|
323
|
+
inAlloc.destroy()
|
|
324
|
+
outAlloc.destroy()
|
|
325
|
+
src.recycle()
|
|
326
|
+
src = dst
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
val maskPixels = IntArray(maskWidth * maskHeight)
|
|
330
|
+
maskBuffer.rewind()
|
|
331
|
+
for (i in maskPixels.indices) {
|
|
332
|
+
val raw = if (maskBuffer.remaining() >= 4) maskBuffer.float else 0f
|
|
333
|
+
val conf = when {
|
|
334
|
+
raw < MASK_THRESHOLD_LOW -> 0f
|
|
335
|
+
raw > MASK_THRESHOLD_HIGH -> 1f
|
|
336
|
+
else -> (raw - MASK_THRESHOLD_LOW) / (MASK_THRESHOLD_HIGH - MASK_THRESHOLD_LOW)
|
|
337
|
+
}
|
|
338
|
+
val alpha = (conf * 255).toInt().coerceIn(0, 255)
|
|
339
|
+
maskPixels[i] = (alpha shl 24) or 0x00FFFFFF
|
|
340
|
+
}
|
|
341
|
+
val maskBmp = Bitmap.createBitmap(maskPixels, maskWidth, maskHeight, Bitmap.Config.ARGB_8888)
|
|
342
|
+
val scaledMask = if (maskWidth != width || maskHeight != height) {
|
|
343
|
+
val s = Bitmap.createScaledBitmap(maskBmp, width, height, true)
|
|
344
|
+
maskBmp.recycle()
|
|
345
|
+
s
|
|
346
|
+
} else maskBmp
|
|
347
|
+
|
|
348
|
+
// Build person layer: original pixels with alpha from mask.
|
|
349
|
+
// At edges, alpha is partial → lerp between blurred BG and original → no halo.
|
|
350
|
+
val personBitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
351
|
+
val personCanvas = Canvas(personBitmap)
|
|
352
|
+
personCanvas.drawBitmap(original, 0f, 0f, null)
|
|
353
|
+
val dstInPaint = Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_IN) }
|
|
354
|
+
personCanvas.drawBitmap(scaledMask, 0f, 0f, dstInPaint)
|
|
355
|
+
|
|
356
|
+
val canvas = Canvas(compositeBitmap!!)
|
|
357
|
+
canvas.drawBitmap(blurredBitmap!!, 0f, 0f, null)
|
|
358
|
+
canvas.drawBitmap(personBitmap, 0f, 0f, null)
|
|
359
|
+
scaledMask.recycle()
|
|
360
|
+
personBitmap.recycle()
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// =====================================================================
|
|
364
|
+
// Color Space Conversions
|
|
365
|
+
// =====================================================================
|
|
366
|
+
|
|
367
|
+
private fun i420ToBitmap(buffer: VideoFrame.Buffer, width: Int, height: Int): Bitmap? {
|
|
368
|
+
return try {
|
|
369
|
+
val i420 = buffer.toI420() ?: return null
|
|
370
|
+
|
|
371
|
+
val yPlane = i420.dataY
|
|
372
|
+
val uPlane = i420.dataU
|
|
373
|
+
val vPlane = i420.dataV
|
|
374
|
+
val yStride = i420.strideY
|
|
375
|
+
val uStride = i420.strideU
|
|
376
|
+
val vStride = i420.strideV
|
|
377
|
+
|
|
378
|
+
val nv21 = ByteArray(width * height * 3 / 2)
|
|
379
|
+
|
|
380
|
+
for (row in 0 until height) {
|
|
381
|
+
yPlane.position(row * yStride)
|
|
382
|
+
yPlane.get(nv21, row * width, width)
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
val chromaHeight = height / 2
|
|
386
|
+
val chromaWidth = width / 2
|
|
387
|
+
for (row in 0 until chromaHeight) {
|
|
388
|
+
for (col in 0 until chromaWidth) {
|
|
389
|
+
val nv21Offset = width * height + row * width + col * 2
|
|
390
|
+
vPlane.position(row * vStride + col)
|
|
391
|
+
nv21[nv21Offset] = vPlane.get()
|
|
392
|
+
uPlane.position(row * uStride + col)
|
|
393
|
+
nv21[nv21Offset + 1] = uPlane.get()
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
i420.release()
|
|
398
|
+
|
|
399
|
+
val yuvImage = android.graphics.YuvImage(
|
|
400
|
+
nv21, android.graphics.ImageFormat.NV21, width, height, null
|
|
401
|
+
)
|
|
402
|
+
val out = java.io.ByteArrayOutputStream()
|
|
403
|
+
yuvImage.compressToJpeg(android.graphics.Rect(0, 0, width, height), 85, out)
|
|
404
|
+
val jpegBytes = out.toByteArray()
|
|
405
|
+
val decoded = android.graphics.BitmapFactory.decodeByteArray(jpegBytes, 0, jpegBytes.size)
|
|
406
|
+
decoded?.copy(Bitmap.Config.ARGB_8888, true)?.also {
|
|
407
|
+
decoded.recycle()
|
|
408
|
+
}
|
|
409
|
+
} catch (e: Exception) {
|
|
410
|
+
Log.e(TAG, "I420→Bitmap error", e)
|
|
411
|
+
null
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
private fun bitmapToI420Buffer(bitmap: Bitmap, width: Int, height: Int): JavaI420Buffer {
|
|
416
|
+
val argbBuffer = ByteBuffer.allocateDirect(width * height * 4)
|
|
417
|
+
bitmap.copyPixelsToBuffer(argbBuffer)
|
|
418
|
+
argbBuffer.rewind()
|
|
419
|
+
|
|
420
|
+
val i420Buffer = JavaI420Buffer.allocate(width, height)
|
|
421
|
+
val yBuf = i420Buffer.dataY
|
|
422
|
+
val uBuf = i420Buffer.dataU
|
|
423
|
+
val vBuf = i420Buffer.dataV
|
|
424
|
+
|
|
425
|
+
for (y in 0 until height) {
|
|
426
|
+
for (x in 0 until width) {
|
|
427
|
+
val idx = (y * width + x) * 4
|
|
428
|
+
val r = argbBuffer.get(idx).toInt() and 0xFF
|
|
429
|
+
val g = argbBuffer.get(idx + 1).toInt() and 0xFF
|
|
430
|
+
val b = argbBuffer.get(idx + 2).toInt() and 0xFF
|
|
431
|
+
|
|
432
|
+
// BT.601 RGB → YUV
|
|
433
|
+
val yVal = ((66 * r + 129 * g + 25 * b + 128) shr 8) + 16
|
|
434
|
+
yBuf.put(y * i420Buffer.strideY + x, yVal.coerceIn(0, 255).toByte())
|
|
435
|
+
|
|
436
|
+
if (y % 2 == 0 && x % 2 == 0) {
|
|
437
|
+
val uVal = ((-38 * r - 74 * g + 112 * b + 128) shr 8) + 128
|
|
438
|
+
val vVal = ((112 * r - 94 * g - 18 * b + 128) shr 8) + 128
|
|
439
|
+
val chromaIdx = (y / 2) * i420Buffer.strideU + (x / 2)
|
|
440
|
+
uBuf.put(chromaIdx, uVal.coerceIn(0, 255).toByte())
|
|
441
|
+
vBuf.put(chromaIdx, vVal.coerceIn(0, 255).toByte())
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
return i420Buffer
|
|
447
|
+
}
|
|
448
|
+
}
|
|
@@ -50,6 +50,7 @@ const REQUIRED_PERMISSIONS = [
|
|
|
50
50
|
"android.permission.FOREGROUND_SERVICE_MICROPHONE",
|
|
51
51
|
"android.permission.FOREGROUND_SERVICE_CAMERA"
|
|
52
52
|
];
|
|
53
|
+
const MLKIT_SEGMENTATION_DEP = 'implementation("com.google.mlkit:segmentation-selfie:16.0.0-beta6")';
|
|
53
54
|
function getAndroidPackagePath(projectRoot) {
|
|
54
55
|
var _a, _b;
|
|
55
56
|
const manifestPath = path.join(projectRoot, 'android/app/src/main/AndroidManifest.xml');
|
|
@@ -74,10 +75,15 @@ function getAndroidPackagePath(projectRoot) {
|
|
|
74
75
|
}
|
|
75
76
|
return path.join(projectRoot, 'android/app/src/main/java', ...packageName.split('.'));
|
|
76
77
|
}
|
|
78
|
+
function getPackageName(projectRoot) {
|
|
79
|
+
return getAndroidPackagePath(projectRoot)
|
|
80
|
+
.replace(path.join(projectRoot, 'android/app/src/main/java') + path.sep, '')
|
|
81
|
+
.split(path.sep)
|
|
82
|
+
.join('.');
|
|
83
|
+
}
|
|
77
84
|
function copyAndPatchJavaFiles(projectRoot) {
|
|
78
85
|
const srcDir = path.join(__dirname, 'static/java');
|
|
79
86
|
const destDir = getAndroidPackagePath(projectRoot);
|
|
80
|
-
// Récupère le packageName sous forme de string (ex: com.reactnativeapirtc)
|
|
81
87
|
const packageName = destDir
|
|
82
88
|
.replace(path.join(projectRoot, 'android/app/src/main/java') + path.sep, '')
|
|
83
89
|
.split(path.sep)
|
|
@@ -90,13 +96,43 @@ function copyAndPatchJavaFiles(projectRoot) {
|
|
|
90
96
|
const dest = path.join(destDir, file);
|
|
91
97
|
if (fs.existsSync(src)) {
|
|
92
98
|
let content = fs.readFileSync(src, 'utf8');
|
|
93
|
-
// Remplace la ligne de package par la bonne valeur
|
|
94
99
|
content = content.replace(/^package\s+[\w.]+;/m, `package ${packageName};`);
|
|
95
100
|
fs.writeFileSync(dest, content, 'utf8');
|
|
96
101
|
console.log(`Copied and patched ${file} to android sources with package: ${packageName}`);
|
|
97
102
|
}
|
|
98
103
|
});
|
|
99
104
|
}
|
|
105
|
+
function copyAndPatchKotlinFiles(projectRoot) {
|
|
106
|
+
const srcDir = path.join(__dirname, 'static/kotlin');
|
|
107
|
+
const destDir = getAndroidPackagePath(projectRoot);
|
|
108
|
+
const packageName = destDir
|
|
109
|
+
.replace(path.join(projectRoot, 'android/app/src/main/java') + path.sep, '')
|
|
110
|
+
.split(path.sep)
|
|
111
|
+
.join('.');
|
|
112
|
+
if (!fs.existsSync(destDir)) {
|
|
113
|
+
fs.mkdirSync(destDir, { recursive: true });
|
|
114
|
+
}
|
|
115
|
+
const kotlinFiles = [
|
|
116
|
+
'BackgroundBlurPackage.kt',
|
|
117
|
+
'BackgroundBlurModule.kt',
|
|
118
|
+
'BlurVideoProcessor.kt',
|
|
119
|
+
'BackgroundImageProcessor.kt',
|
|
120
|
+
];
|
|
121
|
+
kotlinFiles.forEach(file => {
|
|
122
|
+
const src = path.join(srcDir, file);
|
|
123
|
+
const dest = path.join(destDir, file);
|
|
124
|
+
if (fs.existsSync(src)) {
|
|
125
|
+
let content = fs.readFileSync(src, 'utf8');
|
|
126
|
+
// Kotlin package declaration has no trailing semicolon
|
|
127
|
+
content = content.replace(/^package\s+[\w.]+/m, `package ${packageName}`);
|
|
128
|
+
fs.writeFileSync(dest, content, 'utf8');
|
|
129
|
+
console.log(`Copied and patched ${file} to android sources with package: ${packageName}`);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
console.warn(`Kotlin source not found: ${src}`);
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
}
|
|
100
136
|
const withAndroidPermissions = (config) => {
|
|
101
137
|
return (0, config_plugins_1.withAndroidManifest)(config, (config) => {
|
|
102
138
|
const manifest = config.modResults.manifest;
|
|
@@ -116,20 +152,28 @@ const withAndroidPermissions = (config) => {
|
|
|
116
152
|
return config;
|
|
117
153
|
});
|
|
118
154
|
};
|
|
155
|
+
const withMlKitGradleDep = (config) => {
|
|
156
|
+
return (0, config_plugins_1.withAppBuildGradle)(config, (config) => {
|
|
157
|
+
if (!config.modResults.contents.includes('segmentation-selfie')) {
|
|
158
|
+
config.modResults.contents = config.modResults.contents.replace(/(\s*dependencies\s*\{)/, `$1\n ${MLKIT_SEGMENTATION_DEP}`);
|
|
159
|
+
console.log('Added ML Kit segmentation-selfie dependency to app/build.gradle');
|
|
160
|
+
}
|
|
161
|
+
return config;
|
|
162
|
+
});
|
|
163
|
+
};
|
|
119
164
|
const withAndroidPlugin = (config, props) => {
|
|
120
165
|
console.error('withAndroidPlugin called with props:', props);
|
|
121
166
|
let updatedConfig = (0, config_plugins_1.withMainApplication)(config, (config) => {
|
|
122
167
|
const mainApplication = config.modResults;
|
|
123
168
|
if (props.enableMediaProjectionService === false) {
|
|
124
|
-
// If the feature is disabled, we can return early
|
|
125
169
|
console.warn('Media Projection Service is disabled, skipping modifications.');
|
|
126
170
|
return config;
|
|
127
171
|
}
|
|
128
|
-
//
|
|
129
|
-
if (!mainApplication.contents.includes('import
|
|
172
|
+
// Add WebRTCModuleOptions import
|
|
173
|
+
if (!mainApplication.contents.includes('import com.oney.WebRTCModule.WebRTCModuleOptions')) {
|
|
130
174
|
mainApplication.contents = mainApplication.contents.replace(/(package\s+[\w.]+(?:\r?\n))/, `$1\nimport com.oney.WebRTCModule.WebRTCModuleOptions\n`);
|
|
131
175
|
}
|
|
132
|
-
//
|
|
176
|
+
// Add WebRTCModuleOptions code at start of onCreate()
|
|
133
177
|
const onCreateRegex = /override fun onCreate\(\) \{\n/;
|
|
134
178
|
const customCode = [
|
|
135
179
|
' val options: WebRTCModuleOptions = WebRTCModuleOptions.getInstance()',
|
|
@@ -138,14 +182,10 @@ const withAndroidPlugin = (config, props) => {
|
|
|
138
182
|
if (!mainApplication.contents.includes('WebRTCModuleOptions.getInstance()')) {
|
|
139
183
|
mainApplication.contents = mainApplication.contents.replace(onCreateRegex, match => `${match}${customCode}`);
|
|
140
184
|
}
|
|
141
|
-
//
|
|
142
|
-
// Récupère le packageName automatiquement
|
|
185
|
+
// Add AppLifecyclePackage import and registration
|
|
143
186
|
const packageName = (() => {
|
|
144
187
|
try {
|
|
145
|
-
return
|
|
146
|
-
.replace(path.join(config.modRequest.projectRoot, 'android/app/src/main/java') + path.sep, '')
|
|
147
|
-
.split(path.sep)
|
|
148
|
-
.join('.');
|
|
188
|
+
return getPackageName(config.modRequest.projectRoot);
|
|
149
189
|
}
|
|
150
190
|
catch (_a) {
|
|
151
191
|
return null;
|
|
@@ -157,15 +197,31 @@ const withAndroidPlugin = (config, props) => {
|
|
|
157
197
|
mainApplication.contents = mainApplication.contents.replace(/(package\s+[\w.]+(?:\r?\n))/, `$1${importLine}`);
|
|
158
198
|
}
|
|
159
199
|
}
|
|
160
|
-
if (!mainApplication.contents.includes('
|
|
161
|
-
mainApplication.contents = mainApplication.contents.replace(/(
|
|
162
|
-
return `${p1}\n packages.add(AppLifecyclePackage())${p2}`;
|
|
163
|
-
});
|
|
200
|
+
if (!mainApplication.contents.includes('add(AppLifecyclePackage())')) {
|
|
201
|
+
mainApplication.contents = mainApplication.contents.replace(/(\/\/ add\(MyReactNativePackage\(\)\))/, match => `${match}\n add(AppLifecyclePackage())`);
|
|
164
202
|
}
|
|
165
|
-
//
|
|
203
|
+
// Add BackgroundBlurPackage if enableVideoEffects is true
|
|
204
|
+
if (props.enableVideoEffects !== false) {
|
|
205
|
+
if (packageName) {
|
|
206
|
+
const blurImportLine = `import ${packageName}.BackgroundBlurPackage\n`;
|
|
207
|
+
if (!mainApplication.contents.includes(blurImportLine.trim())) {
|
|
208
|
+
mainApplication.contents = mainApplication.contents.replace(/(package\s+[\w.]+(?:\r?\n))/, `$1${blurImportLine}`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
if (!mainApplication.contents.includes('add(BackgroundBlurPackage())')) {
|
|
212
|
+
mainApplication.contents = mainApplication.contents.replace(/(\/\/ add\(MyReactNativePackage\(\)\))/, match => `${match}\n add(BackgroundBlurPackage())`);
|
|
213
|
+
}
|
|
214
|
+
// Copy Kotlin native files
|
|
215
|
+
copyAndPatchKotlinFiles(config.modRequest.projectRoot);
|
|
216
|
+
}
|
|
217
|
+
// Copy Java native files
|
|
166
218
|
copyAndPatchJavaFiles(config.modRequest.projectRoot);
|
|
167
219
|
return config;
|
|
168
220
|
});
|
|
221
|
+
// Add ML Kit Gradle dependency when video effects are enabled
|
|
222
|
+
if (props.enableVideoEffects !== false) {
|
|
223
|
+
updatedConfig = withMlKitGradleDep(updatedConfig);
|
|
224
|
+
}
|
|
169
225
|
updatedConfig = withAndroidPermissions(updatedConfig);
|
|
170
226
|
return updatedConfig;
|
|
171
227
|
};
|
package/build/withPlugin.d.ts
CHANGED
package/build/withPlugin.js
CHANGED
|
@@ -9,6 +9,7 @@ const withIosBroadcastExtension_1 = __importDefault(require("./withIosBroadcastE
|
|
|
9
9
|
const withIosRPKFiles_1 = __importDefault(require("./withIosRPKFiles"));
|
|
10
10
|
const withPlugin = (config, props = {
|
|
11
11
|
enableMediaProjectionService: true,
|
|
12
|
+
enableVideoEffects: true,
|
|
12
13
|
appleTeamId: process.env.EXPO_APPLE_TEAM_ID || 'APPLE_TEAM_ID_NOT_SET',
|
|
13
14
|
}) => {
|
|
14
15
|
config = (0, withAndroidPlugin_1.default)(config, props);
|