@volcengine/react-native-live-push 1.1.3-rc.1 → 1.3.0-rc.0

package/android/src/main/java/com/volcengine/velive/rn/push/mixer/MixerView.java

@@ -0,0 +1,328 @@
+package com.volcengine.velive.rn.push.mixer;
+
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.os.Handler;
+import android.os.Looper;
+import android.view.ViewTreeObserver;
+import android.widget.FrameLayout;
+import androidx.annotation.Nullable;
+import com.facebook.react.bridge.Arguments;
+import com.facebook.react.bridge.ReadableArray;
+import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.uimanager.ThemedReactContext;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Custom MixerView that supports different capture modes
+ */
+public class MixerView extends FrameLayout {
+  private ThemedReactContext context;
+  private String viewId;
+
+  // Capture configuration
+  private String captureMode = "onchange";
+  private float captureFramerate = 5f; // realTime mode default 5 fps
+
+  // Capture state
+  private Handler captureHandler;
+  private Runnable captureRunnable;
+  private ViewTreeObserver.OnGlobalLayoutListener layoutListener;
+  private AtomicBoolean isCapturing = new AtomicBoolean(false);
+  private AtomicBoolean isDestroyed = new AtomicBoolean(false);
+
+  // Performance optimization
+  private Bitmap lastBitmap;
+  private long lastCaptureTime = 0;
+  private final Object captureLock = new Object();
+
+  // Add callback support for MixerManager integration
+  private MixerManager.BitmapCaptureCallback bitmapCaptureCallback;
+
+  public MixerView(ThemedReactContext context) {
+    super(context);
+    this.context = context;
+    this.captureHandler = new Handler(Looper.getMainLooper());
+    initializeCapture();
+  }
+
+  private void initializeCapture() {
+    // Initialize layout change listener
+    layoutListener = this::onViewChanged;
+  }
+
+  public void setCaptureMode(String mode) {
+    if (mode.equals(this.captureMode))
+      return;
+
+    stopCapture();
+    this.captureMode = mode;
+    startCapture();
+  }
+
+  public void setCaptureFramerate(float framerate) {
+    this.captureFramerate = Math.max(1f, Math.min(60f, framerate));
+    if ("realtime".equals(captureMode)) {
+      restartRealtimeCapture();
+    }
+  }
+
+  private void startCapture() {
+    if (isDestroyed.get())
+      return;
+
+    switch (captureMode) {
+    case "onchange":
+      startOnChangeCapture();
+      break;
+    case "realtime":
+      startRealtimeCapture();
+      break;
+    case "manual":
+      // Manual mode doesn't auto-start
+      break;
+    }
+  }
+
+  private void stopCapture() {
+    // Remove listeners
+    if (getViewTreeObserver().isAlive()) {
+      if (layoutListener != null) {
+        getViewTreeObserver().removeOnGlobalLayoutListener(layoutListener);
+      }
+    }
+
+    // Stop realtime capture
+    if (captureRunnable != null) {
+      captureHandler.removeCallbacks(captureRunnable);
+      captureRunnable = null;
+    }
+
+    isCapturing.set(false);
+  }
+
+  private void startOnChangeCapture() {
+    // onChange mode triggers every 2 seconds
+    long interval = 2000; // 2 seconds
+
+    captureRunnable = new Runnable() {
+      @Override
+      public void run() {
+        if (!isDestroyed.get() && "onchange".equals(captureMode)) {
+          performCapture("onchange");
+          captureHandler.postDelayed(this, interval);
+        }
+      }
+    };
+    captureHandler.postDelayed(captureRunnable, interval);
+  }
+
+  private void startRealtimeCapture() {
+    long interval = (long)(1000f / captureFramerate);
+
+    captureRunnable = new Runnable() {
+      @Override
+      public void run() {
+        if (!isDestroyed.get() && "realtime".equals(captureMode)) {
+          performCapture("realtime");
+          captureHandler.postDelayed(this, interval);
+        }
+      }
+    };
+    captureHandler.postDelayed(captureRunnable, interval);
+  }
+
+  private void restartRealtimeCapture() {
+    if ("realtime".equals(captureMode)) {
+      stopCapture();
+      startRealtimeCapture();
+    }
+  }
+
+  private void onViewChanged() {
+    if (isDestroyed.get()) {
+      return;
+    }
+
+    if ("onchange".equals(captureMode)) {
+      performCapture("onchange");
+    }
+  }
+
+  private void performCapture(String trigger) {
+    if (isCapturing.get() || isDestroyed.get()) {
+      return;
+    }
+
+    long currentTime = System.currentTimeMillis();
+    long minInterval = 33; // 30 FPS ~ 33ms per frame
+    if (currentTime - lastCaptureTime < minInterval) {
+      return;
+    }
+    lastCaptureTime = currentTime;
+
+    synchronized (captureLock) {
+      if (isCapturing.compareAndSet(false, true)) {
+        Bitmap bitmap = null;
+        try {
+          bitmap = createBitmap();
+          if (bitmap != null) {
+            processBitmap(bitmap, trigger);
+          }
+        } catch (OutOfMemoryError e) {
+          System.gc();
+          // If an OOM occurs and the bitmap has not been passed to
+          // processBitmap, recycle it
+          if (bitmap != null && !bitmap.isRecycled()) {
+            bitmap.recycle();
+          }
+        } catch (Exception e) {
+          // If any other exception occurs and the bitmap has not been passed to
+          // processBitmap, recycle it
+          if (bitmap != null && !bitmap.isRecycled()) {
+            bitmap.recycle();
+          }
+        } finally {
+          isCapturing.set(false);
+        }
+      }
+    }
+  }
+
+  private Bitmap createBitmap() {
+    if (getWidth() <= 0 || getHeight() <= 0)
+      return null;
+
+    try {
+      Bitmap bitmap =
+          Bitmap.createBitmap(getWidth(), getHeight(), Bitmap.Config.ARGB_8888);
+      Canvas canvas = new Canvas(bitmap);
+      draw(canvas);
+      return bitmap;
+    } catch (OutOfMemoryError e) {
+      // Try with lower quality
+      try {
+        Bitmap bitmap =
+            Bitmap.createBitmap(getWidth(), getHeight(), Bitmap.Config.RGB_565);
+        Canvas canvas = new Canvas(bitmap);
+        draw(canvas);
+        return bitmap;
+      } catch (Exception ex) {
+        return null;
+      }
+    }
+  }
+
+  private void processBitmap(Bitmap bitmap, String trigger) {
+    if (bitmap == null || bitmap.isRecycled()) {
+      if (bitmapCaptureCallback != null) {
+        bitmapCaptureCallback.onCaptureError("Bitmap is null or recycled");
+      }
+      return;
+    }
+
+    // Call callback first if set (for MixerManager)
+    if (bitmapCaptureCallback != null) {
+      try {
+        // only capture if the callback is set
+        bitmapCaptureCallback.onBitmapCaptured(bitmap);
+      } catch (Exception e) {
+        if (bitmapCaptureCallback != null) {
+          bitmapCaptureCallback.onCaptureError("Callback error: " +
+                                               e.getMessage());
+        }
+      }
+    } else {
+      // Send event to JS (existing functionality)
+      try {
+        WritableMap event = Arguments.createMap();
+        event.putString("trigger", trigger);
+        event.putBoolean("success", true);
+        event.putInt("bitmapWidth", bitmap.getWidth());
+        event.putInt("bitmapHeight", bitmap.getHeight());
+      } catch (IllegalStateException e) {
+        // Ignore
+      }
+    }
+
+    // Clean up old bitmap
+    if (lastBitmap != null && !lastBitmap.isRecycled()) {
+      lastBitmap.recycle();
+    }
+
+    // If no callback is set, immediately release current bitmap to avoid memory accumulation
+    // If callback is set, bitmap will be managed by the callback receiver
+    if (bitmapCaptureCallback == null) {
+      if (bitmap != null && !bitmap.isRecycled()) {
+        bitmap.recycle();
+      }
+      lastBitmap = null;
+    } else {
+      lastBitmap = bitmap;
+    }
+  }
+
+  public void handleCommand(String command, @Nullable ReadableArray args) {
+    switch (command) {
+    case "capture":
+      performCapture("manual");
+      break;
+    case "startCapture":
+      startCapture();
+      break;
+    case "stopCapture":
+      stopCapture();
+      break;
+    case "setCaptureMode":
+      if (args != null && args.size() > 0) {
+        setCaptureMode(args.getString(0));
+      }
+      break;
+    }
+  }
+
+  @Override
+  protected void onAttachedToWindow() {
+    super.onAttachedToWindow();
+    if (viewId != null) {
+      MixerManager.cachedMixedViews.put(viewId, this);
+      // Notify MixerManager that view is ready, check for pending callbacks
+      MixerManager.onViewReady(viewId, this);
+    }
+  }
+
+  @Override
+  protected void onDetachedFromWindow() {
+    super.onDetachedFromWindow();
+    if (viewId != null) {
+      MixerManager.cachedMixedViews.remove(viewId);
+    }
+    cleanup();
+  }
+
+  public void setViewId(String viewId) {
+    this.viewId = viewId;
+    MixerManager.cachedMixedViews.put(viewId, this);
+    // Notify MixerManager that view is ready, check for pending callbacks
+    MixerManager.onViewReady(viewId, this);
+  }
+
+  // Add callback support for MixerManager integration
+  public void
+  setBitmapCaptureCallback(MixerManager.BitmapCaptureCallback callback) {
+    this.bitmapCaptureCallback = callback;
+  }
+
+  // Public method for manual capture (called by MixerManager)
+  public void performManualCapture() { performCapture("manual"); }
+
+  private void cleanup() {
+    isDestroyed.set(true);
+    stopCapture();
+
+    if (lastBitmap != null && !lastBitmap.isRecycled()) {
+      lastBitmap.recycle();
+      lastBitmap = null;
+    }
+  }
+}

package/android/src/main/java/com/volcengine/velive/rn/push/mixer/MixerViewManager.java

@@ -0,0 +1,43 @@
+package com.volcengine.velive.rn.push.mixer;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import com.facebook.react.uimanager.ViewGroupManager;
+import com.facebook.react.uimanager.ThemedReactContext;
+import com.facebook.react.uimanager.annotations.ReactProp;
+
+public class MixerViewManager extends ViewGroupManager<MixerView> {
+  public static final String NAME = "VeLiveMixView";
+
+  @NonNull
+  @Override
+  public String getName() {
+    return NAME;
+  }
+
+  @NonNull
+  @Override
+  protected MixerView
+  createViewInstance(@NonNull ThemedReactContext reactContext) {
+    return new MixerView(reactContext);
+  }
+
+  @ReactProp(name = "captureMode")
+  public void setCaptureMode(MixerView view, @Nullable String captureMode) {
+    view.setCaptureMode(captureMode != null ? captureMode : "onchange");
+  }
+
+  @ReactProp(name = "captureFramerate", defaultFloat = 5f)
+  public void setCaptureFramerate(MixerView view, float framerate) {
+    view.setCaptureFramerate(framerate);
+  }
+
+  @ReactProp(name = "viewId")
+  public void setViewId(MixerView view, @Nullable String viewId) {
+    if (viewId != null) {
+      view.setViewId(viewId);
+    }
+  }
+}
+
+

package/android/src/main/java/com/volcengine/velive/rn/push/mixer/TextureMgr.java

@@ -0,0 +1,168 @@
+package com.volcengine.velive.rn.push.mixer;
+
+import android.graphics.Bitmap;
+import android.opengl.GLES20;
+import android.opengl.GLUtils;
+
+import com.pandora.common.env.Env;
+import com.ss.avframework.opengl.GLThreadManager;
+import com.ss.avframework.opengl.GlUtil;
+
+import java.nio.ByteBuffer;
+
+public class TextureMgr {
+    private int texture;
+    private int width;
+    private int height;
+    public TextureMgr(int width, int height) {
+        this.width = width;
+        this.height = height;
+        GLThreadManager.getMainGlHandle().post(() -> {
+            if (texture <= 0) {
+                texture = GlUtil.generateTexture(GLES20.GL_TEXTURE_2D);
+                GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
+                GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
+                GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, width, height, 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null);
+                GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                GLES20.glFinish();
+                GlUtil.checkNoGLES2Error("clearBackgroundTex");
+            }
+        });
+    }
+
+    public void cleanup() {
+        GLThreadManager.getMainGlHandle().post(() -> {
+            if (texture > 0) {
+                GLES20.glDeleteTextures(1, new int[]{texture}, 0);
+                texture = 0;
+            }
+        });
+    }
+
+    public interface RenderListener {
+        void doBusiness(int texture);
+    }
+
+    public void dealWithTexture(ByteBuffer byteBuffer, RenderListener listener) {
+        GLThreadManager.getMainGlHandle().post(new Runnable() {
+            @Override
+            public void run() {
+                if (texture > 0) {
+                    YuvHelper.NV21ToBitmap bm = null;
+                    Bitmap bmp = null;
+                    Bitmap rotatedBmp = null;
+                    try {
+                        bm = new YuvHelper.NV21ToBitmap(Env.getApplicationContext());
+                        bmp = bm.nv21ToBitmap(byteBuffer.array(), width, height);
+                        
+                        if (bmp == null || bmp.isRecycled()) {
+                            return;
+                        }
+                        
+                        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
+                        rotatedBmp = YuvHelper.rotateBitmap(bmp, 0, false, true);
+                        
+                        if (rotatedBmp != null && !rotatedBmp.isRecycled()) {
+                            GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, rotatedBmp, 0);
+                            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                            GLES20.glFlush();
+                            listener.doBusiness(texture);
+                        } else {
+                            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                        }
+                    } catch (OutOfMemoryError e) {
+                        System.gc();
+                    } catch (Exception e) {
+                        // Ignore
+                    } finally {
+                        if (bmp != null && !bmp.isRecycled()) {
+                            bmp.recycle();
+                        }
+                        if (rotatedBmp != null && rotatedBmp != bmp && !rotatedBmp.isRecycled()) {
+                            rotatedBmp.recycle();
+                        }
+                        if (bm != null) {
+                            bm.cleanup();
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    public void dealI420WithTexture(ByteBuffer byteBuffer, RenderListener listener) {
+        GLThreadManager.getMainGlHandle().post(new Runnable() {
+            @Override
+            public void run() {
+                if (texture > 0) {
+                    Bitmap bm = null;
+                    Bitmap rotatedBm = null;
+                    try {
+                        bm = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+                        bm.copyPixelsFromBuffer(byteBuffer);
+                        
+                        if (bm.isRecycled()) {
+                            return;
+                        }
+                        
+                        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
+                        rotatedBm = YuvHelper.rotateBitmap(bm, 0, false, true);
+                        
+                        if (rotatedBm != null && !rotatedBm.isRecycled()) {
+                            GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, rotatedBm, 0);
+                            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                            GLES20.glFlush();
+                            listener.doBusiness(texture);
+                        } else {
+                            GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                        }
+                    } catch (OutOfMemoryError e) {
+                        System.gc();
+                    } catch (Exception e) {
+                        // Ignore
+                    } finally {
+                        if (bm != null && !bm.isRecycled()) {
+                            bm.recycle();
+                        }
+                        if (rotatedBm != null && rotatedBm != bm && !rotatedBm.isRecycled()) {
+                            rotatedBm.recycle();
+                        }
+                    }
+                }
+            }
+        });
+    }
+
+    public void dealWithTexture(Bitmap bitmap, RenderListener listener) {
+        GLThreadManager.getMainGlHandle().post(() -> {
+            if (texture > 0) {
+                Bitmap processedBitmap = null;
+                boolean needRecycle = false;
+                try {
+                    if (bitmap == null || bitmap.isRecycled()) {
+                        return;
+                    }
+                    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, texture);
+                    processedBitmap = YuvHelper.rotateBitmap(bitmap, 0, false, true);
+                    needRecycle = (processedBitmap != null && processedBitmap != bitmap);
+                    if (processedBitmap != null && !processedBitmap.isRecycled()) {
+                        GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, 0, processedBitmap, 0);
+                        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                        GLES20.glFlush();
+                        listener.doBusiness(texture);
+                    } else {
+                        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
+                    }
+                } catch (OutOfMemoryError e) {
+                    System.gc();
+                } catch (Exception e) {
+                    // Ignore
+                } finally {
+                    if (needRecycle && processedBitmap != null && !processedBitmap.isRecycled()) {
+                        processedBitmap.recycle();
+                    }
+                }
+            }
+        });
+    }
+}

package/android/src/main/java/com/volcengine/velive/rn/push/mixer/YuvHelper.java

@@ -0,0 +1,165 @@
+package com.volcengine.velive.rn.push.mixer;
+
+import android.content.Context;
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.renderscript.Allocation;
+import android.renderscript.Element;
+import android.renderscript.RenderScript;
+import android.renderscript.ScriptIntrinsicYuvToRGB;
+import android.renderscript.Type;
+
+import java.nio.ByteBuffer;
+
+public class YuvHelper {
+    private static RenderScript globalRS = null;
+    public static synchronized RenderScript getGlobalRS(Context context) {
+        if (globalRS == null) {
+            globalRS = RenderScript.create(context.getApplicationContext());
+        }
+        return globalRS;
+    }
+    public static void destroyGlobalRS() {
+        if (globalRS != null) {
+            globalRS.destroy();
+            globalRS = null;
+        }
+    }
+    public static class NV21ToBitmap {
+        private RenderScript rs;
+        private ScriptIntrinsicYuvToRGB yuvToRgbIntrinsic;
+        private Type.Builder yuvType, rgbaType;
+
+        public NV21ToBitmap(Context context) {
+            rs = YuvHelper.getGlobalRS(context);
+            yuvToRgbIntrinsic = ScriptIntrinsicYuvToRGB.create(rs, Element.U8_4(rs));
+            yuvType = new Type.Builder(rs, Element.U8(rs));
+            rgbaType = new Type.Builder(rs, Element.RGBA_8888(rs));
+        }
+
+        public void cleanup() {
+            try {
+                if (yuvToRgbIntrinsic != null) {
+                    yuvToRgbIntrinsic.destroy();
+                    yuvToRgbIntrinsic = null;
+                }
+                // 不销毁全局 rs
+                // rs = null;
+            } catch (Exception e) {
+                // Ignore
+            }
+        }
+
+        public Bitmap nv21ToBitmap(byte[] nv21, int width, int height) {
+            Allocation in = null;
+            Allocation out = null;
+            try {
+                yuvType.setX(nv21.length);
+                in = Allocation.createTyped(rs, yuvType.create(), Allocation.USAGE_SCRIPT);
+                in.copyFrom(nv21);
+
+                rgbaType.setX(width).setY(height);
+                out = Allocation.createTyped(rs, rgbaType.create(), Allocation.USAGE_SCRIPT);
+                yuvToRgbIntrinsic.setInput(in);
+                yuvToRgbIntrinsic.forEach(out);
+
+                Bitmap bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+                out.copyTo(bmp);
+
+                return bmp;
+            } finally {
+                if (in != null) {
+                    in.destroy();
+                }
+                if (out != null) {
+                    out.destroy();
+                }
+            }
+        }
+    }
+
+    public static void NV21toI420(ByteBuffer input, ByteBuffer output, final int width, final int height) {
+        final int size = width * height;
+        final int quarter = size / 4;
+        final int v0 = size + quarter;
+
+        System.arraycopy(input.array(), 0, output.array(), 0, size);
+
+        for (int u = size, v = v0, o = size; u < v0; u++, v++, o += 2) {
+            output.array()[v] = input.array()[o];
+            output.array()[u] = input.array()[o + 1];
+        }
+        output.limit(width * height * 3 / 2);
+        output.position(0);
+    }
+
+    public static void NV21toNV12(ByteBuffer input, ByteBuffer output, final int width, final int height) {
+        final int size = width * height;
+
+        System.arraycopy(input.array(), 0, output.array(), 0, size);
+        for (int i = size; i < 1.5f * size; i += 2) {
+            output.array()[i] = input.array()[i + 1];
+            output.array()[i + 1] = input.array()[i];
+        }
+        output.position(0);
+        output.limit(width * height * 3 / 2);
+    }
+
+    public static void NV12toI420(ByteBuffer input, ByteBuffer output, final int width, final int height) {
+        final int size = width * height;
+        final int quarter = size / 4;
+        final int v0 = size + quarter;
+
+        System.arraycopy(input.array(), 0, output.array(), 0, size);
+
+        for (int u = size, v = v0, o = size; u < v0; u++, v++, o += 2) {
+            output.array()[v] = input.array()[o + 1];
+            output.array()[u] = input.array()[o];
+        }
+    }
+
+    public static void I420toNV21(ByteBuffer input, ByteBuffer output, final int width, final int height) {
+        final int size = width * height;
+        final int quarter = size / 4;
+        final int v0 = size + quarter;
+
+        System.arraycopy(input.array(), 0, output.array(), 0, size);
+
+        for (int u = size, v = v0, o = size; u < v0; u++, v++, o += 2) {
+            output.array()[o + 1] = input.array()[v];
+            output.array()[o] = input.array()[u];
+        }
+    }
+
+    public static Bitmap rotateBitmap(Bitmap origin, float alpha, boolean horizontalMirror, boolean verticalMirror) {
+        if (origin == null || origin.isRecycled()) {
+            return null;
+        }
+        
+        try {
+            int width = origin.getWidth();
+            int height = origin.getHeight();
+            Matrix matrix = new Matrix();
+            float sx = 1;
+            float sy = 1;
+            if (horizontalMirror) {
+                sx = -1;
+            }
+            if (verticalMirror) {
+                sy = -1;
+            }
+            matrix.postScale(sx, sy);
+            matrix.postRotate(alpha);
+            
+            if (origin.isRecycled()) {
+                return null;
+            }
+            
+            return Bitmap.createBitmap(origin, 0, 0, width, height, matrix, false);
+        } catch (IllegalStateException e) {
+            return null;
+        } catch (Exception e) {
+            return null;
+        }
+    }
+}