@shopify/react-native-skia 1.2.3 → 1.3.0

package/android/CMakeLists.txt

@@ -48,6 +48,7 @@ add_library(
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/SkiaOpenGLSurfaceFactory.cpp"
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/GrAHardwareBufferUtils.cpp"
         "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/AHardwareBufferUtils.cpp"
+        "${PROJECT_SOURCE_DIR}/cpp/rnskia-android/RNSkAndroidVideo.cpp"
 
         "${PROJECT_SOURCE_DIR}/../cpp/jsi/JsiHostObject.cpp"
         "${PROJECT_SOURCE_DIR}/../cpp/jsi/JsiValue.cpp"

package/android/cpp/jni/JniPlatformContext.cpp

@@ -75,6 +75,29 @@ TSelf JniPlatformContext::initHybrid(jni::alias_ref<jhybridobject> jThis,
   return makeCxxInstance(jThis, pixelDensity);
 }
 
+jni::global_ref<jobject>
+JniPlatformContext::createVideo(const std::string &url) {
+  jni::ThreadScope ts; // Manages JNI thread attachment/detachment
+
+  // Get the JNI environment
+  JNIEnv *env = facebook::jni::Environment::current();
+
+  // Convert std::string to jstring
+  jstring jUrl = env->NewStringUTF(url.c_str());
+
+  // Get the method ID for the createVideo method
+  // Replace "Lcom/yourpackage/RNSkVideo;" with the actual return type
+  // descriptor
+  static auto method =
+      javaPart_->getClass()->getMethod<jobject(jstring)>("createVideo");
+
+  // Call the method and receive a local reference to the video object
+  auto videoObject = method(javaPart_.get(), jUrl);
+  // Clean up the jstring local reference
+  auto result = jni::make_global(videoObject);
+  return result;
+}
+
 sk_sp<SkImage> JniPlatformContext::takeScreenshotFromViewTag(size_t tag) {
   // Call the java method for creating a view screenshot as a bitmap:
   auto env = jni::Environment::current();

package/android/cpp/jni/include/JniPlatformContext.h

@@ -50,6 +50,8 @@ public:
     _onNotifyDrawLoop = callback;
   }
 
+  jni::global_ref<jobject> createVideo(const std::string &url);
+
 private:
   friend HybridBase;
   jni::global_ref<JniPlatformContext::javaobject> javaPart_;

package/android/cpp/rnskia-android/RNSkAndroidPlatformContext.h

@@ -11,6 +11,7 @@
 
 #include "AHardwareBufferUtils.h"
 #include "JniPlatformContext.h"
+#include "RNSkAndroidVideo.h"
 #include "RNSkPlatformContext.h"
 #include "SkiaOpenGLSurfaceFactory.h"
 
@@ -57,6 +58,11 @@ public:
     return SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(buffer);
   }
 
+  std::shared_ptr<RNSkVideo> createVideo(const std::string &url) override {
+    auto jniVideo = _jniPlatformContext->createVideo(url);
+    return std::make_shared<RNSkAndroidVideo>(jniVideo);
+  }
+
   void releaseNativeBuffer(uint64_t pointer) override {
 #if __ANDROID_API__ >= 26
     AHardwareBuffer *buffer = reinterpret_cast<AHardwareBuffer *>(pointer);

package/android/cpp/rnskia-android/RNSkAndroidVideo.cpp

@@ -0,0 +1,92 @@
+#include <memory>
+#include <string>
+
+#if __ANDROID_API__ >= 26
+#include <android/hardware_buffer_jni.h>
+#endif
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+
+#include "include/core/SkImage.h"
+
+#pragma clang diagnostic pop
+
+#include "RNSkAndroidVideo.h"
+#include "SkiaOpenGLSurfaceFactory.h"
+
+namespace RNSkia {
+
+namespace jsi = facebook::jsi;
+namespace jni = facebook::jni;
+
+RNSkAndroidVideo::RNSkAndroidVideo(jni::global_ref<jobject> jniVideo)
+    : _jniVideo(jniVideo) {
+#if __ANDROID_API__ < 26
+  throw std::runtime_error("Skia Videos are only support on API 26 and above");
+#endif
+}
+
+RNSkAndroidVideo::~RNSkAndroidVideo() {}
+
+sk_sp<SkImage> RNSkAndroidVideo::nextImage(double *timeStamp) {
+#if __ANDROID_API__ >= 26
+  JNIEnv *env = facebook::jni::Environment::current();
+  // Get the Java class and method ID
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid =
+      env->GetMethodID(cls, "nextImage", "()Landroid/hardware/HardwareBuffer;");
+  if (mid == nullptr) {
+    // Method not found, handle error
+    RNSkLogger::logToConsole("nextImage method not found");
+    return nullptr;
+  }
+  // Call the Java method
+  jobject jHardwareBuffer = env->CallObjectMethod(_jniVideo.get(), mid);
+  if (jHardwareBuffer == nullptr) {
+    RNSkLogger::logToConsole("Buffer not found");
+    // Null return, handle error
+    return nullptr;
+  }
+  // Convert jobject to AHardwareBuffer
+  AHardwareBuffer *buffer =
+      AHardwareBuffer_fromHardwareBuffer(env, jHardwareBuffer);
+  return SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(buffer);
+#else
+  return nullptr;
+#endif
+}
+
+double RNSkAndroidVideo::duration() {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "getDuration", "()D");
+  if (!mid) {
+    RNSkLogger::logToConsole("getDuration method not found");
+    return 0.0;
+  }
+  return env->CallDoubleMethod(_jniVideo.get(), mid);
+}
+double RNSkAndroidVideo::framerate() {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "getFrameRate", "()D");
+  if (!mid) {
+    RNSkLogger::logToConsole("getFrameRate method not found");
+    return 0.0;
+  }
+  return env->CallDoubleMethod(_jniVideo.get(), mid);
+}
+
+void RNSkAndroidVideo::seek(double timestamp) {
+  JNIEnv *env = facebook::jni::Environment::current();
+  jclass cls = env->GetObjectClass(_jniVideo.get());
+  jmethodID mid = env->GetMethodID(cls, "seek", "(J)V");
+  if (!mid) {
+    RNSkLogger::logToConsole("seek method not found");
+    return;
+  }
+  env->CallVoidMethod(_jniVideo.get(), mid, static_cast<jlong>(timestamp));
+}
+
+} // namespace RNSkia

package/android/cpp/rnskia-android/RNSkAndroidVideo.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <string>
+
+#include <fbjni/fbjni.h>
+#include <jni.h>
+#include <jsi/jsi.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+
+#include "include/core/SkImage.h"
+
+#pragma clang diagnostic pop
+
+#include "RNSkVideo.h"
+
+namespace RNSkia {
+
+namespace jsi = facebook::jsi;
+namespace jni = facebook::jni;
+
+class RNSkAndroidVideo : public RNSkVideo {
+private:
+  jni::global_ref<jobject> _jniVideo;
+
+public:
+  explicit RNSkAndroidVideo(jni::global_ref<jobject> jniVideo);
+  ~RNSkAndroidVideo();
+  sk_sp<SkImage> nextImage(double *timeStamp = nullptr) override;
+  double duration() override;
+  double framerate() override;
+  void seek(double timestamp) override;
+};
+
+} // namespace RNSkia

package/android/cpp/rnskia-android/SkiaOpenGLSurfaceFactory.cpp

@@ -37,7 +37,6 @@ SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(void *buffer,
   // TODO: find out if we can detect, which graphic buffers support
   // GR_GL_TEXTURE_2D
   case AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM:
-  case AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM:
     format = GrBackendFormats::MakeGL(GR_GL_RGBA8, GR_GL_TEXTURE_EXTERNAL);
   case AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT:
     format = GrBackendFormats::MakeGL(GR_GL_RGBA16F, GR_GL_TEXTURE_EXTERNAL);
@@ -64,9 +63,10 @@ SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(void *buffer,
       const_cast<AHardwareBuffer *>(hardwareBuffer), description.width,
       description.height, &deleteImageProc, &updateImageProc, &deleteImageCtx,
       false, format, false);
-  if (!backendTex.isValid()) [[unlikely]] {
-    throw std::runtime_error(
+  if (!backendTex.isValid()) {
+    RNSkLogger::logToConsole(
         "Failed to convert HardwareBuffer to OpenGL Texture!");
+    return nullptr;
   }
   sk_sp<SkImage> image = SkImages::BorrowTextureFrom(
       ThreadContextHolder::ThreadSkiaOpenGLContext.directContext.get(),

package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java

@@ -38,6 +38,11 @@ public class PlatformContext {
         mHybridData = initHybrid(reactContext.getResources().getDisplayMetrics().density);
     }
 
+    @DoNotStrip
+    public Object createVideo(String url) {
+        return new RNSkVideo(mContext, url);
+    }
+
     private byte[] getStreamAsBytes(InputStream is) throws IOException {
         ByteArrayOutputStream buffer = new ByteArrayOutputStream();
         int nRead;

package/android/src/main/java/com/shopify/reactnative/skia/RNSkVideo.java

@@ -0,0 +1,185 @@
+package com.shopify.reactnative.skia;
+
+import android.content.Context;
+import android.graphics.ImageFormat;
+import android.hardware.HardwareBuffer;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.MediaCodec;
+import android.media.MediaExtractor;
+import android.media.MediaFormat;
+import android.net.Uri;
+import android.os.Build;
+import android.view.Surface;
+
+import androidx.annotation.RequiresApi;
+
+import com.facebook.jni.annotations.DoNotStrip;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class RNSkVideo {
+    private final Uri uri;
+    private final Context context;
+
+    private MediaExtractor extractor;
+    private MediaCodec decoder;
+    private ImageReader imageReader;
+    private Surface outputSurface;
+    private double durationMs;
+    private double frameRate;
+
+    RNSkVideo(Context context, String localUri) {
+        this.uri = Uri.parse(localUri);
+        this.context = context;
+        this.initializeReader();
+    }
+
+    private void initializeReader() {
+        extractor = new MediaExtractor();
+        try {
+            extractor.setDataSource(context, this.uri, null);
+            int trackIndex = selectVideoTrack(extractor);
+            if (trackIndex < 0) {
+                throw new RuntimeException("No video track found in " + this.uri);
+            }
+            extractor.selectTrack(trackIndex);
+            MediaFormat format = extractor.getTrackFormat(trackIndex);
+            // Retrieve and store video properties
+            if (format.containsKey(MediaFormat.KEY_DURATION)) {
+                durationMs = format.getLong(MediaFormat.KEY_DURATION) / 1000;  // Convert microseconds to milliseconds
+            }
+            if (format.containsKey(MediaFormat.KEY_FRAME_RATE)) {
+                frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE);
+            }
+            int width = format.getInteger(MediaFormat.KEY_WIDTH);
+            int height = format.getInteger(MediaFormat.KEY_HEIGHT);
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+                imageReader = ImageReader.newInstance(
+                        width,
+                        height,
+                        ImageFormat.PRIVATE,
+                        2,
+                        HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
+                );
+            } else {
+                imageReader = ImageReader.newInstance(width, height,
+                        ImageFormat.PRIVATE, 2);
+            }
+            outputSurface = imageReader.getSurface();
+
+            // Create a decoder for the format
+            String mime = format.getString(MediaFormat.KEY_MIME);
+            decoder = MediaCodec.createDecoderByType(mime);
+
+            // Note: Use an output Surface for rendering if necessary, otherwise handle buffers
+            decoder.configure(format, outputSurface, null, 0);
+            decoder.start();
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to initialize extractor or decoder", e);
+        }
+    }
+
+    @DoNotStrip
+    public double getDuration() {
+        return durationMs;
+    }
+
+    @DoNotStrip
+
+    public double getFrameRate() {
+        return frameRate;
+    }
+
+    @DoNotStrip
+    public HardwareBuffer nextImage() {
+        if (!decoderOutputAvailable()) {
+            decodeFrame();
+        }
+
+        Image image = imageReader.acquireLatestImage();
+        if (image != null) {
+            HardwareBuffer hardwareBuffer = image.getHardwareBuffer();
+            image.close();  // Make sure to close the Image to free up the buffer
+            return hardwareBuffer;
+        }
+        return null;
+    }
+
+    @DoNotStrip
+    public void seek(long timestamp) {
+        // Seek to the closest sync frame at or before the specified time
+        extractor.seekTo(timestamp * 1000, MediaExtractor.SEEK_TO_PREVIOUS_SYNC);
+        // Flush the codec to reset internal state and buffers
+        if (decoder != null) {
+            decoder.flush();
+        }
+    }
+
+    private int selectVideoTrack(MediaExtractor extractor) {
+        int numTracks = extractor.getTrackCount();
+        for (int i = 0; i < numTracks; i++) {
+            MediaFormat format = extractor.getTrackFormat(i);
+            String mime = format.getString(MediaFormat.KEY_MIME);
+            if (mime.startsWith("video/")) {
+                return i;
+            }
+        }
+        return -1;
+    }
+
+    private boolean decoderOutputAvailable() {
+        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+        int outputBufferId = decoder.dequeueOutputBuffer(info, 0);
+        if (outputBufferId >= 0) {
+            // If a buffer is available, release it immediately back since we are just checking
+            decoder.releaseOutputBuffer(outputBufferId, true);
+            return true;
+        }
+        return false;
+    }
+
+    private void decodeFrame() {
+        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
+        long timeoutUs = 10000;
+        boolean isEOS = false;
+
+        int inputBufferId = decoder.dequeueInputBuffer(timeoutUs);
+        if (inputBufferId >= 0) {
+            ByteBuffer inputBuffer = decoder.getInputBuffer(inputBufferId);
+            int sampleSize = extractor.readSampleData(inputBuffer, 0);
+            if (sampleSize < 0) {
+                // End of stream, make sure to send this information to the decoder
+                decoder.queueInputBuffer(inputBufferId, 0, 0, 0L, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+                isEOS = true;
+            } else {
+                long presentationTimeUs = extractor.getSampleTime();
+                decoder.queueInputBuffer(inputBufferId, 0, sampleSize, presentationTimeUs, 0);
+                extractor.advance();
+            }
+        }
+
+        int outputBufferId = decoder.dequeueOutputBuffer(info, timeoutUs);
+        if (outputBufferId >= 0) {
+            // If we have a valid buffer, release it to make it available to the ImageReader's surface
+            decoder.releaseOutputBuffer(outputBufferId, true);
+
+            if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
+                isEOS = true;
+            }
+        }
+    }
+
+    public void release() {
+        if (decoder != null) {
+            decoder.stop();
+            decoder.release();
+            decoder = null;
+        }
+        if (extractor != null) {
+            extractor.release();
+            extractor = null;
+        }
+    }
+}

package/cpp/api/JsiSkApi.h

@@ -50,6 +50,7 @@
 #include "JsiSkTypefaceFactory.h"
 #include "JsiSkTypefaceFontProviderFactory.h"
 #include "JsiSkVertices.h"
+#include "JsiVideo.h"
 
 namespace RNSkia {
 
@@ -67,6 +68,7 @@ public:
     // We create the system font manager eagerly since it has proven to be too
     // slow to do it on demand
     JsiSkFontMgrFactory::getFontMgr(getContext());
+    installFunction("Video", JsiVideo::createCtor(context));
     installFunction("Font", JsiSkFont::createCtor(context));
     installFunction("Paint", JsiSkPaint::createCtor(context));
     installFunction("RSXform", JsiSkRSXform::createCtor(context));

package/cpp/api/JsiVideo.h

@@ -0,0 +1,87 @@
+#pragma once
+
+#include <memory>
+#include <numeric>
+#include <utility>
+#include <vector>
+
+#include "JsiSkHostObjects.h"
+#include "RNSkLog.h"
+#include <jsi/jsi.h>
+
+#include "JsiSkPaint.h"
+#include "JsiSkPoint.h"
+#include "JsiSkRect.h"
+#include "JsiSkTypeface.h"
+
+#include "RNSkVideo.h"
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+
+#include "include/core/SkFont.h"
+#include "include/core/SkFontMetrics.h"
+
+#pragma clang diagnostic pop
+
+namespace RNSkia {
+
+namespace jsi = facebook::jsi;
+
+class JsiVideo : public JsiSkWrappingSharedPtrHostObject<RNSkVideo> {
+public:
+  EXPORT_JSI_API_TYPENAME(JsiVideo, Video)
+
+  JSI_HOST_FUNCTION(nextImage) {
+    double timestamp = 0;
+    auto video = getObject();
+    auto image = video->nextImage(&timestamp);
+    if (!image) {
+      return jsi::Value::null();
+    }
+    return jsi::Object::createFromHostObject(
+        runtime, std::make_shared<JsiSkImage>(getContext(), std::move(image)));
+  }
+
+  JSI_HOST_FUNCTION(duration) { return getObject()->duration(); }
+
+  JSI_HOST_FUNCTION(framerate) { return getObject()->framerate(); }
+
+  JSI_HOST_FUNCTION(seek) {
+    double timestamp = arguments[0].asNumber();
+    getObject()->seek(timestamp);
+    return jsi::Value::undefined();
+  }
+
+  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiVideo, nextImage),
+                       JSI_EXPORT_FUNC(JsiVideo, duration),
+                       JSI_EXPORT_FUNC(JsiVideo, framerate),
+                       JSI_EXPORT_FUNC(JsiVideo, seek),
+                       JSI_EXPORT_FUNC(JsiVideo, dispose))
+
+  JsiVideo(std::shared_ptr<RNSkPlatformContext> context,
+           std::shared_ptr<RNSkVideo> video)
+      : JsiSkWrappingSharedPtrHostObject(std::move(context), std::move(video)) {
+  }
+
+  /**
+   * Creates the function for construction a new instance of the SkFont
+   * wrapper
+   * @param context Platform context
+   * @return A function for creating a new host object wrapper for the SkFont
+   * class
+   */
+  static const jsi::HostFunctionType
+  createCtor(std::shared_ptr<RNSkPlatformContext> context) {
+    return JSI_HOST_FUNCTION_LAMBDA {
+      auto url = arguments[0].asString(runtime).utf8(runtime);
+      auto video = context->createVideo(url);
+      // Return the newly constructed object
+      return jsi::Object::createFromHostObject(
+          runtime,
+          std::make_shared<JsiVideo>(std::move(context), std::move(video)));
+    };
+  }
+};
+
+} // namespace RNSkia

package/cpp/rnskia/RNSkPlatformContext.h

@@ -10,6 +10,7 @@
 #include <utility>
 
 #include "RNSkDispatchQueue.h"
+#include "RNSkVideo.h"
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -146,6 +147,8 @@ public:
 
   virtual uint64_t makeNativeBuffer(sk_sp<SkImage> image) = 0;
 
+  virtual std::shared_ptr<RNSkVideo> createVideo(const std::string &url) = 0;
+
   /**
    * Return the Platform specific font manager
    */

package/cpp/rnskia/RNSkVideo.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <string>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+
+#include "include/core/SkImage.h"
+
+#pragma clang diagnostic pop
+
+namespace RNSkia {
+
+class RNSkVideo {
+public:
+  virtual ~RNSkVideo() = default;
+  virtual sk_sp<SkImage> nextImage(double *timeStamp = nullptr) = 0;
+  virtual double duration() = 0;
+  virtual double framerate() = 0;
+  virtual void seek(double timestamp) = 0;
+};
+
+} // namespace RNSkia

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.h

@@ -65,6 +65,8 @@ public:
 
   void releaseNativeBuffer(uint64_t pointer) override;
 
+  std::shared_ptr<RNSkVideo> createVideo(const std::string &url) override;
+
   virtual void performStreamOperation(
       const std::string &sourceUri,
       const std::function<void(std::unique_ptr<SkStreamAsset>)> &op) override;

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm

@@ -5,6 +5,7 @@
 #include <thread>
 #include <utility>
 
+#include "RNSkiOSVideo.h"
 #import "SkiaCVPixelBufferUtils.h"
 #import "SkiaMetalSurfaceFactory.h"
 
@@ -143,6 +144,11 @@ uint64_t RNSkiOSPlatformContext::makeNativeBuffer(sk_sp<SkImage> image) {
   return reinterpret_cast<uint64_t>(pixelBuffer);
 }
 
+std::shared_ptr<RNSkVideo>
+RNSkiOSPlatformContext::createVideo(const std::string &url) {
+  return std::make_shared<RNSkiOSVideo>(url, this);
+}
+
 void RNSkiOSPlatformContext::raiseError(const std::exception &err) {
   RCTFatal(RCTErrorWithMessage([NSString stringWithUTF8String:err.what()]));
 }

package/ios/RNSkia-iOS/RNSkiOSVideo.h

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <string>
+
+#include <AVFoundation/AVFoundation.h>
+#include <CoreVideo/CoreVideo.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+
+#include "include/core/SkImage.h"
+
+#pragma clang diagnostic pop
+
+#include "RNSkPlatformContext.h"
+#include "RNSkVideo.h"
+
+namespace RNSkia {
+
+class RNSkiOSVideo : public RNSkVideo {
+private:
+  std::string _url;
+  AVAssetReader *_reader = nullptr;
+  AVAssetReaderTrackOutput *_trackOutput = nullptr;
+  RNSkPlatformContext *_context;
+  double _duration = 0;
+  double _framerate = 0;
+  void setupReader(CMTimeRange timeRange);
+  NSDictionary *getOutputSettings();
+
+public:
+  RNSkiOSVideo(std::string url, RNSkPlatformContext *context);
+  ~RNSkiOSVideo();
+  sk_sp<SkImage> nextImage(double *timeStamp = nullptr) override;
+  double duration() override;
+  double framerate() override;
+  void seek(double timestamp) override;
+};
+
+} // namespace RNSkia