@shopify/react-native-skia 1.2.0 → 1.2.2

package/android/cpp/rnskia-android/RNSkAndroidPlatformContext.h

@@ -53,18 +53,18 @@ public:
     return SkiaOpenGLSurfaceFactory::makeOffscreenSurface(width, height);
   }
 
-  sk_sp<SkImage> makeImageFromPlatformBuffer(void *buffer) override {
+  sk_sp<SkImage> makeImageFromNativeBuffer(void *buffer) override {
     return SkiaOpenGLSurfaceFactory::makeImageFromHardwareBuffer(buffer);
   }
 
-  void releasePlatformBuffer(uint64_t pointer) override {
+  void releaseNativeBuffer(uint64_t pointer) override {
 #if __ANDROID_API__ >= 26
     AHardwareBuffer *buffer = reinterpret_cast<AHardwareBuffer *>(pointer);
     AHardwareBuffer_release(buffer);
 #endif
   }
 
-  uint64_t makePlatformBuffer(sk_sp<SkImage> image) override {
+  uint64_t makeNativeBuffer(sk_sp<SkImage> image) override {
 #if __ANDROID_API__ >= 26
     auto bytesPerPixel = image->imageInfo().bytesPerPixel();
     int bytesPerRow = image->width() * bytesPerPixel;

package/android/src/main/java/com/shopify/reactnative/skia/ViewScreenshotService.java

@@ -162,16 +162,21 @@ public class ViewScreenshotService {
                 latch.await(SURFACE_VIEW_READ_PIXELS_TIMEOUT, TimeUnit.SECONDS);
             } catch (Exception e) {
                 Log.e(TAG, "Cannot PixelCopy for " + sv, e);
+                drawSurfaceViewFromCache(canvas, sv, paint, opacity);
             }
         } else {
-            Bitmap cache = sv.getDrawingCache();
-            if (cache != null) {
-                canvas.save();
-                applyTransformations(canvas, sv);
-                paint.setAlpha(Math.round(opacity * 255)); // Set paint alpha based on opacity
-                canvas.drawBitmap(cache, 0, 0, paint);
-                canvas.restore();
-            }
+            drawSurfaceViewFromCache(canvas, sv, paint, opacity);
+        }
+    }
+
+    private static void drawSurfaceViewFromCache(Canvas canvas, SurfaceView sv, Paint paint, float opacity) {
+        Bitmap cache = sv.getDrawingCache();
+        if (cache != null) {
+            canvas.save();
+            applyTransformations(canvas, sv);
+            paint.setAlpha(Math.round(opacity * 255)); // Set paint alpha based on opacity
+            canvas.drawBitmap(cache, 0, 0, paint);
+            canvas.restore();
         }
     }
 

package/cpp/api/JsiNativeBuffer.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <memory>
+#include <utility>
+
+#include <jsi/jsi.h>
+
+#include "JsiSkImage.h"
+
+namespace RNSkia {
+
+namespace jsi = facebook::jsi;
+
+/**
+ Implementation of the ParagraphBuilderFactory for making ParagraphBuilder JSI
+ object
+ */
+class JsiNativeBufferFactory : public JsiSkHostObject {
+public:
+  JSI_HOST_FUNCTION(MakeFromImage) {
+    auto image = JsiSkImage::fromValue(runtime, arguments[0]);
+    image->makeNonTextureImage();
+    uint64_t pointer = getContext()->makeNativeBuffer(image);
+    return jsi::BigInt::fromUint64(runtime, pointer);
+  }
+
+  JSI_HOST_FUNCTION(Release) {
+
+    jsi::BigInt pointer = arguments[0].asBigInt(runtime);
+    const uintptr_t nativeBufferPointer = pointer.asUint64(runtime);
+
+    getContext()->releaseNativeBuffer(nativeBufferPointer);
+    return jsi::Value::undefined();
+  }
+
+  JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiNativeBufferFactory, Release),
+                       JSI_EXPORT_FUNC(JsiNativeBufferFactory, MakeFromImage))
+
+  explicit JsiNativeBufferFactory(std::shared_ptr<RNSkPlatformContext> context)
+      : JsiSkHostObject(std::move(context)) {}
+};
+
+} // namespace RNSkia

package/cpp/api/JsiSkApi.h

@@ -6,7 +6,7 @@
 
 #include "JsiSkHostObjects.h"
 
-#include "JsiPlatformBuffer.h"
+#include "JsiNativeBuffer.h"
 #include "JsiSkAnimatedImage.h"
 #include "JsiSkAnimatedImageFactory.h"
 #include "JsiSkColor.h"
@@ -124,8 +124,8 @@ public:
         "ParagraphBuilder",
         std::make_shared<JsiSkParagraphBuilderFactory>(context));
 
-    installReadonlyProperty(
-        "PlatformBuffer", std::make_shared<JsiPlatformBufferFactory>(context));
+    installReadonlyProperty("NativeBuffer",
+                            std::make_shared<JsiNativeBufferFactory>(context));
   }
 };
 } // namespace RNSkia

package/cpp/api/JsiSkImageFactory.h

@@ -27,11 +27,11 @@ public:
         runtime, std::make_shared<JsiSkImage>(getContext(), std::move(image)));
   }
 
-  JSI_HOST_FUNCTION(MakeImageFromPlatformBuffer) {
+  JSI_HOST_FUNCTION(MakeImageFromNativeBuffer) {
     jsi::BigInt pointer = arguments[0].asBigInt(runtime);
     const uintptr_t platformBufferPointer = pointer.asUint64(runtime);
     void *rawPointer = reinterpret_cast<void *>(platformBufferPointer);
-    auto image = getContext()->makeImageFromPlatformBuffer(rawPointer);
+    auto image = getContext()->makeImageFromNativeBuffer(rawPointer);
     if (image == nullptr) {
       throw std::runtime_error("Failed to convert PlatformBuffer to SkImage!");
     }
@@ -81,7 +81,7 @@ public:
   JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromEncoded),
                        JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImageFromViewTag),
                        JSI_EXPORT_FUNC(JsiSkImageFactory,
-                                       MakeImageFromPlatformBuffer),
+                                       MakeImageFromNativeBuffer),
                        JSI_EXPORT_FUNC(JsiSkImageFactory, MakeImage))
 
   explicit JsiSkImageFactory(std::shared_ptr<RNSkPlatformContext> context)

package/cpp/rnskia/RNSkPlatformContext.h

@@ -134,17 +134,17 @@ public:
   virtual sk_sp<SkSurface> makeOffscreenSurface(int width, int height) = 0;
 
   /**
-   * Creates an image from a native platform buffer.
-   * - On iOS, this is a `CMSampleBuffer`
+   * Creates an image from a native buffer.
+   * - On iOS, this is a `CVPixelBufferRef`
    * - On Android, this is a `AHardwareBuffer*`
    * @param buffer The native platform buffer.
    * @return sk_sp<SkImage>
    */
-  virtual sk_sp<SkImage> makeImageFromPlatformBuffer(void *buffer) = 0;
+  virtual sk_sp<SkImage> makeImageFromNativeBuffer(void *buffer) = 0;
 
-  virtual void releasePlatformBuffer(uint64_t pointer) = 0;
+  virtual void releaseNativeBuffer(uint64_t pointer) = 0;
 
-  virtual uint64_t makePlatformBuffer(sk_sp<SkImage> image) = 0;
+  virtual uint64_t makeNativeBuffer(sk_sp<SkImage> image) = 0;
 
   /**
    * Return the Platform specific font manager

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.h

@@ -59,11 +59,11 @@ public:
 
   sk_sp<SkImage> takeScreenshotFromViewTag(size_t tag) override;
 
-  sk_sp<SkImage> makeImageFromPlatformBuffer(void *buffer) override;
+  sk_sp<SkImage> makeImageFromNativeBuffer(void *buffer) override;
 
-  uint64_t makePlatformBuffer(sk_sp<SkImage> image) override;
+  uint64_t makeNativeBuffer(sk_sp<SkImage> image) override;
 
-  void releasePlatformBuffer(uint64_t pointer) override;
+  void releaseNativeBuffer(uint64_t pointer) override;
 
   virtual void performStreamOperation(
       const std::string &sourceUri,

package/ios/RNSkia-iOS/RNSkiOSPlatformContext.mm

@@ -1,11 +1,12 @@
-#include "RNSkiOSPlatformContext.h"
+#import "RNSkiOSPlatformContext.h"
 
 #import <CoreMedia/CMSampleBuffer.h>
 #import <React/RCTUtils.h>
 #include <thread>
 #include <utility>
 
-#include "SkiaMetalSurfaceFactory.h"
+#import "SkiaCVPixelBufferUtils.h"
+#import "SkiaMetalSurfaceFactory.h"
 
 #pragma clang diagnostic push
 #pragma clang diagnostic ignored "-Wdocumentation"
@@ -57,87 +58,89 @@ void RNSkiOSPlatformContext::performStreamOperation(
   std::thread(loader).detach();
 }
 
-void RNSkiOSPlatformContext::releasePlatformBuffer(uint64_t pointer) {
-  CMSampleBufferRef sampleBuffer = reinterpret_cast<CMSampleBufferRef>(pointer);
-  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-  if (sampleBuffer) {
-    CFRelease(sampleBuffer);
-  }
+void RNSkiOSPlatformContext::releaseNativeBuffer(uint64_t pointer) {
+  CVPixelBufferRef pixelBuffer = reinterpret_cast<CVPixelBufferRef>(pointer);
   if (pixelBuffer) {
     CFRelease(pixelBuffer);
   }
 }
 
-uint64_t RNSkiOSPlatformContext::makePlatformBuffer(sk_sp<SkImage> image) {
+uint64_t RNSkiOSPlatformContext::makeNativeBuffer(sk_sp<SkImage> image) {
+  // 0. If Image is not in BGRA, convert to BGRA as only BGRA is supported.
+  if (image->colorType() != kBGRA_8888_SkColorType) {
+    // on iOS, 32_BGRA is the only supported RGB format for CVPixelBuffers.
+    image = image->makeColorTypeAndColorSpace(
+        ThreadContextHolder::ThreadSkiaMetalContext.skContext.get(),
+        kBGRA_8888_SkColorType, SkColorSpace::MakeSRGB());
+    if (image == nullptr) {
+      throw std::runtime_error(
+          "Failed to convert image to BGRA_8888 colortype! Only BGRA_8888 "
+          "PlatformBuffers are supported.");
+    }
+  }
+
+  // 1. Get image info
   auto bytesPerPixel = image->imageInfo().bytesPerPixel();
   int bytesPerRow = image->width() * bytesPerPixel;
   auto buf = SkData::MakeUninitialized(image->width() * image->height() *
                                        bytesPerPixel);
   SkImageInfo info = SkImageInfo::Make(image->width(), image->height(),
                                        image->colorType(), image->alphaType());
+  // 2. Copy pixels into our buffer
   image->readPixels(nullptr, info, const_cast<void *>(buf->data()), bytesPerRow,
                     0, 0);
-  auto pixelData = const_cast<void *>(buf->data());
 
-  // Create a CVPixelBuffer from the raw pixel data
-  CVPixelBufferRef pixelBuffer = nullptr;
-  // OSType pixelFormatType = MapSkColorTypeToOSType(image->colorType());
-
-  // You will need to fill in the details for creating the pixel buffer
-  // CVPixelBufferCreateWithBytes or CVPixelBufferCreateWithPlanarBytes
-  // Create the CVPixelBuffer with the image data
-  void *context = static_cast<void *>(
-      new sk_sp<SkData>(buf)); // Create a copy for the context
-  CVReturn r = CVPixelBufferCreateWithBytes(
-      nullptr, // allocator
-      image->width(), image->height(), kCVPixelFormatType_32BGRA,
-      pixelData,                                         // pixel data
-      bytesPerRow,                                       // bytes per row
-      [](void *releaseRefCon, const void *baseAddress) { // release callback
-        auto buf = static_cast<sk_sp<SkData> *>(releaseRefCon);
-        buf->reset(); // This effectively calls unref on the SkData object
-        delete buf;   // Cleanup the dynamically allocated context
-      },
-      context,     // release callback context
-      nullptr,     // pixel buffer attributes
-      &pixelBuffer // the newly created pixel buffer
-  );
-
-  if (r != kCVReturnSuccess) {
-    return 0; // or handle error appropriately
+  // 3. Create an IOSurface (GPU + CPU memory)
+  CFMutableDictionaryRef dict = CFDictionaryCreateMutable(
+      kCFAllocatorDefault, 0, &kCFTypeDictionaryKeyCallBacks,
+      &kCFTypeDictionaryValueCallBacks);
+  int width = image->width();
+  int height = image->height();
+  int pitch = width * bytesPerPixel;
+  int size = width * height * bytesPerPixel;
+  OSType pixelFormat = kCVPixelFormatType_32BGRA;
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerRow,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pitch));
+  CFDictionarySetValue(
+      dict, kIOSurfaceBytesPerElement,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bytesPerPixel));
+  CFDictionarySetValue(
+      dict, kIOSurfaceWidth,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width));
+  CFDictionarySetValue(
+      dict, kIOSurfaceHeight,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height));
+  CFDictionarySetValue(
+      dict, kIOSurfacePixelFormat,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pixelFormat));
+  CFDictionarySetValue(
+      dict, kIOSurfaceAllocSize,
+      CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &size));
+  IOSurfaceRef surface = IOSurfaceCreate(dict);
+  if (surface == nil) {
+    throw std::runtime_error("Failed to create " + std::to_string(width) + "x" +
+                             std::to_string(height) + " IOSurface!");
   }
 
-  // Wrap the CVPixelBuffer in a CMSampleBuffer
-  CMSampleBufferRef sampleBuffer = nullptr;
-
-  CMFormatDescriptionRef formatDescription = nullptr;
-  CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer,
-                                               &formatDescription);
+  // 4. Copy over the memory from the pixels into the IOSurface
+  IOSurfaceLock(surface, 0, nil);
+  void *base = IOSurfaceGetBaseAddress(surface);
+  memcpy(base, buf->data(), buf->size());
+  IOSurfaceUnlock(surface, 0, nil);
 
-  // Assuming no specific timing is required, we initialize the timing info to
-  // zero.
-  CMSampleTimingInfo timingInfo = {0};
-  timingInfo.duration = kCMTimeInvalid; // Indicate an unknown duration.
-  timingInfo.presentationTimeStamp = kCMTimeZero; // Start at time zero.
-  timingInfo.decodeTimeStamp = kCMTimeInvalid;    // No specific decode time.
-
-  // Create the sample buffer.
-  OSStatus status = CMSampleBufferCreateReadyWithImageBuffer(
-      kCFAllocatorDefault, pixelBuffer, formatDescription, &timingInfo,
-      &sampleBuffer);
-
-  if (status != noErr) {
-    if (formatDescription) {
-      CFRelease(formatDescription);
-    }
-    if (pixelBuffer) {
-      CFRelease(pixelBuffer);
-    }
-    return 0;
+  // 5. Create a CVPixelBuffer from the IOSurface
+  CVPixelBufferRef pixelBuffer = nullptr;
+  CVReturn result =
+      CVPixelBufferCreateWithIOSurface(nil, surface, nil, &pixelBuffer);
+  if (result != kCVReturnSuccess) {
+    throw std::runtime_error(
+        "Failed to create CVPixelBuffer from SkImage! Return value: " +
+        std::to_string(result));
   }
 
-  // Return sampleBuffer casted to uint64_t
-  return reinterpret_cast<uint64_t>(sampleBuffer);
+  // 8. Return CVPixelBuffer casted to uint64_t
+  return reinterpret_cast<uint64_t>(pixelBuffer);
 }
 
 void RNSkiOSPlatformContext::raiseError(const std::exception &err) {
@@ -149,35 +152,9 @@ sk_sp<SkSurface> RNSkiOSPlatformContext::makeOffscreenSurface(int width,
   return SkiaMetalSurfaceFactory::makeOffscreenSurface(width, height);
 }
 
-sk_sp<SkImage>
-RNSkiOSPlatformContext::makeImageFromPlatformBuffer(void *buffer) {
-  CMSampleBufferRef sampleBuffer = (CMSampleBufferRef)buffer;
-  // DO the CPU transfer (debugging only)
-  //  Step 1: Extract the CVPixelBufferRef from the CMSampleBufferRef
-  CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-
-  // Step 2: Lock the pixel buffer to access the raw pixel data
-  CVPixelBufferLockBaseAddress(pixelBuffer, 0);
-
-  // Step 3: Get information about the image
-  void *baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer);
-  size_t width = CVPixelBufferGetWidth(pixelBuffer);
-  size_t height = CVPixelBufferGetHeight(pixelBuffer);
-  size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
-
-  // Assuming the pixel format is 32BGRA, which is common for iOS video frames.
-  // You might need to adjust this based on the actual pixel format.
-  SkImageInfo info = SkImageInfo::Make(width, height, kRGBA_8888_SkColorType,
-                                       kUnpremul_SkAlphaType);
-
-  // Step 4: Create an SkImage from the pixel buffer
-  sk_sp<SkData> data =
-      SkData::MakeWithoutCopy(baseAddress, height * bytesPerRow);
-  sk_sp<SkImage> image = SkImages::RasterFromData(info, data, bytesPerRow);
-  auto texture = SkiaMetalSurfaceFactory::makeTextureFromImage(image);
-  // Step 5: Unlock the pixel buffer
-  CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
-  return texture;
+sk_sp<SkImage> RNSkiOSPlatformContext::makeImageFromNativeBuffer(void *buffer) {
+  CVPixelBufferRef sampleBuffer = (CVPixelBufferRef)buffer;
+  return SkiaMetalSurfaceFactory::makeTextureFromCVPixelBuffer(sampleBuffer);
 }
 
 sk_sp<SkFontMgr> RNSkiOSPlatformContext::createFontMgr() {

package/ios/RNSkia-iOS/SkiaCVPixelBufferUtils.h

@@ -0,0 +1,119 @@
+//
+//  SkiaCVPixelBufferUtils.h
+//  react-native-skia
+//
+//  Created by Marc Rousavy on 10.04.24.
+//
+
+#pragma once
+
+#import <vector>
+
+#import <CoreMedia/CMSampleBuffer.h>
+#import <CoreVideo/CVMetalTextureCache.h>
+#import <MetalKit/MetalKit.h>
+
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wdocumentation"
+#import "include/core/SkColorSpace.h"
+#import "include/core/SkImage.h"
+#import "include/gpu/GrBackendSurface.h"
+#import "include/gpu/GrYUVABackendTextures.h"
+#pragma clang diagnostic pop
+
+/**
+ Holds a Metal Texture.
+ When the `TextureHolder` is destroyed, the underlying Metal Texture
+ is marked as invalid and might be overwritten by the Texture Cache,
+ so make sure to delete the `TextureHolder` only when the `SkImage`
+ has been deleted.
+
+ For example, use the `releaseProc` parameter in `BorrowImageFromTexture`
+ to delete the `TextureHolder`.
+ */
+class TextureHolder {
+public:
+  /**
+   Create a new instance of TextureHolder which holds
+   the given `CVMetalTextureRef`.
+
+   The given `CVMetalTextureRef` is assumed to already be
+   retained with `CFRetain`, and will later be manually
+   released with `CFRelease`.
+   */
+  TextureHolder(CVMetalTextureRef texture);
+  ~TextureHolder();
+
+  /**
+   Converts this Texture to a Skia GrBackendTexture.
+   */
+  GrBackendTexture toGrBackendTexture();
+
+private:
+  CVMetalTextureRef _texture;
+};
+
+/**
+ Same as `TextureHolder`, but for multiple planar textures (e.g. YUVA)
+ */
+class MultiTexturesHolder {
+public:
+  ~MultiTexturesHolder();
+  void addTexture(TextureHolder *texture);
+
+private:
+  std::vector<TextureHolder *> _textures;
+};
+
+class SkiaCVPixelBufferUtils {
+public:
+  enum class CVPixelBufferBaseFormat { rgb, yuv };
+
+  /**
+   Get the base format (currently only RGB) of the PixelBuffer.
+   Depending on the base-format, different methods have to be used to create
+   Skia buffers.
+   */
+  static CVPixelBufferBaseFormat
+  getCVPixelBufferBaseFormat(CVPixelBufferRef pixelBuffer);
+
+  class RGB {
+  public:
+    /**
+     Creates a GPU-backed Skia Texture (SkImage) with the given RGB
+     CVPixelBuffer.
+     */
+    static sk_sp<SkImage>
+    makeSkImageFromCVPixelBuffer(GrDirectContext *context,
+                                 CVPixelBufferRef pixelBuffer);
+
+  private:
+    static SkColorType getCVPixelBufferColorType(CVPixelBufferRef pixelBuffer);
+  };
+
+  class YUV {
+  public:
+    /**
+     Creates a GPU-backed Skia Texture (SkImage) with the given YUV
+     CVPixelBuffer.
+     */
+    static sk_sp<SkImage>
+    makeSkImageFromCVPixelBuffer(GrDirectContext *context,
+                                 CVPixelBufferRef pixelBuffer);
+
+  private:
+    static SkYUVAInfo::PlaneConfig getPlaneConfig(OSType pixelFormat);
+    static SkYUVAInfo::Subsampling getSubsampling(OSType pixelFormat);
+    static SkYUVColorSpace getColorspace(OSType pixelFormat);
+    static SkYUVAInfo getYUVAInfoForCVPixelBuffer(CVPixelBufferRef pixelBuffer);
+  };
+
+private:
+  static CVMetalTextureCacheRef getTextureCache();
+  static TextureHolder *
+  getSkiaTextureForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                      size_t planeIndex);
+  static MTLPixelFormat
+  getMTLPixelFormatForCVPixelBufferPlane(CVPixelBufferRef pixelBuffer,
+                                         size_t planeIndex);
+};