@stream-io/video-react-native-sdk 1.0.0-rc2.0 → 1.0.1

package/ios/PictureInPicture/StreamPictureInPictureVideoRenderer.swift

@@ -0,0 +1,250 @@
+//
+// Copyright © 2024 Stream.io Inc. All rights reserved.
+//
+
+import Combine
+import Foundation
+
+/// A view that can be used to render an instance of `RTCVideoTrack`
+final class StreamPictureInPictureVideoRenderer: UIView, RTCVideoRenderer {
+    
+    /// The rendering track.
+    var track: RTCVideoTrack? {
+        didSet {
+            // Whenever the track changes we perform the following operations if possible:
+            // - stopFrameStreaming for the old track
+            // - startFrameStreaming for the new track and only if we are already
+            // in picture-in-picture.
+            guard oldValue != track else { return }
+            prepareForTrackRendering(oldValue)
+        }
+    }
+    
+    /// The layer that renders the track's frames.
+    var displayLayer: CALayer { contentView.layer }
+
+    /// A policy defining how the Picture in Picture window should be resized in order to better fit
+    /// the rendering frame size.
+    var pictureInPictureWindowSizePolicy: PictureInPictureWindowSizePolicy
+    
+    /// The publisher which is used to streamline the frames received from the track.
+    private let bufferPublisher: PassthroughSubject<CMSampleBuffer, Never> = .init()
+    
+    /// The view that contains the rendering layer.
+    private lazy var contentView: SampleBufferVideoCallView = {
+        let contentView = SampleBufferVideoCallView()
+        contentView.translatesAutoresizingMaskIntoConstraints = false
+        contentView.contentMode = .scaleAspectFill
+        contentView.videoGravity = .resizeAspectFill
+        contentView.preventsDisplaySleepDuringVideoPlayback = true
+        return contentView
+    }()
+    
+    /// The transformer used to transform and downsample a RTCVideoFrame's buffer.
+    private var bufferTransformer = StreamBufferTransformer()
+    
+    /// The cancellable used to control the bufferPublisher stream.
+    private var bufferUpdatesCancellable: AnyCancellable?
+    
+    /// The view's size.
+    /// - Note: We are using this property instead for `frame.size` or `bounds.size` so we can
+    /// access it from any thread.
+    private var contentSize: CGSize = .zero
+    
+    /// The track's size.
+    private var trackSize: CGSize = .zero {
+        didSet {
+            guard trackSize != oldValue else { return }
+            didUpdateTrackSize()
+        }
+    }
+    
+    /// A property that defines if the RTCVideoFrame instances that will be rendered need to be resized
+    /// to fid the view's contentSize.
+    private var requiresResize = false {
+        didSet { bufferTransformer.requiresResize = requiresResize }
+    }
+    
+    /// As we are operate in smaller rendering bounds we skip frames depending on this property's value
+    /// to improve performance.
+    /// - Note: The number of frames to skip is being calculated based on the ``trackSize`` and
+    /// ``contentSize``. It takes into account also the ``sizeRatioThreshold``
+    private var noOfFramesToSkipAfterRendering = 1
+    
+    /// The number of frames that we have skipped so far. This is used as a step counter in the
+    /// ``renderFrame(_:)``.
+    private var skippedFrames = 0
+    
+    /// We render frames every time the stepper/counter value is 0 and have a valid trackSize.
+    private var shouldRenderFrame: Bool { skippedFrames == 0 && trackSize != .zero }
+    
+    /// A size ratio threshold used to determine if resizing is required.
+    /// - Note: It seems that Picture-in-Picture doesn't like rendering frames that are bigger than its
+    /// window size. For this reason, we are setting the resizeThreshold to `1`.
+    private let resizeRequiredSizeRatioThreshold: CGFloat = 1
+    
+    /// A size ratio threshold used to determine if skipping frames is required.
+    private let sizeRatioThreshold: CGFloat = 15
+    
+    // MARK: - Lifecycle
+    
+    @available(*, unavailable)
+    required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") }
+    
+    init(windowSizePolicy: PictureInPictureWindowSizePolicy) {
+        pictureInPictureWindowSizePolicy = windowSizePolicy
+        super.init(frame: .zero)
+        setUp()
+    }
+    
+    override func willMove(toWindow newWindow: UIWindow?) {
+        super.willMove(toWindow: newWindow)
+        // Depending on the window we are moving we either start or stop
+        // streaming frames from the track.
+        if newWindow != nil {
+            startFrameStreaming(for: track, on: newWindow)
+        } else {
+            stopFrameStreaming(for: track)
+        }
+    }
+    
+    override func layoutSubviews() {
+        super.layoutSubviews()
+        contentSize = frame.size
+    }
+    
+    // MARK: - Rendering lifecycle
+    
+    /// This method is being called from WebRTC and asks the container to set its size to the track's size.
+    func setSize(_ size: CGSize) {
+        trackSize = size
+    }
+    
+    func renderFrame(_ frame: RTCVideoFrame?) {
+        guard let frame = frame else {
+            return
+        }
+        
+        // Update the trackSize and re-calculate rendering properties if the size
+        // has changed.
+        trackSize = .init(width: Int(frame.width), height: Int(frame.height))
+        
+        defer {
+            handleFrameSkippingIfRequired()
+        }
+        
+        guard shouldRenderFrame else {
+            return
+        }
+
+        if
+            let yuvBuffer = bufferTransformer.transformAndResizeIfRequired(frame, targetSize: contentSize)?
+            .buffer as? StreamRTCYUVBuffer,
+            let sampleBuffer = yuvBuffer.sampleBuffer {
+            bufferPublisher.send(sampleBuffer)
+        } 
+    }
+    
+    // MARK: - Private helpers
+    
+    /// Set up the view's hierarchy.
+    private func setUp() {
+        addSubview(contentView)
+        NSLayoutConstraint.activate([
+            contentView.leadingAnchor.constraint(equalTo: leadingAnchor),
+            contentView.trailingAnchor.constraint(equalTo: trailingAnchor),
+            contentView.topAnchor.constraint(equalTo: topAnchor),
+            contentView.bottomAnchor.constraint(equalTo: bottomAnchor)
+        ])
+    }
+    
+    /// A method used to process the frame's buffer and enqueue on the rendering view.
+    private func process(_ buffer: CMSampleBuffer) {
+        guard
+            bufferUpdatesCancellable != nil,
+            let trackId = track?.trackId,
+            buffer.isValid
+        else {
+            contentView.renderingComponent.flush()
+            return
+        }
+        
+        if #available(iOS 14.0, *) {
+            if contentView.renderingComponent.requiresFlushToResumeDecoding == true {
+                contentView.renderingComponent.flush()
+            }
+        }
+        
+        if contentView.renderingComponent.isReadyForMoreMediaData {
+            contentView.renderingComponent.enqueue(buffer)
+        }
+    }
+    
+    /// A method used to start consuming frames from the track.
+    /// - Note: In order to avoid unnecessary processing, we only start consuming track's frames when
+    /// the view has been added on a window (which means that picture-in-picture view is visible).
+    private func startFrameStreaming(
+        for track: RTCVideoTrack?,
+        on window: UIWindow?
+    ) {
+        guard window != nil, let track else { return }
+        
+        bufferUpdatesCancellable = bufferPublisher
+            .receive(on: DispatchQueue.main)
+            .sink { [weak self] in self?.process($0) }
+        
+        track.add(self)
+    }
+    
+    /// A method that stops the frame consumption from the track. Used automatically when the rendering
+    /// view move's away from the window or when the track changes.
+    private func stopFrameStreaming(for track: RTCVideoTrack?) {
+        guard bufferUpdatesCancellable != nil else { return }
+        bufferUpdatesCancellable?.cancel()
+        bufferUpdatesCancellable = nil
+        track?.remove(self)
+        contentView.renderingComponent.flush()
+    }
+    
+    /// A method used to calculate rendering required properties, every time the trackSize changes.
+    private func didUpdateTrackSize() {
+        guard contentSize != .zero, trackSize != .zero else { return }
+        
+        let widthDiffRatio = trackSize.width / contentSize.width
+        let heightDiffRatio = trackSize.height / contentSize.height
+        requiresResize = widthDiffRatio >= resizeRequiredSizeRatioThreshold || heightDiffRatio >= resizeRequiredSizeRatioThreshold
+        let requiresFramesSkipping = widthDiffRatio >= sizeRatioThreshold || heightDiffRatio >= sizeRatioThreshold
+        
+        /// Skipping frames is decided based on how much bigger is the incoming frame's size compared
+        /// to PiP window's size.
+        noOfFramesToSkipAfterRendering = requiresFramesSkipping ? max(Int(max(Int(widthDiffRatio), Int(heightDiffRatio)) / 2), 1) :
+        0
+        skippedFrames = 0
+
+         /// We update the provided windowSizePolicy with the size of the track we received, transformed
+        /// to the value that fits.
+        pictureInPictureWindowSizePolicy.trackSize = trackSize
+    }
+    
+    /// A method used to handle the frameSkipping(step) during frame consumption.
+    private func handleFrameSkippingIfRequired() {
+        if noOfFramesToSkipAfterRendering > 0 {
+            if skippedFrames == noOfFramesToSkipAfterRendering {
+                skippedFrames = 0
+            } else {
+                skippedFrames += 1
+            }
+        } else if skippedFrames > 0 {
+            skippedFrames = 0
+        }
+    }
+    
+    /// A method used to prepare the view for a new track rendering.
+    private func prepareForTrackRendering(_ oldValue: RTCVideoTrack?) {
+        stopFrameStreaming(for: oldValue)
+        noOfFramesToSkipAfterRendering = 0
+        skippedFrames = 0
+        requiresResize = false
+        startFrameStreaming(for: track, on: window)
+    }
+}

package/ios/PictureInPicture/StreamPixelBufferPool.swift

@@ -0,0 +1,118 @@
+//
+// Copyright © 2024 Stream.io Inc. All rights reserved.
+//
+
+import CoreVideo
+import Foundation
+
+/// A class managing a pool of pixel buffers to optimize memory usage during video processing.
+///
+/// A CVPixelBufferPool is instrumental in optimizing video processing and real-time image manipulation by
+/// enhancing memory efficiency and performance. It achieves this by reusing pixel buffers, which curtails
+/// the frequent allocation and deallocation of memory, a common bottleneck in real-time processing.
+///
+/// This reuse not only boosts performance but also aids in effective resource management, maintaining a
+/// controlled memory footprint and preventing memory pressure. The pool ensures that all buffers share
+/// consistent attributes like size and format, crucial for operations expecting uniform input.
+///
+/// Additionally, it reduces memory fragmentation, contributing to overall system stability and efficiency.
+/// Essentially, CVPixelBufferPool streamlines handling high volumes of image or video frames, ensuring
+/// optimized, consistent, and fast processing.
+final class StreamPixelBufferPool {
+
+    /// Errors that can be encountered while managing the pixel buffer pool.
+    enum Error: LocalizedError {
+
+        /// Error indicating the pixel buffer pool cannot allocate more buffers without exceeding its limit.
+        case returnWouldExceedAllocationThreshold
+
+        /// Error indicating an unknown issue occurred, including the size of the buffer attempted.
+        case unknown(CGSize)
+
+        /// Error indicating that the bufferPool failed to initialise.
+        case unavailableBufferPool
+
+        /// A human-readable description of the error.
+        var errorDescription: String? {
+            switch self {
+            case .returnWouldExceedAllocationThreshold:
+                return "BufferPool is out of buffers, dropping frame"
+            case let .unknown(bufferSize):
+                return "An unknown error occurred while trying to dequeue a pixelBuffer for size: \(bufferSize)"
+            case .unavailableBufferPool:
+                return "BufferPool is unavailable."
+            }
+        }
+    }
+
+    /// The maximum number of pixel buffers that the pool can allocate.
+    let maxNoOfBuffers: Int
+
+    /// The size of each pixel buffer in the pool.
+    let bufferSize: CGSize
+
+    /// The pixel format type of each pixel buffer in the pool.
+    let pixelFormat: OSType
+
+    /// The underlying CVPixelBufferPool managed by this class.
+    private var pool: CVPixelBufferPool?
+
+    /// Initializes a new pixel buffer pool with specified characteristics.
+    ///
+    /// - Parameters:
+    ///   - bufferSize: The size for each buffer in the pool.
+    ///   - pixelFormat: The pixel format type, defaults to 32-bit BGRA.
+    ///   - maxNoOfBuffers: The maximum number of buffers the pool can allocate, defaults to 5.
+    init(
+        bufferSize: CGSize,
+        pixelFormat: OSType = kCVPixelFormatType_32BGRA,
+        maxNoOfBuffers: Int = 5
+    ) {
+        self.bufferSize = bufferSize
+        self.pixelFormat = pixelFormat
+        self.maxNoOfBuffers = maxNoOfBuffers
+
+        var cvPool: CVPixelBufferPool?
+        let poolAttributes: [String: Any] = [
+            kCVPixelBufferPoolMinimumBufferCountKey as String: maxNoOfBuffers
+        ]
+        let pixelBufferAttributes: [String: Any] = [
+            kCVPixelBufferPixelFormatTypeKey as String: Int(pixelFormat),
+            kCVPixelBufferWidthKey as String: Int(bufferSize.width),
+            kCVPixelBufferHeightKey as String: Int(bufferSize.height),
+            kCVPixelBufferIOSurfacePropertiesKey as String: [:]
+        ]
+        CVPixelBufferPoolCreate(
+            nil,
+            poolAttributes as CFDictionary,
+            pixelBufferAttributes as CFDictionary,
+            &cvPool
+        )
+        pool = cvPool
+    }
+
+    /// Dequeues a pixel buffer from the pool or throws an error if none are available.
+    ///
+    /// - Returns: A pixel buffer from the pool.
+    /// - Throws: An `Error` if it cannot dequeue a pixel buffer.
+    func dequeuePixelBuffer() throws -> CVPixelBuffer {
+        guard let pool = self.pool else {
+            throw Error.unavailableBufferPool
+        }
+
+        var pixelBuffer: CVPixelBuffer?
+        let error = CVPixelBufferPoolCreatePixelBuffer(
+            nil,
+            pool,
+            &pixelBuffer
+        )
+
+        if error == kCVReturnWouldExceedAllocationThreshold {
+            throw Error.returnWouldExceedAllocationThreshold
+        } else if let pixelBuffer {
+            return pixelBuffer
+        } else {
+            throw Error.unknown(bufferSize)
+        }
+    }
+}

package/ios/PictureInPicture/StreamPixelBufferRepository.swift

@@ -0,0 +1,98 @@
+//
+// Copyright © 2024 Stream.io Inc. All rights reserved.
+//
+
+import CoreVideo
+import Foundation
+
+/// A repository class that manages multiple pools of pixel buffers to efficiently handle various sizes and formats.
+final class StreamPixelBufferRepository {
+
+    /// A private struct to use as a key in the dictionary that maps buffer characteristics to their respective pools.
+    private struct Key: Hashable {
+        /// The width of the pixel buffers in the pool.
+        var width: Int
+        /// The height of the pixel buffers in the pool.
+        var height: Int
+        /// The pixel format type of the pixel buffers in the pool.
+        var pixelFormat: OSType
+
+        /// Initializes a new key with a given size and pixel format.
+        /// - Parameters:
+        ///   - size: The size of the pixel buffers.
+        ///   - pixelFormat: The pixel format type.
+        init(_ size: CGSize, pixelFormat: OSType) {
+            width = Int(size.width)
+            height = Int(size.height)
+            self.pixelFormat = pixelFormat
+        }
+    }
+
+    /// A dictionary that maps unique keys to specific `StreamPixelBufferPool` instances.
+    private var pools: [Key: StreamPixelBufferPool] = [:]
+
+    /// A dispatch queue used for synchronizing access to the pixel buffer pools.
+    private let queue = UnfairQueue()
+
+    /// Attempts to dequeue a pixel buffer from the appropriate pool or creates a new pool if necessary.
+    ///
+    /// This method looks up the pool corresponding to the specified size and pixel format.
+    /// If a pool doesn't exist yet, it creates a new one and then tries to dequeue a pixel buffer from it.
+    ///
+    /// - Parameters:
+    ///   - size: The size of the pixel buffer required.
+    ///   - pixelFormat: The pixel format of the buffer, defaults to 32-bit BGRA.
+    /// - Returns: A pixel buffer that matches the requested size and format.
+    /// - Throws: An error if a pixel buffer cannot be dequeued or created.
+    func dequeuePixelBuffer(
+        of size: CGSize,
+        pixelFormat: OSType = kCVPixelFormatType_32BGRA
+    ) throws -> CVPixelBuffer {
+        let key = Key(size, pixelFormat: pixelFormat)
+        return try queue.sync {
+            if let targetPool = pools[key] {
+                return try targetPool.dequeuePixelBuffer()
+            } else {
+                let targetPool = StreamPixelBufferPool(
+                    bufferSize: size,
+                    pixelFormat: pixelFormat
+                )
+                pools[key] = targetPool
+                return try targetPool.dequeuePixelBuffer()
+            }
+        }
+    }
+}
+
+final class UnfairQueue {
+
+    /// The unfair lock variable, managed as an unsafe mutable pointer to `os_unfair_lock`.
+    private let lock: os_unfair_lock_t
+
+    /// Initializes a new instance of `UnfairQueue`.
+    ///
+    /// It allocates memory for an `os_unfair_lock` and initializes it.
+    init() {
+        lock = UnsafeMutablePointer<os_unfair_lock>.allocate(capacity: 1)
+        lock.initialize(to: os_unfair_lock())
+    }
+
+    /// Deinitializes the instance, deallocating the unfair lock.
+    deinit {
+        lock.deallocate()
+    }
+
+    /// Executes a block of code, ensuring mutual exclusion via an unfair lock.
+    ///
+    /// The method locks before the block executes and unlocks after the block completes.
+    /// It's designed to be exception-safe, unlocking even if an error is thrown within the block.
+    ///
+    /// - Parameter block: The block of code to execute safely under the lock.
+    /// - Returns: The value returned by the block, if any.
+    /// - Throws: Rethrows any errors that are thrown by the block.
+    func sync<T>(_ block: () throws -> T) rethrows -> T {
+        os_unfair_lock_lock(lock)
+        defer { os_unfair_lock_unlock(lock) }
+        return try block()
+    }
+}