@stream-io/video-react-native-sdk 1.37.0 → 1.37.1-beta.0

package/ios/TracksRecorder/RecorderAudioRenderTap.swift

@@ -0,0 +1,154 @@
+//
+// Copyright © 2026 Stream.io Inc. All rights reserved.
+//
+
+import AVFoundation
+import CoreMedia
+import Foundation
+import WebRTC
+
+/// Render-side audio tap used by `TracksRecorderManager`. Implements
+/// `RTCAudioCustomProcessingDelegate` and is installed on
+/// `RTCDefaultAudioProcessingModule.renderPreProcessingDelegate` for the
+/// duration of a recording.
+///
+/// The render path WebRTC is using:
+/// ```
+/// SFU → decoder → audio mixer → renderPreProcessingDelegate →
+///   render-side processing → speaker
+/// ```
+///
+/// What the delegate sees per call: an `RTCAudioBuffer` that holds the
+/// **post-mix** decoded audio about to be played to the speaker. In a
+/// self-sub-only call, that's exactly the SFU echo of the local mic. In a
+/// call with multiple remote participants the buffer contains the
+/// post-mix output (everyone blended together).
+///
+/// **Important:** `RTCAudioBuffer` exposes `rawBuffer(forChannel:)` as
+/// `UnsafeMutablePointer<Float>` in **FloatS16** format — i.e. Float32 values
+/// in the Int16 range -32768…32767. Cast/clamp to `Int16` for the PCM
+/// destination buffer (no normalisation needed).
+///
+/// **Threading:** all three protocol methods run on a WebRTC audio
+/// processing thread. The buffer handler closure is invoked from there; the
+/// caller is responsible for hopping queues if needed.
+///
+/// **Lifetime:** `RTCDefaultAudioProcessingModule.renderPreProcessingDelegate`
+/// is `weak`, so the manager must keep this instance alive for the duration
+/// of recording.
+@objc public final class RecorderAudioRenderTap: NSObject, RTCAudioCustomProcessingDelegate {
+
+    typealias BufferHandler = (AVAudioPCMBuffer) -> Void
+
+    private let bufferHandler: BufferHandler
+
+    /// When `true`, the WebRTC `RTCAudioBuffer` is zero-filled in place
+    /// *after* the samples have been copied into the recording PCM buffer.
+    /// The recording keeps the original audio; everything downstream of
+    /// this delegate (render-side APM → audio device module → speaker)
+    /// sees silence. This yields "audio in the file, silence at the
+    /// speaker" without disrupting the recording — `track.setVolume(0)`
+    /// / `track.isEnabled = false` mutes apply *before* this tap and
+    /// would silence the recording too.
+    ///
+    /// Side effect to be aware of: this mutes the entire post-mix
+    /// playback, not just one track. In a self-sub-only call post-mix ==
+    /// loopback, so it's effectively a per-track mute. With other remote
+    /// participants in the call they would be muted at the speaker too
+    /// while recording is active.
+    private let muteOriginal: Bool
+
+    private var processingSampleRate: Double = 0
+    private var processingChannels: Int = 0
+    private var avFormat: AVAudioFormat?
+
+    /// Atomic-style call counter — exposes whether the APM is invoking
+    /// `audioProcessingProcess(audioBuffer:)` at all.
+    private let counterLock = NSLock()
+    private var _callCount: Int = 0
+    @objc public var callCount: Int {
+        counterLock.lock()
+        defer { counterLock.unlock() }
+        return _callCount
+    }
+
+    init(muteOriginal: Bool, bufferHandler: @escaping BufferHandler) {
+        self.muteOriginal = muteOriginal
+        self.bufferHandler = bufferHandler
+        super.init()
+    }
+
+    // MARK: - RTCAudioCustomProcessingDelegate
+
+    public func audioProcessingInitialize(sampleRate: Int, channels: Int) {
+        processingSampleRate = Double(sampleRate)
+        processingChannels = channels
+        avFormat = AVAudioFormat(
+            commonFormat: .pcmFormatInt16,
+            sampleRate: processingSampleRate,
+            channels: AVAudioChannelCount(channels),
+            interleaved: false
+        )
+        NSLog("[TracksRecorder] RenderTap initialize sampleRate=%d channels=%d",
+              sampleRate, channels)
+    }
+
+    public func audioProcessingProcess(audioBuffer: RTCAudioBuffer) {
+        counterLock.lock()
+        _callCount += 1
+        let count = _callCount
+        counterLock.unlock()
+
+        if count == 1 {
+            NSLog("[TracksRecorder] RenderTap FIRST call frames=%d channels=%d",
+                  audioBuffer.frames, audioBuffer.channels)
+        } else if count % 100 == 0 {
+            NSLog("[TracksRecorder] RenderTap call #%d", count)
+        }
+
+        guard let format = avFormat else { return }
+        let frames = Int(audioBuffer.frames)
+        let channels = Int(audioBuffer.channels)
+        guard frames > 0, channels > 0 else { return }
+
+        guard let pcm = AVAudioPCMBuffer(
+            pcmFormat: format,
+            frameCapacity: AVAudioFrameCount(frames)
+        ) else {
+            return
+        }
+        pcm.frameLength = AVAudioFrameCount(frames)
+        guard let dst = pcm.int16ChannelData else { return }
+
+        // Copy each channel: FloatS16 (Float32 in Int16 range) → Int16.
+        // No normalisation needed — values already span the Int16 range.
+        for ch in 0..<channels {
+            let src = audioBuffer.rawBuffer(forChannel: ch)
+            let dstChannel = dst[ch]
+            for i in 0..<frames {
+                let v = src[i]
+                if v >= 32767 {
+                    dstChannel[i] = Int16.max
+                } else if v <= -32768 {
+                    dstChannel[i] = Int16.min
+                } else {
+                    dstChannel[i] = Int16(v)
+                }
+
+                // If `muteOriginal` is on, zero the source buffer in the same pass
+                // so the data continuing downstream to the speaker is silence.
+                if muteOriginal {
+                    src[i] = 0
+                }
+            }
+        }
+
+        bufferHandler(pcm)
+    }
+
+    public func audioProcessingRelease() {
+        avFormat = nil
+        // Deliberately preserve `_callCount` — useful in end-of-recording
+        // diagnostics even after release.
+    }
+}

package/ios/TracksRecorder/RecorderVideoSink.swift

@@ -0,0 +1,137 @@
+//
+// Copyright © 2026 Stream.io Inc. All rights reserved.
+//
+
+import CoreMedia
+import CoreVideo
+import Foundation
+import WebRTC
+
+/// Per-track video sink used by `TracksRecorderManager`. Implements
+/// `RTCVideoRenderer` so it can be attached directly to an `RTCVideoTrack`.
+///
+/// Each delivered `RTCVideoFrame` is normalised to a CVPixelBuffer in the
+/// hardware H.264 encoder's native format — **NV12**
+/// (`kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange`).
+/// `RTCCVPixelBuffer` sources (camera passthrough) are forwarded with
+/// the underlying pixel buffer unchanged. `RTCI420Buffer` and other YUV
+/// sources are converted into a fresh IOSurface-backed NV12 buffer via
+/// a plane reorder (no color-space conversion required).
+///
+/// **Why NV12 and not BGRA?** AVAssetWriter's hardware encoder accepts
+/// both, but BGRA requires an internal colour-space conversion that
+/// fails with VideoToolbox `-16364` on certain stride/alignment
+/// combinations a few frames in. NV12 is the encoder's native input;
+/// passing it directly bypasses the failure entirely.
+///
+/// Threading: `renderFrame` runs on a WebRTC frame-delivery thread; the
+/// callback must be safe to invoke from there. The manager serialises
+/// further access on its own queue.
+@objc final class RecorderVideoSink: NSObject, RTCVideoRenderer {
+
+    typealias FrameHandler = (_ pixelBuffer: CVPixelBuffer, _ width: Int32, _ height: Int32, _ timestampNs: Int64) -> Void
+
+    private let frameHandler: FrameHandler
+
+    init(frameHandler: @escaping FrameHandler) {
+        self.frameHandler = frameHandler
+        super.init()
+    }
+
+    // MARK: - RTCVideoRenderer
+
+    func setSize(_ size: CGSize) {
+        // No-op: pixel buffer dimensions are derived from each incoming frame.
+    }
+
+    func renderFrame(_ frame: RTCVideoFrame?) {
+        guard let frame = frame, frame.width > 0, frame.height > 0 else { return }
+
+        let pixelBuffer: CVPixelBuffer?
+        if let cvBuffer = frame.buffer as? RTCCVPixelBuffer {
+            // Camera passthrough — already a CVPixelBuffer (typically NV12 on iOS).
+            pixelBuffer = cvBuffer.pixelBuffer
+        } else if let i420 = frame.buffer as? RTCI420Buffer {
+            pixelBuffer = Self.makeNV12PixelBuffer(fromI420: i420)
+        } else {
+            // Other YUV variants — normalise via toI420() first.
+            let i420 = frame.buffer.toI420()
+            if let concrete = i420 as? RTCI420Buffer {
+                pixelBuffer = Self.makeNV12PixelBuffer(fromI420: concrete)
+            } else {
+                pixelBuffer = nil
+            }
+        }
+
+        guard let outputBuffer = pixelBuffer else { return }
+        frameHandler(outputBuffer, frame.width, frame.height, frame.timeStampNs)
+    }
+
+    // MARK: - I420 → NV12
+
+    /// Allocates a fresh IOSurface-backed NV12 `CVPixelBuffer` and copies
+    /// the I420 source's planes into it (Y as-is, U+V interleaved into the
+    /// UV plane). No colour-space conversion — this is purely a plane
+    /// reorder, so the operation is both fast and bit-exact.
+    private static func makeNV12PixelBuffer(fromI420 i420: RTCI420Buffer) -> CVPixelBuffer? {
+        let width = Int(i420.width)
+        let height = Int(i420.height)
+        let chromaWidth = Int(i420.chromaWidth)
+        let chromaHeight = Int(i420.chromaHeight)
+
+        var pixelBuffer: CVPixelBuffer?
+        let attrs: [String: Any] = [
+            kCVPixelBufferIOSurfacePropertiesKey as String: [:] as [String: Any],
+        ]
+        let status = CVPixelBufferCreate(
+            kCFAllocatorDefault,
+            width,
+            height,
+            kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
+            attrs as CFDictionary,
+            &pixelBuffer
+        )
+        guard status == kCVReturnSuccess, let buffer = pixelBuffer else { return nil }
+
+        CVPixelBufferLockBaseAddress(buffer, [])
+        defer { CVPixelBufferUnlockBaseAddress(buffer, []) }
+
+        // Plane 0: Y — direct copy.
+        guard let yDest = CVPixelBufferGetBaseAddressOfPlane(buffer, 0) else { return nil }
+        let yDestStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 0)
+        let ySrcStride = Int(i420.strideY)
+        let ySrc = UnsafeRawPointer(i420.dataY)
+        if ySrcStride == yDestStride {
+            memcpy(yDest, ySrc, ySrcStride * height)
+        } else {
+            let copyBytes = min(ySrcStride, yDestStride)
+            for row in 0..<height {
+                memcpy(
+                    yDest.advanced(by: row * yDestStride),
+                    ySrc.advanced(by: row * ySrcStride),
+                    copyBytes
+                )
+            }
+        }
+
+        // Plane 1: UV — interleave I420's U and V planes.
+        guard let uvDestRaw = CVPixelBufferGetBaseAddressOfPlane(buffer, 1) else { return nil }
+        let uvDestStride = CVPixelBufferGetBytesPerRowOfPlane(buffer, 1)
+        let uvDest = uvDestRaw.assumingMemoryBound(to: UInt8.self)
+        let uSrcStride = Int(i420.strideU)
+        let vSrcStride = Int(i420.strideV)
+        let uSrc = i420.dataU
+        let vSrc = i420.dataV
+        for row in 0..<chromaHeight {
+            let uRow = uSrc.advanced(by: row * uSrcStride)
+            let vRow = vSrc.advanced(by: row * vSrcStride)
+            let uvRow = uvDest.advanced(by: row * uvDestStride)
+            for col in 0..<chromaWidth {
+                uvRow[col * 2] = uRow[col]
+                uvRow[col * 2 + 1] = vRow[col]
+            }
+        }
+
+        return buffer
+    }
+}

package/ios/TracksRecorder/TracksRecorderManager.swift

@@ -0,0 +1,327 @@
+//
+// Copyright © 2026 Stream.io Inc. All rights reserved.
+//
+
+import AVFoundation
+import CoreMedia
+import Foundation
+import WebRTC
+
+/// Orchestrator for the React Native track recorder. Owns the
+/// `AVAssetWriter`, the recording lifecycle, the writer-start gate, and
+/// the terminal-completion barrier. Delegates the encoder + sink + drain
+/// work to `VideoPipeline` and `AudioPipeline` respectively (composed via
+/// `PipelineHost`).
+///
+/// The `startRecording` completion is the *lifecycle* signal — it fires once at the terminal moment of the
+/// recording (auto-stop timer expired, manual stop, or fatal error),
+/// carrying the resulting file URL or an error. The `stopRecording`
+/// completion is just a synchronization point — `void` — so callers can
+/// `await stopTrackRecording(); await getStreamRecordings()` without
+/// racing the disk flush.
+@objc public final class TracksRecorderManager: NSObject, PipelineHost {
+
+    @objc public static let shared = TracksRecorderManager()
+
+    // MARK: - Configuration
+
+    @objc public var recordingsDirectory: URL {
+        let tmp = NSTemporaryDirectory()
+        let url = URL(fileURLWithPath: tmp).appendingPathComponent("StreamRecordings", isDirectory: true)
+        try? FileManager.default.createDirectory(
+            at: url,
+            withIntermediateDirectories: true,
+            attributes: nil
+        )
+        return url
+    }
+
+    // MARK: - PipelineHost
+
+    let queue = DispatchQueue(label: "io.stream.video.tracks-recorder")
+
+    private(set) var assetWriter: AVAssetWriter?
+    private(set) var isRecording = false
+
+    // MARK: - State
+
+    private var videoPipeline: VideoPipeline?
+    private var audioPipeline: AudioPipeline?
+
+    private var outputURL: URL?
+    private var recordingCompletion: ((URL?, NSError?) -> Void)?
+    private var isCompleted = false
+
+    private var pendingPipelines = 0
+    private var recordingStartHostTimeNs: UInt64?
+    private var autoStopTimer: DispatchSourceTimer?
+
+    // MARK: - Public API
+
+    @objc public func startRecording(
+        videoTrackId: String?,
+        maxDurationMs: Int,
+        targetWidth: Int,
+        targetHeight: Int,
+        webRTCModule: WebRTCModule,
+        completion: @escaping (URL?, NSError?) -> Void
+    ) {
+        queue.async { [weak self] in
+            guard let self = self else { return }
+
+            if self.isRecording {
+                completion(nil, makeRecorderError("recording_in_progress", code: 1))
+                return
+            }
+
+            let videoTrack = videoTrackId
+                .flatMap { webRTCModule.track(forId: $0) } as? RTCVideoTrack
+
+            // Resolve the APM for the render-side audio tap before opening
+            // any files — so a missing/incompatible APM fails fast. Audio
+            // is captured through the APM render-pre delegate (post-mix
+            // decoded audio); no per-track lookup is needed.
+            let apmId = WebRTCModuleOptions.sharedInstance().audioProcessingModule
+            guard let apm = apmId as? RTCDefaultAudioProcessingModule else {
+                completion(nil, makeRecorderError("audio_processing_module_unavailable", code: 5))
+                return
+            }
+            if apm.renderPreProcessingDelegate != nil {
+                completion(nil, makeRecorderError("recording_blocked_by_audio_render_tap", code: 6))
+                return
+            }
+
+            let dir = self.recordingsDirectory
+            let timestamp = Int(Date().timeIntervalSince1970 * 1000)
+            let outputURL = dir.appendingPathComponent("recording_\(timestamp).mp4")
+
+            let writer: AVAssetWriter
+            do {
+                writer = try AVAssetWriter(url: outputURL, fileType: .mp4)
+            } catch {
+                completion(nil, error as NSError)
+                return
+            }
+
+            self.assetWriter = writer
+            self.outputURL = outputURL
+            self.recordingCompletion = completion
+            self.isCompleted = false
+            self.recordingStartHostTimeNs = nil
+            self.pendingPipelines = 0
+            self.isRecording = true
+
+            if let videoTrack = videoTrack {
+                let pipeline = VideoPipeline(
+                    host: self,
+                    videoTrack: videoTrack,
+                    targetWidth: Int32(targetWidth),
+                    targetHeight: Int32(targetHeight)
+                )
+                self.videoPipeline = pipeline
+                self.pendingPipelines += 1
+                pipeline.start()
+            }
+
+            let audio = AudioPipeline(host: self, apm: apm)
+            self.audioPipeline = audio
+            self.pendingPipelines += 1
+            audio.start()
+
+            if maxDurationMs > 0 {
+                let timer = DispatchSource.makeTimerSource(queue: self.queue)
+                timer.schedule(deadline: .now() + .milliseconds(maxDurationMs))
+                timer.setEventHandler { [weak self] in
+                    self?.stopRecording { }
+                }
+                timer.resume()
+                self.autoStopTimer = timer
+            }
+
+            NSLog("[TracksRecorder] recording started video=%@ audio=YES → %@",
+                  videoTrack != nil ? "YES" : "NO",
+                  outputURL.absoluteString)
+        }
+    }
+
+    @objc public func stopRecording(completion: @escaping () -> Void) {
+        queue.async { [weak self] in
+            guard let self = self else {
+                DispatchQueue.main.async { completion() }
+                return
+            }
+
+            if !self.isRecording {
+                DispatchQueue.main.async { completion() }
+                return
+            }
+
+            self.autoStopTimer?.cancel()
+            self.autoStopTimer = nil
+
+            // Detach sinks/taps so no more frames arrive on the recorder path.
+            self.videoPipeline?.detachSink()
+            self.audioPipeline?.detachSink()
+
+            let video = self.videoPipeline
+            let audio = self.audioPipeline
+            let writer = self.assetWriter
+            let outputURL = self.outputURL
+
+            self.isRecording = false
+
+            // If the writer never started (no encoded sample ever made it
+            // out), there's nothing to finalise.
+            guard let assetWriter = writer, assetWriter.status == .writing else {
+                NSLog("[TracksRecorder] stopRecording: writer not in .writing (status=%ld) — no file produced",
+                      Int(writer?.status.rawValue ?? -1))
+                video?.logSummary()
+                audio?.logSummary()
+                self.fireTerminalCompletion(url: nil, error: writer?.error as NSError?)
+                self.cleanupAfterStop()
+                DispatchQueue.main.async { completion() }
+                return
+            }
+
+            video?.markInputAsFinished()
+            audio?.markInputAsFinished()
+
+            assetWriter.finishWriting { [weak self] in
+                guard let self = self else {
+                    DispatchQueue.main.async { completion() }
+                    return
+                }
+                self.queue.async {
+                    let resolved: URL? = (assetWriter.status == .completed) ? outputURL : nil
+                    let writerError: NSError? = (assetWriter.status == .failed)
+                        ? (assetWriter.error as NSError?)
+                        : nil
+                    video?.logSummary()
+                    audio?.logSummary()
+                    NSLog("[TracksRecorder] recording finalised → %@",
+                          resolved?.absoluteString ?? "(no file produced)")
+                    self.fireTerminalCompletion(url: resolved, error: writerError)
+                    self.cleanupAfterStop()
+                    DispatchQueue.main.async { completion() }
+                }
+            }
+        }
+    }
+
+    @objc public func clearRecordingsDirectory(completion: @escaping (NSError?) -> Void) {
+        queue.async { [weak self] in
+            guard let self = self else { return }
+            let fm = FileManager.default
+            do {
+                let contents = try fm.contentsOfDirectory(
+                    at: self.recordingsDirectory,
+                    includingPropertiesForKeys: nil,
+                    options: []
+                )
+                for url in contents {
+                    try? fm.removeItem(at: url)
+                }
+                DispatchQueue.main.async { completion(nil) }
+            } catch {
+                DispatchQueue.main.async { completion(error as NSError) }
+            }
+        }
+    }
+
+    @objc public func listRecordings() -> [URL] {
+        let fm = FileManager.default
+        guard let contents = try? fm.contentsOfDirectory(
+            at: recordingsDirectory,
+            includingPropertiesForKeys: [.contentModificationDateKey],
+            options: [.skipsHiddenFiles]
+        ) else {
+            return []
+        }
+        return contents.sorted { lhs, rhs in
+            let lDate = (try? lhs.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) ?? .distantPast
+            let rDate = (try? rhs.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate) ?? .distantPast
+            return lDate > rDate
+        }
+    }
+
+    // MARK: - PipelineHost
+
+    func seedOriginNs(_ timestampNs: UInt64) -> UInt64 {
+        if let existing = recordingStartHostTimeNs { return existing }
+        recordingStartHostTimeNs = timestampNs
+        return timestampNs
+    }
+
+    func onTrackAdded() {
+        pendingPipelines = max(0, pendingPipelines - 1)
+        maybeStartSession()
+    }
+
+    func onFatalError(_ error: NSError) {
+        fireTerminalCompletion(url: nil, error: error)
+        cleanupAfterFailure()
+    }
+
+    // MARK: - Internal helpers
+
+    /// Starts the writer once every active pipeline has reported its input
+    /// added. Calling `startWriting()` / `startSession(.zero)` before all
+    /// inputs are present would leave un-added tracks orphaned; the gate
+    /// is load-bearing.
+    private func maybeStartSession() {
+        guard let writer = assetWriter else { return }
+        if pendingPipelines > 0 { return }
+        if writer.status != .unknown { return }
+
+        guard writer.startWriting() else {
+            let err = (writer.error as NSError?) ?? makeRecorderError("start_failed", code: 3)
+            fireTerminalCompletion(url: nil, error: err)
+            cleanupAfterFailure()
+            return
+        }
+        // The session origin is `.zero`; PTS values are computed relative
+        // to `recordingStartHostTimeNs` before being passed to
+        // `VTCompressionSessionEncodeFrame`, so the first encoded sample
+        // already carries pts=0.
+        writer.startSession(atSourceTime: .zero)
+        NSLog("[TracksRecorder] writer.startWriting() + startSession(.zero)")
+    }
+
+    private func fireTerminalCompletion(url: URL?, error: NSError?) {
+        guard !isCompleted else { return }
+        isCompleted = true
+        let cb = recordingCompletion
+        recordingCompletion = nil
+        DispatchQueue.main.async { cb?(url, error) }
+    }
+
+    private func cleanupAfterFailure() {
+        videoPipeline?.detachSink()
+        audioPipeline?.detachSink()
+        autoStopTimer?.cancel()
+        autoStopTimer = nil
+        resetTransientState()
+    }
+
+    private func cleanupAfterStop() {
+        resetTransientState()
+    }
+
+    /// Resets every transient field to its initial value. Single source of
+    /// truth for "the manager is between recordings". Does NOT release
+    /// native resources — the caller must have already torn down the
+    /// pipelines and the writer.
+    private func resetTransientState() {
+        videoPipeline = nil
+        audioPipeline = nil
+        assetWriter = nil
+        outputURL = nil
+        recordingCompletion = nil
+        isCompleted = false
+        isRecording = false
+        pendingPipelines = 0
+        recordingStartHostTimeNs = nil
+        autoStopTimer?.cancel()
+        autoStopTimer = nil
+    }
+}