@livekit/react-native 2.5.0 → 2.6.0

package/ios/LKAudioProcessingManager.m

@@ -0,0 +1,63 @@
+#import "LKAudioProcessingManager.h"
+#import "LKAudioProcessingAdapter.h"
+
+@implementation LKAudioProcessingManager
+
++ (instancetype)sharedInstance {
+    static dispatch_once_t onceToken;
+    static LKAudioProcessingManager* sharedInstance = nil;
+    dispatch_once(&onceToken, ^{
+        sharedInstance = [[self alloc] init];
+    });
+    return sharedInstance;
+}
+
+- (instancetype)init {
+    if (self = [super init]) {
+        _audioProcessingModule = [[RTCDefaultAudioProcessingModule alloc] init];
+        _capturePostProcessingAdapter = [[LKAudioProcessingAdapter alloc] init];
+        _renderPreProcessingAdapter = [[LKAudioProcessingAdapter alloc] init];
+        _audioProcessingModule.capturePostProcessingDelegate = _capturePostProcessingAdapter;
+        _audioProcessingModule.renderPreProcessingDelegate = _renderPreProcessingAdapter;
+    }
+    return self;
+}
+
+- (void)addLocalAudioRenderer:(nonnull id<RTCAudioRenderer>)renderer {
+  [_capturePostProcessingAdapter addAudioRenderer:renderer];
+}
+
+- (void)removeLocalAudioRenderer:(nonnull id<RTCAudioRenderer>)renderer {
+  [_capturePostProcessingAdapter removeAudioRenderer:renderer];
+}
+
+- (void)addRemoteAudioRenderer:(nonnull id<RTCAudioRenderer>)renderer {
+  [_renderPreProcessingAdapter addAudioRenderer:renderer];
+}
+
+- (void)removeRemoteAudioRenderer:(nonnull id<RTCAudioRenderer>)renderer {
+  [_renderPreProcessingAdapter removeAudioRenderer:renderer];
+}
+
+- (void)addCapturePostProcessor:(nonnull id<LKExternalAudioProcessingDelegate>)processor {
+    [_capturePostProcessingAdapter addProcessing:processor];
+}
+
+- (void)removeCapturePostProcessor:(nonnull id<LKExternalAudioProcessingDelegate>)processor {
+    [_capturePostProcessingAdapter removeProcessing:processor];
+}
+
+- (void)addRenderPreProcessor:(nonnull id<LKExternalAudioProcessingDelegate>)processor {
+    [_renderPreProcessingAdapter addProcessing:processor];
+}
+
+- (void)removeRenderPreProcessor:(nonnull id<LKExternalAudioProcessingDelegate>)processor {
+    [_renderPreProcessingAdapter removeProcessing:processor];
+}
+
+- (void)clearProcessors {
+    // TODO
+}
+
+
+@end

package/ios/LivekitReactNative-Bridging-Header.h

@@ -1,2 +1,4 @@
 #import <React/RCTBridgeModule.h>
 #import <React/RCTViewManager.h>
+#import "WebRTCModule.h"
+#import "WebRTCModule+RTCMediaStream.h"

package/ios/LivekitReactNative.h

@@ -3,12 +3,17 @@
 //  LivekitReactNative
 //
 //  Created by David Liu on 9/4/22.
-//  Copyright © 2022 LiveKit. All rights reserved.
+//  Copyright © 2022-2025 LiveKit. All rights reserved.
 //
 #import <React/RCTBridgeModule.h>
+#import <WebRTC/WebRTC.h>
+#import <React/RCTEventEmitter.h>
 
-@interface LivekitReactNative : NSObject <RCTBridgeModule>
-
+@class AudioRendererManager;
+@interface LivekitReactNative : RCTEventEmitter <RCTBridgeModule>
+@property(nonatomic, strong) AudioRendererManager* _Nonnull audioRendererManager;
 +(void)setup;
-
 @end
+
+extern NSString * _Nonnull const kEventVolumeProcessed;
+extern NSString * _Nonnull const kEventMultibandProcessed;

package/ios/LivekitReactNative.m

@@ -1,15 +1,23 @@
 #import "AudioUtils.h"
 #import "LivekitReactNative.h"
+#import "LKAudioProcessingManager.h"
 #import "WebRTCModule.h"
 #import "WebRTCModuleOptions.h"
 #import <WebRTC/RTCAudioSession.h>
 #import <WebRTC/RTCAudioSessionConfiguration.h>
 #import <AVFAudio/AVFAudio.h>
 #import <AVKit/AVKit.h>
+#import "livekit_react_native-Swift.h"
+
+NSString *const kEventVolumeProcessed = @"LK_VOLUME_PROCESSED";
+NSString *const kEventMultibandProcessed = @"LK_MULTIBAND_PROCESSED";
 
 @implementation LivekitReactNative
+
+
 RCT_EXPORT_MODULE();
 
+
 -(instancetype)init {
     if(self = [super init]) {
         
@@ -38,6 +46,7 @@ RCT_EXPORT_MODULE();
     RTCVideoEncoderFactorySimulcast *simulcastVideoEncoderFactory = [[RTCVideoEncoderFactorySimulcast alloc] initWithPrimary:videoEncoderFactory fallback:videoEncoderFactory];
     WebRTCModuleOptions *options = [WebRTCModuleOptions sharedInstance];
     options.videoEncoderFactory = simulcastVideoEncoderFactory;
+    options.audioProcessingModule = LKAudioProcessingManager.sharedInstance.audioProcessingModule;
 }
 
 /// Configure default audio config for WebRTC
@@ -123,13 +132,13 @@ RCT_EXPORT_METHOD(selectAudioOutput:(NSString *)deviceId
 RCT_EXPORT_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration){
     RTCAudioSession* session = [RTCAudioSession sharedInstance];
     RTCAudioSessionConfiguration* config = [RTCAudioSessionConfiguration webRTCConfiguration];
-
+    
     NSString* appleAudioCategory = configuration[@"audioCategory"];
     NSArray* appleAudioCategoryOptions = configuration[@"audioCategoryOptions"];
     NSString* appleAudioMode = configuration[@"audioMode"];
     
     [session lockForConfiguration];
-
+    
     NSError* error = nil;
     BOOL categoryChanged = NO;
     if(appleAudioCategoryOptions != nil) {
@@ -151,7 +160,7 @@ RCT_EXPORT_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration){
             }
         }
     }
-
+    
     if(appleAudioCategory != nil) {
         categoryChanged = YES;
         config.category = [AudioUtils audioSessionCategoryFromString:appleAudioCategory];
@@ -164,7 +173,7 @@ RCT_EXPORT_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration){
             error = nil;
         }
     }
-
+    
     if(appleAudioMode != nil) {
         config.mode = [AudioUtils audioSessionModeFromString:appleAudioMode];
         [session setMode:config.mode error:&error];
@@ -173,7 +182,76 @@ RCT_EXPORT_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration){
             error = nil;
         }
     }
-
+    
     [session unlockForConfiguration];
 }
+
+-(AudioRendererManager *)audioRendererManager {
+    if(!_audioRendererManager) {
+        _audioRendererManager = [[AudioRendererManager alloc] initWithBridge:self.bridge];
+    }
+    
+    return _audioRendererManager;
+}
+
+RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(createVolumeProcessor:(nonnull NSNumber *)pcId
+                                       trackId:(nonnull NSString *)trackId) {
+    
+    
+    VolumeAudioRenderer *renderer = [[VolumeAudioRenderer alloc] initWithIntervalMs:40.0 eventEmitter:self];
+    
+    NSString *reactTag = [self.audioRendererManager registerRenderer:renderer];
+    renderer.reactTag = reactTag;
+    [self.audioRendererManager attachWithRenderer:renderer pcId:pcId trackId:trackId];
+    return reactTag;
+}
+
+RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(deleteVolumeProcessor:(nonnull NSString *)reactTag
+                                       pcId:(nonnull NSNumber *)pcId
+                                       trackId:(nonnull NSString *)trackId) {
+    
+    [self.audioRendererManager detachWithRendererByTag:reactTag pcId:pcId trackId:trackId];
+    [self.audioRendererManager unregisterRendererForReactTag:reactTag];
+    
+    return nil;
+}
+
+RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(createMultibandVolumeProcessor:(NSDictionary *)options
+                                       pcId:(nonnull NSNumber *)pcId
+                                       trackId:(nonnull NSString *)trackId) {
+    
+    NSInteger bands = [(NSNumber *)options[@"bands"] integerValue];
+    float minFrequency = [(NSNumber *)options[@"minFrequency"] floatValue];
+    float maxFrequency = [(NSNumber *)options[@"maxFrequency"] floatValue];
+    float intervalMs = [(NSNumber *)options[@"updateInterval"] floatValue];
+    MultibandVolumeAudioRenderer *renderer = [[MultibandVolumeAudioRenderer alloc] initWithBands:bands
+                                                                                    minFrequency:minFrequency
+                                                                                    maxFrequency:maxFrequency
+                                                                                      intervalMs:intervalMs
+                                                                                    eventEmitter:self];
+    
+    NSString *reactTag = [self.audioRendererManager registerRenderer:renderer];
+    renderer.reactTag = reactTag;
+    [self.audioRendererManager attachWithRenderer:renderer pcId:pcId trackId:trackId];
+    return reactTag;
+}
+
+RCT_EXPORT_BLOCKING_SYNCHRONOUS_METHOD(deleteMultibandVolumeProcessor:(nonnull NSString *)reactTag
+                                       pcId:(nonnull NSNumber *)pcId
+                                       trackId:(nonnull NSString *)trackId) {
+    
+    [self.audioRendererManager detachWithRendererByTag:reactTag pcId:pcId trackId:trackId];
+    [self.audioRendererManager unregisterRendererForReactTag:reactTag];
+    
+    return nil;
+}
+
+
+- (NSArray<NSString *> *)supportedEvents {
+    return @[
+        kEventVolumeProcessed,
+        kEventMultibandProcessed,
+    ];
+}
+
 @end

package/ios/Logging.swift

@@ -0,0 +1,4 @@
+public func lklog(_ object: Any, functionName: String = #function, fileName: String = #file, lineNumber: Int = #line) {
+  let className = (fileName as NSString).lastPathComponent
+  print("\(className).\(functionName):\(lineNumber) : \(object)\n")
+}

package/ios/audio/AVAudioPCMBuffer.swift

@@ -0,0 +1,136 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Accelerate
+import AVFoundation
+
+public extension AVAudioPCMBuffer {
+    func resample(toSampleRate targetSampleRate: Double) -> AVAudioPCMBuffer? {
+        let sourceFormat = format
+
+        if sourceFormat.sampleRate == targetSampleRate {
+            // Already targetSampleRate.
+            return self
+        }
+
+        // Define the source format (from the input buffer) and the target format.
+        guard let targetFormat = AVAudioFormat(commonFormat: sourceFormat.commonFormat,
+                                               sampleRate: targetSampleRate,
+                                               channels: sourceFormat.channelCount,
+                                               interleaved: sourceFormat.isInterleaved)
+        else {
+            print("Failed to create target format.")
+            return nil
+        }
+
+        guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else {
+            print("Failed to create audio converter.")
+            return nil
+        }
+
+        let capacity = targetFormat.sampleRate * Double(frameLength) / sourceFormat.sampleRate
+
+        guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount(capacity)) else {
+            print("Failed to create converted buffer.")
+            return nil
+        }
+
+        var isDone = false
+        let inputBlock: AVAudioConverterInputBlock = { _, outStatus in
+            if isDone {
+                outStatus.pointee = .noDataNow
+                return nil
+            }
+            outStatus.pointee = .haveData
+            isDone = true
+            return self
+        }
+
+        var error: NSError?
+        let status = converter.convert(to: convertedBuffer, error: &error, withInputFrom: inputBlock)
+
+        if status == .error {
+            print("Conversion failed: \(error?.localizedDescription ?? "Unknown error")")
+            return nil
+        }
+
+        // Adjust frame length to the actual amount of data written
+        convertedBuffer.frameLength = convertedBuffer.frameCapacity
+
+        return convertedBuffer
+    }
+
+    /// Convert PCM buffer to specified common format.
+    /// Currently supports conversion from Int16 to Float32.
+    func convert(toCommonFormat commonFormat: AVAudioCommonFormat) -> AVAudioPCMBuffer? {
+        // Check if conversion is needed
+        guard format.commonFormat != commonFormat else {
+            return self
+        }
+
+        // Check if the conversion is supported
+        guard format.commonFormat == .pcmFormatInt16, commonFormat == .pcmFormatFloat32 else {
+            print("Unsupported conversion: only Int16 to Float32 is supported")
+            return nil
+        }
+
+        // Create output format
+        guard let outputFormat = AVAudioFormat(commonFormat: commonFormat,
+                                               sampleRate: format.sampleRate,
+                                               channels: format.channelCount,
+                                               interleaved: false)
+        else {
+            print("Failed to create output audio format")
+            return nil
+        }
+
+        // Create output buffer
+        guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat,
+                                                  frameCapacity: frameCapacity)
+        else {
+            print("Failed to create output PCM buffer")
+            return nil
+        }
+
+        outputBuffer.frameLength = frameLength
+
+        let channelCount = Int(format.channelCount)
+        let frameCount = Int(frameLength)
+
+        // Ensure the source buffer has Int16 data
+        guard let int16Data = int16ChannelData else {
+            print("Source buffer doesn't contain Int16 data")
+            return nil
+        }
+
+        // Ensure the output buffer has Float32 data
+        guard let floatData = outputBuffer.floatChannelData else {
+            print("Failed to get float channel data from output buffer")
+            return nil
+        }
+
+        // Convert Int16 to Float32 and normalize to [-1.0, 1.0]
+        let scale = Float(Int16.max)
+        var scalar = 1.0 / scale
+
+        for channel in 0 ..< channelCount {
+            vDSP_vflt16(int16Data[channel], 1, floatData[channel], 1, vDSP_Length(frameCount))
+            vDSP_vsmul(floatData[channel], 1, &scalar, floatData[channel], 1, vDSP_Length(frameCount))
+        }
+
+        return outputBuffer
+    }
+}

package/ios/audio/AudioProcessing.swift

@@ -0,0 +1,163 @@
+/*
+ * Copyright 2025 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Accelerate
+import AVFoundation
+import Foundation
+import WebRTC
+
+public struct AudioLevel {
+    /// Linear Scale RMS Value
+    public let average: Float
+    public let peak: Float
+}
+
+public extension RTCAudioBuffer {
+    /// Convert to AVAudioPCMBuffer Int16 format.
+    @objc
+    func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? {
+        guard let audioFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
+                                              sampleRate: Double(frames * 100),
+                                              channels: AVAudioChannelCount(channels),
+                                              interleaved: false),
+            let pcmBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat,
+                                             frameCapacity: AVAudioFrameCount(frames))
+        else { return nil }
+
+        pcmBuffer.frameLength = AVAudioFrameCount(frames)
+
+        guard let targetBufferPointer = pcmBuffer.int16ChannelData else { return nil }
+
+        for i in 0 ..< channels {
+            let sourceBuffer = rawBuffer(forChannel: i)
+            let targetBuffer = targetBufferPointer[i]
+            // sourceBuffer is in the format of [Int16] but is stored in 32-bit alignment, we need to pack the Int16 data correctly.
+
+            for frame in 0 ..< frames {
+                // Cast and pack the source 32-bit Int16 data into the target 16-bit buffer
+                let clampedValue = max(Float(Int16.min), min(Float(Int16.max), sourceBuffer[frame]))
+                targetBuffer[frame] = Int16(clampedValue)
+            }
+        }
+
+        return pcmBuffer
+    }
+}
+
+public extension AVAudioPCMBuffer {
+    /// Computes Peak and Linear Scale RMS Value (Average) for all channels.
+    func audioLevels() -> [AudioLevel] {
+        var result: [AudioLevel] = []
+        guard let data = floatChannelData else {
+            // Not containing float data
+            return result
+        }
+
+        for i in 0 ..< Int(format.channelCount) {
+            let channelData = data[i]
+            var max: Float = 0.0
+            vDSP_maxv(channelData, stride, &max, vDSP_Length(frameLength))
+            var rms: Float = 0.0
+            vDSP_rmsqv(channelData, stride, &rms, vDSP_Length(frameLength))
+
+            // No conversion to dB, return linear scale values directly
+            result.append(AudioLevel(average: rms, peak: max))
+        }
+
+        return result
+    }
+}
+
+public extension Sequence where Iterator.Element == AudioLevel {
+    /// Combines all elements into a single audio level by computing the average value of all elements.
+    func combine() -> AudioLevel? {
+        var count = 0
+        let totalSums: (averageSum: Float, peakSum: Float) = reduce((averageSum: 0.0, peakSum: 0.0)) { totals, audioLevel in
+            count += 1
+            return (totals.averageSum + audioLevel.average,
+                    totals.peakSum + audioLevel.peak)
+        }
+
+        guard count > 0 else { return nil }
+
+        return AudioLevel(average: totalSums.averageSum / Float(count),
+                          peak: totalSums.peakSum / Float(count))
+    }
+}
+
+public class AudioVisualizeProcessor {
+    static let bufferSize = 1024
+
+    // MARK: - Public
+
+    public let minFrequency: Float
+    public let maxFrequency: Float
+    public let minDB: Float
+    public let maxDB: Float
+    public let bandsCount: Int
+
+    private var bands: [Float]?
+
+    // MARK: - Private
+
+    private let ringBuffer = RingBuffer<Float>(size: AudioVisualizeProcessor.bufferSize)
+    private let processor: FFTProcessor
+
+    public init(minFrequency: Float = 10,
+                maxFrequency: Float = 8000,
+                minDB: Float = -32.0,
+                maxDB: Float = 32.0,
+                bandsCount: Int = 100)
+    {
+        self.minFrequency = minFrequency
+        self.maxFrequency = maxFrequency
+        self.minDB = minDB
+        self.maxDB = maxDB
+        self.bandsCount = bandsCount
+
+        processor = FFTProcessor(bufferSize: Self.bufferSize)
+        bands = [Float](repeating: 0.0, count: bandsCount)
+    }
+
+    public func process(pcmBuffer: AVAudioPCMBuffer) -> [Float]? {
+        guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return nil }
+        guard let floatChannelData = pcmBuffer.floatChannelData else { return nil }
+
+        // Get the float array.
+        let floats = Array(UnsafeBufferPointer(start: floatChannelData[0], count: Int(pcmBuffer.frameLength)))
+        ringBuffer.write(floats)
+
+        // Get full-size buffer if available, otherwise return
+        guard let buffer = ringBuffer.read() else { return nil }
+
+        // Process FFT and compute frequency bands
+        let fftRes = processor.process(buffer: buffer)
+        let bands = fftRes.computeBands(
+            minFrequency: minFrequency,
+            maxFrequency: maxFrequency,
+            bandsCount: bandsCount,
+            sampleRate: Float(pcmBuffer.format.sampleRate)
+        )
+
+        let headroom = maxDB - minDB
+
+        // Normalize magnitudes (already in decibels)
+        return bands.magnitudes.map { magnitude in
+            let adjustedMagnitude = max(0, magnitude + abs(minDB))
+            return min(1.0, adjustedMagnitude / headroom)
+        }
+    }
+}

package/ios/audio/AudioRendererManager.swift

@@ -0,0 +1,72 @@
+import Foundation
+import WebRTC
+
+public class AudioRendererManager: NSObject {
+    private let bridge: RCTBridge
+    public private(set) var renderers: [String: RTCAudioRenderer] = [:]
+    
+    @objc
+    public init(bridge: RCTBridge) {
+        self.bridge = bridge
+    }
+    
+    @objc
+    public func registerRenderer(_ audioRenderer: RTCAudioRenderer) -> String {
+        let reactTag = NSUUID().uuidString
+        self.renderers[reactTag] = audioRenderer
+        return reactTag
+    }
+    
+    @objc
+    public func unregisterRenderer(forReactTag: String) {
+        self.renderers.removeValue(forKey: forReactTag)
+    }
+    
+    @objc
+    public func unregisterRenderer(_ audioRenderer: RTCAudioRenderer) {
+        self.renderers = self.renderers.filter({ $0.value !== audioRenderer })
+    }
+    
+    @objc
+    public func attach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) {
+        let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule
+        guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack
+        else {
+            lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)")
+            return
+        }
+        
+        if (pcId == -1) {
+            LKAudioProcessingManager.sharedInstance().addLocalAudioRenderer(renderer);
+        } else {
+            track.add(renderer)
+        }
+    }
+    
+    @objc
+    public func detach(rendererByTag reactTag:String, pcId: NSNumber, trackId: String){
+        guard let renderer = self.renderers[reactTag]
+        else {
+            lklog("couldn't find renderer: tag: \(reactTag)")
+            return
+        }
+        
+        detach(renderer: renderer, pcId: pcId, trackId: trackId)
+    }
+    
+    @objc
+    public func detach(renderer: RTCAudioRenderer, pcId: NSNumber, trackId: String) {
+        let webrtcModule = self.bridge.module(for: WebRTCModule.self) as! WebRTCModule
+        guard let track = webrtcModule.track(forId: trackId, pcId: pcId) as? RTCAudioTrack
+        else {
+            lklog("couldn't find audio track: pcId: \(pcId), trackId: \(trackId)")
+            return
+        }
+        
+        if (pcId == -1) {
+            LKAudioProcessingManager.sharedInstance().removeLocalAudioRenderer(renderer);
+        } else {
+            track.remove(renderer)
+        }
+    }
+}