@livekit/react-native 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/README.md +4 -3
  2. package/android/build.gradle +2 -1
  3. package/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt +61 -5
  4. package/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt +81 -4
  5. package/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt +6 -0
  6. package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt +2 -0
  7. package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt +27 -0
  8. package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt +52 -0
  9. package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt +72 -0
  10. package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkManager.kt +75 -0
  11. package/android/src/main/java/com/livekit/reactnative/audio/processing/CustomAudioProcessingFactory.kt +78 -0
  12. package/android/src/main/java/com/livekit/reactnative/audio/processing/MultibandVolumeProcessor.kt +181 -0
  13. package/android/src/main/java/com/livekit/reactnative/audio/processing/VolumeProcessor.kt +67 -0
  14. package/android/src/main/java/com/livekit/reactnative/audio/processing/fft/FFTAudioAnalyzer.kt +224 -0
  15. package/ios/LKAudioProcessingAdapter.h +26 -0
  16. package/ios/LKAudioProcessingAdapter.m +117 -0
  17. package/ios/LKAudioProcessingManager.h +34 -0
  18. package/ios/LKAudioProcessingManager.m +63 -0
  19. package/ios/LivekitReactNative-Bridging-Header.h +2 -0
  20. package/ios/LivekitReactNative.h +9 -4
  21. package/ios/LivekitReactNative.m +83 -5
  22. package/ios/Logging.swift +4 -0
  23. package/ios/audio/AVAudioPCMBuffer.swift +136 -0
  24. package/ios/audio/AudioProcessing.swift +163 -0
  25. package/ios/audio/AudioRendererManager.swift +72 -0
  26. package/ios/audio/FFTProcessor.swift +147 -0
  27. package/ios/audio/MultibandVolumeAudioRenderer.swift +65 -0
  28. package/ios/audio/RingBuffer.swift +51 -0
  29. package/ios/audio/VolumeAudioRenderer.swift +48 -0
  30. package/lib/commonjs/LKNativeModule.js +18 -0
  31. package/lib/commonjs/LKNativeModule.js.map +1 -0
  32. package/lib/commonjs/components/BarVisualizer.js +192 -0
  33. package/lib/commonjs/components/BarVisualizer.js.map +1 -0
  34. package/lib/commonjs/events/EventEmitter.js +45 -0
  35. package/lib/commonjs/events/EventEmitter.js.map +1 -0
  36. package/lib/commonjs/hooks/useMultibandTrackVolume.js +64 -0
  37. package/lib/commonjs/hooks/useMultibandTrackVolume.js.map +1 -0
  38. package/lib/commonjs/hooks/useTrackVolume.js +45 -0
  39. package/lib/commonjs/hooks/useTrackVolume.js.map +1 -0
  40. package/lib/commonjs/hooks.js +24 -0
  41. package/lib/commonjs/hooks.js.map +1 -1
  42. package/lib/commonjs/index.js +14 -0
  43. package/lib/commonjs/index.js.map +1 -1
  44. package/lib/module/LKNativeModule.js +12 -0
  45. package/lib/module/LKNativeModule.js.map +1 -0
  46. package/lib/module/components/BarVisualizer.js +182 -0
  47. package/lib/module/components/BarVisualizer.js.map +1 -0
  48. package/lib/module/events/EventEmitter.js +36 -0
  49. package/lib/module/events/EventEmitter.js.map +1 -0
  50. package/lib/module/hooks/useMultibandTrackVolume.js +58 -0
  51. package/lib/module/hooks/useMultibandTrackVolume.js.map +1 -0
  52. package/lib/module/hooks/useTrackVolume.js +39 -0
  53. package/lib/module/hooks/useTrackVolume.js.map +1 -0
  54. package/lib/module/hooks.js +2 -0
  55. package/lib/module/hooks.js.map +1 -1
  56. package/lib/module/index.js +3 -0
  57. package/lib/module/index.js.map +1 -1
  58. package/lib/typescript/lib/commonjs/LKNativeModule.d.ts +3 -0
  59. package/lib/typescript/lib/commonjs/components/BarVisualizer.d.ts +32 -0
  60. package/lib/typescript/lib/commonjs/events/EventEmitter.d.ts +4 -0
  61. package/lib/typescript/lib/commonjs/hooks/useMultibandTrackVolume.d.ts +8 -0
  62. package/lib/typescript/lib/commonjs/hooks/useTrackVolume.d.ts +8 -0
  63. package/lib/typescript/lib/module/LKNativeModule.d.ts +2 -0
  64. package/lib/typescript/lib/module/components/BarVisualizer.d.ts +10 -0
  65. package/lib/typescript/lib/module/events/EventEmitter.d.ts +3 -0
  66. package/lib/typescript/lib/module/hooks/useMultibandTrackVolume.d.ts +7 -0
  67. package/lib/typescript/lib/module/hooks/useTrackVolume.d.ts +7 -0
  68. package/lib/typescript/lib/module/hooks.d.ts +2 -0
  69. package/lib/typescript/lib/module/index.d.ts +1 -0
  70. package/lib/typescript/src/LKNativeModule.d.ts +2 -0
  71. package/lib/typescript/src/components/BarVisualizer.d.ts +49 -0
  72. package/lib/typescript/src/events/EventEmitter.d.ts +6 -0
  73. package/lib/typescript/src/hooks/useMultibandTrackVolume.d.ts +31 -0
  74. package/lib/typescript/src/hooks/useTrackVolume.d.ts +9 -0
  75. package/lib/typescript/src/hooks.d.ts +2 -0
  76. package/lib/typescript/src/index.d.ts +1 -0
  77. package/livekit-react-native.podspec +1 -1
  78. package/package.json +7 -6
  79. package/src/LKNativeModule.ts +19 -0
  80. package/src/components/BarVisualizer.tsx +252 -0
  81. package/src/events/EventEmitter.ts +51 -0
  82. package/src/hooks/useMultibandTrackVolume.ts +97 -0
  83. package/src/hooks/useTrackVolume.ts +62 -0
  84. package/src/hooks.ts +2 -0
  85. package/src/index.tsx +3 -0
package/README.md CHANGED
@@ -416,9 +416,10 @@ Apache License 2.0
416
416
  <br/><table>
417
417
  <thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead>
418
418
  <tbody>
419
- <tr><td>Realtime SDKs</td><td><a href="https://github.com/livekit/components-js">React Components</a> · <a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/components-swift">Swift Components</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <b>React Native</b> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (web)</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity (beta)</a></td></tr><tr></tr>
420
- <tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a></td></tr><tr></tr>
421
- <tr><td>Agents Frameworks</td><td><a href="https://github.com/livekit/agents">Python</a> · <a href="https://github.com/livekit/agent-playground">Playground</a></td></tr><tr></tr>
419
+ <tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <b>React Native</b> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>
420
+ <tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a> · <a href="https://github.com/pabloFuente/livekit-server-sdk-dotnet">.NET (community)</a></td></tr><tr></tr>
421
+ <tr><td>UI Components</td><td><a href="https://github.com/livekit/components-js">React</a> · <a href="https://github.com/livekit/components-android">Android Compose</a> · <a href="https://github.com/livekit/components-swift">SwiftUI</a></td></tr><tr></tr>
422
+ <tr><td>Agents Frameworks</td><td><a href="https://github.com/livekit/agents">Python</a> · <a href="https://github.com/livekit/agents-js">Node.js</a> · <a href="https://github.com/livekit/agent-playground">Playground</a></td></tr><tr></tr>
422
423
  <tr><td>Services</td><td><a href="https://github.com/livekit/livekit">LiveKit server</a> · <a href="https://github.com/livekit/egress">Egress</a> · <a href="https://github.com/livekit/ingress">Ingress</a> · <a href="https://github.com/livekit/sip">SIP</a></td></tr><tr></tr>
423
424
  <tr><td>Resources</td><td><a href="https://docs.livekit.io">Docs</a> · <a href="https://github.com/livekit-examples">Example apps</a> · <a href="https://livekit.io/cloud">Cloud</a> · <a href="https://docs.livekit.io/home/self-hosting/deployment">Self-hosting</a> · <a href="https://github.com/livekit/livekit-cli">CLI</a></td></tr>
424
425
  </tbody>
@@ -131,6 +131,7 @@ dependencies {
131
131
  implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
132
132
  api 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a'
133
133
  api 'io.github.webrtc-sdk:android:125.6422.02'
134
+ implementation "com.github.paramsen:noise:2.0.0"
134
135
  implementation project(':livekit_react-native-webrtc')
135
- implementation "androidx.annotation:annotation:1.4.0"
136
+ implementation "androidx.annotation:annotation:1.9.1"
136
137
  }
@@ -1,27 +1,56 @@
1
1
  package com.livekit.reactnative
2
2
 
3
+ import android.annotation.SuppressLint
3
4
  import android.app.Application
4
5
  import android.content.Context
5
6
  import android.os.Build
6
7
  import com.livekit.reactnative.audio.AudioType
7
- import com.livekit.reactnative.video.CustomVideoEncoderFactory
8
+ import com.livekit.reactnative.audio.processing.AudioProcessingController
9
+ import com.livekit.reactnative.audio.processing.AudioRecordSamplesDispatcher
10
+ import com.livekit.reactnative.audio.processing.CustomAudioProcessingController
8
11
  import com.livekit.reactnative.video.CustomVideoDecoderFactory
12
+ import com.livekit.reactnative.video.CustomVideoEncoderFactory
9
13
  import com.oney.WebRTCModule.WebRTCModuleOptions
10
14
  import org.webrtc.audio.JavaAudioDeviceModule
15
+ import java.util.concurrent.Callable
11
16
 
12
17
  object LiveKitReactNative {
13
18
 
14
- private lateinit var adm: JavaAudioDeviceModule
19
+
20
+ private var audioType: AudioType = AudioType.CommunicationAudioType()
21
+
22
+ @SuppressLint("StaticFieldLeak")
23
+ private var adm: JavaAudioDeviceModule? = null
15
24
 
16
25
  val audioDeviceModule: JavaAudioDeviceModule
17
26
  get() {
18
- if(!::adm.isInitialized) {
19
- throw IllegalStateException("Audio device module is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
27
+ val adm = this.adm
28
+ ?: throw IllegalStateException("Audio device module is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
29
+ return adm
30
+ }
31
+
32
+ private lateinit var _audioProcessingController: AudioProcessingController
33
+
34
+ val audioProcessingController: AudioProcessingController
35
+ get() {
36
+ if (!::_audioProcessingController.isInitialized) {
37
+ throw IllegalStateException("audioProcessingController is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
20
38
  }
39
+ return _audioProcessingController
40
+ }
21
41
 
22
- return adm
42
+
43
+ lateinit var _audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
44
+
45
+ val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
46
+ get() {
47
+ if (!::_audioRecordSamplesDispatcher.isInitialized) {
48
+ throw IllegalStateException("audioRecordSamplesDispatcher is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
49
+ }
50
+ return _audioRecordSamplesDispatcher
23
51
  }
24
52
 
53
+
25
54
  /**
26
55
  * Initializes components required for LiveKit to work on Android.
27
56
  *
@@ -34,19 +63,46 @@ object LiveKitReactNative {
34
63
  context: Context,
35
64
  audioType: AudioType = AudioType.CommunicationAudioType()
36
65
  ) {
66
+ _audioRecordSamplesDispatcher = AudioRecordSamplesDispatcher()
67
+
68
+ this.audioType = audioType
37
69
  val options = WebRTCModuleOptions.getInstance()
38
70
  options.videoEncoderFactory = CustomVideoEncoderFactory(null, true, true)
39
71
  options.videoDecoderFactory = CustomVideoDecoderFactory()
40
72
  options.enableMediaProjectionService = true
41
73
 
74
+ setupAdm(context)
75
+ options.audioDeviceModule = adm
76
+ }
77
+
78
+ private fun setupAdm(context: Context) {
42
79
  val useHardwareAudioProcessing = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q
43
80
 
44
81
  adm = JavaAudioDeviceModule.builder(context)
45
82
  .setUseHardwareAcousticEchoCanceler(useHardwareAudioProcessing)
46
83
  .setUseHardwareNoiseSuppressor(useHardwareAudioProcessing)
47
84
  .setAudioAttributes(audioType.audioAttributes)
85
+ .setSamplesReadyCallback(audioRecordSamplesDispatcher)
48
86
  .createAudioDeviceModule()
87
+ }
49
88
 
89
+ internal fun invalidate(context: Context) {
90
+ val options = WebRTCModuleOptions.getInstance()
91
+ if (options.audioDeviceModule == adm) {
92
+ options.audioDeviceModule = null
93
+ }
94
+ adm?.release()
95
+ adm = null
96
+
97
+ setupAdm(context)
50
98
  options.audioDeviceModule = adm
99
+
100
+ // CustomAudioProcessingController can't be instantiated before WebRTC is loaded.
101
+ options.audioProcessingFactoryFactory = Callable {
102
+ val apc = CustomAudioProcessingController()
103
+ _audioProcessingController = apc
104
+ return@Callable apc.externalAudioProcessor
105
+ }
106
+
51
107
  }
52
108
  }
@@ -1,17 +1,26 @@
1
1
  package com.livekit.reactnative
2
2
 
3
- import android.annotation.SuppressLint
4
- import android.content.Context
5
3
  import android.media.AudioAttributes
6
- import com.facebook.react.bridge.*
4
+ import android.util.Log
5
+ import com.facebook.react.bridge.Arguments
6
+ import com.facebook.react.bridge.Promise
7
+ import com.facebook.react.bridge.ReactApplicationContext
8
+ import com.facebook.react.bridge.ReactContextBaseJavaModule
9
+ import com.facebook.react.bridge.ReactMethod
10
+ import com.facebook.react.bridge.ReadableMap
7
11
  import com.livekit.reactnative.audio.AudioDeviceKind
8
12
  import com.livekit.reactnative.audio.AudioManagerUtils
9
13
  import com.livekit.reactnative.audio.AudioSwitchManager
14
+ import com.livekit.reactnative.audio.processing.AudioSinkManager
15
+ import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
16
+ import com.livekit.reactnative.audio.processing.VolumeProcessor
10
17
  import org.webrtc.audio.WebRtcAudioTrackHelper
18
+ import kotlin.time.Duration.Companion.milliseconds
11
19
 
12
20
 
13
21
  class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
14
22
 
23
+ val audioSinkManager = AudioSinkManager(reactContext)
15
24
  val audioManager = AudioSwitchManager(reactContext.applicationContext)
16
25
  override fun getName(): String {
17
26
  return "LivekitReactNative"
@@ -115,9 +124,77 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
115
124
  promise.resolve(Arguments.makeNativeArray(deviceIds))
116
125
  }
117
126
 
118
- @ReactMethod
127
+ @ReactMethod(isBlockingSynchronousMethod = true)
119
128
  fun selectAudioOutput(deviceId: String, promise: Promise) {
120
129
  audioManager.selectAudioOutput(AudioDeviceKind.fromTypeName(deviceId))
121
130
  promise.resolve(null)
122
131
  }
132
+
133
+ @ReactMethod(isBlockingSynchronousMethod = true)
134
+ fun createVolumeProcessor(pcId: Int, trackId: String): String {
135
+ val processor = VolumeProcessor(reactApplicationContext)
136
+ val reactTag = audioSinkManager.registerSink(processor)
137
+ audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
138
+ processor.reactTag = reactTag
139
+
140
+ return reactTag
141
+ }
142
+
143
+ @ReactMethod(isBlockingSynchronousMethod = true)
144
+ fun deleteVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
145
+ audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
146
+ audioSinkManager.unregisterSink(reactTag)
147
+ }
148
+
149
+ @ReactMethod(isBlockingSynchronousMethod = true)
150
+ fun createMultibandVolumeProcessor(options: ReadableMap, pcId: Int, trackId: String): String {
151
+ val bands = options.getInt("bands")
152
+ val minFrequency = options.getDouble("minFrequency")
153
+ val maxFrequency = options.getDouble("maxFrequency")
154
+ val intervalMs = options.getDouble("updateInterval")
155
+
156
+ val processor = MultibandVolumeProcessor(
157
+ minFrequency = minFrequency.toFloat(),
158
+ maxFrequency = maxFrequency.toFloat(),
159
+ barCount = bands,
160
+ interval = intervalMs.milliseconds,
161
+ reactContext = reactApplicationContext
162
+ )
163
+ val reactTag = audioSinkManager.registerSink(processor)
164
+ processor.reactTag = reactTag
165
+ audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
166
+
167
+ processor.start()
168
+
169
+ return reactTag
170
+ }
171
+
172
+ @ReactMethod(isBlockingSynchronousMethod = true)
173
+ fun deleteMultibandVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
174
+ val volumeProcessor =
175
+ audioSinkManager.getSink(reactTag) ?: throw IllegalArgumentException("Can't find volume processor for $reactTag")
176
+ audioSinkManager.detachSinkFromTrack(volumeProcessor, pcId, trackId)
177
+ audioSinkManager.unregisterSink(volumeProcessor)
178
+ val multibandVolumeProcessor = volumeProcessor as? MultibandVolumeProcessor
179
+
180
+ if (multibandVolumeProcessor != null) {
181
+ multibandVolumeProcessor.release()
182
+ } else {
183
+ Log.w(name, "deleteMultibandVolumeProcessor called, but non-MultibandVolumeProcessor found?!")
184
+ }
185
+ }
186
+
187
+ @ReactMethod
188
+ fun addListener(eventName: String?) {
189
+ // Keep: Required for RN built in Event Emitter Calls.
190
+ }
191
+
192
+ @ReactMethod
193
+ fun removeListeners(count: Int?) {
194
+ // Keep: Required for RN built in Event Emitter Calls.
195
+ }
196
+
197
+ override fun invalidate() {
198
+ LiveKitReactNative.invalidate(reactApplicationContext)
199
+ }
123
200
  }
@@ -0,0 +1,6 @@
1
+ package com.livekit.reactnative.audio.events
2
+
3
+ enum class Events {
4
+ LK_VOLUME_PROCESSED,
5
+ LK_MULTIBAND_PROCESSED,
6
+ }
@@ -0,0 +1,2 @@
1
+ package com.livekit.reactnative.audio.processing
2
+ data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int)
@@ -0,0 +1,27 @@
1
+ package com.livekit.reactnative.audio.processing
2
+
3
+ /**
4
+ * Interface for controlling external audio processing.
5
+ */
6
+ interface AudioProcessingController {
7
+ /**
8
+ * the audio processor to be used for capture post processing.
9
+ */
10
+ var capturePostProcessor: AudioProcessorInterface?
11
+
12
+ /**
13
+ * the audio processor to be used for render pre processing.
14
+ */
15
+ var renderPreProcessor: AudioProcessorInterface?
16
+
17
+ /**
18
+ * whether to bypass mode the render pre processing.
19
+ */
20
+ var bypassRenderPreProcessing: Boolean
21
+
22
+ /**
23
+ * whether to bypass the capture post processing.
24
+ */
25
+ var bypassCapturePostProcessing: Boolean
26
+
27
+ }
@@ -0,0 +1,52 @@
1
+ package com.livekit.reactnative.audio.processing
2
+
3
+ import java.nio.ByteBuffer
4
+
5
+ /**
6
+ * Interface for external audio processing.
7
+ */
8
+ interface AudioProcessorInterface {
9
+ /**
10
+ * Check if the audio processing is enabled.
11
+ */
12
+ fun isEnabled(): Boolean
13
+
14
+ /**
15
+ * Get the name of the audio processing.
16
+ */
17
+ fun getName(): String
18
+
19
+ /**
20
+ * Initialize the audio processing.
21
+ *
22
+ * Note: audio processing methods will be called regardless of whether
23
+ * [isEnabled] returns true or not.
24
+ */
25
+ fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int)
26
+
27
+ /**
28
+ * Called when the sample rate has changed.
29
+ *
30
+ * Note: audio processing methods will be called regardless of whether
31
+ * [isEnabled] returns true or not.
32
+ */
33
+ fun resetAudioProcessing(newRate: Int)
34
+
35
+ /**
36
+ * Process the audio frame (10ms).
37
+ *
38
+ * Note: audio processing methods will be called regardless of whether
39
+ * [isEnabled] returns true or not.
40
+ */
41
+ fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer)
42
+ }
43
+
44
+ /**
45
+ * @suppress
46
+ */
47
+ interface AuthedAudioProcessorInterface : AudioProcessorInterface {
48
+ /**
49
+ * @suppress
50
+ */
51
+ fun authenticate(url: String, token: String)
52
+ }
@@ -0,0 +1,72 @@
1
+ /*
2
+ * Copyright 2024 LiveKit, Inc.
3
+ *
4
+ * Licensed under the Apache License, Version 2.0 (the "License");
5
+ * you may not use this file except in compliance with the License.
6
+ * You may obtain a copy of the License at
7
+ *
8
+ * http://www.apache.org/licenses/LICENSE-2.0
9
+ *
10
+ * Unless required by applicable law or agreed to in writing, software
11
+ * distributed under the License is distributed on an "AS IS" BASIS,
12
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ * See the License for the specific language governing permissions and
14
+ * limitations under the License.
15
+ */
16
+
17
+ package com.livekit.reactnative.audio.processing
18
+
19
+ import android.media.AudioFormat
20
+ import android.os.SystemClock
21
+ import org.webrtc.AudioTrackSink
22
+ import org.webrtc.audio.JavaAudioDeviceModule
23
+ import java.nio.ByteBuffer
24
+
25
+ /**
26
+ * Dispatches recorded audio samples from the local microphone.
27
+ */
28
+ class AudioRecordSamplesDispatcher : JavaAudioDeviceModule.SamplesReadyCallback {
29
+
30
+ private val sinks = mutableSetOf<AudioTrackSink>()
31
+
32
+ @Synchronized
33
+ fun registerSink(sink: AudioTrackSink) {
34
+ sinks.add(sink)
35
+ }
36
+
37
+ @Synchronized
38
+ fun unregisterSink(sink: AudioTrackSink) {
39
+ sinks.remove(sink)
40
+ }
41
+
42
+ // Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8
43
+ // Default audio data format is PCM 16 bits per sample.
44
+ // Guaranteed to be supported by all devices
45
+ private fun getBytesPerSample(audioFormat: Int): Int {
46
+ return when (audioFormat) {
47
+ AudioFormat.ENCODING_PCM_8BIT -> 1
48
+ AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
49
+ AudioFormat.ENCODING_PCM_FLOAT -> 4
50
+ AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
51
+ else -> throw IllegalArgumentException("Bad audio format $audioFormat")
52
+ }
53
+ }
54
+
55
+ @Synchronized
56
+ override fun onWebRtcAudioRecordSamplesReady(samples: JavaAudioDeviceModule.AudioSamples) {
57
+ val bitsPerSample = getBytesPerSample(samples.audioFormat) * 8
58
+ val numFrames = samples.sampleRate / 100 // 10ms worth of samples.
59
+ val timestamp = SystemClock.elapsedRealtime()
60
+ for (sink in sinks) {
61
+ val byteBuffer = ByteBuffer.wrap(samples.data)
62
+ sink.onData(
63
+ byteBuffer,
64
+ bitsPerSample,
65
+ samples.sampleRate,
66
+ samples.channelCount,
67
+ numFrames,
68
+ timestamp,
69
+ )
70
+ }
71
+ }
72
+ }
@@ -0,0 +1,75 @@
1
+ package com.livekit.reactnative.audio.processing
2
+
3
+ import com.facebook.react.bridge.ReactContext
4
+ import com.livekit.reactnative.LiveKitReactNative
5
+ import com.oney.WebRTCModule.WebRTCModule
6
+ import org.webrtc.AudioTrack
7
+ import org.webrtc.AudioTrackSink
8
+ import java.util.UUID
9
+
10
+ private const val LOCAL_PC_ID = -1
11
+
12
+ class AudioSinkManager(val reactContext: ReactContext) {
13
+
14
+ private val sinks = mutableMapOf<String, AudioTrackSink>()
15
+
16
+ /**
17
+ * Registers a sink to this manager.
18
+ * @return the tag to identify this sink in future calls, such as [getSink] or [unregisterSink]
19
+ */
20
+ fun registerSink(sink: AudioTrackSink): String {
21
+ val reactTag = UUID.randomUUID().toString()
22
+ sinks[reactTag] = sink
23
+
24
+ return reactTag
25
+ }
26
+
27
+ /**
28
+ * Unregisters a sink from this manager. Does not detach the sink from tracks.
29
+ */
30
+ fun unregisterSink(reactTag: String) {
31
+ sinks.remove(reactTag)
32
+ }
33
+
34
+ /**
35
+ * Unregisters a sink from this manager. Does not detach the sink from tracks.
36
+ */
37
+ fun unregisterSink(sink: AudioTrackSink) {
38
+ sinks.filterNot { entry -> entry.value == sink }
39
+ }
40
+
41
+ fun getSink(reactTag: String) = sinks[reactTag]
42
+
43
+ fun attachSinkToTrack(sink: AudioTrackSink, pcId: Int, trackId: String) {
44
+ val webRTCModule =
45
+ reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!")
46
+
47
+ val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack
48
+ ?: throw IllegalArgumentException("Couldn't find audio track for pcID:${pcId}, trackId:${trackId}")
49
+
50
+ if (pcId == LOCAL_PC_ID) {
51
+ LiveKitReactNative.audioRecordSamplesDispatcher.registerSink(sink)
52
+ } else {
53
+ track.addSink(sink)
54
+ }
55
+ }
56
+
57
+ fun detachSinkFromTrack(sink: AudioTrackSink, pcId: Int, trackId: String) {
58
+ val webRTCModule =
59
+ reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!")
60
+ val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack
61
+ ?: return // fail silently
62
+
63
+ if (pcId == LOCAL_PC_ID) {
64
+ LiveKitReactNative.audioRecordSamplesDispatcher.unregisterSink(sink)
65
+ } else {
66
+ track.removeSink(sink)
67
+ }
68
+ }
69
+
70
+ fun detachSinkFromTrack(sinkReactTag: String, pcId: Int, trackId: String) {
71
+ val sink = sinks[sinkReactTag]
72
+ ?: throw IllegalArgumentException("Couldn't find audio sink for react tag: $sinkReactTag")
73
+ detachSinkFromTrack(sink, pcId, trackId)
74
+ }
75
+ }
@@ -0,0 +1,78 @@
1
+ package com.livekit.reactnative.audio.processing
2
+
3
+ import org.webrtc.ExternalAudioProcessingFactory
4
+ import java.nio.ByteBuffer
5
+
6
+ /**
7
+ * @suppress
8
+ */
9
+ class CustomAudioProcessingController(
10
+ /**
11
+ * the audio processor to be used for capture post processing.
12
+ */
13
+ capturePostProcessor: AudioProcessorInterface? = null,
14
+
15
+ /**
16
+ * the audio processor to be used for render pre processing.
17
+ */
18
+ renderPreProcessor: AudioProcessorInterface? = null,
19
+
20
+ /**
21
+ * whether to bypass mode the render pre processing.
22
+ */
23
+ bypassRenderPreProcessing: Boolean = false,
24
+
25
+ /**
26
+ * whether to bypass the capture post processing.
27
+ */
28
+ bypassCapturePostProcessing: Boolean = false,
29
+ ) : AudioProcessingController {
30
+
31
+ val externalAudioProcessor = ExternalAudioProcessingFactory()
32
+
33
+ override var capturePostProcessor: AudioProcessorInterface? = capturePostProcessor
34
+ set(value) {
35
+ field = value
36
+ externalAudioProcessor.setCapturePostProcessing(
37
+ value.toAudioProcessing(),
38
+ )
39
+ }
40
+
41
+ override var renderPreProcessor: AudioProcessorInterface? = renderPreProcessor
42
+ set(value) {
43
+ field = value
44
+ externalAudioProcessor.setRenderPreProcessing(
45
+ value.toAudioProcessing(),
46
+ )
47
+ }
48
+
49
+ override var bypassCapturePostProcessing: Boolean = bypassCapturePostProcessing
50
+ set(value) {
51
+ externalAudioProcessor.setBypassFlagForCapturePost(value)
52
+ }
53
+
54
+ override var bypassRenderPreProcessing: Boolean = bypassRenderPreProcessing
55
+ set(value) {
56
+ externalAudioProcessor.setBypassFlagForRenderPre(value)
57
+ }
58
+
59
+ private class AudioProcessingBridge(
60
+ var audioProcessing: AudioProcessorInterface? = null,
61
+ ) : ExternalAudioProcessingFactory.AudioProcessing {
62
+ override fun initialize(sampleRateHz: Int, numChannels: Int) {
63
+ audioProcessing?.initializeAudioProcessing(sampleRateHz, numChannels)
64
+ }
65
+
66
+ override fun reset(newRate: Int) {
67
+ audioProcessing?.resetAudioProcessing(newRate)
68
+ }
69
+
70
+ override fun process(numBands: Int, numFrames: Int, buffer: ByteBuffer?) {
71
+ audioProcessing?.processAudio(numBands, numFrames, buffer!!)
72
+ }
73
+ }
74
+
75
+ private fun AudioProcessorInterface?.toAudioProcessing(): ExternalAudioProcessingFactory.AudioProcessing {
76
+ return AudioProcessingBridge(this)
77
+ }
78
+ }