@livekit/react-native 2.5.1 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/android/build.gradle +2 -1
- package/android/src/main/java/com/livekit/reactnative/LiveKitReactNative.kt +61 -5
- package/android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt +81 -4
- package/android/src/main/java/com/livekit/reactnative/audio/events/Events.kt +6 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioFormat.kt +2 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt +27 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt +52 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioRecordSamplesDispatcher.kt +72 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkManager.kt +75 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/CustomAudioProcessingFactory.kt +78 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/MultibandVolumeProcessor.kt +181 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/VolumeProcessor.kt +67 -0
- package/android/src/main/java/com/livekit/reactnative/audio/processing/fft/FFTAudioAnalyzer.kt +224 -0
- package/ios/LKAudioProcessingAdapter.h +26 -0
- package/ios/LKAudioProcessingAdapter.m +117 -0
- package/ios/LKAudioProcessingManager.h +34 -0
- package/ios/LKAudioProcessingManager.m +63 -0
- package/ios/LivekitReactNative-Bridging-Header.h +2 -0
- package/ios/LivekitReactNative.h +9 -4
- package/ios/LivekitReactNative.m +83 -5
- package/ios/Logging.swift +4 -0
- package/ios/audio/AVAudioPCMBuffer.swift +136 -0
- package/ios/audio/AudioProcessing.swift +163 -0
- package/ios/audio/AudioRendererManager.swift +72 -0
- package/ios/audio/FFTProcessor.swift +147 -0
- package/ios/audio/MultibandVolumeAudioRenderer.swift +65 -0
- package/ios/audio/RingBuffer.swift +51 -0
- package/ios/audio/VolumeAudioRenderer.swift +48 -0
- package/lib/commonjs/LKNativeModule.js +18 -0
- package/lib/commonjs/LKNativeModule.js.map +1 -0
- package/lib/commonjs/components/BarVisualizer.js +192 -0
- package/lib/commonjs/components/BarVisualizer.js.map +1 -0
- package/lib/commonjs/events/EventEmitter.js +45 -0
- package/lib/commonjs/events/EventEmitter.js.map +1 -0
- package/lib/commonjs/hooks/useMultibandTrackVolume.js +64 -0
- package/lib/commonjs/hooks/useMultibandTrackVolume.js.map +1 -0
- package/lib/commonjs/hooks/useTrackVolume.js +45 -0
- package/lib/commonjs/hooks/useTrackVolume.js.map +1 -0
- package/lib/commonjs/hooks.js +24 -0
- package/lib/commonjs/hooks.js.map +1 -1
- package/lib/commonjs/index.js +14 -0
- package/lib/commonjs/index.js.map +1 -1
- package/lib/module/LKNativeModule.js +12 -0
- package/lib/module/LKNativeModule.js.map +1 -0
- package/lib/module/components/BarVisualizer.js +182 -0
- package/lib/module/components/BarVisualizer.js.map +1 -0
- package/lib/module/events/EventEmitter.js +36 -0
- package/lib/module/events/EventEmitter.js.map +1 -0
- package/lib/module/hooks/useMultibandTrackVolume.js +58 -0
- package/lib/module/hooks/useMultibandTrackVolume.js.map +1 -0
- package/lib/module/hooks/useTrackVolume.js +39 -0
- package/lib/module/hooks/useTrackVolume.js.map +1 -0
- package/lib/module/hooks.js +2 -0
- package/lib/module/hooks.js.map +1 -1
- package/lib/module/index.js +3 -0
- package/lib/module/index.js.map +1 -1
- package/lib/typescript/lib/commonjs/LKNativeModule.d.ts +3 -0
- package/lib/typescript/lib/commonjs/components/BarVisualizer.d.ts +32 -0
- package/lib/typescript/lib/commonjs/events/EventEmitter.d.ts +4 -0
- package/lib/typescript/lib/commonjs/hooks/useMultibandTrackVolume.d.ts +8 -0
- package/lib/typescript/lib/commonjs/hooks/useTrackVolume.d.ts +8 -0
- package/lib/typescript/lib/module/LKNativeModule.d.ts +2 -0
- package/lib/typescript/lib/module/components/BarVisualizer.d.ts +10 -0
- package/lib/typescript/lib/module/events/EventEmitter.d.ts +3 -0
- package/lib/typescript/lib/module/hooks/useMultibandTrackVolume.d.ts +7 -0
- package/lib/typescript/lib/module/hooks/useTrackVolume.d.ts +7 -0
- package/lib/typescript/lib/module/hooks.d.ts +2 -0
- package/lib/typescript/lib/module/index.d.ts +1 -0
- package/lib/typescript/src/LKNativeModule.d.ts +2 -0
- package/lib/typescript/src/components/BarVisualizer.d.ts +49 -0
- package/lib/typescript/src/events/EventEmitter.d.ts +6 -0
- package/lib/typescript/src/hooks/useMultibandTrackVolume.d.ts +31 -0
- package/lib/typescript/src/hooks/useTrackVolume.d.ts +9 -0
- package/lib/typescript/src/hooks.d.ts +2 -0
- package/lib/typescript/src/index.d.ts +1 -0
- package/livekit-react-native.podspec +1 -1
- package/package.json +5 -5
- package/src/LKNativeModule.ts +19 -0
- package/src/components/BarVisualizer.tsx +252 -0
- package/src/events/EventEmitter.ts +51 -0
- package/src/hooks/useMultibandTrackVolume.ts +97 -0
- package/src/hooks/useTrackVolume.ts +62 -0
- package/src/hooks.ts +2 -0
- package/src/index.tsx +3 -0
package/README.md
CHANGED
|
@@ -416,9 +416,10 @@ Apache License 2.0
|
|
|
416
416
|
<br/><table>
|
|
417
417
|
<thead><tr><th colspan="2">LiveKit Ecosystem</th></tr></thead>
|
|
418
418
|
<tbody>
|
|
419
|
-
<tr><td>
|
|
420
|
-
<tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a></td></tr><tr></tr>
|
|
421
|
-
<tr><td>
|
|
419
|
+
<tr><td>LiveKit SDKs</td><td><a href="https://github.com/livekit/client-sdk-js">Browser</a> · <a href="https://github.com/livekit/client-sdk-swift">iOS/macOS/visionOS</a> · <a href="https://github.com/livekit/client-sdk-android">Android</a> · <a href="https://github.com/livekit/client-sdk-flutter">Flutter</a> · <b>React Native</b> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/client-sdk-unity">Unity</a> · <a href="https://github.com/livekit/client-sdk-unity-web">Unity (WebGL)</a></td></tr><tr></tr>
|
|
420
|
+
<tr><td>Server APIs</td><td><a href="https://github.com/livekit/node-sdks">Node.js</a> · <a href="https://github.com/livekit/server-sdk-go">Golang</a> · <a href="https://github.com/livekit/server-sdk-ruby">Ruby</a> · <a href="https://github.com/livekit/server-sdk-kotlin">Java/Kotlin</a> · <a href="https://github.com/livekit/python-sdks">Python</a> · <a href="https://github.com/livekit/rust-sdks">Rust</a> · <a href="https://github.com/agence104/livekit-server-sdk-php">PHP (community)</a> · <a href="https://github.com/pabloFuente/livekit-server-sdk-dotnet">.NET (community)</a></td></tr><tr></tr>
|
|
421
|
+
<tr><td>UI Components</td><td><a href="https://github.com/livekit/components-js">React</a> · <a href="https://github.com/livekit/components-android">Android Compose</a> · <a href="https://github.com/livekit/components-swift">SwiftUI</a></td></tr><tr></tr>
|
|
422
|
+
<tr><td>Agents Frameworks</td><td><a href="https://github.com/livekit/agents">Python</a> · <a href="https://github.com/livekit/agents-js">Node.js</a> · <a href="https://github.com/livekit/agent-playground">Playground</a></td></tr><tr></tr>
|
|
422
423
|
<tr><td>Services</td><td><a href="https://github.com/livekit/livekit">LiveKit server</a> · <a href="https://github.com/livekit/egress">Egress</a> · <a href="https://github.com/livekit/ingress">Ingress</a> · <a href="https://github.com/livekit/sip">SIP</a></td></tr><tr></tr>
|
|
423
424
|
<tr><td>Resources</td><td><a href="https://docs.livekit.io">Docs</a> · <a href="https://github.com/livekit-examples">Example apps</a> · <a href="https://livekit.io/cloud">Cloud</a> · <a href="https://docs.livekit.io/home/self-hosting/deployment">Self-hosting</a> · <a href="https://github.com/livekit/livekit-cli">CLI</a></td></tr>
|
|
424
425
|
</tbody>
|
package/android/build.gradle
CHANGED
|
@@ -131,6 +131,7 @@ dependencies {
|
|
|
131
131
|
implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
|
132
132
|
api 'com.github.davidliu:audioswitch:89582c47c9a04c62f90aa5e57251af4800a62c9a'
|
|
133
133
|
api 'io.github.webrtc-sdk:android:125.6422.02'
|
|
134
|
+
implementation "com.github.paramsen:noise:2.0.0"
|
|
134
135
|
implementation project(':livekit_react-native-webrtc')
|
|
135
|
-
implementation "androidx.annotation:annotation:1.
|
|
136
|
+
implementation "androidx.annotation:annotation:1.9.1"
|
|
136
137
|
}
|
|
@@ -1,27 +1,56 @@
|
|
|
1
1
|
package com.livekit.reactnative
|
|
2
2
|
|
|
3
|
+
import android.annotation.SuppressLint
|
|
3
4
|
import android.app.Application
|
|
4
5
|
import android.content.Context
|
|
5
6
|
import android.os.Build
|
|
6
7
|
import com.livekit.reactnative.audio.AudioType
|
|
7
|
-
import com.livekit.reactnative.
|
|
8
|
+
import com.livekit.reactnative.audio.processing.AudioProcessingController
|
|
9
|
+
import com.livekit.reactnative.audio.processing.AudioRecordSamplesDispatcher
|
|
10
|
+
import com.livekit.reactnative.audio.processing.CustomAudioProcessingController
|
|
8
11
|
import com.livekit.reactnative.video.CustomVideoDecoderFactory
|
|
12
|
+
import com.livekit.reactnative.video.CustomVideoEncoderFactory
|
|
9
13
|
import com.oney.WebRTCModule.WebRTCModuleOptions
|
|
10
14
|
import org.webrtc.audio.JavaAudioDeviceModule
|
|
15
|
+
import java.util.concurrent.Callable
|
|
11
16
|
|
|
12
17
|
object LiveKitReactNative {
|
|
13
18
|
|
|
14
|
-
|
|
19
|
+
|
|
20
|
+
private var audioType: AudioType = AudioType.CommunicationAudioType()
|
|
21
|
+
|
|
22
|
+
@SuppressLint("StaticFieldLeak")
|
|
23
|
+
private var adm: JavaAudioDeviceModule? = null
|
|
15
24
|
|
|
16
25
|
val audioDeviceModule: JavaAudioDeviceModule
|
|
17
26
|
get() {
|
|
18
|
-
|
|
19
|
-
throw IllegalStateException("Audio device module is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
|
|
27
|
+
val adm = this.adm
|
|
28
|
+
?: throw IllegalStateException("Audio device module is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
|
|
29
|
+
return adm
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private lateinit var _audioProcessingController: AudioProcessingController
|
|
33
|
+
|
|
34
|
+
val audioProcessingController: AudioProcessingController
|
|
35
|
+
get() {
|
|
36
|
+
if (!::_audioProcessingController.isInitialized) {
|
|
37
|
+
throw IllegalStateException("audioProcessingController is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
|
|
20
38
|
}
|
|
39
|
+
return _audioProcessingController
|
|
40
|
+
}
|
|
21
41
|
|
|
22
|
-
|
|
42
|
+
|
|
43
|
+
lateinit var _audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
|
|
44
|
+
|
|
45
|
+
val audioRecordSamplesDispatcher: AudioRecordSamplesDispatcher
|
|
46
|
+
get() {
|
|
47
|
+
if (!::_audioRecordSamplesDispatcher.isInitialized) {
|
|
48
|
+
throw IllegalStateException("audioRecordSamplesDispatcher is not initialized! Did you remember to call LiveKitReactNative.setup in your Application.onCreate?")
|
|
49
|
+
}
|
|
50
|
+
return _audioRecordSamplesDispatcher
|
|
23
51
|
}
|
|
24
52
|
|
|
53
|
+
|
|
25
54
|
/**
|
|
26
55
|
* Initializes components required for LiveKit to work on Android.
|
|
27
56
|
*
|
|
@@ -34,19 +63,46 @@ object LiveKitReactNative {
|
|
|
34
63
|
context: Context,
|
|
35
64
|
audioType: AudioType = AudioType.CommunicationAudioType()
|
|
36
65
|
) {
|
|
66
|
+
_audioRecordSamplesDispatcher = AudioRecordSamplesDispatcher()
|
|
67
|
+
|
|
68
|
+
this.audioType = audioType
|
|
37
69
|
val options = WebRTCModuleOptions.getInstance()
|
|
38
70
|
options.videoEncoderFactory = CustomVideoEncoderFactory(null, true, true)
|
|
39
71
|
options.videoDecoderFactory = CustomVideoDecoderFactory()
|
|
40
72
|
options.enableMediaProjectionService = true
|
|
41
73
|
|
|
74
|
+
setupAdm(context)
|
|
75
|
+
options.audioDeviceModule = adm
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private fun setupAdm(context: Context) {
|
|
42
79
|
val useHardwareAudioProcessing = Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q
|
|
43
80
|
|
|
44
81
|
adm = JavaAudioDeviceModule.builder(context)
|
|
45
82
|
.setUseHardwareAcousticEchoCanceler(useHardwareAudioProcessing)
|
|
46
83
|
.setUseHardwareNoiseSuppressor(useHardwareAudioProcessing)
|
|
47
84
|
.setAudioAttributes(audioType.audioAttributes)
|
|
85
|
+
.setSamplesReadyCallback(audioRecordSamplesDispatcher)
|
|
48
86
|
.createAudioDeviceModule()
|
|
87
|
+
}
|
|
49
88
|
|
|
89
|
+
internal fun invalidate(context: Context) {
|
|
90
|
+
val options = WebRTCModuleOptions.getInstance()
|
|
91
|
+
if (options.audioDeviceModule == adm) {
|
|
92
|
+
options.audioDeviceModule = null
|
|
93
|
+
}
|
|
94
|
+
adm?.release()
|
|
95
|
+
adm = null
|
|
96
|
+
|
|
97
|
+
setupAdm(context)
|
|
50
98
|
options.audioDeviceModule = adm
|
|
99
|
+
|
|
100
|
+
// CustomAudioProcessingController can't be instantiated before WebRTC is loaded.
|
|
101
|
+
options.audioProcessingFactoryFactory = Callable {
|
|
102
|
+
val apc = CustomAudioProcessingController()
|
|
103
|
+
_audioProcessingController = apc
|
|
104
|
+
return@Callable apc.externalAudioProcessor
|
|
105
|
+
}
|
|
106
|
+
|
|
51
107
|
}
|
|
52
108
|
}
|
|
@@ -1,17 +1,26 @@
|
|
|
1
1
|
package com.livekit.reactnative
|
|
2
2
|
|
|
3
|
-
import android.annotation.SuppressLint
|
|
4
|
-
import android.content.Context
|
|
5
3
|
import android.media.AudioAttributes
|
|
6
|
-
import
|
|
4
|
+
import android.util.Log
|
|
5
|
+
import com.facebook.react.bridge.Arguments
|
|
6
|
+
import com.facebook.react.bridge.Promise
|
|
7
|
+
import com.facebook.react.bridge.ReactApplicationContext
|
|
8
|
+
import com.facebook.react.bridge.ReactContextBaseJavaModule
|
|
9
|
+
import com.facebook.react.bridge.ReactMethod
|
|
10
|
+
import com.facebook.react.bridge.ReadableMap
|
|
7
11
|
import com.livekit.reactnative.audio.AudioDeviceKind
|
|
8
12
|
import com.livekit.reactnative.audio.AudioManagerUtils
|
|
9
13
|
import com.livekit.reactnative.audio.AudioSwitchManager
|
|
14
|
+
import com.livekit.reactnative.audio.processing.AudioSinkManager
|
|
15
|
+
import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
|
|
16
|
+
import com.livekit.reactnative.audio.processing.VolumeProcessor
|
|
10
17
|
import org.webrtc.audio.WebRtcAudioTrackHelper
|
|
18
|
+
import kotlin.time.Duration.Companion.milliseconds
|
|
11
19
|
|
|
12
20
|
|
|
13
21
|
class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
|
|
14
22
|
|
|
23
|
+
val audioSinkManager = AudioSinkManager(reactContext)
|
|
15
24
|
val audioManager = AudioSwitchManager(reactContext.applicationContext)
|
|
16
25
|
override fun getName(): String {
|
|
17
26
|
return "LivekitReactNative"
|
|
@@ -115,9 +124,77 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
|
|
|
115
124
|
promise.resolve(Arguments.makeNativeArray(deviceIds))
|
|
116
125
|
}
|
|
117
126
|
|
|
118
|
-
@ReactMethod
|
|
127
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
119
128
|
fun selectAudioOutput(deviceId: String, promise: Promise) {
|
|
120
129
|
audioManager.selectAudioOutput(AudioDeviceKind.fromTypeName(deviceId))
|
|
121
130
|
promise.resolve(null)
|
|
122
131
|
}
|
|
132
|
+
|
|
133
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
134
|
+
fun createVolumeProcessor(pcId: Int, trackId: String): String {
|
|
135
|
+
val processor = VolumeProcessor(reactApplicationContext)
|
|
136
|
+
val reactTag = audioSinkManager.registerSink(processor)
|
|
137
|
+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
|
|
138
|
+
processor.reactTag = reactTag
|
|
139
|
+
|
|
140
|
+
return reactTag
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
144
|
+
fun deleteVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
|
|
145
|
+
audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
|
|
146
|
+
audioSinkManager.unregisterSink(reactTag)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
150
|
+
fun createMultibandVolumeProcessor(options: ReadableMap, pcId: Int, trackId: String): String {
|
|
151
|
+
val bands = options.getInt("bands")
|
|
152
|
+
val minFrequency = options.getDouble("minFrequency")
|
|
153
|
+
val maxFrequency = options.getDouble("maxFrequency")
|
|
154
|
+
val intervalMs = options.getDouble("updateInterval")
|
|
155
|
+
|
|
156
|
+
val processor = MultibandVolumeProcessor(
|
|
157
|
+
minFrequency = minFrequency.toFloat(),
|
|
158
|
+
maxFrequency = maxFrequency.toFloat(),
|
|
159
|
+
barCount = bands,
|
|
160
|
+
interval = intervalMs.milliseconds,
|
|
161
|
+
reactContext = reactApplicationContext
|
|
162
|
+
)
|
|
163
|
+
val reactTag = audioSinkManager.registerSink(processor)
|
|
164
|
+
processor.reactTag = reactTag
|
|
165
|
+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
|
|
166
|
+
|
|
167
|
+
processor.start()
|
|
168
|
+
|
|
169
|
+
return reactTag
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
@ReactMethod(isBlockingSynchronousMethod = true)
|
|
173
|
+
fun deleteMultibandVolumeProcessor(reactTag: String, pcId: Int, trackId: String) {
|
|
174
|
+
val volumeProcessor =
|
|
175
|
+
audioSinkManager.getSink(reactTag) ?: throw IllegalArgumentException("Can't find volume processor for $reactTag")
|
|
176
|
+
audioSinkManager.detachSinkFromTrack(volumeProcessor, pcId, trackId)
|
|
177
|
+
audioSinkManager.unregisterSink(volumeProcessor)
|
|
178
|
+
val multibandVolumeProcessor = volumeProcessor as? MultibandVolumeProcessor
|
|
179
|
+
|
|
180
|
+
if (multibandVolumeProcessor != null) {
|
|
181
|
+
multibandVolumeProcessor.release()
|
|
182
|
+
} else {
|
|
183
|
+
Log.w(name, "deleteMultibandVolumeProcessor called, but non-MultibandVolumeProcessor found?!")
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
@ReactMethod
|
|
188
|
+
fun addListener(eventName: String?) {
|
|
189
|
+
// Keep: Required for RN built in Event Emitter Calls.
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
@ReactMethod
|
|
193
|
+
fun removeListeners(count: Int?) {
|
|
194
|
+
// Keep: Required for RN built in Event Emitter Calls.
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
override fun invalidate() {
|
|
198
|
+
LiveKitReactNative.invalidate(reactApplicationContext)
|
|
199
|
+
}
|
|
123
200
|
}
|
package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessingController.kt
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
package com.livekit.reactnative.audio.processing
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Interface for controlling external audio processing.
|
|
5
|
+
*/
|
|
6
|
+
interface AudioProcessingController {
|
|
7
|
+
/**
|
|
8
|
+
* the audio processor to be used for capture post processing.
|
|
9
|
+
*/
|
|
10
|
+
var capturePostProcessor: AudioProcessorInterface?
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* the audio processor to be used for render pre processing.
|
|
14
|
+
*/
|
|
15
|
+
var renderPreProcessor: AudioProcessorInterface?
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* whether to bypass mode the render pre processing.
|
|
19
|
+
*/
|
|
20
|
+
var bypassRenderPreProcessing: Boolean
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* whether to bypass the capture post processing.
|
|
24
|
+
*/
|
|
25
|
+
var bypassCapturePostProcessing: Boolean
|
|
26
|
+
|
|
27
|
+
}
|
package/android/src/main/java/com/livekit/reactnative/audio/processing/AudioProcessorInterface.kt
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
package com.livekit.reactnative.audio.processing
|
|
2
|
+
|
|
3
|
+
import java.nio.ByteBuffer
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Interface for external audio processing.
|
|
7
|
+
*/
|
|
8
|
+
interface AudioProcessorInterface {
|
|
9
|
+
/**
|
|
10
|
+
* Check if the audio processing is enabled.
|
|
11
|
+
*/
|
|
12
|
+
fun isEnabled(): Boolean
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Get the name of the audio processing.
|
|
16
|
+
*/
|
|
17
|
+
fun getName(): String
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Initialize the audio processing.
|
|
21
|
+
*
|
|
22
|
+
* Note: audio processing methods will be called regardless of whether
|
|
23
|
+
* [isEnabled] returns true or not.
|
|
24
|
+
*/
|
|
25
|
+
fun initializeAudioProcessing(sampleRateHz: Int, numChannels: Int)
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Called when the sample rate has changed.
|
|
29
|
+
*
|
|
30
|
+
* Note: audio processing methods will be called regardless of whether
|
|
31
|
+
* [isEnabled] returns true or not.
|
|
32
|
+
*/
|
|
33
|
+
fun resetAudioProcessing(newRate: Int)
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Process the audio frame (10ms).
|
|
37
|
+
*
|
|
38
|
+
* Note: audio processing methods will be called regardless of whether
|
|
39
|
+
* [isEnabled] returns true or not.
|
|
40
|
+
*/
|
|
41
|
+
fun processAudio(numBands: Int, numFrames: Int, buffer: ByteBuffer)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* @suppress
|
|
46
|
+
*/
|
|
47
|
+
interface AuthedAudioProcessorInterface : AudioProcessorInterface {
|
|
48
|
+
/**
|
|
49
|
+
* @suppress
|
|
50
|
+
*/
|
|
51
|
+
fun authenticate(url: String, token: String)
|
|
52
|
+
}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright 2024 LiveKit, Inc.
|
|
3
|
+
*
|
|
4
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
5
|
+
* you may not use this file except in compliance with the License.
|
|
6
|
+
* You may obtain a copy of the License at
|
|
7
|
+
*
|
|
8
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
9
|
+
*
|
|
10
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
11
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
12
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
13
|
+
* See the License for the specific language governing permissions and
|
|
14
|
+
* limitations under the License.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
package com.livekit.reactnative.audio.processing
|
|
18
|
+
|
|
19
|
+
import android.media.AudioFormat
|
|
20
|
+
import android.os.SystemClock
|
|
21
|
+
import org.webrtc.AudioTrackSink
|
|
22
|
+
import org.webrtc.audio.JavaAudioDeviceModule
|
|
23
|
+
import java.nio.ByteBuffer
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Dispatches recorded audio samples from the local microphone.
|
|
27
|
+
*/
|
|
28
|
+
class AudioRecordSamplesDispatcher : JavaAudioDeviceModule.SamplesReadyCallback {
|
|
29
|
+
|
|
30
|
+
private val sinks = mutableSetOf<AudioTrackSink>()
|
|
31
|
+
|
|
32
|
+
@Synchronized
|
|
33
|
+
fun registerSink(sink: AudioTrackSink) {
|
|
34
|
+
sinks.add(sink)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
@Synchronized
|
|
38
|
+
fun unregisterSink(sink: AudioTrackSink) {
|
|
39
|
+
sinks.remove(sink)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Reference from Android code, AudioFormat.getBytesPerSample. BitPerSample / 8
|
|
43
|
+
// Default audio data format is PCM 16 bits per sample.
|
|
44
|
+
// Guaranteed to be supported by all devices
|
|
45
|
+
private fun getBytesPerSample(audioFormat: Int): Int {
|
|
46
|
+
return when (audioFormat) {
|
|
47
|
+
AudioFormat.ENCODING_PCM_8BIT -> 1
|
|
48
|
+
AudioFormat.ENCODING_PCM_16BIT, AudioFormat.ENCODING_IEC61937, AudioFormat.ENCODING_DEFAULT -> 2
|
|
49
|
+
AudioFormat.ENCODING_PCM_FLOAT -> 4
|
|
50
|
+
AudioFormat.ENCODING_INVALID -> throw IllegalArgumentException("Bad audio format $audioFormat")
|
|
51
|
+
else -> throw IllegalArgumentException("Bad audio format $audioFormat")
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
@Synchronized
|
|
56
|
+
override fun onWebRtcAudioRecordSamplesReady(samples: JavaAudioDeviceModule.AudioSamples) {
|
|
57
|
+
val bitsPerSample = getBytesPerSample(samples.audioFormat) * 8
|
|
58
|
+
val numFrames = samples.sampleRate / 100 // 10ms worth of samples.
|
|
59
|
+
val timestamp = SystemClock.elapsedRealtime()
|
|
60
|
+
for (sink in sinks) {
|
|
61
|
+
val byteBuffer = ByteBuffer.wrap(samples.data)
|
|
62
|
+
sink.onData(
|
|
63
|
+
byteBuffer,
|
|
64
|
+
bitsPerSample,
|
|
65
|
+
samples.sampleRate,
|
|
66
|
+
samples.channelCount,
|
|
67
|
+
numFrames,
|
|
68
|
+
timestamp,
|
|
69
|
+
)
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
package com.livekit.reactnative.audio.processing
|
|
2
|
+
|
|
3
|
+
import com.facebook.react.bridge.ReactContext
|
|
4
|
+
import com.livekit.reactnative.LiveKitReactNative
|
|
5
|
+
import com.oney.WebRTCModule.WebRTCModule
|
|
6
|
+
import org.webrtc.AudioTrack
|
|
7
|
+
import org.webrtc.AudioTrackSink
|
|
8
|
+
import java.util.UUID
|
|
9
|
+
|
|
10
|
+
private const val LOCAL_PC_ID = -1
|
|
11
|
+
|
|
12
|
+
class AudioSinkManager(val reactContext: ReactContext) {
|
|
13
|
+
|
|
14
|
+
private val sinks = mutableMapOf<String, AudioTrackSink>()
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Registers a sink to this manager.
|
|
18
|
+
* @return the tag to identify this sink in future calls, such as [getSink] or [unregisterSink]
|
|
19
|
+
*/
|
|
20
|
+
fun registerSink(sink: AudioTrackSink): String {
|
|
21
|
+
val reactTag = UUID.randomUUID().toString()
|
|
22
|
+
sinks[reactTag] = sink
|
|
23
|
+
|
|
24
|
+
return reactTag
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Unregisters a sink from this manager. Does not detach the sink from tracks.
|
|
29
|
+
*/
|
|
30
|
+
fun unregisterSink(reactTag: String) {
|
|
31
|
+
sinks.remove(reactTag)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Unregisters a sink from this manager. Does not detach the sink from tracks.
|
|
36
|
+
*/
|
|
37
|
+
fun unregisterSink(sink: AudioTrackSink) {
|
|
38
|
+
sinks.filterNot { entry -> entry.value == sink }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
fun getSink(reactTag: String) = sinks[reactTag]
|
|
42
|
+
|
|
43
|
+
fun attachSinkToTrack(sink: AudioTrackSink, pcId: Int, trackId: String) {
|
|
44
|
+
val webRTCModule =
|
|
45
|
+
reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!")
|
|
46
|
+
|
|
47
|
+
val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack
|
|
48
|
+
?: throw IllegalArgumentException("Couldn't find audio track for pcID:${pcId}, trackId:${trackId}")
|
|
49
|
+
|
|
50
|
+
if (pcId == LOCAL_PC_ID) {
|
|
51
|
+
LiveKitReactNative.audioRecordSamplesDispatcher.registerSink(sink)
|
|
52
|
+
} else {
|
|
53
|
+
track.addSink(sink)
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
fun detachSinkFromTrack(sink: AudioTrackSink, pcId: Int, trackId: String) {
|
|
58
|
+
val webRTCModule =
|
|
59
|
+
reactContext.getNativeModule(WebRTCModule::class.java) ?: throw IllegalArgumentException("Couldn't find WebRTC module!")
|
|
60
|
+
val track = webRTCModule.getTrack(pcId, trackId) as? AudioTrack
|
|
61
|
+
?: return // fail silently
|
|
62
|
+
|
|
63
|
+
if (pcId == LOCAL_PC_ID) {
|
|
64
|
+
LiveKitReactNative.audioRecordSamplesDispatcher.unregisterSink(sink)
|
|
65
|
+
} else {
|
|
66
|
+
track.removeSink(sink)
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
fun detachSinkFromTrack(sinkReactTag: String, pcId: Int, trackId: String) {
|
|
71
|
+
val sink = sinks[sinkReactTag]
|
|
72
|
+
?: throw IllegalArgumentException("Couldn't find audio sink for react tag: $sinkReactTag")
|
|
73
|
+
detachSinkFromTrack(sink, pcId, trackId)
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
package com.livekit.reactnative.audio.processing
|
|
2
|
+
|
|
3
|
+
import org.webrtc.ExternalAudioProcessingFactory
|
|
4
|
+
import java.nio.ByteBuffer
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* @suppress
|
|
8
|
+
*/
|
|
9
|
+
class CustomAudioProcessingController(
|
|
10
|
+
/**
|
|
11
|
+
* the audio processor to be used for capture post processing.
|
|
12
|
+
*/
|
|
13
|
+
capturePostProcessor: AudioProcessorInterface? = null,
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* the audio processor to be used for render pre processing.
|
|
17
|
+
*/
|
|
18
|
+
renderPreProcessor: AudioProcessorInterface? = null,
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* whether to bypass mode the render pre processing.
|
|
22
|
+
*/
|
|
23
|
+
bypassRenderPreProcessing: Boolean = false,
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* whether to bypass the capture post processing.
|
|
27
|
+
*/
|
|
28
|
+
bypassCapturePostProcessing: Boolean = false,
|
|
29
|
+
) : AudioProcessingController {
|
|
30
|
+
|
|
31
|
+
val externalAudioProcessor = ExternalAudioProcessingFactory()
|
|
32
|
+
|
|
33
|
+
override var capturePostProcessor: AudioProcessorInterface? = capturePostProcessor
|
|
34
|
+
set(value) {
|
|
35
|
+
field = value
|
|
36
|
+
externalAudioProcessor.setCapturePostProcessing(
|
|
37
|
+
value.toAudioProcessing(),
|
|
38
|
+
)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
override var renderPreProcessor: AudioProcessorInterface? = renderPreProcessor
|
|
42
|
+
set(value) {
|
|
43
|
+
field = value
|
|
44
|
+
externalAudioProcessor.setRenderPreProcessing(
|
|
45
|
+
value.toAudioProcessing(),
|
|
46
|
+
)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
override var bypassCapturePostProcessing: Boolean = bypassCapturePostProcessing
|
|
50
|
+
set(value) {
|
|
51
|
+
externalAudioProcessor.setBypassFlagForCapturePost(value)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
override var bypassRenderPreProcessing: Boolean = bypassRenderPreProcessing
|
|
55
|
+
set(value) {
|
|
56
|
+
externalAudioProcessor.setBypassFlagForRenderPre(value)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private class AudioProcessingBridge(
|
|
60
|
+
var audioProcessing: AudioProcessorInterface? = null,
|
|
61
|
+
) : ExternalAudioProcessingFactory.AudioProcessing {
|
|
62
|
+
override fun initialize(sampleRateHz: Int, numChannels: Int) {
|
|
63
|
+
audioProcessing?.initializeAudioProcessing(sampleRateHz, numChannels)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
override fun reset(newRate: Int) {
|
|
67
|
+
audioProcessing?.resetAudioProcessing(newRate)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
override fun process(numBands: Int, numFrames: Int, buffer: ByteBuffer?) {
|
|
71
|
+
audioProcessing?.processAudio(numBands, numFrames, buffer!!)
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
private fun AudioProcessorInterface?.toAudioProcessing(): ExternalAudioProcessingFactory.AudioProcessing {
|
|
76
|
+
return AudioProcessingBridge(this)
|
|
77
|
+
}
|
|
78
|
+
}
|