lq-audio-studio 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/build.gradle +2 -1
- package/android/consumer-rules.pro +1 -0
- package/android/src/main/java/com/aozora/audio/AozoraWebRtcApm.java +36 -0
- package/android/src/main/java/net/siteed/audiostudio/AudioProcessing.kt +95 -0
- package/android/src/main/java/net/siteed/audiostudio/AudioRecorderManager.kt +145 -20
- package/android/src/main/java/net/siteed/audiostudio/Pcm16FrameAccumulator.kt +47 -0
- package/android/src/main/java/net/siteed/audiostudio/RecordingConfig.kt +45 -0
- package/android/src/main/java/net/siteed/audiostudio/VoiceProcessingEffects.kt +16 -5
- package/android/src/main/java/net/siteed/audiostudio/WebRtcAudioFrameProcessor.kt +118 -0
- package/android/src/main/jniLibs/arm64-v8a/libaozora_apm.so +0 -0
- package/android/src/main/webrtc-apm-notices/README.md +151 -0
- package/android/src/main/webrtc-apm-notices/THIRD_PARTY_NOTICES.md +25 -0
- package/android/src/main/webrtc-apm-notices/licenses/AUTHORS +7 -0
- package/android/src/main/webrtc-apm-notices/licenses/COPYING +29 -0
- package/android/src/main/webrtc-apm-notices/licenses/android-ndk/NOTICE +9997 -0
- package/android/src/main/webrtc-apm-notices/licenses/android-ndk/NOTICE.toolchain +15302 -0
- package/android/src/main/webrtc-apm-notices/licenses/subprojects/abseil-cpp-20230125.1/AUTHORS +6 -0
- package/android/src/main/webrtc-apm-notices/licenses/subprojects/abseil-cpp-20230125.1/LICENSE +203 -0
- package/android/src/main/webrtc-apm-notices/licenses/subprojects/abseil-cpp-20230125.1/LICENSE.build +19 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/LICENSE +29 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/PATENTS +24 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/common_audio/third_party/ooura/LICENSE +8 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/common_audio/third_party/spl_sqrt_floor/LICENSE +27 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/modules/third_party/fft/LICENSE +25 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/third_party/pffft/LICENSE +1 -0
- package/android/src/main/webrtc-apm-notices/licenses/webrtc/third_party/rnnoise/COPYING +1 -0
- package/android/src/main/webrtc-apm-notices/licenses/wrapper-LICENSE +27 -0
- package/android/src/main/webrtc-apm-notices/manifest.json +150 -0
- package/build/types/AudioStudio.types.d.ts +35 -1
- package/package.json +1 -1
- package/src/AudioStudio.types.ts +47 -0
package/android/build.gradle
CHANGED
|
@@ -81,6 +81,7 @@ android {
|
|
|
81
81
|
versionCode 1
|
|
82
82
|
versionName "0.1.0"
|
|
83
83
|
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
|
|
84
|
+
consumerProguardFiles "consumer-rules.pro"
|
|
84
85
|
externalNativeBuild {
|
|
85
86
|
cmake {
|
|
86
87
|
cppFlags "-frtti -fexceptions -Wall"
|
|
@@ -140,4 +141,4 @@ kotlin {
|
|
|
140
141
|
jvmToolchain {
|
|
141
142
|
languageVersion.set(JavaLanguageVersion.of(17))
|
|
142
143
|
}
|
|
143
|
-
}
|
|
144
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
-keep class com.aozora.audio.AozoraWebRtcApm { *; }
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
package com.aozora.audio;
|
|
2
|
+
|
|
3
|
+
/** Stable JNI boundary for the bundled WebRTC Audio Processing Module. */
|
|
4
|
+
public final class AozoraWebRtcApm {
|
|
5
|
+
private static final boolean AVAILABLE;
|
|
6
|
+
|
|
7
|
+
static {
|
|
8
|
+
boolean loaded;
|
|
9
|
+
try {
|
|
10
|
+
System.loadLibrary("aozora_apm");
|
|
11
|
+
loaded = true;
|
|
12
|
+
} catch (Throwable error) {
|
|
13
|
+
loaded = false;
|
|
14
|
+
}
|
|
15
|
+
AVAILABLE = loaded;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
private AozoraWebRtcApm() {}
|
|
19
|
+
|
|
20
|
+
public static boolean isAvailable() {
|
|
21
|
+
return AVAILABLE;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
public static native long nativeCreate(
|
|
25
|
+
int sampleRate,
|
|
26
|
+
int channels,
|
|
27
|
+
boolean echoCancellation,
|
|
28
|
+
boolean noiseSuppression,
|
|
29
|
+
boolean gainControl
|
|
30
|
+
);
|
|
31
|
+
public static native int nativeProcessReverse(long handle, short[] samples);
|
|
32
|
+
public static native int nativeProcessCapture(long handle, short[] samples);
|
|
33
|
+
public static native void nativeSetStreamDelay(long handle, int delayMs);
|
|
34
|
+
public static native int nativeGetVoiceDetected(long handle);
|
|
35
|
+
public static native void nativeRelease(long handle);
|
|
36
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
package net.siteed.audiostudio
|
|
2
|
+
|
|
3
|
+
internal enum class AudioProcessingBackend(val wireValue: String) {
|
|
4
|
+
DISABLED("disabled"),
|
|
5
|
+
SYSTEM("system"),
|
|
6
|
+
WEBRTC("webrtc"),
|
|
7
|
+
UNAVAILABLE("unavailable"),
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
internal data class AudioProcessingRequest(
|
|
11
|
+
val echoCancellation: Boolean,
|
|
12
|
+
val noiseSuppression: Boolean,
|
|
13
|
+
val automaticGainControl: Boolean,
|
|
14
|
+
) {
|
|
15
|
+
val anyEnabled: Boolean
|
|
16
|
+
get() = echoCancellation || noiseSuppression || automaticGainControl
|
|
17
|
+
|
|
18
|
+
companion object {
|
|
19
|
+
fun from(config: AndroidAudioProcessingConfig) = AudioProcessingRequest(
|
|
20
|
+
echoCancellation = config.echoCancellation,
|
|
21
|
+
noiseSuppression = config.noiseSuppression,
|
|
22
|
+
automaticGainControl = config.automaticGainControl,
|
|
23
|
+
)
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
internal data class SystemAudioProcessingStatus(
|
|
28
|
+
val aec: Boolean = false,
|
|
29
|
+
val noiseSuppression: Boolean = false,
|
|
30
|
+
val automaticGainControl: Boolean = false,
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
internal data class AudioProcessingPlan(
|
|
34
|
+
val system: AudioProcessingRequest,
|
|
35
|
+
val software: AudioProcessingRequest,
|
|
36
|
+
) {
|
|
37
|
+
companion object {
|
|
38
|
+
fun create(
|
|
39
|
+
config: AndroidAudioProcessingConfig,
|
|
40
|
+
systemStatus: SystemAudioProcessingStatus,
|
|
41
|
+
): AudioProcessingPlan {
|
|
42
|
+
val requested = AudioProcessingRequest.from(config)
|
|
43
|
+
val system = when (config.mode) {
|
|
44
|
+
AndroidAudioProcessingMode.SOFTWARE -> AudioProcessingRequest(false, false, false)
|
|
45
|
+
else -> requested
|
|
46
|
+
}
|
|
47
|
+
val software = when (config.mode) {
|
|
48
|
+
AndroidAudioProcessingMode.SYSTEM -> AudioProcessingRequest(false, false, false)
|
|
49
|
+
AndroidAudioProcessingMode.SOFTWARE -> requested
|
|
50
|
+
AndroidAudioProcessingMode.SYSTEM_WITH_SOFTWARE_FALLBACK -> AudioProcessingRequest(
|
|
51
|
+
echoCancellation = requested.echoCancellation && !systemStatus.aec,
|
|
52
|
+
noiseSuppression = requested.noiseSuppression && !systemStatus.noiseSuppression,
|
|
53
|
+
automaticGainControl = requested.automaticGainControl && !systemStatus.automaticGainControl,
|
|
54
|
+
)
|
|
55
|
+
}
|
|
56
|
+
return AudioProcessingPlan(system = system, software = software)
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
internal data class AudioProcessingStatus(
|
|
62
|
+
val mode: String,
|
|
63
|
+
val aec: AudioProcessingBackend,
|
|
64
|
+
val noiseSuppression: AudioProcessingBackend,
|
|
65
|
+
val automaticGainControl: AudioProcessingBackend,
|
|
66
|
+
val reverseStreamConnected: Boolean,
|
|
67
|
+
val sampleRate: Int,
|
|
68
|
+
val channels: Int,
|
|
69
|
+
val frameDurationMs: Int = 10,
|
|
70
|
+
val compressedOutputProcessed: Boolean = false,
|
|
71
|
+
) {
|
|
72
|
+
fun toMap(): Map<String, Any> = mapOf(
|
|
73
|
+
"mode" to mode,
|
|
74
|
+
"aec" to aec.wireValue,
|
|
75
|
+
"noiseSuppression" to noiseSuppression.wireValue,
|
|
76
|
+
"automaticGainControl" to automaticGainControl.wireValue,
|
|
77
|
+
"reverseStreamConnected" to reverseStreamConnected,
|
|
78
|
+
"sampleRate" to sampleRate,
|
|
79
|
+
"channels" to channels,
|
|
80
|
+
"frameDurationMs" to frameDurationMs,
|
|
81
|
+
"compressedOutputProcessed" to compressedOutputProcessed,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
companion object {
|
|
85
|
+
fun disabled(sampleRate: Int = 0, channels: Int = 0) = AudioProcessingStatus(
|
|
86
|
+
mode = "disabled",
|
|
87
|
+
aec = AudioProcessingBackend.DISABLED,
|
|
88
|
+
noiseSuppression = AudioProcessingBackend.DISABLED,
|
|
89
|
+
automaticGainControl = AudioProcessingBackend.DISABLED,
|
|
90
|
+
reverseStreamConnected = false,
|
|
91
|
+
sampleRate = sampleRate,
|
|
92
|
+
channels = channels,
|
|
93
|
+
)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
@@ -151,6 +151,10 @@ class AudioRecorderManager(
|
|
|
151
151
|
private val audioSourceLifecycle = AudioSourceLifecycle<AudioSourceResolver.Resolution>()
|
|
152
152
|
/** Platform AEC/ANS/AGC attached to the current VOICE_COMMUNICATION AudioRecord session. */
|
|
153
153
|
private var voiceProcessingEffects: VoiceProcessingEffects? = null
|
|
154
|
+
/** WebRTC APM processing only the effects the platform session could not enable. */
|
|
155
|
+
private var softwareAudioProcessor: WebRtcAudioFrameProcessor? = null
|
|
156
|
+
private var audioProcessingConfigInUse: AndroidAudioProcessingConfig? = null
|
|
157
|
+
private var systemAudioProcessingStatus = SystemAudioProcessingStatus()
|
|
154
158
|
|
|
155
159
|
/**
|
|
156
160
|
* The resolved source, for use after the recorders exist. Reading this before resolution
|
|
@@ -1113,6 +1117,7 @@ class AudioRecorderManager(
|
|
|
1113
1117
|
// The source actually opened, which may differ from the request when the
|
|
1114
1118
|
// device does not support it (#428).
|
|
1115
1119
|
"androidAudioSource" to requireAudioSource.name,
|
|
1120
|
+
"audioProcessing" to currentAudioProcessingStatus().toMap(),
|
|
1116
1121
|
"compression" to if (compressedFile != null) bundleOf(
|
|
1117
1122
|
"mimeType" to if (recordingConfig.output.compressed.format == "aac") "audio/aac" else "audio/opus",
|
|
1118
1123
|
"bitrate" to recordingConfig.output.compressed.bitrate,
|
|
@@ -1564,18 +1569,13 @@ class AudioRecorderManager(
|
|
|
1564
1569
|
return false
|
|
1565
1570
|
}
|
|
1566
1571
|
|
|
1567
|
-
//
|
|
1568
|
-
//
|
|
1569
|
-
// AudioFlinger dump showed 0 Effect Chains). Bind each available platform
|
|
1570
|
-
// AEC/ANS/AGC effect to this concrete AudioRecord session so the system has
|
|
1571
|
-
// an explicit chain. A route change or interruption creates a new session
|
|
1572
|
-
// and reaches this block again; teardown releases the previous chain first.
|
|
1572
|
+
// Rebuild the complete processing chain with each AudioRecord session. USB
|
|
1573
|
+
// route changes and interruption recovery replace the Android session ID.
|
|
1573
1574
|
releaseVoiceProcessingEffects()
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
}
|
|
1575
|
+
configureAudioProcessing(
|
|
1576
|
+
currentSourceName = currentSource.name,
|
|
1577
|
+
audioSessionId = checkNotNull(audioRecord).audioSessionId,
|
|
1578
|
+
)
|
|
1579
1579
|
}
|
|
1580
1580
|
return true
|
|
1581
1581
|
|
|
@@ -2174,7 +2174,8 @@ class AudioRecorderManager(
|
|
|
2174
2174
|
"isPaused" to false,
|
|
2175
2175
|
"mime" to mimeType,
|
|
2176
2176
|
"size" to 0,
|
|
2177
|
-
"interval" to if (::recordingConfig.isInitialized) recordingConfig.interval else 0
|
|
2177
|
+
"interval" to if (::recordingConfig.isInitialized) recordingConfig.interval else 0,
|
|
2178
|
+
"audioProcessing" to currentAudioProcessingStatus().toMap(),
|
|
2178
2179
|
)
|
|
2179
2180
|
}
|
|
2180
2181
|
|
|
@@ -2226,6 +2227,7 @@ class AudioRecorderManager(
|
|
|
2226
2227
|
"interval" to recordingConfig.interval,
|
|
2227
2228
|
"maxDurationMs" to if (recordingConfig.maxDurationMs > 0) recordingConfig.maxDurationMs else null,
|
|
2228
2229
|
"maxDurationReached" to maxDurationReached,
|
|
2230
|
+
"audioProcessing" to currentAudioProcessingStatus().toMap(),
|
|
2229
2231
|
"compression" to compressionBundle
|
|
2230
2232
|
)
|
|
2231
2233
|
}
|
|
@@ -2317,6 +2319,9 @@ class AudioRecorderManager(
|
|
|
2317
2319
|
} else {
|
|
2318
2320
|
null
|
|
2319
2321
|
}
|
|
2322
|
+
// Keep the analysis accumulator in the same scope as the APM tail flush.
|
|
2323
|
+
// Otherwise the final partial capture frame cannot be forwarded to analysis.
|
|
2324
|
+
val accumulatedAnalysisData = ByteArrayOutputStream()
|
|
2320
2325
|
|
|
2321
2326
|
try {
|
|
2322
2327
|
// Write audio data directly to the file (if not skipping)
|
|
@@ -2325,7 +2330,6 @@ class AudioRecorderManager(
|
|
|
2325
2330
|
|
|
2326
2331
|
// Buffer to accumulate data
|
|
2327
2332
|
accumulatedAudioData = ByteArrayOutputStream()
|
|
2328
|
-
val accumulatedAnalysisData = ByteArrayOutputStream() // Separate buffer for analysis
|
|
2329
2333
|
audioFileHandler.writeWavHeader(
|
|
2330
2334
|
accumulatedAudioData!!,
|
|
2331
2335
|
recordingConfig.sampleRate,
|
|
@@ -2380,6 +2384,7 @@ class AudioRecorderManager(
|
|
|
2380
2384
|
shouldProcessAnalysis = recordingConfig.enableProcessing &&
|
|
2381
2385
|
(isFirstAnalysis || timeSinceLastAnalysis >= recordingConfig.intervalAnalysis)
|
|
2382
2386
|
|
|
2387
|
+
var processedAudioData: ByteArray? = null
|
|
2383
2388
|
val bytesRead = synchronized(audioRecordLock) {
|
|
2384
2389
|
if (
|
|
2385
2390
|
sessionId != recordingSession ||
|
|
@@ -2429,26 +2434,31 @@ class AudioRecorderManager(
|
|
|
2429
2434
|
// budget left and its next failure silently suppressed.
|
|
2430
2435
|
failedDeliveryAttempts.remove(RecordingErrorKind.INPUT_READ)
|
|
2431
2436
|
failedDeliveryAttempts.remove(RecordingErrorKind.INPUT_STATE)
|
|
2437
|
+
processedAudioData = softwareAudioProcessor
|
|
2438
|
+
?.processCapture(audioData, bytes)
|
|
2439
|
+
?: audioData.copyOf(bytes)
|
|
2432
2440
|
}
|
|
2433
2441
|
}
|
|
2434
2442
|
} ?: -1 // Handle null case
|
|
2435
2443
|
}
|
|
2436
2444
|
|
|
2437
|
-
|
|
2445
|
+
val processedChunk = processedAudioData
|
|
2446
|
+
if (bytesRead > 0 && processedChunk != null && processedChunk.isNotEmpty()) {
|
|
2447
|
+
val processedBytesRead = processedChunk.size
|
|
2438
2448
|
// Only write to file if primary output is enabled
|
|
2439
2449
|
if (fos != null) {
|
|
2440
|
-
fos.write(
|
|
2441
|
-
cachedPrimaryFileSize +=
|
|
2450
|
+
fos.write(processedChunk, 0, processedBytesRead)
|
|
2451
|
+
cachedPrimaryFileSize += processedBytesRead // Update cached file size
|
|
2442
2452
|
}
|
|
2443
|
-
totalDataSize +=
|
|
2453
|
+
totalDataSize += processedBytesRead
|
|
2444
2454
|
|
|
2445
|
-
accumulatedAudioData?.write(
|
|
2455
|
+
accumulatedAudioData?.write(processedChunk, 0, processedBytesRead)
|
|
2446
2456
|
|
|
2447
2457
|
// Always accumulate data for analysis if enabled (moved outside shouldProcessAnalysis check)
|
|
2448
2458
|
if (recordingConfig.enableProcessing) {
|
|
2449
2459
|
// Check buffer size to prevent OOM on low-RAM devices with extreme configs
|
|
2450
|
-
if (accumulatedAnalysisData.size() +
|
|
2451
|
-
accumulatedAnalysisData.write(
|
|
2460
|
+
if (accumulatedAnalysisData.size() + processedBytesRead <= MAX_ANALYSIS_BUFFER_SIZE) {
|
|
2461
|
+
accumulatedAnalysisData.write(processedChunk, 0, processedBytesRead)
|
|
2452
2462
|
} else {
|
|
2453
2463
|
LogUtils.w(CLASS_NAME, "Analysis buffer size limit reached (${accumulatedAnalysisData.size()} bytes). Skipping data to prevent OOM.")
|
|
2454
2464
|
}
|
|
@@ -2521,6 +2531,23 @@ class AudioRecorderManager(
|
|
|
2521
2531
|
}
|
|
2522
2532
|
}
|
|
2523
2533
|
} finally {
|
|
2534
|
+
// APM retains fewer than 10 ms while completing its next frame. Preserve
|
|
2535
|
+
// that raw tail on stop instead of shortening the recording or padding it.
|
|
2536
|
+
val processingTail = synchronized(audioRecordLock) {
|
|
2537
|
+
softwareAudioProcessor?.flushCapture() ?: ByteArray(0)
|
|
2538
|
+
}
|
|
2539
|
+
if (processingTail.isNotEmpty()) {
|
|
2540
|
+
fos?.write(processingTail)
|
|
2541
|
+
if (fos != null) cachedPrimaryFileSize += processingTail.size
|
|
2542
|
+
totalDataSize += processingTail.size
|
|
2543
|
+
accumulatedAudioData?.write(processingTail)
|
|
2544
|
+
if (
|
|
2545
|
+
recordingConfig.enableProcessing &&
|
|
2546
|
+
accumulatedAnalysisData.size() + processingTail.size <= MAX_ANALYSIS_BUFFER_SIZE
|
|
2547
|
+
) {
|
|
2548
|
+
accumulatedAnalysisData.write(processingTail)
|
|
2549
|
+
}
|
|
2550
|
+
}
|
|
2524
2551
|
// Flush and close the file output stream if it was opened
|
|
2525
2552
|
try {
|
|
2526
2553
|
fos?.flush()
|
|
@@ -2948,6 +2975,104 @@ class AudioRecorderManager(
|
|
|
2948
2975
|
private fun releaseVoiceProcessingEffects() {
|
|
2949
2976
|
voiceProcessingEffects?.release()
|
|
2950
2977
|
voiceProcessingEffects = null
|
|
2978
|
+
softwareAudioProcessor?.release()
|
|
2979
|
+
softwareAudioProcessor = null
|
|
2980
|
+
audioProcessingConfigInUse = null
|
|
2981
|
+
systemAudioProcessingStatus = SystemAudioProcessingStatus()
|
|
2982
|
+
}
|
|
2983
|
+
|
|
2984
|
+
private fun configureAudioProcessing(currentSourceName: String, audioSessionId: Int) {
|
|
2985
|
+
val config = recordingConfig.audioProcessing
|
|
2986
|
+
?: if (currentSourceName == "voiceCommunication") AndroidAudioProcessingConfig() else null
|
|
2987
|
+
if (config == null) return
|
|
2988
|
+
|
|
2989
|
+
audioProcessingConfigInUse = config
|
|
2990
|
+
val requested = AudioProcessingRequest.from(config)
|
|
2991
|
+
if (config.mode != AndroidAudioProcessingMode.SOFTWARE && requested.anyEnabled) {
|
|
2992
|
+
voiceProcessingEffects = VoiceProcessingEffects.attach(audioSessionId, requested)
|
|
2993
|
+
systemAudioProcessingStatus = checkNotNull(voiceProcessingEffects).status
|
|
2994
|
+
}
|
|
2995
|
+
val plan = AudioProcessingPlan.create(config, systemAudioProcessingStatus)
|
|
2996
|
+
softwareAudioProcessor = WebRtcAudioFrameProcessor.create(
|
|
2997
|
+
sampleRate = recordingConfig.sampleRate,
|
|
2998
|
+
channels = recordingConfig.channels,
|
|
2999
|
+
encoding = recordingConfig.encoding,
|
|
3000
|
+
request = plan.software,
|
|
3001
|
+
)
|
|
3002
|
+
val status = currentAudioProcessingStatus()
|
|
3003
|
+
LogUtils.i(
|
|
3004
|
+
CLASS_NAME,
|
|
3005
|
+
"3A selected mode=${status.mode} AEC=${status.aec.wireValue} " +
|
|
3006
|
+
"NS=${status.noiseSuppression.wireValue} AGC=${status.automaticGainControl.wireValue} " +
|
|
3007
|
+
"reverseStreamConnected=${status.reverseStreamConnected}"
|
|
3008
|
+
)
|
|
3009
|
+
}
|
|
3010
|
+
|
|
3011
|
+
private fun currentAudioProcessingStatus(): AudioProcessingStatus {
|
|
3012
|
+
val config = audioProcessingConfigInUse
|
|
3013
|
+
?: return AudioProcessingStatus.disabled(
|
|
3014
|
+
sampleRate = if (::recordingConfig.isInitialized) recordingConfig.sampleRate else 0,
|
|
3015
|
+
channels = if (::recordingConfig.isInitialized) recordingConfig.channels else 0,
|
|
3016
|
+
)
|
|
3017
|
+
val requested = AudioProcessingRequest.from(config)
|
|
3018
|
+
val software = softwareAudioProcessor
|
|
3019
|
+
val softwareRequest = software?.request
|
|
3020
|
+
val reverseStreamConnected = software?.reverseStreamConnected == true
|
|
3021
|
+
|
|
3022
|
+
fun backend(
|
|
3023
|
+
requestedEffect: Boolean,
|
|
3024
|
+
systemEnabled: Boolean,
|
|
3025
|
+
softwareEnabled: Boolean,
|
|
3026
|
+
): AudioProcessingBackend = when {
|
|
3027
|
+
!requestedEffect -> AudioProcessingBackend.DISABLED
|
|
3028
|
+
systemEnabled -> AudioProcessingBackend.SYSTEM
|
|
3029
|
+
software != null && software.operational && softwareEnabled -> AudioProcessingBackend.WEBRTC
|
|
3030
|
+
else -> AudioProcessingBackend.UNAVAILABLE
|
|
3031
|
+
}
|
|
3032
|
+
|
|
3033
|
+
return AudioProcessingStatus(
|
|
3034
|
+
mode = config.mode.wireValue,
|
|
3035
|
+
aec = backend(
|
|
3036
|
+
requested.echoCancellation,
|
|
3037
|
+
systemAudioProcessingStatus.aec,
|
|
3038
|
+
// Creating AEC3 is not evidence that echo cancellation is active.
|
|
3039
|
+
// It only becomes effective after real speaker PCM reaches the reverse stream.
|
|
3040
|
+
softwareRequest?.echoCancellation == true && reverseStreamConnected,
|
|
3041
|
+
),
|
|
3042
|
+
noiseSuppression = backend(
|
|
3043
|
+
requested.noiseSuppression,
|
|
3044
|
+
systemAudioProcessingStatus.noiseSuppression,
|
|
3045
|
+
softwareRequest?.noiseSuppression == true,
|
|
3046
|
+
),
|
|
3047
|
+
automaticGainControl = backend(
|
|
3048
|
+
requested.automaticGainControl,
|
|
3049
|
+
systemAudioProcessingStatus.automaticGainControl,
|
|
3050
|
+
softwareRequest?.automaticGainControl == true,
|
|
3051
|
+
),
|
|
3052
|
+
reverseStreamConnected = reverseStreamConnected,
|
|
3053
|
+
sampleRate = recordingConfig.sampleRate,
|
|
3054
|
+
channels = recordingConfig.channels,
|
|
3055
|
+
)
|
|
3056
|
+
}
|
|
3057
|
+
|
|
3058
|
+
/**
|
|
3059
|
+
* A native playback path can feed exact speaker PCM here. A JS/Base64 10 ms API is
|
|
3060
|
+
* intentionally not exposed because scheduler jitter makes AEC delay unreliable.
|
|
3061
|
+
*/
|
|
3062
|
+
internal fun processPlaybackReference(
|
|
3063
|
+
pcm16: ByteArray,
|
|
3064
|
+
sampleRate: Int,
|
|
3065
|
+
channels: Int,
|
|
3066
|
+
delayMs: Int,
|
|
3067
|
+
): Boolean = synchronized(audioRecordLock) {
|
|
3068
|
+
if (
|
|
3069
|
+
!::recordingConfig.isInitialized ||
|
|
3070
|
+
sampleRate != recordingConfig.sampleRate ||
|
|
3071
|
+
channels != recordingConfig.channels
|
|
3072
|
+
) {
|
|
3073
|
+
return@synchronized false
|
|
3074
|
+
}
|
|
3075
|
+
softwareAudioProcessor?.processReverse(pcm16, pcm16.size, delayMs) == true
|
|
2951
3076
|
}
|
|
2952
3077
|
|
|
2953
3078
|
@RequiresApi(Build.VERSION_CODES.Q)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
package net.siteed.audiostudio
|
|
2
|
+
|
|
3
|
+
import java.io.ByteArrayOutputStream
|
|
4
|
+
|
|
5
|
+
/** Converts arbitrary PCM16 read sizes into exact fixed-size frames without reordering data. */
|
|
6
|
+
internal class Pcm16FrameAccumulator(private val frameBytes: Int) {
|
|
7
|
+
init {
|
|
8
|
+
require(frameBytes > 0 && frameBytes % 2 == 0) { "PCM16 frame size must be positive and even" }
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
private var pending = ByteArray(0)
|
|
12
|
+
|
|
13
|
+
@Synchronized
|
|
14
|
+
fun consume(
|
|
15
|
+
data: ByteArray,
|
|
16
|
+
length: Int,
|
|
17
|
+
transform: (ByteArray) -> ByteArray,
|
|
18
|
+
): ByteArray {
|
|
19
|
+
require(length in 0..data.size) { "Invalid PCM byte count" }
|
|
20
|
+
if (length == 0) return ByteArray(0)
|
|
21
|
+
|
|
22
|
+
val combined = ByteArray(pending.size + length)
|
|
23
|
+
pending.copyInto(combined)
|
|
24
|
+
data.copyInto(combined, destinationOffset = pending.size, endIndex = length)
|
|
25
|
+
|
|
26
|
+
val completeBytes = combined.size - (combined.size % frameBytes)
|
|
27
|
+
if (completeBytes == 0) {
|
|
28
|
+
pending = combined
|
|
29
|
+
return ByteArray(0)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
val output = ByteArrayOutputStream(completeBytes)
|
|
33
|
+
var offset = 0
|
|
34
|
+
while (offset < completeBytes) {
|
|
35
|
+
val frame = combined.copyOfRange(offset, offset + frameBytes)
|
|
36
|
+
val transformed = transform(frame)
|
|
37
|
+
require(transformed.size == frameBytes) { "PCM frame transform changed frame size" }
|
|
38
|
+
output.write(transformed)
|
|
39
|
+
offset += frameBytes
|
|
40
|
+
}
|
|
41
|
+
pending = combined.copyOfRange(completeBytes, combined.size)
|
|
42
|
+
return output.toByteArray()
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
@Synchronized
|
|
46
|
+
fun drain(): ByteArray = pending.also { pending = ByteArray(0) }
|
|
47
|
+
}
|
|
@@ -4,6 +4,43 @@ import android.media.AudioFormat
|
|
|
4
4
|
import android.os.Build
|
|
5
5
|
import java.io.File
|
|
6
6
|
|
|
7
|
+
enum class AndroidAudioProcessingMode(val wireValue: String) {
|
|
8
|
+
SYSTEM("system"),
|
|
9
|
+
SYSTEM_WITH_SOFTWARE_FALLBACK("systemWithSoftwareFallback"),
|
|
10
|
+
SOFTWARE("software");
|
|
11
|
+
|
|
12
|
+
companion object {
|
|
13
|
+
fun fromWireValue(value: String): AndroidAudioProcessingMode =
|
|
14
|
+
entries.firstOrNull { it.wireValue == value }
|
|
15
|
+
?: throw IllegalArgumentException(
|
|
16
|
+
"android.audioProcessing.mode must be system, systemWithSoftwareFallback, or software"
|
|
17
|
+
)
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
data class AndroidAudioProcessingConfig(
|
|
22
|
+
val mode: AndroidAudioProcessingMode = AndroidAudioProcessingMode.SYSTEM_WITH_SOFTWARE_FALLBACK,
|
|
23
|
+
val echoCancellation: Boolean = true,
|
|
24
|
+
val noiseSuppression: Boolean = true,
|
|
25
|
+
val automaticGainControl: Boolean = true,
|
|
26
|
+
) {
|
|
27
|
+
companion object {
|
|
28
|
+
fun fromValue(value: Any?): AndroidAudioProcessingConfig? {
|
|
29
|
+
if (value == null) return null
|
|
30
|
+
val map = value as? Map<*, *>
|
|
31
|
+
?: throw IllegalArgumentException("android.audioProcessing must be an object")
|
|
32
|
+
val mode = (map["mode"] as? String)?.let(AndroidAudioProcessingMode::fromWireValue)
|
|
33
|
+
?: AndroidAudioProcessingMode.SYSTEM_WITH_SOFTWARE_FALLBACK
|
|
34
|
+
return AndroidAudioProcessingConfig(
|
|
35
|
+
mode = mode,
|
|
36
|
+
echoCancellation = map["echoCancellation"] as? Boolean ?: true,
|
|
37
|
+
noiseSuppression = map["noiseSuppression"] as? Boolean ?: true,
|
|
38
|
+
automaticGainControl = map["automaticGainControl"] as? Boolean ?: true,
|
|
39
|
+
)
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
7
44
|
// New output configuration structure
|
|
8
45
|
data class OutputConfig(
|
|
9
46
|
val primary: PrimaryOutput = PrimaryOutput(),
|
|
@@ -67,6 +104,8 @@ data class RecordingConfig(
|
|
|
67
104
|
val audioFocusStrategy: String? = null,
|
|
68
105
|
/** Requested MediaRecorder.AudioSource: mic, voiceRecognition, voiceCommunication, unprocessed, auto. */
|
|
69
106
|
val audioSource: String? = null,
|
|
107
|
+
/** Android platform 3A and WebRTC APM fallback policy. */
|
|
108
|
+
val audioProcessing: AndroidAudioProcessingConfig? = null,
|
|
70
109
|
val bufferDurationSeconds: Double? = null,
|
|
71
110
|
val streamFormat: String = "raw",
|
|
72
111
|
val maxDurationMs: Long = 0L,
|
|
@@ -148,6 +187,11 @@ data class RecordingConfig(
|
|
|
148
187
|
val androidConfig = options["android"] as? Map<String, Any>
|
|
149
188
|
val audioFocusStrategy = androidConfig?.get("audioFocusStrategy") as? String
|
|
150
189
|
val audioSource = androidConfig?.get("audioSource") as? String
|
|
190
|
+
val audioProcessing = try {
|
|
191
|
+
AndroidAudioProcessingConfig.fromValue(androidConfig?.get("audioProcessing"))
|
|
192
|
+
} catch (error: IllegalArgumentException) {
|
|
193
|
+
return Result.failure(error)
|
|
194
|
+
}
|
|
151
195
|
|
|
152
196
|
// Initialize the recording configuration with cleaned directory path
|
|
153
197
|
val tempRecordingConfig = RecordingConfig(
|
|
@@ -176,6 +220,7 @@ data class RecordingConfig(
|
|
|
176
220
|
deviceDisconnectionBehavior = deviceDisconnectionBehavior,
|
|
177
221
|
audioFocusStrategy = audioFocusStrategy,
|
|
178
222
|
audioSource = audioSource,
|
|
223
|
+
audioProcessing = audioProcessing,
|
|
179
224
|
bufferDurationSeconds = (options["bufferDurationSeconds"] as? Number)?.toDouble(),
|
|
180
225
|
streamFormat = options.getStringOrDefault("streamFormat", "raw"),
|
|
181
226
|
maxDurationMs = options.getNumberOrDefault("maxDurationMs", 0L),
|
|
@@ -16,6 +16,7 @@ import android.media.audiofx.NoiseSuppressor
|
|
|
16
16
|
*/
|
|
17
17
|
internal class VoiceProcessingEffects private constructor(
|
|
18
18
|
private val effects: List<AudioEffect>,
|
|
19
|
+
val status: SystemAudioProcessingStatus,
|
|
19
20
|
) {
|
|
20
21
|
fun release() {
|
|
21
22
|
effects.asReversed().forEach { effect ->
|
|
@@ -30,20 +31,23 @@ internal class VoiceProcessingEffects private constructor(
|
|
|
30
31
|
companion object {
|
|
31
32
|
private const val CLASS_NAME = "VoiceProcessingEffects"
|
|
32
33
|
|
|
33
|
-
fun attach(
|
|
34
|
+
fun attach(
|
|
35
|
+
audioSessionId: Int,
|
|
36
|
+
request: AudioProcessingRequest = AudioProcessingRequest(true, true, true),
|
|
37
|
+
): VoiceProcessingEffects {
|
|
34
38
|
val attached = mutableListOf<AudioEffect>()
|
|
35
39
|
|
|
36
|
-
attachEffect(
|
|
40
|
+
if (request.echoCancellation) attachEffect(
|
|
37
41
|
label = "AEC",
|
|
38
42
|
available = AcousticEchoCanceler.isAvailable(),
|
|
39
43
|
create = { AcousticEchoCanceler.create(audioSessionId) },
|
|
40
44
|
)?.let(attached::add)
|
|
41
|
-
attachEffect(
|
|
45
|
+
if (request.noiseSuppression) attachEffect(
|
|
42
46
|
label = "ANS",
|
|
43
47
|
available = NoiseSuppressor.isAvailable(),
|
|
44
48
|
create = { NoiseSuppressor.create(audioSessionId) },
|
|
45
49
|
)?.let(attached::add)
|
|
46
|
-
attachEffect(
|
|
50
|
+
if (request.automaticGainControl) attachEffect(
|
|
47
51
|
label = "AGC",
|
|
48
52
|
available = AutomaticGainControl.isAvailable(),
|
|
49
53
|
create = { AutomaticGainControl.create(audioSessionId) },
|
|
@@ -56,7 +60,14 @@ internal class VoiceProcessingEffects private constructor(
|
|
|
56
60
|
"ANS=${hasType(attached, NoiseSuppressor::class.java)}, " +
|
|
57
61
|
"AGC=${hasType(attached, AutomaticGainControl::class.java)})"
|
|
58
62
|
)
|
|
59
|
-
return VoiceProcessingEffects(
|
|
63
|
+
return VoiceProcessingEffects(
|
|
64
|
+
effects = attached,
|
|
65
|
+
status = SystemAudioProcessingStatus(
|
|
66
|
+
aec = hasType(attached, AcousticEchoCanceler::class.java),
|
|
67
|
+
noiseSuppression = hasType(attached, NoiseSuppressor::class.java),
|
|
68
|
+
automaticGainControl = hasType(attached, AutomaticGainControl::class.java),
|
|
69
|
+
)
|
|
70
|
+
)
|
|
60
71
|
}
|
|
61
72
|
|
|
62
73
|
private fun attachEffect(
|