@siteed/audio-studio 3.1.0 → 3.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -1
- package/README.md +97 -50
- package/android/src/androidTest/java/net/siteed/audiostudio/AudioFinalMetadataContractInstrumentedTest.kt +190 -0
- package/android/src/androidTest/java/net/siteed/audiostudio/AudioRecorderInstrumentedTest.kt +29 -83
- package/android/src/androidTest/java/net/siteed/audiostudio/AudioRecorderPerformanceInstrumentedTest.kt +17 -1
- package/android/src/androidTest/java/net/siteed/audiostudio/OpusRangeDecodeRegressionInstrumentedTest.kt +186 -0
- package/android/src/main/java/net/siteed/audiostudio/AudioProcessor.kt +473 -380
- package/android/src/main/java/net/siteed/audiostudio/AudioStreamDecoder.kt +640 -0
- package/android/src/main/java/net/siteed/audiostudio/AudioStudioModule.kt +187 -13
- package/android/src/main/java/net/siteed/audiostudio/AudioTrimmer.kt +174 -212
- package/android/src/main/java/net/siteed/audiostudio/Constants.kt +4 -0
- package/build/cjs/AudioAnalysis/AudioAnalysis.types.js.map +1 -1
- package/build/cjs/AudioAnalysis/extractPreview.js +92 -15
- package/build/cjs/AudioAnalysis/extractPreview.js.map +1 -1
- package/build/cjs/AudioAnalysis/extractPreviewBars.js +134 -0
- package/build/cjs/AudioAnalysis/extractPreviewBars.js.map +1 -0
- package/build/cjs/errors/AudioExtractionError.js +127 -0
- package/build/cjs/errors/AudioExtractionError.js.map +1 -0
- package/build/cjs/errors/AudioStreamError.js +152 -0
- package/build/cjs/errors/AudioStreamError.js.map +1 -0
- package/build/cjs/errors/AudioStreamError.test.js +61 -0
- package/build/cjs/errors/AudioStreamError.test.js.map +1 -0
- package/build/cjs/index.js +12 -1
- package/build/cjs/index.js.map +1 -1
- package/build/cjs/streamAudioData.js +467 -0
- package/build/cjs/streamAudioData.js.map +1 -0
- package/build/esm/AudioAnalysis/AudioAnalysis.types.js.map +1 -1
- package/build/esm/AudioAnalysis/extractPreview.js +92 -15
- package/build/esm/AudioAnalysis/extractPreview.js.map +1 -1
- package/build/esm/AudioAnalysis/extractPreviewBars.js +128 -0
- package/build/esm/AudioAnalysis/extractPreviewBars.js.map +1 -0
- package/build/esm/errors/AudioExtractionError.js +122 -0
- package/build/esm/errors/AudioExtractionError.js.map +1 -0
- package/build/esm/errors/AudioStreamError.js +147 -0
- package/build/esm/errors/AudioStreamError.js.map +1 -0
- package/build/esm/errors/AudioStreamError.test.js +59 -0
- package/build/esm/errors/AudioStreamError.test.js.map +1 -0
- package/build/esm/index.js +5 -1
- package/build/esm/index.js.map +1 -1
- package/build/esm/streamAudioData.js +460 -0
- package/build/esm/streamAudioData.js.map +1 -0
- package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts +79 -0
- package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts.map +1 -1
- package/build/types/AudioAnalysis/extractPreview.d.ts +2 -2
- package/build/types/AudioAnalysis/extractPreview.d.ts.map +1 -1
- package/build/types/AudioAnalysis/extractPreviewBars.d.ts +12 -0
- package/build/types/AudioAnalysis/extractPreviewBars.d.ts.map +1 -0
- package/build/types/errors/AudioExtractionError.d.ts +24 -0
- package/build/types/errors/AudioExtractionError.d.ts.map +1 -0
- package/build/types/errors/AudioStreamError.d.ts +25 -0
- package/build/types/errors/AudioStreamError.d.ts.map +1 -0
- package/build/types/errors/AudioStreamError.test.d.ts +2 -0
- package/build/types/errors/AudioStreamError.test.d.ts.map +1 -0
- package/build/types/index.d.ts +8 -1
- package/build/types/index.d.ts.map +1 -1
- package/build/types/streamAudioData.d.ts +114 -0
- package/build/types/streamAudioData.d.ts.map +1 -0
- package/ios/AudioProcessingHelpers.swift +10 -5
- package/ios/AudioProcessor.swift +99 -0
- package/ios/AudioStreamDecoder.swift +523 -0
- package/ios/AudioStudioModule.swift +210 -3
- package/ios/AudioStudioTests/AudioStreamDecoderTests.swift +128 -0
- package/package.json +7 -7
- package/src/AudioAnalysis/AudioAnalysis.types.ts +82 -0
- package/src/AudioAnalysis/extractPreview.ts +118 -17
- package/src/AudioAnalysis/extractPreviewBars.ts +193 -0
- package/src/errors/AudioExtractionError.ts +167 -0
- package/src/errors/AudioStreamError.test.ts +65 -0
- package/src/errors/AudioStreamError.ts +185 -0
- package/src/index.ts +34 -0
- package/src/streamAudioData.ts +654 -0
|
@@ -9,6 +9,10 @@ private let recordingInterruptedEvent: String = "onRecordingInterrupted"
|
|
|
9
9
|
private let deviceChangedEvent: String = "deviceChangedEvent"
|
|
10
10
|
private let trimProgressEvent: String = "TrimProgress"
|
|
11
11
|
private let errorEvent: String = "error"
|
|
12
|
+
private let audioStreamChunkEvent: String = "AudioDataStreamChunk"
|
|
13
|
+
private let audioStreamProgressEvent: String = "AudioDataStreamProgress"
|
|
14
|
+
private let audioStreamCompleteEvent: String = "AudioDataStreamComplete"
|
|
15
|
+
private let audioStreamErrorEvent: String = "AudioDataStreamError"
|
|
12
16
|
private let DEFAULT_SEGMENT_DURATION_MS = 100
|
|
13
17
|
private let audioDeviceTypeBuiltinMic = "builtin_mic"
|
|
14
18
|
private let audioDeviceTypeBluetooth = "bluetooth"
|
|
@@ -18,13 +22,16 @@ private let audioDeviceTypeWiredHeadphones = "wired_headphones"
|
|
|
18
22
|
private let audioDeviceTypeSpeaker = "speaker"
|
|
19
23
|
private let audioDeviceTypeUnknown = "unknown"
|
|
20
24
|
|
|
21
|
-
public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceManagerDelegate {
|
|
25
|
+
public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceManagerDelegate, AudioStreamDecoderDelegate {
|
|
22
26
|
private var streamManager = AudioStreamManager()
|
|
23
27
|
private let notificationCenter = UNUserNotificationCenter.current()
|
|
24
28
|
private let notificationIdentifier = "audio_recording_notification"
|
|
25
29
|
private var deviceManager = AudioDeviceManager()
|
|
26
30
|
private var deviceChangeObserver: Any?
|
|
27
31
|
|
|
32
|
+
private let streamDecodersLock = NSLock()
|
|
33
|
+
private var streamDecoders: [String: AudioStreamDecoder] = [:]
|
|
34
|
+
|
|
28
35
|
// Serial queue for AVAudioEngine lifecycle ops (prepare/start/stop).
|
|
29
36
|
// Prevents concurrent mutation of shared engine state and keeps callers
|
|
30
37
|
// off the main thread to avoid UI freezes during heavy native init.
|
|
@@ -43,7 +50,11 @@ public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceM
|
|
|
43
50
|
recordingInterruptedEvent,
|
|
44
51
|
deviceChangedEvent,
|
|
45
52
|
trimProgressEvent,
|
|
46
|
-
errorEvent
|
|
53
|
+
errorEvent,
|
|
54
|
+
audioStreamChunkEvent,
|
|
55
|
+
audioStreamProgressEvent,
|
|
56
|
+
audioStreamCompleteEvent,
|
|
57
|
+
audioStreamErrorEvent
|
|
47
58
|
])
|
|
48
59
|
|
|
49
60
|
OnCreate {
|
|
@@ -163,6 +174,10 @@ public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceM
|
|
|
163
174
|
}
|
|
164
175
|
})
|
|
165
176
|
}
|
|
177
|
+
AsyncFunction("extractPreviewBars") { (options: [String: Any], promise: Promise) in
|
|
178
|
+
extractPreviewBars(options: options, promise: promise)
|
|
179
|
+
}
|
|
180
|
+
|
|
166
181
|
|
|
167
182
|
|
|
168
183
|
/// Asynchronously starts audio recording with the given settings.
|
|
@@ -897,8 +912,141 @@ public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceM
|
|
|
897
912
|
}
|
|
898
913
|
}
|
|
899
914
|
}
|
|
915
|
+
|
|
916
|
+
AsyncFunction("streamAudioData") { (options: [String: Any], promise: Promise) in
|
|
917
|
+
guard let requestId = options["requestId"] as? String,
|
|
918
|
+
let fileUri = options["fileUri"] as? String else {
|
|
919
|
+
promise.reject(
|
|
920
|
+
"ERR_AUDIO_STREAM_INVALID_RANGE",
|
|
921
|
+
"fileUri and requestId are required"
|
|
922
|
+
)
|
|
923
|
+
return
|
|
924
|
+
}
|
|
925
|
+
|
|
926
|
+
let streamFormat = options["streamFormat"] as? String ?? "float32"
|
|
927
|
+
guard streamFormat == "float32" else {
|
|
928
|
+
promise.reject(
|
|
929
|
+
"ERR_AUDIO_STREAM_UNSUPPORTED_FORMAT",
|
|
930
|
+
"Only streamFormat='float32' is supported"
|
|
931
|
+
)
|
|
932
|
+
return
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
let opts = AudioStreamDecoder.Options(
|
|
936
|
+
requestId: requestId,
|
|
937
|
+
fileUri: fileUri,
|
|
938
|
+
startTimeMs: options["startTimeMs"] as? Double,
|
|
939
|
+
endTimeMs: options["endTimeMs"] as? Double,
|
|
940
|
+
targetSampleRate: options["targetSampleRate"] as? Double
|
|
941
|
+
?? (options["sampleRate"] as? Double),
|
|
942
|
+
channels: options["channels"] as? Int,
|
|
943
|
+
normalizeAudio: options["normalizeAudio"] as? Bool ?? true,
|
|
944
|
+
chunkDurationMs: options["chunkDurationMs"] as? Int ?? 1000,
|
|
945
|
+
maxChunkBytes: options["maxChunkBytes"] as? Int,
|
|
946
|
+
maxBufferedChunks: options["maxBufferedChunks"] as? Int ?? 4
|
|
947
|
+
)
|
|
948
|
+
|
|
949
|
+
let decoder = AudioStreamDecoder(options: opts)
|
|
950
|
+
decoder.delegate = self
|
|
951
|
+
self.streamDecodersLock.lock()
|
|
952
|
+
if self.streamDecoders[requestId] != nil {
|
|
953
|
+
self.streamDecodersLock.unlock()
|
|
954
|
+
promise.reject(
|
|
955
|
+
"ERR_AUDIO_STREAM_BUSY",
|
|
956
|
+
"requestId already in use"
|
|
957
|
+
)
|
|
958
|
+
return
|
|
959
|
+
}
|
|
960
|
+
self.streamDecoders[requestId] = decoder
|
|
961
|
+
self.streamDecodersLock.unlock()
|
|
962
|
+
decoder.start()
|
|
963
|
+
promise.resolve(["requestId": requestId])
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
AsyncFunction("cancelStreamAudioData") { (requestId: String, promise: Promise) in
|
|
967
|
+
self.streamDecodersLock.lock()
|
|
968
|
+
let decoder = self.streamDecoders[requestId]
|
|
969
|
+
self.streamDecodersLock.unlock()
|
|
970
|
+
decoder?.cancel()
|
|
971
|
+
promise.resolve(["requestId": requestId, "cancelled": decoder != nil])
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
Function("acknowledgeStreamAudioChunk") { (requestId: String, chunkIndex: Int) in
|
|
975
|
+
self.streamDecodersLock.lock()
|
|
976
|
+
let decoder = self.streamDecoders[requestId]
|
|
977
|
+
self.streamDecodersLock.unlock()
|
|
978
|
+
decoder?.acknowledgeChunk(chunkIndex)
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
AsyncFunction("getAudioDecodeCapabilities") { (promise: Promise) in
|
|
982
|
+
promise.resolve([
|
|
983
|
+
"platform": "ios",
|
|
984
|
+
"supportedInputFormats": [
|
|
985
|
+
"audio/wav",
|
|
986
|
+
"audio/aac",
|
|
987
|
+
"audio/mp4",
|
|
988
|
+
"audio/mpeg",
|
|
989
|
+
"audio/x-m4a",
|
|
990
|
+
"audio/caf",
|
|
991
|
+
"audio/aiff",
|
|
992
|
+
],
|
|
993
|
+
"supportedOutputFormats": ["float32"],
|
|
994
|
+
"supportsCancellation": true,
|
|
995
|
+
"supportsBackpressure": true,
|
|
996
|
+
"supportsTimeRange": true,
|
|
997
|
+
"supportsTargetSampleRate": true,
|
|
998
|
+
"supportsChannelMixing": true,
|
|
999
|
+
"knownLimitations": [
|
|
1000
|
+
"Opus/WebM input depends on AVFoundation codec availability for the iOS version."
|
|
1001
|
+
],
|
|
1002
|
+
])
|
|
1003
|
+
}
|
|
900
1004
|
}
|
|
901
|
-
|
|
1005
|
+
|
|
1006
|
+
private func releaseStreamDecoder(_ requestId: String) {
|
|
1007
|
+
streamDecodersLock.lock()
|
|
1008
|
+
streamDecoders.removeValue(forKey: requestId)
|
|
1009
|
+
streamDecodersLock.unlock()
|
|
1010
|
+
}
|
|
1011
|
+
|
|
1012
|
+
// MARK: - AudioStreamDecoderDelegate
|
|
1013
|
+
|
|
1014
|
+
public func streamDecoder(
|
|
1015
|
+
_ decoder: AudioStreamDecoder,
|
|
1016
|
+
didEmitChunk payload: [String: Any]
|
|
1017
|
+
) {
|
|
1018
|
+
sendEvent(audioStreamChunkEvent, payload)
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
public func streamDecoder(
|
|
1022
|
+
_ decoder: AudioStreamDecoder,
|
|
1023
|
+
didReportProgress payload: [String: Any]
|
|
1024
|
+
) {
|
|
1025
|
+
sendEvent(audioStreamProgressEvent, payload)
|
|
1026
|
+
}
|
|
1027
|
+
|
|
1028
|
+
public func streamDecoder(
|
|
1029
|
+
_ decoder: AudioStreamDecoder,
|
|
1030
|
+
didCompleteWith payload: [String: Any]
|
|
1031
|
+
) {
|
|
1032
|
+
if let requestId = payload["requestId"] as? String {
|
|
1033
|
+
releaseStreamDecoder(requestId)
|
|
1034
|
+
}
|
|
1035
|
+
sendEvent(audioStreamCompleteEvent, payload)
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
public func streamDecoder(
|
|
1039
|
+
_ decoder: AudioStreamDecoder,
|
|
1040
|
+
didFailWith payload: [String: Any]
|
|
1041
|
+
) {
|
|
1042
|
+
if let requestId = payload["requestId"] as? String,
|
|
1043
|
+
let code = payload["code"] as? String,
|
|
1044
|
+
code != "ERR_AUDIO_STREAM_CANCELLED" {
|
|
1045
|
+
releaseStreamDecoder(requestId)
|
|
1046
|
+
}
|
|
1047
|
+
sendEvent(audioStreamErrorEvent, payload)
|
|
1048
|
+
}
|
|
1049
|
+
|
|
902
1050
|
func audioStreamManager(_ manager: AudioStreamManager, didReceiveInterruption info: [String: Any]) {
|
|
903
1051
|
Logger.debug("AudioStudioModule", "Delegate: didReceiveInterruption: \(info)")
|
|
904
1052
|
// Convert iOS interruption events to match the TypeScript types
|
|
@@ -1052,6 +1200,65 @@ public class AudioStudioModule: Module, AudioStreamManagerDelegate, AudioDeviceM
|
|
|
1052
1200
|
}
|
|
1053
1201
|
|
|
1054
1202
|
/// Clears all audio files stored in the document directory.
|
|
1203
|
+
private func extractPreviewBars(options: [String: Any], promise: Promise) {
|
|
1204
|
+
Logger.debug("AudioStudioModule", "extractPreviewBars called with options: \(options)")
|
|
1205
|
+
guard let fileUri = options["fileUri"] as? String else {
|
|
1206
|
+
promise.reject("INVALID_ARGUMENTS", "Invalid file URI provided")
|
|
1207
|
+
return
|
|
1208
|
+
}
|
|
1209
|
+
|
|
1210
|
+
let url = URL(string: fileUri) ?? URL(fileURLWithPath: fileUri.replacingOccurrences(of: "file://", with: ""))
|
|
1211
|
+
let numberOfBars = (options["numberOfBars"] as? NSNumber)?.intValue ?? 100
|
|
1212
|
+
let startTimeMs = (options["startTimeMs"] as? NSNumber)?.doubleValue
|
|
1213
|
+
let endTimeMs = (options["endTimeMs"] as? NSNumber)?.doubleValue
|
|
1214
|
+
let decodingOptions = options["decodingOptions"] as? [String: Any]
|
|
1215
|
+
let silenceRmsThreshold = (decodingOptions?["silenceRmsThreshold"] as? NSNumber)?.floatValue ?? 0.01
|
|
1216
|
+
|
|
1217
|
+
DispatchQueue.global().async {
|
|
1218
|
+
self.resolvePreviewBars(
|
|
1219
|
+
url: url,
|
|
1220
|
+
numberOfBars: numberOfBars,
|
|
1221
|
+
startTimeMs: startTimeMs,
|
|
1222
|
+
endTimeMs: endTimeMs,
|
|
1223
|
+
silenceRmsThreshold: silenceRmsThreshold,
|
|
1224
|
+
promise: promise
|
|
1225
|
+
)
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
|
|
1229
|
+
private func resolvePreviewBars(
|
|
1230
|
+
url: URL,
|
|
1231
|
+
numberOfBars: Int,
|
|
1232
|
+
startTimeMs: Double?,
|
|
1233
|
+
endTimeMs: Double?,
|
|
1234
|
+
silenceRmsThreshold: Float,
|
|
1235
|
+
promise: Promise
|
|
1236
|
+
) {
|
|
1237
|
+
do {
|
|
1238
|
+
let audioProcessor = try previewBarsProcessor(for: url)
|
|
1239
|
+
guard let result = audioProcessor.extractPreviewBars(
|
|
1240
|
+
numberOfBars: numberOfBars,
|
|
1241
|
+
startTimeMs: startTimeMs,
|
|
1242
|
+
endTimeMs: endTimeMs,
|
|
1243
|
+
silenceRmsThreshold: silenceRmsThreshold
|
|
1244
|
+
) else {
|
|
1245
|
+
promise.reject("PROCESSING_ERROR", "Failed to extract preview bars")
|
|
1246
|
+
return
|
|
1247
|
+
}
|
|
1248
|
+
promise.resolve(result)
|
|
1249
|
+
} catch {
|
|
1250
|
+
promise.reject("PROCESSING_ERROR", "Failed to initialize audio processor: \(error.localizedDescription)")
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
|
|
1254
|
+
private func previewBarsProcessor(for url: URL) throws -> AudioProcessor {
|
|
1255
|
+
return try AudioProcessor(url: url, resolve: { _ in
|
|
1256
|
+
Logger.warn("AudioStudioModule", "extractPreviewBars: AudioProcessor resolve called unexpectedly.")
|
|
1257
|
+
}, reject: { code, message in
|
|
1258
|
+
Logger.warn("AudioStudioModule", "extractPreviewBars: AudioProcessor reject called unexpectedly: \(code) - \(message)")
|
|
1259
|
+
})
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1055
1262
|
private func clearAudioFiles() {
|
|
1056
1263
|
let fileURLs = listAudioFiles() // This now returns full URLs as strings
|
|
1057
1264
|
fileURLs.forEach { fileURLString in
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
@testable import AudioStudio
|
|
3
|
+
|
|
4
|
+
final class AudioStreamDecoderTests: XCTestCase {
|
|
5
|
+
|
|
6
|
+
// MARK: - Sample sanitization
|
|
7
|
+
|
|
8
|
+
func testSafeFloatToInt16ReplacesNonFinite() {
|
|
9
|
+
XCTAssertEqual(safeFloatToInt16(Float.nan), 0)
|
|
10
|
+
XCTAssertEqual(safeFloatToInt16(Float.infinity), Int16.max)
|
|
11
|
+
XCTAssertEqual(safeFloatToInt16(-Float.infinity), Int16.min)
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
func testSafeFloatToInt16ClampsOutOfRange() {
|
|
15
|
+
XCTAssertEqual(safeFloatToInt16(2.0), Int16.max)
|
|
16
|
+
XCTAssertEqual(safeFloatToInt16(-2.0), Int16.min)
|
|
17
|
+
XCTAssertEqual(safeFloatToInt16(0.0), 0)
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
func testSafeFloatToInt16IdentityAtUnityIsBounded() {
|
|
21
|
+
// The previous Swift `Int16(1.0 * Float(Int16.max))` trap requires
|
|
22
|
+
// the result of the multiplication to fit Int16. The new helper
|
|
23
|
+
// must produce Int16.max for sample == 1.0 without trapping.
|
|
24
|
+
XCTAssertEqual(safeFloatToInt16(1.0), Int16.max)
|
|
25
|
+
XCTAssertEqual(safeFloatToInt16(-1.0), -Int16.max)
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
func testSafeFloatToInt32ReplacesNonFinite() {
|
|
29
|
+
XCTAssertEqual(safeFloatToInt32(Float.nan), 0)
|
|
30
|
+
XCTAssertEqual(safeFloatToInt32(Float.infinity), Int32.max)
|
|
31
|
+
XCTAssertEqual(safeFloatToInt32(-Float.infinity), Int32.min)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
func testSafeFloatToInt32ClampsOutOfRange() {
|
|
35
|
+
XCTAssertEqual(safeFloatToInt32(5.0), Int32.max)
|
|
36
|
+
XCTAssertEqual(safeFloatToInt32(-5.0), Int32.min)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// MARK: - Decoder option bounds
|
|
40
|
+
|
|
41
|
+
func testDecoderOptionsClampsChunkDuration() {
|
|
42
|
+
let opts = AudioStreamDecoder.Options(
|
|
43
|
+
requestId: "test",
|
|
44
|
+
fileUri: "/dev/null",
|
|
45
|
+
startTimeMs: nil,
|
|
46
|
+
endTimeMs: nil,
|
|
47
|
+
targetSampleRate: nil,
|
|
48
|
+
channels: nil,
|
|
49
|
+
normalizeAudio: true,
|
|
50
|
+
chunkDurationMs: 5,
|
|
51
|
+
maxChunkBytes: nil,
|
|
52
|
+
maxBufferedChunks: 0
|
|
53
|
+
)
|
|
54
|
+
XCTAssertEqual(opts.chunkDurationMs, 10, "chunkDurationMs floor is 10ms")
|
|
55
|
+
XCTAssertEqual(opts.maxBufferedChunks, 1, "maxBufferedChunks floor is 1")
|
|
56
|
+
|
|
57
|
+
let bigOpts = AudioStreamDecoder.Options(
|
|
58
|
+
requestId: "big",
|
|
59
|
+
fileUri: "/dev/null",
|
|
60
|
+
startTimeMs: nil,
|
|
61
|
+
endTimeMs: nil,
|
|
62
|
+
targetSampleRate: nil,
|
|
63
|
+
channels: nil,
|
|
64
|
+
normalizeAudio: true,
|
|
65
|
+
chunkDurationMs: 999_999,
|
|
66
|
+
maxChunkBytes: nil,
|
|
67
|
+
maxBufferedChunks: 99
|
|
68
|
+
)
|
|
69
|
+
XCTAssertEqual(bigOpts.chunkDurationMs, 60_000, "chunkDurationMs ceiling is 60s")
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// MARK: - Decoder event contract
|
|
73
|
+
|
|
74
|
+
final class CaptureDelegate: AudioStreamDecoderDelegate {
|
|
75
|
+
var chunks: [[String: Any]] = []
|
|
76
|
+
var progressEvents: [[String: Any]] = []
|
|
77
|
+
var completePayload: [String: Any]?
|
|
78
|
+
var errorPayload: [String: Any]?
|
|
79
|
+
let done = XCTestExpectation(description: "decoder terminal event")
|
|
80
|
+
|
|
81
|
+
func streamDecoder(_ decoder: AudioStreamDecoder, didEmitChunk payload: [String: Any]) {
|
|
82
|
+
chunks.append(payload)
|
|
83
|
+
if let idx = payload["chunkIndex"] as? Int {
|
|
84
|
+
decoder.acknowledgeChunk(idx)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
func streamDecoder(_ decoder: AudioStreamDecoder, didReportProgress payload: [String: Any]) {
|
|
88
|
+
progressEvents.append(payload)
|
|
89
|
+
}
|
|
90
|
+
func streamDecoder(_ decoder: AudioStreamDecoder, didCompleteWith payload: [String: Any]) {
|
|
91
|
+
completePayload = payload
|
|
92
|
+
done.fulfill()
|
|
93
|
+
}
|
|
94
|
+
func streamDecoder(_ decoder: AudioStreamDecoder, didFailWith payload: [String: Any]) {
|
|
95
|
+
errorPayload = payload
|
|
96
|
+
// Some flows emit an error then a complete; let complete fulfill.
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
func testDecoderEmitsFileNotFoundForMissingPath() {
|
|
101
|
+
let delegate = CaptureDelegate()
|
|
102
|
+
let opts = AudioStreamDecoder.Options(
|
|
103
|
+
requestId: "missing",
|
|
104
|
+
fileUri: "/tmp/this-file-does-not-exist-\(UUID().uuidString).wav",
|
|
105
|
+
startTimeMs: nil,
|
|
106
|
+
endTimeMs: nil,
|
|
107
|
+
targetSampleRate: nil,
|
|
108
|
+
channels: nil,
|
|
109
|
+
normalizeAudio: true,
|
|
110
|
+
chunkDurationMs: 100,
|
|
111
|
+
maxChunkBytes: nil,
|
|
112
|
+
maxBufferedChunks: 2
|
|
113
|
+
)
|
|
114
|
+
let decoder = AudioStreamDecoder(options: opts)
|
|
115
|
+
decoder.delegate = delegate
|
|
116
|
+
decoder.start()
|
|
117
|
+
// Error path never calls complete, so wait directly on the error.
|
|
118
|
+
let exp = XCTestExpectation(description: "error received")
|
|
119
|
+
DispatchQueue.global().asyncAfter(deadline: .now() + 0.5) {
|
|
120
|
+
if delegate.errorPayload != nil {
|
|
121
|
+
exp.fulfill()
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
wait(for: [exp], timeout: 2.0)
|
|
125
|
+
XCTAssertEqual(delegate.errorPayload?["code"] as? String, "ERR_AUDIO_STREAM_FILE_NOT_FOUND")
|
|
126
|
+
XCTAssertEqual(delegate.errorPayload?["requestId"] as? String, "missing")
|
|
127
|
+
}
|
|
128
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@siteed/audio-studio",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.2.0-beta.1",
|
|
4
4
|
"description": "Comprehensive audio processing library for React Native and Expo with recording, analysis, visualization, and streaming capabilities across iOS, Android, and web",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "commonjs",
|
|
@@ -84,11 +84,11 @@
|
|
|
84
84
|
"lint:fix": "expo-module lint --fix",
|
|
85
85
|
"test": "expo-module test",
|
|
86
86
|
"test:android": "yarn test:android:unit && yarn test:android:instrumented",
|
|
87
|
-
"test:android:unit": "cd ../../apps/playground/android && ./gradlew :siteed-
|
|
88
|
-
"test:android:instrumented": "cd ../../apps/playground/android && ./gradlew :siteed-
|
|
89
|
-
"test:android:unit:watch": "cd ../../apps/playground/android && ./gradlew :siteed-
|
|
87
|
+
"test:android:unit": "cd ../../apps/playground/android && ./gradlew :siteed-audio-studio:test",
|
|
88
|
+
"test:android:instrumented": "cd ../../apps/playground/android && ./gradlew :siteed-audio-studio:connectedAndroidTest",
|
|
89
|
+
"test:android:unit:watch": "cd ../../apps/playground/android && ./gradlew :siteed-audio-studio:test --continuous",
|
|
90
90
|
"test:ios": "cd ../../apps/playground/ios && xcodebuild -workspace AudioDevPlayground.xcworkspace -scheme AudioDevPlayground -destination 'platform=iOS Simulator,name=iPhone 15' build",
|
|
91
|
-
"test:coverage": "cd ../../apps/playground/android && ./gradlew :siteed-
|
|
91
|
+
"test:coverage": "cd ../../apps/playground/android && ./gradlew :siteed-audio-studio:jacocoTestReport",
|
|
92
92
|
"typecheck": "tsc --noEmit",
|
|
93
93
|
"docgen": "typedoc src/index.ts --plugin typedoc-plugin-markdown --readme none --out ../../documentation_site/docs/api-reference/API && node ../../scripts/escape-mdx-generics.js ../../documentation_site/docs/api-reference",
|
|
94
94
|
"prepare": "yarn build && node -e \"require('fs').renameSync('./plugin/build/index.d.ts', './plugin/build/index.d.cts')\"",
|
|
@@ -125,8 +125,8 @@
|
|
|
125
125
|
"expo-modules-core": "~3.0.0",
|
|
126
126
|
"jest": "^29.7.0",
|
|
127
127
|
"prettier": "^3.2.5",
|
|
128
|
-
"react": "19.
|
|
129
|
-
"react-native": "0.
|
|
128
|
+
"react": "19.2.0",
|
|
129
|
+
"react-native": "0.83.6",
|
|
130
130
|
"rimraf": "^6.0.1",
|
|
131
131
|
"size-limit": "^11.1.4",
|
|
132
132
|
"ts-node": "^10.9.2",
|
|
@@ -14,6 +14,12 @@ export interface DecodingConfig {
|
|
|
14
14
|
targetBitDepth?: BitDepth
|
|
15
15
|
/** Whether to normalize audio levels (Android and Web) */
|
|
16
16
|
normalizeAudio?: boolean
|
|
17
|
+
/**
|
|
18
|
+
* RMS threshold below which a segment is flagged silent.
|
|
19
|
+
* Range 0..1. Default 0.01.
|
|
20
|
+
* Currently applied as a JS post-process so the same behavior holds across iOS/Android/Web.
|
|
21
|
+
*/
|
|
22
|
+
silenceRmsThreshold?: number
|
|
17
23
|
}
|
|
18
24
|
|
|
19
25
|
/**
|
|
@@ -163,6 +169,69 @@ export interface AudioRangeOptions {
|
|
|
163
169
|
* Options for generating a quick preview of audio waveform.
|
|
164
170
|
* This is optimized for UI rendering with a specified number of points.
|
|
165
171
|
*/
|
|
172
|
+
export interface PreviewBar {
|
|
173
|
+
/** Stable zero-based bar identifier. */
|
|
174
|
+
id: number
|
|
175
|
+
/** Peak amplitude for this bar, normalized to 0..1. */
|
|
176
|
+
amplitude: number
|
|
177
|
+
/** Root mean square amplitude for this bar, normalized to 0..1. */
|
|
178
|
+
rms: number
|
|
179
|
+
/** Whether this bar is below the configured silence RMS threshold. */
|
|
180
|
+
silent: boolean
|
|
181
|
+
/** Bar start time in milliseconds from the extracted range start. */
|
|
182
|
+
startTimeMs: number
|
|
183
|
+
/** Bar end time in milliseconds from the extracted range start. */
|
|
184
|
+
endTimeMs: number
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Compact preview-bars result for UI waveform rendering.
|
|
189
|
+
* Unlike `AudioAnalysis`, this intentionally omits full `DataPoint` feature data.
|
|
190
|
+
*/
|
|
191
|
+
export interface PreviewBarsResult {
|
|
192
|
+
bars: PreviewBar[]
|
|
193
|
+
durationMs: number
|
|
194
|
+
sampleRate: number
|
|
195
|
+
numberOfChannels: number
|
|
196
|
+
bitDepth: number
|
|
197
|
+
samples: number
|
|
198
|
+
/** Requested bar count before native/platform clamping. */
|
|
199
|
+
requestedNumberOfBars: number
|
|
200
|
+
/** Approximate duration represented by each bar. */
|
|
201
|
+
barDurationMs: number
|
|
202
|
+
amplitudeRange: {
|
|
203
|
+
min: number
|
|
204
|
+
max: number
|
|
205
|
+
}
|
|
206
|
+
rmsRange: {
|
|
207
|
+
min: number
|
|
208
|
+
max: number
|
|
209
|
+
}
|
|
210
|
+
extractionTimeMs: number
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Options for extracting compact waveform preview bars for UI rendering.
|
|
215
|
+
*/
|
|
216
|
+
export interface PreviewBarsOptions extends AudioRangeOptions {
|
|
217
|
+
/** URI of the audio file to analyze */
|
|
218
|
+
fileUri: string
|
|
219
|
+
/**
|
|
220
|
+
* Total number of bars to generate for the preview.
|
|
221
|
+
* @default 100
|
|
222
|
+
*/
|
|
223
|
+
numberOfBars?: number
|
|
224
|
+
/** Optional logger for debugging. */
|
|
225
|
+
logger?: ConsoleLike
|
|
226
|
+
/** Optional configuration for decoding the audio file. */
|
|
227
|
+
decodingOptions?: DecodingConfig
|
|
228
|
+
/**
|
|
229
|
+
* Optional callback fired once per compact bar after extraction resolves.
|
|
230
|
+
* Native progressive streaming is not implied by this callback.
|
|
231
|
+
*/
|
|
232
|
+
onBarReady?: (bar: PreviewBar, index: number, total: number) => void
|
|
233
|
+
}
|
|
234
|
+
|
|
166
235
|
export interface PreviewOptions extends AudioRangeOptions {
|
|
167
236
|
/** URI of the audio file to analyze */
|
|
168
237
|
fileUri: string
|
|
@@ -184,6 +253,19 @@ export interface PreviewOptions extends AudioRangeOptions {
|
|
|
184
253
|
* - normalizeAudio: false
|
|
185
254
|
*/
|
|
186
255
|
decodingOptions?: DecodingConfig
|
|
256
|
+
/**
|
|
257
|
+
* Optional callback fired once per data point as the preview becomes available.
|
|
258
|
+
* Today the native module returns the full analysis in one shot; the points are then
|
|
259
|
+
* micro-batched on the JS side so consumers can render bars incrementally.
|
|
260
|
+
* Native progressive streaming is a future enhancement.
|
|
261
|
+
*/
|
|
262
|
+
onPointReady?: (point: DataPoint, index: number, total: number) => void
|
|
263
|
+
/**
|
|
264
|
+
* Optional cancellation signal for JS-side progressive point emission.
|
|
265
|
+
* Aborting does not cancel native extraction after it has started, but it
|
|
266
|
+
* stops any queued `onPointReady` callbacks from an older request.
|
|
267
|
+
*/
|
|
268
|
+
signal?: AbortSignal
|
|
187
269
|
}
|
|
188
270
|
|
|
189
271
|
/**
|
|
@@ -1,34 +1,135 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { mapExtractionError } from '../errors/AudioExtractionError'
|
|
2
|
+
import { PreviewOptions, AudioAnalysis, DataPoint } from './AudioAnalysis.types'
|
|
2
3
|
import { extractAudioAnalysis } from './extractAudioAnalysis'
|
|
3
4
|
|
|
5
|
+
const DEFAULT_SILENCE_THRESHOLD = 0.01
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Apply a silence threshold to the data points by recomputing the `silent` flag from rms.
|
|
9
|
+
* Returns a new array (does not mutate the source).
|
|
10
|
+
*/
|
|
11
|
+
function applySilenceThreshold(
|
|
12
|
+
dataPoints: DataPoint[],
|
|
13
|
+
threshold: number
|
|
14
|
+
): DataPoint[] {
|
|
15
|
+
return dataPoints.map((p) => ({
|
|
16
|
+
...p,
|
|
17
|
+
silent: p.rms < threshold,
|
|
18
|
+
}))
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const SMALL_TOTAL_INSTANT_THRESHOLD = 50
|
|
22
|
+
const PROGRESSIVE_BATCH_DELAY_MS = 30
|
|
23
|
+
const PROGRESSIVE_BATCH_COUNT = 8
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Schedule progressive emission of points after the native one-shot resolve.
|
|
27
|
+
* Native progressive streaming is a future enhancement; today the points are
|
|
28
|
+
* micro-batched on the JS side so consumers (and the agentic recipe runner)
|
|
29
|
+
* can observe an in-flight `pointsReceived < totalPoints` window.
|
|
30
|
+
*/
|
|
31
|
+
function emitPointsProgressively(
|
|
32
|
+
dataPoints: DataPoint[],
|
|
33
|
+
onPointReady: NonNullable<PreviewOptions['onPointReady']>,
|
|
34
|
+
signal?: PreviewOptions['signal'],
|
|
35
|
+
logger?: PreviewOptions['logger']
|
|
36
|
+
): void {
|
|
37
|
+
const total = dataPoints.length
|
|
38
|
+
if (total === 0) return
|
|
39
|
+
|
|
40
|
+
const safeEmit = (point: DataPoint, index: number) => {
|
|
41
|
+
if (signal?.aborted) return
|
|
42
|
+
try {
|
|
43
|
+
onPointReady(point, index, total)
|
|
44
|
+
} catch (err) {
|
|
45
|
+
// Swallow callback errors so a buggy consumer cannot break extraction.
|
|
46
|
+
logger?.warn?.('extractPreview onPointReady callback failed', err)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (signal?.aborted) return
|
|
51
|
+
if (total <= SMALL_TOTAL_INSTANT_THRESHOLD) {
|
|
52
|
+
for (let i = 0; i < total; i++) safeEmit(dataPoints[i], i)
|
|
53
|
+
return
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// First quarter flushes immediately so the UI shows something within a frame.
|
|
57
|
+
const firstFlushCount = Math.max(1, Math.floor(total / 4))
|
|
58
|
+
for (let i = 0; i < firstFlushCount; i++) safeEmit(dataPoints[i], i)
|
|
59
|
+
|
|
60
|
+
if (firstFlushCount >= total) return
|
|
61
|
+
|
|
62
|
+
const remaining = total - firstFlushCount
|
|
63
|
+
const batchSize = Math.max(
|
|
64
|
+
1,
|
|
65
|
+
Math.ceil(remaining / PROGRESSIVE_BATCH_COUNT)
|
|
66
|
+
)
|
|
67
|
+
let cursor = firstFlushCount
|
|
68
|
+
const pump = () => {
|
|
69
|
+
if (signal?.aborted) return
|
|
70
|
+
const end = Math.min(total, cursor + batchSize)
|
|
71
|
+
for (let i = cursor; i < end; i++) safeEmit(dataPoints[i], i)
|
|
72
|
+
cursor = end
|
|
73
|
+
if (cursor < total) {
|
|
74
|
+
setTimeout(pump, PROGRESSIVE_BATCH_DELAY_MS)
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
setTimeout(pump, PROGRESSIVE_BATCH_DELAY_MS)
|
|
78
|
+
}
|
|
79
|
+
|
|
4
80
|
/**
|
|
5
81
|
* Generates a simplified preview of the audio waveform for quick visualization.
|
|
6
82
|
* Ideal for UI rendering with a specified number of points.
|
|
7
83
|
*
|
|
8
84
|
* @param options - The options for the preview, including file URI and time range.
|
|
9
85
|
* @returns A promise that resolves to the audio preview data.
|
|
86
|
+
* @throws {AudioExtractionError} when the underlying extraction fails.
|
|
10
87
|
*/
|
|
11
88
|
export async function extractPreview({
|
|
12
89
|
fileUri,
|
|
13
90
|
numberOfPoints = 100,
|
|
14
91
|
startTimeMs = 0,
|
|
15
|
-
endTimeMs = 30000,
|
|
92
|
+
endTimeMs = 30000,
|
|
16
93
|
decodingOptions,
|
|
17
94
|
logger,
|
|
95
|
+
onPointReady,
|
|
96
|
+
signal,
|
|
18
97
|
}: PreviewOptions): Promise<AudioAnalysis> {
|
|
19
|
-
const durationMs = endTimeMs - startTimeMs
|
|
20
|
-
const segmentDurationMs = Math.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
98
|
+
const durationMs = Math.max(1, endTimeMs - startTimeMs)
|
|
99
|
+
const segmentDurationMs = Math.max(
|
|
100
|
+
1,
|
|
101
|
+
Math.floor(durationMs / numberOfPoints)
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
let analysis: AudioAnalysis
|
|
105
|
+
try {
|
|
106
|
+
analysis = await extractAudioAnalysis({
|
|
107
|
+
fileUri,
|
|
108
|
+
startTimeMs,
|
|
109
|
+
endTimeMs,
|
|
110
|
+
logger,
|
|
111
|
+
segmentDurationMs,
|
|
112
|
+
decodingOptions,
|
|
113
|
+
})
|
|
114
|
+
} catch (err) {
|
|
115
|
+
throw mapExtractionError(err, fileUri)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const threshold =
|
|
119
|
+
decodingOptions?.silenceRmsThreshold ?? DEFAULT_SILENCE_THRESHOLD
|
|
120
|
+
const adjusted: AudioAnalysis = {
|
|
121
|
+
...analysis,
|
|
122
|
+
dataPoints: applySilenceThreshold(analysis.dataPoints, threshold),
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (onPointReady) {
|
|
126
|
+
emitPointsProgressively(
|
|
127
|
+
adjusted.dataPoints,
|
|
128
|
+
onPointReady,
|
|
129
|
+
signal,
|
|
130
|
+
logger
|
|
131
|
+
)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
return adjusted
|
|
34
135
|
}
|