@siteed/audio-studio 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/CHANGELOG.md +535 -0
  2. package/LICENSE +21 -0
  3. package/README.md +167 -0
  4. package/android/build.gradle +143 -0
  5. package/android/src/androidTest/assets/chorus.wav +0 -0
  6. package/android/src/androidTest/assets/jfk.wav +0 -0
  7. package/android/src/androidTest/assets/osr_us_000_0010_8k.wav +0 -0
  8. package/android/src/androidTest/assets/recorder_hello_world.wav +0 -0
  9. package/android/src/androidTest/java/net/siteed/audiostudio/AudioProcessorInstrumentedTest.kt +197 -0
  10. package/android/src/androidTest/java/net/siteed/audiostudio/AudioRecorderInstrumentedTest.kt +541 -0
  11. package/android/src/androidTest/java/net/siteed/audiostudio/AudioRecorderPerformanceInstrumentedTest.kt +234 -0
  12. package/android/src/androidTest/java/net/siteed/audiostudio/integration/AudioFocusStrategyIntegrationTest.kt +332 -0
  13. package/android/src/androidTest/java/net/siteed/audiostudio/integration/BufferDurationIntegrationTest.kt +324 -0
  14. package/android/src/androidTest/java/net/siteed/audiostudio/integration/CompressedOnlyOutputTest.kt +253 -0
  15. package/android/src/androidTest/java/net/siteed/audiostudio/integration/DeviceDisconnectionFallbackTest.kt +218 -0
  16. package/android/src/androidTest/java/net/siteed/audiostudio/integration/EventEmissionIntervalTest.kt +120 -0
  17. package/android/src/androidTest/java/net/siteed/audiostudio/integration/M4aFormatTest.kt +345 -0
  18. package/android/src/androidTest/java/net/siteed/audiostudio/integration/OutputControlIntegrationTest.kt +340 -0
  19. package/android/src/androidTest/java/net/siteed/audiostudio/integration/PcmStreamingDurationTest.kt +252 -0
  20. package/android/src/androidTest/java/net/siteed/audiostudio/integration/README.md +95 -0
  21. package/android/src/androidTest/java/net/siteed/audiostudio/integration/run_integration_tests.sh +43 -0
  22. package/android/src/main/AndroidManifest.xml +30 -0
  23. package/android/src/main/CMakeLists.txt +29 -0
  24. package/android/src/main/java/net/siteed/audiostudio/AudioAnalysisData.kt +188 -0
  25. package/android/src/main/java/net/siteed/audiostudio/AudioDataEncoder.kt +9 -0
  26. package/android/src/main/java/net/siteed/audiostudio/AudioDeviceManager.kt +1741 -0
  27. package/android/src/main/java/net/siteed/audiostudio/AudioFeaturesNative.kt +26 -0
  28. package/android/src/main/java/net/siteed/audiostudio/AudioFileHandler.kt +136 -0
  29. package/android/src/main/java/net/siteed/audiostudio/AudioFormatUtils.kt +354 -0
  30. package/android/src/main/java/net/siteed/audiostudio/AudioNotificationsManager.kt +439 -0
  31. package/android/src/main/java/net/siteed/audiostudio/AudioProcessor.kt +2237 -0
  32. package/android/src/main/java/net/siteed/audiostudio/AudioRecorderManager.kt +2163 -0
  33. package/android/src/main/java/net/siteed/audiostudio/AudioRecordingService.kt +167 -0
  34. package/android/src/main/java/net/siteed/audiostudio/AudioStudioModule.kt +1112 -0
  35. package/android/src/main/java/net/siteed/audiostudio/AudioTrimmer.kt +1099 -0
  36. package/android/src/main/java/net/siteed/audiostudio/Constants.kt +37 -0
  37. package/android/src/main/java/net/siteed/audiostudio/EventSender.kt +7 -0
  38. package/android/src/main/java/net/siteed/audiostudio/FFT.kt +100 -0
  39. package/android/src/main/java/net/siteed/audiostudio/Features.kt +98 -0
  40. package/android/src/main/java/net/siteed/audiostudio/LogUtils.kt +93 -0
  41. package/android/src/main/java/net/siteed/audiostudio/MelSpectrogramNative.kt +36 -0
  42. package/android/src/main/java/net/siteed/audiostudio/NotificationConfig.kt +72 -0
  43. package/android/src/main/java/net/siteed/audiostudio/PermissionUtils.kt +68 -0
  44. package/android/src/main/java/net/siteed/audiostudio/RecordingActionReceiver.kt +59 -0
  45. package/android/src/main/java/net/siteed/audiostudio/RecordingConfig.kt +259 -0
  46. package/android/src/main/java/net/siteed/audiostudio/WaveformConfig.kt +19 -0
  47. package/android/src/main/java/net/siteed/audiostudio/WaveformRenderer.kt +159 -0
  48. package/android/src/main/jni/AudioFeaturesJNI.cpp +152 -0
  49. package/android/src/main/jni/MelSpectrogramJNI.cpp +165 -0
  50. package/android/src/main/res/drawable/ic_default_action_icon.xml +16 -0
  51. package/android/src/main/res/drawable/ic_microphone.xml +13 -0
  52. package/android/src/main/res/drawable/ic_pause.xml +10 -0
  53. package/android/src/main/res/drawable/ic_play.xml +10 -0
  54. package/android/src/main/res/drawable/ic_stop.xml +10 -0
  55. package/android/src/main/res/layout/notification_recording.xml +37 -0
  56. package/android/src/test/java/net/siteed/audiostudio/AudioFileHandlerTest.kt +279 -0
  57. package/android/src/test/java/net/siteed/audiostudio/AudioFocusStrategyTest.kt +249 -0
  58. package/android/src/test/java/net/siteed/audiostudio/AudioFormatTest.kt +151 -0
  59. package/android/src/test/java/net/siteed/audiostudio/AudioFormatUtilsTest.kt +273 -0
  60. package/android/src/test/java/net/siteed/audiostudio/DeviceDisconnectionFallbackUnitTest.kt +140 -0
  61. package/android/src/test/resources/chorus.wav +0 -0
  62. package/android/src/test/resources/generate_test_audio.py +94 -0
  63. package/android/src/test/resources/jfk.wav +0 -0
  64. package/android/src/test/resources/osr_us_000_0010_8k.wav +0 -0
  65. package/android/src/test/resources/recorder_hello_world.wav +0 -0
  66. package/app.plugin.js +3 -0
  67. package/build/cjs/AudioAnalysis/AudioAnalysis.types.js +4 -0
  68. package/build/cjs/AudioAnalysis/AudioAnalysis.types.js.map +1 -0
  69. package/build/cjs/AudioAnalysis/audioFeaturesWasm.js +164 -0
  70. package/build/cjs/AudioAnalysis/audioFeaturesWasm.js.map +1 -0
  71. package/build/cjs/AudioAnalysis/extractAudioAnalysis.js +213 -0
  72. package/build/cjs/AudioAnalysis/extractAudioAnalysis.js.map +1 -0
  73. package/build/cjs/AudioAnalysis/extractAudioData.js +21 -0
  74. package/build/cjs/AudioAnalysis/extractAudioData.js.map +1 -0
  75. package/build/cjs/AudioAnalysis/extractMelSpectrogram.js +90 -0
  76. package/build/cjs/AudioAnalysis/extractMelSpectrogram.js.map +1 -0
  77. package/build/cjs/AudioAnalysis/extractPreview.js +28 -0
  78. package/build/cjs/AudioAnalysis/extractPreview.js.map +1 -0
  79. package/build/cjs/AudioAnalysis/extractWaveform.js +18 -0
  80. package/build/cjs/AudioAnalysis/extractWaveform.js.map +1 -0
  81. package/build/cjs/AudioAnalysis/melSpectrogramWasm.js +149 -0
  82. package/build/cjs/AudioAnalysis/melSpectrogramWasm.js.map +1 -0
  83. package/build/cjs/AudioDeviceManager.js +688 -0
  84. package/build/cjs/AudioDeviceManager.js.map +1 -0
  85. package/build/cjs/AudioRecorder.provider.js +78 -0
  86. package/build/cjs/AudioRecorder.provider.js.map +1 -0
  87. package/build/cjs/AudioStudio.native.js +8 -0
  88. package/build/cjs/AudioStudio.native.js.map +1 -0
  89. package/build/cjs/AudioStudio.types.js +11 -0
  90. package/build/cjs/AudioStudio.types.js.map +1 -0
  91. package/build/cjs/AudioStudio.web.js +708 -0
  92. package/build/cjs/AudioStudio.web.js.map +1 -0
  93. package/build/cjs/AudioStudioModule.js +718 -0
  94. package/build/cjs/AudioStudioModule.js.map +1 -0
  95. package/build/cjs/WebRecorder.web.js +865 -0
  96. package/build/cjs/WebRecorder.web.js.map +1 -0
  97. package/build/cjs/constants/platformLimitations.js +99 -0
  98. package/build/cjs/constants/platformLimitations.js.map +1 -0
  99. package/build/cjs/constants.js +20 -0
  100. package/build/cjs/constants.js.map +1 -0
  101. package/build/cjs/events.js +29 -0
  102. package/build/cjs/events.js.map +1 -0
  103. package/build/cjs/hooks/useAudioDevices.js +179 -0
  104. package/build/cjs/hooks/useAudioDevices.js.map +1 -0
  105. package/build/cjs/index.js +64 -0
  106. package/build/cjs/index.js.map +1 -0
  107. package/build/cjs/trimAudio.js +76 -0
  108. package/build/cjs/trimAudio.js.map +1 -0
  109. package/build/cjs/useAudioRecorder.js +535 -0
  110. package/build/cjs/useAudioRecorder.js.map +1 -0
  111. package/build/cjs/utils/BlobFix.js +502 -0
  112. package/build/cjs/utils/BlobFix.js.map +1 -0
  113. package/build/cjs/utils/audioProcessing.js +136 -0
  114. package/build/cjs/utils/audioProcessing.js.map +1 -0
  115. package/build/cjs/utils/cleanNativeOptions.js +22 -0
  116. package/build/cjs/utils/cleanNativeOptions.js.map +1 -0
  117. package/build/cjs/utils/concatenateBuffers.js +25 -0
  118. package/build/cjs/utils/concatenateBuffers.js.map +1 -0
  119. package/build/cjs/utils/convertPCMToFloat32.js +124 -0
  120. package/build/cjs/utils/convertPCMToFloat32.js.map +1 -0
  121. package/build/cjs/utils/crc32.js +52 -0
  122. package/build/cjs/utils/crc32.js.map +1 -0
  123. package/build/cjs/utils/encodingToBitDepth.js +17 -0
  124. package/build/cjs/utils/encodingToBitDepth.js.map +1 -0
  125. package/build/cjs/utils/getWavFileInfo.js +96 -0
  126. package/build/cjs/utils/getWavFileInfo.js.map +1 -0
  127. package/build/cjs/utils/writeWavHeader.js +88 -0
  128. package/build/cjs/utils/writeWavHeader.js.map +1 -0
  129. package/build/cjs/workers/InlineFeaturesExtractor.web.js +294 -0
  130. package/build/cjs/workers/InlineFeaturesExtractor.web.js.map +1 -0
  131. package/build/cjs/workers/inlineAudioWebWorker.web.js +190 -0
  132. package/build/cjs/workers/inlineAudioWebWorker.web.js.map +1 -0
  133. package/build/cjs/workers/wasmGlueString.web.js +27 -0
  134. package/build/cjs/workers/wasmGlueString.web.js.map +1 -0
  135. package/build/esm/AudioAnalysis/AudioAnalysis.types.js +3 -0
  136. package/build/esm/AudioAnalysis/AudioAnalysis.types.js.map +1 -0
  137. package/build/esm/AudioAnalysis/audioFeaturesWasm.js +126 -0
  138. package/build/esm/AudioAnalysis/audioFeaturesWasm.js.map +1 -0
  139. package/build/esm/AudioAnalysis/extractAudioAnalysis.js +205 -0
  140. package/build/esm/AudioAnalysis/extractAudioAnalysis.js.map +1 -0
  141. package/build/esm/AudioAnalysis/extractAudioData.js +14 -0
  142. package/build/esm/AudioAnalysis/extractAudioData.js.map +1 -0
  143. package/build/esm/AudioAnalysis/extractMelSpectrogram.js +86 -0
  144. package/build/esm/AudioAnalysis/extractMelSpectrogram.js.map +1 -0
  145. package/build/esm/AudioAnalysis/extractPreview.js +25 -0
  146. package/build/esm/AudioAnalysis/extractPreview.js.map +1 -0
  147. package/build/esm/AudioAnalysis/extractWaveform.js +11 -0
  148. package/build/esm/AudioAnalysis/extractWaveform.js.map +1 -0
  149. package/build/esm/AudioAnalysis/melSpectrogramWasm.js +111 -0
  150. package/build/esm/AudioAnalysis/melSpectrogramWasm.js.map +1 -0
  151. package/build/esm/AudioDeviceManager.js +681 -0
  152. package/build/esm/AudioDeviceManager.js.map +1 -0
  153. package/build/esm/AudioRecorder.provider.js +40 -0
  154. package/build/esm/AudioRecorder.provider.js.map +1 -0
  155. package/build/esm/AudioStudio.native.js +6 -0
  156. package/build/esm/AudioStudio.native.js.map +1 -0
  157. package/build/esm/AudioStudio.types.js +8 -0
  158. package/build/esm/AudioStudio.types.js.map +1 -0
  159. package/build/esm/AudioStudio.web.js +704 -0
  160. package/build/esm/AudioStudio.web.js.map +1 -0
  161. package/build/esm/AudioStudioModule.js +713 -0
  162. package/build/esm/AudioStudioModule.js.map +1 -0
  163. package/build/esm/WebRecorder.web.js +861 -0
  164. package/build/esm/WebRecorder.web.js.map +1 -0
  165. package/build/esm/constants/platformLimitations.js +90 -0
  166. package/build/esm/constants/platformLimitations.js.map +1 -0
  167. package/build/esm/constants.js +17 -0
  168. package/build/esm/constants.js.map +1 -0
  169. package/build/esm/events.js +21 -0
  170. package/build/esm/events.js.map +1 -0
  171. package/build/esm/hooks/useAudioDevices.js +176 -0
  172. package/build/esm/hooks/useAudioDevices.js.map +1 -0
  173. package/build/esm/index.js +23 -0
  174. package/build/esm/index.js.map +1 -0
  175. package/build/esm/trimAudio.js +69 -0
  176. package/build/esm/trimAudio.js.map +1 -0
  177. package/build/esm/useAudioRecorder.js +529 -0
  178. package/build/esm/useAudioRecorder.js.map +1 -0
  179. package/build/esm/utils/BlobFix.js +498 -0
  180. package/build/esm/utils/BlobFix.js.map +1 -0
  181. package/build/esm/utils/audioProcessing.js +133 -0
  182. package/build/esm/utils/audioProcessing.js.map +1 -0
  183. package/build/esm/utils/cleanNativeOptions.js +19 -0
  184. package/build/esm/utils/cleanNativeOptions.js.map +1 -0
  185. package/build/esm/utils/concatenateBuffers.js +21 -0
  186. package/build/esm/utils/concatenateBuffers.js.map +1 -0
  187. package/build/esm/utils/convertPCMToFloat32.js +120 -0
  188. package/build/esm/utils/convertPCMToFloat32.js.map +1 -0
  189. package/build/esm/utils/crc32.js +50 -0
  190. package/build/esm/utils/crc32.js.map +1 -0
  191. package/build/esm/utils/encodingToBitDepth.js +13 -0
  192. package/build/esm/utils/encodingToBitDepth.js.map +1 -0
  193. package/build/esm/utils/getWavFileInfo.js +92 -0
  194. package/build/esm/utils/getWavFileInfo.js.map +1 -0
  195. package/build/esm/utils/writeWavHeader.js +84 -0
  196. package/build/esm/utils/writeWavHeader.js.map +1 -0
  197. package/build/esm/workers/InlineFeaturesExtractor.web.js +291 -0
  198. package/build/esm/workers/InlineFeaturesExtractor.web.js.map +1 -0
  199. package/build/esm/workers/inlineAudioWebWorker.web.js +187 -0
  200. package/build/esm/workers/inlineAudioWebWorker.web.js.map +1 -0
  201. package/build/esm/workers/wasmGlueString.web.js +24 -0
  202. package/build/esm/workers/wasmGlueString.web.js.map +1 -0
  203. package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts +198 -0
  204. package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts.map +1 -0
  205. package/build/types/AudioAnalysis/audioFeaturesWasm.d.ts +24 -0
  206. package/build/types/AudioAnalysis/audioFeaturesWasm.d.ts.map +1 -0
  207. package/build/types/AudioAnalysis/extractAudioAnalysis.d.ts +74 -0
  208. package/build/types/AudioAnalysis/extractAudioAnalysis.d.ts.map +1 -0
  209. package/build/types/AudioAnalysis/extractAudioData.d.ts +3 -0
  210. package/build/types/AudioAnalysis/extractAudioData.d.ts.map +1 -0
  211. package/build/types/AudioAnalysis/extractMelSpectrogram.d.ts +20 -0
  212. package/build/types/AudioAnalysis/extractMelSpectrogram.d.ts.map +1 -0
  213. package/build/types/AudioAnalysis/extractPreview.d.ts +11 -0
  214. package/build/types/AudioAnalysis/extractPreview.d.ts.map +1 -0
  215. package/build/types/AudioAnalysis/extractWaveform.d.ts +8 -0
  216. package/build/types/AudioAnalysis/extractWaveform.d.ts.map +1 -0
  217. package/build/types/AudioAnalysis/melSpectrogramWasm.d.ts +16 -0
  218. package/build/types/AudioAnalysis/melSpectrogramWasm.d.ts.map +1 -0
  219. package/build/types/AudioDeviceManager.d.ts +187 -0
  220. package/build/types/AudioDeviceManager.d.ts.map +1 -0
  221. package/build/types/AudioRecorder.provider.d.ts +11 -0
  222. package/build/types/AudioRecorder.provider.d.ts.map +1 -0
  223. package/build/types/AudioStudio.native.d.ts +3 -0
  224. package/build/types/AudioStudio.native.d.ts.map +1 -0
  225. package/build/types/AudioStudio.types.d.ts +760 -0
  226. package/build/types/AudioStudio.types.d.ts.map +1 -0
  227. package/build/types/AudioStudio.web.d.ts +96 -0
  228. package/build/types/AudioStudio.web.d.ts.map +1 -0
  229. package/build/types/AudioStudioModule.d.ts +3 -0
  230. package/build/types/AudioStudioModule.d.ts.map +1 -0
  231. package/build/types/WebRecorder.web.d.ts +208 -0
  232. package/build/types/WebRecorder.web.d.ts.map +1 -0
  233. package/build/types/constants/platformLimitations.d.ts +40 -0
  234. package/build/types/constants/platformLimitations.d.ts.map +1 -0
  235. package/build/types/constants.d.ts +14 -0
  236. package/build/types/constants.d.ts.map +1 -0
  237. package/build/types/events.d.ts +29 -0
  238. package/build/types/events.d.ts.map +1 -0
  239. package/build/types/hooks/useAudioDevices.d.ts +15 -0
  240. package/build/types/hooks/useAudioDevices.d.ts.map +1 -0
  241. package/build/types/index.d.ts +21 -0
  242. package/build/types/index.d.ts.map +1 -0
  243. package/build/types/trimAudio.d.ts +25 -0
  244. package/build/types/trimAudio.d.ts.map +1 -0
  245. package/build/types/useAudioRecorder.d.ts +22 -0
  246. package/build/types/useAudioRecorder.d.ts.map +1 -0
  247. package/build/types/utils/BlobFix.d.ts +9 -0
  248. package/build/types/utils/BlobFix.d.ts.map +1 -0
  249. package/build/types/utils/audioProcessing.d.ts +24 -0
  250. package/build/types/utils/audioProcessing.d.ts.map +1 -0
  251. package/build/types/utils/cleanNativeOptions.d.ts +15 -0
  252. package/build/types/utils/cleanNativeOptions.d.ts.map +1 -0
  253. package/build/types/utils/concatenateBuffers.d.ts +8 -0
  254. package/build/types/utils/concatenateBuffers.d.ts.map +1 -0
  255. package/build/types/utils/convertPCMToFloat32.d.ts +13 -0
  256. package/build/types/utils/convertPCMToFloat32.d.ts.map +1 -0
  257. package/build/types/utils/crc32.d.ts +7 -0
  258. package/build/types/utils/crc32.d.ts.map +1 -0
  259. package/build/types/utils/encodingToBitDepth.d.ts +5 -0
  260. package/build/types/utils/encodingToBitDepth.d.ts.map +1 -0
  261. package/build/types/utils/getWavFileInfo.d.ts +26 -0
  262. package/build/types/utils/getWavFileInfo.d.ts.map +1 -0
  263. package/build/types/utils/writeWavHeader.d.ts +34 -0
  264. package/build/types/utils/writeWavHeader.d.ts.map +1 -0
  265. package/build/types/workers/InlineFeaturesExtractor.web.d.ts +2 -0
  266. package/build/types/workers/InlineFeaturesExtractor.web.d.ts.map +1 -0
  267. package/build/types/workers/inlineAudioWebWorker.web.d.ts +2 -0
  268. package/build/types/workers/inlineAudioWebWorker.web.d.ts.map +1 -0
  269. package/build/types/workers/wasmGlueString.web.d.ts +2 -0
  270. package/build/types/workers/wasmGlueString.web.d.ts.map +1 -0
  271. package/cpp/AudioFeatures.cpp +274 -0
  272. package/cpp/AudioFeatures.h +85 -0
  273. package/cpp/AudioFeaturesBridge.cpp +146 -0
  274. package/cpp/AudioFeaturesBridge.h +47 -0
  275. package/cpp/MelSpectrogram.cpp +227 -0
  276. package/cpp/MelSpectrogram.h +82 -0
  277. package/cpp/MelSpectrogramBridge.cpp +112 -0
  278. package/cpp/MelSpectrogramBridge.h +33 -0
  279. package/cpp/kiss_fft/COPYING +11 -0
  280. package/cpp/kiss_fft/_kiss_fft_guts.h +167 -0
  281. package/cpp/kiss_fft/kiss_fft.c +424 -0
  282. package/cpp/kiss_fft/kiss_fft.h +160 -0
  283. package/cpp/kiss_fft/kiss_fft_log.h +36 -0
  284. package/cpp/kiss_fft/kiss_fftr.c +155 -0
  285. package/cpp/kiss_fft/kiss_fftr.h +54 -0
  286. package/expo-module.config.json +10 -0
  287. package/ios/AudioAnalysisData.swift +74 -0
  288. package/ios/AudioDeviceManager.swift +670 -0
  289. package/ios/AudioFeaturesWrapper.h +21 -0
  290. package/ios/AudioFeaturesWrapper.mm +63 -0
  291. package/ios/AudioNotificationManager.swift +154 -0
  292. package/ios/AudioProcessingHelpers.swift +797 -0
  293. package/ios/AudioProcessor.swift +1191 -0
  294. package/ios/AudioStreamError.swift +7 -0
  295. package/ios/AudioStreamManager.swift +2369 -0
  296. package/ios/AudioStreamManagerDelegate.swift +16 -0
  297. package/ios/AudioStudio.podspec +39 -0
  298. package/ios/AudioStudioModule.swift +1111 -0
  299. package/ios/AudioStudioTests/AudioFileHandlerTests.swift +338 -0
  300. package/ios/AudioStudioTests/AudioFormatUtilsTests.swift +331 -0
  301. package/ios/AudioStudioTests/AudioTestHelpers.swift +130 -0
  302. package/ios/AudioStudioTests/CompressedOnlyOutputTests.swift +294 -0
  303. package/ios/AudioStudioTests/EventEmissionIntervalTests.swift +105 -0
  304. package/ios/AudioStudioTests/Info.plist +22 -0
  305. package/ios/AudioStudioTests/README.md +39 -0
  306. package/ios/AudioStudioTests/SimpleAudioTest.swift +98 -0
  307. package/ios/AudioStudioTests/TestAudioGenerator.swift +75 -0
  308. package/ios/DataPoint.swift +54 -0
  309. package/ios/DecodingConfig.swift +59 -0
  310. package/ios/FFT.swift +62 -0
  311. package/ios/Features.swift +95 -0
  312. package/ios/ISSUE_IOS.md +68 -0
  313. package/ios/Logger.swift +39 -0
  314. package/ios/MelSpectrogramWrapper.h +30 -0
  315. package/ios/MelSpectrogramWrapper.mm +97 -0
  316. package/ios/NotificationExtension.swift +15 -0
  317. package/ios/RecordingResult.swift +22 -0
  318. package/ios/RecordingSettings.swift +311 -0
  319. package/ios/WaveformExtractor.swift +105 -0
  320. package/ios/tests/README.md +41 -0
  321. package/ios/tests/integration/buffer_and_fallback_test.swift +178 -0
  322. package/ios/tests/integration/buffer_duration_test.swift +185 -0
  323. package/ios/tests/integration/compressed_only_output_test.swift +271 -0
  324. package/ios/tests/integration/output_control_test.swift +322 -0
  325. package/ios/tests/integration/run_integration_tests.sh +37 -0
  326. package/ios/tests/opus_support_test_macos.swift +154 -0
  327. package/ios/tests/standalone/audio_processing_test.swift +144 -0
  328. package/ios/tests/standalone/audio_recording_test.swift +277 -0
  329. package/ios/tests/standalone/audio_streaming_test.swift +249 -0
  330. package/ios/tests/standalone/standalone_test.swift +144 -0
  331. package/package.json +146 -0
  332. package/plugin/build/index.cjs +194 -0
  333. package/plugin/build/index.d.cts +22 -0
  334. package/plugin/build/index.js +194 -0
  335. package/plugin/src/index.ts +285 -0
  336. package/plugin/tsconfig.json +10 -0
  337. package/plugin/tsconfig.tsbuildinfo +1 -0
  338. package/prebuilt/wasm/mel-spectrogram.js +18 -0
  339. package/src/AudioAnalysis/AudioAnalysis.types.ts +226 -0
  340. package/src/AudioAnalysis/audio-features-wasm.d.ts +37 -0
  341. package/src/AudioAnalysis/audioFeaturesWasm.ts +200 -0
  342. package/src/AudioAnalysis/extractAudioAnalysis.ts +350 -0
  343. package/src/AudioAnalysis/extractAudioData.ts +17 -0
  344. package/src/AudioAnalysis/extractMelSpectrogram.ts +140 -0
  345. package/src/AudioAnalysis/extractPreview.ts +34 -0
  346. package/src/AudioAnalysis/extractWaveform.ts +22 -0
  347. package/src/AudioAnalysis/mel-spectrogram-wasm.d.ts +48 -0
  348. package/src/AudioAnalysis/melSpectrogramWasm.ts +179 -0
  349. package/src/AudioDeviceManager.ts +800 -0
  350. package/src/AudioRecorder.provider.tsx +57 -0
  351. package/src/AudioStudio.native.ts +6 -0
  352. package/src/AudioStudio.types.ts +899 -0
  353. package/src/AudioStudio.web.ts +911 -0
  354. package/src/AudioStudioModule.ts +984 -0
  355. package/src/WebRecorder.web.ts +1114 -0
  356. package/src/constants/platformLimitations.ts +118 -0
  357. package/src/constants.ts +21 -0
  358. package/src/events.ts +63 -0
  359. package/src/hooks/useAudioDevices.ts +213 -0
  360. package/src/index.ts +67 -0
  361. package/src/trimAudio.ts +94 -0
  362. package/src/types/crc-32.d.ts +9 -0
  363. package/src/useAudioRecorder.tsx +784 -0
  364. package/src/utils/BlobFix.ts +561 -0
  365. package/src/utils/audioProcessing.ts +205 -0
  366. package/src/utils/cleanNativeOptions.ts +18 -0
  367. package/src/utils/concatenateBuffers.ts +24 -0
  368. package/src/utils/convertPCMToFloat32.ts +170 -0
  369. package/src/utils/crc32.ts +59 -0
  370. package/src/utils/encodingToBitDepth.ts +18 -0
  371. package/src/utils/getWavFileInfo.ts +132 -0
  372. package/src/utils/writeWavHeader.ts +115 -0
  373. package/src/workers/InlineFeaturesExtractor.web.tsx +291 -0
  374. package/src/workers/inlineAudioWebWorker.web.tsx +186 -0
  375. package/src/workers/wasmGlueString.web.ts +23 -0
@@ -0,0 +1,1114 @@
1
+ // packages/audio-studio/src/WebRecorder.web.ts
2
+
3
+ import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
4
+ import {
5
+ initMelStreamingWasm,
6
+ computeMelFrameWasm,
7
+ } from './AudioAnalysis/melSpectrogramWasm'
8
+ import { ConsoleLike, RecordingConfig } from './AudioStudio.types'
9
+ import {
10
+ EmitAudioAnalysisFunction,
11
+ EmitAudioEventFunction,
12
+ } from './AudioStudio.web'
13
+ import { encodingToBitDepth } from './utils/encodingToBitDepth'
14
+ import { writeWavHeader } from './utils/writeWavHeader'
15
+ import { InlineFeaturesExtractor } from './workers/InlineFeaturesExtractor.web'
16
+ import { InlineAudioWebWorker } from './workers/inlineAudioWebWorker.web'
17
+ import { wasmGlueJs } from './workers/wasmGlueString.web'
18
+
19
+ interface AudioWorkletEvent {
20
+ data: {
21
+ command: string
22
+ recordedData?: Float32Array
23
+ sampleRate?: number
24
+ position?: number
25
+ message?: string // For debug messages
26
+ }
27
+ }
28
+
29
+ interface AudioFeaturesEvent {
30
+ data: {
31
+ command: string
32
+ result: AudioAnalysis
33
+ }
34
+ }
35
+
36
+ const DEFAULT_WEB_BITDEPTH = 32
37
+ const DEFAULT_SEGMENT_DURATION_MS = 100
38
+ const DEFAULT_WEB_INTERVAL = 500
39
+ const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1
40
+
41
+ const TAG = 'WebRecorder'
42
+
43
+ export class WebRecorder {
44
+ public audioContext: AudioContext
45
+ private audioWorkletNode!: AudioWorkletNode
46
+ private featureExtractorWorker?: Worker
47
+ private featureExtractorWorkerUrl?: string
48
+ private source: MediaStreamAudioSourceNode
49
+ private emitAudioEventCallback: EmitAudioEventFunction
50
+ private emitAudioAnalysisCallback: EmitAudioAnalysisFunction
51
+ private config: RecordingConfig
52
+ private position: number = 0
53
+ private numberOfChannels: number // Number of audio channels
54
+ private bitDepth: number // Bit depth of the audio
55
+ private exportBitDepth: number // Bit depth of the audio
56
+ private audioAnalysisData: AudioAnalysis // Keep updating the full audio analysis data with latest events
57
+ private readonly logger?: ConsoleLike
58
+ private compressedMediaRecorder: MediaRecorder | null = null
59
+ private compressedChunks: Blob[] = []
60
+ private compressedSize: number = 0
61
+ private pendingCompressedChunk: Blob | null = null
62
+ private dataPointIdCounter: number = 0 // Add this property to track the counter
63
+ private deviceDisconnectionHandler: (() => void) | null = null
64
+ private readonly mediaStream: MediaStream | null = null
65
+ private readonly onInterruptionCallback?: (event: {
66
+ reason: string
67
+ isPaused: boolean
68
+ timestamp: number
69
+ }) => void
70
+ private _isDeviceDisconnected: boolean = false
71
+ private pcmData: Float32Array | null = null // Store original PCM data
72
+ private totalSampleCount: number = 0
73
+ private melWasmReady: boolean = false
74
+ private melWasmInitPromise: Promise<void> | null = null
75
+ private melPendingChunks: { chunk: Float32Array; sampleRate: number }[] = []
76
+ private pendingMelFrames: (number[] | null)[] = [] // Queued mel frames to attach to datapoints
77
+
78
+ /**
79
+ * Flag to indicate whether this is the first audio chunk after a device switch
80
+ * Used to maintain proper duration counting
81
+ */
82
+ public isFirstChunkAfterSwitch: boolean = false
83
+
84
+ /**
85
+ * Gets whether the recording device has been disconnected
86
+ */
87
+ get isDeviceDisconnected(): boolean {
88
+ return this._isDeviceDisconnected
89
+ }
90
+
91
+ /**
92
+ * Initializes a new WebRecorder instance for audio recording and processing
93
+ * @param audioContext - The AudioContext to use for recording
94
+ * @param source - The MediaStreamAudioSourceNode providing the audio input
95
+ * @param recordingConfig - Configuration options for the recording
96
+ * @param emitAudioEventCallback - Callback function for audio data events
97
+ * @param emitAudioAnalysisCallback - Callback function for audio analysis events
98
+ * @param onInterruption - Callback for recording interruptions
99
+ * @param logger - Optional logger for debugging information
100
+ */
101
+ constructor({
102
+ audioContext,
103
+ source,
104
+ recordingConfig,
105
+ emitAudioEventCallback,
106
+ emitAudioAnalysisCallback,
107
+ onInterruption,
108
+ logger,
109
+ }: {
110
+ audioContext: AudioContext
111
+ source: MediaStreamAudioSourceNode
112
+ recordingConfig: RecordingConfig
113
+ emitAudioEventCallback: EmitAudioEventFunction
114
+ emitAudioAnalysisCallback: EmitAudioAnalysisFunction
115
+ onInterruption?: (event: {
116
+ reason: string
117
+ isPaused: boolean
118
+ timestamp: number
119
+ }) => void
120
+ logger?: ConsoleLike
121
+ }) {
122
+ this.audioContext = audioContext
123
+ this.source = source
124
+ this.emitAudioEventCallback = emitAudioEventCallback
125
+ this.emitAudioAnalysisCallback = emitAudioAnalysisCallback
126
+ this.config = recordingConfig
127
+ this.logger = logger
128
+
129
+ const audioContextFormat = this.checkAudioContextFormat({
130
+ sampleRate: this.audioContext.sampleRate,
131
+ })
132
+ this.logger?.debug('Initialized WebRecorder with config:', {
133
+ sampleRate: audioContextFormat.sampleRate,
134
+ bitDepth: audioContextFormat.bitDepth,
135
+ numberOfChannels: audioContextFormat.numberOfChannels,
136
+ })
137
+
138
+ this.bitDepth = audioContextFormat.bitDepth
139
+ this.numberOfChannels =
140
+ audioContextFormat.numberOfChannels ||
141
+ DEFAULT_WEB_NUMBER_OF_CHANNELS // Default to 1 if not available
142
+ this.exportBitDepth =
143
+ encodingToBitDepth({
144
+ encoding: recordingConfig.encoding ?? 'pcm_32bit',
145
+ }) ||
146
+ audioContextFormat.bitDepth ||
147
+ DEFAULT_WEB_BITDEPTH
148
+
149
+ this.audioAnalysisData = {
150
+ amplitudeRange: { min: 0, max: 0 },
151
+ rmsRange: { min: 0, max: 0 },
152
+ dataPoints: [],
153
+ durationMs: 0,
154
+ samples: 0,
155
+ bitDepth: this.bitDepth,
156
+ numberOfChannels: this.numberOfChannels,
157
+ sampleRate: this.config.sampleRate || this.audioContext.sampleRate,
158
+ segmentDurationMs:
159
+ this.config.segmentDurationMs ?? DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms segments
160
+ extractionTimeMs: 0,
161
+ }
162
+
163
+ if (recordingConfig.enableProcessing) {
164
+ this.initFeatureExtractorWorker()
165
+ if (recordingConfig.features?.melSpectrogram) {
166
+ const sr =
167
+ this.config.sampleRate || this.audioContext.sampleRate
168
+ const segMs =
169
+ this.config.segmentDurationMs ?? DEFAULT_SEGMENT_DURATION_MS
170
+ const windowSamples = Math.floor((sr * segMs) / 1000)
171
+ this.melWasmInitPromise = initMelStreamingWasm(
172
+ sr,
173
+ 128,
174
+ 2048,
175
+ windowSamples,
176
+ windowSamples
177
+ )
178
+ .then(() => {
179
+ this.melWasmReady = true
180
+ this.logger?.log('Mel WASM streaming processor ready')
181
+ // Process any chunks that arrived during init
182
+ for (const pending of this.melPendingChunks) {
183
+ this.computeMelFrames(
184
+ pending.chunk,
185
+ pending.sampleRate
186
+ )
187
+ }
188
+ this.melPendingChunks = []
189
+ })
190
+ .catch((err) => {
191
+ console.error(`[${TAG}] Failed to init mel WASM:`, err)
192
+ this.melWasmInitPromise = null
193
+ this.melPendingChunks = []
194
+ })
195
+ }
196
+ }
197
+
198
+ // Initialize compressed recording if enabled
199
+ if (recordingConfig.output?.compressed?.enabled) {
200
+ this.initializeCompressedRecorder()
201
+ }
202
+
203
+ this.mediaStream = source.mediaStream
204
+ this.onInterruptionCallback = onInterruption
205
+
206
+ // Setup device disconnection detection
207
+ this.setupDeviceDisconnectionDetection()
208
+ }
209
+
210
+ /**
211
+ * Initializes the audio worklet using an inline script
212
+ * Creates and connects the audio processing pipeline
213
+ */
214
+ async init() {
215
+ try {
216
+ // Create and use inline audio worklet
217
+ const blob = new Blob([InlineAudioWebWorker], {
218
+ type: 'application/javascript',
219
+ })
220
+ const url = URL.createObjectURL(blob)
221
+ await this.audioContext.audioWorklet.addModule(url)
222
+
223
+ this.audioWorkletNode = new AudioWorkletNode(
224
+ this.audioContext,
225
+ 'recorder-processor'
226
+ )
227
+
228
+ this.audioWorkletNode.port.onmessage = async (
229
+ event: AudioWorkletEvent
230
+ ) => {
231
+ const command = event.data.command
232
+ if (command === 'debug') {
233
+ this.logger?.debug(`[AudioWorklet] ${event.data.message}`)
234
+ return
235
+ }
236
+
237
+ if (command !== 'newData') return
238
+
239
+ const pcmBufferFloat = event.data.recordedData
240
+ if (!pcmBufferFloat) {
241
+ this.logger?.warn('Received empty audio buffer', event)
242
+ return
243
+ }
244
+
245
+ // Process data in smaller chunks and emit immediately
246
+ const sampleRate =
247
+ event.data.sampleRate ?? this.audioContext.sampleRate
248
+ // Use chunk size from config interval or default to 2 seconds
249
+ const intervalMs = this.config.interval ?? DEFAULT_WEB_INTERVAL
250
+ const chunkSize = Math.floor(sampleRate * (intervalMs / 1000))
251
+ const duration = pcmBufferFloat.length / sampleRate
252
+
253
+ // Use incoming position if provided by worklet, otherwise use our tracked position
254
+ const incomingPosition =
255
+ typeof event.data.position === 'number'
256
+ ? event.data.position
257
+ : this.position
258
+
259
+ // Simple position tracking for logging (no duplicate filtering)
260
+ this.logger?.debug(
261
+ `Audio chunk: position=${incomingPosition.toFixed(3)}s, size=${pcmBufferFloat.length}`
262
+ )
263
+
264
+ // Calculate bytes per sample based on bit depth
265
+ const bytesPerSample = this.bitDepth / 8
266
+
267
+ // Emit chunks without storing them
268
+ for (let i = 0; i < pcmBufferFloat.length; i += chunkSize) {
269
+ const chunk = pcmBufferFloat.slice(i, i + chunkSize)
270
+ const chunkPosition = incomingPosition + i / sampleRate
271
+
272
+ // Calculate byte positions and samples
273
+ const startPosition = Math.floor(i * bytesPerSample)
274
+ const endPosition = Math.floor(
275
+ (i + chunk.length) * bytesPerSample
276
+ )
277
+ const samples = chunk.length // Number of samples in this chunk
278
+
279
+ // Only store PCM data if primary output is enabled
280
+ const shouldStoreUncompressed =
281
+ this.config.output?.primary?.enabled ?? true
282
+
283
+ // Store PCM chunks when needed - this is for the final WAV file
284
+ if (shouldStoreUncompressed) {
285
+ // Store the original Float32Array data for later WAV creation
286
+ this.appendPcmData(chunk)
287
+ this.totalSampleCount += chunk.length
288
+ }
289
+
290
+ this.computeMelFrames(chunk, sampleRate)
291
+
292
+ // Process features if enabled
293
+ if (
294
+ this.config.enableProcessing &&
295
+ this.featureExtractorWorker
296
+ ) {
297
+ this.featureExtractorWorker.postMessage({
298
+ command: 'process',
299
+ channelData: chunk,
300
+ sampleRate,
301
+ segmentDurationMs:
302
+ this.config.segmentDurationMs ??
303
+ DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms
304
+ bitDepth: this.bitDepth,
305
+ fullAudioDurationMs: chunkPosition * 1000,
306
+ numberOfChannels: this.numberOfChannels,
307
+ features: this.config.features,
308
+ intervalAnalysis: this.config.intervalAnalysis,
309
+ startPosition,
310
+ endPosition,
311
+ samples,
312
+ })
313
+ }
314
+
315
+ // Prepare compression data if available
316
+ const compression = this.pendingCompressedChunk
317
+ ? {
318
+ data: this.pendingCompressedChunk,
319
+ size: this.pendingCompressedChunk.size,
320
+ totalSize: this.compressedSize,
321
+ mimeType: 'audio/webm',
322
+ format:
323
+ this.config.output?.compressed?.format ??
324
+ 'opus',
325
+ bitrate:
326
+ this.config.output?.compressed?.bitrate ??
327
+ 128000,
328
+ }
329
+ : undefined
330
+
331
+ // Emit chunk immediately - whether compressed or not
332
+ this.emitAudioEventCallback({
333
+ data: chunk,
334
+ position: chunkPosition,
335
+ compression,
336
+ })
337
+
338
+ // Reset pending compressed chunk after we've used it
339
+ this.pendingCompressedChunk = null
340
+ }
341
+
342
+ // Update our position based on the worklet's position if provided
343
+ this.position = incomingPosition + duration
344
+ }
345
+
346
+ // Ensure we use all relevant settings from config
347
+ const recordSampleRate = this.audioContext.sampleRate
348
+ const exportSampleRate =
349
+ this.config.sampleRate ?? this.audioContext.sampleRate
350
+ const channels = this.config.channels ?? this.numberOfChannels
351
+ const interval = this.config.interval ?? DEFAULT_WEB_INTERVAL
352
+
353
+ this.logger?.debug(`WebRecorder initialized with config:`, {
354
+ recordSampleRate,
355
+ exportSampleRate,
356
+ bitDepth: this.bitDepth,
357
+ exportBitDepth: this.exportBitDepth,
358
+ channels,
359
+ interval,
360
+ position: this.position,
361
+ deviceId: this.config.deviceId ?? 'default',
362
+ compression: this.config.output?.compressed
363
+ ? {
364
+ enabled: this.config.output.compressed.enabled,
365
+ format: this.config.output.compressed.format,
366
+ bitrate: this.config.output.compressed.bitrate,
367
+ }
368
+ : 'disabled',
369
+ })
370
+
371
+ // Initialize the worklet with all settings from config
372
+ this.audioWorkletNode.port.postMessage({
373
+ command: 'init',
374
+ recordSampleRate,
375
+ exportSampleRate,
376
+ bitDepth: this.bitDepth,
377
+ exportBitDepth: this.exportBitDepth,
378
+ channels,
379
+ interval,
380
+ position: this.position, // Pass the current position to the processor
381
+ enableLogging: true,
382
+ streamFormat: this.config.streamFormat,
383
+ })
384
+
385
+ // Connect the source to the AudioWorkletNode and start recording
386
+ this.source.connect(this.audioWorkletNode)
387
+ this.audioWorkletNode.connect(this.audioContext.destination)
388
+ } catch (error) {
389
+ console.error(`[${TAG}] Failed to initialize WebRecorder`, error)
390
+ }
391
+ }
392
+
393
+ /**
394
+ * Append new PCM data to the existing buffer
395
+ * @param newData New Float32Array data to append
396
+ */
397
+ private appendPcmData(newData: Float32Array): void {
398
+ // Clone the incoming data to ensure it's not modified
399
+ const dataToAdd = new Float32Array(newData)
400
+
401
+ if (!this.pcmData) {
402
+ // First chunk - create a copy to avoid references to original data
403
+ this.pcmData = new Float32Array(dataToAdd)
404
+ return
405
+ }
406
+
407
+ // Create a new buffer with increased size
408
+ const newBuffer = new Float32Array(
409
+ this.pcmData.length + dataToAdd.length
410
+ )
411
+
412
+ // Copy existing data
413
+ newBuffer.set(this.pcmData)
414
+
415
+ // Append new data
416
+ newBuffer.set(dataToAdd, this.pcmData.length)
417
+
418
+ // Replace existing buffer
419
+ this.pcmData = newBuffer
420
+ }
421
+
422
+ /**
423
+ * Initializes the feature extractor worker for audio analysis
424
+ * Creates an inline worker from a blob for audio feature extraction
425
+ */
426
+ initFeatureExtractorWorker() {
427
+ try {
428
+ const blob = new Blob([wasmGlueJs, '\n', InlineFeaturesExtractor], {
429
+ type: 'application/javascript',
430
+ })
431
+ const url = URL.createObjectURL(blob)
432
+ this.featureExtractorWorkerUrl = url
433
+ this.featureExtractorWorker = new Worker(url)
434
+ this.featureExtractorWorker.onmessage =
435
+ this.handleFeatureExtractorMessage.bind(this)
436
+ this.featureExtractorWorker.onerror = (error) => {
437
+ console.error(`[${TAG}] Feature extractor worker error:`, error)
438
+ }
439
+
440
+ // Initialize worker with counter if needed
441
+ if (this.dataPointIdCounter > 0) {
442
+ this.featureExtractorWorker.postMessage({
443
+ command: 'resetCounter',
444
+ value: this.dataPointIdCounter,
445
+ })
446
+ this.logger?.debug(
447
+ `Initialized worker with counter value ${this.dataPointIdCounter}`
448
+ )
449
+ }
450
+
451
+ this.logger?.log(
452
+ 'Feature extractor worker initialized successfully'
453
+ )
454
+ } catch (error) {
455
+ console.error(
456
+ `[${TAG}] Failed to initialize feature extractor worker`,
457
+ error
458
+ )
459
+ }
460
+ }
461
+
462
+ /**
463
+ * Processes audio analysis results from the feature extractor worker
464
+ * Updates the audio analysis data and emits events
465
+ * @param event - The event containing audio analysis results
466
+ */
467
+ handleFeatureExtractorMessage(event: AudioFeaturesEvent) {
468
+ if (event.data.command !== 'features') return
469
+
470
+ const segmentResult = event.data.result
471
+
472
+ // Attach WASM-computed mel frames to datapoints
473
+ if (this.pendingMelFrames.length > 0) {
474
+ for (const dp of segmentResult.dataPoints) {
475
+ if (this.pendingMelFrames.length === 0) break
476
+ const melFrame = this.pendingMelFrames.shift()
477
+ if (melFrame) {
478
+ if (!dp.features) dp.features = {}
479
+ dp.features.melSpectrogram = melFrame
480
+ }
481
+ }
482
+ }
483
+
484
+ const uniqueNewDataPoints = this.filterUniqueDataPoints(
485
+ segmentResult.dataPoints
486
+ )
487
+
488
+ // Update counter based on the highest ID seen
489
+ this.updateDataPointCounter(uniqueNewDataPoints)
490
+
491
+ // Update analysis data with the new results
492
+ this.updateAudioAnalysisData(segmentResult, uniqueNewDataPoints)
493
+
494
+ // Send filtered result to avoid duplicate IDs
495
+ const filteredSegmentResult = {
496
+ ...segmentResult,
497
+ dataPoints: uniqueNewDataPoints,
498
+ }
499
+
500
+ this.emitAudioAnalysisCallback(filteredSegmentResult)
501
+ }
502
+
503
+ /**
504
+ * Compute mel spectrogram frames via WASM C++ for each segment in the chunk.
505
+ * Frames are queued in pendingMelFrames and attached to datapoints in handleFeatureExtractorMessage.
506
+ */
507
+ private computeMelFrames(chunk: Float32Array, sampleRate: number): void {
508
+ if (!this.config.features?.melSpectrogram) return
509
+ if (!this.melWasmReady) {
510
+ // Buffer chunks while WASM is still initializing
511
+ if (this.melWasmInitPromise) {
512
+ this.melPendingChunks.push({
513
+ chunk: new Float32Array(chunk),
514
+ sampleRate,
515
+ })
516
+ }
517
+ return
518
+ }
519
+ const segMs =
520
+ this.config.segmentDurationMs ?? DEFAULT_SEGMENT_DURATION_MS
521
+ const samplesPerSeg = Math.floor((sampleRate * segMs) / 1000)
522
+ const numSegs = Math.floor(chunk.length / samplesPerSeg)
523
+ for (let s = 0; s < numSegs; s++) {
524
+ const seg = chunk.slice(s * samplesPerSeg, (s + 1) * samplesPerSeg)
525
+ this.pendingMelFrames.push(computeMelFrameWasm(seg))
526
+ }
527
+ const rem = chunk.length % samplesPerSeg
528
+ if (rem > samplesPerSeg / 4) {
529
+ const seg = chunk.slice(numSegs * samplesPerSeg)
530
+ this.pendingMelFrames.push(computeMelFrameWasm(seg))
531
+ }
532
+
533
+ // Cap queue to prevent unbounded memory growth if consumer falls behind
534
+ const MAX_PENDING_MEL_FRAMES = 200
535
+ if (this.pendingMelFrames.length > MAX_PENDING_MEL_FRAMES) {
536
+ this.pendingMelFrames = this.pendingMelFrames.slice(
537
+ -MAX_PENDING_MEL_FRAMES
538
+ )
539
+ }
540
+ }
541
+
542
+ /**
543
+ * Filters out data points with duplicate IDs
544
+ */
545
+ private filterUniqueDataPoints(dataPoints: any[]): any[] {
546
+ // Track existing IDs to prevent duplicates
547
+ const existingIds = new Set(
548
+ this.audioAnalysisData.dataPoints.map((dp) => dp.id)
549
+ )
550
+
551
+ // Filter out datapoints with duplicate IDs
552
+ const uniquePoints = dataPoints.filter((dp) => !existingIds.has(dp.id))
553
+
554
+ // Log filtered duplicates if any
555
+ if (uniquePoints.length < dataPoints.length && this.logger?.warn) {
556
+ this.logger.warn(
557
+ `Filtered ${dataPoints.length - uniquePoints.length} duplicate datapoints`
558
+ )
559
+ }
560
+
561
+ return uniquePoints
562
+ }
563
+
564
+ /**
565
+ * Updates the counter based on the highest ID in datapoints
566
+ */
567
+ private updateDataPointCounter(dataPoints: any[]): void {
568
+ if (dataPoints.length === 0) return
569
+
570
+ const lastDataPoint = dataPoints[dataPoints.length - 1]
571
+ if (lastDataPoint && typeof lastDataPoint.id === 'number') {
572
+ const nextIdValue = lastDataPoint.id + 1
573
+ if (nextIdValue > this.dataPointIdCounter) {
574
+ this.dataPointIdCounter = nextIdValue
575
+ this.logger?.debug(
576
+ `Counter updated to ${this.dataPointIdCounter}`
577
+ )
578
+ }
579
+ }
580
+ }
581
+
582
+ /**
583
+ * Updates audio analysis data with segment results
584
+ */
585
+ private updateAudioAnalysisData(
586
+ segmentResult: AudioAnalysis,
587
+ uniqueDataPoints: any[]
588
+ ): void {
589
+ // Add unique data points to our analysis data
590
+ this.audioAnalysisData.dataPoints.push(...uniqueDataPoints)
591
+ this.audioAnalysisData.durationMs += segmentResult.durationMs
592
+ this.audioAnalysisData.sampleRate = segmentResult.sampleRate
593
+
594
+ // Update amplitude range if present
595
+ if (segmentResult.amplitudeRange) {
596
+ this.audioAnalysisData.amplitudeRange = this.mergeRange(
597
+ this.audioAnalysisData.amplitudeRange,
598
+ segmentResult.amplitudeRange
599
+ )
600
+ }
601
+
602
+ // Update RMS range if present
603
+ if (segmentResult.rmsRange) {
604
+ this.audioAnalysisData.rmsRange = this.mergeRange(
605
+ this.audioAnalysisData.rmsRange,
606
+ segmentResult.rmsRange
607
+ )
608
+ }
609
+ }
610
+
611
+ /**
612
+ * Merges value ranges
613
+ */
614
+ private mergeRange(
615
+ existing: { min: number; max: number } | undefined,
616
+ newRange: { min: number; max: number }
617
+ ): { min: number; max: number } {
618
+ if (!existing) return { ...newRange }
619
+
620
+ return {
621
+ min: Math.min(existing.min, newRange.min),
622
+ max: Math.max(existing.max, newRange.max),
623
+ }
624
+ }
625
+
626
+ /**
627
+ * Reset the data point counter to a specific value or zero
628
+ * @param startCounterFrom Optional value to start the counter from (for continuing from previous recordings)
629
+ */
630
+ resetDataPointCounter(startCounterFrom?: number): void {
631
+ // Set the counter with the passed value or 0
632
+ this.dataPointIdCounter = startCounterFrom ?? 0
633
+ this.logger?.debug(
634
+ `Reset data point counter to ${this.dataPointIdCounter}`
635
+ )
636
+
637
+ // Update worker counter if available
638
+ if (this.featureExtractorWorker) {
639
+ this.featureExtractorWorker.postMessage({
640
+ command: 'resetCounter',
641
+ value: this.dataPointIdCounter,
642
+ })
643
+ } else {
644
+ this.logger?.warn(
645
+ 'No feature extractor worker available to update counter'
646
+ )
647
+ }
648
+ }
649
+
650
+ /**
651
+ * Get the current data point counter value
652
+ * @returns The current value of the data point counter
653
+ */
654
+ getDataPointCounter(): number {
655
+ return this.dataPointIdCounter
656
+ }
657
+
658
+ /**
659
+ * Prepares the recorder for continuity after device switch
660
+ * Sets up all necessary state to maintain proper recording continuity
661
+ */
662
+ prepareForDeviceSwitch(): void {
663
+ this.isFirstChunkAfterSwitch = true
664
+ this.logger?.debug(
665
+ `Prepared for device switch at position ${this.position}s`
666
+ )
667
+ }
668
+
669
+ /**
670
+ * Starts the audio recording process
671
+ * Connects the audio nodes and begins capturing audio data
672
+ * @param preserveCounters If true, do not reset the counter (used for device switching)
673
+ */
674
+ start(preserveCounters = false) {
675
+ this.source.connect(this.audioWorkletNode)
676
+ this.audioWorkletNode.connect(this.audioContext.destination)
677
+
678
+ // Only reset the counter when not preserving state (e.g., for a fresh recording)
679
+ if (!preserveCounters) {
680
+ this.logger?.debug(
681
+ 'Starting fresh recording, resetting counter to 0'
682
+ )
683
+ this.resetDataPointCounter(0) // Explicitly reset to 0 for new recordings
684
+ this.isFirstChunkAfterSwitch = false
685
+
686
+ // Clear PCM data for new recording
687
+ this.pcmData = null
688
+ this.totalSampleCount = 0
689
+ } else {
690
+ this.logger?.debug(
691
+ `Preserving counter at ${this.dataPointIdCounter} during device switch`
692
+ )
693
+ }
694
+
695
+ if (this.compressedMediaRecorder) {
696
+ this.compressedMediaRecorder.start(this.config.interval ?? 1000)
697
+ }
698
+ }
699
+
700
+ /**
701
+ * Creates a WAV file from the stored PCM data
702
+ */
703
+ private createWavFromPcmData(): Blob | null {
704
+ try {
705
+ // Check if we have PCM data
706
+ if (!this.pcmData || this.pcmData.length === 0) {
707
+ this.logger?.warn('No PCM data available to create WAV file')
708
+ return null
709
+ }
710
+
711
+ const sampleRate =
712
+ this.config.sampleRate ?? this.audioContext.sampleRate
713
+ const channels = this.numberOfChannels || 1
714
+
715
+ // Convert float32 PCM data to 16-bit PCM for WAV
716
+ const bytesPerSample = 2 // 16-bit = 2 bytes
717
+ const dataLength = this.pcmData.length * bytesPerSample
718
+ const buffer = new ArrayBuffer(dataLength)
719
+ const view = new DataView(buffer)
720
+
721
+ // Convert Float32Array (-1 to 1) to Int16Array (-32768 to 32767)
722
+ for (let i = 0; i < this.pcmData.length; i++) {
723
+ const sample = Math.max(-1, Math.min(1, this.pcmData[i]))
724
+ const int16Value = Math.round(sample * 32767)
725
+ view.setInt16(i * 2, int16Value, true)
726
+ }
727
+
728
+ // Use the existing writeWavHeader utility to add a WAV header
729
+ const wavBuffer = writeWavHeader({
730
+ buffer,
731
+ sampleRate,
732
+ numChannels: channels,
733
+ bitDepth: 16,
734
+ isFloat: false,
735
+ })
736
+
737
+ return new Blob([wavBuffer], { type: 'audio/wav' })
738
+ } catch (error) {
739
+ this.logger?.error('Error creating WAV file from PCM data:', error)
740
+ return null
741
+ }
742
+ }
743
+
744
+ /**
745
+ * Stops the audio recording process and returns the recorded data
746
+ * @returns Promise resolving to an object containing compressed and/or uncompressed blobs
747
+ */
748
+ async stop(): Promise<{ compressedBlob?: Blob; uncompressedBlob?: Blob }> {
749
+ try {
750
+ // Stop any compressed recording first
751
+ if (
752
+ this.compressedMediaRecorder &&
753
+ this.compressedMediaRecorder.state !== 'inactive'
754
+ ) {
755
+ this.compressedMediaRecorder.stop()
756
+ }
757
+
758
+ // Wait for any pending compressed chunks to be processed
759
+ if (this.compressedMediaRecorder) {
760
+ // Small delay to ensure all data is processed
761
+ await new Promise((resolve) => setTimeout(resolve, 100))
762
+ }
763
+
764
+ // Create uncompressed WAV file from the PCM data
765
+ let uncompressedBlob: Blob | undefined
766
+
767
+ // Only create WAV if we have PCM data
768
+ if (this.pcmData && this.pcmData.length > 0) {
769
+ uncompressedBlob = this.createWavFromPcmData() || undefined
770
+ }
771
+
772
+ // Return the compressed and/or uncompressed blobs if available
773
+ return {
774
+ compressedBlob:
775
+ this.compressedChunks.length > 0
776
+ ? new Blob(this.compressedChunks, {
777
+ type: 'audio/webm;codecs=opus',
778
+ })
779
+ : undefined,
780
+ uncompressedBlob,
781
+ }
782
+ } finally {
783
+ this.cleanup()
784
+ // Reset the chunks array
785
+ this.compressedChunks = []
786
+ this.compressedSize = 0
787
+ this.pendingCompressedChunk = null
788
+ this.pcmData = null
789
+ this.totalSampleCount = 0
790
+ this.dataPointIdCounter = 0 // Reset counter
791
+ }
792
+ }
793
+
794
+ /**
795
+ * Cleans up resources when recording is stopped
796
+ * Closes audio context and disconnects nodes
797
+ */
798
+ public cleanup() {
799
+ // Remove device disconnection handler
800
+ if (this.deviceDisconnectionHandler) {
801
+ this.deviceDisconnectionHandler()
802
+ this.deviceDisconnectionHandler = null
803
+ }
804
+
805
+ // Check if AudioContext is already closed before attempting to close it
806
+ if (this.audioContext && this.audioContext.state !== 'closed') {
807
+ this.audioContext.close().catch((e) => {
808
+ // Log closure errors but continue cleanup
809
+ this.logger?.warn('Error closing AudioContext:', e)
810
+ })
811
+ }
812
+
813
+ // Safely disconnect audioWorkletNode if it exists
814
+ if (this.audioWorkletNode) {
815
+ try {
816
+ this.audioWorkletNode.disconnect()
817
+ } catch (e) {
818
+ // Log disconnection errors but continue cleanup
819
+ this.logger?.warn('Error disconnecting audioWorkletNode:', e)
820
+ }
821
+ }
822
+
823
+ // Safely disconnect source if it exists
824
+ if (this.source) {
825
+ try {
826
+ this.source.disconnect()
827
+ } catch (e) {
828
+ // Log disconnection errors but continue cleanup
829
+ this.logger?.warn('Error disconnecting source:', e)
830
+ }
831
+ }
832
+
833
+ // Terminate feature extractor worker and revoke blob URL
834
+ if (this.featureExtractorWorker) {
835
+ this.featureExtractorWorker.terminate()
836
+ this.featureExtractorWorker = undefined
837
+ }
838
+ if (this.featureExtractorWorkerUrl) {
839
+ URL.revokeObjectURL(this.featureExtractorWorkerUrl)
840
+ this.featureExtractorWorkerUrl = undefined
841
+ }
842
+
843
+ // Always stop media stream tracks to release hardware resources
844
+ this.stopMediaStreamTracks()
845
+
846
+ // Mark as disconnected to prevent future errors
847
+ this._isDeviceDisconnected = true
848
+ }
849
+
850
+ /**
851
+ * Pauses the audio recording process
852
+ * Disconnects audio nodes and pauses the media recorder
853
+ */
854
+ pause() {
855
+ try {
856
+ // Note: We're just pausing, not disconnecting the device
857
+ // Simply disconnect nodes temporarily without marking device as disconnected
858
+ this.source.disconnect(this.audioWorkletNode)
859
+ this.audioWorkletNode.disconnect(this.audioContext.destination)
860
+ this.audioWorkletNode.port.postMessage({ command: 'pause' })
861
+
862
+ if (this.compressedMediaRecorder?.state === 'recording') {
863
+ this.compressedMediaRecorder.pause()
864
+ }
865
+
866
+ this.logger?.debug('Recording paused successfully')
867
+ } catch (error) {
868
+ this.logger?.error('Error in pause(): ', error)
869
+ // Already disconnected, just ignore and continue
870
+ }
871
+ }
872
+
873
+ /**
874
+ * Stops all media stream tracks to release hardware resources
875
+ * Ensures recording indicators (like microphone icon) are turned off
876
+ */
877
+ public stopMediaStreamTracks() {
878
+ // Stop all audio tracks to stop the recording icon
879
+ if (this.mediaStream) {
880
+ const tracks = this.mediaStream.getTracks()
881
+ tracks.forEach((track) => track.stop())
882
+ } else if (this.source?.mediaStream) {
883
+ const tracks = this.source.mediaStream.getTracks()
884
+ tracks.forEach((track) => track.stop())
885
+ }
886
+ }
887
+
888
+ /**
889
+ * Determines the audio format capabilities of the current audio context
890
+ * @param sampleRate - The sample rate to check
891
+ * @returns Object containing format information (sample rate, bit depth, channels)
892
+ */
893
+ private checkAudioContextFormat({ sampleRate }: { sampleRate: number }) {
894
+ // Create a silent AudioBuffer
895
+ const frameCount = sampleRate * 1.0 // 1 second buffer
896
+ const audioBuffer = this.audioContext.createBuffer(
897
+ 1,
898
+ frameCount,
899
+ sampleRate
900
+ )
901
+
902
+ // Check the format
903
+ const channelData = audioBuffer.getChannelData(0)
904
+ const bitDepth = channelData.BYTES_PER_ELEMENT * 8 // 4 bytes per element means 32-bit
905
+
906
+ return {
907
+ sampleRate: audioBuffer.sampleRate,
908
+ bitDepth,
909
+ numberOfChannels: audioBuffer.numberOfChannels,
910
+ }
911
+ }
912
+
913
+ /**
914
+ * Resumes a paused recording
915
+ * Reconnects audio nodes and resumes the media recorder
916
+ */
917
+ resume() {
918
+ // If device was disconnected, we can't resume
919
+ if (this._isDeviceDisconnected) {
920
+ this.logger?.warn('Cannot resume recording: device disconnected')
921
+ return
922
+ }
923
+
924
+ try {
925
+ this.source.connect(this.audioWorkletNode)
926
+ this.audioWorkletNode.connect(this.audioContext.destination)
927
+ this.audioWorkletNode.port.postMessage({ command: 'resume' })
928
+ this.compressedMediaRecorder?.resume()
929
+ } catch (error: unknown) {
930
+ this.logger?.error('Error in resume(): ', error)
931
+ // Rethrow the error to inform callers
932
+ throw new Error(
933
+ `Failed to resume recording: ${error instanceof Error ? error.message : 'unknown error'}`
934
+ )
935
+ }
936
+ }
937
+
938
+ /**
939
+ * Initializes the compressed media recorder if compression is enabled
940
+ * Sets up event handlers for compressed audio data
941
+ */
942
+ private initializeCompressedRecorder() {
943
+ try {
944
+ const mimeType = 'audio/webm;codecs=opus'
945
+ if (!MediaRecorder.isTypeSupported(mimeType)) {
946
+ this.logger?.warn(
947
+ 'Opus compression not supported in this browser'
948
+ )
949
+ return
950
+ }
951
+
952
+ this.compressedMediaRecorder = new MediaRecorder(
953
+ this.source.mediaStream,
954
+ {
955
+ mimeType,
956
+ audioBitsPerSecond:
957
+ this.config.output?.compressed?.bitrate ?? 128000,
958
+ }
959
+ )
960
+
961
+ this.compressedMediaRecorder.ondataavailable = (event) => {
962
+ if (event.data.size > 0) {
963
+ // Store the compressed chunk for final blob creation
964
+ this.compressedChunks.push(event.data)
965
+ this.compressedSize += event.data.size
966
+
967
+ // Store the pending compressed chunk for the next PCM chunk to use
968
+ this.pendingCompressedChunk = event.data
969
+ }
970
+ }
971
+ } catch (error) {
972
+ this.logger?.error(
973
+ 'Failed to initialize compressed recorder:',
974
+ error
975
+ )
976
+ // Setting to null to indicate initialization failed
977
+ this.compressedMediaRecorder = null
978
+ }
979
+ }
980
+
981
+ /**
982
+ * Processes features if enabled
983
+ */
984
+ processFeatures(
985
+ chunk: Float32Array,
986
+ sampleRate: number,
987
+ chunkPosition: number,
988
+ startPosition: number,
989
+ endPosition: number,
990
+ samples: number
991
+ ) {
992
+ this.computeMelFrames(chunk, sampleRate)
993
+
994
+ if (this.config.enableProcessing && this.featureExtractorWorker) {
995
+ this.featureExtractorWorker.postMessage({
996
+ command: 'process',
997
+ channelData: chunk,
998
+ sampleRate,
999
+ segmentDurationMs:
1000
+ this.config.segmentDurationMs ??
1001
+ DEFAULT_SEGMENT_DURATION_MS, // Default to 100ms
1002
+ bitDepth: this.bitDepth,
1003
+ fullAudioDurationMs: chunkPosition * 1000,
1004
+ numberOfChannels: this.numberOfChannels,
1005
+ features: this.config.features,
1006
+ intervalAnalysis: this.config.intervalAnalysis,
1007
+ startPosition,
1008
+ endPosition,
1009
+ samples,
1010
+ })
1011
+ }
1012
+ }
1013
+
1014
+ /**
1015
+ * Sets up detection for device disconnection events
1016
+ */
1017
+ private setupDeviceDisconnectionDetection() {
1018
+ if (!this.mediaStream) return
1019
+
1020
+ // Function to handle track ending (which happens on device disconnection)
1021
+ const handleTrackEnded = () => {
1022
+ this.logger?.warn('Audio track ended - device disconnected')
1023
+ this._isDeviceDisconnected = true
1024
+
1025
+ // Use the callback to notify parent component about device disconnection
1026
+ if (this.onInterruptionCallback) {
1027
+ this.onInterruptionCallback({
1028
+ reason: 'deviceDisconnected',
1029
+ isPaused: true,
1030
+ timestamp: Date.now(),
1031
+ })
1032
+ this.logger?.debug('Notified about device disconnection')
1033
+ }
1034
+
1035
+ // Ensure we disconnect nodes to prevent zombie recordings
1036
+ if (this.audioWorkletNode) {
1037
+ this.audioWorkletNode.port.postMessage({
1038
+ command: 'deviceDisconnected',
1039
+ })
1040
+
1041
+ try {
1042
+ this.source.disconnect(this.audioWorkletNode)
1043
+ this.audioWorkletNode.disconnect()
1044
+ } catch (e) {
1045
+ // Ignore disconnection errors as the track might already be gone
1046
+ this.logger?.warn(
1047
+ 'Error disconnecting audioWorkletNode:',
1048
+ e
1049
+ )
1050
+ }
1051
+ }
1052
+ }
1053
+
1054
+ // Add listeners to all audio tracks
1055
+ const tracks = this.mediaStream.getAudioTracks()
1056
+ tracks.forEach((track) => {
1057
+ track.addEventListener('ended', handleTrackEnded)
1058
+ })
1059
+
1060
+ // Store the handler for cleanup
1061
+ this.deviceDisconnectionHandler = () => {
1062
+ tracks.forEach((track) => {
1063
+ track.removeEventListener('ended', handleTrackEnded)
1064
+ })
1065
+ }
1066
+ }
1067
+
1068
+ /**
1069
+ * Explicitly set the position for continuous recording across device switches
1070
+ * @param position The position in seconds to continue from
1071
+ */
1072
+ setPosition(position: number): void {
1073
+ if (position >= 0) {
1074
+ this.position = position
1075
+ this.logger?.debug(`Position explicitly set to ${position} seconds`)
1076
+ } else {
1077
+ this.logger?.warn(`Invalid position value: ${position}, ignoring`)
1078
+ }
1079
+ }
1080
+
1081
+ /**
1082
+ * Get the current position in seconds
1083
+ * @returns The current position
1084
+ */
1085
+ getPosition(): number {
1086
+ return this.position
1087
+ }
1088
+
1089
+ /**
1090
+ * Gets the current compressed chunks
1091
+ * @returns Array of current compressed audio chunks
1092
+ */
1093
+ getCompressedChunks(): Blob[] {
1094
+ return [...this.compressedChunks]
1095
+ }
1096
+
1097
+ /**
1098
+ * Sets the compressed chunks from a previous recorder
1099
+ * @param chunks Array of compressed chunks from a previous recorder
1100
+ */
1101
+ setCompressedChunks(chunks: Blob[]): void {
1102
+ if (chunks && chunks.length > 0) {
1103
+ this.logger?.debug(
1104
+ `Adding ${chunks.length} compressed chunks from previous device`
1105
+ )
1106
+ this.compressedChunks = [...chunks, ...this.compressedChunks]
1107
+ // Update size
1108
+ this.compressedSize = this.compressedChunks.reduce(
1109
+ (size, chunk) => size + chunk.size,
1110
+ 0
1111
+ )
1112
+ }
1113
+ }
1114
+ }