@siteed/audio-studio 3.0.2 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -1
- package/android/src/main/java/net/siteed/audiostudio/AudioRecorderManager.kt +41 -35
- package/android/src/main/java/net/siteed/audiostudio/AudioStudioModule.kt +500 -479
- package/build/cjs/AudioAnalysis/AudioAnalysis.types.js.map +1 -1
- package/build/cjs/AudioAnalysis/audioFeaturesWasm.js +10 -7
- package/build/cjs/AudioAnalysis/audioFeaturesWasm.js.map +1 -1
- package/build/cjs/AudioAnalysis/audioFeaturesWasm.web.js +78 -97
- package/build/cjs/AudioAnalysis/audioFeaturesWasm.web.js.map +1 -1
- package/build/cjs/AudioAnalysis/extractAudioAnalysis.js +15 -12
- package/build/cjs/AudioAnalysis/extractAudioAnalysis.js.map +1 -1
- package/build/cjs/AudioAnalysis/extractAudioData.js +144 -2
- package/build/cjs/AudioAnalysis/extractAudioData.js.map +1 -1
- package/build/cjs/AudioAnalysis/melSpectrogramWasm.web.js +9 -56
- package/build/cjs/AudioAnalysis/melSpectrogramWasm.web.js.map +1 -1
- package/build/cjs/AudioAnalysis/wasmConfig.js +4 -4
- package/build/cjs/AudioAnalysis/wasmConfig.js.map +1 -1
- package/build/cjs/AudioAnalysis/wasmLoader.web.js +79 -0
- package/build/cjs/AudioAnalysis/wasmLoader.web.js.map +1 -0
- package/build/cjs/AudioStudioModule.js +4 -599
- package/build/cjs/AudioStudioModule.js.map +1 -1
- package/build/cjs/trimAudio.js +227 -0
- package/build/cjs/trimAudio.js.map +1 -1
- package/build/cjs/utils/encodeCompressedAudio.web.js +65 -0
- package/build/cjs/utils/encodeCompressedAudio.web.js.map +1 -0
- package/build/cjs/utils/resampleAudioBuffer.web.js +25 -0
- package/build/cjs/utils/resampleAudioBuffer.web.js.map +1 -0
- package/build/esm/AudioAnalysis/AudioAnalysis.types.js.map +1 -1
- package/build/esm/AudioAnalysis/audioFeaturesWasm.js +8 -5
- package/build/esm/AudioAnalysis/audioFeaturesWasm.js.map +1 -1
- package/build/esm/AudioAnalysis/audioFeaturesWasm.web.js +76 -62
- package/build/esm/AudioAnalysis/audioFeaturesWasm.web.js.map +1 -1
- package/build/esm/AudioAnalysis/extractAudioAnalysis.js +15 -12
- package/build/esm/AudioAnalysis/extractAudioAnalysis.js.map +1 -1
- package/build/esm/AudioAnalysis/extractAudioData.js +144 -2
- package/build/esm/AudioAnalysis/extractAudioData.js.map +1 -1
- package/build/esm/AudioAnalysis/melSpectrogramWasm.web.js +9 -23
- package/build/esm/AudioAnalysis/melSpectrogramWasm.web.js.map +1 -1
- package/build/esm/AudioAnalysis/wasmConfig.js +4 -4
- package/build/esm/AudioAnalysis/wasmConfig.js.map +1 -1
- package/build/esm/AudioAnalysis/wasmLoader.web.js +43 -0
- package/build/esm/AudioAnalysis/wasmLoader.web.js.map +1 -0
- package/build/esm/AudioStudioModule.js +4 -596
- package/build/esm/AudioStudioModule.js.map +1 -1
- package/build/esm/trimAudio.js +227 -0
- package/build/esm/trimAudio.js.map +1 -1
- package/build/esm/utils/encodeCompressedAudio.web.js +62 -0
- package/build/esm/utils/encodeCompressedAudio.web.js.map +1 -0
- package/build/esm/utils/resampleAudioBuffer.web.js +22 -0
- package/build/esm/utils/resampleAudioBuffer.web.js.map +1 -0
- package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts +11 -0
- package/build/types/AudioAnalysis/AudioAnalysis.types.d.ts.map +1 -1
- package/build/types/AudioAnalysis/audioFeaturesWasm.d.ts +5 -9
- package/build/types/AudioAnalysis/audioFeaturesWasm.d.ts.map +1 -1
- package/build/types/AudioAnalysis/audioFeaturesWasm.web.d.ts +35 -16
- package/build/types/AudioAnalysis/audioFeaturesWasm.web.d.ts.map +1 -1
- package/build/types/AudioAnalysis/extractAudioAnalysis.d.ts.map +1 -1
- package/build/types/AudioAnalysis/extractAudioData.d.ts +2 -2
- package/build/types/AudioAnalysis/extractAudioData.d.ts.map +1 -1
- package/build/types/AudioAnalysis/melSpectrogramWasm.web.d.ts.map +1 -1
- package/build/types/AudioAnalysis/wasmLoader.web.d.ts +3 -0
- package/build/types/AudioAnalysis/wasmLoader.web.d.ts.map +1 -0
- package/build/types/AudioStudioModule.d.ts.map +1 -1
- package/build/types/trimAudio.d.ts.map +1 -1
- package/build/types/utils/encodeCompressedAudio.web.d.ts +10 -0
- package/build/types/utils/encodeCompressedAudio.web.d.ts.map +1 -0
- package/build/types/utils/resampleAudioBuffer.web.d.ts +2 -0
- package/build/types/utils/resampleAudioBuffer.web.d.ts.map +1 -0
- package/ios/AudioStreamManager.swift +135 -89
- package/ios/AudioStudioModule.swift +239 -216
- package/package.json +1 -1
- package/src/AudioAnalysis/AudioAnalysis.types.ts +12 -0
- package/src/AudioAnalysis/audioFeaturesWasm.ts +17 -22
- package/src/AudioAnalysis/audioFeaturesWasm.web.ts +102 -94
- package/src/AudioAnalysis/extractAudioAnalysis.ts +23 -20
- package/src/AudioAnalysis/extractAudioData.ts +186 -4
- package/src/AudioAnalysis/melSpectrogramWasm.web.ts +10 -27
- package/src/AudioAnalysis/wasmConfig.ts +4 -4
- package/src/AudioAnalysis/wasmLoader.web.ts +53 -0
- package/src/AudioStudioModule.ts +6 -854
- package/src/trimAudio.ts +351 -0
- package/src/utils/encodeCompressedAudio.web.ts +78 -0
- package/src/utils/resampleAudioBuffer.web.ts +39 -0
- package/build/cjs/AudioAnalysis/extractWaveform.js +0 -18
- package/build/cjs/AudioAnalysis/extractWaveform.js.map +0 -1
- package/build/esm/AudioAnalysis/extractWaveform.js +0 -11
- package/build/esm/AudioAnalysis/extractWaveform.js.map +0 -1
- package/build/types/AudioAnalysis/extractWaveform.d.ts +0 -8
- package/build/types/AudioAnalysis/extractWaveform.d.ts.map +0 -1
- package/src/AudioAnalysis/extractWaveform.ts +0 -22
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"AudioAnalysis.types.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/AudioAnalysis.types.ts"],"names":[],"mappings":"AAAA,iEAAiE","sourcesContent":["// packages/audio-studio/src/AudioAnalysis/AudioAnalysis.types.ts\n\nimport { BitDepth, ConsoleLike } from '../AudioStudio.types'\n\n/**\n * Represents the configuration for decoding audio data.\n */\nexport interface DecodingConfig {\n /** Target sample rate for decoded audio (Android and Web) */\n targetSampleRate?: number\n /** Target number of channels (Android and Web) */\n targetChannels?: number\n /** Target bit depth (Android and Web) */\n targetBitDepth?: BitDepth\n /** Whether to normalize audio levels (Android and Web) */\n normalizeAudio?: boolean\n}\n\n/**\n * Represents speech-related features extracted from audio.\n */\nexport interface SpeechFeatures {\n isActive: boolean // Whether speech is detected in this segment\n speakerId?: number // Optional speaker identification\n // Could add more speech-related features here like:\n // confidence: number\n // language?: string\n // sentiment?: number\n // etc.\n}\n\n/**\n * Represents various audio features extracted from an audio signal.\n */\nexport interface AudioFeatures {\n energy?: number // The infinite integral of the squared signal, representing the overall energy of the audio.\n mfcc?: number[] // Mel-frequency cepstral coefficients, describing the short-term power spectrum of a sound.\n rms?: number // Root mean square value, indicating the amplitude of the audio signal.\n minAmplitude?: number // Minimum amplitude value in the audio signal.\n maxAmplitude?: number // Maximum amplitude value in the audio signal.\n zcr?: number // Zero-crossing rate, indicating the rate at which the signal changes sign.\n spectralCentroid?: number // The center of mass of the spectrum, indicating the brightness of the sound.\n spectralFlatness?: number // Measure of the flatness of the spectrum, indicating how noise-like the signal is.\n spectralRolloff?: number // The frequency below which a specified percentage (usually 85%) of the total spectral energy lies.\n spectralBandwidth?: number // The width of the spectrum, indicating the range of frequencies present.\n chromagram?: number[] // Chromagram, representing the 12 different pitch classes of the audio.\n tempo?: number // Estimated tempo of the audio signal, measured in beats per minute (BPM).\n hnr?: number // Harmonics-to-noise ratio, indicating the proportion of harmonics to noise in the audio signal.\n melSpectrogram?: number[] // Mel-scaled spectrogram representation of the audio.\n spectralContrast?: number[] // Spectral contrast features representing the difference between peaks and valleys.\n tonnetz?: number[] // Tonal network features representing harmonic relationships.\n pitch?: number // Pitch of the audio signal, measured in Hertz (Hz).\n crc32?: number // crc32 checksum of the audio signal, used to verify the integrity of the audio.\n}\n\n/**\n * Options to specify which audio features to extract.\n * Note: Advanced features (spectral features, chromagram, pitch, etc.) are experimental,\n * especially during live recording, due to high processing requirements.\n */\nexport interface AudioFeaturesOptions {\n // Basic features - well optimized\n energy?: boolean\n rms?: boolean\n zcr?: boolean\n\n // Advanced features - experimental, may impact performance in live recording\n mfcc?: boolean\n spectralCentroid?: boolean\n spectralFlatness?: boolean\n spectralRolloff?: boolean\n spectralBandwidth?: boolean\n chromagram?: boolean\n tempo?: boolean\n hnr?: boolean\n melSpectrogram?: boolean\n spectralContrast?: boolean\n tonnetz?: boolean\n pitch?: boolean\n\n // Utility\n crc32?: boolean\n}\n\n/**\n * Represents a single data point in the audio analysis.\n */\nexport interface DataPoint {\n id: number\n amplitude: number // Peak amplitude for the segment\n rms: number // Root mean square value\n dB: number // dBFS (decibels relative to full scale) computed from RMS value\n silent: boolean // Always computed\n features?: AudioFeatures\n speech?: SpeechFeatures\n startTime?: number\n endTime?: number\n // start / end position in bytes\n startPosition?: number\n endPosition?: number\n // number of audio samples for this point (samples size depends on bit depth)\n samples?: number\n}\n\n/**\n * Represents the complete data from the audio analysis.\n */\nexport interface AudioAnalysis {\n segmentDurationMs: number // Duration of each segment in milliseconds\n durationMs: number // Duration of the audio in milliseconds\n /**\n * Bit depth used for audio analysis processing.\n *\n * **Important**: This represents the internal processing bit depth, which may differ\n * from the recording bit depth. Audio is typically converted to 32-bit float for\n * analysis to ensure precision in calculations, regardless of the original recording format.\n *\n * Platform behavior:\n * - iOS: Always 32 (float processing)\n * - Android: Always 32 (float processing)\n * - Web: Always 32 (Web Audio API standard)\n *\n * The actual recorded file will maintain the requested bit depth (8, 16, or 32).\n */\n bitDepth: number\n samples: number // Size of the audio in bytes\n numberOfChannels: number // Number of audio channels\n sampleRate: number // Sample rate of the audio\n dataPoints: DataPoint[] // Array of data points from the analysis.\n amplitudeRange: {\n min: number\n max: number\n }\n rmsRange: {\n min: number\n max: number\n }\n extractionTimeMs: number // Time taken to extract/process the analysis in milliseconds\n // TODO: speaker changes into a broader speech analysis section\n speechAnalysis?: {\n speakerChanges: {\n timestamp: number\n speakerId: number\n }[]\n // Could add more speech analysis data here like:\n // dominantSpeaker?: number\n // totalSpeechDuration?: number\n // speakerStats?: { [speakerId: number]: { duration: number, segments: number } }\n }\n}\n\n/**\n * Options for specifying a time range within an audio file.\n */\nexport interface AudioRangeOptions {\n /** Start time in milliseconds */\n startTimeMs?: number\n /** End time in milliseconds */\n endTimeMs?: number\n}\n\n/**\n * Options for generating a quick preview of audio waveform.\n * This is optimized for UI rendering with a specified number of points.\n */\nexport interface PreviewOptions extends AudioRangeOptions {\n /** URI of the audio file to analyze */\n fileUri: string\n /**\n * Total number of points to generate for the preview.\n * @default 100\n */\n numberOfPoints?: number\n /**\n * Optional logger for debugging.\n */\n logger?: ConsoleLike\n /**\n * Optional configuration for decoding the audio file.\n * Defaults to:\n * - targetSampleRate: undefined (keep original)\n * - targetChannels: undefined (keep original)\n * - targetBitDepth: 16\n * - normalizeAudio: false\n */\n decodingOptions?: DecodingConfig\n}\n\n/**\n * Options for mel-spectrogram extraction\n *\n * @experimental This feature is experimental and currently only available on Android.\n * The API may change in future versions.\n */\nexport interface ExtractMelSpectrogramOptions {\n fileUri?: string // Path to audio file\n arrayBuffer?: ArrayBuffer // Raw audio buffer\n windowSizeMs: number // Window size in ms (e.g., 25)\n hopLengthMs: number // Hop length in ms (e.g., 10)\n nMels: number // Number of mel filters (e.g., 60)\n fMin?: number // Min frequency (default: 0)\n fMax?: number // Max frequency (default: sampleRate / 2)\n windowType?: 'hann' | 'hamming' // Window function (default: 'hann')\n normalize?: boolean // Mean normalization (default: false)\n logScale?: boolean // Log scaling of mel energies (default: true)\n decodingOptions?: DecodingConfig // Audio decoding settings\n /** Optional start time in ms. If neither startTimeMs nor endTimeMs is set, defaults to 0. */\n startTimeMs?: number\n /** Optional end time in ms. Clamped so that the range does not exceed MAX_DURATION_MS (30 s). */\n endTimeMs?: number\n logger?: ConsoleLike\n}\n\n/**\n * Return type for mel spectrogram extraction\n *\n * @experimental This feature is experimental and currently only available on Android.\n * The API may change in future versions.\n */\nexport interface MelSpectrogram {\n spectrogram: number[][] // 2D array [time][mel]\n sampleRate: number // Audio sample rate\n nMels: number // Number of mel filters\n timeSteps: number // Number of time frames\n durationMs: number // Audio duration in ms\n}\n"]}
|
|
1
|
+
{"version":3,"file":"AudioAnalysis.types.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/AudioAnalysis.types.ts"],"names":[],"mappings":"AAAA,iEAAiE","sourcesContent":["// packages/audio-studio/src/AudioAnalysis/AudioAnalysis.types.ts\n\nimport { BitDepth, ConsoleLike } from '../AudioStudio.types'\n\n/**\n * Represents the configuration for decoding audio data.\n */\nexport interface DecodingConfig {\n /** Target sample rate for decoded audio (Android and Web) */\n targetSampleRate?: number\n /** Target number of channels (Android and Web) */\n targetChannels?: number\n /** Target bit depth (Android and Web) */\n targetBitDepth?: BitDepth\n /** Whether to normalize audio levels (Android and Web) */\n normalizeAudio?: boolean\n}\n\n/**\n * Represents speech-related features extracted from audio.\n */\nexport interface SpeechFeatures {\n isActive: boolean // Whether speech is detected in this segment\n speakerId?: number // Optional speaker identification\n // Could add more speech-related features here like:\n // confidence: number\n // language?: string\n // sentiment?: number\n // etc.\n}\n\n/**\n * Represents various audio features extracted from an audio signal.\n */\nexport interface AudioFeatures {\n energy?: number // The infinite integral of the squared signal, representing the overall energy of the audio.\n mfcc?: number[] // Mel-frequency cepstral coefficients, describing the short-term power spectrum of a sound.\n rms?: number // Root mean square value, indicating the amplitude of the audio signal.\n minAmplitude?: number // Minimum amplitude value in the audio signal.\n maxAmplitude?: number // Maximum amplitude value in the audio signal.\n zcr?: number // Zero-crossing rate, indicating the rate at which the signal changes sign.\n spectralCentroid?: number // The center of mass of the spectrum, indicating the brightness of the sound.\n spectralFlatness?: number // Measure of the flatness of the spectrum, indicating how noise-like the signal is.\n spectralRolloff?: number // The frequency below which a specified percentage (usually 85%) of the total spectral energy lies.\n spectralBandwidth?: number // The width of the spectrum, indicating the range of frequencies present.\n chromagram?: number[] // Chromagram, representing the 12 different pitch classes of the audio.\n tempo?: number // Estimated tempo of the audio signal, measured in beats per minute (BPM).\n hnr?: number // Harmonics-to-noise ratio, indicating the proportion of harmonics to noise in the audio signal.\n melSpectrogram?: number[] // Mel-scaled spectrogram representation of the audio.\n spectralContrast?: number[] // Spectral contrast features representing the difference between peaks and valleys.\n tonnetz?: number[] // Tonal network features representing harmonic relationships.\n pitch?: number // Pitch of the audio signal, measured in Hertz (Hz).\n crc32?: number // crc32 checksum of the audio signal, used to verify the integrity of the audio.\n}\n\n/**\n * Options to specify which audio features to extract.\n * Note: Advanced features (spectral features, chromagram, pitch, etc.) are experimental,\n * especially during live recording, due to high processing requirements.\n */\nexport interface AudioFeaturesOptions {\n // Basic features - well optimized\n energy?: boolean\n rms?: boolean\n zcr?: boolean\n\n // Advanced features - experimental, may impact performance in live recording\n mfcc?: boolean\n spectralCentroid?: boolean\n spectralFlatness?: boolean\n spectralRolloff?: boolean\n spectralBandwidth?: boolean\n chromagram?: boolean\n tempo?: boolean\n hnr?: boolean\n melSpectrogram?: boolean\n spectralContrast?: boolean\n tonnetz?: boolean\n pitch?: boolean\n\n // Utility\n crc32?: boolean\n}\n\n/**\n * Represents a single data point in the audio analysis.\n */\nexport interface DataPoint {\n id: number\n amplitude: number // Peak amplitude for the segment\n rms: number // Root mean square value\n dB: number // dBFS (decibels relative to full scale) computed from RMS value\n silent: boolean // Always computed\n features?: AudioFeatures\n speech?: SpeechFeatures\n startTime?: number\n endTime?: number\n // start / end position in bytes\n startPosition?: number\n endPosition?: number\n // number of audio samples for this point (samples size depends on bit depth)\n samples?: number\n}\n\n/**\n * Represents the complete data from the audio analysis.\n */\nexport interface AudioAnalysis {\n segmentDurationMs: number // Duration of each segment in milliseconds\n durationMs: number // Duration of the audio in milliseconds\n /**\n * Bit depth used for audio analysis processing.\n *\n * **Important**: This represents the internal processing bit depth, which may differ\n * from the recording bit depth. Audio is typically converted to 32-bit float for\n * analysis to ensure precision in calculations, regardless of the original recording format.\n *\n * Platform behavior:\n * - iOS: Always 32 (float processing)\n * - Android: Always 32 (float processing)\n * - Web: Always 32 (Web Audio API standard)\n *\n * The actual recorded file will maintain the requested bit depth (8, 16, or 32).\n */\n bitDepth: number\n samples: number // Size of the audio in bytes\n numberOfChannels: number // Number of audio channels\n sampleRate: number // Sample rate of the audio\n dataPoints: DataPoint[] // Array of data points from the analysis.\n amplitudeRange: {\n min: number\n max: number\n }\n rmsRange: {\n min: number\n max: number\n }\n extractionTimeMs: number // Time taken to extract/process the analysis in milliseconds\n // TODO: speaker changes into a broader speech analysis section\n speechAnalysis?: {\n speakerChanges: {\n timestamp: number\n speakerId: number\n }[]\n // Could add more speech analysis data here like:\n // dominantSpeaker?: number\n // totalSpeechDuration?: number\n // speakerStats?: { [speakerId: number]: { duration: number, segments: number } }\n }\n}\n\n/**\n * Options for specifying a time range within an audio file.\n */\nexport interface AudioRangeOptions {\n /** Start time in milliseconds */\n startTimeMs?: number\n /** End time in milliseconds */\n endTimeMs?: number\n}\n\n/**\n * Options for generating a quick preview of audio waveform.\n * This is optimized for UI rendering with a specified number of points.\n */\nexport interface PreviewOptions extends AudioRangeOptions {\n /** URI of the audio file to analyze */\n fileUri: string\n /**\n * Total number of points to generate for the preview.\n * @default 100\n */\n numberOfPoints?: number\n /**\n * Optional logger for debugging.\n */\n logger?: ConsoleLike\n /**\n * Optional configuration for decoding the audio file.\n * Defaults to:\n * - targetSampleRate: undefined (keep original)\n * - targetChannels: undefined (keep original)\n * - targetBitDepth: 16\n * - normalizeAudio: false\n */\n decodingOptions?: DecodingConfig\n}\n\n/**\n * Options for mel-spectrogram extraction\n *\n * @experimental This feature is experimental and currently only available on Android.\n * The API may change in future versions.\n */\nexport interface ExtractMelSpectrogramOptions {\n fileUri?: string // Path to audio file\n arrayBuffer?: ArrayBuffer // Raw audio buffer\n windowSizeMs: number // Window size in ms (e.g., 25)\n hopLengthMs: number // Hop length in ms (e.g., 10)\n nMels: number // Number of mel filters (e.g., 60)\n fMin?: number // Min frequency (default: 0)\n fMax?: number // Max frequency (default: sampleRate / 2)\n windowType?: 'hann' | 'hamming' // Window function (default: 'hann')\n normalize?: boolean // Mean normalization (default: false)\n logScale?: boolean // Log scaling of mel energies (default: true)\n decodingOptions?: DecodingConfig // Audio decoding settings\n /** Optional start time in ms. If neither startTimeMs nor endTimeMs is set, defaults to 0. */\n startTimeMs?: number\n /** Optional end time in ms. Clamped so that the range does not exceed MAX_DURATION_MS (30 s). */\n endTimeMs?: number\n logger?: ConsoleLike\n}\n\n/**\n * Result type for WASM-based audio feature extraction.\n */\nexport interface AudioFeaturesWasmResult {\n spectralCentroid: number\n spectralFlatness: number\n spectralRolloff: number\n spectralBandwidth: number\n mfcc: number[]\n chromagram: number[]\n}\n\n/**\n * Return type for mel spectrogram extraction\n *\n * @experimental This feature is experimental and currently only available on Android.\n * The API may change in future versions.\n */\nexport interface MelSpectrogram {\n spectrogram: number[][] // 2D array [time][mel]\n sampleRate: number // Audio sample rate\n nMels: number // Number of mel filters\n timeSteps: number // Number of time frames\n durationMs: number // Audio duration in ms\n}\n"]}
|
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
// Native stub — WASM audio features is web-only.
|
|
2
|
-
export
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
2
|
+
export class AudioFeaturesStreamingSession {
|
|
3
|
+
static async create(_sampleRate, _fftLength, _nMfcc, _nMelFilters, _computeMfcc, _computeChroma) {
|
|
4
|
+
throw new Error('WASM audio features is not available on native');
|
|
5
|
+
}
|
|
6
|
+
computeFrame(_samples) {
|
|
7
|
+
return null;
|
|
8
|
+
}
|
|
9
|
+
dispose() { }
|
|
7
10
|
}
|
|
8
11
|
export async function computeAudioFeaturesWasm(_audioData, _sampleRate, _fftLength, _nMfcc, _nMelFilters, _computeMfcc, _computeChroma) {
|
|
9
12
|
throw new Error('WASM audio features is not available on native');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audioFeaturesWasm.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/audioFeaturesWasm.ts"],"names":[],"mappings":"AAAA,iDAAiD;
|
|
1
|
+
{"version":3,"file":"audioFeaturesWasm.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/audioFeaturesWasm.ts"],"names":[],"mappings":"AAAA,iDAAiD;AAIjD,MAAM,OAAO,6BAA6B;IACtC,MAAM,CAAC,KAAK,CAAC,MAAM,CACf,WAAmB,EACnB,UAAmB,EACnB,MAAe,EACf,YAAqB,EACrB,YAAsB,EACtB,cAAwB;QAExB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;IACrE,CAAC;IAED,YAAY,CAAC,QAAsB;QAC/B,OAAO,IAAI,CAAA;IACf,CAAC;IAED,OAAO,KAAU,CAAC;CACrB;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,UAAwB,EACxB,WAAmB,EACnB,UAAmB,EACnB,MAAe,EACf,YAAqB,EACrB,YAAsB,EACtB,cAAwB;IAExB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;AACrE,CAAC","sourcesContent":["// Native stub — WASM audio features is web-only.\n\nimport type { AudioFeaturesWasmResult } from './AudioAnalysis.types'\n\nexport class AudioFeaturesStreamingSession {\n static async create(\n _sampleRate: number,\n _fftLength?: number,\n _nMfcc?: number,\n _nMelFilters?: number,\n _computeMfcc?: boolean,\n _computeChroma?: boolean\n ): Promise<AudioFeaturesStreamingSession> {\n throw new Error('WASM audio features is not available on native')\n }\n\n computeFrame(_samples: Float32Array): AudioFeaturesWasmResult | null {\n return null\n }\n\n dispose(): void {}\n}\n\nexport async function computeAudioFeaturesWasm(\n _audioData: Float32Array,\n _sampleRate: number,\n _fftLength?: number,\n _nMfcc?: number,\n _nMelFilters?: number,\n _computeMfcc?: boolean,\n _computeChroma?: boolean\n): Promise<AudioFeaturesWasmResult> {\n throw new Error('WASM audio features is not available on native')\n}\n"]}
|
|
@@ -1,21 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
let modulePromise = null;
|
|
3
|
-
function getModule() {
|
|
4
|
-
if (!modulePromise) {
|
|
5
|
-
modulePromise = (async () => {
|
|
6
|
-
const url = getMelSpectrogramWasmUrl();
|
|
7
|
-
// webpackIgnore + @vite-ignore prevent bundlers from trying to resolve the URL
|
|
8
|
-
const mod = await import(
|
|
9
|
-
/* webpackIgnore: true */ /* @vite-ignore */ url);
|
|
10
|
-
const factory = mod.default ?? mod;
|
|
11
|
-
return factory();
|
|
12
|
-
})().catch((err) => {
|
|
13
|
-
modulePromise = null;
|
|
14
|
-
throw err;
|
|
15
|
-
});
|
|
16
|
-
}
|
|
17
|
-
return modulePromise;
|
|
18
|
-
}
|
|
1
|
+
import { getWasmModule } from './wasmLoader.web';
|
|
19
2
|
// --- Struct layout for CAudioFeaturesResult (wasm32) ---
|
|
20
3
|
// Offset 0: float spectralCentroid (4 bytes)
|
|
21
4
|
// Offset 4: float spectralFlatness (4 bytes)
|
|
@@ -59,52 +42,83 @@ function readResult(Module, ptr) {
|
|
|
59
42
|
};
|
|
60
43
|
}
|
|
61
44
|
// --- Streaming (per-frame) API ---
|
|
62
|
-
let streamingModule = null;
|
|
63
|
-
let streamingFramePtr = 0;
|
|
64
|
-
let streamingFrameCapacity = 0;
|
|
65
|
-
let streamingResultPtr = 0;
|
|
66
|
-
/**
|
|
67
|
-
* Initialise the WASM streaming audio features processor.
|
|
68
|
-
* Call once before computeAudioFeaturesFrameWasm().
|
|
69
|
-
*/
|
|
70
|
-
export async function initAudioFeaturesWasm(sampleRate, fftLength = 1024, nMfcc = 13, nMelFilters = 26, computeMfcc = true, computeChroma = true) {
|
|
71
|
-
const Module = await getModule();
|
|
72
|
-
streamingModule = Module;
|
|
73
|
-
Module._audio_features_init(sampleRate, fftLength, nMfcc, nMelFilters, computeMfcc ? 1 : 0, computeChroma ? 1 : 0);
|
|
74
|
-
// Pre-allocate result struct on WASM heap
|
|
75
|
-
if (streamingResultPtr)
|
|
76
|
-
Module._free(streamingResultPtr);
|
|
77
|
-
streamingResultPtr = Module._malloc(STRUCT_SIZE);
|
|
78
|
-
// Zero-initialize to prevent freeing garbage pointers on first use
|
|
79
|
-
Module.HEAPU8.fill(0, streamingResultPtr, streamingResultPtr + STRUCT_SIZE);
|
|
80
|
-
// Frame input buffer allocated on demand
|
|
81
|
-
streamingFrameCapacity = 0;
|
|
82
|
-
streamingFramePtr = 0;
|
|
83
|
-
}
|
|
84
45
|
/**
|
|
85
|
-
*
|
|
86
|
-
*
|
|
46
|
+
* Encapsulates a single WASM streaming audio features session.
|
|
47
|
+
* Each instance owns its own WASM heap allocations; multiple sessions
|
|
48
|
+
* can exist concurrently without interfering with each other.
|
|
49
|
+
*
|
|
50
|
+
* Usage:
|
|
51
|
+
* const session = await AudioFeaturesStreamingSession.create(sampleRate)
|
|
52
|
+
* try {
|
|
53
|
+
* for (const frame of frames) {
|
|
54
|
+
* const result = session.computeFrame(frame)
|
|
55
|
+
* }
|
|
56
|
+
* } finally {
|
|
57
|
+
* session.dispose()
|
|
58
|
+
* }
|
|
87
59
|
*/
|
|
88
|
-
export
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
60
|
+
export class AudioFeaturesStreamingSession {
|
|
61
|
+
module;
|
|
62
|
+
framePtr = 0;
|
|
63
|
+
frameCapacity = 0;
|
|
64
|
+
resultPtr = 0;
|
|
65
|
+
constructor(module) {
|
|
66
|
+
this.module = module;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Initialise a new streaming session. Loads the WASM module if needed.
|
|
70
|
+
*/
|
|
71
|
+
static async create(sampleRate, fftLength = 1024, nMfcc = 13, nMelFilters = 26, computeMfcc = true, computeChroma = true) {
|
|
72
|
+
const Module = await getWasmModule();
|
|
73
|
+
const session = new AudioFeaturesStreamingSession(Module);
|
|
74
|
+
Module._audio_features_init(sampleRate, fftLength, nMfcc, nMelFilters, computeMfcc ? 1 : 0, computeChroma ? 1 : 0);
|
|
75
|
+
// Pre-allocate result struct on WASM heap
|
|
76
|
+
session.resultPtr = Module._malloc(STRUCT_SIZE);
|
|
77
|
+
// Zero-initialize to prevent freeing garbage pointers on first use
|
|
78
|
+
Module.HEAPU8.fill(0, session.resultPtr, session.resultPtr + STRUCT_SIZE);
|
|
79
|
+
return session;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Compute audio features for a single frame.
|
|
83
|
+
* Returns null on error or if the session has been disposed.
|
|
84
|
+
*/
|
|
85
|
+
computeFrame(samples) {
|
|
86
|
+
if (!this.resultPtr)
|
|
87
|
+
return null;
|
|
88
|
+
const Module = this.module;
|
|
89
|
+
// (Re-)allocate frame input buffer if needed
|
|
90
|
+
if (samples.length > this.frameCapacity) {
|
|
91
|
+
if (this.framePtr)
|
|
92
|
+
Module._free(this.framePtr);
|
|
93
|
+
this.framePtr = Module._malloc(samples.length * 4);
|
|
94
|
+
this.frameCapacity = samples.length;
|
|
95
|
+
}
|
|
96
|
+
// Copy samples to WASM heap
|
|
97
|
+
Module.HEAPF32.set(samples, this.framePtr >> 2);
|
|
98
|
+
const ok = Module._audio_features_compute_frame(this.framePtr, samples.length, this.resultPtr);
|
|
99
|
+
if (!ok)
|
|
100
|
+
return null;
|
|
101
|
+
const result = readResult(Module, this.resultPtr);
|
|
102
|
+
// Free internal arrays (mfcc, chromagram) allocated by C
|
|
103
|
+
Module._audio_features_free_arrays(this.resultPtr);
|
|
104
|
+
return result;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Free all WASM heap allocations owned by this session.
|
|
108
|
+
* The session must not be used after calling dispose().
|
|
109
|
+
*/
|
|
110
|
+
dispose() {
|
|
111
|
+
const Module = this.module;
|
|
112
|
+
if (this.framePtr) {
|
|
113
|
+
Module._free(this.framePtr);
|
|
114
|
+
this.framePtr = 0;
|
|
115
|
+
this.frameCapacity = 0;
|
|
116
|
+
}
|
|
117
|
+
if (this.resultPtr) {
|
|
118
|
+
Module._free(this.resultPtr);
|
|
119
|
+
this.resultPtr = 0;
|
|
120
|
+
}
|
|
98
121
|
}
|
|
99
|
-
// Copy samples to WASM heap
|
|
100
|
-
Module.HEAPF32.set(samples, streamingFramePtr >> 2);
|
|
101
|
-
const ok = Module._audio_features_compute_frame(streamingFramePtr, samples.length, streamingResultPtr);
|
|
102
|
-
if (!ok)
|
|
103
|
-
return null;
|
|
104
|
-
const result = readResult(Module, streamingResultPtr);
|
|
105
|
-
// Free internal arrays (mfcc, chromagram) allocated by C
|
|
106
|
-
Module._audio_features_free_arrays(streamingResultPtr);
|
|
107
|
-
return result;
|
|
108
122
|
}
|
|
109
123
|
// --- Batch API ---
|
|
110
124
|
/**
|
|
@@ -112,7 +126,7 @@ export function computeAudioFeaturesFrameWasm(samples) {
|
|
|
112
126
|
* Lazy-loads the WASM module on first call.
|
|
113
127
|
*/
|
|
114
128
|
export async function computeAudioFeaturesWasm(audioData, sampleRate, fftLength = 1024, nMfcc = 13, nMelFilters = 26, computeMfcc = true, computeChroma = true) {
|
|
115
|
-
const Module = await
|
|
129
|
+
const Module = await getWasmModule();
|
|
116
130
|
const numSamples = audioData.length;
|
|
117
131
|
const inputPtr = Module._malloc(numSamples * 4);
|
|
118
132
|
Module.HEAPF32.set(audioData, inputPtr >> 2);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"audioFeaturesWasm.web.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/audioFeaturesWasm.web.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,cAAc,CAAA;AAWvD,IAAI,aAAa,GAA4C,IAAI,CAAA;AAEjE,SAAS,SAAS;IACd,IAAI,CAAC,aAAa,EAAE,CAAC;QACjB,aAAa,GAAG,CAAC,KAAK,IAAI,EAAE;YACxB,MAAM,GAAG,GAAG,wBAAwB,EAAE,CAAA;YACtC,+EAA+E;YAC/E,MAAM,GAAG,GAAG,MAAM,MAAM;YACpB,yBAAyB,CAAC,kBAAkB,CAAC,GAAG,CACnD,CAAA;YACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,IAAI,GAAG,CAAA;YAClC,OAAO,OAAO,EAAsC,CAAA;QACxD,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACf,aAAa,GAAG,IAAI,CAAA;YACpB,MAAM,GAAG,CAAA;QACb,CAAC,CAAC,CAAA;IACN,CAAC;IACD,OAAO,aAAa,CAAA;AACxB,CAAC;AAED,0DAA0D;AAC1D,+CAA+C;AAC/C,+CAA+C;AAC/C,+CAA+C;AAC/C,+CAA+C;AAC/C,uDAAuD;AACvD,+CAA+C;AAC/C,uDAAuD;AACvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,EAAE,CAAA;AAEtB,SAAS,UAAU,CACf,MAA+B,EAC/B,GAAW;IAEX,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACtD,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAC1D,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IACzD,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,OAAO,CAAC,CAAA;IAE5D,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAClD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAClD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAEpD,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,OAAO,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,SAAS,IAAI,CAAC,CAAA;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAC/C,CAAC;IACL,CAAC;IAED,OAAO;QACH,gBAAgB;QAChB,gBAAgB;QAChB,eAAe;QACf,iBAAiB;QACjB,IAAI;QACJ,UAAU;KACb,CAAA;AACL,CAAC;AAED,oCAAoC;AAEpC,IAAI,eAAe,GAAmC,IAAI,CAAA;AAC1D,IAAI,iBAAiB,GAAG,CAAC,CAAA;AACzB,IAAI,sBAAsB,GAAG,CAAC,CAAA;AAC9B,IAAI,kBAAkB,GAAG,CAAC,CAAA;AAE1B;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACvC,UAAkB,EAClB,SAAS,GAAG,IAAI,EAChB,KAAK,GAAG,EAAE,EACV,WAAW,GAAG,EAAE,EAChB,WAAW,GAAG,IAAI,EAClB,aAAa,GAAG,IAAI;IAEpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAA;IAChC,eAAe,GAAG,MAAM,CAAA;IAExB,MAAM,CAAC,oBAAoB,CACvB,UAAU,EACV,SAAS,EACT,KAAK,EACL,WAAW,EACX,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnB,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxB,CAAA;IAED,0CAA0C;IAC1C,IAAI,kBAAkB;QAAE,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAA;IACxD,kBAAkB,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;IAChD,mEAAmE;IACnE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,kBAAkB,EAAE,kBAAkB,GAAG,WAAW,CAAC,CAAA;IAE3E,yCAAyC;IACzC,sBAAsB,GAAG,CAAC,CAAA;IAC1B,iBAAiB,GAAG,CAAC,CAAA;AACzB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,6BAA6B,CACzC,OAAqB;IAErB,IAAI,CAAC,eAAe,IAAI,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAA;IACxD,MAAM,MAAM,GAAG,eAAe,CAAA;IAE9B,6CAA6C;IAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC1C,IAAI,iBAAiB;YAAE,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAA;QACtD,iBAAiB,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QACtD,sBAAsB,GAAG,OAAO,CAAC,MAAM,CAAA;IAC3C,CAAC;IAED,4BAA4B;IAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,iBAAiB,IAAI,CAAC,CAAC,CAAA;IAEnD,MAAM,EAAE,GAAG,MAAM,CAAC,6BAA6B,CAC3C,iBAAiB,EACjB,OAAO,CAAC,MAAM,EACd,kBAAkB,CACrB,CAAA;IACD,IAAI,CAAC,EAAE;QAAE,OAAO,IAAI,CAAA;IAEpB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IAErD,yDAAyD;IACzD,MAAM,CAAC,2BAA2B,CAAC,kBAAkB,CAAC,CAAA;IAEtD,OAAO,MAAM,CAAA;AACjB,CAAC;AAED,oBAAoB;AAEpB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,SAAuB,EACvB,UAAkB,EAClB,SAAS,GAAG,IAAI,EAChB,KAAK,GAAG,EAAE,EACV,WAAW,GAAG,EAAE,EAChB,WAAW,GAAG,IAAI,EAClB,aAAa,GAAG,IAAI;IAEpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAA;IAEhC,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAA;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAA;IAC/C,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,MAAM,CAAC,uBAAuB,CAC5C,QAAQ,EACR,UAAU,EACV,UAAU,EACV,SAAS,EACT,KAAK,EACL,WAAW,EACX,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnB,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxB,CAAA;IAED,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAEtB,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAC5C,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAA;IAEtC,OAAO,MAAM,CAAA;AACjB,CAAC","sourcesContent":["import type { AudioFeaturesWasmModule } from './audio-features-wasm'\nimport { getMelSpectrogramWasmUrl } from './wasmConfig'\n\nexport interface AudioFeaturesWasmResult {\n spectralCentroid: number\n spectralFlatness: number\n spectralRolloff: number\n spectralBandwidth: number\n mfcc: number[]\n chromagram: number[]\n}\n\nlet modulePromise: Promise<AudioFeaturesWasmModule> | null = null\n\nfunction getModule(): Promise<AudioFeaturesWasmModule> {\n if (!modulePromise) {\n modulePromise = (async () => {\n const url = getMelSpectrogramWasmUrl()\n // webpackIgnore + @vite-ignore prevent bundlers from trying to resolve the URL\n const mod = await import(\n /* webpackIgnore: true */ /* @vite-ignore */ url\n )\n const factory = mod.default ?? mod\n return factory() as Promise<AudioFeaturesWasmModule>\n })().catch((err) => {\n modulePromise = null\n throw err\n })\n }\n return modulePromise\n}\n\n// --- Struct layout for CAudioFeaturesResult (wasm32) ---\n// Offset 0: float spectralCentroid (4 bytes)\n// Offset 4: float spectralFlatness (4 bytes)\n// Offset 8: float spectralRolloff (4 bytes)\n// Offset 12: float spectralBandwidth (4 bytes)\n// Offset 16: float* mfcc (4 bytes pointer)\n// Offset 20: int mfccCount (4 bytes)\n// Offset 24: float* chromagram (4 bytes pointer)\n// Offset 28: int chromagramCount (4 bytes)\nconst STRUCT_SIZE = 32\n\nfunction readResult(\n Module: AudioFeaturesWasmModule,\n ptr: number\n): AudioFeaturesWasmResult {\n const spectralCentroid = Module.getValue(ptr, 'float')\n const spectralFlatness = Module.getValue(ptr + 4, 'float')\n const spectralRolloff = Module.getValue(ptr + 8, 'float')\n const spectralBandwidth = Module.getValue(ptr + 12, 'float')\n\n const mfccPtr = Module.getValue(ptr + 16, 'i32')\n const mfccCount = Module.getValue(ptr + 20, 'i32')\n const chromaPtr = Module.getValue(ptr + 24, 'i32')\n const chromaCount = Module.getValue(ptr + 28, 'i32')\n\n const mfcc: number[] = []\n if (mfccPtr && mfccCount > 0) {\n const offset = mfccPtr >> 2\n for (let i = 0; i < mfccCount; i++) {\n mfcc.push(Module.HEAPF32[offset + i])\n }\n }\n\n const chromagram: number[] = []\n if (chromaPtr && chromaCount > 0) {\n const offset = chromaPtr >> 2\n for (let i = 0; i < chromaCount; i++) {\n chromagram.push(Module.HEAPF32[offset + i])\n }\n }\n\n return {\n spectralCentroid,\n spectralFlatness,\n spectralRolloff,\n spectralBandwidth,\n mfcc,\n chromagram,\n }\n}\n\n// --- Streaming (per-frame) API ---\n\nlet streamingModule: AudioFeaturesWasmModule | null = null\nlet streamingFramePtr = 0\nlet streamingFrameCapacity = 0\nlet streamingResultPtr = 0\n\n/**\n * Initialise the WASM streaming audio features processor.\n * Call once before computeAudioFeaturesFrameWasm().\n */\nexport async function initAudioFeaturesWasm(\n sampleRate: number,\n fftLength = 1024,\n nMfcc = 13,\n nMelFilters = 26,\n computeMfcc = true,\n computeChroma = true\n): Promise<void> {\n const Module = await getModule()\n streamingModule = Module\n\n Module._audio_features_init(\n sampleRate,\n fftLength,\n nMfcc,\n nMelFilters,\n computeMfcc ? 1 : 0,\n computeChroma ? 1 : 0\n )\n\n // Pre-allocate result struct on WASM heap\n if (streamingResultPtr) Module._free(streamingResultPtr)\n streamingResultPtr = Module._malloc(STRUCT_SIZE)\n // Zero-initialize to prevent freeing garbage pointers on first use\n Module.HEAPU8.fill(0, streamingResultPtr, streamingResultPtr + STRUCT_SIZE)\n\n // Frame input buffer allocated on demand\n streamingFrameCapacity = 0\n streamingFramePtr = 0\n}\n\n/**\n * Compute audio features for a single frame via WASM C++.\n * Returns null if not initialised or on error.\n */\nexport function computeAudioFeaturesFrameWasm(\n samples: Float32Array\n): AudioFeaturesWasmResult | null {\n if (!streamingModule || !streamingResultPtr) return null\n const Module = streamingModule\n\n // (Re-)allocate frame input buffer if needed\n if (samples.length > streamingFrameCapacity) {\n if (streamingFramePtr) Module._free(streamingFramePtr)\n streamingFramePtr = Module._malloc(samples.length * 4)\n streamingFrameCapacity = samples.length\n }\n\n // Copy samples to WASM heap\n Module.HEAPF32.set(samples, streamingFramePtr >> 2)\n\n const ok = Module._audio_features_compute_frame(\n streamingFramePtr,\n samples.length,\n streamingResultPtr\n )\n if (!ok) return null\n\n const result = readResult(Module, streamingResultPtr)\n\n // Free internal arrays (mfcc, chromagram) allocated by C\n Module._audio_features_free_arrays(streamingResultPtr)\n\n return result\n}\n\n// --- Batch API ---\n\n/**\n * Compute audio features for a buffer of samples via WASM C++.\n * Lazy-loads the WASM module on first call.\n */\nexport async function computeAudioFeaturesWasm(\n audioData: Float32Array,\n sampleRate: number,\n fftLength = 1024,\n nMfcc = 13,\n nMelFilters = 26,\n computeMfcc = true,\n computeChroma = true\n): Promise<AudioFeaturesWasmResult> {\n const Module = await getModule()\n\n const numSamples = audioData.length\n const inputPtr = Module._malloc(numSamples * 4)\n Module.HEAPF32.set(audioData, inputPtr >> 2)\n\n const resultPtr = Module._audio_features_compute(\n inputPtr,\n numSamples,\n sampleRate,\n fftLength,\n nMfcc,\n nMelFilters,\n computeMfcc ? 1 : 0,\n computeChroma ? 1 : 0\n )\n\n Module._free(inputPtr)\n\n if (resultPtr === 0) {\n throw new Error('audio_features_compute returned null')\n }\n\n const result = readResult(Module, resultPtr)\n Module._audio_features_free(resultPtr)\n\n return result\n}\n"]}
|
|
1
|
+
{"version":3,"file":"audioFeaturesWasm.web.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/audioFeaturesWasm.web.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAEhD,0DAA0D;AAC1D,+CAA+C;AAC/C,+CAA+C;AAC/C,+CAA+C;AAC/C,+CAA+C;AAC/C,uDAAuD;AACvD,+CAA+C;AAC/C,uDAAuD;AACvD,+CAA+C;AAC/C,MAAM,WAAW,GAAG,EAAE,CAAA;AAEtB,SAAS,UAAU,CACf,MAA+B,EAC/B,GAAW;IAEX,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACtD,MAAM,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAC1D,MAAM,eAAe,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IACzD,MAAM,iBAAiB,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,OAAO,CAAC,CAAA;IAE5D,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAClD,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAClD,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,EAAE,KAAK,CAAC,CAAA;IAEpD,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,OAAO,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,CAAA;QAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC;IACL,CAAC;IAED,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,MAAM,GAAG,SAAS,IAAI,CAAC,CAAA;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACnC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAC/C,CAAC;IACL,CAAC;IAED,OAAO;QACH,gBAAgB;QAChB,gBAAgB;QAChB,eAAe;QACf,iBAAiB;QACjB,IAAI;QACJ,UAAU;KACb,CAAA;AACL,CAAC;AAED,oCAAoC;AAEpC;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,6BAA6B;IAC9B,MAAM,CAAyB;IAC/B,QAAQ,GAAG,CAAC,CAAA;IACZ,aAAa,GAAG,CAAC,CAAA;IACjB,SAAS,GAAG,CAAC,CAAA;IAErB,YAAoB,MAA+B;QAC/C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACxB,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CACf,UAAkB,EAClB,SAAS,GAAG,IAAI,EAChB,KAAK,GAAG,EAAE,EACV,WAAW,GAAG,EAAE,EAChB,WAAW,GAAG,IAAI,EAClB,aAAa,GAAG,IAAI;QAEpB,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAA;QACpC,MAAM,OAAO,GAAG,IAAI,6BAA6B,CAAC,MAAM,CAAC,CAAA;QAEzD,MAAM,CAAC,oBAAoB,CACvB,UAAU,EACV,SAAS,EACT,KAAK,EACL,WAAW,EACX,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnB,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxB,CAAA;QAED,0CAA0C;QAC1C,OAAO,CAAC,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAA;QAC/C,mEAAmE;QACnE,MAAM,CAAC,MAAM,CAAC,IAAI,CACd,CAAC,EACD,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,SAAS,GAAG,WAAW,CAClC,CAAA;QAED,OAAO,OAAO,CAAA;IAClB,CAAC;IAED;;;OAGG;IACH,YAAY,CAAC,OAAqB;QAC9B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAA;QAChC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAE1B,6CAA6C;QAC7C,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,IAAI,IAAI,CAAC,QAAQ;gBAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC9C,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;YAClD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;QACvC,CAAC;QAED,4BAA4B;QAC5B,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAA;QAE/C,MAAM,EAAE,GAAG,MAAM,CAAC,6BAA6B,CAC3C,IAAI,CAAC,QAAQ,EACb,OAAO,CAAC,MAAM,EACd,IAAI,CAAC,SAAS,CACjB,CAAA;QACD,IAAI,CAAC,EAAE;YAAE,OAAO,IAAI,CAAA;QAEpB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;QAEjD,yDAAyD;QACzD,MAAM,CAAC,2BAA2B,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QAElD,OAAO,MAAM,CAAA;IACjB,CAAC;IAED;;;OAGG;IACH,OAAO;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC1B,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YAC3B,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAA;YACjB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAA;QAC1B,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAC5B,IAAI,CAAC,SAAS,GAAG,CAAC,CAAA;QACtB,CAAC;IACL,CAAC;CACJ;AAED,oBAAoB;AAEpB;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC1C,SAAuB,EACvB,UAAkB,EAClB,SAAS,GAAG,IAAI,EAChB,KAAK,GAAG,EAAE,EACV,WAAW,GAAG,EAAE,EAChB,WAAW,GAAG,IAAI,EAClB,aAAa,GAAG,IAAI;IAEpB,MAAM,MAAM,GAAG,MAAM,aAAa,EAAE,CAAA;IAEpC,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAA;IACnC,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAA;IAC/C,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,MAAM,CAAC,uBAAuB,CAC5C,QAAQ,EACR,UAAU,EACV,UAAU,EACV,SAAS,EACT,KAAK,EACL,WAAW,EACX,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACnB,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACxB,CAAA;IAED,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAEtB,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;IAC5C,MAAM,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAA;IAEtC,OAAO,MAAM,CAAA;AACjB,CAAC","sourcesContent":["import type { AudioFeaturesWasmResult } from './AudioAnalysis.types'\nimport type { AudioFeaturesWasmModule } from './audio-features-wasm'\nimport { getWasmModule } from './wasmLoader.web'\n\n// --- Struct layout for CAudioFeaturesResult (wasm32) ---\n// Offset 0: float spectralCentroid (4 bytes)\n// Offset 4: float spectralFlatness (4 bytes)\n// Offset 8: float spectralRolloff (4 bytes)\n// Offset 12: float spectralBandwidth (4 bytes)\n// Offset 16: float* mfcc (4 bytes pointer)\n// Offset 20: int mfccCount (4 bytes)\n// Offset 24: float* chromagram (4 bytes pointer)\n// Offset 28: int chromagramCount (4 bytes)\nconst STRUCT_SIZE = 32\n\nfunction readResult(\n Module: AudioFeaturesWasmModule,\n ptr: number\n): AudioFeaturesWasmResult {\n const spectralCentroid = Module.getValue(ptr, 'float')\n const spectralFlatness = Module.getValue(ptr + 4, 'float')\n const spectralRolloff = Module.getValue(ptr + 8, 'float')\n const spectralBandwidth = Module.getValue(ptr + 12, 'float')\n\n const mfccPtr = Module.getValue(ptr + 16, 'i32')\n const mfccCount = Module.getValue(ptr + 20, 'i32')\n const chromaPtr = Module.getValue(ptr + 24, 'i32')\n const chromaCount = Module.getValue(ptr + 28, 'i32')\n\n const mfcc: number[] = []\n if (mfccPtr && mfccCount > 0) {\n const offset = mfccPtr >> 2\n for (let i = 0; i < mfccCount; i++) {\n mfcc.push(Module.HEAPF32[offset + i])\n }\n }\n\n const chromagram: number[] = []\n if (chromaPtr && chromaCount > 0) {\n const offset = chromaPtr >> 2\n for (let i = 0; i < chromaCount; i++) {\n chromagram.push(Module.HEAPF32[offset + i])\n }\n }\n\n return {\n spectralCentroid,\n spectralFlatness,\n spectralRolloff,\n spectralBandwidth,\n mfcc,\n chromagram,\n }\n}\n\n// --- Streaming (per-frame) API ---\n\n/**\n * Encapsulates a single WASM streaming audio features session.\n * Each instance owns its own WASM heap allocations; multiple sessions\n * can exist concurrently without interfering with each other.\n *\n * Usage:\n * const session = await AudioFeaturesStreamingSession.create(sampleRate)\n * try {\n * for (const frame of frames) {\n * const result = session.computeFrame(frame)\n * }\n * } finally {\n * session.dispose()\n * }\n */\nexport class AudioFeaturesStreamingSession {\n private module: AudioFeaturesWasmModule\n private framePtr = 0\n private frameCapacity = 0\n private resultPtr = 0\n\n private constructor(module: AudioFeaturesWasmModule) {\n this.module = module\n }\n\n /**\n * Initialise a new streaming session. Loads the WASM module if needed.\n */\n static async create(\n sampleRate: number,\n fftLength = 1024,\n nMfcc = 13,\n nMelFilters = 26,\n computeMfcc = true,\n computeChroma = true\n ): Promise<AudioFeaturesStreamingSession> {\n const Module = await getWasmModule()\n const session = new AudioFeaturesStreamingSession(Module)\n\n Module._audio_features_init(\n sampleRate,\n fftLength,\n nMfcc,\n nMelFilters,\n computeMfcc ? 1 : 0,\n computeChroma ? 1 : 0\n )\n\n // Pre-allocate result struct on WASM heap\n session.resultPtr = Module._malloc(STRUCT_SIZE)\n // Zero-initialize to prevent freeing garbage pointers on first use\n Module.HEAPU8.fill(\n 0,\n session.resultPtr,\n session.resultPtr + STRUCT_SIZE\n )\n\n return session\n }\n\n /**\n * Compute audio features for a single frame.\n * Returns null on error or if the session has been disposed.\n */\n computeFrame(samples: Float32Array): AudioFeaturesWasmResult | null {\n if (!this.resultPtr) return null\n const Module = this.module\n\n // (Re-)allocate frame input buffer if needed\n if (samples.length > this.frameCapacity) {\n if (this.framePtr) Module._free(this.framePtr)\n this.framePtr = Module._malloc(samples.length * 4)\n this.frameCapacity = samples.length\n }\n\n // Copy samples to WASM heap\n Module.HEAPF32.set(samples, this.framePtr >> 2)\n\n const ok = Module._audio_features_compute_frame(\n this.framePtr,\n samples.length,\n this.resultPtr\n )\n if (!ok) return null\n\n const result = readResult(Module, this.resultPtr)\n\n // Free internal arrays (mfcc, chromagram) allocated by C\n Module._audio_features_free_arrays(this.resultPtr)\n\n return result\n }\n\n /**\n * Free all WASM heap allocations owned by this session.\n * The session must not be used after calling dispose().\n */\n dispose(): void {\n const Module = this.module\n if (this.framePtr) {\n Module._free(this.framePtr)\n this.framePtr = 0\n this.frameCapacity = 0\n }\n if (this.resultPtr) {\n Module._free(this.resultPtr)\n this.resultPtr = 0\n }\n }\n}\n\n// --- Batch API ---\n\n/**\n * Compute audio features for a buffer of samples via WASM C++.\n * Lazy-loads the WASM module on first call.\n */\nexport async function computeAudioFeaturesWasm(\n audioData: Float32Array,\n sampleRate: number,\n fftLength = 1024,\n nMfcc = 13,\n nMelFilters = 26,\n computeMfcc = true,\n computeChroma = true\n): Promise<AudioFeaturesWasmResult> {\n const Module = await getWasmModule()\n\n const numSamples = audioData.length\n const inputPtr = Module._malloc(numSamples * 4)\n Module.HEAPF32.set(audioData, inputPtr >> 2)\n\n const resultPtr = Module._audio_features_compute(\n inputPtr,\n numSamples,\n sampleRate,\n fftLength,\n nMfcc,\n nMelFilters,\n computeMfcc ? 1 : 0,\n computeChroma ? 1 : 0\n )\n\n Module._free(inputPtr)\n\n if (resultPtr === 0) {\n throw new Error('audio_features_compute returned null')\n }\n\n const result = readResult(Module, resultPtr)\n Module._audio_features_free(resultPtr)\n\n return result\n}\n"]}
|
|
@@ -7,6 +7,13 @@ import crc32 from '../utils/crc32';
|
|
|
7
7
|
import { getWavFileInfo } from '../utils/getWavFileInfo';
|
|
8
8
|
import { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web';
|
|
9
9
|
import { wasmGlueJs } from '../workers/wasmGlueString.web';
|
|
10
|
+
function createAnalysisWorker() {
|
|
11
|
+
const blob = new Blob([wasmGlueJs, '\n', InlineFeaturesExtractor], {
|
|
12
|
+
type: 'application/javascript',
|
|
13
|
+
});
|
|
14
|
+
const workerUrl = URL.createObjectURL(blob);
|
|
15
|
+
return { worker: new Worker(workerUrl), workerUrl };
|
|
16
|
+
}
|
|
10
17
|
function calculateCRC32ForDataPoint(data) {
|
|
11
18
|
// Convert float array to byte array for CRC32
|
|
12
19
|
const byteArray = new Uint8Array(data.length * 4);
|
|
@@ -49,12 +56,12 @@ export async function extractAudioAnalysis(props) {
|
|
|
49
56
|
});
|
|
50
57
|
const channelData = processedBuffer.buffer.getChannelData(0);
|
|
51
58
|
// Create worker blob: WASM glue (defines createMelSpectrogramModule) + worker code
|
|
52
|
-
const
|
|
53
|
-
const workerUrl = URL.createObjectURL(blob);
|
|
54
|
-
const worker = new Worker(workerUrl);
|
|
59
|
+
const { worker, workerUrl } = createAnalysisWorker();
|
|
55
60
|
return new Promise((resolve, reject) => {
|
|
56
61
|
worker.onmessage = (event) => {
|
|
57
62
|
if (event.data.error) {
|
|
63
|
+
URL.revokeObjectURL(workerUrl);
|
|
64
|
+
worker.terminate();
|
|
58
65
|
reject(new Error(event.data.error));
|
|
59
66
|
return;
|
|
60
67
|
}
|
|
@@ -156,18 +163,14 @@ arrayBuffer, bitDepth, durationMs, sampleRate, numberOfChannels, features, logge
|
|
|
156
163
|
const endIndex = length ? startIndex + length : channelData.length;
|
|
157
164
|
const constrainedChannelData = channelData.slice(startIndex, endIndex);
|
|
158
165
|
return new Promise((resolve, reject) => {
|
|
159
|
-
const
|
|
160
|
-
type: 'application/javascript',
|
|
161
|
-
});
|
|
162
|
-
const url = URL.createObjectURL(blob);
|
|
163
|
-
const worker = new Worker(url);
|
|
166
|
+
const { worker, workerUrl } = createAnalysisWorker();
|
|
164
167
|
worker.onmessage = (event) => {
|
|
165
|
-
URL.revokeObjectURL(
|
|
168
|
+
URL.revokeObjectURL(workerUrl);
|
|
166
169
|
worker.terminate();
|
|
167
170
|
resolve(event.data.result);
|
|
168
171
|
};
|
|
169
172
|
worker.onerror = (error) => {
|
|
170
|
-
URL.revokeObjectURL(
|
|
173
|
+
URL.revokeObjectURL(workerUrl);
|
|
171
174
|
worker.terminate();
|
|
172
175
|
reject(error);
|
|
173
176
|
};
|
|
@@ -191,13 +194,13 @@ arrayBuffer, bitDepth, durationMs, sampleRate, numberOfChannels, features, logge
|
|
|
191
194
|
fileUri,
|
|
192
195
|
segmentDurationMs,
|
|
193
196
|
});
|
|
194
|
-
const res = await AudioStudioModule.extractAudioAnalysis({
|
|
197
|
+
const res = await AudioStudioModule.extractAudioAnalysis(cleanNativeOptions({
|
|
195
198
|
fileUri,
|
|
196
199
|
segmentDurationMs,
|
|
197
200
|
features,
|
|
198
201
|
position,
|
|
199
202
|
length,
|
|
200
|
-
});
|
|
203
|
+
}));
|
|
201
204
|
logger?.log(`extractAudioAnalysis`, res);
|
|
202
205
|
return res;
|
|
203
206
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"extractAudioAnalysis.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/extractAudioAnalysis.ts"],"names":[],"mappings":"AAQA,OAAO,iBAAiB,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAOpC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,EAAE,cAAc,EAAe,MAAM,yBAAyB,CAAA;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAA;AAE1D,SAAS,0BAA0B,CAAC,IAAkB;IAClD,8CAA8C;IAC9C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AAC/B,CAAC;AAwDD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,KAAgC;IAEhC,MAAM,EACF,OAAO,EACP,WAAW,EACX,eAAe,EACf,MAAM,EACN,iBAAiB,GAAG,GAAG,EACvB,QAAQ,GACX,GAAG,KAAK,CAAA;IAET,IAAI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC;YACD,2BAA2B;YAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY;gBACxC,MAAc,CAAC,kBAAkB,CAAC,CAAC;gBACpC,UAAU,EAAE,eAAe,EAAE,gBAAgB,IAAI,KAAK;aACzD,CAAC,CAAA;YAEF,IAAI,CAAC;gBACD,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAAC;oBAC7C,WAAW;oBACX,OAAO;oBACP,gBAAgB,EACZ,eAAe,EAAE,gBAAgB,IAAI,KAAK;oBAC9C,cAAc,EAAE,eAAe,EAAE,cAAc,IAAI,CAAC;oBACpD,cAAc,EAAE,eAAe,EAAE,cAAc,IAAI,KAAK;oBACxD,WAAW,EACP,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;oBAC1D,SAAS,EACL,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACtD,QAAQ,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;oBAC1D,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;oBACpD,YAAY,EAAE,8BAA8B;oBAC5C,MAAM;iBACT,CAAC,CAAA;gBAEF,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;gBAE5D,mFAAmF;gBACnF,MAAM,IAAI,GAAG,IAAI,IAAI,CACjB,CAAC,UAAU,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAC3C,EAAE,IAAI,EAAE,wBAAwB,EAAE,CACrC,CAAA;gBACD,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;gBAC3C,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAA;gBAEpC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACnC,MAAM,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;wBACzB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BACnB,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;4BACnC,OAAM;wBACV,CAAC;wBAED,MAAM,MAAM,GAAkB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAA;wBAC/C,sDAAsD;wBACtD,IAAI,QAAQ,EAAE,KAAK,EAAE,CAAC;4BAClB,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAChC,CAAC,eAAe,CAAC,UAAU;gCACvB,iBAAiB,CAAC;gCAClB,IAAI,CACX,CAAA;4BAED,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CACrC,CAAC,KAAgB,EAAE,KAAa,EAAE,EAAE;gCAChC,MAAM,WAAW,GACb,KAAK,GAAG,iBAAiB,CAAA;gCAC7B,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACjC,WAAW,EACX,WAAW,GAAG,iBAAiB,CAClC,CAAA;gCAED,OAAO;oCACH,GAAG,KAAK;oCACR,QAAQ,EAAE;wCACN,GAAG,KAAK,CAAC,QAAQ;wCACjB,KAAK,EAAE,0BAA0B,CAC7B,WAAW,CACd;qCACJ;iCACJ,CAAA;4BACL,CAAC,CACJ,CAAA;wBACL,CAAC;wBAED,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;wBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;wBAClB,OAAO,CAAC,MAAM,CAAC,CAAA;oBACnB,CAAC,CAAA;oBAED,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;wBACvB,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;wBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;wBAClB,MAAM,CAAC,KAAK,CAAC,CAAA;oBACjB,CAAC,CAAA;oBAED,MAAM,CAAC,WAAW,CAAC;wBACf,WAAW;wBACX,UAAU,EAAE,eAAe,CAAC,UAAU;wBACtC,iBAAiB;wBACjB,QAAQ,EAAE,eAAe,EAAE,cAAc,IAAI,EAAE;wBAC/C,gBAAgB,EAAE,eAAe,CAAC,QAAQ;wBAC1C,mBAAmB,EAAE,eAAe,CAAC,UAAU;wBAC/C,2BAA2B;wBAC3B,QAAQ;qBACX,CAAC,CAAA;gBACN,CAAC,CAAC,CAAA;YACN,CAAC;oBAAS,CAAC;gBACP,MAAM,YAAY,CAAC,KAAK,EAAE,CAAA;YAC9B,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,EAAE,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAA;YAChD,MAAM,KAAK,CAAA;QACf,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,EACF,MAAM,EAAE,OAAO,EACf,WAAW,EAAE,YAAY,EACzB,GAAG,aAAa,EACnB,GAAG,KAAK,CAAA;QACT,8DAA8D;QAC9D,OAAO,MAAM,iBAAiB,CAAC,oBAAoB,CAC/C,kBAAkB,CAAC,aAAa,CAAC,CACpC,CAAA;IACL,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EAAE,EACxC,OAAO,EACP,iBAAiB,GAAG,GAAG,EAAE,mBAAmB;AAC5C,WAAW,EACX,QAAQ,EACR,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,QAAQ,EACR,MAAM,EACN,QAAQ,GAAG,CAAC,EACZ,MAAM,GACqB,EAA0B,EAAE;IACvD,IAAI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QACrE,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,MAAM,EAAE,GAAG,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAA;YACxC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAQ,CAAC,CAAA;YAEtC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACX,4BAA4B,QAAQ,CAAC,UAAU,EAAE,CACpD,CAAA;YACL,CAAC;YAED,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAA;YAC1C,MAAM,EAAE,GAAG,CAAC,iBAAiB,EAAE,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;QACvE,CAAC;QAED,kEAAkE;QAClE,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,GAAG,CACP,iCAAiC,QAAQ,QAAQ,UAAU,CAAC,UAAU,EAAE,EACxE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAC3B,CAAA;QAED,IAAI,cAAc,GAAG,QAAQ,CAAA;QAC7B,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,MAAM,EAAE,GAAG,CACP,qEAAqE,CACxE,CAAA;YACD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAA;YACjD,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAA;QACtC,CAAC;QACD,MAAM,EAAE,GAAG,CAAC,uCAAuC,cAAc,EAAE,CAAC,CAAA;QAEpE,MAAM,EACF,SAAS,EAAE,WAAW,EACtB,GAAG,EACH,GAAG,GACN,GAAG,MAAM,mBAAmB,CAAC;YAC1B,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,cAAc;SAC3B,CAAC,CAAA;QACF,MAAM,EAAE,GAAG,CACP,mDAAmD,WAAW,CAAC,MAAM,aAAa,GAAG,OAAO,GAAG,IAAI,CACtG,CAAA;QAED,oEAAoE;QACpE,MAAM,UAAU,GAAG,QAAQ,CAAA;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAA;QAClE,MAAM,sBAAsB,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAAE;gBAC/D,IAAI,EAAE,wBAAwB;aACjC,CAAC,CAAA;YACF,MAAM,GAAG,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;YACrC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,GAAG,CAAC,CAAA;YAE9B,MAAM,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;gBACzB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;gBACxB,MAAM,CAAC,SAAS,EAAE,CAAA;gBAClB,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC9B,CAAC,CAAA;YAED,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;gBACvB,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAA;gBACxB,MAAM,CAAC,SAAS,EAAE,CAAA;gBAClB,MAAM,CAAC,KAAK,CAAC,CAAA;YACjB,CAAC,CAAA;YAED,MAAM,CAAC,WAAW,CAAC;gBACf,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,sBAAsB;gBACnC,UAAU;gBACV,iBAAiB;gBACjB,MAAM;gBACN,QAAQ;gBACR,mBAAmB,EAAE,UAAU;gBAC/B,gBAAgB;aACnB,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;IACN,CAAC;SAAM,CAAC;QACJ,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;QAC1C,CAAC;QACD,MAAM,EAAE,GAAG,CAAC,sBAAsB,EAAE;YAChC,OAAO;YACP,iBAAiB;SACpB,CAAC,CAAA;QACF,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,oBAAoB,CAAC;YACrD,OAAO;YACP,iBAAiB;YACjB,QAAQ;YACR,QAAQ;YACR,MAAM;SACT,CAAC,CAAA;QACF,MAAM,EAAE,GAAG,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAA;QACxC,OAAO,GAAG,CAAA;IACd,CAAC;AACL,CAAC,CAAA","sourcesContent":["// packages/audio-studio/src/AudioAnalysis/extractAudioAnalysis.ts\n/**\n * This module provides functions for extracting and analyzing audio data.\n * - `extractAudioAnalysis`: For detailed analysis with customizable ranges and decoding options.\n * - `extractWavAudioAnalysis`: For analyzing WAV files without decoding, preserving original PCM values.\n * - `extractPreview`: For generating quick previews of audio waveforms, optimized for UI rendering.\n */\nimport { ConsoleLike } from '../AudioStudio.types'\nimport AudioStudioModule from '../AudioStudioModule'\nimport { isWeb } from '../constants'\nimport {\n AudioAnalysis,\n AudioFeaturesOptions,\n DataPoint,\n DecodingConfig,\n} from './AudioAnalysis.types'\nimport { processAudioBuffer } from '../utils/audioProcessing'\nimport { cleanNativeOptions } from '../utils/cleanNativeOptions'\nimport { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'\nimport crc32 from '../utils/crc32'\nimport { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'\nimport { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'\nimport { wasmGlueJs } from '../workers/wasmGlueString.web'\n\nfunction calculateCRC32ForDataPoint(data: Float32Array): number {\n // Convert float array to byte array for CRC32\n const byteArray = new Uint8Array(data.length * 4)\n const dataView = new DataView(byteArray.buffer)\n\n for (let i = 0; i < data.length; i++) {\n dataView.setFloat32(i * 4, data[i], true)\n }\n\n return crc32.buf(byteArray)\n}\n\nexport interface ExtractWavAudioAnalysisProps {\n fileUri?: string // should provide either fileUri or arrayBuffer\n wavMetadata?: WavFileInfo\n arrayBuffer?: ArrayBuffer\n bitDepth?: number\n durationMs?: number\n sampleRate?: number\n numberOfChannels?: number\n position?: number // Optional number of bytes to skip. Default is 0\n length?: number // Optional number of bytes to read.\n segmentDurationMs?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.\n features?: AudioFeaturesOptions\n featuresExtratorUrl?: string\n logger?: ConsoleLike\n decodingOptions?: DecodingConfig\n}\n\n// Define base options interface with common properties\ninterface BaseExtractOptions {\n fileUri?: string\n arrayBuffer?: ArrayBuffer\n /**\n * Duration of each analysis segment in milliseconds. Defaults to 100ms if not specified.\n */\n segmentDurationMs?: number\n features?: AudioFeaturesOptions\n decodingOptions?: DecodingConfig\n logger?: ConsoleLike\n}\n\n// Time-based range options\ninterface TimeRangeOptions extends BaseExtractOptions {\n startTimeMs?: number\n endTimeMs?: number\n position?: never\n length?: never\n}\n\n// Byte-based range options\ninterface ByteRangeOptions extends BaseExtractOptions {\n position?: number\n length?: number\n startTimeMs?: never\n endTimeMs?: never\n}\n\n/**\n * Options for extracting audio analysis.\n * - For time-based analysis, provide `startTimeMs` and `endTimeMs`.\n * - For byte-based analysis, provide `position` and `length`.\n * - Do not mix time and byte ranges.\n */\nexport type ExtractAudioAnalysisProps = TimeRangeOptions | ByteRangeOptions\n\n/**\n * Extracts detailed audio analysis from the specified audio file or buffer.\n * Supports either time-based or byte-based ranges for flexibility in analysis.\n *\n * @param props - The options for extraction, including file URI, ranges, and decoding settings.\n * @returns A promise that resolves to the audio analysis data.\n * @throws {Error} If both time and byte ranges are provided or if required parameters are missing.\n */\nexport async function extractAudioAnalysis(\n props: ExtractAudioAnalysisProps\n): Promise<AudioAnalysis> {\n const {\n fileUri,\n arrayBuffer,\n decodingOptions,\n logger,\n segmentDurationMs = 100,\n features,\n } = props\n\n if (isWeb) {\n try {\n // Create AudioContext here\n const audioContext = new (window.AudioContext ||\n (window as any).webkitAudioContext)({\n sampleRate: decodingOptions?.targetSampleRate ?? 16000,\n })\n\n try {\n const processedBuffer = await processAudioBuffer({\n arrayBuffer,\n fileUri,\n targetSampleRate:\n decodingOptions?.targetSampleRate ?? 16000,\n targetChannels: decodingOptions?.targetChannels ?? 1,\n normalizeAudio: decodingOptions?.normalizeAudio ?? false,\n startTimeMs:\n 'startTimeMs' in props ? props.startTimeMs : undefined,\n endTimeMs:\n 'endTimeMs' in props ? props.endTimeMs : undefined,\n position: 'position' in props ? props.position : undefined,\n length: 'length' in props ? props.length : undefined,\n audioContext, // Pass the context we created\n logger,\n })\n\n const channelData = processedBuffer.buffer.getChannelData(0)\n\n // Create worker blob: WASM glue (defines createMelSpectrogramModule) + worker code\n const blob = new Blob(\n [wasmGlueJs, '\\n', InlineFeaturesExtractor],\n { type: 'application/javascript' }\n )\n const workerUrl = URL.createObjectURL(blob)\n const worker = new Worker(workerUrl)\n\n return new Promise((resolve, reject) => {\n worker.onmessage = (event) => {\n if (event.data.error) {\n reject(new Error(event.data.error))\n return\n }\n\n const result: AudioAnalysis = event.data.result\n // Calculate CRC32 after worker completes if requested\n if (features?.crc32) {\n const samplesPerSegment = Math.floor(\n (processedBuffer.sampleRate *\n segmentDurationMs) /\n 1000\n )\n\n result.dataPoints = result.dataPoints.map(\n (point: DataPoint, index: number) => {\n const startSample =\n index * samplesPerSegment\n const segmentData = channelData.slice(\n startSample,\n startSample + samplesPerSegment\n )\n\n return {\n ...point,\n features: {\n ...point.features,\n crc32: calculateCRC32ForDataPoint(\n segmentData\n ),\n },\n }\n }\n )\n }\n\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n resolve(result)\n }\n\n worker.onerror = (error) => {\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n reject(error)\n }\n\n worker.postMessage({\n channelData,\n sampleRate: processedBuffer.sampleRate,\n segmentDurationMs,\n bitDepth: decodingOptions?.targetBitDepth ?? 32,\n numberOfChannels: processedBuffer.channels,\n fullAudioDurationMs: processedBuffer.durationMs,\n // enableLogging: !!logger,\n features,\n })\n })\n } finally {\n await audioContext.close()\n }\n } catch (error) {\n logger?.error('Failed to process audio:', error)\n throw error\n }\n } else {\n // Strip non-serializable fields — logger and arrayBuffer cause\n // \"Cannot convert '[object Object]' to a Kotlin type\" on Android.\n const {\n logger: _logger,\n arrayBuffer: _arrayBuffer,\n ...nativeOptions\n } = props\n // Clean undefined values to avoid Android Kotlin bridge crash\n return await AudioStudioModule.extractAudioAnalysis(\n cleanNativeOptions(nativeOptions)\n )\n }\n}\n\n/**\n * Analyzes WAV files without decoding, preserving original PCM values.\n * Use this function when you need to ensure the analysis matches other software by avoiding any transformations.\n *\n * @param props - The options for WAV analysis, including file URI and range.\n * @returns A promise that resolves to the audio analysis data.\n */\nexport const extractRawWavAnalysis = async ({\n fileUri,\n segmentDurationMs = 100, // Default to 100ms\n arrayBuffer,\n bitDepth,\n durationMs,\n sampleRate,\n numberOfChannels,\n features,\n logger,\n position = 0,\n length,\n}: ExtractWavAudioAnalysisProps): Promise<AudioAnalysis> => {\n if (isWeb) {\n if (!arrayBuffer && !fileUri) {\n throw new Error('Either arrayBuffer or fileUri must be provided')\n }\n\n if (!arrayBuffer) {\n logger?.log(`fetching fileUri`, fileUri)\n const response = await fetch(fileUri!)\n\n if (!response.ok) {\n throw new Error(\n `Failed to fetch fileUri: ${response.statusText}`\n )\n }\n\n arrayBuffer = await response.arrayBuffer()\n logger?.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)\n }\n\n // Create a new copy of the ArrayBuffer to avoid detachment issues\n const bufferCopy = arrayBuffer.slice(0)\n logger?.log(\n `extractAudioAnalysis bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,\n bufferCopy.slice(0, 100)\n )\n\n let actualBitDepth = bitDepth\n if (!actualBitDepth) {\n logger?.log(\n `extractAudioAnalysis bitDepth not provided -- getting wav file info`\n )\n const fileInfo = await getWavFileInfo(bufferCopy)\n actualBitDepth = fileInfo.bitDepth\n }\n logger?.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)\n\n const {\n pcmValues: channelData,\n min,\n max,\n } = await convertPCMToFloat32({\n buffer: arrayBuffer,\n bitDepth: actualBitDepth,\n })\n logger?.log(\n `extractAudioAnalysis convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`\n )\n\n // Apply position and length constraints to channelData if specified\n const startIndex = position\n const endIndex = length ? startIndex + length : channelData.length\n const constrainedChannelData = channelData.slice(startIndex, endIndex)\n\n return new Promise((resolve, reject) => {\n const blob = new Blob([wasmGlueJs, '\\n', InlineFeaturesExtractor], {\n type: 'application/javascript',\n })\n const url = URL.createObjectURL(blob)\n const worker = new Worker(url)\n\n worker.onmessage = (event) => {\n URL.revokeObjectURL(url)\n worker.terminate()\n resolve(event.data.result)\n }\n\n worker.onerror = (error) => {\n URL.revokeObjectURL(url)\n worker.terminate()\n reject(error)\n }\n\n worker.postMessage({\n command: 'process',\n channelData: constrainedChannelData,\n sampleRate,\n segmentDurationMs,\n logger,\n bitDepth,\n fullAudioDurationMs: durationMs,\n numberOfChannels,\n })\n })\n } else {\n if (!fileUri) {\n throw new Error('fileUri is required')\n }\n logger?.log(`extractAudioAnalysis`, {\n fileUri,\n segmentDurationMs,\n })\n const res = await AudioStudioModule.extractAudioAnalysis({\n fileUri,\n segmentDurationMs,\n features,\n position,\n length,\n })\n logger?.log(`extractAudioAnalysis`, res)\n return res\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"extractAudioAnalysis.js","sourceRoot":"","sources":["../../../src/AudioAnalysis/extractAudioAnalysis.ts"],"names":[],"mappings":"AAQA,OAAO,iBAAiB,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAA;AAOpC,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAA;AAC7D,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,8BAA8B,CAAA;AAClE,OAAO,KAAK,MAAM,gBAAgB,CAAA;AAClC,OAAO,EAAE,cAAc,EAAe,MAAM,yBAAyB,CAAA;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,wCAAwC,CAAA;AAChF,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAA;AAE1D,SAAS,oBAAoB;IACzB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,uBAAuB,CAAC,EAAE;QAC/D,IAAI,EAAE,wBAAwB;KACjC,CAAC,CAAA;IACF,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAA;IAC3C,OAAO,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,CAAA;AACvD,CAAC;AAED,SAAS,0BAA0B,CAAC,IAAkB;IAClD,8CAA8C;IAC9C,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;IACjD,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IAE/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;IAC7C,CAAC;IAED,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;AAC/B,CAAC;AAwDD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACtC,KAAgC;IAEhC,MAAM,EACF,OAAO,EACP,WAAW,EACX,eAAe,EACf,MAAM,EACN,iBAAiB,GAAG,GAAG,EACvB,QAAQ,GACX,GAAG,KAAK,CAAA;IAET,IAAI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC;YACD,2BAA2B;YAC3B,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,YAAY;gBACxC,MAAc,CAAC,kBAAkB,CAAC,CAAC;gBACpC,UAAU,EAAE,eAAe,EAAE,gBAAgB,IAAI,KAAK;aACzD,CAAC,CAAA;YAEF,IAAI,CAAC;gBACD,MAAM,eAAe,GAAG,MAAM,kBAAkB,CAAC;oBAC7C,WAAW;oBACX,OAAO;oBACP,gBAAgB,EACZ,eAAe,EAAE,gBAAgB,IAAI,KAAK;oBAC9C,cAAc,EAAE,eAAe,EAAE,cAAc,IAAI,CAAC;oBACpD,cAAc,EAAE,eAAe,EAAE,cAAc,IAAI,KAAK;oBACxD,WAAW,EACP,aAAa,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;oBAC1D,SAAS,EACL,WAAW,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;oBACtD,QAAQ,EAAE,UAAU,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;oBAC1D,MAAM,EAAE,QAAQ,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;oBACpD,YAAY,EAAE,8BAA8B;oBAC5C,MAAM;iBACT,CAAC,CAAA;gBAEF,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC,CAAA;gBAE5D,mFAAmF;gBACnF,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAA;gBAEpD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBACnC,MAAM,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;wBACzB,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;4BACnB,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;4BAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;4BAClB,MAAM,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;4BACnC,OAAM;wBACV,CAAC;wBAED,MAAM,MAAM,GAAkB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAA;wBAC/C,sDAAsD;wBACtD,IAAI,QAAQ,EAAE,KAAK,EAAE,CAAC;4BAClB,MAAM,iBAAiB,GAAG,IAAI,CAAC,KAAK,CAChC,CAAC,eAAe,CAAC,UAAU;gCACvB,iBAAiB,CAAC;gCAClB,IAAI,CACX,CAAA;4BAED,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CACrC,CAAC,KAAgB,EAAE,KAAa,EAAE,EAAE;gCAChC,MAAM,WAAW,GACb,KAAK,GAAG,iBAAiB,CAAA;gCAC7B,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CACjC,WAAW,EACX,WAAW,GAAG,iBAAiB,CAClC,CAAA;gCAED,OAAO;oCACH,GAAG,KAAK;oCACR,QAAQ,EAAE;wCACN,GAAG,KAAK,CAAC,QAAQ;wCACjB,KAAK,EAAE,0BAA0B,CAC7B,WAAW,CACd;qCACJ;iCACJ,CAAA;4BACL,CAAC,CACJ,CAAA;wBACL,CAAC;wBAED,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;wBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;wBAClB,OAAO,CAAC,MAAM,CAAC,CAAA;oBACnB,CAAC,CAAA;oBAED,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;wBACvB,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;wBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;wBAClB,MAAM,CAAC,KAAK,CAAC,CAAA;oBACjB,CAAC,CAAA;oBAED,MAAM,CAAC,WAAW,CAAC;wBACf,WAAW;wBACX,UAAU,EAAE,eAAe,CAAC,UAAU;wBACtC,iBAAiB;wBACjB,QAAQ,EAAE,eAAe,EAAE,cAAc,IAAI,EAAE;wBAC/C,gBAAgB,EAAE,eAAe,CAAC,QAAQ;wBAC1C,mBAAmB,EAAE,eAAe,CAAC,UAAU;wBAC/C,2BAA2B;wBAC3B,QAAQ;qBACX,CAAC,CAAA;gBACN,CAAC,CAAC,CAAA;YACN,CAAC;oBAAS,CAAC;gBACP,MAAM,YAAY,CAAC,KAAK,EAAE,CAAA;YAC9B,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,EAAE,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAA;YAChD,MAAM,KAAK,CAAA;QACf,CAAC;IACL,CAAC;SAAM,CAAC;QACJ,+DAA+D;QAC/D,kEAAkE;QAClE,MAAM,EACF,MAAM,EAAE,OAAO,EACf,WAAW,EAAE,YAAY,EACzB,GAAG,aAAa,EACnB,GAAG,KAAK,CAAA;QACT,8DAA8D;QAC9D,OAAO,MAAM,iBAAiB,CAAC,oBAAoB,CAC/C,kBAAkB,CAAC,aAAa,CAAC,CACpC,CAAA;IACL,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,EAAE,EACxC,OAAO,EACP,iBAAiB,GAAG,GAAG,EAAE,mBAAmB;AAC5C,WAAW,EACX,QAAQ,EACR,UAAU,EACV,UAAU,EACV,gBAAgB,EAChB,QAAQ,EACR,MAAM,EACN,QAAQ,GAAG,CAAC,EACZ,MAAM,GACqB,EAA0B,EAAE;IACvD,IAAI,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAA;QACrE,CAAC;QAED,IAAI,CAAC,WAAW,EAAE,CAAC;YACf,MAAM,EAAE,GAAG,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAA;YACxC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAQ,CAAC,CAAA;YAEtC,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CACX,4BAA4B,QAAQ,CAAC,UAAU,EAAE,CACpD,CAAA;YACL,CAAC;YAED,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAA;YAC1C,MAAM,EAAE,GAAG,CAAC,iBAAiB,EAAE,WAAW,CAAC,UAAU,EAAE,WAAW,CAAC,CAAA;QACvE,CAAC;QAED,kEAAkE;QAClE,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QACvC,MAAM,EAAE,GAAG,CACP,iCAAiC,QAAQ,QAAQ,UAAU,CAAC,UAAU,EAAE,EACxE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAC3B,CAAA;QAED,IAAI,cAAc,GAAG,QAAQ,CAAA;QAC7B,IAAI,CAAC,cAAc,EAAE,CAAC;YAClB,MAAM,EAAE,GAAG,CACP,qEAAqE,CACxE,CAAA;YACD,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,CAAA;YACjD,cAAc,GAAG,QAAQ,CAAC,QAAQ,CAAA;QACtC,CAAC;QACD,MAAM,EAAE,GAAG,CAAC,uCAAuC,cAAc,EAAE,CAAC,CAAA;QAEpE,MAAM,EACF,SAAS,EAAE,WAAW,EACtB,GAAG,EACH,GAAG,GACN,GAAG,MAAM,mBAAmB,CAAC;YAC1B,MAAM,EAAE,WAAW;YACnB,QAAQ,EAAE,cAAc;SAC3B,CAAC,CAAA;QACF,MAAM,EAAE,GAAG,CACP,mDAAmD,WAAW,CAAC,MAAM,aAAa,GAAG,OAAO,GAAG,IAAI,CACtG,CAAA;QAED,oEAAoE;QACpE,MAAM,UAAU,GAAG,QAAQ,CAAA;QAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAA;QAClE,MAAM,sBAAsB,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAA;QAEtE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACnC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,oBAAoB,EAAE,CAAA;YAEpD,MAAM,CAAC,SAAS,GAAG,CAAC,KAAK,EAAE,EAAE;gBACzB,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;gBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;gBAClB,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAC9B,CAAC,CAAA;YAED,MAAM,CAAC,OAAO,GAAG,CAAC,KAAK,EAAE,EAAE;gBACvB,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAA;gBAC9B,MAAM,CAAC,SAAS,EAAE,CAAA;gBAClB,MAAM,CAAC,KAAK,CAAC,CAAA;YACjB,CAAC,CAAA;YAED,MAAM,CAAC,WAAW,CAAC;gBACf,OAAO,EAAE,SAAS;gBAClB,WAAW,EAAE,sBAAsB;gBACnC,UAAU;gBACV,iBAAiB;gBACjB,MAAM;gBACN,QAAQ;gBACR,mBAAmB,EAAE,UAAU;gBAC/B,gBAAgB;aACnB,CAAC,CAAA;QACN,CAAC,CAAC,CAAA;IACN,CAAC;SAAM,CAAC;QACJ,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;QAC1C,CAAC;QACD,MAAM,EAAE,GAAG,CAAC,sBAAsB,EAAE;YAChC,OAAO;YACP,iBAAiB;SACpB,CAAC,CAAA;QACF,MAAM,GAAG,GAAG,MAAM,iBAAiB,CAAC,oBAAoB,CACpD,kBAAkB,CAAC;YACf,OAAO;YACP,iBAAiB;YACjB,QAAQ;YACR,QAAQ;YACR,MAAM;SACT,CAAC,CACL,CAAA;QACD,MAAM,EAAE,GAAG,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAA;QACxC,OAAO,GAAG,CAAA;IACd,CAAC;AACL,CAAC,CAAA","sourcesContent":["// packages/audio-studio/src/AudioAnalysis/extractAudioAnalysis.ts\n/**\n * This module provides functions for extracting and analyzing audio data.\n * - `extractAudioAnalysis`: For detailed analysis with customizable ranges and decoding options.\n * - `extractWavAudioAnalysis`: For analyzing WAV files without decoding, preserving original PCM values.\n * - `extractPreview`: For generating quick previews of audio waveforms, optimized for UI rendering.\n */\nimport { ConsoleLike } from '../AudioStudio.types'\nimport AudioStudioModule from '../AudioStudioModule'\nimport { isWeb } from '../constants'\nimport {\n AudioAnalysis,\n AudioFeaturesOptions,\n DataPoint,\n DecodingConfig,\n} from './AudioAnalysis.types'\nimport { processAudioBuffer } from '../utils/audioProcessing'\nimport { cleanNativeOptions } from '../utils/cleanNativeOptions'\nimport { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'\nimport crc32 from '../utils/crc32'\nimport { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'\nimport { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'\nimport { wasmGlueJs } from '../workers/wasmGlueString.web'\n\nfunction createAnalysisWorker(): { worker: Worker; workerUrl: string } {\n const blob = new Blob([wasmGlueJs, '\\n', InlineFeaturesExtractor], {\n type: 'application/javascript',\n })\n const workerUrl = URL.createObjectURL(blob)\n return { worker: new Worker(workerUrl), workerUrl }\n}\n\nfunction calculateCRC32ForDataPoint(data: Float32Array): number {\n // Convert float array to byte array for CRC32\n const byteArray = new Uint8Array(data.length * 4)\n const dataView = new DataView(byteArray.buffer)\n\n for (let i = 0; i < data.length; i++) {\n dataView.setFloat32(i * 4, data[i], true)\n }\n\n return crc32.buf(byteArray)\n}\n\nexport interface ExtractWavAudioAnalysisProps {\n fileUri?: string // should provide either fileUri or arrayBuffer\n wavMetadata?: WavFileInfo\n arrayBuffer?: ArrayBuffer\n bitDepth?: number\n durationMs?: number\n sampleRate?: number\n numberOfChannels?: number\n position?: number // Optional number of bytes to skip. Default is 0\n length?: number // Optional number of bytes to read.\n segmentDurationMs?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.\n features?: AudioFeaturesOptions\n featuresExtratorUrl?: string\n logger?: ConsoleLike\n decodingOptions?: DecodingConfig\n}\n\n// Define base options interface with common properties\ninterface BaseExtractOptions {\n fileUri?: string\n arrayBuffer?: ArrayBuffer\n /**\n * Duration of each analysis segment in milliseconds. Defaults to 100ms if not specified.\n */\n segmentDurationMs?: number\n features?: AudioFeaturesOptions\n decodingOptions?: DecodingConfig\n logger?: ConsoleLike\n}\n\n// Time-based range options\ninterface TimeRangeOptions extends BaseExtractOptions {\n startTimeMs?: number\n endTimeMs?: number\n position?: never\n length?: never\n}\n\n// Byte-based range options\ninterface ByteRangeOptions extends BaseExtractOptions {\n position?: number\n length?: number\n startTimeMs?: never\n endTimeMs?: never\n}\n\n/**\n * Options for extracting audio analysis.\n * - For time-based analysis, provide `startTimeMs` and `endTimeMs`.\n * - For byte-based analysis, provide `position` and `length`.\n * - Do not mix time and byte ranges.\n */\nexport type ExtractAudioAnalysisProps = TimeRangeOptions | ByteRangeOptions\n\n/**\n * Extracts detailed audio analysis from the specified audio file or buffer.\n * Supports either time-based or byte-based ranges for flexibility in analysis.\n *\n * @param props - The options for extraction, including file URI, ranges, and decoding settings.\n * @returns A promise that resolves to the audio analysis data.\n * @throws {Error} If both time and byte ranges are provided or if required parameters are missing.\n */\nexport async function extractAudioAnalysis(\n props: ExtractAudioAnalysisProps\n): Promise<AudioAnalysis> {\n const {\n fileUri,\n arrayBuffer,\n decodingOptions,\n logger,\n segmentDurationMs = 100,\n features,\n } = props\n\n if (isWeb) {\n try {\n // Create AudioContext here\n const audioContext = new (window.AudioContext ||\n (window as any).webkitAudioContext)({\n sampleRate: decodingOptions?.targetSampleRate ?? 16000,\n })\n\n try {\n const processedBuffer = await processAudioBuffer({\n arrayBuffer,\n fileUri,\n targetSampleRate:\n decodingOptions?.targetSampleRate ?? 16000,\n targetChannels: decodingOptions?.targetChannels ?? 1,\n normalizeAudio: decodingOptions?.normalizeAudio ?? false,\n startTimeMs:\n 'startTimeMs' in props ? props.startTimeMs : undefined,\n endTimeMs:\n 'endTimeMs' in props ? props.endTimeMs : undefined,\n position: 'position' in props ? props.position : undefined,\n length: 'length' in props ? props.length : undefined,\n audioContext, // Pass the context we created\n logger,\n })\n\n const channelData = processedBuffer.buffer.getChannelData(0)\n\n // Create worker blob: WASM glue (defines createMelSpectrogramModule) + worker code\n const { worker, workerUrl } = createAnalysisWorker()\n\n return new Promise((resolve, reject) => {\n worker.onmessage = (event) => {\n if (event.data.error) {\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n reject(new Error(event.data.error))\n return\n }\n\n const result: AudioAnalysis = event.data.result\n // Calculate CRC32 after worker completes if requested\n if (features?.crc32) {\n const samplesPerSegment = Math.floor(\n (processedBuffer.sampleRate *\n segmentDurationMs) /\n 1000\n )\n\n result.dataPoints = result.dataPoints.map(\n (point: DataPoint, index: number) => {\n const startSample =\n index * samplesPerSegment\n const segmentData = channelData.slice(\n startSample,\n startSample + samplesPerSegment\n )\n\n return {\n ...point,\n features: {\n ...point.features,\n crc32: calculateCRC32ForDataPoint(\n segmentData\n ),\n },\n }\n }\n )\n }\n\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n resolve(result)\n }\n\n worker.onerror = (error) => {\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n reject(error)\n }\n\n worker.postMessage({\n channelData,\n sampleRate: processedBuffer.sampleRate,\n segmentDurationMs,\n bitDepth: decodingOptions?.targetBitDepth ?? 32,\n numberOfChannels: processedBuffer.channels,\n fullAudioDurationMs: processedBuffer.durationMs,\n // enableLogging: !!logger,\n features,\n })\n })\n } finally {\n await audioContext.close()\n }\n } catch (error) {\n logger?.error('Failed to process audio:', error)\n throw error\n }\n } else {\n // Strip non-serializable fields — logger and arrayBuffer cause\n // \"Cannot convert '[object Object]' to a Kotlin type\" on Android.\n const {\n logger: _logger,\n arrayBuffer: _arrayBuffer,\n ...nativeOptions\n } = props\n // Clean undefined values to avoid Android Kotlin bridge crash\n return await AudioStudioModule.extractAudioAnalysis(\n cleanNativeOptions(nativeOptions)\n )\n }\n}\n\n/**\n * Analyzes WAV files without decoding, preserving original PCM values.\n * Use this function when you need to ensure the analysis matches other software by avoiding any transformations.\n *\n * @param props - The options for WAV analysis, including file URI and range.\n * @returns A promise that resolves to the audio analysis data.\n */\nexport const extractRawWavAnalysis = async ({\n fileUri,\n segmentDurationMs = 100, // Default to 100ms\n arrayBuffer,\n bitDepth,\n durationMs,\n sampleRate,\n numberOfChannels,\n features,\n logger,\n position = 0,\n length,\n}: ExtractWavAudioAnalysisProps): Promise<AudioAnalysis> => {\n if (isWeb) {\n if (!arrayBuffer && !fileUri) {\n throw new Error('Either arrayBuffer or fileUri must be provided')\n }\n\n if (!arrayBuffer) {\n logger?.log(`fetching fileUri`, fileUri)\n const response = await fetch(fileUri!)\n\n if (!response.ok) {\n throw new Error(\n `Failed to fetch fileUri: ${response.statusText}`\n )\n }\n\n arrayBuffer = await response.arrayBuffer()\n logger?.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)\n }\n\n // Create a new copy of the ArrayBuffer to avoid detachment issues\n const bufferCopy = arrayBuffer.slice(0)\n logger?.log(\n `extractAudioAnalysis bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,\n bufferCopy.slice(0, 100)\n )\n\n let actualBitDepth = bitDepth\n if (!actualBitDepth) {\n logger?.log(\n `extractAudioAnalysis bitDepth not provided -- getting wav file info`\n )\n const fileInfo = await getWavFileInfo(bufferCopy)\n actualBitDepth = fileInfo.bitDepth\n }\n logger?.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)\n\n const {\n pcmValues: channelData,\n min,\n max,\n } = await convertPCMToFloat32({\n buffer: arrayBuffer,\n bitDepth: actualBitDepth,\n })\n logger?.log(\n `extractAudioAnalysis convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`\n )\n\n // Apply position and length constraints to channelData if specified\n const startIndex = position\n const endIndex = length ? startIndex + length : channelData.length\n const constrainedChannelData = channelData.slice(startIndex, endIndex)\n\n return new Promise((resolve, reject) => {\n const { worker, workerUrl } = createAnalysisWorker()\n\n worker.onmessage = (event) => {\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n resolve(event.data.result)\n }\n\n worker.onerror = (error) => {\n URL.revokeObjectURL(workerUrl)\n worker.terminate()\n reject(error)\n }\n\n worker.postMessage({\n command: 'process',\n channelData: constrainedChannelData,\n sampleRate,\n segmentDurationMs,\n logger,\n bitDepth,\n fullAudioDurationMs: durationMs,\n numberOfChannels,\n })\n })\n } else {\n if (!fileUri) {\n throw new Error('fileUri is required')\n }\n logger?.log(`extractAudioAnalysis`, {\n fileUri,\n segmentDurationMs,\n })\n const res = await AudioStudioModule.extractAudioAnalysis(\n cleanNativeOptions({\n fileUri,\n segmentDurationMs,\n features,\n position,\n length,\n })\n )\n logger?.log(`extractAudioAnalysis`, res)\n return res\n }\n}\n"]}
|
|
@@ -1,10 +1,152 @@
|
|
|
1
1
|
import AudioStudioModule from '../AudioStudioModule';
|
|
2
2
|
import { isWeb } from '../constants';
|
|
3
|
+
import { processAudioBuffer } from '../utils/audioProcessing';
|
|
3
4
|
import { cleanNativeOptions } from '../utils/cleanNativeOptions';
|
|
5
|
+
import crc32 from '../utils/crc32';
|
|
6
|
+
import { writeWavHeader } from '../utils/writeWavHeader';
|
|
4
7
|
export const extractAudioData = async (props) => {
|
|
5
8
|
if (isWeb) {
|
|
6
|
-
|
|
7
|
-
|
|
9
|
+
try {
|
|
10
|
+
const { fileUri, position, length, startTimeMs, endTimeMs, decodingOptions, includeNormalizedData, includeBase64Data, includeWavHeader = false, logger, } = props;
|
|
11
|
+
logger?.debug('EXTRACT AUDIO - Step 1: Initial request', {
|
|
12
|
+
fileUri,
|
|
13
|
+
extractionParams: {
|
|
14
|
+
position,
|
|
15
|
+
length,
|
|
16
|
+
startTimeMs,
|
|
17
|
+
endTimeMs,
|
|
18
|
+
},
|
|
19
|
+
decodingOptions: {
|
|
20
|
+
targetSampleRate: decodingOptions?.targetSampleRate ?? 16000,
|
|
21
|
+
targetChannels: decodingOptions?.targetChannels ?? 1,
|
|
22
|
+
targetBitDepth: decodingOptions?.targetBitDepth ?? 16,
|
|
23
|
+
normalizeAudio: decodingOptions?.normalizeAudio ?? false,
|
|
24
|
+
},
|
|
25
|
+
outputOptions: {
|
|
26
|
+
includeNormalizedData,
|
|
27
|
+
includeBase64Data,
|
|
28
|
+
includeWavHeader,
|
|
29
|
+
},
|
|
30
|
+
});
|
|
31
|
+
// Process the audio using shared helper function
|
|
32
|
+
const processedBuffer = await processAudioBuffer({
|
|
33
|
+
fileUri,
|
|
34
|
+
targetSampleRate: decodingOptions?.targetSampleRate ?? 16000,
|
|
35
|
+
targetChannels: decodingOptions?.targetChannels ?? 1,
|
|
36
|
+
normalizeAudio: decodingOptions?.normalizeAudio ?? false,
|
|
37
|
+
position,
|
|
38
|
+
length,
|
|
39
|
+
startTimeMs,
|
|
40
|
+
endTimeMs,
|
|
41
|
+
logger,
|
|
42
|
+
});
|
|
43
|
+
logger?.debug('EXTRACT AUDIO - Step 2: Audio processing complete', {
|
|
44
|
+
processedData: {
|
|
45
|
+
samples: processedBuffer.samples,
|
|
46
|
+
sampleRate: processedBuffer.sampleRate,
|
|
47
|
+
channels: processedBuffer.channels,
|
|
48
|
+
durationMs: processedBuffer.durationMs,
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
const channelData = processedBuffer.channelData;
|
|
52
|
+
const bitDepth = (decodingOptions?.targetBitDepth ?? 16);
|
|
53
|
+
const bytesPerSample = bitDepth / 8;
|
|
54
|
+
const numSamples = processedBuffer.samples;
|
|
55
|
+
logger?.debug('EXTRACT AUDIO - Step 3: PCM conversion setup', {
|
|
56
|
+
channelData: {
|
|
57
|
+
length: channelData.length,
|
|
58
|
+
first: channelData[0],
|
|
59
|
+
last: channelData[channelData.length - 1],
|
|
60
|
+
},
|
|
61
|
+
calculation: {
|
|
62
|
+
bitDepth,
|
|
63
|
+
bytesPerSample,
|
|
64
|
+
numSamples,
|
|
65
|
+
expectedBytes: numSamples * bytesPerSample,
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
// Create PCM data with correct length based on original byte length
|
|
69
|
+
const pcmData = new Uint8Array(numSamples * bytesPerSample);
|
|
70
|
+
let offset = 0;
|
|
71
|
+
// Convert Float32 samples to PCM format
|
|
72
|
+
for (let i = 0; i < numSamples; i++) {
|
|
73
|
+
const sample = channelData[i];
|
|
74
|
+
const value = Math.max(-1, Math.min(1, sample));
|
|
75
|
+
// Convert to 16-bit signed integer
|
|
76
|
+
let intValue = Math.round(value * 32767);
|
|
77
|
+
// Handle negative values correctly
|
|
78
|
+
if (intValue < 0) {
|
|
79
|
+
intValue = 65536 + intValue;
|
|
80
|
+
}
|
|
81
|
+
// Write as little-endian
|
|
82
|
+
pcmData[offset++] = intValue & 255; // Low byte
|
|
83
|
+
pcmData[offset++] = (intValue >> 8) & 255; // High byte
|
|
84
|
+
}
|
|
85
|
+
const durationMs = Math.round((numSamples / processedBuffer.sampleRate) * 1000);
|
|
86
|
+
logger?.debug('EXTRACT AUDIO - Step 4: Final output', {
|
|
87
|
+
pcmData: {
|
|
88
|
+
length: pcmData.length,
|
|
89
|
+
first: pcmData[0],
|
|
90
|
+
last: pcmData[pcmData.length - 1],
|
|
91
|
+
},
|
|
92
|
+
timing: {
|
|
93
|
+
numSamples,
|
|
94
|
+
sampleRate: processedBuffer.sampleRate,
|
|
95
|
+
durationMs,
|
|
96
|
+
shouldBe3000ms: endTimeMs
|
|
97
|
+
? endTimeMs - (startTimeMs ?? 0) === 3000
|
|
98
|
+
: undefined,
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
const result = {
|
|
102
|
+
pcmData: new Uint8Array(pcmData.buffer),
|
|
103
|
+
sampleRate: processedBuffer.sampleRate,
|
|
104
|
+
channels: processedBuffer.channels,
|
|
105
|
+
bitDepth,
|
|
106
|
+
durationMs,
|
|
107
|
+
format: `pcm_${bitDepth}bit`,
|
|
108
|
+
samples: numSamples,
|
|
109
|
+
};
|
|
110
|
+
// Add WAV header if requested
|
|
111
|
+
if (includeWavHeader) {
|
|
112
|
+
logger?.debug('EXTRACT AUDIO - Step 5: Adding WAV header', {
|
|
113
|
+
originalLength: pcmData.length,
|
|
114
|
+
newLength: result.pcmData.length,
|
|
115
|
+
firstBytes: Array.from(result.pcmData.slice(0, 44)), // WAV header is 44 bytes
|
|
116
|
+
});
|
|
117
|
+
const wavBuffer = writeWavHeader({
|
|
118
|
+
buffer: pcmData.buffer.slice(0, pcmData.length),
|
|
119
|
+
sampleRate: processedBuffer.sampleRate,
|
|
120
|
+
numChannels: processedBuffer.channels,
|
|
121
|
+
bitDepth,
|
|
122
|
+
});
|
|
123
|
+
result.pcmData = new Uint8Array(wavBuffer);
|
|
124
|
+
result.hasWavHeader = true;
|
|
125
|
+
}
|
|
126
|
+
if (includeNormalizedData) {
|
|
127
|
+
result.normalizedData = channelData;
|
|
128
|
+
}
|
|
129
|
+
if (includeBase64Data) {
|
|
130
|
+
result.base64Data = btoa(String.fromCharCode(...new Uint8Array(pcmData.buffer)));
|
|
131
|
+
}
|
|
132
|
+
if (props.computeChecksum) {
|
|
133
|
+
result.checksum = crc32.buf(pcmData);
|
|
134
|
+
}
|
|
135
|
+
logger?.debug('EXTRACT AUDIO - Step 3: PCM conversion complete', {
|
|
136
|
+
pcmStats: {
|
|
137
|
+
length: pcmData.length,
|
|
138
|
+
bytesPerSample,
|
|
139
|
+
totalSamples: numSamples,
|
|
140
|
+
firstBytes: Array.from(pcmData.slice(0, 16)),
|
|
141
|
+
lastBytes: Array.from(pcmData.slice(-16)),
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
return result;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
props.logger?.error('EXTRACT AUDIO - Error:', error);
|
|
148
|
+
throw error;
|
|
149
|
+
}
|
|
8
150
|
}
|
|
9
151
|
// Native: only pass serializable fields — logger causes crash on Android
|
|
10
152
|
const { logger: _logger, ...nativeOptions } = props;
|