@siteed/audio-studio 3.0.0

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -0,0 +1,350 @@
+// packages/audio-studio/src/AudioAnalysis/extractAudioAnalysis.ts
+/**
+ * This module provides functions for extracting and analyzing audio data.
+ * - `extractAudioAnalysis`: For detailed analysis with customizable ranges and decoding options.
+ * - `extractWavAudioAnalysis`: For analyzing WAV files without decoding, preserving original PCM values.
+ * - `extractPreview`: For generating quick previews of audio waveforms, optimized for UI rendering.
+ */
+import { ConsoleLike } from '../AudioStudio.types'
+import AudioStudioModule from '../AudioStudioModule'
+import { isWeb } from '../constants'
+import {
+    AudioAnalysis,
+    AudioFeaturesOptions,
+    DataPoint,
+    DecodingConfig,
+} from './AudioAnalysis.types'
+import { processAudioBuffer } from '../utils/audioProcessing'
+import { cleanNativeOptions } from '../utils/cleanNativeOptions'
+import { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'
+import crc32 from '../utils/crc32'
+import { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'
+import { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'
+import { wasmGlueJs } from '../workers/wasmGlueString.web'
+
+function calculateCRC32ForDataPoint(data: Float32Array): number {
+    // Convert float array to byte array for CRC32
+    const byteArray = new Uint8Array(data.length * 4)
+    const dataView = new DataView(byteArray.buffer)
+
+    for (let i = 0; i < data.length; i++) {
+        dataView.setFloat32(i * 4, data[i], true)
+    }
+
+    return crc32.buf(byteArray)
+}
+
+export interface ExtractWavAudioAnalysisProps {
+    fileUri?: string // should provide either fileUri or arrayBuffer
+    wavMetadata?: WavFileInfo
+    arrayBuffer?: ArrayBuffer
+    bitDepth?: number
+    durationMs?: number
+    sampleRate?: number
+    numberOfChannels?: number
+    position?: number // Optional number of bytes to skip. Default is 0
+    length?: number // Optional number of bytes to read.
+    segmentDurationMs?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.
+    features?: AudioFeaturesOptions
+    featuresExtratorUrl?: string
+    logger?: ConsoleLike
+    decodingOptions?: DecodingConfig
+}
+
+// Define base options interface with common properties
+interface BaseExtractOptions {
+    fileUri?: string
+    arrayBuffer?: ArrayBuffer
+    /**
+     * Duration of each analysis segment in milliseconds. Defaults to 100ms if not specified.
+     */
+    segmentDurationMs?: number
+    features?: AudioFeaturesOptions
+    decodingOptions?: DecodingConfig
+    logger?: ConsoleLike
+}
+
+// Time-based range options
+interface TimeRangeOptions extends BaseExtractOptions {
+    startTimeMs?: number
+    endTimeMs?: number
+    position?: never
+    length?: never
+}
+
+// Byte-based range options
+interface ByteRangeOptions extends BaseExtractOptions {
+    position?: number
+    length?: number
+    startTimeMs?: never
+    endTimeMs?: never
+}
+
+/**
+ * Options for extracting audio analysis.
+ * - For time-based analysis, provide `startTimeMs` and `endTimeMs`.
+ * - For byte-based analysis, provide `position` and `length`.
+ * - Do not mix time and byte ranges.
+ */
+export type ExtractAudioAnalysisProps = TimeRangeOptions | ByteRangeOptions
+
+/**
+ * Extracts detailed audio analysis from the specified audio file or buffer.
+ * Supports either time-based or byte-based ranges for flexibility in analysis.
+ *
+ * @param props - The options for extraction, including file URI, ranges, and decoding settings.
+ * @returns A promise that resolves to the audio analysis data.
+ * @throws {Error} If both time and byte ranges are provided or if required parameters are missing.
+ */
+export async function extractAudioAnalysis(
+    props: ExtractAudioAnalysisProps
+): Promise<AudioAnalysis> {
+    const {
+        fileUri,
+        arrayBuffer,
+        decodingOptions,
+        logger,
+        segmentDurationMs = 100,
+        features,
+    } = props
+
+    if (isWeb) {
+        try {
+            // Create AudioContext here
+            const audioContext = new (window.AudioContext ||
+                (window as any).webkitAudioContext)({
+                sampleRate: decodingOptions?.targetSampleRate ?? 16000,
+            })
+
+            try {
+                const processedBuffer = await processAudioBuffer({
+                    arrayBuffer,
+                    fileUri,
+                    targetSampleRate:
+                        decodingOptions?.targetSampleRate ?? 16000,
+                    targetChannels: decodingOptions?.targetChannels ?? 1,
+                    normalizeAudio: decodingOptions?.normalizeAudio ?? false,
+                    startTimeMs:
+                        'startTimeMs' in props ? props.startTimeMs : undefined,
+                    endTimeMs:
+                        'endTimeMs' in props ? props.endTimeMs : undefined,
+                    position: 'position' in props ? props.position : undefined,
+                    length: 'length' in props ? props.length : undefined,
+                    audioContext, // Pass the context we created
+                    logger,
+                })
+
+                const channelData = processedBuffer.buffer.getChannelData(0)
+
+                // Create worker blob: WASM glue (defines createMelSpectrogramModule) + worker code
+                const blob = new Blob(
+                    [wasmGlueJs, '\n', InlineFeaturesExtractor],
+                    { type: 'application/javascript' }
+                )
+                const workerUrl = URL.createObjectURL(blob)
+                const worker = new Worker(workerUrl)
+
+                return new Promise((resolve, reject) => {
+                    worker.onmessage = (event) => {
+                        if (event.data.error) {
+                            reject(new Error(event.data.error))
+                            return
+                        }
+
+                        const result: AudioAnalysis = event.data.result
+                        // Calculate CRC32 after worker completes if requested
+                        if (features?.crc32) {
+                            const samplesPerSegment = Math.floor(
+                                (processedBuffer.sampleRate *
+                                    segmentDurationMs) /
+                                    1000
+                            )
+
+                            result.dataPoints = result.dataPoints.map(
+                                (point: DataPoint, index: number) => {
+                                    const startSample =
+                                        index * samplesPerSegment
+                                    const segmentData = channelData.slice(
+                                        startSample,
+                                        startSample + samplesPerSegment
+                                    )
+
+                                    return {
+                                        ...point,
+                                        features: {
+                                            ...point.features,
+                                            crc32: calculateCRC32ForDataPoint(
+                                                segmentData
+                                            ),
+                                        },
+                                    }
+                                }
+                            )
+                        }
+
+                        URL.revokeObjectURL(workerUrl)
+                        worker.terminate()
+                        resolve(result)
+                    }
+
+                    worker.onerror = (error) => {
+                        URL.revokeObjectURL(workerUrl)
+                        worker.terminate()
+                        reject(error)
+                    }
+
+                    worker.postMessage({
+                        channelData,
+                        sampleRate: processedBuffer.sampleRate,
+                        segmentDurationMs,
+                        bitDepth: decodingOptions?.targetBitDepth ?? 32,
+                        numberOfChannels: processedBuffer.channels,
+                        fullAudioDurationMs: processedBuffer.durationMs,
+                        // enableLogging: !!logger,
+                        features,
+                    })
+                })
+            } finally {
+                await audioContext.close()
+            }
+        } catch (error) {
+            logger?.error('Failed to process audio:', error)
+            throw error
+        }
+    } else {
+        // Strip non-serializable fields — logger and arrayBuffer cause
+        // "Cannot convert '[object Object]' to a Kotlin type" on Android.
+        const {
+            logger: _logger,
+            arrayBuffer: _arrayBuffer,
+            ...nativeOptions
+        } = props
+        // Clean undefined values to avoid Android Kotlin bridge crash
+        return await AudioStudioModule.extractAudioAnalysis(
+            cleanNativeOptions(nativeOptions)
+        )
+    }
+}
+
+/**
+ * Analyzes WAV files without decoding, preserving original PCM values.
+ * Use this function when you need to ensure the analysis matches other software by avoiding any transformations.
+ *
+ * @param props - The options for WAV analysis, including file URI and range.
+ * @returns A promise that resolves to the audio analysis data.
+ */
+export const extractRawWavAnalysis = async ({
+    fileUri,
+    segmentDurationMs = 100, // Default to 100ms
+    arrayBuffer,
+    bitDepth,
+    durationMs,
+    sampleRate,
+    numberOfChannels,
+    features,
+    logger,
+    position = 0,
+    length,
+}: ExtractWavAudioAnalysisProps): Promise<AudioAnalysis> => {
+    if (isWeb) {
+        if (!arrayBuffer && !fileUri) {
+            throw new Error('Either arrayBuffer or fileUri must be provided')
+        }
+
+        if (!arrayBuffer) {
+            logger?.log(`fetching fileUri`, fileUri)
+            const response = await fetch(fileUri!)
+
+            if (!response.ok) {
+                throw new Error(
+                    `Failed to fetch fileUri: ${response.statusText}`
+                )
+            }
+
+            arrayBuffer = await response.arrayBuffer()
+            logger?.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)
+        }
+
+        // Create a new copy of the ArrayBuffer to avoid detachment issues
+        const bufferCopy = arrayBuffer.slice(0)
+        logger?.log(
+            `extractAudioAnalysis bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
+            bufferCopy.slice(0, 100)
+        )
+
+        let actualBitDepth = bitDepth
+        if (!actualBitDepth) {
+            logger?.log(
+                `extractAudioAnalysis bitDepth not provided -- getting wav file info`
+            )
+            const fileInfo = await getWavFileInfo(bufferCopy)
+            actualBitDepth = fileInfo.bitDepth
+        }
+        logger?.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)
+
+        const {
+            pcmValues: channelData,
+            min,
+            max,
+        } = await convertPCMToFloat32({
+            buffer: arrayBuffer,
+            bitDepth: actualBitDepth,
+        })
+        logger?.log(
+            `extractAudioAnalysis convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`
+        )
+
+        // Apply position and length constraints to channelData if specified
+        const startIndex = position
+        const endIndex = length ? startIndex + length : channelData.length
+        const constrainedChannelData = channelData.slice(startIndex, endIndex)
+
+        return new Promise((resolve, reject) => {
+            const blob = new Blob([wasmGlueJs, '\n', InlineFeaturesExtractor], {
+                type: 'application/javascript',
+            })
+            const url = URL.createObjectURL(blob)
+            const worker = new Worker(url)
+
+            worker.onmessage = (event) => {
+                URL.revokeObjectURL(url)
+                worker.terminate()
+                resolve(event.data.result)
+            }
+
+            worker.onerror = (error) => {
+                URL.revokeObjectURL(url)
+                worker.terminate()
+                reject(error)
+            }
+
+            worker.postMessage({
+                command: 'process',
+                channelData: constrainedChannelData,
+                sampleRate,
+                segmentDurationMs,
+                logger,
+                bitDepth,
+                fullAudioDurationMs: durationMs,
+                numberOfChannels,
+            })
+        })
+    } else {
+        if (!fileUri) {
+            throw new Error('fileUri is required')
+        }
+        logger?.log(`extractAudioAnalysis`, {
+            fileUri,
+            segmentDurationMs,
+        })
+        const res = await AudioStudioModule.extractAudioAnalysis({
+            fileUri,
+            segmentDurationMs,
+            features,
+            position,
+            length,
+        })
+        logger?.log(`extractAudioAnalysis`, res)
+        return res
+    }
+}

package/src/AudioAnalysis/extractAudioData.ts

@@ -0,0 +1,17 @@
+import { ExtractAudioDataOptions } from '../AudioStudio.types'
+import AudioStudioModule from '../AudioStudioModule'
+import { isWeb } from '../constants'
+import { cleanNativeOptions } from '../utils/cleanNativeOptions'
+
+export const extractAudioData = async (props: ExtractAudioDataOptions) => {
+    if (isWeb) {
+        // Web implementation handles logger natively in AudioStudioModule.ts
+        return await AudioStudioModule.extractAudioData(props)
+    }
+    // Native: only pass serializable fields — logger causes crash on Android
+    const { logger: _logger, ...nativeOptions } = props
+    // Clean undefined values to avoid Android Kotlin bridge crash
+    return await AudioStudioModule.extractAudioData(
+        cleanNativeOptions(nativeOptions)
+    )
+}

package/src/AudioAnalysis/extractMelSpectrogram.ts

@@ -0,0 +1,140 @@
+/**
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+
+import { AudioStudioModule } from '..'
+import { isWeb } from '../constants'
+import {
+    ExtractMelSpectrogramOptions,
+    MelSpectrogram,
+} from './AudioAnalysis.types'
+import { computeMelSpectrogramWasm } from './melSpectrogramWasm'
+import {
+    processAudioBuffer,
+    ProcessedAudioData,
+} from '../utils/audioProcessing'
+import { cleanNativeOptions } from '../utils/cleanNativeOptions'
+
+/**
+ * Maximum duration in milliseconds that extractMelSpectrogram will process in a single call.
+ * The C++ core requires the entire trimmed range as a contiguous float array in memory,
+ * so this bound prevents OOM on all platforms. Callers needing longer ranges can iterate
+ * in windows of this size using startTimeMs/endTimeMs.
+ */
+export const MAX_DURATION_MS = 30_000
+
+/**
+ * Extracts a mel spectrogram from audio data
+ *
+ * @experimental This feature is experimental.
+ * Uses shared C++ implementation on all platforms (native on iOS/Android, WASM on web).
+ */
+export async function extractMelSpectrogram(
+    options: ExtractMelSpectrogramOptions
+): Promise<MelSpectrogram> {
+    let {
+        fileUri,
+        arrayBuffer,
+        windowSizeMs,
+        hopLengthMs,
+        nMels,
+        fMin = 0,
+        fMax,
+        windowType = 'hann',
+        normalize = false,
+        logScale = true,
+        decodingOptions,
+        startTimeMs,
+        endTimeMs,
+        logger,
+    } = options
+
+    // Apply max duration guard
+    if (startTimeMs == null && endTimeMs == null) {
+        startTimeMs = 0
+        endTimeMs = MAX_DURATION_MS
+        logger?.warn?.(
+            `extractMelSpectrogram: no time range specified, defaulting to 0–${MAX_DURATION_MS}ms`
+        )
+    } else {
+        const start = startTimeMs ?? 0
+        const end = endTimeMs ?? start + MAX_DURATION_MS
+        if (end - start > MAX_DURATION_MS) {
+            endTimeMs = start + MAX_DURATION_MS
+            logger?.warn?.(
+                `extractMelSpectrogram: requested range ${end - start}ms exceeds max ${MAX_DURATION_MS}ms, clamping endTimeMs to ${endTimeMs}`
+            )
+        }
+    }
+
+    if (isWeb) {
+        // Create audio context
+        const audioContext = new (window.AudioContext ||
+            (window as any).webkitAudioContext)()
+
+        try {
+            // Process audio data using the existing utility
+            const processedAudio: ProcessedAudioData = await processAudioBuffer(
+                {
+                    arrayBuffer,
+                    fileUri,
+                    targetSampleRate:
+                        decodingOptions?.targetSampleRate || 16000,
+                    targetChannels: decodingOptions?.targetChannels || 1,
+                    normalizeAudio: decodingOptions?.normalizeAudio ?? false,
+                    startTimeMs,
+                    endTimeMs,
+                    audioContext,
+                    logger: options.logger,
+                }
+            )
+
+            // Calculate window and hop size in samples
+            const sampleRate = processedAudio.sampleRate
+            const windowSize = Math.floor((windowSizeMs * sampleRate) / 1000)
+            const hopLength = Math.floor((hopLengthMs * sampleRate) / 1000)
+            const maxFreq = fMax || sampleRate / 2
+
+            // Extract the mel spectrogram via WASM (same C++ as native)
+            const spectrogram = await computeMelSpectrogramWasm(
+                processedAudio.channelData,
+                sampleRate,
+                nMels,
+                windowSize,
+                hopLength,
+                fMin,
+                maxFreq,
+                windowType,
+                normalize,
+                logScale
+            )
+
+            const timeSteps = spectrogram.length
+
+            return {
+                spectrogram,
+                sampleRate,
+                nMels,
+                timeSteps,
+                durationMs: processedAudio.durationMs,
+            }
+        } catch (error) {
+            logger?.error('Error extracting mel spectrogram:', error)
+            throw error
+        } finally {
+            // Close the audio context
+            await audioContext.close()
+        }
+    }
+    // Strip logger/arrayBuffer (non-serializable) then clean undefined values
+    // to avoid Android "Cannot convert '[object Object]' to Kotlin type" crash
+    const {
+        logger: _logger,
+        arrayBuffer: _arrayBuffer,
+        ...nativeOptions
+    } = options
+    return AudioStudioModule.extractMelSpectrogram(
+        cleanNativeOptions(nativeOptions)
+    )
+}

package/src/AudioAnalysis/extractPreview.ts

@@ -0,0 +1,34 @@
+import { PreviewOptions, AudioAnalysis } from './AudioAnalysis.types'
+import { extractAudioAnalysis } from './extractAudioAnalysis'
+
+/**
+ * Generates a simplified preview of the audio waveform for quick visualization.
+ * Ideal for UI rendering with a specified number of points.
+ *
+ * @param options - The options for the preview, including file URI and time range.
+ * @returns A promise that resolves to the audio preview data.
+ */
+export async function extractPreview({
+    fileUri,
+    numberOfPoints = 100,
+    startTimeMs = 0,
+    endTimeMs = 30000, // First 30 seconds
+    decodingOptions,
+    logger,
+}: PreviewOptions): Promise<AudioAnalysis> {
+    const durationMs = endTimeMs - startTimeMs
+    const segmentDurationMs = Math.floor(durationMs / numberOfPoints)
+
+    // Call extractAudioAnalysis with calculated parameters
+    const analysis = await extractAudioAnalysis({
+        fileUri,
+        startTimeMs,
+        endTimeMs,
+        logger,
+        segmentDurationMs,
+        decodingOptions,
+    })
+
+    // Transform the result into AudioPreview format
+    return analysis
+}

package/src/AudioAnalysis/extractWaveform.ts

@@ -0,0 +1,22 @@
+import AudioStudioModule from '../AudioStudioModule'
+
+export interface ExtractWaveformProps {
+    fileUri: string
+    numberOfSamples: number
+    offset?: number
+    length?: number
+}
+export const extractWaveform = async ({
+    fileUri,
+    numberOfSamples,
+    offset = 0,
+    length,
+}: ExtractWaveformProps): Promise<unknown> => {
+    const res = await AudioStudioModule.extractAudioAnalysis({
+        fileUri,
+        numberOfSamples,
+        offset,
+        length,
+    })
+    return res
+}

package/src/AudioAnalysis/mel-spectrogram-wasm.d.ts

@@ -0,0 +1,48 @@
+/** Type declarations for the Emscripten-generated mel-spectrogram WASM module */
+
+export interface MelSpectrogramWasmModule {
+    _mel_spectrogram_compute(
+        samples: number,
+        numSamples: number,
+        sampleRate: number,
+        fftLength: number,
+        windowSizeSamples: number,
+        hopLengthSamples: number,
+        nMels: number,
+        fMin: number,
+        fMax: number,
+        windowType: number,
+        logScale: number,
+        normalize: number
+    ): number
+
+    _mel_spectrogram_free(resultPtr: number): void
+
+    _mel_spectrogram_init(
+        sampleRate: number,
+        fftLength: number,
+        windowSizeSamples: number,
+        hopLengthSamples: number,
+        nMels: number,
+        fMin: number,
+        fMax: number,
+        windowType: number
+    ): void
+
+    _mel_spectrogram_compute_frame(
+        framePtr: number,
+        frameSize: number,
+        melOutputPtr: number
+    ): number
+
+    _mel_spectrogram_get_n_mels(): number
+
+    _malloc(size: number): number
+    _free(ptr: number): void
+
+    HEAPF32: Float32Array
+    HEAPU8: Uint8Array
+    getValue(ptr: number, type: string): number
+}
+
+export default function createMelSpectrogramModule(): Promise<MelSpectrogramWasmModule>