@siteed/expo-audio-studio 2.1.1

package/src/AudioAnalysis/AudioAnalysis.types.ts

@@ -0,0 +1,202 @@
+// packages/expo-audio-stream/src/AudioAnalysis/AudioAnalysis.types.ts
+
+import { BitDepth, ConsoleLike } from '../ExpoAudioStream.types'
+
+/**
+ * Represents the configuration for decoding audio data.
+ */
+export interface DecodingConfig {
+    /** Target sample rate for decoded audio (Android and Web) */
+    targetSampleRate?: number
+    /** Target number of channels (Android and Web) */
+    targetChannels?: number
+    /** Target bit depth (Android and Web) */
+    targetBitDepth?: BitDepth
+    /** Whether to normalize audio levels (Android and Web) */
+    normalizeAudio?: boolean
+}
+
+/**
+ * Represents speech-related features extracted from audio.
+ */
+export interface SpeechFeatures {
+    isActive: boolean // Whether speech is detected in this segment
+    speakerId?: number // Optional speaker identification
+    // Could add more speech-related features here like:
+    // confidence: number
+    // language?: string
+    // sentiment?: number
+    // etc.
+}
+
+/**
+ * Represents various audio features extracted from an audio signal.
+ */
+export interface AudioFeatures {
+    energy?: number // The infinite integral of the squared signal, representing the overall energy of the audio.
+    mfcc?: number[] // Mel-frequency cepstral coefficients, describing the short-term power spectrum of a sound.
+    rms?: number // Root mean square value, indicating the amplitude of the audio signal.
+    minAmplitude?: number // Minimum amplitude value in the audio signal.
+    maxAmplitude?: number // Maximum amplitude value in the audio signal.
+    zcr?: number // Zero-crossing rate, indicating the rate at which the signal changes sign.
+    spectralCentroid?: number // The center of mass of the spectrum, indicating the brightness of the sound.
+    spectralFlatness?: number // Measure of the flatness of the spectrum, indicating how noise-like the signal is.
+    spectralRolloff?: number // The frequency below which a specified percentage (usually 85%) of the total spectral energy lies.
+    spectralBandwidth?: number // The width of the spectrum, indicating the range of frequencies present.
+    chromagram?: number[] // Chromagram, representing the 12 different pitch classes of the audio.
+    tempo?: number // Estimated tempo of the audio signal, measured in beats per minute (BPM).
+    hnr?: number // Harmonics-to-noise ratio, indicating the proportion of harmonics to noise in the audio signal.
+    melSpectrogram?: number[] // Mel-scaled spectrogram representation of the audio.
+    spectralContrast?: number[] // Spectral contrast features representing the difference between peaks and valleys.
+    tonnetz?: number[] // Tonal network features representing harmonic relationships.
+    pitch?: number // Pitch of the audio signal, measured in Hertz (Hz).
+    crc32?: number // crc32 checksum of the audio signal, used to verify the integrity of the audio.
+}
+
+/**
+ * Options to specify which audio features to extract.
+ */
+export interface AudioFeaturesOptions {
+    energy?: boolean
+    mfcc?: boolean
+    rms?: boolean
+    zcr?: boolean
+    spectralCentroid?: boolean
+    spectralFlatness?: boolean
+    spectralRolloff?: boolean
+    spectralBandwidth?: boolean
+    chromagram?: boolean
+    tempo?: boolean
+    hnr?: boolean
+    melSpectrogram?: boolean
+    spectralContrast?: boolean
+    tonnetz?: boolean
+    pitch?: boolean
+    crc32?: boolean
+}
+
+/**
+ * Represents a single data point in the audio analysis.
+ */
+export interface DataPoint {
+    id: number
+    amplitude: number // Peak amplitude for the segment
+    rms: number // Root mean square value
+    dB: number // dBFS (decibels relative to full scale) computed from RMS value
+    silent: boolean // Always computed
+    features?: AudioFeatures
+    speech?: SpeechFeatures
+    startTime?: number
+    endTime?: number
+    // start / end position in bytes
+    startPosition?: number
+    endPosition?: number
+    // number of audio samples for this point (samples size depends on bit depth)
+    samples?: number
+}
+
+/**
+ * Represents the complete data from the audio analysis.
+ */
+export interface AudioAnalysis {
+    segmentDurationMs: number // Duration of each segment in milliseconds
+    durationMs: number // Duration of the audio in milliseconds
+    bitDepth: number // Bit depth of the audio
+    samples: number // Size of the audio in bytes
+    numberOfChannels: number // Number of audio channels
+    sampleRate: number // Sample rate of the audio
+    dataPoints: DataPoint[] // Array of data points from the analysis.
+    amplitudeRange: {
+        min: number
+        max: number
+    }
+    rmsRange: {
+        min: number
+        max: number
+    }
+    // TODO: speaker changes into a broader speech analysis section
+    speechAnalysis?: {
+        speakerChanges: {
+            timestamp: number
+            speakerId: number
+        }[]
+        // Could add more speech analysis data here like:
+        // dominantSpeaker?: number
+        // totalSpeechDuration?: number
+        // speakerStats?: { [speakerId: number]: { duration: number, segments: number } }
+    }
+}
+
+/**
+ * Options for specifying a time range within an audio file.
+ */
+export interface AudioRangeOptions {
+    /** Start time in milliseconds */
+    startTimeMs?: number
+    /** End time in milliseconds */
+    endTimeMs?: number
+}
+
+/**
+ * Options for generating a quick preview of audio waveform.
+ * This is optimized for UI rendering with a specified number of points.
+ */
+export interface PreviewOptions extends AudioRangeOptions {
+    /** URI of the audio file to analyze */
+    fileUri: string
+    /**
+     * Total number of points to generate for the preview.
+     * @default 100
+     */
+    numberOfPoints?: number
+    /**
+     * Optional logger for debugging.
+     */
+    logger?: ConsoleLike
+    /**
+     * Optional configuration for decoding the audio file.
+     * Defaults to:
+     * - targetSampleRate: undefined (keep original)
+     * - targetChannels: undefined (keep original)
+     * - targetBitDepth: 16
+     * - normalizeAudio: false
+     */
+    decodingOptions?: DecodingConfig
+}
+
+/**
+ * Options for mel-spectrogram extraction
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface ExtractMelSpectrogramOptions {
+    fileUri?: string // Path to audio file
+    arrayBuffer?: ArrayBuffer // Raw audio buffer
+    windowSizeMs: number // Window size in ms (e.g., 25)
+    hopLengthMs: number // Hop length in ms (e.g., 10)
+    nMels: number // Number of mel filters (e.g., 60)
+    fMin?: number // Min frequency (default: 0)
+    fMax?: number // Max frequency (default: sampleRate / 2)
+    windowType?: 'hann' | 'hamming' // Window function (default: 'hann')
+    normalize?: boolean // Mean normalization (default: false)
+    logScale?: boolean // Log scaling of mel energies (default: true)
+    decodingOptions?: DecodingConfig // Audio decoding settings
+    startTimeMs?: number // Optional start time
+    endTimeMs?: number // Optional end time
+    logger?: ConsoleLike
+}
+
+/**
+ * Return type for mel spectrogram extraction
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface MelSpectrogram {
+    spectrogram: number[][] // 2D array [time][mel]
+    sampleRate: number // Audio sample rate
+    nMels: number // Number of mel filters
+    timeSteps: number // Number of time frames
+    durationMs: number // Audio duration in ms
+}

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -0,0 +1,333 @@
+// packages/expo-audio-stream/src/AudioAnalysis/extractAudioAnalysis.ts
+/**
+ * This module provides functions for extracting and analyzing audio data.
+ * - `extractAudioAnalysis`: For detailed analysis with customizable ranges and decoding options.
+ * - `extractWavAudioAnalysis`: For analyzing WAV files without decoding, preserving original PCM values.
+ * - `extractPreview`: For generating quick previews of audio waveforms, optimized for UI rendering.
+ */
+import crc32 from 'crc-32'
+
+import { ConsoleLike } from '../ExpoAudioStream.types'
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+import { isWeb } from '../constants'
+import {
+    AudioAnalysis,
+    AudioFeaturesOptions,
+    DataPoint,
+    DecodingConfig,
+} from './AudioAnalysis.types'
+import { processAudioBuffer } from '../utils/audioProcessing'
+import { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'
+import { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'
+import { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'
+
+function calculateCRC32ForDataPoint(data: Float32Array): number {
+    // Convert float array to byte array for CRC32
+    const byteArray = new Uint8Array(data.length * 4)
+    const dataView = new DataView(byteArray.buffer)
+
+    for (let i = 0; i < data.length; i++) {
+        dataView.setFloat32(i * 4, data[i], true)
+    }
+
+    return crc32.buf(byteArray)
+}
+
+export interface ExtractWavAudioAnalysisProps {
+    fileUri?: string // should provide either fileUri or arrayBuffer
+    wavMetadata?: WavFileInfo
+    arrayBuffer?: ArrayBuffer
+    bitDepth?: number
+    durationMs?: number
+    sampleRate?: number
+    numberOfChannels?: number
+    position?: number // Optional number of bytes to skip. Default is 0
+    length?: number // Optional number of bytes to read.
+    segmentDurationMs?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.
+    features?: AudioFeaturesOptions
+    featuresExtratorUrl?: string
+    logger?: ConsoleLike
+    decodingOptions?: DecodingConfig
+}
+
+// Define base options interface with common properties
+interface BaseExtractOptions {
+    fileUri?: string
+    arrayBuffer?: ArrayBuffer
+    /**
+     * Duration of each analysis segment in milliseconds. Defaults to 100ms if not specified.
+     */
+    segmentDurationMs?: number
+    features?: AudioFeaturesOptions
+    decodingOptions?: DecodingConfig
+    logger?: ConsoleLike
+}
+
+// Time-based range options
+interface TimeRangeOptions extends BaseExtractOptions {
+    startTimeMs?: number
+    endTimeMs?: number
+    position?: never
+    length?: never
+}
+
+// Byte-based range options
+interface ByteRangeOptions extends BaseExtractOptions {
+    position?: number
+    length?: number
+    startTimeMs?: never
+    endTimeMs?: never
+}
+
+/**
+ * Options for extracting audio analysis.
+ * - For time-based analysis, provide `startTimeMs` and `endTimeMs`.
+ * - For byte-based analysis, provide `position` and `length`.
+ * - Do not mix time and byte ranges.
+ */
+export type ExtractAudioAnalysisProps = TimeRangeOptions | ByteRangeOptions
+
+/**
+ * Extracts detailed audio analysis from the specified audio file or buffer.
+ * Supports either time-based or byte-based ranges for flexibility in analysis.
+ *
+ * @param props - The options for extraction, including file URI, ranges, and decoding settings.
+ * @returns A promise that resolves to the audio analysis data.
+ * @throws {Error} If both time and byte ranges are provided or if required parameters are missing.
+ */
+export async function extractAudioAnalysis(
+    props: ExtractAudioAnalysisProps
+): Promise<AudioAnalysis> {
+    const {
+        fileUri,
+        arrayBuffer,
+        decodingOptions,
+        logger,
+        segmentDurationMs = 100,
+        features,
+    } = props
+
+    if (isWeb) {
+        try {
+            // Create AudioContext here
+            const audioContext = new (window.AudioContext ||
+                (window as any).webkitAudioContext)({
+                sampleRate: decodingOptions?.targetSampleRate ?? 16000,
+            })
+
+            try {
+                const processedBuffer = await processAudioBuffer({
+                    arrayBuffer,
+                    fileUri,
+                    targetSampleRate:
+                        decodingOptions?.targetSampleRate ?? 16000,
+                    targetChannels: decodingOptions?.targetChannels ?? 1,
+                    normalizeAudio: decodingOptions?.normalizeAudio ?? false,
+                    startTimeMs:
+                        'startTimeMs' in props ? props.startTimeMs : undefined,
+                    endTimeMs:
+                        'endTimeMs' in props ? props.endTimeMs : undefined,
+                    position: 'position' in props ? props.position : undefined,
+                    length: 'length' in props ? props.length : undefined,
+                    audioContext, // Pass the context we created
+                    logger,
+                })
+
+                const channelData = processedBuffer.buffer.getChannelData(0)
+
+                // Create and initialize the worker
+                const blob = new Blob([InlineFeaturesExtractor], {
+                    type: 'application/javascript',
+                })
+                const workerUrl = URL.createObjectURL(blob)
+                const worker = new Worker(workerUrl)
+
+                return new Promise((resolve, reject) => {
+                    worker.onmessage = (event) => {
+                        if (event.data.error) {
+                            reject(new Error(event.data.error))
+                            return
+                        }
+
+                        const result: AudioAnalysis = event.data.result
+                        // Calculate CRC32 after worker completes if requested
+                        if (features?.crc32) {
+                            const samplesPerSegment = Math.floor(
+                                (processedBuffer.sampleRate *
+                                    segmentDurationMs) /
+                                    1000
+                            )
+
+                            result.dataPoints = result.dataPoints.map(
+                                (point: DataPoint, index: number) => {
+                                    const startSample =
+                                        index * samplesPerSegment
+                                    const segmentData = channelData.slice(
+                                        startSample,
+                                        startSample + samplesPerSegment
+                                    )
+
+                                    return {
+                                        ...point,
+                                        features: {
+                                            ...point.features,
+                                            crc32: calculateCRC32ForDataPoint(
+                                                segmentData
+                                            ),
+                                        },
+                                    }
+                                }
+                            )
+                        }
+
+                        URL.revokeObjectURL(workerUrl)
+                        worker.terminate()
+                        resolve(result)
+                    }
+
+                    worker.onerror = (error) => {
+                        URL.revokeObjectURL(workerUrl)
+                        worker.terminate()
+                        reject(error)
+                    }
+
+                    worker.postMessage({
+                        channelData,
+                        sampleRate: processedBuffer.sampleRate,
+                        segmentDurationMs,
+                        bitDepth: decodingOptions?.targetBitDepth ?? 32,
+                        numberOfChannels: processedBuffer.channels,
+                        // enableLogging: !!logger,
+                        features,
+                    })
+                })
+            } finally {
+                await audioContext.close()
+            }
+        } catch (error) {
+            logger?.error('Failed to process audio:', error)
+            throw error
+        }
+    } else {
+        return await ExpoAudioStreamModule.extractAudioAnalysis(props)
+    }
+}
+
+/**
+ * Analyzes WAV files without decoding, preserving original PCM values.
+ * Use this function when you need to ensure the analysis matches other software by avoiding any transformations.
+ *
+ * @param props - The options for WAV analysis, including file URI and range.
+ * @returns A promise that resolves to the audio analysis data.
+ */
+export const extractRawWavAnalysis = async ({
+    fileUri,
+    segmentDurationMs = 100, // Default to 100ms
+    arrayBuffer,
+    bitDepth,
+    durationMs,
+    sampleRate,
+    numberOfChannels,
+    features,
+    logger,
+    position = 0,
+    length,
+}: ExtractWavAudioAnalysisProps): Promise<AudioAnalysis> => {
+    if (isWeb) {
+        if (!arrayBuffer && !fileUri) {
+            throw new Error('Either arrayBuffer or fileUri must be provided')
+        }
+
+        if (!arrayBuffer) {
+            logger?.log(`fetching fileUri`, fileUri)
+            const response = await fetch(fileUri!)
+
+            if (!response.ok) {
+                throw new Error(
+                    `Failed to fetch fileUri: ${response.statusText}`
+                )
+            }
+
+            arrayBuffer = await response.arrayBuffer()
+            logger?.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)
+        }
+
+        // Create a new copy of the ArrayBuffer to avoid detachment issues
+        const bufferCopy = arrayBuffer.slice(0)
+        logger?.log(
+            `extractAudioAnalysis bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
+            bufferCopy.slice(0, 100)
+        )
+
+        let actualBitDepth = bitDepth
+        if (!actualBitDepth) {
+            logger?.log(
+                `extractAudioAnalysis bitDepth not provided -- getting wav file info`
+            )
+            const fileInfo = await getWavFileInfo(bufferCopy)
+            actualBitDepth = fileInfo.bitDepth
+        }
+        logger?.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)
+
+        const {
+            pcmValues: channelData,
+            min,
+            max,
+        } = await convertPCMToFloat32({
+            buffer: arrayBuffer,
+            bitDepth: actualBitDepth,
+        })
+        logger?.log(
+            `extractAudioAnalysis convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`
+        )
+
+        // Apply position and length constraints to channelData if specified
+        const startIndex = position
+        const endIndex = length ? startIndex + length : channelData.length
+        const constrainedChannelData = channelData.slice(startIndex, endIndex)
+
+        return new Promise((resolve, reject) => {
+            const blob = new Blob([InlineFeaturesExtractor], {
+                type: 'application/javascript',
+            })
+            const url = URL.createObjectURL(blob)
+            const worker = new Worker(url)
+
+            worker.onmessage = (event) => {
+                resolve(event.data.result)
+            }
+
+            worker.onerror = (error) => {
+                reject(error)
+            }
+
+            worker.postMessage({
+                command: 'process',
+                channelData: constrainedChannelData,
+                sampleRate,
+                segmentDurationMs,
+                logger,
+                bitDepth,
+                fullAudioDurationMs: durationMs,
+                numberOfChannels,
+            })
+        })
+    } else {
+        if (!fileUri) {
+            throw new Error('fileUri is required')
+        }
+        logger?.log(`extractAudioAnalysis`, {
+            fileUri,
+            segmentDurationMs,
+        })
+        const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+            fileUri,
+            segmentDurationMs,
+            features,
+            position,
+            length,
+        })
+        logger?.log(`extractAudioAnalysis`, res)
+        return res
+    }
+}

package/src/AudioAnalysis/extractAudioData.ts

@@ -0,0 +1,6 @@
+import { ExtractAudioDataOptions } from '../ExpoAudioStream.types'
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+
+export const extractAudioData = async (props: ExtractAudioDataOptions) => {
+    return await ExpoAudioStreamModule.extractAudioData(props)
+}

package/src/AudioAnalysis/extractMelSpectrogram.ts

@@ -0,0 +1,144 @@
+/**
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions. The web implementation is a placeholder.
+ */
+
+import { ExpoAudioStreamModule } from '..'
+import { isWeb } from '../constants'
+import {
+    ExtractMelSpectrogramOptions,
+    MelSpectrogram,
+} from './AudioAnalysis.types'
+import {
+    processAudioBuffer,
+    ProcessedAudioData,
+} from '../utils/audioProcessing'
+
+/**
+ * Extracts a mel spectrogram from audio data
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The iOS implementation will throw an "UNSUPPORTED_PLATFORM" error.
+ * The web implementation is a placeholder that returns dummy data.
+ */
+export async function extractMelSpectrogram(
+    options: ExtractMelSpectrogramOptions
+): Promise<MelSpectrogram> {
+    const {
+        fileUri,
+        arrayBuffer,
+        windowSizeMs,
+        hopLengthMs,
+        nMels,
+        fMin = 0,
+        fMax,
+        windowType = 'hann',
+        normalize = false,
+        logScale = true,
+        decodingOptions,
+        startTimeMs,
+        endTimeMs,
+        logger,
+    } = options
+
+    if (isWeb) {
+        // Create audio context
+        const audioContext = new (window.AudioContext ||
+            (window as any).webkitAudioContext)()
+
+        try {
+            // Process audio data using the existing utility
+            const processedAudio: ProcessedAudioData = await processAudioBuffer(
+                {
+                    arrayBuffer,
+                    fileUri,
+                    targetSampleRate:
+                        decodingOptions?.targetSampleRate || 16000,
+                    targetChannels: decodingOptions?.targetChannels || 1,
+                    normalizeAudio: decodingOptions?.normalizeAudio ?? false,
+                    startTimeMs,
+                    endTimeMs,
+                    audioContext,
+                    logger: options.logger,
+                }
+            )
+
+            // Calculate window and hop size in samples
+            const sampleRate = processedAudio.sampleRate
+            const windowSize = Math.floor((windowSizeMs * sampleRate) / 1000)
+            const hopLength = Math.floor((hopLengthMs * sampleRate) / 1000)
+            const maxFreq = fMax || sampleRate / 2
+
+            // Extract the mel spectrogram from the processed audio
+            const spectrogram = computeMelSpectrogram(
+                processedAudio.channelData,
+                sampleRate,
+                nMels,
+                windowSize,
+                hopLength,
+                fMin,
+                maxFreq,
+                windowType,
+                normalize,
+                logScale
+            )
+
+            const timeSteps = spectrogram.length
+
+            return {
+                spectrogram,
+                sampleRate,
+                nMels,
+                timeSteps,
+                durationMs: processedAudio.durationMs,
+            }
+        } catch (error) {
+            logger?.error('Error extracting mel spectrogram:', error)
+            throw error
+        } finally {
+            // Close the audio context
+            await audioContext.close()
+        }
+    }
+    return ExpoAudioStreamModule.extractMelSpectrogram(options)
+}
+
+/**
+ * Computes a mel spectrogram from audio data
+ * 
+ * @experimental This is a placeholder implementation that returns dummy data.
+ * The actual implementation will be added in a future version.
+ */
+function computeMelSpectrogram(
+    audioData: Float32Array,
+    sampleRate: number,
+    nMels: number,
+    windowSize: number,
+    hopLength: number,
+    fMin: number,
+    fMax: number,
+    windowType: 'hann' | 'hamming',
+    normalize: boolean,
+    logScale: boolean
+): number[][] {
+    // Placeholder for the actual implementation
+    // This would include:
+    // 1. Windowing the audio data using the specified window type
+    // 2. Computing the STFT (Short-Time Fourier Transform)
+    // 3. Converting to power spectrogram
+    // 4. Applying mel filterbanks
+    // 5. Taking the logarithm if logScale is true
+    // 6. Normalizing if normalize is true
+
+    // For now, return a dummy implementation
+    const numFrames =
+        Math.floor((audioData.length - windowSize) / hopLength) + 1
+    const spectrogram: number[][] = []
+
+    // Create dummy mel spectrogram data
+    for (let i = 0; i < numFrames; i++) {
+        spectrogram.push(Array(nMels).fill(0))
+    }
+
+    return spectrogram
+}