@siteed/audio-studio 3.0.0

package/src/AudioAnalysis/AudioAnalysis.types.ts

@@ -0,0 +1,226 @@
+// packages/audio-studio/src/AudioAnalysis/AudioAnalysis.types.ts
+
+import { BitDepth, ConsoleLike } from '../AudioStudio.types'
+
+/**
+ * Represents the configuration for decoding audio data.
+ */
+export interface DecodingConfig {
+    /** Target sample rate for decoded audio (Android and Web) */
+    targetSampleRate?: number
+    /** Target number of channels (Android and Web) */
+    targetChannels?: number
+    /** Target bit depth (Android and Web) */
+    targetBitDepth?: BitDepth
+    /** Whether to normalize audio levels (Android and Web) */
+    normalizeAudio?: boolean
+}
+
+/**
+ * Represents speech-related features extracted from audio.
+ */
+export interface SpeechFeatures {
+    isActive: boolean // Whether speech is detected in this segment
+    speakerId?: number // Optional speaker identification
+    // Could add more speech-related features here like:
+    // confidence: number
+    // language?: string
+    // sentiment?: number
+    // etc.
+}
+
+/**
+ * Represents various audio features extracted from an audio signal.
+ */
+export interface AudioFeatures {
+    energy?: number // The infinite integral of the squared signal, representing the overall energy of the audio.
+    mfcc?: number[] // Mel-frequency cepstral coefficients, describing the short-term power spectrum of a sound.
+    rms?: number // Root mean square value, indicating the amplitude of the audio signal.
+    minAmplitude?: number // Minimum amplitude value in the audio signal.
+    maxAmplitude?: number // Maximum amplitude value in the audio signal.
+    zcr?: number // Zero-crossing rate, indicating the rate at which the signal changes sign.
+    spectralCentroid?: number // The center of mass of the spectrum, indicating the brightness of the sound.
+    spectralFlatness?: number // Measure of the flatness of the spectrum, indicating how noise-like the signal is.
+    spectralRolloff?: number // The frequency below which a specified percentage (usually 85%) of the total spectral energy lies.
+    spectralBandwidth?: number // The width of the spectrum, indicating the range of frequencies present.
+    chromagram?: number[] // Chromagram, representing the 12 different pitch classes of the audio.
+    tempo?: number // Estimated tempo of the audio signal, measured in beats per minute (BPM).
+    hnr?: number // Harmonics-to-noise ratio, indicating the proportion of harmonics to noise in the audio signal.
+    melSpectrogram?: number[] // Mel-scaled spectrogram representation of the audio.
+    spectralContrast?: number[] // Spectral contrast features representing the difference between peaks and valleys.
+    tonnetz?: number[] // Tonal network features representing harmonic relationships.
+    pitch?: number // Pitch of the audio signal, measured in Hertz (Hz).
+    crc32?: number // crc32 checksum of the audio signal, used to verify the integrity of the audio.
+}
+
+/**
+ * Options to specify which audio features to extract.
+ * Note: Advanced features (spectral features, chromagram, pitch, etc.) are experimental,
+ * especially during live recording, due to high processing requirements.
+ */
+export interface AudioFeaturesOptions {
+    // Basic features - well optimized
+    energy?: boolean
+    rms?: boolean
+    zcr?: boolean
+
+    // Advanced features - experimental, may impact performance in live recording
+    mfcc?: boolean
+    spectralCentroid?: boolean
+    spectralFlatness?: boolean
+    spectralRolloff?: boolean
+    spectralBandwidth?: boolean
+    chromagram?: boolean
+    tempo?: boolean
+    hnr?: boolean
+    melSpectrogram?: boolean
+    spectralContrast?: boolean
+    tonnetz?: boolean
+    pitch?: boolean
+
+    // Utility
+    crc32?: boolean
+}
+
+/**
+ * Represents a single data point in the audio analysis.
+ */
+export interface DataPoint {
+    id: number
+    amplitude: number // Peak amplitude for the segment
+    rms: number // Root mean square value
+    dB: number // dBFS (decibels relative to full scale) computed from RMS value
+    silent: boolean // Always computed
+    features?: AudioFeatures
+    speech?: SpeechFeatures
+    startTime?: number
+    endTime?: number
+    // start / end position in bytes
+    startPosition?: number
+    endPosition?: number
+    // number of audio samples for this point (samples size depends on bit depth)
+    samples?: number
+}
+
+/**
+ * Represents the complete data from the audio analysis.
+ */
+export interface AudioAnalysis {
+    segmentDurationMs: number // Duration of each segment in milliseconds
+    durationMs: number // Duration of the audio in milliseconds
+    /**
+     * Bit depth used for audio analysis processing.
+     *
+     * **Important**: This represents the internal processing bit depth, which may differ
+     * from the recording bit depth. Audio is typically converted to 32-bit float for
+     * analysis to ensure precision in calculations, regardless of the original recording format.
+     *
+     * Platform behavior:
+     * - iOS: Always 32 (float processing)
+     * - Android: Always 32 (float processing)
+     * - Web: Always 32 (Web Audio API standard)
+     *
+     * The actual recorded file will maintain the requested bit depth (8, 16, or 32).
+     */
+    bitDepth: number
+    samples: number // Size of the audio in bytes
+    numberOfChannels: number // Number of audio channels
+    sampleRate: number // Sample rate of the audio
+    dataPoints: DataPoint[] // Array of data points from the analysis.
+    amplitudeRange: {
+        min: number
+        max: number
+    }
+    rmsRange: {
+        min: number
+        max: number
+    }
+    extractionTimeMs: number // Time taken to extract/process the analysis in milliseconds
+    // TODO: speaker changes into a broader speech analysis section
+    speechAnalysis?: {
+        speakerChanges: {
+            timestamp: number
+            speakerId: number
+        }[]
+        // Could add more speech analysis data here like:
+        // dominantSpeaker?: number
+        // totalSpeechDuration?: number
+        // speakerStats?: { [speakerId: number]: { duration: number, segments: number } }
+    }
+}
+
+/**
+ * Options for specifying a time range within an audio file.
+ */
+export interface AudioRangeOptions {
+    /** Start time in milliseconds */
+    startTimeMs?: number
+    /** End time in milliseconds */
+    endTimeMs?: number
+}
+
+/**
+ * Options for generating a quick preview of audio waveform.
+ * This is optimized for UI rendering with a specified number of points.
+ */
+export interface PreviewOptions extends AudioRangeOptions {
+    /** URI of the audio file to analyze */
+    fileUri: string
+    /**
+     * Total number of points to generate for the preview.
+     * @default 100
+     */
+    numberOfPoints?: number
+    /**
+     * Optional logger for debugging.
+     */
+    logger?: ConsoleLike
+    /**
+     * Optional configuration for decoding the audio file.
+     * Defaults to:
+     * - targetSampleRate: undefined (keep original)
+     * - targetChannels: undefined (keep original)
+     * - targetBitDepth: 16
+     * - normalizeAudio: false
+     */
+    decodingOptions?: DecodingConfig
+}
+
+/**
+ * Options for mel-spectrogram extraction
+ *
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface ExtractMelSpectrogramOptions {
+    fileUri?: string // Path to audio file
+    arrayBuffer?: ArrayBuffer // Raw audio buffer
+    windowSizeMs: number // Window size in ms (e.g., 25)
+    hopLengthMs: number // Hop length in ms (e.g., 10)
+    nMels: number // Number of mel filters (e.g., 60)
+    fMin?: number // Min frequency (default: 0)
+    fMax?: number // Max frequency (default: sampleRate / 2)
+    windowType?: 'hann' | 'hamming' // Window function (default: 'hann')
+    normalize?: boolean // Mean normalization (default: false)
+    logScale?: boolean // Log scaling of mel energies (default: true)
+    decodingOptions?: DecodingConfig // Audio decoding settings
+    /** Optional start time in ms. If neither startTimeMs nor endTimeMs is set, defaults to 0. */
+    startTimeMs?: number
+    /** Optional end time in ms. Clamped so that the range does not exceed MAX_DURATION_MS (30 s). */
+    endTimeMs?: number
+    logger?: ConsoleLike
+}
+
+/**
+ * Return type for mel spectrogram extraction
+ *
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface MelSpectrogram {
+    spectrogram: number[][] // 2D array [time][mel]
+    sampleRate: number // Audio sample rate
+    nMels: number // Number of mel filters
+    timeSteps: number // Number of time frames
+    durationMs: number // Audio duration in ms
+}

package/src/AudioAnalysis/audio-features-wasm.d.ts

@@ -0,0 +1,37 @@
+/** Type declarations for audio features functions in the mel-spectrogram WASM module */
+
+import type { MelSpectrogramWasmModule } from './mel-spectrogram-wasm'
+
+export interface AudioFeaturesWasmModule extends MelSpectrogramWasmModule {
+    _audio_features_compute(
+        samples: number,
+        numSamples: number,
+        sampleRate: number,
+        fftLength: number,
+        nMfcc: number,
+        nMelFilters: number,
+        computeMfcc: number,
+        computeChroma: number
+    ): number
+
+    _audio_features_free(resultPtr: number): void
+
+    _audio_features_init(
+        sampleRate: number,
+        fftLength: number,
+        nMfcc: number,
+        nMelFilters: number,
+        computeMfcc: number,
+        computeChroma: number
+    ): void
+
+    _audio_features_compute_frame(
+        samples: number,
+        numSamples: number,
+        resultPtr: number
+    ): number
+
+    _audio_features_free_arrays(resultPtr: number): void
+
+    _audio_features_get_n_mfcc(): number
+}

package/src/AudioAnalysis/audioFeaturesWasm.ts

@@ -0,0 +1,200 @@
+import type { AudioFeaturesWasmModule } from './audio-features-wasm'
+
+export interface AudioFeaturesWasmResult {
+    spectralCentroid: number
+    spectralFlatness: number
+    spectralRolloff: number
+    spectralBandwidth: number
+    mfcc: number[]
+    chromagram: number[]
+}
+
+let modulePromise: Promise<AudioFeaturesWasmModule> | null = null
+
+function getModule(): Promise<AudioFeaturesWasmModule> {
+    if (!modulePromise) {
+        modulePromise = (async () => {
+            // Same WASM module as mel spectrogram (now includes audio features)
+            // @ts-expect-error -- prebuilt Emscripten JS glue has no .d.ts
+            const mod = await import('../../prebuilt/wasm/mel-spectrogram.js')
+            const factory = mod.default ?? mod
+            return factory() as Promise<AudioFeaturesWasmModule>
+        })().catch((err) => {
+            modulePromise = null
+            throw err
+        })
+    }
+    return modulePromise
+}
+
+// --- Struct layout for CAudioFeaturesResult (wasm32) ---
+// Offset  0: float spectralCentroid  (4 bytes)
+// Offset  4: float spectralFlatness  (4 bytes)
+// Offset  8: float spectralRolloff   (4 bytes)
+// Offset 12: float spectralBandwidth (4 bytes)
+// Offset 16: float* mfcc             (4 bytes pointer)
+// Offset 20: int mfccCount           (4 bytes)
+// Offset 24: float* chromagram       (4 bytes pointer)
+// Offset 28: int chromagramCount     (4 bytes)
+const STRUCT_SIZE = 32
+
+function readResult(
+    Module: AudioFeaturesWasmModule,
+    ptr: number
+): AudioFeaturesWasmResult {
+    const spectralCentroid = Module.getValue(ptr, 'float')
+    const spectralFlatness = Module.getValue(ptr + 4, 'float')
+    const spectralRolloff = Module.getValue(ptr + 8, 'float')
+    const spectralBandwidth = Module.getValue(ptr + 12, 'float')
+
+    const mfccPtr = Module.getValue(ptr + 16, 'i32')
+    const mfccCount = Module.getValue(ptr + 20, 'i32')
+    const chromaPtr = Module.getValue(ptr + 24, 'i32')
+    const chromaCount = Module.getValue(ptr + 28, 'i32')
+
+    const mfcc: number[] = []
+    if (mfccPtr && mfccCount > 0) {
+        const offset = mfccPtr >> 2
+        for (let i = 0; i < mfccCount; i++) {
+            mfcc.push(Module.HEAPF32[offset + i])
+        }
+    }
+
+    const chromagram: number[] = []
+    if (chromaPtr && chromaCount > 0) {
+        const offset = chromaPtr >> 2
+        for (let i = 0; i < chromaCount; i++) {
+            chromagram.push(Module.HEAPF32[offset + i])
+        }
+    }
+
+    return {
+        spectralCentroid,
+        spectralFlatness,
+        spectralRolloff,
+        spectralBandwidth,
+        mfcc,
+        chromagram,
+    }
+}
+
+// --- Streaming (per-frame) API ---
+
+let streamingModule: AudioFeaturesWasmModule | null = null
+let streamingFramePtr = 0
+let streamingFrameCapacity = 0
+let streamingResultPtr = 0
+
+/**
+ * Initialise the WASM streaming audio features processor.
+ * Call once before computeAudioFeaturesFrameWasm().
+ */
+export async function initAudioFeaturesWasm(
+    sampleRate: number,
+    fftLength = 1024,
+    nMfcc = 13,
+    nMelFilters = 26,
+    computeMfcc = true,
+    computeChroma = true
+): Promise<void> {
+    const Module = await getModule()
+    streamingModule = Module
+
+    Module._audio_features_init(
+        sampleRate,
+        fftLength,
+        nMfcc,
+        nMelFilters,
+        computeMfcc ? 1 : 0,
+        computeChroma ? 1 : 0
+    )
+
+    // Pre-allocate result struct on WASM heap
+    if (streamingResultPtr) Module._free(streamingResultPtr)
+    streamingResultPtr = Module._malloc(STRUCT_SIZE)
+    // Zero-initialize to prevent freeing garbage pointers on first use
+    Module.HEAPU8.fill(0, streamingResultPtr, streamingResultPtr + STRUCT_SIZE)
+
+    // Frame input buffer allocated on demand
+    streamingFrameCapacity = 0
+    streamingFramePtr = 0
+}
+
+/**
+ * Compute audio features for a single frame via WASM C++.
+ * Returns null if not initialised or on error.
+ */
+export function computeAudioFeaturesFrameWasm(
+    samples: Float32Array
+): AudioFeaturesWasmResult | null {
+    if (!streamingModule || !streamingResultPtr) return null
+    const Module = streamingModule
+
+    // (Re-)allocate frame input buffer if needed
+    if (samples.length > streamingFrameCapacity) {
+        if (streamingFramePtr) Module._free(streamingFramePtr)
+        streamingFramePtr = Module._malloc(samples.length * 4)
+        streamingFrameCapacity = samples.length
+    }
+
+    // Copy samples to WASM heap
+    Module.HEAPF32.set(samples, streamingFramePtr >> 2)
+
+    const ok = Module._audio_features_compute_frame(
+        streamingFramePtr,
+        samples.length,
+        streamingResultPtr
+    )
+    if (!ok) return null
+
+    const result = readResult(Module, streamingResultPtr)
+
+    // Free internal arrays (mfcc, chromagram) allocated by C
+    Module._audio_features_free_arrays(streamingResultPtr)
+
+    return result
+}
+
+// --- Batch API ---
+
+/**
+ * Compute audio features for a buffer of samples via WASM C++.
+ * Lazy-loads the WASM module on first call.
+ */
+export async function computeAudioFeaturesWasm(
+    audioData: Float32Array,
+    sampleRate: number,
+    fftLength = 1024,
+    nMfcc = 13,
+    nMelFilters = 26,
+    computeMfcc = true,
+    computeChroma = true
+): Promise<AudioFeaturesWasmResult> {
+    const Module = await getModule()
+
+    const numSamples = audioData.length
+    const inputPtr = Module._malloc(numSamples * 4)
+    Module.HEAPF32.set(audioData, inputPtr >> 2)
+
+    const resultPtr = Module._audio_features_compute(
+        inputPtr,
+        numSamples,
+        sampleRate,
+        fftLength,
+        nMfcc,
+        nMelFilters,
+        computeMfcc ? 1 : 0,
+        computeChroma ? 1 : 0
+    )
+
+    Module._free(inputPtr)
+
+    if (resultPtr === 0) {
+        throw new Error('audio_features_compute returned null')
+    }
+
+    const result = readResult(Module, resultPtr)
+    Module._audio_features_free(resultPtr)
+
+    return result
+}