@siteed/expo-audio-stream 1.1.5 → 1.1.7

package/src/utils/getWavFileInfo.ts

@@ -0,0 +1,132 @@
+// packages/expo-audio-stream/src/utils/getWavFileInfo.ts
+
+import { BitDepth, SampleRate } from '../ExpoAudioStream.types'
+import {
+    DATA_CHUNK_ID,
+    DEFAULT_BIT_DEPTH,
+    DEFAULT_SAMPLE_RATE,
+    FMT_CHUNK_ID,
+    INFO_CHUNK_ID,
+    RIFF_HEADER,
+    WAVE_HEADER,
+} from '../constants'
+
+// Audio format descriptions
+const AUDIO_FORMATS: { [key: number]: string } = {
+    1: 'PCM',
+    3: 'IEEE float',
+    6: '8-bit ITU-T G.711 A-law',
+    7: '8-bit ITU-T G.711 µ-law',
+    65534: 'WAVE_FORMAT_EXTENSIBLE',
+}
+
+/**
+ * Interface representing the metadata of a WAV file.
+ */
+export interface WavFileInfo {
+    sampleRate: SampleRate
+    numChannels: number
+    bitDepth: BitDepth
+    size: number // in bytes
+    durationMs: number // in ms
+    audioFormatDescription: string // Description of the audio format
+    byteRate: number // Average bytes per second
+    blockAlign: number // Number of bytes for one sample including all channels
+    creationDateTime?: string // Optional creation date and time
+    comments?: string // Optional comments or tags
+    compressionType?: string // Optional compression type
+    dataChunkOffset: number // Position of the first data chunk
+}
+
+/**
+ * Extracts metadata from a WAV buffer.
+ *
+ * @param arrayBuffer - The array buffer containing the WAV data.
+ * @returns A promise that resolves to the extracted metadata.
+ */
+export const getWavFileInfo = async (
+    arrayBuffer: ArrayBuffer
+): Promise<WavFileInfo> => {
+    const view = new DataView(arrayBuffer)
+
+    // Check if the file is a valid RIFF/WAVE file
+    const riffHeader = view.getUint32(0, false)
+    const waveHeader = view.getUint32(8, false)
+    if (riffHeader !== RIFF_HEADER || waveHeader !== WAVE_HEADER) {
+        throw new Error('Invalid WAV file')
+    }
+
+    // Initialize variables for the metadata
+    let fmtChunkOffset = 12
+    let sampleRate: SampleRate = DEFAULT_SAMPLE_RATE
+    let numChannels = 0
+    let bitDepth: BitDepth = DEFAULT_BIT_DEPTH
+    let dataChunkSize = 0
+    let audioFormat = 0
+    let byteRate = 0
+    let blockAlign = 0
+    let creationDateTime = ''
+    let comments = ''
+    let dataChunkOffset = 0
+
+    // Parse chunks to find the "fmt " and "data" chunks
+    while (fmtChunkOffset < view.byteLength) {
+        const chunkId = view.getUint32(fmtChunkOffset, false)
+        const chunkSize = view.getUint32(fmtChunkOffset + 4, true)
+        if (chunkId === FMT_CHUNK_ID) {
+            // "fmt "
+            audioFormat = view.getUint16(fmtChunkOffset + 8, true)
+            if (!AUDIO_FORMATS[audioFormat]) {
+                throw new Error('Unsupported WAV file format')
+            }
+            numChannels = view.getUint16(fmtChunkOffset + 10, true)
+            sampleRate = view.getUint32(fmtChunkOffset + 12, true) as SampleRate
+            byteRate = view.getUint32(fmtChunkOffset + 16, true)
+            blockAlign = view.getUint16(fmtChunkOffset + 20, true)
+            bitDepth = view.getUint16(fmtChunkOffset + 22, true) as BitDepth
+        } else if (chunkId === DATA_CHUNK_ID) {
+            // "data"
+            dataChunkSize = chunkSize
+            dataChunkOffset = fmtChunkOffset + 8 // Position after chunk header
+            break
+        } else if (chunkId === INFO_CHUNK_ID) {
+            // "INFO"
+            // Read INFO chunk (assuming it contains a text-based creation date/time and comments)
+            const infoStart = fmtChunkOffset + 8
+            const infoText = new TextDecoder().decode(
+                new Uint8Array(
+                    arrayBuffer.slice(infoStart, infoStart + chunkSize)
+                )
+            )
+            const infoParts = infoText.split('\0')
+            creationDateTime = infoParts[0]
+            comments = infoParts[1]
+        }
+        fmtChunkOffset += 8 + chunkSize
+    }
+
+    if (!sampleRate || !numChannels || !bitDepth || !dataChunkSize) {
+        throw new Error('Incomplete WAV file information')
+    }
+
+    // Calculate duration
+    const bytesPerSample = bitDepth / 8
+    const numSamples = dataChunkSize / (numChannels * bytesPerSample)
+    const durationMs = (numSamples / sampleRate) * 1000
+
+    return {
+        sampleRate,
+        numChannels,
+        bitDepth,
+        size: arrayBuffer.byteLength,
+        durationMs,
+        audioFormatDescription: AUDIO_FORMATS[audioFormat],
+        byteRate,
+        blockAlign,
+        creationDateTime: creationDateTime || undefined,
+        comments: comments || undefined,
+        compressionType:
+            audioFormat === 1 ? 'None' : AUDIO_FORMATS[audioFormat],
+        dataChunkOffset,
+    }
+}

package/src/utils/writeWavHeader.ts

@@ -0,0 +1,114 @@
+// packages/expo-audio-stream/src/utils/writeWavHeader.ts
+
+/**
+ * Options for creating a WAV header.
+ */
+export interface WavHeaderOptions {
+    /** Optional buffer containing audio data. If provided, it will be combined with the header. */
+    buffer?: ArrayBuffer
+    /** The sample rate of the audio in Hz (e.g., 44100). */
+    sampleRate: number
+    /** The number of audio channels (e.g., 1 for mono, 2 for stereo). */
+    numChannels: number
+    /** The bit depth of the audio (e.g., 16, 24, or 32). */
+    bitDepth: number
+}
+
+/**
+ * Writes or updates a WAV (RIFF) header based on the provided options.
+ *
+ * This function can be used in three ways:
+ * 1. To create a standalone WAV header (when no buffer is provided).
+ * 2. To create a WAV header and combine it with existing audio data (when a buffer without a header is provided).
+ * 3. To update an existing WAV header in the provided buffer.
+ *
+ * For streaming audio where the final size is unknown, this function sets the size fields
+ * to the maximum 32-bit value (0xFFFFFFFF). These can be updated later using the
+ * `updateWavHeaderSize` function once the final size is known.
+ *
+ * @param options - The options for creating or updating the WAV header.
+ * @returns An ArrayBuffer containing the WAV header, or the header combined with the provided audio data.
+ *
+ * @throws {Error} Throws an error if the provided options are invalid or if the buffer is too small.
+ *
+ * @example
+ * // Create a standalone WAV header
+ * const header = writeWavHeader({
+ *   sampleRate: 44100,
+ *   numChannels: 2,
+ *   bitDepth: 16
+ * });
+ *
+ * @example
+ * // Create a WAV header and combine it with audio data
+ * const completeWav = writeWavHeader({
+ *   buffer: audioData,
+ *   sampleRate: 44100,
+ *   numChannels: 2,
+ *   bitDepth: 16
+ * });
+ */
+export const writeWavHeader = ({
+    buffer,
+    sampleRate,
+    numChannels,
+    bitDepth,
+}: WavHeaderOptions): ArrayBuffer => {
+    const bytesPerSample = bitDepth / 8
+    const blockAlign = numChannels * bytesPerSample
+    const byteRate = sampleRate * blockAlign
+
+    // Function to write a string to the DataView
+    const writeString = (view: DataView, offset: number, string: string) => {
+        for (let i = 0; i < string.length; i++) {
+            view.setUint8(offset + i, string.charCodeAt(i))
+        }
+    }
+
+    // Function to write or update the header
+    const writeHeader = (view: DataView, dataSize: number = 0xffffffff) => {
+        writeString(view, 0, 'RIFF') // ChunkID
+        view.setUint32(4, 36 + dataSize, true) // ChunkSize
+        writeString(view, 8, 'WAVE') // Format
+        writeString(view, 12, 'fmt ') // Subchunk1ID
+        view.setUint32(16, 16, true) // Subchunk1Size (16 for PCM)
+        view.setUint16(20, bitDepth === 32 ? 3 : 1, true) // AudioFormat (3 for float, 1 for PCM)
+        view.setUint16(22, numChannels, true) // NumChannels
+        view.setUint32(24, sampleRate, true) // SampleRate
+        view.setUint32(28, byteRate, true) // ByteRate
+        view.setUint16(32, blockAlign, true) // BlockAlign
+        view.setUint16(34, bitDepth, true) // BitsPerSample
+        writeString(view, 36, 'data') // Subchunk2ID
+        view.setUint32(40, dataSize, true) // Subchunk2Size
+    }
+
+    if (buffer) {
+        if (buffer.byteLength < 44) {
+            throw new Error('Buffer is too small to contain a valid WAV header')
+        }
+
+        const view = new DataView(buffer)
+
+        // Check if the buffer already has a WAV header by looking for "RIFF" at the start
+        const existingHeader = view.getUint32(0, false) === 0x52494646 // "RIFF" in ASCII
+
+        if (existingHeader) {
+            // Update the existing header
+            writeHeader(view, buffer.byteLength - 44)
+            return buffer
+        } else {
+            // Combine the new header with the existing buffer
+            const newBuffer = new ArrayBuffer(44 + buffer.byteLength)
+            const newView = new DataView(newBuffer)
+            writeHeader(newView, buffer.byteLength)
+            new Uint8Array(newBuffer).set(new Uint8Array(buffer), 44)
+            return newBuffer
+        }
+    } else {
+        // Create a standalone header
+        const headerBuffer = new ArrayBuffer(44)
+        const view = new DataView(headerBuffer)
+        writeHeader(view)
+        return headerBuffer
+    }
+}

package/src/workers/InlineFeaturesExtractor.web.tsx

@@ -0,0 +1,312 @@
+export const InlineFeaturesExtractor = `
+// Unique ID counter
+let uniqueIdCounter = 0
+
+self.onmessage = function (event) {
+    const {
+        channelData, // this is only the newly recorded data when live recording.
+        sampleRate,
+        pointsPerSecond,
+        algorithm,
+        bitDepth,
+        fullAudioDurationMs,
+        numberOfChannels,
+        features: _features,
+    } = event.data
+
+    console.log('[AudioFeaturesExtractor] Worker received message', event.data)
+    const features = _features || {}
+
+    const SILENCE_THRESHOLD = 0.01
+    const MIN_SILENCE_DURATION = 1.5 * sampleRate // 1.5 seconds of silence
+    const SPEECH_INERTIA_DURATION = 0.1 * sampleRate // Speech inertia duration in samples
+    const RMS_THRESHOLD = 0.01
+    const ZCR_THRESHOLD = 0.1
+
+    // Placeholder functions for feature extraction
+    const extractMFCC = (segmentData, sampleRate) => {
+        // Implement MFCC extraction logic here
+        return []
+    }
+
+    const extractSpectralCentroid = (segmentData, sampleRate) => {
+        const magnitudeSpectrum = segmentData.map((v) => v * v)
+        const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        if (sum === 0) return 0
+
+        const weightedSum = magnitudeSpectrum.reduce(
+            (acc, value, index) => acc + index * value,
+            0
+        )
+        return (
+            ((weightedSum / sum) * (sampleRate / 2)) / magnitudeSpectrum.length
+        )
+    }
+
+    const extractSpectralFlatness = (segmentData) => {
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const geometricMean = Math.exp(
+            magnitudeSpectrum
+                .map((v) => Math.log(v + Number.MIN_VALUE))
+                .reduce((a, b) => a + b) / magnitudeSpectrum.length
+        )
+        const arithmeticMean =
+            magnitudeSpectrum.reduce((a, b) => a + b) / magnitudeSpectrum.length
+        return arithmeticMean === 0 ? 0 : geometricMean / arithmeticMean
+    }
+
+    const extractSpectralRollOff = (segmentData, sampleRate) => {
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const totalEnergy = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        const rollOffThreshold = totalEnergy * 0.85
+        let cumulativeEnergy = 0
+
+        for (let i = 0; i < magnitudeSpectrum.length; i++) {
+            cumulativeEnergy += magnitudeSpectrum[i]
+            if (cumulativeEnergy >= rollOffThreshold) {
+                return (i / magnitudeSpectrum.length) * (sampleRate / 2)
+            }
+        }
+
+        return 0
+    }
+
+    const extractSpectralBandwidth = (segmentData, sampleRate) => {
+        const centroid = extractSpectralCentroid(segmentData, sampleRate)
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        if (sum === 0) return 0
+
+        const weightedSum = magnitudeSpectrum.reduce(
+            (acc, value, index) => acc + value * Math.pow(index - centroid, 2),
+            0
+        )
+        return Math.sqrt(weightedSum / sum)
+    }
+
+    const extractChromagram = (segmentData, sampleRate) => {
+        return [] // TODO implement
+    }
+
+    const extractHNR = (segmentData) => {
+        const frameSize = segmentData.length
+        const autocorrelation = new Float32Array(frameSize)
+
+        // Compute the autocorrelation of the segment data
+        for (let i = 0; i < frameSize; i++) {
+            let sum = 0
+            for (let j = 0; j < frameSize - i; j++) {
+                sum += segmentData[j] * segmentData[j + i]
+            }
+            autocorrelation[i] = sum
+        }
+
+        // Find the maximum autocorrelation value (excluding the zero lag)
+        const maxAutocorrelation = Math.max(...autocorrelation.subarray(1))
+
+        // Compute the HNR
+        return autocorrelation[0] !== 0
+            ? 10 *
+                  Math.log10(
+                      maxAutocorrelation /
+                          (autocorrelation[0] - maxAutocorrelation)
+                  )
+            : 0
+    }
+
+    const extractWaveform = (
+        channelData, // Float32Array
+        sampleRate, // number
+        pointsPerSecond, // number
+        algorithm // string
+    ) => {
+        const totalSamples = channelData.length
+        const segmentDuration = totalSamples / sampleRate
+        const totalPoints = Math.max(
+            Math.ceil(segmentDuration * pointsPerSecond),
+            1
+        )
+        const pointInterval = Math.ceil(totalSamples / totalPoints)
+        const dataPoints = []
+        let minAmplitude = Infinity
+        let maxAmplitude = -Infinity
+        let silenceStart = null
+        let lastSpeechEnd = -Infinity
+        let isSpeech = false
+
+        const expectedPoints = segmentDuration * pointsPerSecond
+        const samplesPerPoint = Math.ceil(channelData.length / expectedPoints)
+
+        for (let i = 0; i < expectedPoints; i++) {
+            const start = i * samplesPerPoint
+            const end = Math.min(start + samplesPerPoint, totalSamples)
+
+            let sumSquares = 0
+            let zeroCrossings = 0
+            let prevValue = channelData[start]
+            let localMinAmplitude = Infinity
+            let localMaxAmplitude = -Infinity
+            let hasNonZeroValue = false
+
+            // compute values for the segment
+            for (let j = start; j < end; j++) {
+                const value = channelData[j]
+                sumSquares += value * value
+                if (j > start && value * prevValue < 0) {
+                    zeroCrossings++
+                }
+                prevValue = value
+
+                // We need to keep absolute value otherwise we cannot visualize properly
+                const absValue = Math.abs(value)
+                localMinAmplitude = Math.min(localMinAmplitude, absValue)
+                localMaxAmplitude = Math.max(localMaxAmplitude, absValue)
+
+                if (value !== 0) {
+                    hasNonZeroValue = true
+                }
+            }
+
+            // Post-processing checks
+            if (!hasNonZeroValue) {
+                // All values are zero
+                localMinAmplitude = 0
+                localMaxAmplitude = 0
+            }
+
+            const rms = Math.sqrt(sumSquares / (end - start))
+            minAmplitude = Math.min(minAmplitude, localMinAmplitude)
+            maxAmplitude = Math.max(maxAmplitude, localMaxAmplitude)
+
+            const energy = sumSquares
+            const zcr = zeroCrossings / (end - start)
+
+            const silent = rms < SILENCE_THRESHOLD
+            const dB = 20 * Math.log10(rms)
+
+            if (silent) {
+                if (silenceStart === null) {
+                    silenceStart = start
+                } else if (start - silenceStart > MIN_SILENCE_DURATION) {
+                    // Silence detected for longer than the threshold, set amplitude to 0
+                    localMaxAmplitude = 0
+                    localMinAmplitude = 0
+                    isSpeech = false
+                }
+            } else {
+                silenceStart = null
+                if (
+                    !isSpeech &&
+                    start - lastSpeechEnd < SPEECH_INERTIA_DURATION
+                ) {
+                    isSpeech = true
+                }
+                lastSpeechEnd = end
+            }
+
+            const activeSpeech =
+                (rms > RMS_THRESHOLD && zcr > ZCR_THRESHOLD) ||
+                (isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION)
+
+            if (activeSpeech) {
+                isSpeech = true
+                lastSpeechEnd = end
+            } else {
+                isSpeech = false
+            }
+
+            const bytesPerSample = bitDepth / 8
+            const startPosition = start * bytesPerSample * numberOfChannels // Calculate start position in bytes
+            const endPosition = end * bytesPerSample * numberOfChannels // Calculate end position in bytes
+
+            // Compute features
+            const segmentData = channelData.slice(start, end)
+            const mfcc = features.mfcc
+                ? extractMFCC(segmentData, sampleRate)
+                : []
+            const spectralCentroid = features.spectralCentroid
+                ? extractSpectralCentroid(segmentData, sampleRate)
+                : 0
+            const spectralFlatness = features.spectralFlatness
+                ? extractSpectralFlatness(segmentData)
+                : 0
+            const spectralRollOff = features.spectralRollOff
+                ? extractSpectralRollOff(segmentData, sampleRate)
+                : 0
+            const spectralBandwidth = features.spectralBandwidth
+                ? extractSpectralBandwidth(segmentData, sampleRate)
+                : 0
+            const chromagram = features.chromagram
+                ? extractChromagram(segmentData, sampleRate)
+                : []
+            const hnr = features.hnr ? extractHNR(segmentData) : 0
+
+            const peakAmp = Math.max(Math.abs(localMaxAmplitude), Math.abs(localMinAmplitude))
+            const newData = {
+                id: uniqueIdCounter++, // Assign unique ID and increment the counter
+                amplitude: algorithm === 'peak' ? peakAmp : rms,
+                activeSpeech,
+                dB,
+                silent,
+                features: {
+                    energy,
+                    rms,
+                    minAmplitude: localMinAmplitude,
+                    maxAmplitude: localMaxAmplitude,
+                    zcr,
+                    mfcc: [], // Placeholder for MFCC features
+                    spectralCentroid, // Computed spectral centroid
+                    spectralFlatness, // Computed spectral flatness
+                    spectralRollOff, // Computed spectral roll-off
+                    spectralBandwidth, // Computed spectral bandwidth
+                    chromagram, // Computed chromagram
+                    hnr, // Computed HNR
+                },
+                startTime: start / sampleRate,
+                endTime: end / sampleRate,
+                startPosition,
+                endPosition,
+                samples: end - start,
+                speaker: 0, // Assuming speaker detection is to be handled later
+            }
+
+            dataPoints.push(newData)
+        }
+
+        return {
+            pointsPerSecond,
+            amplitudeAlgorithm: algorithm,
+            durationMs: fullAudioDurationMs,
+            bitDepth,
+            samples: totalSamples,
+            numberOfChannels,
+            sampleRate,
+            dataPoints,
+            amplitudeRange: {
+                min: minAmplitude,
+                max: maxAmplitude,
+            },
+            speakerChanges: [], // Placeholder for future speaker detection logic
+        }
+    }
+
+    try {
+        const result = extractWaveform(
+            channelData,
+            sampleRate,
+            pointsPerSecond,
+            algorithm
+        )
+        self.postMessage({
+            command: 'features',
+            result,
+        })
+    } catch (error) {
+        console.error('[AudioFeaturesExtractor] Error in processing', error)
+        self.postMessage({ error: error.message })
+    } finally {
+        // Do not close the worker so it can be re-used for subsequent messages
+        // self.close();
+    }
+}
+`