@siteed/expo-audio-stream 1.0.2 → 1.0.5

package/src/utils/getWavFileInfo.ts

@@ -0,0 +1,132 @@
+// packages/expo-audio-stream/src/utils/getWavFileInfo.ts
+
+import { BitDepth, SampleRate } from '../ExpoAudioStream.types'
+import {
+    DATA_CHUNK_ID,
+    DEFAULT_BIT_DEPTH,
+    DEFAULT_SAMPLE_RATE,
+    FMT_CHUNK_ID,
+    INFO_CHUNK_ID,
+    RIFF_HEADER,
+    WAVE_HEADER,
+} from '../constants'
+
+// Audio format descriptions
+const AUDIO_FORMATS: { [key: number]: string } = {
+    1: 'PCM',
+    3: 'IEEE float',
+    6: '8-bit ITU-T G.711 A-law',
+    7: '8-bit ITU-T G.711 µ-law',
+    65534: 'WAVE_FORMAT_EXTENSIBLE',
+}
+
+/**
+ * Interface representing the metadata of a WAV file.
+ */
+export interface WavFileInfo {
+    sampleRate: SampleRate
+    numChannels: number
+    bitDepth: BitDepth
+    size: number // in bytes
+    durationMs: number // in ms
+    audioFormatDescription: string // Description of the audio format
+    byteRate: number // Average bytes per second
+    blockAlign: number // Number of bytes for one sample including all channels
+    creationDateTime?: string // Optional creation date and time
+    comments?: string // Optional comments or tags
+    compressionType?: string // Optional compression type
+    dataChunkOffset: number // Position of the first data chunk
+}
+
+/**
+ * Extracts metadata from a WAV buffer.
+ *
+ * @param arrayBuffer - The array buffer containing the WAV data.
+ * @returns A promise that resolves to the extracted metadata.
+ */
+export const getWavFileInfo = async (
+    arrayBuffer: ArrayBuffer
+): Promise<WavFileInfo> => {
+    const view = new DataView(arrayBuffer)
+
+    // Check if the file is a valid RIFF/WAVE file
+    const riffHeader = view.getUint32(0, false)
+    const waveHeader = view.getUint32(8, false)
+    if (riffHeader !== RIFF_HEADER || waveHeader !== WAVE_HEADER) {
+        throw new Error('Invalid WAV file')
+    }
+
+    // Initialize variables for the metadata
+    let fmtChunkOffset = 12
+    let sampleRate: SampleRate = DEFAULT_SAMPLE_RATE
+    let numChannels = 0
+    let bitDepth: BitDepth = DEFAULT_BIT_DEPTH
+    let dataChunkSize = 0
+    let audioFormat = 0
+    let byteRate = 0
+    let blockAlign = 0
+    let creationDateTime = ''
+    let comments = ''
+    let dataChunkOffset = 0
+
+    // Parse chunks to find the "fmt " and "data" chunks
+    while (fmtChunkOffset < view.byteLength) {
+        const chunkId = view.getUint32(fmtChunkOffset, false)
+        const chunkSize = view.getUint32(fmtChunkOffset + 4, true)
+        if (chunkId === FMT_CHUNK_ID) {
+            // "fmt "
+            audioFormat = view.getUint16(fmtChunkOffset + 8, true)
+            if (!AUDIO_FORMATS[audioFormat]) {
+                throw new Error('Unsupported WAV file format')
+            }
+            numChannels = view.getUint16(fmtChunkOffset + 10, true)
+            sampleRate = view.getUint32(fmtChunkOffset + 12, true) as SampleRate
+            byteRate = view.getUint32(fmtChunkOffset + 16, true)
+            blockAlign = view.getUint16(fmtChunkOffset + 20, true)
+            bitDepth = view.getUint16(fmtChunkOffset + 22, true) as BitDepth
+        } else if (chunkId === DATA_CHUNK_ID) {
+            // "data"
+            dataChunkSize = chunkSize
+            dataChunkOffset = fmtChunkOffset + 8 // Position after chunk header
+            break
+        } else if (chunkId === INFO_CHUNK_ID) {
+            // "INFO"
+            // Read INFO chunk (assuming it contains a text-based creation date/time and comments)
+            const infoStart = fmtChunkOffset + 8
+            const infoText = new TextDecoder().decode(
+                new Uint8Array(
+                    arrayBuffer.slice(infoStart, infoStart + chunkSize)
+                )
+            )
+            const infoParts = infoText.split('\0')
+            creationDateTime = infoParts[0]
+            comments = infoParts[1]
+        }
+        fmtChunkOffset += 8 + chunkSize
+    }
+
+    if (!sampleRate || !numChannels || !bitDepth || !dataChunkSize) {
+        throw new Error('Incomplete WAV file information')
+    }
+
+    // Calculate duration
+    const bytesPerSample = bitDepth / 8
+    const numSamples = dataChunkSize / (numChannels * bytesPerSample)
+    const durationMs = (numSamples / sampleRate) * 1000
+
+    return {
+        sampleRate,
+        numChannels,
+        bitDepth,
+        size: arrayBuffer.byteLength,
+        durationMs,
+        audioFormatDescription: AUDIO_FORMATS[audioFormat],
+        byteRate,
+        blockAlign,
+        creationDateTime: creationDateTime || undefined,
+        comments: comments || undefined,
+        compressionType:
+            audioFormat === 1 ? 'None' : AUDIO_FORMATS[audioFormat],
+        dataChunkOffset,
+    }
+}

package/src/utils/writeWavHeader.ts

@@ -0,0 +1,56 @@
+// packages/expo-audio-stream/src/utils/writeWavHeader.ts
+interface WavHeaderOptions {
+    buffer: ArrayBuffer
+    sampleRate: number
+    numChannels: number
+    bitDepth: number
+}
+
+export const writeWavHeader = ({
+    buffer,
+    sampleRate,
+    numChannels,
+    bitDepth,
+}: WavHeaderOptions): ArrayBuffer => {
+    const bytesPerSample = bitDepth / 8
+    const numSamples = buffer.byteLength / (numChannels * bytesPerSample)
+    const view = new DataView(buffer)
+    const blockAlign = numChannels * bytesPerSample
+    const byteRate = sampleRate * blockAlign
+
+    // Function to write a string to the DataView
+    const writeString = (view: DataView, offset: number, string: string) => {
+        for (let i = 0; i < string.length; i++) {
+            view.setUint8(offset + i, string.charCodeAt(i))
+        }
+    }
+
+    // Check if the buffer already has a WAV header by looking for "RIFF" at the start
+    const existingHeader = view.getUint32(0, false) === 0x52494646 // "RIFF" in ASCII
+
+    if (!existingHeader) {
+        // Write the WAV header
+        writeString(view, 0, 'RIFF') // ChunkID
+        view.setUint32(4, 36 + numSamples * blockAlign, true) // ChunkSize
+        writeString(view, 8, 'WAVE') // Format
+        writeString(view, 12, 'fmt ') // Subchunk1ID
+        view.setUint32(16, 16, true) // Subchunk1Size (16 for PCM)
+        view.setUint16(20, bitDepth === 32 ? 3 : 1, true) // AudioFormat (3 for float, 1 for PCM)
+        view.setUint16(22, numChannels, true) // NumChannels
+        view.setUint32(24, sampleRate, true) // SampleRate
+        view.setUint32(28, byteRate, true) // ByteRate
+        view.setUint16(32, blockAlign, true) // BlockAlign
+        view.setUint16(34, bitDepth, true) // BitsPerSample
+        writeString(view, 36, 'data') // Subchunk2ID
+        view.setUint32(40, numSamples * blockAlign, true) // Subchunk2Size
+    } else {
+        // Update the existing WAV header if necessary
+        view.setUint32(4, 36 + numSamples * blockAlign, true) // Update ChunkSize
+        view.setUint32(24, sampleRate, true) // Update SampleRate
+        view.setUint32(28, byteRate, true) // Update ByteRate
+        view.setUint16(32, blockAlign, true) // Update BlockAlign
+        view.setUint32(40, numSamples * blockAlign, true) // Update Subchunk2Size
+    }
+
+    return buffer
+}

package/src/workers/InlineFeaturesExtractor.web.tsx

@@ -0,0 +1,302 @@
+export const InlineFeaturesExtractor = `
+
+// Unique ID counter
+let uniqueIdCounter = 0;
+
+self.onmessage = function (event) {
+  const {
+    channelData, // this is only the newly recorded data when live recording.
+    sampleRate,
+    pointsPerSecond,
+    algorithm,
+    bitDepth,
+    fullAudioDurationMs,
+    numberOfChannels,
+    features: _features,
+  } = event.data;
+
+  console.log("[AudioFeaturesExtractor] Worker received message", event.data);
+  const features = _features || {};
+
+  const SILENCE_THRESHOLD = 0.01;
+  const MIN_SILENCE_DURATION = 1.5 * sampleRate; // 1.5 seconds of silence
+  const SPEECH_INERTIA_DURATION = 0.1 * sampleRate; // Speech inertia duration in samples
+  const RMS_THRESHOLD = 0.01;
+  const ZCR_THRESHOLD = 0.1;
+
+  // Placeholder functions for feature extraction
+  const extractMFCC = (segmentData, sampleRate) => {
+    // Implement MFCC extraction logic here
+    return [];
+  };
+
+  const extractSpectralCentroid = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => v * v);
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + index * value,
+      0,
+    );
+    return ((weightedSum / sum) * (sampleRate / 2)) / magnitudeSpectrum.length;
+  };
+
+  const extractSpectralFlatness = (segmentData) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const geometricMean = Math.exp(
+      magnitudeSpectrum
+        .map((v) => Math.log(v + Number.MIN_VALUE))
+        .reduce((a, b) => a + b) / magnitudeSpectrum.length,
+    );
+    const arithmeticMean =
+      magnitudeSpectrum.reduce((a, b) => a + b) / magnitudeSpectrum.length;
+    return arithmeticMean === 0 ? 0 : geometricMean / arithmeticMean;
+  };
+
+  const extractSpectralRollOff = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const totalEnergy = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    const rollOffThreshold = totalEnergy * 0.85;
+    let cumulativeEnergy = 0;
+
+    for (let i = 0; i < magnitudeSpectrum.length; i++) {
+      cumulativeEnergy += magnitudeSpectrum[i];
+      if (cumulativeEnergy >= rollOffThreshold) {
+        return (i / magnitudeSpectrum.length) * (sampleRate / 2);
+      }
+    }
+
+    return 0;
+  };
+
+  const extractSpectralBandwidth = (segmentData, sampleRate) => {
+    const centroid = extractSpectralCentroid(segmentData, sampleRate);
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + value * Math.pow(index - centroid, 2),
+      0,
+    );
+    return Math.sqrt(weightedSum / sum);
+  };
+
+  const extractChromagram = (segmentData, sampleRate) => {
+    return []; // TODO implement
+  };
+
+  const extractHNR = (segmentData) => {
+    const frameSize = segmentData.length;
+    const autocorrelation = new Float32Array(frameSize);
+
+    // Compute the autocorrelation of the segment data
+    for (let i = 0; i < frameSize; i++) {
+      let sum = 0;
+      for (let j = 0; j < frameSize - i; j++) {
+        sum += segmentData[j] * segmentData[j + i];
+      }
+      autocorrelation[i] = sum;
+    }
+
+    // Find the maximum autocorrelation value (excluding the zero lag)
+    const maxAutocorrelation = Math.max(...autocorrelation.subarray(1));
+
+    // Compute the HNR
+    return autocorrelation[0] !== 0
+      ? 10 *
+          Math.log10(
+            maxAutocorrelation / (autocorrelation[0] - maxAutocorrelation),
+          )
+      : 0;
+  };
+
+  const extractWaveform = (
+    channelData, // Float32Array
+    sampleRate, // number
+    pointsPerSecond, // number
+    algorithm, // string
+  ) => {
+    const totalSamples = channelData.length;
+    const segmentDuration = totalSamples / sampleRate;
+    const totalPoints = Math.max(
+      Math.ceil(segmentDuration * pointsPerSecond),
+      1,
+    );
+    const pointInterval = Math.ceil(totalSamples / totalPoints);
+    const dataPoints = [];
+    let minAmplitude = Infinity;
+    let maxAmplitude = -Infinity;
+    let silenceStart = null;
+    let lastSpeechEnd = -Infinity;
+    let isSpeech = false;
+
+    const expectedPoints = segmentDuration * pointsPerSecond;
+    const samplesPerPoint = Math.ceil(channelData.length / expectedPoints);
+
+    for (let i = 0; i < expectedPoints; i++) {
+      const start = i * samplesPerPoint;
+      const end = Math.min(start + samplesPerPoint, totalSamples);
+
+      let sumSquares = 0;
+      let zeroCrossings = 0;
+      let prevValue = channelData[start];
+      let localMinAmplitude = Infinity;
+      let localMaxAmplitude = -Infinity;
+      let hasNonZeroValue = false;
+
+      // compute values for the segment
+      for (let j = start; j < end; j++) {
+        const value = channelData[j];
+        sumSquares += value * value;
+        if (j > start && value * prevValue < 0) {
+          zeroCrossings++;
+        }
+        prevValue = value;
+
+        const absValue = Math.abs(value);
+        localMinAmplitude = Math.min(localMinAmplitude, absValue);
+        localMaxAmplitude = Math.max(localMaxAmplitude, absValue);
+
+        if (absValue !== 0) {
+          hasNonZeroValue = true;
+        }
+      }
+
+      // Post-processing checks
+      if (!hasNonZeroValue) {
+        // All values are zero
+        localMinAmplitude = 0;
+        localMaxAmplitude = 0;
+      }
+
+      const rms = Math.sqrt(sumSquares / (end - start));
+      minAmplitude = Math.min(minAmplitude, rms);
+      maxAmplitude = Math.max(maxAmplitude, rms);
+
+      const energy = sumSquares;
+      const zcr = zeroCrossings / (end - start);
+
+      const silent = rms < SILENCE_THRESHOLD;
+      const dB = 20 * Math.log10(rms);
+
+      if (silent) {
+        if (silenceStart === null) {
+          silenceStart = start;
+        } else if (start - silenceStart > MIN_SILENCE_DURATION) {
+          // Silence detected for longer than the threshold, set amplitude to 0
+          localMaxAmplitude = 0;
+          localMinAmplitude = 0;
+          isSpeech = false;
+        }
+      } else {
+        silenceStart = null;
+        if (!isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION) {
+          isSpeech = true;
+        }
+        lastSpeechEnd = end;
+      }
+
+      const activeSpeech =
+        (rms > RMS_THRESHOLD && zcr > ZCR_THRESHOLD) ||
+        (isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION);
+
+      if (activeSpeech) {
+        isSpeech = true;
+        lastSpeechEnd = end;
+      } else {
+        isSpeech = false;
+      }
+
+      const bytesPerSample = bitDepth / 8;
+      const startPosition = start * bytesPerSample * numberOfChannels; // Calculate start position in bytes
+      const endPosition = end * bytesPerSample * numberOfChannels; // Calculate end position in bytes
+
+      // Compute features
+      const segmentData = channelData.slice(start, end);
+      const mfcc = features.mfcc ? extractMFCC(segmentData, sampleRate) : [];
+      const spectralCentroid = features.spectralCentroid
+        ? extractSpectralCentroid(segmentData, sampleRate)
+        : 0;
+      const spectralFlatness = features.spectralFlatness
+        ? extractSpectralFlatness(segmentData)
+        : 0;
+      const spectralRollOff = features.spectralRollOff
+        ? extractSpectralRollOff(segmentData, sampleRate)
+        : 0;
+      const spectralBandwidth = features.spectralBandwidth
+        ? extractSpectralBandwidth(segmentData, sampleRate)
+        : 0;
+      const chromagram = features.chromagram
+        ? extractChromagram(segmentData, sampleRate)
+        : [];
+      const hnr = features.hnr ? extractHNR(segmentData) : 0;
+
+      const newData = {
+        id: uniqueIdCounter++, // Assign unique ID and increment the counter
+        amplitude: algorithm === "peak" ? localMaxAmplitude : rms,
+        activeSpeech,
+        dB,
+        silent,
+        features: {
+          energy,
+          rms,
+          minAmplitude: localMinAmplitude,
+          maxAmplitude: localMaxAmplitude,
+          zcr,
+          mfcc: [], // Placeholder for MFCC features
+          spectralCentroid, // Computed spectral centroid
+          spectralFlatness, // Computed spectral flatness
+          spectralRollOff, // Computed spectral roll-off
+          spectralBandwidth, // Computed spectral bandwidth
+          chromagram, // Computed chromagram
+          hnr, // Computed HNR
+        },
+        startTime: start / sampleRate,
+        endTime: end / sampleRate,
+        startPosition,
+        endPosition,
+        samples: end - start,
+        speaker: 0, // Assuming speaker detection is to be handled later
+      };
+
+      dataPoints.push(newData);
+    }
+
+    return {
+      pointsPerSecond,
+      durationMs: fullAudioDurationMs,
+      bitDepth,
+      samples: totalSamples,
+      numberOfChannels,
+      sampleRate,
+      dataPoints,
+      amplitudeRange: {
+        min: minAmplitude,
+        max: maxAmplitude,
+      },
+      speakerChanges: [], // Placeholder for future speaker detection logic
+    };
+  };
+
+  try {
+    const result = extractWaveform(
+      channelData,
+      sampleRate,
+      pointsPerSecond,
+      algorithm,
+    );
+    self.postMessage({
+      command: "features",
+      result,
+    });
+  } catch (error) {
+    console.error("[AudioFeaturesExtractor] Error in processing", error);
+    self.postMessage({ error: error.message });
+  } finally {
+    // Do not close the worker so it can be re-used for subsequent messages
+    // self.close();
+  }
+};
+`