@siteed/expo-audio-stream 1.0.2 → 1.0.3

package/src/utils/encodingToBitDepth.ts

@@ -0,0 +1,18 @@
+import { BitDepth, EncodingType } from "../ExpoAudioStream.types";
+
+export const encodingToBitDepth = ({
+  encoding,
+}: {
+  encoding: EncodingType;
+}): BitDepth => {
+  switch (encoding) {
+    case "pcm_32bit":
+      return 32;
+    case "pcm_16bit":
+      return 16;
+    case "pcm_8bit":
+      return 8;
+    default:
+      throw new Error(`Unsupported encoding type: ${encoding}`);
+  }
+};

package/src/utils/getWavFileInfo.ts

@@ -0,0 +1,125 @@
+// packages/expo-audio-stream/src/utils/getWavFileInfo.ts
+
+import { BitDepth, SampleRate } from "../ExpoAudioStream.types";
+import {
+  DATA_CHUNK_ID,
+  DEFAULT_BIT_DEPTH,
+  DEFAULT_SAMPLE_RATE,
+  FMT_CHUNK_ID,
+  INFO_CHUNK_ID,
+  RIFF_HEADER,
+  WAVE_HEADER,
+} from "../constants";
+
+// Audio format descriptions
+const AUDIO_FORMATS: { [key: number]: string } = {
+  1: "PCM",
+  3: "IEEE float",
+  6: "8-bit ITU-T G.711 A-law",
+  7: "8-bit ITU-T G.711 µ-law",
+  65534: "WAVE_FORMAT_EXTENSIBLE",
+};
+
+/**
+ * Interface representing the metadata of a WAV file.
+ */
+export interface WavFileInfo {
+  sampleRate: SampleRate;
+  numChannels: number;
+  bitDepth: BitDepth;
+  size: number; // in bytes
+  durationMs: number; // in ms
+  audioFormatDescription: string; // Description of the audio format
+  byteRate: number; // Average bytes per second
+  blockAlign: number; // Number of bytes for one sample including all channels
+  creationDateTime?: string; // Optional creation date and time
+  comments?: string; // Optional comments or tags
+  compressionType?: string; // Optional compression type
+}
+
+/**
+ * Extracts metadata from a WAV file.
+ *
+ * @param arrayBuffer - The array buffer containing the WAV file data.
+ * @returns A promise that resolves to the extracted metadata.
+ */
+export const getWavFileInfo = async (
+  arrayBuffer: ArrayBuffer,
+): Promise<WavFileInfo> => {
+  const view = new DataView(arrayBuffer);
+
+  // Check if the file is a valid RIFF/WAVE file
+  const riffHeader = view.getUint32(0, false);
+  const waveHeader = view.getUint32(8, false);
+  if (riffHeader !== RIFF_HEADER || waveHeader !== WAVE_HEADER) {
+    throw new Error("Invalid WAV file");
+  }
+
+  // Initialize variables for the metadata
+  let fmtChunkOffset = 12;
+  let sampleRate: SampleRate = DEFAULT_SAMPLE_RATE;
+  let numChannels = 0;
+  let bitDepth: BitDepth = DEFAULT_BIT_DEPTH;
+  let dataChunkSize = 0;
+  let audioFormat = 0;
+  let byteRate = 0;
+  let blockAlign = 0;
+  let creationDateTime = "";
+  let comments = "";
+
+  // Parse chunks to find the "fmt " and "data" chunks
+  while (fmtChunkOffset < view.byteLength) {
+    const chunkId = view.getUint32(fmtChunkOffset, false);
+    const chunkSize = view.getUint32(fmtChunkOffset + 4, true);
+    if (chunkId === FMT_CHUNK_ID) {
+      // "fmt "
+      audioFormat = view.getUint16(fmtChunkOffset + 8, true);
+      if (!AUDIO_FORMATS[audioFormat]) {
+        throw new Error("Unsupported WAV file format");
+      }
+      numChannels = view.getUint16(fmtChunkOffset + 10, true);
+      sampleRate = view.getUint32(fmtChunkOffset + 12, true) as SampleRate;
+      byteRate = view.getUint32(fmtChunkOffset + 16, true);
+      blockAlign = view.getUint16(fmtChunkOffset + 20, true);
+      bitDepth = view.getUint16(fmtChunkOffset + 22, true) as BitDepth;
+    } else if (chunkId === DATA_CHUNK_ID) {
+      // "data"
+      dataChunkSize = chunkSize;
+      break;
+    } else if (chunkId === INFO_CHUNK_ID) {
+      // "INFO"
+      // Read INFO chunk (assuming it contains a text-based creation date/time and comments)
+      const infoStart = fmtChunkOffset + 8;
+      const infoText = new TextDecoder().decode(
+        new Uint8Array(arrayBuffer.slice(infoStart, infoStart + chunkSize)),
+      );
+      const infoParts = infoText.split("\0");
+      creationDateTime = infoParts[0];
+      comments = infoParts[1];
+    }
+    fmtChunkOffset += 8 + chunkSize;
+  }
+
+  if (!sampleRate || !numChannels || !bitDepth || !dataChunkSize) {
+    throw new Error("Incomplete WAV file information");
+  }
+
+  // Calculate duration
+  const bytesPerSample = bitDepth / 8;
+  const numSamples = dataChunkSize / (numChannels * bytesPerSample);
+  const durationMs = (numSamples / sampleRate) * 1000;
+
+  return {
+    sampleRate,
+    numChannels,
+    bitDepth,
+    size: arrayBuffer.byteLength,
+    durationMs,
+    audioFormatDescription: AUDIO_FORMATS[audioFormat],
+    byteRate,
+    blockAlign,
+    creationDateTime: creationDateTime || undefined,
+    comments: comments || undefined,
+    compressionType: audioFormat === 1 ? "None" : AUDIO_FORMATS[audioFormat],
+  };
+};

package/src/utils/writeWavHeader.ts

@@ -0,0 +1,56 @@
+// packages/expo-audio-stream/src/utils/writeWavHeader.ts
+interface WavHeaderOptions {
+  buffer: ArrayBuffer;
+  sampleRate: number;
+  numChannels: number;
+  bitDepth: number;
+}
+
+export const writeWavHeader = ({
+  buffer,
+  sampleRate,
+  numChannels,
+  bitDepth,
+}: WavHeaderOptions): ArrayBuffer => {
+  const bytesPerSample = bitDepth / 8;
+  const numSamples = buffer.byteLength / (numChannels * bytesPerSample);
+  const view = new DataView(buffer);
+  const blockAlign = numChannels * bytesPerSample;
+  const byteRate = sampleRate * blockAlign;
+
+  // Function to write a string to the DataView
+  const writeString = (view: DataView, offset: number, string: string) => {
+    for (let i = 0; i < string.length; i++) {
+      view.setUint8(offset + i, string.charCodeAt(i));
+    }
+  };
+
+  // Check if the buffer already has a WAV header by looking for "RIFF" at the start
+  const existingHeader = view.getUint32(0, false) === 0x52494646; // "RIFF" in ASCII
+
+  if (!existingHeader) {
+    // Write the WAV header
+    writeString(view, 0, "RIFF"); // ChunkID
+    view.setUint32(4, 36 + numSamples * blockAlign, true); // ChunkSize
+    writeString(view, 8, "WAVE"); // Format
+    writeString(view, 12, "fmt "); // Subchunk1ID
+    view.setUint32(16, 16, true); // Subchunk1Size (16 for PCM)
+    view.setUint16(20, bitDepth === 32 ? 3 : 1, true); // AudioFormat (3 for float, 1 for PCM)
+    view.setUint16(22, numChannels, true); // NumChannels
+    view.setUint32(24, sampleRate, true); // SampleRate
+    view.setUint32(28, byteRate, true); // ByteRate
+    view.setUint16(32, blockAlign, true); // BlockAlign
+    view.setUint16(34, bitDepth, true); // BitsPerSample
+    writeString(view, 36, "data"); // Subchunk2ID
+    view.setUint32(40, numSamples * blockAlign, true); // Subchunk2Size
+  } else {
+    // Update the existing WAV header if necessary
+    view.setUint32(4, 36 + numSamples * blockAlign, true); // Update ChunkSize
+    view.setUint32(24, sampleRate, true); // Update SampleRate
+    view.setUint32(28, byteRate, true); // Update ByteRate
+    view.setUint16(32, blockAlign, true); // Update BlockAlign
+    view.setUint32(40, numSamples * blockAlign, true); // Update Subchunk2Size
+  }
+
+  return buffer;
+};

package/src/workers/InlineFeaturesExtractor.web.tsx

@@ -0,0 +1,302 @@
+export const InlineFeaturesExtractor = `
+
+// Unique ID counter
+let uniqueIdCounter = 0;
+
+self.onmessage = function (event) {
+  const {
+    channelData, // this is only the newly recorded data when live recording.
+    sampleRate,
+    pointsPerSecond,
+    algorithm,
+    bitDepth,
+    fullAudioDurationMs,
+    numberOfChannels,
+    features: _features,
+  } = event.data;
+
+  console.log("[AudioFeaturesExtractor] Worker received message", event.data);
+  const features = _features || {};
+
+  const SILENCE_THRESHOLD = 0.01;
+  const MIN_SILENCE_DURATION = 1.5 * sampleRate; // 1.5 seconds of silence
+  const SPEECH_INERTIA_DURATION = 0.1 * sampleRate; // Speech inertia duration in samples
+  const RMS_THRESHOLD = 0.01;
+  const ZCR_THRESHOLD = 0.1;
+
+  // Placeholder functions for feature extraction
+  const extractMFCC = (segmentData, sampleRate) => {
+    // Implement MFCC extraction logic here
+    return [];
+  };
+
+  const extractSpectralCentroid = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => v * v);
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + index * value,
+      0,
+    );
+    return ((weightedSum / sum) * (sampleRate / 2)) / magnitudeSpectrum.length;
+  };
+
+  const extractSpectralFlatness = (segmentData) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const geometricMean = Math.exp(
+      magnitudeSpectrum
+        .map((v) => Math.log(v + Number.MIN_VALUE))
+        .reduce((a, b) => a + b) / magnitudeSpectrum.length,
+    );
+    const arithmeticMean =
+      magnitudeSpectrum.reduce((a, b) => a + b) / magnitudeSpectrum.length;
+    return arithmeticMean === 0 ? 0 : geometricMean / arithmeticMean;
+  };
+
+  const extractSpectralRollOff = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const totalEnergy = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    const rollOffThreshold = totalEnergy * 0.85;
+    let cumulativeEnergy = 0;
+
+    for (let i = 0; i < magnitudeSpectrum.length; i++) {
+      cumulativeEnergy += magnitudeSpectrum[i];
+      if (cumulativeEnergy >= rollOffThreshold) {
+        return (i / magnitudeSpectrum.length) * (sampleRate / 2);
+      }
+    }
+
+    return 0;
+  };
+
+  const extractSpectralBandwidth = (segmentData, sampleRate) => {
+    const centroid = extractSpectralCentroid(segmentData, sampleRate);
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + value * Math.pow(index - centroid, 2),
+      0,
+    );
+    return Math.sqrt(weightedSum / sum);
+  };
+
+  const extractChromagram = (segmentData, sampleRate) => {
+    return []; // TODO implement
+  };
+
+  const extractHNR = (segmentData) => {
+    const frameSize = segmentData.length;
+    const autocorrelation = new Float32Array(frameSize);
+
+    // Compute the autocorrelation of the segment data
+    for (let i = 0; i < frameSize; i++) {
+      let sum = 0;
+      for (let j = 0; j < frameSize - i; j++) {
+        sum += segmentData[j] * segmentData[j + i];
+      }
+      autocorrelation[i] = sum;
+    }
+
+    // Find the maximum autocorrelation value (excluding the zero lag)
+    const maxAutocorrelation = Math.max(...autocorrelation.subarray(1));
+
+    // Compute the HNR
+    return autocorrelation[0] !== 0
+      ? 10 *
+          Math.log10(
+            maxAutocorrelation / (autocorrelation[0] - maxAutocorrelation),
+          )
+      : 0;
+  };
+
+  const extractWaveform = (
+    channelData, // Float32Array
+    sampleRate, // number
+    pointsPerSecond, // number
+    algorithm, // string
+  ) => {
+    const totalSamples = channelData.length;
+    const segmentDuration = totalSamples / sampleRate;
+    const totalPoints = Math.max(
+      Math.ceil(segmentDuration * pointsPerSecond),
+      1,
+    );
+    const pointInterval = Math.ceil(totalSamples / totalPoints);
+    const dataPoints = [];
+    let minAmplitude = Infinity;
+    let maxAmplitude = -Infinity;
+    let silenceStart = null;
+    let lastSpeechEnd = -Infinity;
+    let isSpeech = false;
+
+    const expectedPoints = segmentDuration * pointsPerSecond;
+    const samplesPerPoint = Math.ceil(channelData.length / expectedPoints);
+
+    for (let i = 0; i < expectedPoints; i++) {
+      const start = i * samplesPerPoint;
+      const end = Math.min(start + samplesPerPoint, totalSamples);
+
+      let sumSquares = 0;
+      let zeroCrossings = 0;
+      let prevValue = channelData[start];
+      let localMinAmplitude = Infinity;
+      let localMaxAmplitude = -Infinity;
+      let hasNonZeroValue = false;
+
+      // compute values for the segment
+      for (let j = start; j < end; j++) {
+        const value = channelData[j];
+        sumSquares += value * value;
+        if (j > start && value * prevValue < 0) {
+          zeroCrossings++;
+        }
+        prevValue = value;
+
+        const absValue = Math.abs(value);
+        localMinAmplitude = Math.min(localMinAmplitude, absValue);
+        localMaxAmplitude = Math.max(localMaxAmplitude, absValue);
+
+        if (absValue !== 0) {
+          hasNonZeroValue = true;
+        }
+      }
+
+      // Post-processing checks
+      if (!hasNonZeroValue) {
+        // All values are zero
+        localMinAmplitude = 0;
+        localMaxAmplitude = 0;
+      }
+
+      const rms = Math.sqrt(sumSquares / (end - start));
+      minAmplitude = Math.min(minAmplitude, rms);
+      maxAmplitude = Math.max(maxAmplitude, rms);
+
+      const energy = sumSquares;
+      const zcr = zeroCrossings / (end - start);
+
+      const silent = rms < SILENCE_THRESHOLD;
+      const dB = 20 * Math.log10(rms);
+
+      if (silent) {
+        if (silenceStart === null) {
+          silenceStart = start;
+        } else if (start - silenceStart > MIN_SILENCE_DURATION) {
+          // Silence detected for longer than the threshold, set amplitude to 0
+          localMaxAmplitude = 0;
+          localMinAmplitude = 0;
+          isSpeech = false;
+        }
+      } else {
+        silenceStart = null;
+        if (!isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION) {
+          isSpeech = true;
+        }
+        lastSpeechEnd = end;
+      }
+
+      const activeSpeech =
+        (rms > RMS_THRESHOLD && zcr > ZCR_THRESHOLD) ||
+        (isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION);
+
+      if (activeSpeech) {
+        isSpeech = true;
+        lastSpeechEnd = end;
+      } else {
+        isSpeech = false;
+      }
+
+      const bytesPerSample = bitDepth / 8;
+      const startPosition = start * bytesPerSample * numberOfChannels; // Calculate start position in bytes
+      const endPosition = end * bytesPerSample * numberOfChannels; // Calculate end position in bytes
+
+      // Compute features
+      const segmentData = channelData.slice(start, end);
+      const mfcc = features.mfcc ? extractMFCC(segmentData, sampleRate) : [];
+      const spectralCentroid = features.spectralCentroid
+        ? extractSpectralCentroid(segmentData, sampleRate)
+        : 0;
+      const spectralFlatness = features.spectralFlatness
+        ? extractSpectralFlatness(segmentData)
+        : 0;
+      const spectralRollOff = features.spectralRollOff
+        ? extractSpectralRollOff(segmentData, sampleRate)
+        : 0;
+      const spectralBandwidth = features.spectralBandwidth
+        ? extractSpectralBandwidth(segmentData, sampleRate)
+        : 0;
+      const chromagram = features.chromagram
+        ? extractChromagram(segmentData, sampleRate)
+        : [];
+      const hnr = features.hnr ? extractHNR(segmentData) : 0;
+
+      const newData = {
+        id: uniqueIdCounter++, // Assign unique ID and increment the counter
+        amplitude: algorithm === "peak" ? localMaxAmplitude : rms,
+        activeSpeech,
+        dB,
+        silent,
+        features: {
+          energy,
+          rms,
+          minAmplitude: localMinAmplitude,
+          maxAmplitude: localMaxAmplitude,
+          zcr,
+          mfcc: [], // Placeholder for MFCC features
+          spectralCentroid, // Computed spectral centroid
+          spectralFlatness, // Computed spectral flatness
+          spectralRollOff, // Computed spectral roll-off
+          spectralBandwidth, // Computed spectral bandwidth
+          chromagram, // Computed chromagram
+          hnr, // Computed HNR
+        },
+        startTime: start / sampleRate,
+        endTime: end / sampleRate,
+        startPosition,
+        endPosition,
+        samples: end - start,
+        speaker: 0, // Assuming speaker detection is to be handled later
+      };
+
+      dataPoints.push(newData);
+    }
+
+    return {
+      pointsPerSecond,
+      durationMs: fullAudioDurationMs,
+      bitDepth,
+      samples: totalSamples,
+      numberOfChannels,
+      sampleRate,
+      dataPoints,
+      amplitudeRange: {
+        min: minAmplitude,
+        max: maxAmplitude,
+      },
+      speakerChanges: [], // Placeholder for future speaker detection logic
+    };
+  };
+
+  try {
+    const result = extractWaveform(
+      channelData,
+      sampleRate,
+      pointsPerSecond,
+      algorithm,
+    );
+    self.postMessage({
+      command: "features",
+      result,
+    });
+  } catch (error) {
+    console.error("[AudioFeaturesExtractor] Error in processing", error);
+    self.postMessage({ error: error.message });
+  } finally {
+    // Do not close the worker so it can be re-used for subsequent messages
+    // self.close();
+  }
+};
+`;