@siteed/expo-audio-stream 1.0.1 → 1.0.3

package/src/workers/InlineFeaturesExtractor.web.tsx

@@ -0,0 +1,302 @@
+export const InlineFeaturesExtractor = `
+
+// Unique ID counter
+let uniqueIdCounter = 0;
+
+self.onmessage = function (event) {
+  const {
+    channelData, // this is only the newly recorded data when live recording.
+    sampleRate,
+    pointsPerSecond,
+    algorithm,
+    bitDepth,
+    fullAudioDurationMs,
+    numberOfChannels,
+    features: _features,
+  } = event.data;
+
+  console.log("[AudioFeaturesExtractor] Worker received message", event.data);
+  const features = _features || {};
+
+  const SILENCE_THRESHOLD = 0.01;
+  const MIN_SILENCE_DURATION = 1.5 * sampleRate; // 1.5 seconds of silence
+  const SPEECH_INERTIA_DURATION = 0.1 * sampleRate; // Speech inertia duration in samples
+  const RMS_THRESHOLD = 0.01;
+  const ZCR_THRESHOLD = 0.1;
+
+  // Placeholder functions for feature extraction
+  const extractMFCC = (segmentData, sampleRate) => {
+    // Implement MFCC extraction logic here
+    return [];
+  };
+
+  const extractSpectralCentroid = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => v * v);
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + index * value,
+      0,
+    );
+    return ((weightedSum / sum) * (sampleRate / 2)) / magnitudeSpectrum.length;
+  };
+
+  const extractSpectralFlatness = (segmentData) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const geometricMean = Math.exp(
+      magnitudeSpectrum
+        .map((v) => Math.log(v + Number.MIN_VALUE))
+        .reduce((a, b) => a + b) / magnitudeSpectrum.length,
+    );
+    const arithmeticMean =
+      magnitudeSpectrum.reduce((a, b) => a + b) / magnitudeSpectrum.length;
+    return arithmeticMean === 0 ? 0 : geometricMean / arithmeticMean;
+  };
+
+  const extractSpectralRollOff = (segmentData, sampleRate) => {
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const totalEnergy = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    const rollOffThreshold = totalEnergy * 0.85;
+    let cumulativeEnergy = 0;
+
+    for (let i = 0; i < magnitudeSpectrum.length; i++) {
+      cumulativeEnergy += magnitudeSpectrum[i];
+      if (cumulativeEnergy >= rollOffThreshold) {
+        return (i / magnitudeSpectrum.length) * (sampleRate / 2);
+      }
+    }
+
+    return 0;
+  };
+
+  const extractSpectralBandwidth = (segmentData, sampleRate) => {
+    const centroid = extractSpectralCentroid(segmentData, sampleRate);
+    const magnitudeSpectrum = segmentData.map((v) => Math.abs(v));
+    const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0);
+    if (sum === 0) return 0;
+
+    const weightedSum = magnitudeSpectrum.reduce(
+      (acc, value, index) => acc + value * Math.pow(index - centroid, 2),
+      0,
+    );
+    return Math.sqrt(weightedSum / sum);
+  };
+
+  const extractChromagram = (segmentData, sampleRate) => {
+    return []; // TODO implement
+  };
+
+  const extractHNR = (segmentData) => {
+    const frameSize = segmentData.length;
+    const autocorrelation = new Float32Array(frameSize);
+
+    // Compute the autocorrelation of the segment data
+    for (let i = 0; i < frameSize; i++) {
+      let sum = 0;
+      for (let j = 0; j < frameSize - i; j++) {
+        sum += segmentData[j] * segmentData[j + i];
+      }
+      autocorrelation[i] = sum;
+    }
+
+    // Find the maximum autocorrelation value (excluding the zero lag)
+    const maxAutocorrelation = Math.max(...autocorrelation.subarray(1));
+
+    // Compute the HNR
+    return autocorrelation[0] !== 0
+      ? 10 *
+          Math.log10(
+            maxAutocorrelation / (autocorrelation[0] - maxAutocorrelation),
+          )
+      : 0;
+  };
+
+  const extractWaveform = (
+    channelData, // Float32Array
+    sampleRate, // number
+    pointsPerSecond, // number
+    algorithm, // string
+  ) => {
+    const totalSamples = channelData.length;
+    const segmentDuration = totalSamples / sampleRate;
+    const totalPoints = Math.max(
+      Math.ceil(segmentDuration * pointsPerSecond),
+      1,
+    );
+    const pointInterval = Math.ceil(totalSamples / totalPoints);
+    const dataPoints = [];
+    let minAmplitude = Infinity;
+    let maxAmplitude = -Infinity;
+    let silenceStart = null;
+    let lastSpeechEnd = -Infinity;
+    let isSpeech = false;
+
+    const expectedPoints = segmentDuration * pointsPerSecond;
+    const samplesPerPoint = Math.ceil(channelData.length / expectedPoints);
+
+    for (let i = 0; i < expectedPoints; i++) {
+      const start = i * samplesPerPoint;
+      const end = Math.min(start + samplesPerPoint, totalSamples);
+
+      let sumSquares = 0;
+      let zeroCrossings = 0;
+      let prevValue = channelData[start];
+      let localMinAmplitude = Infinity;
+      let localMaxAmplitude = -Infinity;
+      let hasNonZeroValue = false;
+
+      // compute values for the segment
+      for (let j = start; j < end; j++) {
+        const value = channelData[j];
+        sumSquares += value * value;
+        if (j > start && value * prevValue < 0) {
+          zeroCrossings++;
+        }
+        prevValue = value;
+
+        const absValue = Math.abs(value);
+        localMinAmplitude = Math.min(localMinAmplitude, absValue);
+        localMaxAmplitude = Math.max(localMaxAmplitude, absValue);
+
+        if (absValue !== 0) {
+          hasNonZeroValue = true;
+        }
+      }
+
+      // Post-processing checks
+      if (!hasNonZeroValue) {
+        // All values are zero
+        localMinAmplitude = 0;
+        localMaxAmplitude = 0;
+      }
+
+      const rms = Math.sqrt(sumSquares / (end - start));
+      minAmplitude = Math.min(minAmplitude, rms);
+      maxAmplitude = Math.max(maxAmplitude, rms);
+
+      const energy = sumSquares;
+      const zcr = zeroCrossings / (end - start);
+
+      const silent = rms < SILENCE_THRESHOLD;
+      const dB = 20 * Math.log10(rms);
+
+      if (silent) {
+        if (silenceStart === null) {
+          silenceStart = start;
+        } else if (start - silenceStart > MIN_SILENCE_DURATION) {
+          // Silence detected for longer than the threshold, set amplitude to 0
+          localMaxAmplitude = 0;
+          localMinAmplitude = 0;
+          isSpeech = false;
+        }
+      } else {
+        silenceStart = null;
+        if (!isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION) {
+          isSpeech = true;
+        }
+        lastSpeechEnd = end;
+      }
+
+      const activeSpeech =
+        (rms > RMS_THRESHOLD && zcr > ZCR_THRESHOLD) ||
+        (isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION);
+
+      if (activeSpeech) {
+        isSpeech = true;
+        lastSpeechEnd = end;
+      } else {
+        isSpeech = false;
+      }
+
+      const bytesPerSample = bitDepth / 8;
+      const startPosition = start * bytesPerSample * numberOfChannels; // Calculate start position in bytes
+      const endPosition = end * bytesPerSample * numberOfChannels; // Calculate end position in bytes
+
+      // Compute features
+      const segmentData = channelData.slice(start, end);
+      const mfcc = features.mfcc ? extractMFCC(segmentData, sampleRate) : [];
+      const spectralCentroid = features.spectralCentroid
+        ? extractSpectralCentroid(segmentData, sampleRate)
+        : 0;
+      const spectralFlatness = features.spectralFlatness
+        ? extractSpectralFlatness(segmentData)
+        : 0;
+      const spectralRollOff = features.spectralRollOff
+        ? extractSpectralRollOff(segmentData, sampleRate)
+        : 0;
+      const spectralBandwidth = features.spectralBandwidth
+        ? extractSpectralBandwidth(segmentData, sampleRate)
+        : 0;
+      const chromagram = features.chromagram
+        ? extractChromagram(segmentData, sampleRate)
+        : [];
+      const hnr = features.hnr ? extractHNR(segmentData) : 0;
+
+      const newData = {
+        id: uniqueIdCounter++, // Assign unique ID and increment the counter
+        amplitude: algorithm === "peak" ? localMaxAmplitude : rms,
+        activeSpeech,
+        dB,
+        silent,
+        features: {
+          energy,
+          rms,
+          minAmplitude: localMinAmplitude,
+          maxAmplitude: localMaxAmplitude,
+          zcr,
+          mfcc: [], // Placeholder for MFCC features
+          spectralCentroid, // Computed spectral centroid
+          spectralFlatness, // Computed spectral flatness
+          spectralRollOff, // Computed spectral roll-off
+          spectralBandwidth, // Computed spectral bandwidth
+          chromagram, // Computed chromagram
+          hnr, // Computed HNR
+        },
+        startTime: start / sampleRate,
+        endTime: end / sampleRate,
+        startPosition,
+        endPosition,
+        samples: end - start,
+        speaker: 0, // Assuming speaker detection is to be handled later
+      };
+
+      dataPoints.push(newData);
+    }
+
+    return {
+      pointsPerSecond,
+      durationMs: fullAudioDurationMs,
+      bitDepth,
+      samples: totalSamples,
+      numberOfChannels,
+      sampleRate,
+      dataPoints,
+      amplitudeRange: {
+        min: minAmplitude,
+        max: maxAmplitude,
+      },
+      speakerChanges: [], // Placeholder for future speaker detection logic
+    };
+  };
+
+  try {
+    const result = extractWaveform(
+      channelData,
+      sampleRate,
+      pointsPerSecond,
+      algorithm,
+    );
+    self.postMessage({
+      command: "features",
+      result,
+    });
+  } catch (error) {
+    console.error("[AudioFeaturesExtractor] Error in processing", error);
+    self.postMessage({ error: error.message });
+  } finally {
+    // Do not close the worker so it can be re-used for subsequent messages
+    // self.close();
+  }
+};
+`;

package/src/workers/inlineAudioWebWorker.web.tsx

@@ -0,0 +1,242 @@
+export const InlineAudioWebWorker = `
+const DEFAULT_BIT_DEPTH = 32;
+const DEFAULT_SAMPLE_RATE = 44100;
+
+class RecorderProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.recordedBuffers = []; // Float32Array
+    this.newRecBuffer = []; // Float32Array
+    this.resampledBuffer = []; // Float32Array
+    this.exportIntervalSamples = 0;
+    this.samplesSinceLastExport = 0;
+    this.recordSampleRate = DEFAULT_SAMPLE_RATE; // To be overwritten
+    this.exportSampleRate = DEFAULT_SAMPLE_RATE; // To be overwritten
+    this.recordBitDepth = DEFAULT_BIT_DEPTH; // Default to 32-bit depth
+    this.exportBitDepth = DEFAULT_BIT_DEPTH; // To be overwritten
+    this.numberOfChannels = 1; // Default to 1 channel (mono)
+    this.isRecording = true;
+    this.port.onmessage = this.handleMessage.bind(this);
+  }
+
+  handleMessage(event) {
+    switch (event.data.command) {
+      case "init":
+        this.recordSampleRate = event.data.recordSampleRate;
+        this.exportSampleRate =
+          event.data.exportSampleRate || event.data.recordSampleRate;
+        this.exportIntervalSamples =
+          this.recordSampleRate * (event.data.interval / 1000);
+        if (event.data.numberOfChannels) {
+          this.numberOfChannels = event.data.numberOfChannels;
+        }
+        if (event.data.recordBitDepth) {
+          this.recordBitDepth = event.data.recordBitDepth;
+        }
+        this.exportBitDepth =
+          event.data.exportBitDepth || this.recordBitDepth || DEFAULT_BIT_DEPTH;
+        break;
+      case "stop":
+        this.isRecording = false;
+        this.getAllRecordedData()
+          .then((fullRecordedData) => {
+            this.port.postMessage({
+              command: "recordedData",
+              recordedData: fullRecordedData,
+              bitDepth: this.exportBitDepth,
+              sampleRate: this.exportSampleRate,
+            });
+            return fullRecordedData;
+          })
+          .catch((error) => {
+            console.error("Error extracting recorded data:", error);
+          });
+        break;
+    }
+  }
+
+  process(inputs, outputs, parameters) {
+    if (!this.isRecording) return true;
+    const input = inputs[0];
+    if (input.length > 0) {
+      const newBuffer = new Float32Array(input[0]);
+      this.newRecBuffer.push(newBuffer);
+      this.recordedBuffers.push(newBuffer);
+      this.samplesSinceLastExport += newBuffer.length;
+
+      if (this.samplesSinceLastExport >= this.exportIntervalSamples) {
+        this.exportNewData();
+        this.samplesSinceLastExport = 0;
+      }
+    }
+    return true;
+  }
+
+  mergeBuffers(bufferArray, recLength) {
+    const result = new Float32Array(recLength);
+    let offset = 0;
+    for (let i = 0; i < bufferArray.length; i++) {
+      result.set(bufferArray[i], offset);
+      offset += bufferArray[i].length;
+    }
+    return result;
+  }
+
+  floatTo16BitPCM(input) {
+    const output = new Int16Array(input.length);
+    for (let i = 0; i < input.length; i++) {
+      const s = Math.max(-1, Math.min(1, input[i]));
+      output[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+    }
+    console.debug(
+      "Float to 16-bit PCM conversion complete. Output byte length:",
+      output.byteLength,
+    );
+    return output;
+  }
+
+  floatTo32BitPCM(input) {
+    const output = new Int32Array(input.length);
+    for (let i = 0; i < input.length; i++) {
+      const s = Math.max(-1, Math.min(1, input[i]));
+      output[i] = s < 0 ? s * 0x80000000 : s * 0x7fffffff;
+    }
+    console.debug(
+      "Float to 32-bit PCM conversion complete. Output byte length:",
+      output.byteLength,
+    );
+    return output;
+  }
+
+  resample(samples, targetSampleRate) {
+    if (this.recordSampleRate === targetSampleRate) {
+      return samples;
+    }
+    const resampledBuffer = new Float32Array(
+      (samples.length * targetSampleRate) / this.recordSampleRate,
+    );
+    const ratio = this.recordSampleRate / targetSampleRate;
+    let offset = 0;
+    for (let i = 0; i < resampledBuffer.length; i++) {
+      const nextOffset = Math.floor((i + 1) * ratio);
+      let accum = 0;
+      let count = 0;
+      for (let j = offset; j < nextOffset && j < samples.length; j++) {
+        accum += samples[j];
+        count++;
+      }
+      resampledBuffer[i] = accum / count;
+      offset = nextOffset;
+    }
+    return resampledBuffer;
+  }
+
+  async resampleBuffer(buffer, targetSampleRate) {
+    if (typeof OfflineAudioContext === "undefined") {
+      return this.resample(buffer, targetSampleRate);
+    }
+
+    if (this.recordSampleRate === targetSampleRate) {
+      return buffer;
+    }
+    const offlineContext = new OfflineAudioContext(
+      this.numberOfChannels,
+      buffer.length,
+      this.recordSampleRate,
+    );
+    const sourceBuffer = offlineContext.createBuffer(
+      this.numberOfChannels,
+      buffer.length,
+      this.recordSampleRate,
+    );
+    sourceBuffer.copyToChannel(buffer, 0);
+
+    const bufferSource = offlineContext.createBufferSource();
+    bufferSource.buffer = sourceBuffer;
+    bufferSource.connect(offlineContext.destination);
+    bufferSource.start();
+
+    const renderedBuffer = await offlineContext.startRendering();
+
+    const resampledBuffer = new Float32Array(renderedBuffer.length);
+    renderedBuffer.copyFromChannel(resampledBuffer, 0);
+
+    return resampledBuffer;
+  }
+
+  async exportNewData() {
+    // Calculate the total length of the new recorded buffers
+    const length = this.newRecBuffer.reduce(
+      (acc, buffer) => acc + buffer.length,
+      0,
+    );
+
+    // Merge all new recorded buffers into a single buffer
+    const mergedBuffer = this.mergeBuffers(this.newRecBuffer, length);
+
+    const resampledBuffer = await this.resampleBuffer(
+      mergedBuffer,
+      this.exportSampleRate,
+    );
+
+    let finalBuffer = resampledBuffer; // Float32Array
+    if (this.recordBitDepth !== this.exportBitDepth) {
+      if (this.exportBitDepth === 16) {
+        finalBuffer = this.floatTo16BitPCM(resampledBuffer);
+      } else if (this.exportBitDepth === 32) {
+        finalBuffer = this.floatTo32BitPCM(resampledBuffer);
+      }
+    }
+
+    const originalSize = mergedBuffer.byteLength;
+    const resampledSize = resampledBuffer.byteLength;
+    const finalSize = finalBuffer.byteLength;
+
+    // Clear the new recorded buffers after they have been processed
+    this.newRecBuffer.length = 0;
+
+    // Post the message to the main thread
+    // The first argument is the message data, containing the encoded WAV buffer
+    // The second argument is the transfer list, which transfers ownership of the ArrayBuffer
+    // to the main thread, avoiding the need to copy the buffer and improving performance
+    // this.port.postMessage({ recordedData: encodedWav.buffer, sampleRate: this.recordSampleRate }, [encodedWav.buffer]);
+    this.port.postMessage({
+      command: "newData",
+      recordedData: finalBuffer.buffer,
+      sampleRate: this.exportSampleRate,
+      bitDepth: this.exportBitDepth,
+    });
+  }
+
+  async getAllRecordedData() {
+    const length = this.recordedBuffers.reduce(
+      (acc, buffer) => acc + buffer.length,
+      0,
+    );
+    const mergedBuffer = this.mergeBuffers(this.recordedBuffers, length);
+    const resampledBuffer = await this.resampleBuffer(
+      mergedBuffer,
+      this.exportSampleRate,
+    );
+    // Convert to the desired bit depth if necessary
+    let finalBuffer = resampledBuffer;
+    if (this.recordBitDepth !== this.exportBitDepth) {
+      if (this.exportBitDepth === 16) {
+        finalBuffer = this.floatTo16BitPCM(resampledBuffer);
+      } else if (this.exportBitDepth === 32) {
+        finalBuffer = this.floatTo32BitPCM(resampledBuffer);
+      }
+    }
+
+    const originalSize = mergedBuffer.byteLength;
+    const resampledSize = resampledBuffer.byteLength;
+    const finalSize = finalBuffer.byteLength;
+
+    this.recordedBuffers.length = 0; // Clear the buffers after extraction
+
+    return finalBuffer.buffer;
+  }
+}
+
+registerProcessor("recorder-processor", RecorderProcessor);
+`;

package/build/ExpoAudioStreamModule.web.d.ts

@@ -1,37 +0,0 @@
-import { EventEmitter } from "expo-modules-core";
-import { AudioStreamResult, RecordingConfig, StartAudioStreamResult } from "./ExpoAudioStream.types";
-declare class ExpoAudioStreamWeb extends EventEmitter {
-    mediaRecorder: MediaRecorder | null;
-    audioChunks: Blob[];
-    isRecording: boolean;
-    isPaused: boolean;
-    recordingStartTime: number;
-    pausedTime: number;
-    currentDurationMs: number;
-    currentSize: number;
-    currentInterval: number;
-    lastEmittedSize: number;
-    lastEmittedTime: number;
-    streamUuid: string | null;
-    constructor();
-    getMediaStream(): Promise<MediaStream>;
-    startRecording(options?: RecordingConfig): Promise<StartAudioStreamResult>;
-    setupRecordingListeners(): void;
-    emitAudioEvent({ data, position }: {
-        data: Blob;
-        position: number;
-    }): void;
-    generateUUID(): string;
-    stopRecording(): Promise<AudioStreamResult | null>;
-    pauseRecording(): Promise<void>;
-    status(): {
-        isRecording: boolean;
-        isPaused: boolean;
-        duration: number;
-        size: number;
-        interval: number;
-    };
-}
-declare const _default: ExpoAudioStreamWeb;
-export default _default;
-//# sourceMappingURL=ExpoAudioStreamModule.web.d.ts.map

package/build/ExpoAudioStreamModule.web.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"ExpoAudioStreamModule.web.d.ts","sourceRoot":"","sources":["../src/ExpoAudioStreamModule.web.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EAEL,iBAAiB,EACjB,eAAe,EACf,sBAAsB,EACvB,MAAM,yBAAyB,CAAC;AAGjC,cAAM,kBAAmB,SAAQ,YAAY;IAC3C,aAAa,EAAE,aAAa,GAAG,IAAI,CAAC;IACpC,WAAW,EAAE,IAAI,EAAE,CAAC;IACpB,WAAW,EAAE,OAAO,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,UAAU,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;;IA4BpB,cAAc;IAUd,cAAc,CAAC,OAAO,GAAE,eAAoB;IAwBlD,uBAAuB;IA4BvB,cAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;QAAE,IAAI,EAAE,IAAI,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;IAiBnE,YAAY;IAUN,aAAa,IAAI,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAelD,cAAc;IAcpB,MAAM;;;;;;;CASP;;AAED,wBAAwC"}