@siteed/expo-audio-stream 1.0.2 → 1.0.3

package/src/AudioRecorder.provider.tsx

@@ -1,18 +1,46 @@
+// packages/expo-audio-stream/src/AudioRecorder.provider.tsx
 import React, { createContext, useContext } from "react";
 
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
 import {
-  UseAudioRecorderProps,
-  UseAudioRecorderState,
-  useAudioRecorder,
-} from "./useAudioRecording";
+  AudioRecordingResult,
+  RecordingConfig,
+  StartRecordingResult,
+} from "./ExpoAudioStream.types";
+import { UseAudioRecorderProps, useAudioRecorder } from "./useAudioRecorder";
 
-const AudioRecorderContext = createContext<UseAudioRecorderState>({
+export interface UseAudioRecorderState {
+  startRecording: (_: RecordingConfig) => Promise<StartRecordingResult>;
+  stopRecording: () => Promise<AudioRecordingResult | null>;
+  pauseRecording: () => void;
+  resumeRecording: () => void;
+  isRecording: boolean;
+  isPaused: boolean;
+  durationMs: number; // Duration of the recording
+  size: number; // Size in bytes of the recorded audio
+  analysisData?: AudioAnalysisData;
+}
+
+const initContext: UseAudioRecorderState = {
   isRecording: false,
   isPaused: false,
   durationMs: 0,
   size: 0,
-  // other properties filled on useAudioRecorder
-} as UseAudioRecorderState);
+  startRecording: async () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  stopRecording: async () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  pauseRecording: () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  resumeRecording: () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+};
+
+const AudioRecorderContext = createContext<UseAudioRecorderState>(initContext);
 
 interface AudioRecorderProviderProps {
   children: React.ReactNode;

package/src/ExpoAudioStream.types.ts

@@ -1,33 +1,8 @@
-export interface AudioEventPayload {
-  encoded?: string;
-  buffer?: ArrayBuffer;
-  fileUri: string;
-  lastEmittedSize: number;
-  position: number;
-  deltaSize: number;
-  totalSize: number;
-  mimeType: string;
-  streamUuid: string;
-}
-
-export interface AudioStreamResult {
-  fileUri: string;
-  webAudioUri?: string;
-  durationMs: number;
-  size: number;
-  mimeType: string;
-  channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
-}
-
-export interface StartAudioStreamResult {
-  fileUri: string;
-  mimeType: string;
-  channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
-}
+// packages/expo-audio-stream/src/ExpoAudioStream.types.ts
+import {
+  AudioAnalysisData,
+  AudioFeaturesOptions,
+} from "./AudioAnalysis/AudioAnalysis.types";
 
 export interface AudioStreamStatus {
   isRecording: boolean;
@@ -46,74 +21,41 @@ export interface AudioDataEvent {
   totalSize: number;
 }
 
-export interface AudioFeatures {
-  energy: number;
-  mfcc: number[];
-  rms: number;
-  minAmplitude: number;
-  maxAmplitude: number;
-  zcr: number;
-  spectralCentroid: number;
-  spectralFlatness: number;
-  spectralRolloff: number;
-  spectralBandwidth: number;
-  chromagram: number[];
-  tempo: number;
-  hnr: number;
+export interface AudioEventPayload {
+  encoded?: string;
+  buffer?: ArrayBuffer;
+  fileUri: string;
+  lastEmittedSize: number;
+  position: number;
+  deltaSize: number;
+  totalSize: number;
+  mimeType: string;
+  streamUuid: string;
 }
 
-export interface AudioFeaturesOptions {
-  energy?: boolean;
-  mfcc?: boolean;
-  rms?: boolean;
-  zcr?: boolean;
-  spectralCentroid?: boolean;
-  spectralFlatness?: boolean;
-  spectralRolloff?: boolean;
-  spectralBandwidth?: boolean;
-  chromagram?: boolean;
-  tempo?: boolean;
-  hnr?: boolean;
-}
+export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
+export type SampleRate = 16000 | 44100 | 48000;
+export type BitDepth = 8 | 16 | 32;
 
-export interface DataPoint {
-  id: number;
-  amplitude: number;
-  activeSpeech?: boolean;
-  dB?: number;
-  silent?: boolean;
-  features?: AudioFeatures;
-  startTime?: number;
-  endTime?: number;
-  // start / end position in bytes
-  startPosition?: number;
-  endPosition?: number;
-  // number of audio samples for this point (samples size depends on bit depth)
-  samples?: number;
-  // Id of the speaker for this point
-  speaker?: number;
+export interface AudioRecordingResult {
+  fileUri: string;
+  webAudioUri?: string;
+  durationMs: number;
+  size: number;
+  mimeType: string;
+  channels: number;
+  bitDepth: BitDepth;
+  sampleRate: SampleRate;
 }
 
-export interface AudioAnalysisData {
-  pointsPerSecond: number; // How many consolidated value per second
-  durationMs: number; // Duration of the audio in milliseconds
-  bitDepth: number; // Bit depth of the audio
-  samples: number; // Size of the audio in bytes
-  numberOfChannels: number; // Number of audio channels
-  sampleRate: number; // Sample rate of the audio
-  dataPoints: DataPoint[];
-  amplitudeRange: {
-    min: number;
-    max: number;
-  };
-  speakerChanges?: {
-    timestamp: number;
-    speaker: number;
-  }[];
+export interface StartRecordingResult {
+  fileUri: string;
+  mimeType: string;
+  channels?: number;
+  bitDepth?: BitDepth;
+  sampleRate?: SampleRate;
 }
 
-export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
-export type SampleRate = 16000 | 44100 | 48000;
 export interface RecordingConfig {
   sampleRate?: SampleRate; // Sample rate for recording
   channels?: 1 | 2; // 1 or 2 (MONO or STEREO)
@@ -121,15 +63,12 @@ export interface RecordingConfig {
   interval?: number; // Interval in milliseconds at which to emit recording data
 
   // Optional parameters for audio processing
-  //TODO remove maxRecentDataDuration - should be replaced by maxDataPoints to 100.
-  maxRecentDataDuration?: number; // Maximum duration of recent data to keep for processing (default is 10.0 seconds)
   enableProcessing?: boolean; // Boolean to enable/disable audio processing (default is false)
   pointsPerSecond?: number; // Number of data points to extract per second of audio (default is 1000)
   algorithm?: string; // Algorithm to use for extraction (default is "rms")
   features?: AudioFeaturesOptions; // Feature options to extract (default is empty)
 
   // Optional paramters from web
-
   onAudioStream?: (_: AudioDataEvent) => Promise<void>; // Callback function to handle audio stream
   onProcessingResult?: (_: AudioAnalysisData) => Promise<void>; // Callback function to handle processing results
 }

package/src/ExpoAudioStream.web.ts

@@ -1,17 +1,18 @@
 // src/ExpoAudioStreamModule.web.ts
-import debug from "debug";
 import { EventEmitter } from "expo-modules-core";
 
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
 import {
-  AudioAnalysisData,
   AudioEventPayload,
-  AudioStreamResult,
+  AudioRecordingResult,
   AudioStreamStatus,
+  BitDepth,
   RecordingConfig,
-  StartAudioStreamResult,
+  StartRecordingResult,
 } from "./ExpoAudioStream.types";
-import { WebRecorder } from "./WebRecorder";
-import { encodingToBitDepth } from "./utils";
+import { WebRecorder } from "./WebRecorder.web";
+import { getLogger } from "./logger";
+import { encodingToBitDepth } from "./utils/encodingToBitDepth";
 
 export interface EmitAudioEventProps {
   data: ArrayBuffer;
@@ -25,8 +26,8 @@ export interface ExpoAudioStreamWebProps {
   featuresExtratorUrl: string;
 }
 
-// const log = debug("expo-audio-stream:useAudioRecording");
-const log = console;
+const logger = getLogger("ExpoAudioStreamWeb");
+
 export class ExpoAudioStreamWeb extends EventEmitter {
   customRecorder: WebRecorder | null;
   audioChunks: ArrayBuffer[];
@@ -42,7 +43,7 @@ export class ExpoAudioStreamWeb extends EventEmitter {
   streamUuid: string | null;
   extension: "webm" | "wav" = "wav"; // Default extension is 'webm'
   recordingConfig?: RecordingConfig;
-  bitDepth: number; // Bit depth of the audio
+  bitDepth: BitDepth; // Bit depth of the audio
   audioWorkletUrl: string;
   featuresExtratorUrl: string;
 
@@ -118,7 +119,7 @@ export class ExpoAudioStreamWeb extends EventEmitter {
         this.lastEmittedSize = this.currentSize;
       },
       emitAudioAnalysisCallback: (audioAnalysisData: AudioAnalysisData) => {
-        console.log(`Emitted AudioAnalysis:`, audioAnalysisData);
+        logger.log(`Emitted AudioAnalysis:`, audioAnalysisData);
         this.emit("AudioAnalysis", audioAnalysisData);
       },
     });
@@ -127,7 +128,7 @@ export class ExpoAudioStreamWeb extends EventEmitter {
 
     // // Set a timer to stop recording after 5 seconds
     // setTimeout(() => {
-    //   console.log("AUTO Stopping recording");
+    //   logger.log("AUTO Stopping recording");
     //   this.customRecorder?.stopAndPlay();
     //   this.isRecording = false;
     // }, 3000);
@@ -140,7 +141,7 @@ export class ExpoAudioStreamWeb extends EventEmitter {
     this.lastEmittedTime = 0;
     this.streamUuid = Date.now().toString();
     const fileUri = `${this.streamUuid}.${this.extension}`;
-    const streamConfig: StartAudioStreamResult = {
+    const streamConfig: StartRecordingResult = {
       fileUri,
       mimeType: `audio/${this.extension}`,
       bitDepth: this.bitDepth,
@@ -167,14 +168,14 @@ export class ExpoAudioStreamWeb extends EventEmitter {
   }
 
   // Stop recording
-  async stopRecording(): Promise<AudioStreamResult | null> {
+  async stopRecording(): Promise<AudioRecordingResult | null> {
     if (this.customRecorder) {
       const fullPcmBuffer = await this.customRecorder.stop();
-      log.debug(`Stopped recording`, fullPcmBuffer);
+      logger.debug(`Stopped recording`, fullPcmBuffer);
     }
     this.isRecording = false;
     this.currentDurationMs = Date.now() - this.recordingStartTime;
-    const result: AudioStreamResult = {
+    const result: AudioRecordingResult = {
       fileUri: `${this.streamUuid}.${this.extension}`,
       bitDepth: this.bitDepth,
       channels: this.recordingConfig?.channels ?? 1,
@@ -194,7 +195,7 @@ export class ExpoAudioStreamWeb extends EventEmitter {
     }
 
     if (this.customRecorder) {
-      this.customRecorder.stop();
+      this.customRecorder.pause();
     }
     this.isPaused = true;
     this.pausedTime = Date.now();

package/src/ExpoAudioStreamModule.ts

@@ -9,8 +9,13 @@ import {
 let ExpoAudioStreamModule: any;
 
 if (Platform.OS === "web") {
+  let instance: ExpoAudioStreamWeb | null = null;
+
   ExpoAudioStreamModule = (webProps: ExpoAudioStreamWebProps) => {
-    return new ExpoAudioStreamWeb(webProps);
+    if (!instance) {
+      instance = new ExpoAudioStreamWeb(webProps);
+    }
+    return instance;
   };
 } else {
   ExpoAudioStreamModule = requireNativeModule("ExpoAudioStream");

package/src/{WebRecorder.ts → WebRecorder.web.ts}

@@ -1,10 +1,15 @@
 // src/WebRecorder.ts
-import { AudioAnalysisData, RecordingConfig } from "./ExpoAudioStream.types";
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
+import { RecordingConfig } from "./ExpoAudioStream.types";
 import {
   EmitAudioAnalysisFunction,
   EmitAudioEventFunction,
 } from "./ExpoAudioStream.web";
-import { encodingToBitDepth } from "./utils";
+import { getLogger } from "./logger";
+import { encodingToBitDepth } from "./utils/encodingToBitDepth";
+import { InlineFeaturesExtractor } from "./workers/InlineFeaturesExtractor.web";
+import { InlineAudioWebWorker } from "./workers/inlineAudioWebWorker.web";
+
 interface AudioWorkletEvent {
   data: {
     command: string;
@@ -25,12 +30,13 @@ const DEFAULT_WEB_POINTS_PER_SECOND = 10;
 const DEFAULT_WEB_INTERVAL = 500;
 const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1;
 
-// const log = debug("expo-audio-stream:WebRecorder");
-const log = console.log;
+const TAG = "WebRecorder";
+const logger = getLogger(TAG);
+
 export class WebRecorder {
   private audioContext: AudioContext;
   private audioWorkletNode!: AudioWorkletNode;
-  private featureExtractorWorker: Worker;
+  private featureExtractorWorker?: Worker;
   private source: MediaStreamAudioSourceNode;
   private audioWorkletUrl: string;
   private emitAudioEventCallback: EmitAudioEventFunction;
@@ -72,7 +78,7 @@ export class WebRecorder {
     const audioContextFormat = this.checkAudioContextFormat({
       sampleRate: this.audioContext.sampleRate,
     });
-    log("Initialized WebRecorder with config:", {
+    logger.debug("Initialized WebRecorder with config:", {
       sampleRate: audioContextFormat.sampleRate,
       bitDepth: audioContextFormat.bitDepth,
       numberOfChannels: audioContextFormat.numberOfChannels,
@@ -101,26 +107,22 @@ export class WebRecorder {
       speakerChanges: [],
     };
 
-    // Initialize the feature extractor worker
-    //TODO: create audio feature extractor from a Blob instead of url since we cannot include the url directly in the library
-    // We keep the url during dev and use the blob in production.
-    this.featureExtractorWorker = new Worker(
-      new URL(featuresExtratorUrl, window.location.href),
-    );
-    this.featureExtractorWorker.onmessage =
-      this.handleFeatureExtractorMessage.bind(this);
+    if (recordingConfig.enableProcessing) {
+      this.initFeatureExtractorWorker();
+    }
   }
 
   async init() {
     try {
-      // TODO: Use the inline processor script for the audio worklet if the script is not available
-      // const blob = new Blob([InlineProcessorScrippt], {
-      //   type: "application/javascript",
-      // });
-      // const url = URL.createObjectURL(blob);
-      // await this.audioContext.audioWorklet.addModule(url);
-      await this.audioContext.audioWorklet.addModule(this.audioWorkletUrl);
-
+      if (!this.audioWorkletUrl) {
+        const blob = new Blob([InlineAudioWebWorker], {
+          type: "application/javascript",
+        });
+        const url = URL.createObjectURL(blob);
+        await this.audioContext.audioWorklet.addModule(url);
+      } else {
+        await this.audioContext.audioWorklet.addModule(this.audioWorkletUrl);
+      }
       this.audioWorkletNode = new AudioWorkletNode(
         this.audioContext,
         "recorder-processor",
@@ -134,7 +136,7 @@ export class WebRecorder {
           return;
         }
         // Handle the audio blob (e.g., send it to the server or process it further)
-        log("Received audio blob from processor", event);
+        logger.debug("Received audio blob from processor", event);
         const pcmBuffer = event.data.recordedData;
 
         if (!pcmBuffer) {
@@ -154,7 +156,7 @@ export class WebRecorder {
         const otherDuration =
           pcmBuffer.byteLength /
           (otherSampleRate * (this.exportBitDepth / this.numberOfChannels)); // Calculate duration of the current buffer
-        log(
+        logger.debug(
           `sampleRate=${sampleRate} Duration: ${duration} -- otherSampleRate=${otherSampleRate} Other duration: ${otherDuration}`,
         );
 
@@ -164,7 +166,7 @@ export class WebRecorder {
         });
         this.position += duration; // Update position
 
-        this.featureExtractorWorker.postMessage(
+        this.featureExtractorWorker?.postMessage(
           {
             command: "process",
             channelData,
@@ -181,7 +183,7 @@ export class WebRecorder {
         );
       };
 
-      log(
+      logger.debug(
         `WebRecorder initialized -- recordSampleRate=${this.audioContext.sampleRate}`,
         this.config,
       );
@@ -200,10 +202,60 @@ export class WebRecorder {
       this.source.connect(this.audioWorkletNode);
       this.audioWorkletNode.connect(this.audioContext.destination);
     } catch (error) {
-      console.error("Failed to initialize WebRecorder", error);
+      console.error(`[${TAG}] Failed to initialize WebRecorder`, error);
+    }
+  }
+
+  initFeatureExtractorWorker(featuresExtratorUrl?: string) {
+    try {
+      if (featuresExtratorUrl) {
+        // Initialize the feature extractor worker
+        //TODO: create audio feature extractor from a Blob instead of url since we cannot include the url directly in the library
+        // We keep the url during dev and use the blob in production.
+        this.featureExtractorWorker = new Worker(
+          new URL(featuresExtratorUrl, window.location.href),
+        );
+        this.featureExtractorWorker.onmessage =
+          this.handleFeatureExtractorMessage.bind(this);
+        this.featureExtractorWorker.onerror = this.handleWorkerError.bind(this);
+      } else {
+        // Fallback to the inline worker if the URL is not provided
+        this.initFallbackWorker();
+      }
+    } catch (error) {
+      console.error(
+        `[${TAG}] Failed to initialize feature extractor worker`,
+        error,
+      );
+      this.initFallbackWorker();
     }
   }
 
+  initFallbackWorker() {
+    try {
+      const blob = new Blob([InlineFeaturesExtractor], {
+        type: "application/javascript",
+      });
+      const url = URL.createObjectURL(blob);
+      this.featureExtractorWorker = new Worker(url);
+      this.featureExtractorWorker.onmessage =
+        this.handleFeatureExtractorMessage.bind(this);
+      this.featureExtractorWorker.onerror = (error) => {
+        console.error(`[${TAG}] Default Inline worker failed`, error);
+      };
+      logger.log("Inline worker initialized successfully");
+    } catch (error) {
+      console.error(
+        `[${TAG}] Failed to initialize Inline Feature Extractor worker`,
+        error,
+      );
+    }
+  }
+
+  handleWorkerError(error: ErrorEvent) {
+    console.error(`[${TAG}] Feature extractor worker error:`, error);
+  }
+
   handleFeatureExtractorMessage(event: AudioFeaturesEvent) {
     if (event.data.command === "features") {
       const segmentResult = event.data.result;
@@ -227,8 +279,8 @@ export class WebRecorder {
         };
       }
       // Handle the extracted features (e.g., emit an event or log them)
-      log("features event segmentResult", segmentResult);
-      log("features event audioAnalysisData", this.audioAnalysisData);
+      logger.debug("features event segmentResult", segmentResult);
+      logger.debug("features event audioAnalysisData", this.audioAnalysisData);
       this.emitAudioAnalysisCallback(segmentResult);
     }
   }
@@ -272,10 +324,10 @@ export class WebRecorder {
                 rawPCMDataFull.byteLength /
                 (this.audioContext.sampleRate *
                   (this.exportBitDepth / this.numberOfChannels));
-              log(
+              logger.debug(
                 `Received recorded data -- Duration: ${duration} vs ${rawPCMDataFull.byteLength / this.audioContext.sampleRate} seconds`,
               );
-              log(
+              logger.debug(
                 `recordedData.length=${rawPCMDataFull.byteLength} vs transmittedData.length=${this.buffers[0].byteLength}`,
               );
 
@@ -330,9 +382,9 @@ export class WebRecorder {
       bufferSource.buffer = audioBuffer;
       bufferSource.connect(this.audioContext.destination);
       bufferSource.start();
-      log("Playing recorded data", recordedData);
+      logger.debug("Playing recorded data", recordedData);
     } catch (error) {
-      console.error(`Failed to play recorded data:`, error);
+      console.error(`[${TAG}] Failed to play recorded data:`, error);
     }
   }
 

package/src/constants.ts

@@ -0,0 +1,18 @@
+// packages/expo-audio-stream/src/constants.ts
+import { Platform } from "react-native";
+
+import { BitDepth, SampleRate } from "./ExpoAudioStream.types";
+
+export const isWeb = Platform.OS === "web";
+export const DEBUG_NAMESPACE = "expo-audio-stream";
+
+// Constants for identifying chunks in a WAV file
+export const RIFF_HEADER = 0x52494646; // "RIFF"
+export const WAVE_HEADER = 0x57415645; // "WAVE"
+export const FMT_CHUNK_ID = 0x666d7420; // "fmt "
+export const DATA_CHUNK_ID = 0x64617461; // "data"
+export const INFO_CHUNK_ID = 0x494e464f; // "INFO"
+
+// Default values
+export const DEFAULT_SAMPLE_RATE: SampleRate = 16000;
+export const DEFAULT_BIT_DEPTH: BitDepth = 32;

package/src/events.ts

@@ -0,0 +1,25 @@
+// packages/expo-audio-stream/src/events.ts
+
+import { EventEmitter, type Subscription } from "expo-modules-core";
+
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
+import { AudioEventPayload } from "./ExpoAudioStream.types";
+import ExpoAudioStreamModule from "./ExpoAudioStreamModule";
+import { getLogger } from "./logger";
+
+const emitter = new EventEmitter(ExpoAudioStreamModule);
+const logger = getLogger("events");
+
+export function addAudioEventListener(
+  listener: (event: AudioEventPayload) => Promise<void>,
+): Subscription {
+  logger.log("Adding listener for AudioData event");
+  return emitter.addListener<AudioEventPayload>("AudioData", listener);
+}
+
+export function addAudioAnalysisListener(
+  listener: (event: AudioAnalysisData) => Promise<void>,
+): Subscription {
+  logger.log("Adding listener for AudioAnalysis event");
+  return emitter.addListener<AudioAnalysisData>("AudioAnalysis", listener);
+}