@siteed/expo-audio-stream 1.0.1 → 1.0.2

package/src/ExpoAudioStream.types.ts

@@ -1,8 +1,6 @@
-import { AudioDataEvent } from "./useAudioRecording";
-
 export interface AudioEventPayload {
   encoded?: string;
-  buffer?: Blob;
+  buffer?: ArrayBuffer;
   fileUri: string;
   lastEmittedSize: number;
   position: number;
@@ -14,7 +12,8 @@ export interface AudioEventPayload {
 
 export interface AudioStreamResult {
   fileUri: string;
-  duration: number;
+  webAudioUri?: string;
+  durationMs: number;
   size: number;
   mimeType: string;
   channels?: number;
@@ -33,17 +32,104 @@ export interface StartAudioStreamResult {
 export interface AudioStreamStatus {
   isRecording: boolean;
   isPaused: boolean;
-  duration: number;
+  durationMs: number;
   size: number;
   interval: number;
   mimeType: string;
 }
 
-export type EncodingType = "pcm_16bit" | "pcm_8bit";
+export interface AudioDataEvent {
+  data: string | ArrayBuffer;
+  position: number;
+  fileUri: string;
+  eventDataSize: number;
+  totalSize: number;
+}
+
+export interface AudioFeatures {
+  energy: number;
+  mfcc: number[];
+  rms: number;
+  minAmplitude: number;
+  maxAmplitude: number;
+  zcr: number;
+  spectralCentroid: number;
+  spectralFlatness: number;
+  spectralRolloff: number;
+  spectralBandwidth: number;
+  chromagram: number[];
+  tempo: number;
+  hnr: number;
+}
+
+export interface AudioFeaturesOptions {
+  energy?: boolean;
+  mfcc?: boolean;
+  rms?: boolean;
+  zcr?: boolean;
+  spectralCentroid?: boolean;
+  spectralFlatness?: boolean;
+  spectralRolloff?: boolean;
+  spectralBandwidth?: boolean;
+  chromagram?: boolean;
+  tempo?: boolean;
+  hnr?: boolean;
+}
+
+export interface DataPoint {
+  id: number;
+  amplitude: number;
+  activeSpeech?: boolean;
+  dB?: number;
+  silent?: boolean;
+  features?: AudioFeatures;
+  startTime?: number;
+  endTime?: number;
+  // start / end position in bytes
+  startPosition?: number;
+  endPosition?: number;
+  // number of audio samples for this point (samples size depends on bit depth)
+  samples?: number;
+  // Id of the speaker for this point
+  speaker?: number;
+}
+
+export interface AudioAnalysisData {
+  pointsPerSecond: number; // How many consolidated value per second
+  durationMs: number; // Duration of the audio in milliseconds
+  bitDepth: number; // Bit depth of the audio
+  samples: number; // Size of the audio in bytes
+  numberOfChannels: number; // Number of audio channels
+  sampleRate: number; // Sample rate of the audio
+  dataPoints: DataPoint[];
+  amplitudeRange: {
+    min: number;
+    max: number;
+  };
+  speakerChanges?: {
+    timestamp: number;
+    speaker: number;
+  }[];
+}
+
+export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
+export type SampleRate = 16000 | 44100 | 48000;
 export interface RecordingConfig {
-  sampleRate?: 16000 | 44100 | 48000;
-  channels?: 1 | 2; // 1 or 2 MONO or STEREO
-  encoding?: EncodingType;
-  interval?: number;
-  onAudioStream?: (_: AudioDataEvent) => Promise<void>;
+  sampleRate?: SampleRate; // Sample rate for recording
+  channels?: 1 | 2; // 1 or 2 (MONO or STEREO)
+  encoding?: EncodingType; // Encoding type for the recording
+  interval?: number; // Interval in milliseconds at which to emit recording data
+
+  // Optional parameters for audio processing
+  //TODO remove maxRecentDataDuration - should be replaced by maxDataPoints to 100.
+  maxRecentDataDuration?: number; // Maximum duration of recent data to keep for processing (default is 10.0 seconds)
+  enableProcessing?: boolean; // Boolean to enable/disable audio processing (default is false)
+  pointsPerSecond?: number; // Number of data points to extract per second of audio (default is 1000)
+  algorithm?: string; // Algorithm to use for extraction (default is "rms")
+  features?: AudioFeaturesOptions; // Feature options to extract (default is empty)
+
+  // Optional paramters from web
+
+  onAudioStream?: (_: AudioDataEvent) => Promise<void>; // Callback function to handle audio stream
+  onProcessingResult?: (_: AudioAnalysisData) => Promise<void>; // Callback function to handle processing results
 }

package/src/ExpoAudioStream.web.ts

@@ -0,0 +1,228 @@
+// src/ExpoAudioStreamModule.web.ts
+import debug from "debug";
+import { EventEmitter } from "expo-modules-core";
+
+import {
+  AudioAnalysisData,
+  AudioEventPayload,
+  AudioStreamResult,
+  AudioStreamStatus,
+  RecordingConfig,
+  StartAudioStreamResult,
+} from "./ExpoAudioStream.types";
+import { WebRecorder } from "./WebRecorder";
+import { encodingToBitDepth } from "./utils";
+
+export interface EmitAudioEventProps {
+  data: ArrayBuffer;
+  position: number;
+}
+export type EmitAudioEventFunction = (_: EmitAudioEventProps) => void;
+export type EmitAudioAnalysisFunction = (_: AudioAnalysisData) => void;
+
+export interface ExpoAudioStreamWebProps {
+  audioWorkletUrl: string;
+  featuresExtratorUrl: string;
+}
+
+// const log = debug("expo-audio-stream:useAudioRecording");
+const log = console;
+export class ExpoAudioStreamWeb extends EventEmitter {
+  customRecorder: WebRecorder | null;
+  audioChunks: ArrayBuffer[];
+  isRecording: boolean;
+  isPaused: boolean;
+  recordingStartTime: number;
+  pausedTime: number;
+  currentDurationMs: number;
+  currentSize: number;
+  currentInterval: number;
+  lastEmittedSize: number;
+  lastEmittedTime: number;
+  streamUuid: string | null;
+  extension: "webm" | "wav" = "wav"; // Default extension is 'webm'
+  recordingConfig?: RecordingConfig;
+  bitDepth: number; // Bit depth of the audio
+  audioWorkletUrl: string;
+  featuresExtratorUrl: string;
+
+  constructor({
+    audioWorkletUrl,
+    featuresExtratorUrl,
+  }: ExpoAudioStreamWebProps) {
+    const mockNativeModule = {
+      addListener: (eventName: string) => {
+        // Not used on web
+      },
+      removeListeners: (count: number) => {
+        // Not used on web
+      },
+    };
+    super(mockNativeModule); // Pass the mock native module to the parent class
+
+    this.customRecorder = null;
+    this.audioChunks = [];
+    this.isRecording = false;
+    this.isPaused = false;
+    this.recordingStartTime = 0;
+    this.pausedTime = 0;
+    this.currentDurationMs = 0;
+    this.currentSize = 0;
+    this.bitDepth = 32; // Default
+    this.currentInterval = 1000; // Default interval in ms
+    this.lastEmittedSize = 0;
+    this.lastEmittedTime = 0;
+    this.streamUuid = null; // Initialize UUID on first recording start
+    this.audioWorkletUrl = audioWorkletUrl;
+    this.featuresExtratorUrl = featuresExtratorUrl;
+  }
+
+  // Utility to handle user media stream
+  async getMediaStream() {
+    try {
+      return await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch (error) {
+      console.error("Failed to get media stream:", error);
+      throw error;
+    }
+  }
+
+  // Start recording with options
+  async startRecording(recordingConfig: RecordingConfig = {}) {
+    if (this.isRecording) {
+      throw new Error("Recording is already in progress");
+    }
+
+    this.bitDepth = encodingToBitDepth({
+      encoding: recordingConfig.encoding ?? "pcm_32bit",
+    });
+
+    const audioContext = new (window.AudioContext ||
+      // @ts-ignore - Allow webkitAudioContext for Safari
+      window.webkitAudioContext)();
+    const stream = await this.getMediaStream();
+
+    const source = audioContext.createMediaStreamSource(stream);
+
+    this.customRecorder = new WebRecorder({
+      audioContext,
+      source,
+      recordingConfig,
+      audioWorkletUrl: this.audioWorkletUrl,
+      featuresExtratorUrl: this.featuresExtratorUrl,
+      emitAudioEventCallback: ({ data, position }: EmitAudioEventProps) => {
+        this.audioChunks.push(data);
+        this.currentSize += data.byteLength;
+        this.emitAudioEvent({ data, position });
+        this.lastEmittedTime = Date.now();
+        this.lastEmittedSize = this.currentSize;
+      },
+      emitAudioAnalysisCallback: (audioAnalysisData: AudioAnalysisData) => {
+        console.log(`Emitted AudioAnalysis:`, audioAnalysisData);
+        this.emit("AudioAnalysis", audioAnalysisData);
+      },
+    });
+    await this.customRecorder.init();
+    this.customRecorder.start();
+
+    // // Set a timer to stop recording after 5 seconds
+    // setTimeout(() => {
+    //   console.log("AUTO Stopping recording");
+    //   this.customRecorder?.stopAndPlay();
+    //   this.isRecording = false;
+    // }, 3000);
+
+    this.isRecording = true;
+    this.recordingConfig = recordingConfig;
+    this.recordingStartTime = Date.now();
+    this.pausedTime = 0;
+    this.lastEmittedSize = 0;
+    this.lastEmittedTime = 0;
+    this.streamUuid = Date.now().toString();
+    const fileUri = `${this.streamUuid}.${this.extension}`;
+    const streamConfig: StartAudioStreamResult = {
+      fileUri,
+      mimeType: `audio/${this.extension}`,
+      bitDepth: this.bitDepth,
+      channels: recordingConfig.channels ?? 1,
+      sampleRate: recordingConfig.sampleRate ?? 44100,
+    };
+    return streamConfig;
+  }
+
+  emitAudioEvent({ data, position }: EmitAudioEventProps) {
+    const fileUri = `${this.streamUuid}.${this.extension}`;
+    const audioEventPayload: AudioEventPayload = {
+      fileUri,
+      mimeType: `audio/${this.extension}`,
+      lastEmittedSize: this.lastEmittedSize, // Since this might be continuously streaming, adjust accordingly
+      deltaSize: data.byteLength,
+      position,
+      totalSize: this.currentSize,
+      buffer: data,
+      streamUuid: this.streamUuid ?? "", // Generate or manage UUID for stream identification
+    };
+
+    this.emit("AudioData", audioEventPayload);
+  }
+
+  // Stop recording
+  async stopRecording(): Promise<AudioStreamResult | null> {
+    if (this.customRecorder) {
+      const fullPcmBuffer = await this.customRecorder.stop();
+      log.debug(`Stopped recording`, fullPcmBuffer);
+    }
+    this.isRecording = false;
+    this.currentDurationMs = Date.now() - this.recordingStartTime;
+    const result: AudioStreamResult = {
+      fileUri: `${this.streamUuid}.${this.extension}`,
+      bitDepth: this.bitDepth,
+      channels: this.recordingConfig?.channels ?? 1,
+      sampleRate: this.recordingConfig?.sampleRate ?? 44100,
+      durationMs: this.currentDurationMs,
+      size: this.currentSize,
+      mimeType: `audio/${this.extension}`,
+    };
+
+    return result;
+  }
+
+  // Pause recording
+  async pauseRecording() {
+    if (!this.isRecording || this.isPaused) {
+      throw new Error("Recording is not active or already paused");
+    }
+
+    if (this.customRecorder) {
+      this.customRecorder.stop();
+    }
+    this.isPaused = true;
+    this.pausedTime = Date.now();
+  }
+
+  // Resume recording
+  async resumeRecording() {
+    if (!this.isPaused) {
+      throw new Error("Recording is not paused");
+    }
+
+    if (this.customRecorder) {
+      this.customRecorder.resume();
+    }
+    this.isPaused = false;
+    this.recordingStartTime += Date.now() - this.pausedTime;
+  }
+
+  // Get current status
+  status() {
+    const status: AudioStreamStatus = {
+      isRecording: this.isRecording,
+      isPaused: this.isPaused,
+      durationMs: Date.now() - this.recordingStartTime,
+      size: this.currentSize,
+      interval: this.currentInterval,
+      mimeType: `audio/${this.extension}`,
+    };
+    return status;
+  }
+}

package/src/ExpoAudioStreamModule.ts

@@ -1,5 +1,19 @@
 import { requireNativeModule } from "expo-modules-core";
+import { Platform } from "react-native";
 
-// It loads the native module object from the JSI or falls back to
-// the bridge module (from NativeModulesProxy) if the remote debugger is on.
-export default requireNativeModule("ExpoAudioStream");
+import {
+  ExpoAudioStreamWeb,
+  ExpoAudioStreamWebProps,
+} from "./ExpoAudioStream.web";
+
+let ExpoAudioStreamModule: any;
+
+if (Platform.OS === "web") {
+  ExpoAudioStreamModule = (webProps: ExpoAudioStreamWebProps) => {
+    return new ExpoAudioStreamWeb(webProps);
+  };
+} else {
+  ExpoAudioStreamModule = requireNativeModule("ExpoAudioStream");
+}
+
+export default ExpoAudioStreamModule;