@siteed/expo-audio-stream 1.0.1 → 1.0.3

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -0,0 +1,136 @@
+// packages/expo-audio-stream/src/AudioAnalysis/extractAudioAnalysis.ts
+import { AudioAnalysisData } from "./AudioAnalysis.types";
+import ExpoAudioStreamModule from "../ExpoAudioStreamModule";
+import { isWeb } from "../constants";
+import { getLogger } from "../logger";
+import { ExtractMetadataProps } from "../useAudioRecorder";
+import { convertPCMToFloat32 } from "../utils/convertPCMToFloat32";
+import { getWavFileInfo } from "../utils/getWavFileInfo";
+import { InlineFeaturesExtractor } from "../workers/InlineFeaturesExtractor.web";
+
+const logger = getLogger("extractAudioAnalysis");
+
+export const extractAudioAnalysis = async ({
+  fileUri,
+  pointsPerSecond = 20,
+  arrayBuffer,
+  bitDepth,
+  skipWavHeader,
+  durationMs,
+  sampleRate,
+  numberOfChannels,
+  algorithm = "rms",
+  features,
+  featuresExtratorUrl,
+}: ExtractMetadataProps): Promise<AudioAnalysisData> => {
+  if (isWeb) {
+    if (!arrayBuffer && !fileUri) {
+      throw new Error("Either arrayBuffer or fileUri must be provided");
+    }
+
+    if (!arrayBuffer) {
+      logger.log(`fetching fileUri`, fileUri);
+      const response = await fetch(fileUri!);
+
+      if (!response.ok) {
+        throw new Error(`Failed to fetch fileUri: ${response.statusText}`);
+      }
+
+      arrayBuffer = (await response.arrayBuffer()).slice(0);
+      logger.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer);
+    }
+
+    // Create a new copy of the ArrayBuffer to avoid detachment issues
+    const bufferCopy = arrayBuffer.slice(0);
+    logger.log(
+      `extractAudioAnalysis skipWavHeader=${skipWavHeader} bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
+      bufferCopy.slice(0, 100),
+    );
+
+    let actualBitDepth = bitDepth;
+    if (!actualBitDepth) {
+      logger.log(
+        `extractAudioAnalysis bitDepth not provided -- getting wav file info`,
+      );
+      const fileInfo = await getWavFileInfo(bufferCopy);
+      actualBitDepth = fileInfo.bitDepth;
+    }
+    logger.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`);
+    // let copyChannelData: Float32Array;
+    // try {
+    //   const audioContext = new (window.AudioContext ||
+    //     // @ts-ignore
+    //     window.webkitAudioContext)();
+    //   const audioBuffer = await audioContext.decodeAudioData(bufferCopy);
+    //   const channelData = audioBuffer.getChannelData(0); // Use only the first channel
+    //   copyChannelData = new Float32Array(channelData); // Create a new Float32Array
+    // } catch (error) {
+    //   console.warn("Failed to decode audio data:", error);
+    //   // Fall back to creating a new Float32Array from the ArrayBuffer if decoding fails
+    //   copyChannelData = new Float32Array(arrayBuffer);
+    // }
+
+    const {
+      pcmValues: channelData,
+      min,
+      max,
+    } = convertPCMToFloat32({
+      buffer: arrayBuffer,
+      bitDepth: actualBitDepth,
+      skipWavHeader,
+    });
+    logger.log(
+      `extractAudioAnalysis skipWaveHeader=${skipWavHeader} convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`,
+    );
+
+    return new Promise((resolve, reject) => {
+      let worker: Worker;
+      if (featuresExtratorUrl) {
+        worker = new Worker(new URL(featuresExtratorUrl, window.location.href));
+      } else {
+        const blob = new Blob([InlineFeaturesExtractor], {
+          type: "application/javascript",
+        });
+        const url = URL.createObjectURL(blob);
+        worker = new Worker(url);
+      }
+
+      worker.onmessage = (event) => {
+        resolve(event.data.result);
+      };
+
+      worker.onerror = (error) => {
+        reject(error);
+      };
+
+      worker.postMessage({
+        command: "process",
+        channelData,
+        sampleRate,
+        pointsPerSecond,
+        algorithm,
+        bitDepth,
+        fullAudioDurationMs: durationMs,
+        numberOfChannels,
+      });
+    });
+  } else {
+    if (!fileUri) {
+      throw new Error("fileUri is required");
+    }
+    logger.log(`extractAudioAnalysis`, {
+      fileUri,
+      pointsPerSecond,
+      algorithm,
+    });
+    const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+      fileUri,
+      pointsPerSecond,
+      skipWavHeader,
+      algorithm,
+      features,
+    });
+    logger.log(`extractAudioAnalysis`, res);
+    return res;
+  }
+};

package/src/AudioAnalysis/extractWaveform.ts

@@ -0,0 +1,25 @@
+import ExpoAudioStreamModule from "../ExpoAudioStreamModule";
+import { getLogger } from "../logger";
+
+const logger = getLogger("extractWaveform");
+export interface ExtractWaveformProps {
+  fileUri: string;
+  numberOfSamples: number;
+  offset?: number;
+  length?: number;
+}
+export const extractWaveform = async ({
+  fileUri,
+  numberOfSamples,
+  offset = 0,
+  length,
+}: ExtractWaveformProps): Promise<unknown> => {
+  const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+    fileUri,
+    numberOfSamples,
+    offset,
+    length,
+  });
+  logger.log(`extractWaveform`, res);
+  return res;
+};

package/src/AudioRecorder.provider.tsx

@@ -1,18 +1,46 @@
+// packages/expo-audio-stream/src/AudioRecorder.provider.tsx
 import React, { createContext, useContext } from "react";
 
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
 import {
-  UseAudioRecorderProps,
-  UseAudioRecorderState,
-  useAudioRecorder,
-} from "./useAudioRecording";
+  AudioRecordingResult,
+  RecordingConfig,
+  StartRecordingResult,
+} from "./ExpoAudioStream.types";
+import { UseAudioRecorderProps, useAudioRecorder } from "./useAudioRecorder";
 
-const AudioRecorderContext = createContext<UseAudioRecorderState>({
+export interface UseAudioRecorderState {
+  startRecording: (_: RecordingConfig) => Promise<StartRecordingResult>;
+  stopRecording: () => Promise<AudioRecordingResult | null>;
+  pauseRecording: () => void;
+  resumeRecording: () => void;
+  isRecording: boolean;
+  isPaused: boolean;
+  durationMs: number; // Duration of the recording
+  size: number; // Size in bytes of the recorded audio
+  analysisData?: AudioAnalysisData;
+}
+
+const initContext: UseAudioRecorderState = {
   isRecording: false,
   isPaused: false,
-  duration: 0,
+  durationMs: 0,
   size: 0,
-  // other properties filled on useAudioRecorder
-} as UseAudioRecorderState);
+  startRecording: async () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  stopRecording: async () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  pauseRecording: () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+  resumeRecording: () => {
+    throw new Error("AudioRecorderProvider not found");
+  },
+};
+
+const AudioRecorderContext = createContext<UseAudioRecorderState>(initContext);
 
 interface AudioRecorderProviderProps {
   children: React.ReactNode;

package/src/ExpoAudioStream.native.ts

@@ -0,0 +1,6 @@
+// src/ExpoAudioStreamModule.ts
+import { requireNativeModule } from "expo-modules-core";
+
+// It loads the native module object from the JSI or falls back to
+// the bridge module (from NativeModulesProxy) if the remote debugger is on.
+export default requireNativeModule("ExpoAudioStream");

package/src/ExpoAudioStream.types.ts

@@ -1,8 +1,29 @@
-import { AudioDataEvent } from "./useAudioRecording";
+// packages/expo-audio-stream/src/ExpoAudioStream.types.ts
+import {
+  AudioAnalysisData,
+  AudioFeaturesOptions,
+} from "./AudioAnalysis/AudioAnalysis.types";
+
+export interface AudioStreamStatus {
+  isRecording: boolean;
+  isPaused: boolean;
+  durationMs: number;
+  size: number;
+  interval: number;
+  mimeType: string;
+}
+
+export interface AudioDataEvent {
+  data: string | ArrayBuffer;
+  position: number;
+  fileUri: string;
+  eventDataSize: number;
+  totalSize: number;
+}
 
 export interface AudioEventPayload {
   encoded?: string;
-  buffer?: Blob;
+  buffer?: ArrayBuffer;
   fileUri: string;
   lastEmittedSize: number;
   position: number;
@@ -12,38 +33,42 @@ export interface AudioEventPayload {
   streamUuid: string;
 }
 
-export interface AudioStreamResult {
+export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
+export type SampleRate = 16000 | 44100 | 48000;
+export type BitDepth = 8 | 16 | 32;
+
+export interface AudioRecordingResult {
   fileUri: string;
-  duration: number;
+  webAudioUri?: string;
+  durationMs: number;
   size: number;
   mimeType: string;
-  channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
+  channels: number;
+  bitDepth: BitDepth;
+  sampleRate: SampleRate;
 }
 
-export interface StartAudioStreamResult {
+export interface StartRecordingResult {
   fileUri: string;
   mimeType: string;
   channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
-}
-
-export interface AudioStreamStatus {
-  isRecording: boolean;
-  isPaused: boolean;
-  duration: number;
-  size: number;
-  interval: number;
-  mimeType: string;
+  bitDepth?: BitDepth;
+  sampleRate?: SampleRate;
 }
 
-export type EncodingType = "pcm_16bit" | "pcm_8bit";
 export interface RecordingConfig {
-  sampleRate?: 16000 | 44100 | 48000;
-  channels?: 1 | 2; // 1 or 2 MONO or STEREO
-  encoding?: EncodingType;
-  interval?: number;
-  onAudioStream?: (_: AudioDataEvent) => Promise<void>;
+  sampleRate?: SampleRate; // Sample rate for recording
+  channels?: 1 | 2; // 1 or 2 (MONO or STEREO)
+  encoding?: EncodingType; // Encoding type for the recording
+  interval?: number; // Interval in milliseconds at which to emit recording data
+
+  // Optional parameters for audio processing
+  enableProcessing?: boolean; // Boolean to enable/disable audio processing (default is false)
+  pointsPerSecond?: number; // Number of data points to extract per second of audio (default is 1000)
+  algorithm?: string; // Algorithm to use for extraction (default is "rms")
+  features?: AudioFeaturesOptions; // Feature options to extract (default is empty)
+
+  // Optional paramters from web
+  onAudioStream?: (_: AudioDataEvent) => Promise<void>; // Callback function to handle audio stream
+  onProcessingResult?: (_: AudioAnalysisData) => Promise<void>; // Callback function to handle processing results
 }

package/src/ExpoAudioStream.web.ts

@@ -0,0 +1,229 @@
+// src/ExpoAudioStreamModule.web.ts
+import { EventEmitter } from "expo-modules-core";
+
+import { AudioAnalysisData } from "./AudioAnalysis/AudioAnalysis.types";
+import {
+  AudioEventPayload,
+  AudioRecordingResult,
+  AudioStreamStatus,
+  BitDepth,
+  RecordingConfig,
+  StartRecordingResult,
+} from "./ExpoAudioStream.types";
+import { WebRecorder } from "./WebRecorder.web";
+import { getLogger } from "./logger";
+import { encodingToBitDepth } from "./utils/encodingToBitDepth";
+
+export interface EmitAudioEventProps {
+  data: ArrayBuffer;
+  position: number;
+}
+export type EmitAudioEventFunction = (_: EmitAudioEventProps) => void;
+export type EmitAudioAnalysisFunction = (_: AudioAnalysisData) => void;
+
+export interface ExpoAudioStreamWebProps {
+  audioWorkletUrl: string;
+  featuresExtratorUrl: string;
+}
+
+const logger = getLogger("ExpoAudioStreamWeb");
+
+export class ExpoAudioStreamWeb extends EventEmitter {
+  customRecorder: WebRecorder | null;
+  audioChunks: ArrayBuffer[];
+  isRecording: boolean;
+  isPaused: boolean;
+  recordingStartTime: number;
+  pausedTime: number;
+  currentDurationMs: number;
+  currentSize: number;
+  currentInterval: number;
+  lastEmittedSize: number;
+  lastEmittedTime: number;
+  streamUuid: string | null;
+  extension: "webm" | "wav" = "wav"; // Default extension is 'webm'
+  recordingConfig?: RecordingConfig;
+  bitDepth: BitDepth; // Bit depth of the audio
+  audioWorkletUrl: string;
+  featuresExtratorUrl: string;
+
+  constructor({
+    audioWorkletUrl,
+    featuresExtratorUrl,
+  }: ExpoAudioStreamWebProps) {
+    const mockNativeModule = {
+      addListener: (eventName: string) => {
+        // Not used on web
+      },
+      removeListeners: (count: number) => {
+        // Not used on web
+      },
+    };
+    super(mockNativeModule); // Pass the mock native module to the parent class
+
+    this.customRecorder = null;
+    this.audioChunks = [];
+    this.isRecording = false;
+    this.isPaused = false;
+    this.recordingStartTime = 0;
+    this.pausedTime = 0;
+    this.currentDurationMs = 0;
+    this.currentSize = 0;
+    this.bitDepth = 32; // Default
+    this.currentInterval = 1000; // Default interval in ms
+    this.lastEmittedSize = 0;
+    this.lastEmittedTime = 0;
+    this.streamUuid = null; // Initialize UUID on first recording start
+    this.audioWorkletUrl = audioWorkletUrl;
+    this.featuresExtratorUrl = featuresExtratorUrl;
+  }
+
+  // Utility to handle user media stream
+  async getMediaStream() {
+    try {
+      return await navigator.mediaDevices.getUserMedia({ audio: true });
+    } catch (error) {
+      console.error("Failed to get media stream:", error);
+      throw error;
+    }
+  }
+
+  // Start recording with options
+  async startRecording(recordingConfig: RecordingConfig = {}) {
+    if (this.isRecording) {
+      throw new Error("Recording is already in progress");
+    }
+
+    this.bitDepth = encodingToBitDepth({
+      encoding: recordingConfig.encoding ?? "pcm_32bit",
+    });
+
+    const audioContext = new (window.AudioContext ||
+      // @ts-ignore - Allow webkitAudioContext for Safari
+      window.webkitAudioContext)();
+    const stream = await this.getMediaStream();
+
+    const source = audioContext.createMediaStreamSource(stream);
+
+    this.customRecorder = new WebRecorder({
+      audioContext,
+      source,
+      recordingConfig,
+      audioWorkletUrl: this.audioWorkletUrl,
+      featuresExtratorUrl: this.featuresExtratorUrl,
+      emitAudioEventCallback: ({ data, position }: EmitAudioEventProps) => {
+        this.audioChunks.push(data);
+        this.currentSize += data.byteLength;
+        this.emitAudioEvent({ data, position });
+        this.lastEmittedTime = Date.now();
+        this.lastEmittedSize = this.currentSize;
+      },
+      emitAudioAnalysisCallback: (audioAnalysisData: AudioAnalysisData) => {
+        logger.log(`Emitted AudioAnalysis:`, audioAnalysisData);
+        this.emit("AudioAnalysis", audioAnalysisData);
+      },
+    });
+    await this.customRecorder.init();
+    this.customRecorder.start();
+
+    // // Set a timer to stop recording after 5 seconds
+    // setTimeout(() => {
+    //   logger.log("AUTO Stopping recording");
+    //   this.customRecorder?.stopAndPlay();
+    //   this.isRecording = false;
+    // }, 3000);
+
+    this.isRecording = true;
+    this.recordingConfig = recordingConfig;
+    this.recordingStartTime = Date.now();
+    this.pausedTime = 0;
+    this.lastEmittedSize = 0;
+    this.lastEmittedTime = 0;
+    this.streamUuid = Date.now().toString();
+    const fileUri = `${this.streamUuid}.${this.extension}`;
+    const streamConfig: StartRecordingResult = {
+      fileUri,
+      mimeType: `audio/${this.extension}`,
+      bitDepth: this.bitDepth,
+      channels: recordingConfig.channels ?? 1,
+      sampleRate: recordingConfig.sampleRate ?? 44100,
+    };
+    return streamConfig;
+  }
+
+  emitAudioEvent({ data, position }: EmitAudioEventProps) {
+    const fileUri = `${this.streamUuid}.${this.extension}`;
+    const audioEventPayload: AudioEventPayload = {
+      fileUri,
+      mimeType: `audio/${this.extension}`,
+      lastEmittedSize: this.lastEmittedSize, // Since this might be continuously streaming, adjust accordingly
+      deltaSize: data.byteLength,
+      position,
+      totalSize: this.currentSize,
+      buffer: data,
+      streamUuid: this.streamUuid ?? "", // Generate or manage UUID for stream identification
+    };
+
+    this.emit("AudioData", audioEventPayload);
+  }
+
+  // Stop recording
+  async stopRecording(): Promise<AudioRecordingResult | null> {
+    if (this.customRecorder) {
+      const fullPcmBuffer = await this.customRecorder.stop();
+      logger.debug(`Stopped recording`, fullPcmBuffer);
+    }
+    this.isRecording = false;
+    this.currentDurationMs = Date.now() - this.recordingStartTime;
+    const result: AudioRecordingResult = {
+      fileUri: `${this.streamUuid}.${this.extension}`,
+      bitDepth: this.bitDepth,
+      channels: this.recordingConfig?.channels ?? 1,
+      sampleRate: this.recordingConfig?.sampleRate ?? 44100,
+      durationMs: this.currentDurationMs,
+      size: this.currentSize,
+      mimeType: `audio/${this.extension}`,
+    };
+
+    return result;
+  }
+
+  // Pause recording
+  async pauseRecording() {
+    if (!this.isRecording || this.isPaused) {
+      throw new Error("Recording is not active or already paused");
+    }
+
+    if (this.customRecorder) {
+      this.customRecorder.pause();
+    }
+    this.isPaused = true;
+    this.pausedTime = Date.now();
+  }
+
+  // Resume recording
+  async resumeRecording() {
+    if (!this.isPaused) {
+      throw new Error("Recording is not paused");
+    }
+
+    if (this.customRecorder) {
+      this.customRecorder.resume();
+    }
+    this.isPaused = false;
+    this.recordingStartTime += Date.now() - this.pausedTime;
+  }
+
+  // Get current status
+  status() {
+    const status: AudioStreamStatus = {
+      isRecording: this.isRecording,
+      isPaused: this.isPaused,
+      durationMs: Date.now() - this.recordingStartTime,
+      size: this.currentSize,
+      interval: this.currentInterval,
+      mimeType: `audio/${this.extension}`,
+    };
+    return status;
+  }
+}

package/src/ExpoAudioStreamModule.ts

@@ -1,5 +1,24 @@
 import { requireNativeModule } from "expo-modules-core";
+import { Platform } from "react-native";
 
-// It loads the native module object from the JSI or falls back to
-// the bridge module (from NativeModulesProxy) if the remote debugger is on.
-export default requireNativeModule("ExpoAudioStream");
+import {
+  ExpoAudioStreamWeb,
+  ExpoAudioStreamWebProps,
+} from "./ExpoAudioStream.web";
+
+let ExpoAudioStreamModule: any;
+
+if (Platform.OS === "web") {
+  let instance: ExpoAudioStreamWeb | null = null;
+
+  ExpoAudioStreamModule = (webProps: ExpoAudioStreamWebProps) => {
+    if (!instance) {
+      instance = new ExpoAudioStreamWeb(webProps);
+    }
+    return instance;
+  };
+} else {
+  ExpoAudioStreamModule = requireNativeModule("ExpoAudioStream");
+}
+
+export default ExpoAudioStreamModule;