@siteed/expo-audio-stream 1.0.2 → 1.0.5

package/src/ExpoAudioStream.types.ts

@@ -1,135 +1,77 @@
-export interface AudioEventPayload {
-  encoded?: string;
-  buffer?: ArrayBuffer;
-  fileUri: string;
-  lastEmittedSize: number;
-  position: number;
-  deltaSize: number;
-  totalSize: number;
-  mimeType: string;
-  streamUuid: string;
-}
-
-export interface AudioStreamResult {
-  fileUri: string;
-  webAudioUri?: string;
-  durationMs: number;
-  size: number;
-  mimeType: string;
-  channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
-}
-
-export interface StartAudioStreamResult {
-  fileUri: string;
-  mimeType: string;
-  channels?: number;
-  bitDepth?: number;
-  sampleRate?: number;
-}
+// packages/expo-audio-stream/src/ExpoAudioStream.types.ts
+import {
+    AmplitudeAlgorithm,
+    AudioAnalysis,
+    AudioFeaturesOptions,
+} from './AudioAnalysis/AudioAnalysis.types'
+import { AudioAnalysisEvent } from './events'
 
 export interface AudioStreamStatus {
-  isRecording: boolean;
-  isPaused: boolean;
-  durationMs: number;
-  size: number;
-  interval: number;
-  mimeType: string;
+    isRecording: boolean
+    isPaused: boolean
+    durationMs: number
+    size: number
+    interval: number
+    mimeType: string
 }
 
 export interface AudioDataEvent {
-  data: string | ArrayBuffer;
-  position: number;
-  fileUri: string;
-  eventDataSize: number;
-  totalSize: number;
+    data: string | Float32Array
+    position: number
+    fileUri: string
+    eventDataSize: number
+    totalSize: number
 }
 
-export interface AudioFeatures {
-  energy: number;
-  mfcc: number[];
-  rms: number;
-  minAmplitude: number;
-  maxAmplitude: number;
-  zcr: number;
-  spectralCentroid: number;
-  spectralFlatness: number;
-  spectralRolloff: number;
-  spectralBandwidth: number;
-  chromagram: number[];
-  tempo: number;
-  hnr: number;
+export type EncodingType = 'pcm_32bit' | 'pcm_16bit' | 'pcm_8bit'
+export type SampleRate = 16000 | 44100 | 48000
+export type BitDepth = 8 | 16 | 32
+
+export interface Chunk {
+    text: string
+    timestamp: [number, number | null]
 }
 
-export interface AudioFeaturesOptions {
-  energy?: boolean;
-  mfcc?: boolean;
-  rms?: boolean;
-  zcr?: boolean;
-  spectralCentroid?: boolean;
-  spectralFlatness?: boolean;
-  spectralRolloff?: boolean;
-  spectralBandwidth?: boolean;
-  chromagram?: boolean;
-  tempo?: boolean;
-  hnr?: boolean;
+export interface TranscriberData {
+    isBusy: boolean
+    text: string
+    chunks: Chunk[]
 }
 
-export interface DataPoint {
-  id: number;
-  amplitude: number;
-  activeSpeech?: boolean;
-  dB?: number;
-  silent?: boolean;
-  features?: AudioFeatures;
-  startTime?: number;
-  endTime?: number;
-  // start / end position in bytes
-  startPosition?: number;
-  endPosition?: number;
-  // number of audio samples for this point (samples size depends on bit depth)
-  samples?: number;
-  // Id of the speaker for this point
-  speaker?: number;
+export interface AudioRecording {
+    fileUri: string
+    filename: string
+    durationMs: number
+    size: number
+    mimeType: string
+    channels: number
+    bitDepth: BitDepth
+    sampleRate: SampleRate
+    transcripts?: TranscriberData[]
+    wavPCMData?: Float32Array // Full PCM data for the recording in WAV format (only on web, for native use the fileUri)
+    analysisData?: AudioAnalysis // Analysis data for the recording depending on enableProcessing flag
 }
 
-export interface AudioAnalysisData {
-  pointsPerSecond: number; // How many consolidated value per second
-  durationMs: number; // Duration of the audio in milliseconds
-  bitDepth: number; // Bit depth of the audio
-  samples: number; // Size of the audio in bytes
-  numberOfChannels: number; // Number of audio channels
-  sampleRate: number; // Sample rate of the audio
-  dataPoints: DataPoint[];
-  amplitudeRange: {
-    min: number;
-    max: number;
-  };
-  speakerChanges?: {
-    timestamp: number;
-    speaker: number;
-  }[];
+export interface StartRecordingResult {
+    fileUri: string
+    mimeType: string
+    channels?: number
+    bitDepth?: BitDepth
+    sampleRate?: SampleRate
 }
 
-export type EncodingType = "pcm_32bit" | "pcm_16bit" | "pcm_8bit";
-export type SampleRate = 16000 | 44100 | 48000;
 export interface RecordingConfig {
-  sampleRate?: SampleRate; // Sample rate for recording
-  channels?: 1 | 2; // 1 or 2 (MONO or STEREO)
-  encoding?: EncodingType; // Encoding type for the recording
-  interval?: number; // Interval in milliseconds at which to emit recording data
-
-  // Optional parameters for audio processing
-  //TODO remove maxRecentDataDuration - should be replaced by maxDataPoints to 100.
-  maxRecentDataDuration?: number; // Maximum duration of recent data to keep for processing (default is 10.0 seconds)
-  enableProcessing?: boolean; // Boolean to enable/disable audio processing (default is false)
-  pointsPerSecond?: number; // Number of data points to extract per second of audio (default is 1000)
-  algorithm?: string; // Algorithm to use for extraction (default is "rms")
-  features?: AudioFeaturesOptions; // Feature options to extract (default is empty)
+    sampleRate?: SampleRate // Sample rate for recording
+    channels?: 1 | 2 // 1 or 2 (MONO or STEREO)
+    encoding?: EncodingType // Encoding type for the recording
+    interval?: number // Interval in milliseconds at which to emit recording data
 
-  // Optional paramters from web
+    // Optional parameters for audio processing
+    enableProcessing?: boolean // Boolean to enable/disable audio processing (default is false)
+    pointsPerSecond?: number // Number of data points to extract per second of audio (default is 1000)
+    algorithm?: AmplitudeAlgorithm // Algorithm to use for amplitude computation (default is "rms")
+    features?: AudioFeaturesOptions // Feature options to extract (default is empty)
 
-  onAudioStream?: (_: AudioDataEvent) => Promise<void>; // Callback function to handle audio stream
-  onProcessingResult?: (_: AudioAnalysisData) => Promise<void>; // Callback function to handle processing results
+    onAudioStream?: (_: AudioDataEvent) => Promise<void> // Callback function to handle audio stream
+    onAudioAnalysis?: (_: AudioAnalysisEvent) => Promise<void> // Callback function to handle audio features extraction results
 }

package/src/ExpoAudioStream.web.ts

@@ -1,228 +1,256 @@
 // src/ExpoAudioStreamModule.web.ts
-import debug from "debug";
-import { EventEmitter } from "expo-modules-core";
+import { EventEmitter } from 'expo-modules-core'
 
+import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
 import {
-  AudioAnalysisData,
-  AudioEventPayload,
-  AudioStreamResult,
-  AudioStreamStatus,
-  RecordingConfig,
-  StartAudioStreamResult,
-} from "./ExpoAudioStream.types";
-import { WebRecorder } from "./WebRecorder";
-import { encodingToBitDepth } from "./utils";
+    AudioRecording,
+    AudioStreamStatus,
+    BitDepth,
+    RecordingConfig,
+    StartRecordingResult,
+} from './ExpoAudioStream.types'
+import { WebRecorder } from './WebRecorder.web'
+import { AudioEventPayload } from './events'
+import { getLogger } from './logger'
+import { concatenateBuffers } from './utils/concatenateBuffers'
+import { encodingToBitDepth } from './utils/encodingToBitDepth'
+import { writeWavHeader } from './utils/writeWavHeader'
 
 export interface EmitAudioEventProps {
-  data: ArrayBuffer;
-  position: number;
+    data: Float32Array
+    position: number
 }
-export type EmitAudioEventFunction = (_: EmitAudioEventProps) => void;
-export type EmitAudioAnalysisFunction = (_: AudioAnalysisData) => void;
+export type EmitAudioEventFunction = (_: EmitAudioEventProps) => void
+export type EmitAudioAnalysisFunction = (_: AudioAnalysis) => void
 
 export interface ExpoAudioStreamWebProps {
-  audioWorkletUrl: string;
-  featuresExtratorUrl: string;
+    audioWorkletUrl: string
+    featuresExtratorUrl: string
 }
 
-// const log = debug("expo-audio-stream:useAudioRecording");
-const log = console;
+const logger = getLogger('ExpoAudioStreamWeb')
+
 export class ExpoAudioStreamWeb extends EventEmitter {
-  customRecorder: WebRecorder | null;
-  audioChunks: ArrayBuffer[];
-  isRecording: boolean;
-  isPaused: boolean;
-  recordingStartTime: number;
-  pausedTime: number;
-  currentDurationMs: number;
-  currentSize: number;
-  currentInterval: number;
-  lastEmittedSize: number;
-  lastEmittedTime: number;
-  streamUuid: string | null;
-  extension: "webm" | "wav" = "wav"; // Default extension is 'webm'
-  recordingConfig?: RecordingConfig;
-  bitDepth: number; // Bit depth of the audio
-  audioWorkletUrl: string;
-  featuresExtratorUrl: string;
-
-  constructor({
-    audioWorkletUrl,
-    featuresExtratorUrl,
-  }: ExpoAudioStreamWebProps) {
-    const mockNativeModule = {
-      addListener: (eventName: string) => {
-        // Not used on web
-      },
-      removeListeners: (count: number) => {
-        // Not used on web
-      },
-    };
-    super(mockNativeModule); // Pass the mock native module to the parent class
-
-    this.customRecorder = null;
-    this.audioChunks = [];
-    this.isRecording = false;
-    this.isPaused = false;
-    this.recordingStartTime = 0;
-    this.pausedTime = 0;
-    this.currentDurationMs = 0;
-    this.currentSize = 0;
-    this.bitDepth = 32; // Default
-    this.currentInterval = 1000; // Default interval in ms
-    this.lastEmittedSize = 0;
-    this.lastEmittedTime = 0;
-    this.streamUuid = null; // Initialize UUID on first recording start
-    this.audioWorkletUrl = audioWorkletUrl;
-    this.featuresExtratorUrl = featuresExtratorUrl;
-  }
-
-  // Utility to handle user media stream
-  async getMediaStream() {
-    try {
-      return await navigator.mediaDevices.getUserMedia({ audio: true });
-    } catch (error) {
-      console.error("Failed to get media stream:", error);
-      throw error;
+    customRecorder: WebRecorder | null
+    audioChunks: ArrayBuffer[]
+    isRecording: boolean
+    isPaused: boolean
+    recordingStartTime: number
+    pausedTime: number
+    currentDurationMs: number
+    currentSize: number
+    currentInterval: number
+    lastEmittedSize: number
+    lastEmittedTime: number
+    streamUuid: string | null
+    extension: 'webm' | 'wav' = 'wav' // Default extension is 'webm'
+    recordingConfig?: RecordingConfig
+    bitDepth: BitDepth // Bit depth of the audio
+    audioWorkletUrl: string
+    featuresExtratorUrl: string
+
+    constructor({
+        audioWorkletUrl,
+        featuresExtratorUrl,
+    }: ExpoAudioStreamWebProps) {
+        const mockNativeModule = {
+            addListener: () => {
+                // Not used on web
+            },
+            removeListeners: () => {
+                // Not used on web
+            },
+        }
+        super(mockNativeModule) // Pass the mock native module to the parent class
+
+        this.customRecorder = null
+        this.audioChunks = []
+        this.isRecording = false
+        this.isPaused = false
+        this.recordingStartTime = 0
+        this.pausedTime = 0
+        this.currentDurationMs = 0
+        this.currentSize = 0
+        this.bitDepth = 32 // Default
+        this.currentInterval = 1000 // Default interval in ms
+        this.lastEmittedSize = 0
+        this.lastEmittedTime = 0
+        this.streamUuid = null // Initialize UUID on first recording start
+        this.audioWorkletUrl = audioWorkletUrl
+        this.featuresExtratorUrl = featuresExtratorUrl
     }
-  }
 
-  // Start recording with options
-  async startRecording(recordingConfig: RecordingConfig = {}) {
-    if (this.isRecording) {
-      throw new Error("Recording is already in progress");
+    // Utility to handle user media stream
+    async getMediaStream() {
+        try {
+            return await navigator.mediaDevices.getUserMedia({ audio: true })
+        } catch (error) {
+            console.error('Failed to get media stream:', error)
+            throw error
+        }
+    }
+
+    // Start recording with options
+    async startRecording(recordingConfig: RecordingConfig = {}) {
+        if (this.isRecording) {
+            throw new Error('Recording is already in progress')
+        }
+
+        this.bitDepth = encodingToBitDepth({
+            encoding: recordingConfig.encoding ?? 'pcm_32bit',
+        })
+
+        const audioContext = new (window.AudioContext ||
+            // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+            // @ts-ignore - Allow webkitAudioContext for Safari
+            window.webkitAudioContext)()
+        const stream = await this.getMediaStream()
+
+        const source = audioContext.createMediaStreamSource(stream)
+
+        this.customRecorder = new WebRecorder({
+            audioContext,
+            source,
+            recordingConfig,
+            audioWorkletUrl: this.audioWorkletUrl,
+            emitAudioEventCallback: ({
+                data,
+                position,
+            }: EmitAudioEventProps) => {
+                this.audioChunks.push(data)
+                this.currentSize += data.byteLength
+                this.emitAudioEvent({ data, position })
+                this.lastEmittedTime = Date.now()
+                this.lastEmittedSize = this.currentSize
+            },
+            emitAudioAnalysisCallback: (audioAnalysisData: AudioAnalysis) => {
+                logger.log(`Emitted AudioAnalysis:`, audioAnalysisData)
+                this.emit('AudioAnalysis', audioAnalysisData)
+            },
+        })
+        await this.customRecorder.init()
+        this.customRecorder.start()
+
+        // // Set a timer to stop recording after 5 seconds
+        // setTimeout(() => {
+        //   logger.log("AUTO Stopping recording");
+        //   this.customRecorder?.stopAndPlay();
+        //   this.isRecording = false;
+        // }, 3000);
+
+        this.isRecording = true
+        this.recordingConfig = recordingConfig
+        this.recordingStartTime = Date.now()
+        this.pausedTime = 0
+        this.lastEmittedSize = 0
+        this.lastEmittedTime = 0
+        this.streamUuid = Date.now().toString()
+        const fileUri = `${this.streamUuid}.${this.extension}`
+        const streamConfig: StartRecordingResult = {
+            fileUri,
+            mimeType: `audio/${this.extension}`,
+            bitDepth: this.bitDepth,
+            channels: recordingConfig.channels ?? 1,
+            sampleRate: recordingConfig.sampleRate ?? 44100,
+        }
+        return streamConfig
     }
 
-    this.bitDepth = encodingToBitDepth({
-      encoding: recordingConfig.encoding ?? "pcm_32bit",
-    });
-
-    const audioContext = new (window.AudioContext ||
-      // @ts-ignore - Allow webkitAudioContext for Safari
-      window.webkitAudioContext)();
-    const stream = await this.getMediaStream();
-
-    const source = audioContext.createMediaStreamSource(stream);
-
-    this.customRecorder = new WebRecorder({
-      audioContext,
-      source,
-      recordingConfig,
-      audioWorkletUrl: this.audioWorkletUrl,
-      featuresExtratorUrl: this.featuresExtratorUrl,
-      emitAudioEventCallback: ({ data, position }: EmitAudioEventProps) => {
-        this.audioChunks.push(data);
-        this.currentSize += data.byteLength;
-        this.emitAudioEvent({ data, position });
-        this.lastEmittedTime = Date.now();
-        this.lastEmittedSize = this.currentSize;
-      },
-      emitAudioAnalysisCallback: (audioAnalysisData: AudioAnalysisData) => {
-        console.log(`Emitted AudioAnalysis:`, audioAnalysisData);
-        this.emit("AudioAnalysis", audioAnalysisData);
-      },
-    });
-    await this.customRecorder.init();
-    this.customRecorder.start();
-
-    // // Set a timer to stop recording after 5 seconds
-    // setTimeout(() => {
-    //   console.log("AUTO Stopping recording");
-    //   this.customRecorder?.stopAndPlay();
-    //   this.isRecording = false;
-    // }, 3000);
-
-    this.isRecording = true;
-    this.recordingConfig = recordingConfig;
-    this.recordingStartTime = Date.now();
-    this.pausedTime = 0;
-    this.lastEmittedSize = 0;
-    this.lastEmittedTime = 0;
-    this.streamUuid = Date.now().toString();
-    const fileUri = `${this.streamUuid}.${this.extension}`;
-    const streamConfig: StartAudioStreamResult = {
-      fileUri,
-      mimeType: `audio/${this.extension}`,
-      bitDepth: this.bitDepth,
-      channels: recordingConfig.channels ?? 1,
-      sampleRate: recordingConfig.sampleRate ?? 44100,
-    };
-    return streamConfig;
-  }
-
-  emitAudioEvent({ data, position }: EmitAudioEventProps) {
-    const fileUri = `${this.streamUuid}.${this.extension}`;
-    const audioEventPayload: AudioEventPayload = {
-      fileUri,
-      mimeType: `audio/${this.extension}`,
-      lastEmittedSize: this.lastEmittedSize, // Since this might be continuously streaming, adjust accordingly
-      deltaSize: data.byteLength,
-      position,
-      totalSize: this.currentSize,
-      buffer: data,
-      streamUuid: this.streamUuid ?? "", // Generate or manage UUID for stream identification
-    };
-
-    this.emit("AudioData", audioEventPayload);
-  }
-
-  // Stop recording
-  async stopRecording(): Promise<AudioStreamResult | null> {
-    if (this.customRecorder) {
-      const fullPcmBuffer = await this.customRecorder.stop();
-      log.debug(`Stopped recording`, fullPcmBuffer);
+    emitAudioEvent({ data, position }: EmitAudioEventProps) {
+        const fileUri = `${this.streamUuid}.${this.extension}`
+        const audioEventPayload: AudioEventPayload = {
+            fileUri,
+            mimeType: `audio/${this.extension}`,
+            lastEmittedSize: this.lastEmittedSize, // Since this might be continuously streaming, adjust accordingly
+            deltaSize: data.byteLength,
+            position,
+            totalSize: this.currentSize,
+            buffer: data,
+            streamUuid: this.streamUuid ?? '', // Generate or manage UUID for stream identification
+        }
+
+        this.emit('AudioData', audioEventPayload)
     }
-    this.isRecording = false;
-    this.currentDurationMs = Date.now() - this.recordingStartTime;
-    const result: AudioStreamResult = {
-      fileUri: `${this.streamUuid}.${this.extension}`,
-      bitDepth: this.bitDepth,
-      channels: this.recordingConfig?.channels ?? 1,
-      sampleRate: this.recordingConfig?.sampleRate ?? 44100,
-      durationMs: this.currentDurationMs,
-      size: this.currentSize,
-      mimeType: `audio/${this.extension}`,
-    };
-
-    return result;
-  }
-
-  // Pause recording
-  async pauseRecording() {
-    if (!this.isRecording || this.isPaused) {
-      throw new Error("Recording is not active or already paused");
+
+    // Stop recording
+    async stopRecording(): Promise<AudioRecording> {
+        if (!this.customRecorder) {
+            throw new Error('Recorder is not initialized')
+        }
+
+        const fullPcmBufferArray = await this.customRecorder.stop()
+
+        // concat all audio chunks
+        logger.debug(`Stopped recording`, fullPcmBufferArray)
+        this.isRecording = false
+        this.currentDurationMs = Date.now() - this.recordingStartTime
+
+        const wavConfig = {
+            buffer: fullPcmBufferArray.buffer,
+            sampleRate: this.recordingConfig?.sampleRate ?? 44100,
+            numChannels: this.recordingConfig?.channels ?? 1,
+            bitDepth: this.bitDepth,
+        }
+        logger.debug(`Writing wav header`, wavConfig)
+        const wavBuffer = writeWavHeader(wavConfig).slice(0)
+
+        // Create blob fileUri from audio chunks
+        const blob = new Blob([wavBuffer], {
+            type: `audio/${this.extension}`,
+        })
+        const fileUri = URL.createObjectURL(blob)
+
+        const result: AudioRecording = {
+            fileUri,
+            filename: `${this.streamUuid}.${this.extension}`,
+            wavPCMData: fullPcmBufferArray,
+            bitDepth: this.bitDepth,
+            channels: this.recordingConfig?.channels ?? 1,
+            sampleRate: this.recordingConfig?.sampleRate ?? 44100,
+            durationMs: this.currentDurationMs,
+            size: this.currentSize,
+            mimeType: `audio/${this.extension}`,
+        }
+
+        return result
     }
 
-    if (this.customRecorder) {
-      this.customRecorder.stop();
+    // Pause recording
+    async pauseRecording() {
+        if (!this.isRecording || this.isPaused) {
+            throw new Error('Recording is not active or already paused')
+        }
+
+        if (this.customRecorder) {
+            this.customRecorder.pause()
+        }
+        this.isPaused = true
+        this.pausedTime = Date.now()
     }
-    this.isPaused = true;
-    this.pausedTime = Date.now();
-  }
-
-  // Resume recording
-  async resumeRecording() {
-    if (!this.isPaused) {
-      throw new Error("Recording is not paused");
+
+    // Resume recording
+    async resumeRecording() {
+        if (!this.isPaused) {
+            throw new Error('Recording is not paused')
+        }
+
+        if (this.customRecorder) {
+            this.customRecorder.resume()
+        }
+        this.isPaused = false
+        this.recordingStartTime += Date.now() - this.pausedTime
     }
 
-    if (this.customRecorder) {
-      this.customRecorder.resume();
+    // Get current status
+    status() {
+        const status: AudioStreamStatus = {
+            isRecording: this.isRecording,
+            isPaused: this.isPaused,
+            durationMs: Date.now() - this.recordingStartTime,
+            size: this.currentSize,
+            interval: this.currentInterval,
+            mimeType: `audio/${this.extension}`,
+        }
+        return status
     }
-    this.isPaused = false;
-    this.recordingStartTime += Date.now() - this.pausedTime;
-  }
-
-  // Get current status
-  status() {
-    const status: AudioStreamStatus = {
-      isRecording: this.isRecording,
-      isPaused: this.isPaused,
-      durationMs: Date.now() - this.recordingStartTime,
-      size: this.currentSize,
-      interval: this.currentInterval,
-      mimeType: `audio/${this.extension}`,
-    };
-    return status;
-  }
 }