@siteed/expo-audio-stream 1.1.5 → 1.1.7

package/src/WebRecorder.web.ts

@@ -0,0 +1,448 @@
+// src/WebRecorder.ts
+import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
+import { RecordingConfig } from './ExpoAudioStream.types'
+import {
+    EmitAudioAnalysisFunction,
+    EmitAudioEventFunction,
+} from './ExpoAudioStream.web'
+import { getLogger } from './logger'
+import { encodingToBitDepth } from './utils/encodingToBitDepth'
+import { WavHeaderOptions, writeWavHeader } from './utils/writeWavHeader'
+import { InlineFeaturesExtractor } from './workers/InlineFeaturesExtractor.web'
+import { InlineAudioWebWorker } from './workers/inlineAudioWebWorker.web'
+
+interface AudioWorkletEvent {
+    data: {
+        command: string
+        recordedData?: Float32Array
+        sampleRate?: number
+    }
+}
+
+interface AudioFeaturesEvent {
+    data: {
+        command: string
+        result: AudioAnalysis
+    }
+}
+
+const DEFAULT_WEB_BITDEPTH = 32
+const DEFAULT_WEB_POINTS_PER_SECOND = 10
+const DEFAULT_WEB_INTERVAL = 500
+const DEFAULT_WEB_NUMBER_OF_CHANNELS = 1
+const DEFAULT_ALGORITHM = 'rms'
+
+const TAG = 'WebRecorder'
+const logger = getLogger(TAG)
+
+export class WebRecorder {
+    private audioContext: AudioContext
+    private audioWorkletNode!: AudioWorkletNode
+    private featureExtractorWorker?: Worker
+    private source: MediaStreamAudioSourceNode
+    private audioWorkletUrl: string
+    private emitAudioEventCallback: EmitAudioEventFunction
+    private emitAudioAnalysisCallback: EmitAudioAnalysisFunction
+    private config: RecordingConfig
+    private position: number // Track the cumulative position
+    private numberOfChannels: number // Number of audio channels
+    private bitDepth: number // Bit depth of the audio
+    private exportBitDepth: number // Bit depth of the audio
+    private buffer: Float32Array // Single buffer to store the audio data
+    private bufferSize: number // Keep track of the buffer size
+    private audioAnalysisData: AudioAnalysis // Keep updating the full audio analysis data with latest events
+
+    constructor({
+        audioContext,
+        source,
+        recordingConfig,
+        audioWorkletUrl,
+        emitAudioEventCallback,
+        emitAudioAnalysisCallback,
+    }: {
+        audioContext: AudioContext
+        source: MediaStreamAudioSourceNode
+        recordingConfig: RecordingConfig
+        audioWorkletUrl: string
+        emitAudioEventCallback: EmitAudioEventFunction
+        emitAudioAnalysisCallback: EmitAudioAnalysisFunction
+    }) {
+        this.audioContext = audioContext
+        this.source = source
+        this.audioWorkletUrl = audioWorkletUrl
+        this.emitAudioEventCallback = emitAudioEventCallback
+        this.emitAudioAnalysisCallback = emitAudioAnalysisCallback
+        this.config = recordingConfig
+        this.position = 0
+
+        const audioContextFormat = this.checkAudioContextFormat({
+            sampleRate: this.audioContext.sampleRate,
+        })
+        logger.debug('Initialized WebRecorder with config:', {
+            sampleRate: audioContextFormat.sampleRate,
+            bitDepth: audioContextFormat.bitDepth,
+            numberOfChannels: audioContextFormat.numberOfChannels,
+        })
+
+        this.bitDepth = audioContextFormat.bitDepth
+        this.numberOfChannels =
+            audioContextFormat.numberOfChannels ||
+            DEFAULT_WEB_NUMBER_OF_CHANNELS // Default to 1 if not available
+        this.exportBitDepth =
+            encodingToBitDepth({
+                encoding: recordingConfig.encoding ?? 'pcm_32bit',
+            }) ||
+            audioContextFormat.bitDepth ||
+            DEFAULT_WEB_BITDEPTH
+
+        // Initialize WAV header
+        const wavHeader = writeWavHeader({
+            sampleRate: this.audioContext.sampleRate,
+            numChannels: this.numberOfChannels,
+            bitDepth: this.exportBitDepth,
+        })
+
+        // Initialize the buffer with WAV header
+        this.buffer = new Float32Array(
+            wavHeader.byteLength / Float32Array.BYTES_PER_ELEMENT
+        )
+        this.bufferSize = this.buffer.byteLength
+
+        // Copy WAV header to Float32Array buffer
+        new Uint8Array(this.buffer.buffer).set(new Uint8Array(wavHeader))
+
+        this.audioAnalysisData = {
+            amplitudeRange: { min: 0, max: 0 },
+            dataPoints: [],
+            durationMs: 0,
+            samples: 0,
+            amplitudeAlgorithm: recordingConfig.algorithm || DEFAULT_ALGORITHM,
+            bitDepth: this.bitDepth,
+            numberOfChannels: this.numberOfChannels,
+            sampleRate: this.config.sampleRate || this.audioContext.sampleRate,
+            pointsPerSecond:
+                this.config.pointsPerSecond || DEFAULT_WEB_POINTS_PER_SECOND,
+            speakerChanges: [],
+        }
+
+        if (recordingConfig.enableProcessing) {
+            this.initFeatureExtractorWorker()
+        }
+    }
+
+    async init() {
+        try {
+            if (!this.audioWorkletUrl) {
+                const blob = new Blob([InlineAudioWebWorker], {
+                    type: 'application/javascript',
+                })
+                const url = URL.createObjectURL(blob)
+                await this.audioContext.audioWorklet.addModule(url)
+            } else {
+                await this.audioContext.audioWorklet.addModule(
+                    this.audioWorkletUrl
+                )
+            }
+            this.audioWorkletNode = new AudioWorkletNode(
+                this.audioContext,
+                'recorder-processor'
+            )
+
+            this.audioWorkletNode.port.onmessage = async (
+                event: AudioWorkletEvent
+            ) => {
+                const command = event.data.command
+                if (command !== 'newData') {
+                    return
+                }
+                // Handle the audio blob (e.g., send it to the server or process it further)
+                logger.debug('Received audio blob from processor', event)
+                const pcmBufferFloat = event.data.recordedData
+
+                if (!pcmBufferFloat) {
+                    return
+                }
+
+                // Concatenate the incoming Float32Array to the existing buffer
+                const newBuffer = new Float32Array(
+                    this.bufferSize + pcmBufferFloat.length
+                )
+                newBuffer.set(this.buffer, 0)
+                newBuffer.set(pcmBufferFloat, this.bufferSize)
+                this.buffer = newBuffer
+                this.bufferSize += pcmBufferFloat.length
+
+                const sampleRate =
+                    event.data.sampleRate ?? this.audioContext.sampleRate
+                const duration = pcmBufferFloat.length / sampleRate // Calculate duration of the current buffer
+
+                this.emitAudioEventCallback({
+                    data: pcmBufferFloat,
+                    position: this.position,
+                })
+                this.position += duration // Update position
+
+                this.featureExtractorWorker?.postMessage(
+                    {
+                        command: 'process',
+                        channelData: pcmBufferFloat,
+                        sampleRate,
+                        pointsPerSecond:
+                            this.config.pointsPerSecond ||
+                            DEFAULT_WEB_POINTS_PER_SECOND,
+                        algorithm: this.config.algorithm || 'rms',
+                        bitDepth: this.bitDepth,
+                        fullAudioDurationMs: this.position * 1000,
+                        numberOfChannels: this.numberOfChannels,
+                        features: this.config.features,
+                    },
+                    []
+                )
+            }
+
+            logger.debug(
+                `WebRecorder initialized -- recordSampleRate=${this.audioContext.sampleRate}`,
+                this.config
+            )
+            this.audioWorkletNode.port.postMessage({
+                command: 'init',
+                recordSampleRate: this.audioContext.sampleRate, // Pass the original sample rate
+                exportSampleRate:
+                    this.config.sampleRate ?? this.audioContext.sampleRate,
+                bitDepth: this.bitDepth,
+                exportBitDepth: this.exportBitDepth,
+                channels: this.numberOfChannels,
+                interval: this.config.interval ?? DEFAULT_WEB_INTERVAL,
+            })
+
+            // Connect the source to the AudioWorkletNode and start recording
+            this.source.connect(this.audioWorkletNode)
+            this.audioWorkletNode.connect(this.audioContext.destination)
+        } catch (error) {
+            console.error(`[${TAG}] Failed to initialize WebRecorder`, error)
+        }
+    }
+
+    initFeatureExtractorWorker(featuresExtratorUrl?: string) {
+        try {
+            if (featuresExtratorUrl) {
+                // Initialize the feature extractor worker
+                //TODO: create audio feature extractor from a Blob instead of url since we cannot include the url directly in the library
+                // We keep the url during dev and use the blob in production.
+                this.featureExtractorWorker = new Worker(
+                    new URL(featuresExtratorUrl, window.location.href)
+                )
+                this.featureExtractorWorker.onmessage =
+                    this.handleFeatureExtractorMessage.bind(this)
+                this.featureExtractorWorker.onerror =
+                    this.handleWorkerError.bind(this)
+            } else {
+                // Fallback to the inline worker if the URL is not provided
+                this.initFallbackWorker()
+            }
+        } catch (error) {
+            console.error(
+                `[${TAG}] Failed to initialize feature extractor worker`,
+                error
+            )
+            this.initFallbackWorker()
+        }
+    }
+
+    initFallbackWorker() {
+        try {
+            const blob = new Blob([InlineFeaturesExtractor], {
+                type: 'application/javascript',
+            })
+            const url = URL.createObjectURL(blob)
+            this.featureExtractorWorker = new Worker(url)
+            this.featureExtractorWorker.onmessage =
+                this.handleFeatureExtractorMessage.bind(this)
+            this.featureExtractorWorker.onerror = (error) => {
+                console.error(`[${TAG}] Default Inline worker failed`, error)
+            }
+            logger.log('Inline worker initialized successfully')
+        } catch (error) {
+            console.error(
+                `[${TAG}] Failed to initialize Inline Feature Extractor worker`,
+                error
+            )
+        }
+    }
+
+    handleWorkerError(error: ErrorEvent) {
+        console.error(`[${TAG}] Feature extractor worker error:`, error)
+    }
+
+    handleFeatureExtractorMessage(event: AudioFeaturesEvent) {
+        if (event.data.command === 'features') {
+            const segmentResult = event.data.result
+
+            // Merge the segment result with the full audio analysis data
+            this.audioAnalysisData.dataPoints.push(...segmentResult.dataPoints)
+            this.audioAnalysisData.speakerChanges?.push(
+                ...(segmentResult.speakerChanges ?? [])
+            )
+            this.audioAnalysisData.durationMs = segmentResult.durationMs
+            if (segmentResult.amplitudeRange) {
+                this.audioAnalysisData.amplitudeRange = {
+                    min: Math.min(
+                        this.audioAnalysisData.amplitudeRange.min,
+                        segmentResult.amplitudeRange.min
+                    ),
+                    max: Math.max(
+                        this.audioAnalysisData.amplitudeRange.max,
+                        segmentResult.amplitudeRange.max
+                    ),
+                }
+            }
+            // Handle the extracted features (e.g., emit an event or log them)
+            logger.debug('features event segmentResult', segmentResult)
+            logger.debug(
+                'features event audioAnalysisData',
+                this.audioAnalysisData
+            )
+            this.emitAudioAnalysisCallback(segmentResult)
+        }
+    }
+
+    start() {
+        this.source.connect(this.audioWorkletNode)
+        this.audioWorkletNode.connect(this.audioContext.destination)
+    }
+
+    stop(): Promise<Float32Array> {
+        return new Promise((resolve, reject) => {
+            try {
+                if (this.audioWorkletNode) {
+                    // this.source.disconnect(this.audioWorkletNode);
+                    // this.audioWorkletNode.disconnect(this.audioContext.destination);
+                    this.audioWorkletNode.port.postMessage({ command: 'stop' })
+
+                    // Set a timeout to reject the promise if no message is received within 5 seconds
+                    const timeout = setTimeout(() => {
+                        this.audioWorkletNode.port.removeEventListener(
+                            'message',
+                            onMessage
+                        )
+                        reject(
+                            new Error(
+                                "Timeout error, audioWorkletNode didn't complete."
+                            )
+                        )
+                    }, 5000)
+
+                    // Listen for the recordedData message to confirm stopping
+                    const onMessage = async (event: AudioWorkletEvent) => {
+                        const command = event.data.command
+                        if (command === 'recordedData') {
+                            clearTimeout(timeout) // Clear the timeout
+
+                            const rawPCMDataFull =
+                                event.data.recordedData?.slice(0)
+
+                            if (!rawPCMDataFull) {
+                                reject(new Error('Failed to get recorded data'))
+                                return
+                            }
+
+                            // Compute duration of the recorded data
+                            const duration =
+                                rawPCMDataFull.byteLength /
+                                (this.audioContext.sampleRate *
+                                    (this.exportBitDepth /
+                                        this.numberOfChannels))
+                            logger.debug(
+                                `Received recorded data -- Duration: ${duration} vs ${rawPCMDataFull.byteLength / this.audioContext.sampleRate} seconds`
+                            )
+                            logger.debug(
+                                `recordedData.length=${rawPCMDataFull.byteLength} vs transmittedData.length=${this.bufferSize}`
+                            )
+
+                            // Remove the event listener after receiving the final data
+                            this.audioWorkletNode.port.removeEventListener(
+                                'message',
+                                onMessage
+                            )
+                            resolve(this.buffer) // Resolve the promise with the collected buffers
+                        }
+                    }
+                    this.audioWorkletNode.port.addEventListener(
+                        'message',
+                        onMessage
+                    )
+                }
+
+                // Stop all media stream tracks to stop the browser recording
+                this.stopMediaStreamTracks()
+            } catch (error) {
+                reject(error)
+            }
+        })
+    }
+
+    pause() {
+        this.source.disconnect(this.audioWorkletNode) // Disconnect the source from the AudioWorkletNode
+        this.audioWorkletNode.disconnect(this.audioContext.destination) // Disconnect the AudioWorkletNode from the destination
+        this.audioWorkletNode.port.postMessage({ command: 'pause' })
+    }
+
+    stopMediaStreamTracks() {
+        // Stop all audio tracks to stop the recording icon
+        const tracks = this.source.mediaStream.getTracks()
+        tracks.forEach((track) => track.stop())
+    }
+
+    async playRecordedData({
+        recordedData,
+    }: {
+        recordedData: ArrayBuffer
+        mimeType?: string
+    }) {
+        try {
+            const blob = new Blob([recordedData])
+            const url = URL.createObjectURL(blob)
+            const response = await fetch(url)
+            const arrayBuffer = await response.arrayBuffer()
+
+            // Decode the audio data
+            const audioBuffer =
+                await this.audioContext.decodeAudioData(arrayBuffer)
+
+            // Create a buffer source node and play the audio
+            const bufferSource = this.audioContext.createBufferSource()
+            bufferSource.buffer = audioBuffer
+            bufferSource.connect(this.audioContext.destination)
+            bufferSource.start()
+            logger.debug('Playing recorded data', recordedData)
+        } catch (error) {
+            console.error(`[${TAG}] Failed to play recorded data:`, error)
+        }
+    }
+
+    private checkAudioContextFormat({ sampleRate }: { sampleRate: number }) {
+        // Create a silent AudioBuffer
+        const frameCount = sampleRate * 1.0 // 1 second buffer
+        const audioBuffer = this.audioContext.createBuffer(
+            1,
+            frameCount,
+            sampleRate
+        )
+
+        // Check the format
+        const channelData = audioBuffer.getChannelData(0)
+        const bitDepth = channelData.BYTES_PER_ELEMENT * 8 // 4 bytes per element means 32-bit
+
+        return {
+            sampleRate: audioBuffer.sampleRate,
+            bitDepth,
+            numberOfChannels: audioBuffer.numberOfChannels,
+        }
+    }
+
+    resume() {
+        this.source.connect(this.audioWorkletNode)
+        this.audioWorkletNode.connect(this.audioContext.destination)
+        this.audioWorkletNode.port.postMessage({ command: 'resume' })
+    }
+}

package/src/constants.ts

@@ -0,0 +1,18 @@
+// packages/expo-audio-stream/src/constants.ts
+import { Platform } from 'react-native'
+
+import { BitDepth, SampleRate } from './ExpoAudioStream.types'
+
+export const isWeb = Platform.OS === 'web'
+export const DEBUG_NAMESPACE = 'expo-audio-stream'
+
+// Constants for identifying chunks in a WAV file
+export const RIFF_HEADER = 0x52494646 // "RIFF"
+export const WAVE_HEADER = 0x57415645 // "WAVE"
+export const FMT_CHUNK_ID = 0x666d7420 // "fmt "
+export const DATA_CHUNK_ID = 0x64617461 // "data"
+export const INFO_CHUNK_ID = 0x494e464f // "INFO"
+
+// Default values
+export const DEFAULT_SAMPLE_RATE: SampleRate = 16000
+export const DEFAULT_BIT_DEPTH: BitDepth = 32

package/src/events.ts

@@ -0,0 +1,39 @@
+// packages/expo-audio-stream/src/events.ts
+
+import { EventEmitter, type Subscription } from 'expo-modules-core'
+
+import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
+import ExpoAudioStreamModule from './ExpoAudioStreamModule'
+import { getLogger } from './logger'
+
+const emitter = new EventEmitter(ExpoAudioStreamModule)
+const logger = getLogger('events')
+
+export interface AudioEventPayload {
+    encoded?: string
+    buffer?: Float32Array
+    fileUri: string
+    lastEmittedSize: number
+    position: number
+    deltaSize: number
+    totalSize: number
+    mimeType: string
+    streamUuid: string
+}
+
+export function addAudioEventListener(
+    listener: (event: AudioEventPayload) => Promise<void>
+): Subscription {
+    logger.log('Adding listener for AudioData event')
+    return emitter.addListener<AudioEventPayload>('AudioData', listener)
+}
+
+// Only aliasing the AudioAnalysis type for the event payload
+export interface AudioAnalysisEvent extends AudioAnalysis {}
+
+export function addAudioAnalysisListener(
+    listener: (event: AudioAnalysisEvent) => Promise<void>
+): Subscription {
+    logger.log('Adding listener for AudioAnalysis event')
+    return emitter.addListener<AudioAnalysisEvent>('AudioAnalysis', listener)
+}

package/src/index.ts

@@ -0,0 +1,24 @@
+// src/index.ts
+
+import { extractAudioAnalysis } from './AudioAnalysis/extractAudioAnalysis'
+import {
+    AudioRecorderProvider,
+    useSharedAudioRecorder,
+} from './AudioRecorder.provider'
+import ExpoAudioStreamModule from './ExpoAudioStreamModule'
+import { useAudioRecorder } from './useAudioRecorder'
+
+export * from './utils/convertPCMToFloat32'
+export * from './utils/getWavFileInfo'
+export * from './utils/writeWavHeader'
+
+export {
+    AudioRecorderProvider,
+    ExpoAudioStreamModule,
+    extractAudioAnalysis,
+    useAudioRecorder,
+    useSharedAudioRecorder,
+}
+
+export type * from './AudioAnalysis/AudioAnalysis.types'
+export type * from './ExpoAudioStream.types'

package/src/logger.ts

@@ -0,0 +1,22 @@
+// packages/expo-audio-stream/src/logger.ts
+import { getLogger as siteedGetLogger } from '@siteed/react-native-logger'
+
+import { DEBUG_NAMESPACE } from './constants'
+
+type ConsoleLike = {
+    log: (message: string, ...args: unknown[]) => void
+    debug: (message: string, ...args: unknown[]) => void
+    warn: (message: string, ...args: unknown[]) => void
+    error: (message: string, ...args: unknown[]) => void
+}
+
+export const getLogger = (tag: string): ConsoleLike => {
+    const baseLogger = siteedGetLogger(`${DEBUG_NAMESPACE}:${tag}`)
+
+    return {
+        log: (...args: unknown[]) => baseLogger.log(...(args as [unknown])),
+        debug: (...args: unknown[]) => baseLogger.debug(...(args as [unknown])),
+        error: (...args: unknown[]) => baseLogger.error(...(args as [unknown])),
+        warn: (...args: unknown[]) => baseLogger.warn(...(args as [unknown])),
+    }
+}