@siteed/expo-audio-stream 1.0.2 → 1.0.5

package/plugin/src/index.ts

@@ -1,53 +1,53 @@
 import {
-  AndroidConfig,
-  ConfigPlugin,
-  withAndroidManifest,
-  withInfoPlist,
-} from "@expo/config-plugins";
+    AndroidConfig,
+    ConfigPlugin,
+    withAndroidManifest,
+    withInfoPlist,
+} from '@expo/config-plugins'
 
-const MICROPHONE_USAGE = "Allow $(PRODUCT_NAME) to access your microphone";
+const MICROPHONE_USAGE = 'Allow $(PRODUCT_NAME) to access your microphone'
 
 const withRecordingPermission: ConfigPlugin<{
-  microphonePermission: string;
+    microphonePermission: string
 }> = (config, existingPerms) => {
-  if (!existingPerms) {
-    console.warn("No previous permissions provided");
-  }
-  config = withInfoPlist(config, (config) => {
-    config.modResults["NSMicrophoneUsageDescription"] = MICROPHONE_USAGE;
-
-    // Add audio to UIBackgroundModes to allow background audio recording
-    const existingBackgroundModes = config.modResults.UIBackgroundModes || [];
-    if (!existingBackgroundModes.includes("audio")) {
-      existingBackgroundModes.push("audio");
+    if (!existingPerms) {
+        console.warn('No previous permissions provided')
     }
-    config.modResults.UIBackgroundModes = existingBackgroundModes;
-
-    return config;
-  });
-
-  config = withAndroidManifest(config, (config) => {
-    const mainApplication = AndroidConfig.Manifest.getMainApplicationOrThrow(
-      config.modResults,
-    );
-
-    AndroidConfig.Manifest.addMetaDataItemToMainApplication(
-      mainApplication,
-      "android.permission.RECORD_AUDIO",
-      MICROPHONE_USAGE,
-    );
-
-    // Add FOREGROUND_SERVICE permission for handling background recording
-    AndroidConfig.Manifest.addMetaDataItemToMainApplication(
-      mainApplication,
-      "android.permission.FOREGROUND_SERVICE",
-      "This apps needs access to the foreground service to record audio in the background",
-    );
-
-    return config;
-  });
-
-  return config;
-};
-
-export default withRecordingPermission;
+    config = withInfoPlist(config, (config) => {
+        config.modResults['NSMicrophoneUsageDescription'] = MICROPHONE_USAGE
+
+        // Add audio to UIBackgroundModes to allow background audio recording
+        const existingBackgroundModes =
+            config.modResults.UIBackgroundModes || []
+        if (!existingBackgroundModes.includes('audio')) {
+            existingBackgroundModes.push('audio')
+        }
+        config.modResults.UIBackgroundModes = existingBackgroundModes
+
+        return config
+    })
+
+    config = withAndroidManifest(config, (config) => {
+        const mainApplication =
+            AndroidConfig.Manifest.getMainApplicationOrThrow(config.modResults)
+
+        AndroidConfig.Manifest.addMetaDataItemToMainApplication(
+            mainApplication,
+            'android.permission.RECORD_AUDIO',
+            MICROPHONE_USAGE
+        )
+
+        // Add FOREGROUND_SERVICE permission for handling background recording
+        AndroidConfig.Manifest.addMetaDataItemToMainApplication(
+            mainApplication,
+            'android.permission.FOREGROUND_SERVICE',
+            'This apps needs access to the foreground service to record audio in the background'
+        )
+
+        return config
+    })
+
+    return config
+}
+
+export default withRecordingPermission

package/plugin/tsconfig.json

@@ -1,14 +1,9 @@
 {
-  "extends": "expo-module-scripts/tsconfig.plugin",
-  "compilerOptions": {
-    "outDir": "build",
-    "rootDir": "src"
-  },
-  "include": [
-    "./src"
-  ],
-  "exclude": [
-    "**/__mocks__/*",
-    "**/__tests__/*"
-  ]
-}
+    "extends": "expo-module-scripts/tsconfig.plugin",
+    "compilerOptions": {
+        "outDir": "build",
+        "rootDir": "src"
+    },
+    "include": ["./src"],
+    "exclude": ["**/__mocks__/*", "**/__tests__/*"]
+}

package/src/AudioAnalysis/AudioAnalysis.types.ts

@@ -0,0 +1,84 @@
+// packages/expo-audio-stream/src/AudioAnalysis/AudioAnalysis.types.ts
+
+/**
+ * Represents various audio features extracted from an audio signal.
+ */
+export interface AudioFeatures {
+    energy: number // The infinite integral of the squared signal, representing the overall energy of the audio.
+    mfcc: number[] // Mel-frequency cepstral coefficients, describing the short-term power spectrum of a sound.
+    rms: number // Root mean square value, indicating the amplitude of the audio signal.
+    minAmplitude: number // Minimum amplitude value in the audio signal.
+    maxAmplitude: number // Maximum amplitude value in the audio signal.
+    zcr: number // Zero-crossing rate, indicating the rate at which the signal changes sign.
+    spectralCentroid: number // The center of mass of the spectrum, indicating the brightness of the sound.
+    spectralFlatness: number // Measure of the flatness of the spectrum, indicating how noise-like the signal is.
+    spectralRolloff: number // The frequency below which a specified percentage (usually 85%) of the total spectral energy lies.
+    spectralBandwidth: number // The width of the spectrum, indicating the range of frequencies present.
+    chromagram: number[] // Chromagram, representing the 12 different pitch classes of the audio.
+    tempo: number // Estimated tempo of the audio signal, measured in beats per minute (BPM).
+    hnr: number // Harmonics-to-noise ratio, indicating the proportion of harmonics to noise in the audio signal.
+}
+
+/**
+ * Options to specify which audio features to extract.
+ */
+export interface AudioFeaturesOptions {
+    energy?: boolean
+    mfcc?: boolean
+    rms?: boolean
+    zcr?: boolean
+    spectralCentroid?: boolean
+    spectralFlatness?: boolean
+    spectralRolloff?: boolean
+    spectralBandwidth?: boolean
+    chromagram?: boolean
+    tempo?: boolean
+    hnr?: boolean
+}
+
+/**
+ * Represents a single data point in the audio analysis.
+ */
+export interface DataPoint {
+    id: number
+    amplitude: number
+    activeSpeech?: boolean
+    dB?: number
+    silent?: boolean
+    features?: AudioFeatures
+    startTime?: number
+    endTime?: number
+    // start / end position in bytes
+    startPosition?: number
+    endPosition?: number
+    // number of audio samples for this point (samples size depends on bit depth)
+    samples?: number
+    // TODO: speaker detection
+    speaker?: number
+}
+
+export type AmplitudeAlgorithm = 'peak' | 'rms'
+
+/**
+ * Represents the complete data from the audio analysis.
+ */
+export interface AudioAnalysis {
+    pointsPerSecond: number // How many consolidated value per second
+    durationMs: number // Duration of the audio in milliseconds
+    bitDepth: number // Bit depth of the audio
+    samples: number // Size of the audio in bytes
+    numberOfChannels: number // Number of audio channels
+    sampleRate: number // Sample rate of the audio
+    dataPoints: DataPoint[] // Array of data points from the analysis.
+    amplitudeAlgorithm: AmplitudeAlgorithm // Algorithm used to calculate amplitude values.
+    amplitudeRange: {
+        min: number
+        max: number
+    }
+    // TODO: speaker detection
+    speakerChanges?: {
+        timestamp: number // Timestamp of the speaker change in milliseconds.
+        speaker: number // Speaker identifier.
+    }[]
+}
+

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -0,0 +1,147 @@
+// packages/expo-audio-stream/src/AudioAnalysis/extractAudioAnalysis.ts
+import {
+    AmplitudeAlgorithm,
+    AudioAnalysis,
+    AudioFeaturesOptions,
+} from './AudioAnalysis.types'
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+import { isWeb } from '../constants'
+import { getLogger } from '../logger'
+import { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'
+import { getWavFileInfo, WavFileInfo } from '../utils/getWavFileInfo'
+import { InlineFeaturesExtractor } from '../workers/InlineFeaturesExtractor.web'
+
+const logger = getLogger('extractAudioAnalysis')
+
+export interface ExtractAudioAnalysisProps {
+    fileUri?: string // should provide either fileUri or arrayBuffer
+    wavMetadata?: WavFileInfo
+    arrayBuffer?: ArrayBuffer
+    bitDepth?: number
+    skipWavHeader?: boolean
+    durationMs?: number
+    sampleRate?: number
+    numberOfChannels?: number
+    algorithm?: AmplitudeAlgorithm
+    position?: number // Optional number of bytes to skip. Default is 0
+    length?: number // Optional number of bytes to read.
+    pointsPerSecond?: number // Optional number of points per second. Use to reduce the number of points and compute the number of datapoints to return.
+    features?: AudioFeaturesOptions
+    featuresExtratorUrl?: string
+}
+
+export const extractAudioAnalysis = async ({
+    fileUri,
+    pointsPerSecond = 20,
+    arrayBuffer,
+    bitDepth,
+    skipWavHeader = true,
+    durationMs,
+    sampleRate,
+    numberOfChannels,
+    algorithm = 'rms',
+    features,
+    featuresExtratorUrl,
+}: ExtractAudioAnalysisProps): Promise<AudioAnalysis> => {
+    if (isWeb) {
+        if (!arrayBuffer && !fileUri) {
+            throw new Error('Either arrayBuffer or fileUri must be provided')
+        }
+
+        if (!arrayBuffer) {
+            logger.log(`fetching fileUri`, fileUri)
+            const response = await fetch(fileUri!)
+
+            if (!response.ok) {
+                throw new Error(
+                    `Failed to fetch fileUri: ${response.statusText}`
+                )
+            }
+
+            arrayBuffer = await response.arrayBuffer()
+            logger.log(`fetched fileUri`, arrayBuffer.byteLength, arrayBuffer)
+        }
+
+        // Create a new copy of the ArrayBuffer to avoid detachment issues
+        const bufferCopy = arrayBuffer.slice(0)
+        logger.log(
+            `extractAudioAnalysis skipWavHeader=${skipWavHeader} bitDepth=${bitDepth} len=${bufferCopy.byteLength}`,
+            bufferCopy.slice(0, 100)
+        )
+
+        let actualBitDepth = bitDepth
+        if (!actualBitDepth) {
+            logger.log(
+                `extractAudioAnalysis bitDepth not provided -- getting wav file info`
+            )
+            const fileInfo = await getWavFileInfo(bufferCopy)
+            actualBitDepth = fileInfo.bitDepth
+        }
+        logger.log(`extractAudioAnalysis actualBitDepth=${actualBitDepth}`)
+
+        const {
+            pcmValues: channelData,
+            min,
+            max,
+        } = await convertPCMToFloat32({
+            buffer: arrayBuffer,
+            bitDepth: actualBitDepth,
+            skipWavHeader,
+        })
+        logger.log(
+            `extractAudioAnalysis skipWaveHeader=${skipWavHeader} convertPCMToFloat32 length=${channelData.length} range: [ ${min} :: ${max} ]`
+        )
+
+        return new Promise((resolve, reject) => {
+            let worker: Worker
+            if (featuresExtratorUrl) {
+                worker = new Worker(
+                    new URL(featuresExtratorUrl, window.location.href)
+                )
+            } else {
+                const blob = new Blob([InlineFeaturesExtractor], {
+                    type: 'application/javascript',
+                })
+                const url = URL.createObjectURL(blob)
+                worker = new Worker(url)
+            }
+
+            worker.onmessage = (event) => {
+                resolve(event.data.result)
+            }
+
+            worker.onerror = (error) => {
+                reject(error)
+            }
+
+            worker.postMessage({
+                command: 'process',
+                channelData,
+                sampleRate,
+                pointsPerSecond,
+                algorithm,
+                bitDepth,
+                fullAudioDurationMs: durationMs,
+                numberOfChannels,
+            })
+        })
+    } else {
+        if (!fileUri) {
+            throw new Error('fileUri is required')
+        }
+        logger.log(`extractAudioAnalysis`, {
+            fileUri,
+            pointsPerSecond,
+            algorithm,
+        })
+        const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+            fileUri,
+            pointsPerSecond,
+            skipWavHeader,
+            algorithm,
+            features,
+        })
+        logger.log(`extractAudioAnalysis`, res)
+        return res
+    }
+}

package/src/AudioAnalysis/extractWaveform.ts

@@ -0,0 +1,25 @@
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+import { getLogger } from '../logger'
+
+const logger = getLogger('extractWaveform')
+export interface ExtractWaveformProps {
+    fileUri: string
+    numberOfSamples: number
+    offset?: number
+    length?: number
+}
+export const extractWaveform = async ({
+    fileUri,
+    numberOfSamples,
+    offset = 0,
+    length,
+}: ExtractWaveformProps): Promise<unknown> => {
+    const res = await ExpoAudioStreamModule.extractAudioAnalysis({
+        fileUri,
+        numberOfSamples,
+        offset,
+        length,
+    })
+    logger.log(`extractWaveform`, res)
+    return res
+}

package/src/AudioRecorder.provider.tsx

@@ -1,42 +1,70 @@
-import React, { createContext, useContext } from "react";
+// packages/expo-audio-stream/src/AudioRecorder.provider.tsx
+import React, { createContext, useContext } from 'react'
 
+import { AudioAnalysis } from './AudioAnalysis/AudioAnalysis.types'
 import {
-  UseAudioRecorderProps,
-  UseAudioRecorderState,
-  useAudioRecorder,
-} from "./useAudioRecording";
+    AudioRecording,
+    RecordingConfig,
+    StartRecordingResult,
+} from './ExpoAudioStream.types'
+import { UseAudioRecorderProps, useAudioRecorder } from './useAudioRecorder'
 
-const AudioRecorderContext = createContext<UseAudioRecorderState>({
-  isRecording: false,
-  isPaused: false,
-  durationMs: 0,
-  size: 0,
-  // other properties filled on useAudioRecorder
-} as UseAudioRecorderState);
+export interface UseAudioRecorderState {
+    startRecording: (_: RecordingConfig) => Promise<StartRecordingResult>
+    stopRecording: () => Promise<AudioRecording | null>
+    pauseRecording: () => void
+    resumeRecording: () => void
+    isRecording: boolean
+    isPaused: boolean
+    durationMs: number // Duration of the recording
+    size: number // Size in bytes of the recorded audio
+    analysisData?: AudioAnalysis // Analysis data for the recording depending on enableProcessing flag
+}
+
+const initContext: UseAudioRecorderState = {
+    isRecording: false,
+    isPaused: false,
+    durationMs: 0,
+    size: 0,
+    startRecording: async () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    stopRecording: async () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    pauseRecording: () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+    resumeRecording: () => {
+        throw new Error('AudioRecorderProvider not found')
+    },
+}
+
+const AudioRecorderContext = createContext<UseAudioRecorderState>(initContext)
 
 interface AudioRecorderProviderProps {
-  children: React.ReactNode;
-  config?: UseAudioRecorderProps;
+    children: React.ReactNode
+    config?: UseAudioRecorderProps
 }
 
 export const AudioRecorderProvider: React.FC<AudioRecorderProviderProps> = ({
-  children,
-  config = {},
+    children,
+    config = {},
 }) => {
-  const audioRecorder = useAudioRecorder(config);
-  return (
-    <AudioRecorderContext.Provider value={audioRecorder}>
-      {children}
-    </AudioRecorderContext.Provider>
-  );
-};
+    const audioRecorder = useAudioRecorder(config)
+    return (
+        <AudioRecorderContext.Provider value={audioRecorder}>
+            {children}
+        </AudioRecorderContext.Provider>
+    )
+}
 
 export const useSharedAudioRecorder = () => {
-  const context = useContext(AudioRecorderContext);
-  if (!context) {
-    throw new Error(
-      "useSharedAudioRecorder must be used within an AudioRecorderProvider",
-    );
-  }
-  return context;
-};
+    const context = useContext(AudioRecorderContext)
+    if (!context) {
+        throw new Error(
+            'useSharedAudioRecorder must be used within an AudioRecorderProvider'
+        )
+    }
+    return context
+}

package/src/ExpoAudioStream.native.ts

@@ -1,6 +1,6 @@
 // src/ExpoAudioStreamModule.ts
-import { requireNativeModule } from "expo-modules-core";
+import { requireNativeModule } from 'expo-modules-core'
 
 // It loads the native module object from the JSI or falls back to
 // the bridge module (from NativeModulesProxy) if the remote debugger is on.
-export default requireNativeModule("ExpoAudioStream");
+export default requireNativeModule('ExpoAudioStream')