@siteed/expo-audio-stream 2.0.0 → 2.1.0

package/ios/ExpoAudioStreamModule.swift

@@ -313,46 +313,136 @@ public class ExpoAudioStreamModule: Module, AudioStreamManagerDelegate {
         /// - Parameters:
         ///   - options: A dictionary containing:
         ///     - `fileUri`: The URI of the audio file.
-        ///     - `startTimeMs`: Start time in milliseconds.
-        ///     - `endTimeMs`: End time in milliseconds.
+        ///     - `mode`: Trim mode ('single', 'keep', or 'remove').
+        ///     - `startTimeMs`: Start time in milliseconds (for 'single' mode).
+        ///     - `endTimeMs`: End time in milliseconds (for 'single' mode).
+        ///     - `ranges`: Array of time ranges (for 'keep' and 'remove' modes).
+        ///     - `outputFileName`: Optional name for the output file.
         ///     - `outputFormat`: Optional output format configuration.
+        ///     - `decodingOptions`: Optional decoding configuration.
         AsyncFunction("trimAudio") { (options: [String: Any], promise: Promise) in
             guard let fileUri = options["fileUri"] as? String,
-                  let startTimeMs = options["startTimeMs"] as? Double,
-                  let endTimeMs = options["endTimeMs"] as? Double,
                   let url = URL(string: fileUri) else {
-                promise.reject("INVALID_ARGUMENTS", "Invalid arguments provided")
+                promise.reject("INVALID_ARGUMENTS", "Invalid file URI provided")
                 return
             }
 
+            let mode = options["mode"] as? String ?? "single"
+            let startTimeMs = options["startTimeMs"] as? Double
+            let endTimeMs = options["endTimeMs"] as? Double
+            let ranges = options["ranges"] as? [[String: Double]]
+            let outputFileName = options["outputFileName"] as? String
             let outputFormat = options["outputFormat"] as? [String: Any]
+            let decodingOptions = options["decodingOptions"] as? [String: Any]
+
+            // Add detailed logging for filename and format options
+            Logger.debug("Trim audio request:")
+            Logger.debug("- Input file: \(fileUri)")
+            Logger.debug("- Mode: \(mode)")
+            Logger.debug("- Output filename: \(outputFileName ?? "not specified (will generate UUID)")")
+            if let format = outputFormat?["format"] as? String {
+                Logger.debug("- Output format: \(format)")
+            } else {
+                Logger.debug("- Output format: not specified (will use default)")
+            }
+
+            // Input validation based on mode
+            switch mode {
+            case "single":
+                guard let start = startTimeMs, let end = endTimeMs else {
+                    promise.reject("INVALID_ARGUMENTS", "startTimeMs and endTimeMs required for 'single' mode")
+                    return
+                }
+                guard start >= 0, end > start else {
+                    promise.reject("INVALID_ARGUMENTS", "Invalid time range")
+                    return
+                }
+            case "keep", "remove":
+                guard let rangesArray = ranges, !rangesArray.isEmpty else {
+                    promise.reject("INVALID_ARGUMENTS", "'ranges' array required for 'keep' or 'remove' mode")
+                    return
+                }
+            default:
+                promise.reject("INVALID_MODE", "Mode must be 'single', 'keep', or 'remove'")
+                return
+            }
 
             DispatchQueue.global().async {
                 do {
                     let audioProcessor = try AudioProcessor(
                         url: url,
-                        resolve: { result in
-                            promise.resolve(result)
-                        },
-                        reject: { code, message in
-                            promise.reject(code, message)
-                        }
+                        resolve: { result in promise.resolve(result) },
+                        reject: { code, message in promise.reject(code, message) }
                     )
-                    
+
+                    let progressCallback: (Float, Int64, Int64) -> Void = { progress, bytesProcessed, totalBytes in
+                        self.sendEvent("TrimProgress", [
+                            "progress": progress,
+                            "bytesProcessed": bytesProcessed,
+                            "totalBytes": totalBytes
+                        ])
+                    }
+
+                    let startTime = CACurrentMediaTime()
                     if let result = audioProcessor.trimAudio(
+                        mode: mode,
                         startTimeMs: startTimeMs,
                         endTimeMs: endTimeMs,
-                        outputFormat: outputFormat
+                        ranges: ranges,
+                        outputFileName: outputFileName,
+                        outputFormat: outputFormat,
+                        decodingOptions: decodingOptions,
+                        progressCallback: progressCallback
                     ) {
-                        promise.resolve([
-                            "uri": result.uri,
-                            "duration": result.duration,
-                            "size": result.size
-                        ])
+                        let processingTimeMs = Int((CACurrentMediaTime() - startTime) * 1000)
+                        var resultDict = result.toDictionary()
+                        resultDict["processingInfo"] = ["durationMs": processingTimeMs]
+                        
+                        let uri = result.uri
+                        Logger.debug("Trim completed successfully in \(processingTimeMs)ms")
+                        Logger.debug("Output file URI: \(uri)")
+                        
+                        // Verify file exists
+                        let fileManager = FileManager.default
+                        if let url = URL(string: uri) {
+                            let exists = fileManager.fileExists(atPath: url.path)
+                            Logger.debug("File exists at path \(url.path): \(exists)")
+                            
+                            // Log filename details
+                            Logger.debug("Filename: \(url.lastPathComponent)")
+                            Logger.debug("File extension: \(url.pathExtension.lowercased())")
+                            
+                            // If format is AAC, ensure we're using the correct extension and MIME type
+                            if let format = outputFormat?["format"] as? String, 
+                               format.lowercased() == "aac" {
+                                
+                                Logger.debug("AAC format detected - ensuring correct metadata")
+                                
+                                // For AAC format, ensure we're using the correct extension and MIME type
+                                if url.pathExtension.lowercased() == "m4a" {
+                                    Logger.debug("File has correct m4a extension for AAC audio")
+                                    
+                                    // Just update the MIME type in the result to ensure correct playback
+                                    if var compression = resultDict["compression"] as? [String: Any] {
+                                        compression["mimeType"] = "audio/mp4"
+                                        resultDict["compression"] = compression
+                                    }
+                                    
+                                    resultDict["mimeType"] = "audio/mp4"
+                                    resultDict["actualFormat"] = "m4a"
+                                } else {
+                                    Logger.debug("Warning: AAC format should use .m4a extension, but found .\(url.pathExtension.lowercased())")
+                                }
+                            }
+                        }
+                        
+                        promise.resolve(resultDict)
                     } else {
+                        Logger.debug("Failed to trim audio")
                         promise.reject("TRIM_ERROR", "Failed to trim audio")
                     }
                 } catch {
+                    Logger.debug("Failed to initialize audio processor: \(error.localizedDescription)")
                     promise.reject("PROCESSING_ERROR", "Failed to initialize audio processor: \(error.localizedDescription)")
                 }
             }
@@ -493,6 +583,23 @@ public class ExpoAudioStreamModule: Module, AudioStreamManagerDelegate {
                 promise.reject("PROCESSING_ERROR", "Failed to process audio file: \(error.localizedDescription)")
             }
         }
+        
+        /// Extracts mel spectrogram data from a file.
+        ///
+        /// - Parameters:
+        ///   - options: A dictionary containing:
+        ///     - `fileUri`: The URI of the audio file.
+        ///     - `pointsPerSecond`: The number of data points to extract per second of audio.
+        ///   - promise: A promise to resolve with the extracted mel spectrogram data or reject with an error.
+        /// - Returns: Promise to be resolved with mel spectrogram data.
+        AsyncFunction("extractMelSpectrogram") { (options: [String: Any], promise: Promise) in
+            // This is a placeholder implementation that will be fully implemented later
+            // Currently, mel spectrogram extraction is only available on Android
+            promise.reject(
+                "UNSUPPORTED_PLATFORM", 
+                "Mel spectrogram extraction is currently only available on Android and is experimental"
+            )
+        }
     }
     
     func audioStreamManager(_ manager: AudioStreamManager, didReceiveInterruption info: [String: Any]) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@siteed/expo-audio-stream",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "stream audio crossplatform",
   "license": "MIT",
   "main": "build/index.js",

package/plugin/build/index.js

@@ -48,7 +48,12 @@ const withRecordingPermission = (config, props) => {
         if (options.iosBackgroundModes?.useAudio === true &&
             enableBackgroundAudio === true &&
             !existingBackgroundModes.includes('audio')) {
-            existingBackgroundModes.push('audio');
+            // Don't automatically add 'audio' background mode as it's only for playback
+            // existingBackgroundModes.push('audio')
+            // Instead, ensure processing mode is used for background recording
+            if (options.iosBackgroundModes?.useProcessing !== true) {
+                console.warn(`${LOG_PREFIX} Warning: Background audio recording requires 'processing' background mode. Please enable 'useProcessing' in iosBackgroundModes.`);
+            }
         }
         if (options.iosBackgroundModes?.useVoIP === true &&
             enablePhoneStateHandling === true) {

package/plugin/src/index.ts

@@ -90,7 +90,15 @@ const withRecordingPermission: ConfigPlugin<AudioStreamPluginOptions> = (
             enableBackgroundAudio === true &&
             !existingBackgroundModes.includes('audio')
         ) {
-            existingBackgroundModes.push('audio')
+            // Don't automatically add 'audio' background mode as it's only for playback
+            // existingBackgroundModes.push('audio')
+
+            // Instead, ensure processing mode is used for background recording
+            if (options.iosBackgroundModes?.useProcessing !== true) {
+                console.warn(
+                    `${LOG_PREFIX} Warning: Background audio recording requires 'processing' background mode. Please enable 'useProcessing' in iosBackgroundModes.`
+                )
+            }
         }
 
         if (

package/src/AudioAnalysis/AudioAnalysis.types.ts

@@ -82,7 +82,7 @@ export interface DataPoint {
     id: number
     amplitude: number // Peak amplitude for the segment
     rms: number // Root mean square value
-    dB: number // Always computed
+    dB: number // dBFS (decibels relative to full scale) computed from RMS value
     silent: boolean // Always computed
     features?: AudioFeatures
     speech?: SpeechFeatures
@@ -163,3 +163,40 @@ export interface PreviewOptions extends AudioRangeOptions {
      */
     decodingOptions?: DecodingConfig
 }
+
+/**
+ * Options for mel-spectrogram extraction
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface ExtractMelSpectrogramOptions {
+    fileUri?: string // Path to audio file
+    arrayBuffer?: ArrayBuffer // Raw audio buffer
+    windowSizeMs: number // Window size in ms (e.g., 25)
+    hopLengthMs: number // Hop length in ms (e.g., 10)
+    nMels: number // Number of mel filters (e.g., 60)
+    fMin?: number // Min frequency (default: 0)
+    fMax?: number // Max frequency (default: sampleRate / 2)
+    windowType?: 'hann' | 'hamming' // Window function (default: 'hann')
+    normalize?: boolean // Mean normalization (default: false)
+    logScale?: boolean // Log scaling of mel energies (default: true)
+    decodingOptions?: DecodingConfig // Audio decoding settings
+    startTimeMs?: number // Optional start time
+    endTimeMs?: number // Optional end time
+    logger?: ConsoleLike
+}
+
+/**
+ * Return type for mel spectrogram extraction
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions.
+ */
+export interface MelSpectrogram {
+    spectrogram: number[][] // 2D array [time][mel]
+    sampleRate: number // Audio sample rate
+    nMels: number // Number of mel filters
+    timeSteps: number // Number of time frames
+    durationMs: number // Audio duration in ms
+}

package/src/AudioAnalysis/extractAudioAnalysis.ts

@@ -7,7 +7,7 @@
  */
 import crc32 from 'crc-32'
 
-import { ConsoleLike, ExtractAudioDataOptions } from '../ExpoAudioStream.types'
+import { ConsoleLike } from '../ExpoAudioStream.types'
 import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
 import { isWeb } from '../constants'
 import {
@@ -15,7 +15,6 @@ import {
     AudioFeaturesOptions,
     DataPoint,
     DecodingConfig,
-    PreviewOptions,
 } from './AudioAnalysis.types'
 import { processAudioBuffer } from '../utils/audioProcessing'
 import { convertPCMToFloat32 } from '../utils/convertPCMToFloat32'
@@ -332,39 +331,3 @@ export const extractRawWavAnalysis = async ({
         return res
     }
 }
-
-/**
- * Generates a simplified preview of the audio waveform for quick visualization.
- * Ideal for UI rendering with a specified number of points.
- *
- * @param options - The options for the preview, including file URI and time range.
- * @returns A promise that resolves to the audio preview data.
- */
-export async function extractPreview({
-    fileUri,
-    numberOfPoints = 100,
-    startTimeMs = 0,
-    endTimeMs = 30000, // First 30 seconds
-    decodingOptions,
-    logger,
-}: PreviewOptions): Promise<AudioAnalysis> {
-    const durationMs = endTimeMs - startTimeMs
-    const segmentDurationMs = Math.floor(durationMs / numberOfPoints)
-
-    // Call extractAudioAnalysis with calculated parameters
-    const analysis = await extractAudioAnalysis({
-        fileUri,
-        startTimeMs,
-        endTimeMs,
-        logger,
-        segmentDurationMs,
-        decodingOptions,
-    })
-
-    // Transform the result into AudioPreview format
-    return analysis
-}
-
-export const extractAudioData = async (props: ExtractAudioDataOptions) => {
-    return await ExpoAudioStreamModule.extractAudioData(props)
-}

package/src/AudioAnalysis/extractAudioData.ts

@@ -0,0 +1,6 @@
+import { ExtractAudioDataOptions } from '../ExpoAudioStream.types'
+import ExpoAudioStreamModule from '../ExpoAudioStreamModule'
+
+export const extractAudioData = async (props: ExtractAudioDataOptions) => {
+    return await ExpoAudioStreamModule.extractAudioData(props)
+}

package/src/AudioAnalysis/extractMelSpectrogram.ts

@@ -0,0 +1,144 @@
+/**
+ * @experimental This feature is experimental and currently only available on Android.
+ * The API may change in future versions. The web implementation is a placeholder.
+ */
+
+import { ExpoAudioStreamModule } from '..'
+import { isWeb } from '../constants'
+import {
+    ExtractMelSpectrogramOptions,
+    MelSpectrogram,
+} from './AudioAnalysis.types'
+import {
+    processAudioBuffer,
+    ProcessedAudioData,
+} from '../utils/audioProcessing'
+
+/**
+ * Extracts a mel spectrogram from audio data
+ * 
+ * @experimental This feature is experimental and currently only available on Android.
+ * The iOS implementation will throw an "UNSUPPORTED_PLATFORM" error.
+ * The web implementation is a placeholder that returns dummy data.
+ */
+export async function extractMelSpectrogram(
+    options: ExtractMelSpectrogramOptions
+): Promise<MelSpectrogram> {
+    const {
+        fileUri,
+        arrayBuffer,
+        windowSizeMs,
+        hopLengthMs,
+        nMels,
+        fMin = 0,
+        fMax,
+        windowType = 'hann',
+        normalize = false,
+        logScale = true,
+        decodingOptions,
+        startTimeMs,
+        endTimeMs,
+        logger,
+    } = options
+
+    if (isWeb) {
+        // Create audio context
+        const audioContext = new (window.AudioContext ||
+            (window as any).webkitAudioContext)()
+
+        try {
+            // Process audio data using the existing utility
+            const processedAudio: ProcessedAudioData = await processAudioBuffer(
+                {
+                    arrayBuffer,
+                    fileUri,
+                    targetSampleRate:
+                        decodingOptions?.targetSampleRate || 16000,
+                    targetChannels: decodingOptions?.targetChannels || 1,
+                    normalizeAudio: decodingOptions?.normalizeAudio ?? false,
+                    startTimeMs,
+                    endTimeMs,
+                    audioContext,
+                    logger: options.logger,
+                }
+            )
+
+            // Calculate window and hop size in samples
+            const sampleRate = processedAudio.sampleRate
+            const windowSize = Math.floor((windowSizeMs * sampleRate) / 1000)
+            const hopLength = Math.floor((hopLengthMs * sampleRate) / 1000)
+            const maxFreq = fMax || sampleRate / 2
+
+            // Extract the mel spectrogram from the processed audio
+            const spectrogram = computeMelSpectrogram(
+                processedAudio.channelData,
+                sampleRate,
+                nMels,
+                windowSize,
+                hopLength,
+                fMin,
+                maxFreq,
+                windowType,
+                normalize,
+                logScale
+            )
+
+            const timeSteps = spectrogram.length
+
+            return {
+                spectrogram,
+                sampleRate,
+                nMels,
+                timeSteps,
+                durationMs: processedAudio.durationMs,
+            }
+        } catch (error) {
+            logger?.error('Error extracting mel spectrogram:', error)
+            throw error
+        } finally {
+            // Close the audio context
+            await audioContext.close()
+        }
+    }
+    return ExpoAudioStreamModule.extractMelSpectrogram(options)
+}
+
+/**
+ * Computes a mel spectrogram from audio data
+ * 
+ * @experimental This is a placeholder implementation that returns dummy data.
+ * The actual implementation will be added in a future version.
+ */
+function computeMelSpectrogram(
+    audioData: Float32Array,
+    sampleRate: number,
+    nMels: number,
+    windowSize: number,
+    hopLength: number,
+    fMin: number,
+    fMax: number,
+    windowType: 'hann' | 'hamming',
+    normalize: boolean,
+    logScale: boolean
+): number[][] {
+    // Placeholder for the actual implementation
+    // This would include:
+    // 1. Windowing the audio data using the specified window type
+    // 2. Computing the STFT (Short-Time Fourier Transform)
+    // 3. Converting to power spectrogram
+    // 4. Applying mel filterbanks
+    // 5. Taking the logarithm if logScale is true
+    // 6. Normalizing if normalize is true
+
+    // For now, return a dummy implementation
+    const numFrames =
+        Math.floor((audioData.length - windowSize) / hopLength) + 1
+    const spectrogram: number[][] = []
+
+    // Create dummy mel spectrogram data
+    for (let i = 0; i < numFrames; i++) {
+        spectrogram.push(Array(nMels).fill(0))
+    }
+
+    return spectrogram
+}

package/src/AudioAnalysis/extractPreview.ts

@@ -0,0 +1,34 @@
+import { PreviewOptions, AudioAnalysis } from './AudioAnalysis.types'
+import { extractAudioAnalysis } from './extractAudioAnalysis'
+
+/**
+ * Generates a simplified preview of the audio waveform for quick visualization.
+ * Ideal for UI rendering with a specified number of points.
+ *
+ * @param options - The options for the preview, including file URI and time range.
+ * @returns A promise that resolves to the audio preview data.
+ */
+export async function extractPreview({
+    fileUri,
+    numberOfPoints = 100,
+    startTimeMs = 0,
+    endTimeMs = 30000, // First 30 seconds
+    decodingOptions,
+    logger,
+}: PreviewOptions): Promise<AudioAnalysis> {
+    const durationMs = endTimeMs - startTimeMs
+    const segmentDurationMs = Math.floor(durationMs / numberOfPoints)
+
+    // Call extractAudioAnalysis with calculated parameters
+    const analysis = await extractAudioAnalysis({
+        fileUri,
+        startTimeMs,
+        endTimeMs,
+        logger,
+        segmentDurationMs,
+        decodingOptions,
+    })
+
+    // Transform the result into AudioPreview format
+    return analysis
+}