@edkimmel/expo-audio-stream 0.2.0

package/ios/ExpoPlayAudioStreamModule.swift

@@ -0,0 +1,436 @@
+import Foundation
+import AVFoundation
+import ExpoModulesCore
+
+let audioDataEvent: String = "AudioData"
+let soundIsPlayedEvent: String = "SoundChunkPlayed"
+let soundIsStartedEvent: String = "SoundStarted"
+let deviceReconnectedEvent: String = "DeviceReconnected"
+
+
+public class ExpoPlayAudioStreamModule: Module, MicrophoneDataDelegate, SoundPlayerDelegate, PipelineEventSender {
+    private var _microphone: Microphone?
+    private var _soundPlayer: SoundPlayer?
+    private var _pipelineIntegration: PipelineIntegration?
+
+    /// Single shared AVAudioEngine used by both SoundPlayer and AudioPipeline.
+    private let sharedAudioEngine = SharedAudioEngine()
+
+    private var microphone: Microphone {
+        if _microphone == nil {
+            _microphone = Microphone()
+            _microphone?.delegate = self
+        }
+        return _microphone!
+    }
+
+    private var soundPlayer: SoundPlayer {
+        if _soundPlayer == nil {
+            _soundPlayer = SoundPlayer()
+            _soundPlayer?.delegate = self
+            _soundPlayer?.setSharedEngine(sharedAudioEngine)
+        }
+        return _soundPlayer!
+    }
+
+    private var pipelineIntegration: PipelineIntegration {
+        if _pipelineIntegration == nil {
+            _pipelineIntegration = PipelineIntegration(eventSender: self, sharedEngine: sharedAudioEngine)
+        }
+        return _pipelineIntegration!
+    }
+
+    private var isAudioSessionInitialized: Bool = false
+
+    // ── PipelineEventSender conformance ───────────────────────────────
+    func sendPipelineEvent(_ eventName: String, _ params: [String: Any]) {
+        sendEvent(eventName, params)
+    }
+
+    public func definition() -> ModuleDefinition {
+        Name("ExpoPlayAudioStream")
+
+        // Defines event names that the module can send to JavaScript.
+        Events([
+            audioDataEvent,
+            soundIsPlayedEvent,
+            soundIsStartedEvent,
+            deviceReconnectedEvent,
+            PipelineIntegration.EVENT_STATE_CHANGED,
+            PipelineIntegration.EVENT_PLAYBACK_STARTED,
+            PipelineIntegration.EVENT_ERROR,
+            PipelineIntegration.EVENT_ZOMBIE_DETECTED,
+            PipelineIntegration.EVENT_UNDERRUN,
+            PipelineIntegration.EVENT_DRAINED,
+            PipelineIntegration.EVENT_AUDIO_FOCUS_LOST,
+            PipelineIntegration.EVENT_AUDIO_FOCUS_RESUMED,
+        ])
+
+        Function("destroy") {
+            self._pipelineIntegration?.destroy()
+            self._pipelineIntegration = nil
+            self.sharedAudioEngine.teardown()
+            if self.isAudioSessionInitialized {
+                let audioSession = AVAudioSession.sharedInstance()
+                try? audioSession.setActive(false, options: .notifyOthersOnDeactivation)
+                self.isAudioSessionInitialized = false
+            }
+            self._microphone = nil
+            self._soundPlayer = nil
+        }
+
+        /// Prompts the user to select the microphone mode.
+        Function("promptMicrophoneModes") {
+            promptForMicrophoneModes()
+        }
+
+        /// Requests microphone permission from the user.
+        AsyncFunction("requestPermissionsAsync") { (promise: Promise) in
+            checkMicrophonePermission { granted in
+                promise.resolve([
+                    "granted": granted,
+                    "canAskAgain": true,
+                    "status": granted ? "granted" : "denied"
+                ])
+            }
+        }
+
+        /// Gets the current microphone permission status.
+        AsyncFunction("getPermissionsAsync") { (promise: Promise) in
+            let status = AVAudioSession.sharedInstance().recordPermission
+            let granted = status == .granted
+            let canAskAgain = status == .undetermined
+            promise.resolve([
+                "granted": granted,
+                "canAskAgain": canAskAgain,
+                "status": granted ? "granted" : (canAskAgain ? "undetermined" : "denied")
+            ])
+        }
+
+        AsyncFunction("playSound") { (base64Chunk: String, turnId: String, encoding: String?, promise: Promise) in
+            Logger.debug("Play sound")
+            do {
+                if !isAudioSessionInitialized {
+                    try ensureAudioSessionInitialized()
+                }
+
+                // Ensure shared engine is configured (playSound may be called without setSoundConfig)
+                if !self.sharedAudioEngine.isConfigured {
+                    try self.sharedAudioEngine.configure(playbackMode: .regular)
+                    self.sharedAudioEngine.addDelegate(self.soundPlayer)
+                }
+
+                // Determine the audio format based on the encoding parameter
+                let commonFormat: AVAudioCommonFormat
+                switch encoding {
+                case "pcm_f32le":
+                    commonFormat = .pcmFormatFloat32
+                case "pcm_s16le", nil:
+                    commonFormat = .pcmFormatInt16
+                default:
+                    Logger.debug("[ExpoPlayAudioStreamModule] Unsupported encoding: \(encoding ?? "nil"), defaulting to PCM_S16LE")
+                    commonFormat = .pcmFormatInt16
+                }
+
+                try soundPlayer.play(audioChunk: base64Chunk, turnId: turnId, resolver: {
+                    _ in promise.resolve(nil)
+                }, rejecter: {code, message, error in
+                    promise.reject(code ?? "ERR_UNKNOWN", message ?? "Unknown error")
+                }, commonFormat: commonFormat)
+            } catch {
+                print("Error enqueuing audio: \(error.localizedDescription)")
+            }
+        }
+
+        AsyncFunction("stopSound") { (promise: Promise) in
+            soundPlayer.stop(promise)
+        }
+
+        AsyncFunction("clearSoundQueueByTurnId") { (turnId: String, promise: Promise) in
+            soundPlayer.clearSoundQueue(turnIdToClear: turnId, resolver: promise)
+        }
+
+        AsyncFunction("startMicrophone") { (options: [String: Any], promise: Promise) in
+            // Create recording settings
+            // Extract settings from provided options, using default values if necessary
+            let sampleRate = options["sampleRate"] as? Double ?? 16000.0 // it fails if not 48000, why?
+            let numberOfChannels = options["channelConfig"] as? Int ?? 1 // Mono channel configuration
+            let bitDepth = options["audioFormat"] as? Int ?? 16 // 16bits
+            let interval = options["interval"] as? Int ?? 1000
+
+            let settings = RecordingSettings(
+                sampleRate: sampleRate,
+                desiredSampleRate: sampleRate,
+                numberOfChannels: numberOfChannels,
+                bitDepth: bitDepth,
+                maxRecentDataDuration: nil,
+                pointsPerSecond: nil
+            )
+
+            if !isAudioSessionInitialized {
+                do {
+                    try ensureAudioSessionInitialized(settings: settings)
+                } catch {
+                    promise.reject("ERROR", "Failed to init audio session \(error.localizedDescription)")
+                    return
+                }
+            }
+
+            if let result = self.microphone.startRecording(settings: settings, intervalMilliseconds: interval) {
+                if let resError = result.error {
+                    promise.reject("ERROR", resError)
+                } else {
+                    let resultDict: [String: Any] = [
+                        "fileUri": result.fileUri ?? "",
+                        "channels": result.channels ?? 1,
+                        "bitDepth": result.bitDepth ?? 16,
+                        "sampleRate": result.sampleRate ?? 48000,
+                        "mimeType": result.mimeType ?? "",
+                    ]
+                    promise.resolve(resultDict)
+                }
+            } else {
+                promise.reject("ERROR", "Failed to start recording.")
+            }
+        }
+
+        /// Stops the microphone recording and releases associated resources
+        /// - Parameter promise: A promise to resolve when microphone recording is stopped
+        /// - Note: This method stops the active recording session, processes any remaining audio data,
+        ///         and releases hardware resources. It should be called when the app no longer needs
+        ///         microphone access to conserve battery and system resources.
+        AsyncFunction("stopMicrophone") { (promise: Promise) in
+            microphone.stopRecording(resolver: promise)
+        }
+
+        Function("toggleSilence") { (isSilent: Bool) in
+            microphone.toggleSilence(isSilent: isSilent)
+        }
+
+        /// Sets the sound player configuration
+        /// - Parameters:
+        ///   - config: A dictionary containing configuration options:
+        ///     - `sampleRate`: The sample rate for audio playback (default is 16000.0).
+        ///     - `playbackMode`: The playback mode ("regular", "voiceProcessing", or "conversation").
+        ///     - `useDefault`: When true, resets to default configuration regardless of other parameters.
+        ///   - promise: A promise to resolve when configuration is updated or reject with an error.
+        AsyncFunction("setSoundConfig") { (config: [String: Any], promise: Promise) in
+            // Check if we should use default configuration
+            let useDefault = config["useDefault"] as? Bool ?? false
+
+            do {
+                if !isAudioSessionInitialized {
+                    try ensureAudioSessionInitialized()
+                }
+
+                if useDefault {
+                    // Reset to default configuration — configure engine for regular mode
+                    Logger.debug("[ExpoPlayAudioStreamModule] Resetting sound configuration to default values")
+                    try self.sharedAudioEngine.configure(playbackMode: .regular)
+                    self.sharedAudioEngine.addDelegate(self.soundPlayer)
+                    try soundPlayer.resetConfigToDefault()
+                } else {
+                    // Extract configuration values from the provided dictionary
+                    let sampleRate = config["sampleRate"] as? Double ?? 16000.0
+                    let playbackModeString = config["playbackMode"] as? String ?? "regular"
+
+                    // Convert string playback mode to enum
+                    let playbackMode: PlaybackMode
+                    switch playbackModeString {
+                    case "voiceProcessing":
+                        playbackMode = .voiceProcessing
+                    case "conversation":
+                        playbackMode = .conversation
+                    default:
+                        playbackMode = .regular
+                    }
+
+                    // Configure shared engine first (handles voice processing)
+                    try self.sharedAudioEngine.configure(playbackMode: playbackMode)
+                    self.sharedAudioEngine.addDelegate(self.soundPlayer)
+
+                    // Create a new SoundConfig object
+                    let soundConfig = SoundConfig(sampleRate: sampleRate, playbackMode: playbackMode)
+
+                    // Update the sound player configuration (attaches node to shared engine)
+                    Logger.debug("[ExpoPlayAudioStreamModule] Setting sound configuration - sampleRate: \(sampleRate), playbackMode: \(playbackModeString)")
+                    try soundPlayer.updateConfig(soundConfig)
+                }
+
+                promise.resolve(nil)
+            } catch {
+                promise.reject("ERROR_CONFIG_UPDATE", "Failed to set sound configuration: \(error.localizedDescription)")
+            }
+        }
+
+        // ── Pipeline functions ────────────────────────────────────────────
+
+        AsyncFunction("connectPipeline") { (options: [String: Any], promise: Promise) in
+            do {
+                if !self.isAudioSessionInitialized {
+                    try self.ensureAudioSessionInitialized()
+                }
+
+                // Parse playback mode from options to configure shared engine
+                let playbackModeString = options["playbackMode"] as? String ?? "regular"
+                let playbackMode: PlaybackMode
+                switch playbackModeString {
+                case "voiceProcessing":
+                    playbackMode = .voiceProcessing
+                case "conversation":
+                    playbackMode = .conversation
+                default:
+                    playbackMode = .regular
+                }
+
+                // Configure shared engine (handles voice processing)
+                try self.sharedAudioEngine.configure(playbackMode: playbackMode)
+
+                let result = try self.pipelineIntegration.connect(options: options)
+
+                // Set the AudioPipeline as the active delegate for route/interruption callbacks
+                self.pipelineIntegration.setAsActiveDelegate(on: self.sharedAudioEngine)
+
+                promise.resolve(result)
+            } catch {
+                promise.reject("PIPELINE_CONNECT_ERROR", error.localizedDescription)
+            }
+        }
+
+        AsyncFunction("pushPipelineAudio") { (options: [String: Any], promise: Promise) in
+            do {
+                try self.pipelineIntegration.pushAudio(options: options)
+                promise.resolve(nil)
+            } catch {
+                promise.reject("PIPELINE_PUSH_ERROR", error.localizedDescription)
+            }
+        }
+
+        Function("pushPipelineAudioSync") { (options: [String: Any]) -> Bool in
+            return self.pipelineIntegration.pushAudioSync(options: options)
+        }
+
+        AsyncFunction("disconnectPipeline") { (promise: Promise) in
+            self.pipelineIntegration.removeAsDelegate(from: self.sharedAudioEngine)
+            self.pipelineIntegration.disconnect()
+            promise.resolve(nil)
+        }
+
+        AsyncFunction("invalidatePipelineTurn") { (options: [String: Any], promise: Promise) in
+            do {
+                try self.pipelineIntegration.invalidateTurn(options: options)
+                promise.resolve(nil)
+            } catch {
+                promise.reject("PIPELINE_INVALIDATE_ERROR", error.localizedDescription)
+            }
+        }
+
+        Function("getPipelineTelemetry") { () -> [String: Any] in
+            return self.pipelineIntegration.getTelemetry()
+        }
+
+        Function("getPipelineState") { () -> String in
+            return self.pipelineIntegration.getState()
+        }
+    }
+
+    private func ensureAudioSessionInitialized(settings recordingSettings: RecordingSettings? = nil) throws {
+        if self.isAudioSessionInitialized { return }
+
+        let audioSession = AVAudioSession.sharedInstance()
+        try audioSession.setCategory(
+            .playAndRecord, mode: .videoChat,
+            options: [.defaultToSpeaker, .allowBluetooth, .allowBluetoothA2DP])
+        if let settings = recordingSettings {
+            try audioSession.setPreferredSampleRate(settings.sampleRate)
+            // Set IO buffer duration short enough to support the desired emission interval.
+            // Use the hardware sample rate (not the desired rate) since this is a hardware-level setting.
+            let hwSampleRate = audioSession.sampleRate > 0 ? audioSession.sampleRate : 48000.0
+            let preferredDuration = 512.0 / hwSampleRate  // ~10.7ms at 48kHz
+            try audioSession.setPreferredIOBufferDuration(preferredDuration)
+        }
+        try audioSession.setActive(true)
+        isAudioSessionInitialized = true
+     }
+
+    // used for voice isolation, experimental
+    private func promptForMicrophoneModes() {
+        guard #available(iOS 15.0, *) else {
+            return
+        }
+
+        if AVCaptureDevice.preferredMicrophoneMode == .voiceIsolation {
+            return
+        }
+
+        AVCaptureDevice.showSystemUserInterface(.microphoneModes)
+    }
+
+    /// Checks microphone permission and calls the completion handler with the result.
+    private func checkMicrophonePermission(completion: @escaping (Bool) -> Void) {
+        switch AVAudioSession.sharedInstance().recordPermission {
+        case .granted:
+            completion(true)
+        case .denied:
+            completion(false)
+        case .undetermined:
+            AVAudioSession.sharedInstance().requestRecordPermission { granted in
+                DispatchQueue.main.async {
+                    completion(granted)
+                }
+            }
+        @unknown default:
+            completion(false)
+        }
+    }
+
+    func onMicrophoneData(_ microphoneData: Data, _ soundLevel: Float?) {
+        let encodedData = microphoneData.base64EncodedString()
+        // Construct the event payload similar to Android
+        let eventBody: [String: Any] = [
+            "fileUri": "",
+            "lastEmittedSize": 0,
+            "position": 0,
+            "encoded": encodedData,
+            "deltaSize": 0,
+            "totalSize": 0,
+            "mimeType": "",
+            "soundLevel": soundLevel ?? -160
+        ]
+        // Emit the event to JavaScript
+        sendEvent(audioDataEvent, eventBody)
+    }
+
+    func onMicrophoneError(_ error: String, _ errorMessage: String) {
+        let eventBody: [String: Any] = [
+            "error": error,
+            "errorMessage": errorMessage,
+            "streamUuid": ""
+        ]
+        sendEvent(audioDataEvent, eventBody)
+    }
+
+    func onDeviceReconnected(_ reason: AVAudioSession.RouteChangeReason) {
+        let reasonString: String
+        switch reason {
+        case .newDeviceAvailable:
+            reasonString = "newDeviceAvailable"
+        case .oldDeviceUnavailable:
+            reasonString = "oldDeviceUnavailable"
+        case .unknown, .categoryChange, .override, .wakeFromSleep, .noSuitableRouteForCategory, .routeConfigurationChange:
+            reasonString = "unknown"
+        @unknown default:
+            reasonString = "unknown"
+        }
+
+        sendEvent(deviceReconnectedEvent, ["reason": reasonString])
+    }
+
+    func onSoundChunkPlayed(_ isFinal: Bool) {
+        sendEvent(soundIsPlayedEvent, ["isFinal": isFinal])
+    }
+
+    func onSoundStartedPlaying() {
+        sendEvent(soundIsStartedEvent)
+    }
+}

package/ios/ExpoPlayAudioStreamView.swift

@@ -0,0 +1,7 @@
+import ExpoModulesCore
+
+// This view will be used as a native component. Make sure to inherit from `ExpoView`
+// to apply the proper styling (e.g. border radius and shadows).
+class ExpoPlayAudioStreamView: ExpoView {
+  
+}

package/ios/JitterBuffer.swift

@@ -0,0 +1,208 @@
+import Foundation
+
+/// Lock-based chunk queue for PCM audio (16-bit signed, little-endian).
+///
+/// Single producer (bridge thread) writes decoded PCM via `write()`.
+/// Single consumer (scheduling thread) drains via `read()`.
+/// All shared state is guarded by an NSLock.
+///
+/// Features:
+///   - Chunk queue: incoming `[Int16]` chunks are stored by reference
+///     (copy-on-write, zero actual copy since neither side mutates after write).
+///     No fixed capacity limit.
+///   - Priming gate: `read()` returns silence until `targetBufferMs` of audio has
+///     accumulated (or `markEndOfStream()` force-primes so the tail drains).
+///   - Silence-fill on underflow: when the buffer has fewer samples than the
+///     consumer requested, the remainder is filled with silence and an underrun
+///     is counted.
+class JitterBuffer {
+    private let sampleRate: Int
+    private let channels: Int
+    private let targetBufferMs: Int
+
+    // ── Chunk queue storage ──────────────────────────────────────────────
+    private var chunks: [[Int16]] = []
+    private var readCursor: Int = 0   // offset into the head chunk
+    private var count: Int = 0        // total live samples across all chunks
+
+    // ── Priming gate ────────────────────────────────────────────────────
+    private let primingSamples: Int
+    private var primed: Bool = false
+
+    // ── End-of-stream ───────────────────────────────────────────────────
+    private var endOfStream: Bool = false
+
+    // ── Lock ────────────────────────────────────────────────────────────
+    private let lock = NSLock()
+
+    // ── Telemetry ───────────────────────────────────────────────────────
+    private(set) var totalWritten: Int64 = 0
+    private(set) var totalRead: Int64 = 0
+    private(set) var underrunCount: Int = 0
+    private(set) var peakLevel: Int = 0
+
+    init(sampleRate: Int, channels: Int, targetBufferMs: Int) {
+        self.sampleRate = sampleRate
+        self.channels = channels
+        self.targetBufferMs = targetBufferMs
+        self.primingSamples = (sampleRate * channels * targetBufferMs) / 1000
+    }
+
+    // ── Producer API ────────────────────────────────────────────────────
+
+    /// Enqueue samples into the chunk queue.
+    ///
+    /// The array is stored directly (Swift CoW means zero actual copy
+    /// as long as neither side mutates after the call).
+    @discardableResult
+    func write(samples: [Int16], offset: Int = 0, length: Int? = nil) -> Int {
+        let len = length ?? samples.count
+        lock.lock()
+        defer { lock.unlock() }
+
+        let chunk: [Int16]
+        if offset == 0 && len == samples.count {
+            chunk = samples
+        } else {
+            chunk = Array(samples[offset..<(offset + len)])
+        }
+
+        chunks.append(chunk)
+        count += len
+
+        // Update peak telemetry
+        if count > peakLevel {
+            peakLevel = count
+        }
+        totalWritten += Int64(len)
+
+        // Check priming
+        if !primed && count >= primingSamples {
+            primed = true
+        }
+
+        return len
+    }
+
+    // ── Consumer API ────────────────────────────────────────────────────
+
+    /// Fill `dest` with up to `length` samples from the chunk queue.
+    ///
+    /// - Not primed & no end-of-stream: fills with silence.
+    /// - Primed: copies available samples; remainder is zero-filled
+    ///   and an underrun is recorded.
+    @discardableResult
+    func read(dest: inout [Int16], offset: Int = 0, length: Int? = nil) -> Int {
+        let len = length ?? dest.count
+        lock.lock()
+        defer { lock.unlock() }
+
+        if !primed {
+            for i in 0..<len {
+                dest[offset + i] = 0
+            }
+            return len
+        }
+
+        var destPos = offset
+        var remaining = len
+
+        while remaining > 0 && !chunks.isEmpty {
+            let chunk = chunks[0]
+            let available = chunk.count - readCursor
+            let toCopy = min(available, remaining)
+
+            for i in 0..<toCopy {
+                dest[destPos + i] = chunk[readCursor + i]
+            }
+            readCursor += toCopy
+            destPos += toCopy
+            remaining -= toCopy
+            count -= toCopy
+
+            if readCursor >= chunk.count {
+                chunks.removeFirst()
+                readCursor = 0
+            }
+        }
+
+        // Silence-fill remainder on underflow
+        if remaining > 0 {
+            for i in 0..<remaining {
+                dest[destPos + i] = 0
+            }
+            // Only count as underrun if we expected more data (not drained)
+            if !endOfStream {
+                underrunCount += 1
+            }
+        }
+
+        totalRead += Int64(len)
+        return len
+    }
+
+    // ── Control API ─────────────────────────────────────────────────────
+
+    /// Mark that the producer will not write any more data for this turn.
+    /// Force-primes the buffer so the consumer can drain whatever remains.
+    func markEndOfStream() {
+        lock.lock()
+        defer { lock.unlock() }
+        endOfStream = true
+        if !primed {
+            primed = true
+        }
+    }
+
+    /// `true` after `markEndOfStream()` was called AND the buffer is empty.
+    func isDrained() -> Bool {
+        lock.lock()
+        defer { lock.unlock() }
+        return endOfStream && count == 0
+    }
+
+    /// Current buffer level in samples (snapshot).
+    func availableSamples() -> Int {
+        lock.lock()
+        defer { lock.unlock() }
+        return count
+    }
+
+    /// Current buffer level converted to milliseconds.
+    func bufferedMs() -> Int {
+        lock.lock()
+        defer { lock.unlock() }
+        let denom = sampleRate * channels
+        guard denom > 0 else { return 0 }
+        return (count * 1000) / denom
+    }
+
+    /// Whether the priming gate is currently open.
+    func isPrimed() -> Bool {
+        lock.lock()
+        defer { lock.unlock() }
+        return primed
+    }
+
+    /// Reset the buffer to its initial empty state.
+    /// Called on turn changes to discard stale audio.
+    func reset() {
+        lock.lock()
+        defer { lock.unlock() }
+        chunks.removeAll()
+        readCursor = 0
+        count = 0
+        primed = false
+        endOfStream = false
+    }
+
+    /// Reset all telemetry counters to zero.
+    func resetTelemetry() {
+        lock.lock()
+        defer { lock.unlock() }
+        totalWritten = 0
+        totalRead = 0
+        underrunCount = 0
+        peakLevel = 0
+    }
+}

package/ios/Logger.swift

@@ -0,0 +1,7 @@
+class Logger {
+    static func debug(_ message: @autoclosure () -> String) {
+        #if DEBUG
+        print(message())
+        #endif
+    }
+}